% use to find bib's not used % grep bibitem cocomost.bbl | gawk '{gsub(/bibitem/,"");gsub(/{/,"");gsub(/}/,"");gsub(/\\/,"");print $0}' \documentclass[journal]{IEEEtran/IEEEtran} \newenvironment{smallitem}{ \setlength{\topsep}{0pt} \setlength{\partopsep}{0pt} \setlength{\parskip}{0pt} \begin{itemize} \setlength{\leftmargin}{.2in} \setlength{\parsep}{0pt} \setlength{\parskip}{0pt} \setlength{\itemsep}{0pt}}{\end{itemize}} \newenvironment{smallnums}{ \setlength{\topsep}{0pt} \setlength{\partopsep}{0pt} \setlength{\parskip}{0pt} \begin{enumerate} \setlength{\leftmargin}{.2in} \setlength{\parsep}{0pt} \setlength{\parskip}{0pt} \setlength{\itemsep}{0pt}}{\end{enumerate}} \newenvironment{hhh}{ \begin{hypo}[?]}{\end{hypo}} \newenvironment{yup}{ \begin{hypo}[\ding{51}]}{\end{hypo}} \newenvironment{nupe}{ \begin{hypo}[\ding{55}]}{\end{hypo}} % stuff for deltas plots \newcommand{\boxplot}[5]{\begin{picture}(100,10) \put(0,0){\line(0,1){8}} \put(100,0){\line(0,1){8}} \put(#1,4){\line(1,0){#2}} \put(#3,4){\circle*{6}} \put(#4,4){\line(1,0){#5}} \put(50,0){\line(0,1){8}} \end{picture}} \newcommand{\aive}{{\"aive}} \newcommand{\bi}{\begin{smallitem}} \newcommand{\ei}{\end{smallitem}} \newcommand{\be}{\begin{smallnums}} \newcommand{\ee}{\end{smallnums}} \newcommand{\bd}{\begin{description}} \newcommand{\ed}{\end{description}} \newcommand{\fig}[1]{Figure~\ref{fig:#1}} \newcommand{\eq}[1]{Equation~\ref{eq:#1}} \newcommand{\hyp}[1]{Hypothesis~\ref{hyp:#1}} % some very useful LaTeX packages include: \usepackage{alltt} \usepackage{cite} % Written by Donald Arseneau % V1.6 and later of IEEEtran pre-defines the format % of the cite.sty package \cite{} output to follow % that of IEEE. Loading the cite package will % result in citation numbers being automatically % sorted and properly "ranged". i.e., % [1], [9], [2], [7], [5], [6] % (without using cite.sty) % will become: % [1], [2], [5]--[7], [9] (using cite.sty) % cite.sty's \cite will automatically add leading % space, if needed. Use cite.sty's noadjust option % (cite.sty V3.8 and later) if you want to turn this % off. cite.sty is already installed on most LaTeX % systems. The latest version can be obtained at: % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/cite/ \usepackage[pdftex]{graphicx} %\fi % However, be warned that pdflatex will require graphics to be in PDF % (not EPS) format and will preclude the use of PostScript based LaTeX % packages such as psfrag.sty and pstricks.sty. IEEE conferences typically % allow PDF graphics (and hence pdfLaTeX). However, IEEE journals do not % (yet) allow image formats other than EPS or TIFF. Therefore, authors of % journal papers should use traditional LaTeX with EPS graphics. % % The path(s) to the graphics files can also be declared: e.g., % \graphicspath{{../eps/}{../ps/}} % if the graphics files are not located in the same directory as the % .tex file. This can be done in each branch of the conditional above % (after graphicx is loaded) to handle the EPS and PDF cases separately. % In this way, full path information will not have to be specified in % each \includegraphics command. % % Note that, when switching from latex to pdflatex and vice-versa, the new % compiler will have to be run twice to clear some warnings. \usepackage{subfigure} % Written by Steven Douglas Cochran % This package makes it easy to put subfigures % in your figures. i.e., "figure 1a and 1b" % Docs are in "Using Imported Graphics in LaTeX2e" % by Keith Reckdahl which also documents the graphicx % package (see above). subfigure.sty is already % installed on most LaTeX systems. The latest version % and documentation can be obtained at: % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/subfigure/ \usepackage{url} % Written by Donald Arseneau % Provides better support for handling and breaking % URLs. url.sty is already installed on most LaTeX % systems. The latest version can be obtained at: % http://www.ctan.org/tex-archive/macros/latex/contrib/other/misc/ % Read the url.sty source comments for usage information. \usepackage{amssymb} \usepackage{amsmath} % From the American Mathematical Society % A popular package that provides many helpful commands % for dealing with mathematics. Note that the AMSmath % package sets \interdisplaylinepenalty to 10000 thus % preventing page breaks from occurring within multiline % equations. Use: \interdisplaylinepenalty=2500 % after loading amsmath to restore such page breaks % as IEEEtran.cls normally does. amsmath.sty is already % installed on most LaTeX systems. The latest version % and documentation can be obtained at: % http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/ \usepackage{pifont} \usepackage{verbatim} % for block comment %\usepackage[dvips]{color} \begin{document} \title{Feature Subset Selection Methods for\\COCOMO Based Software Effort Estimation} \author{Daniel~Baker\thanks{Dan Baker is with the Lane Department of Computer Science, West Virginia University and can be reached at \protect\url{danielryanbaker@gmail.com}},Tim~Menzies~\IEEEmembership{Member,~IEEE}% <-this % stops a space \thanks{Dr. Menzies is with the Lane Department of Computer Science, West Virginia University and can be reached at \protect\url{tim@menzies.us}} \thanks{This research was conducted with funds from the NASA Software Assurance Research Program led by the NASA IV\&V Facility. } \thanks{Manuscript received January 1, 2006; revised XXX YY, 200Z.}}% <-this % stops a space \markboth{IEEE Transactions on Software Engineering,~Vol.~W, No.~X,~YYY~200Z}{Baker, Menzies, Feature Subset Selection} \maketitle \begin{abstract} This paper demonstrates the results of feature subset selection methods for the COCOMO model. Two algorithms are tested, COCOMIN and COCOMOST. COCOMIN was designed as a `strawman` approach to feature subset selection. COCOMOST was designed to efficiently evaluate every subset in the attribute space instead of using heuristics because the domain of software cost estimation is sufficiently data starved to warrant exhaustive methods. \end{abstract} \section{Introduction} %\PARstart{I}{n} 2005...(START WITH QUOTE AND REFERENCE ABOUT MONEY LOST FROM INACURRATE SOFTWARE COST ESTIMATION/BUDGETING) %Currently cost estimation is accomplished using models such as COCOMO~\cite{Boehm81}, (LIST MANY OTHER METHODS WITH REFERENCES). \PARstart{C}{urrently} cost estimation is accomplished using models such as COCOMO~\cite{Boehm81}. These models predict the development cost for a new software project based on past project data. %For an accurate prediction the training data needs quantity, quality, and relevance to the new project(ADD REFERENCES). For an accurate prediction the training data needs quantity, quality, and relevance to the new project. Unfortunately this is difficult in practice and estimates are often made using inadequate training data. %Consequently, these models are plagued with problems including highly inaccurate predictions(FIND BEST REFERENCE) and the variance problem(INSERT REFERENCE). Consequently, these models are plagued with problems including highly inaccurate predictions and the variance problem. %It has been shown that the variance can be reduced by feature subset selection methods that discard irrelevant, redundant, noisy, and unreliable attributes(LOTS OF REF HERE). It has been shown that the variance can be reduced by feature subset selection methods that discard irrelevant, redundant, noisy, and unreliable attributes. This paper explores some of these attribute pruning techniques. %\section{Background: COCOMO and COSEEKMO} %\PARstart{P}{ull} text from COSEEKMO paper. First explain COCOMO. Then justify FSS. Then explain COSEEKMO. Basically say that %COSEEKMO worked but was too slow, so we decided to try the minimal approach (attr - next section). Given that the %datasets were small, we decided extensive searches were ok and tried a complete approach instead of heuristics (next section on cocomost). \section{Background} \subsection{COCOMO} The case study material for this paper uses COCOMO-format data. COCOMO (the COnstructive COst MOdel) was originally developed by Barry Boehm in 1981~\cite{Boehm81} and was extensively revised in 2000~\cite{Boehm00b}. %% COCOMO helps software developers %% reason about the cost and schedule implications of their software %% decisions such as software investment decisions; setting project budgets %% and schedules; negotiating cost, schedule, and performance %% tradeoffs; making software risk management decisions, and making %% software improvement decisions. %% One advantage of COCOMO (and this is why we use it) %% is that unlike many other costing models such as SLIM or SEER, %% COCOMO is an {\em %% open model} with numerous published data~\cite{Boehm81, Boehm00b}. %COCOMO %measures effort in calendar months where one month is 152 % hours (and includes development and management hours). The core intuition behind COCOMO-based estimation is that as a program grows in size, the development effort grows exponentially. More specifically: {\footnotesize \begin{equation}\label{eq:one} effort(person months)=a*\left(KLOC^b\right)*\left(\prod_jEM_j\right) \end{equation} } %XXXwhat about repeatability (folks repeating other people's work), straw man. %also, acceptable if >= cocomo1 stratified results %competent if pred(30) cocomo stratified results:sp %good if pred(25)> 83% Here, $KLOC$ is thousands of delivered source instructions. $KLOC$ can be estimated directly or via a {\em function point estimation}. Function points are a product of five defined data components (inputs, outputs, inquiries, files, external interfaces) and 14 weighted environment characteristics (data comm, performance, reusability, etc.)~\cite{Boehm00b,Capers98}. A 1,000 line Cobol program would typically implement about 14 function points, while a 1,000-line C program would implement about seven\footnote{\url{http://www.qsm.com/FPGearing.html}}. % XXX In \eq{one}, $EM_j$ is one of {\em effort multipliers} such as {\em cplx} (complexity) or {\em pcap} (programmer capability). In order to model the effects of $EM_j$ on development effort, Boehm proposed reusing numeric values which he generated via regression on historical data for each value of $EM_i$. %\begin{figure}[!t] {\footnotesize %\begin{center} %\begin{tabular}{r|r|r|p{1.8in}} %\noindent %Mode & a & b & notes\\\hline %Organic & 3.2 & 1.05 & projects from relatively small software teams develop software %in a highly familiar, in-house environment.\\\hline %Embedded & 2.8& 1.2 & projects operating within (is embedded in) a strongly %coupled complex of hardware, software, regulations, and operational %procedures.\\\hline %Semi-Detached & 3.0 & 1.12 & An intermediary mode between organic and embedded. %\end{tabular} %\end{center}} %\caption{Standard COCOMO 81 development modes.}\label{fig:eso} %\end{figure} %\begin{figure*}{\footnotesize %\begin{center} %\begin{tabular}{|p{0.35in}|p{6.5in}|}\hline %~\newline %~\newline %Data sources&\bd %\item[{\em Coc81:}] 63 records in the %COCOMO 81 format. Source: \cite[p496-497]{Boehm81}. \newline Download from %\url{http://unbox.org/wisp/trunk/cocomo/data/coc81modeTypeLangType.csv}. %\item[{\em Nasa93:}] ~~~ 93 NASA records in the COCOMO 81 format. \newline %Download from \url{http://unbox.org/wisp/trunk/cocomo/data/nasa93.csv}. %\item[{\em CocII:}] 161 records in the COCOMO II format from the COCOMO consortium %(co-ordinated by USC). This data is not in the public domain. %\ed\\\hline %~\newline %~\newline %Data \newline %subsets&\bd %\item[{\em All:}] selects all records from a particular source; %e.g. "coc81\_all''. %\item[{\em Category:}] ~~~~~is a NASA-specific % designation selecting the type of project; e.g. avionics, data capture, etc. %\item[{\em Dev:}] indicates the development methodology; e.g. div.waterfall. %\item[{\em DevEnd:}] ~~~shows the last year of the software project. %\item[{\em Fg:}] selects either ``$f$'' (flight) or ``$g$'' (ground) software. %\item[{\em Kind:}] selects records relating %to the development platform; max= mainframe and mic= microprocessor. %\item[{\em Lang:}] selects records about different development languages. %\item[{\em Project} and {\em center}:]~~~~~~~~~~~~~~~ $nasa93$ % designations selecting records relating to where the software was built and the name %of the project. %\item[{\em Mode=e:}]~\newline %selects records relating to %the {\em embedded} COCOMO 81 development mode. The different COCOMO 81 development models were described %in \fig{eso}. %\item[{\em Mode=o: }]~\newline %selects COCOMO 81 {\em organic} mode records. %\item[{\em Mode=sd: }]~\newline %selects COCOMO 81 {\em semi-detached} mode records. %\item[{\em Org:}] is a $cocII$ designation showing what organization provided the data. %\item[{\em Size:}] is a $cocII$ specific designation grouping the records into (e.g.) those around 100KLOC. %\item[{\em Type:}] selects different $coc81$ designations and include ``bus'' (for business application) %or ``sys'' (for system software). %\item[{\em Year:}] is a $nasa93$ term that %selects the development years, grouped into units of five; e.g. 1970,1971,1972,1973,1974 %are labeled ``1970''. %\ed %\\\hline %\end{tabular} %\end{center}} %\caption{Data sets (top) and %parts (bottom) of the data used in this study.}\label{fig:parts} %\end{figure*} In practice, effort data forms exponential distributions. Appendix B describes methods for using such distributions in effort modeling. Note that in COCOMO 81, Boehm identified three common types of software: {\em embedded, semi-detached}, and {\em organic}. Each has their own characteristic ``$a$'' and ``$b$'' (see \fig{eso}). COCOMO-II ignores these distinctions. This study used data sets in both the COCOMO 81 and COCOMO-II format. For more on the differences between COCOMO 81 and COCOMO-II, see Appendix A. \subsection{Data}\label{sec:data} The software project data we used in this study came from two sources (see \fig{parts}). $Coc81$ is the original COCOMO data used by Boehm to calibrate COCOMO 81. $Nasa93$ comes from a NASA-wide database recorded in the COCOMO 81 format. This data has been in the public domain for several years but few have been aware of it. It can now be found on-line in several places including the PROMISE (Predictor Models in Software Engineering) web site\footnote{\url{http://promise.site.uottawa.ca/SERepository/} and \url{http://unbox.org/wisp/trunk/cocomo/data}.}. $Nasa93$ was originally collected to create a NASA-tuned version of COCOMO, funded by the Space Station Freedom Program. $Nasa93$ contains data from six NASA centers including the Jet Propulsion Laboratory. Hence, it covers a very wide range of software domains, development processes, languages, and complexity as well as fundamental differences in culture and business practices between each center. All of these factors contribute to the large variances observed in this data set. When the $nasa93$ data was collected, it was required that there be multiple interviewers with one person leading the interview and one or two others recording and checking documentation. Each data point was cross-checked with either official records or via independent subjective inputs from other project personnel who fulfilled various roles on the project. After the data was translated into the COCOMO 81 format, the data was reviewed with those who originally provided the data. Once sufficient data existed the data was analyzed to identify outliers and the data values were re-verified with the development teams once again if deemed necessary. This typically required from two to four trips to each NASA center. All of the supporting information was placed in binders, which we still on occasion reference even today. Using Boehm's COCOMO-I ``local calibration`` the $nasa93$ data has been shown to contain large deviations due to the wide variety of projects in that data set, and {\em not} poor data collection. Our belief is that $nasa93$ was collected %and {\em not} poor data collection (ADD REFERENCE TO COSEEKMO PAPER). Our belief is that $nasa93$ was collected using methods equal to, or better, than standard industrial practice. If so, then industrial data would suffer from deviations equal to or lager than those seen in the $nasa93$ data. \begin{figure}[!t] {\footnotesize \begin{center} \begin{tabular}{r|r|r|p{1.8in}} \noindent Mode & a & b & notes\\\hline Organic & 3.2 & 1.05 & projects from relatively small software teams develop software in a highly familiar, in-house environment.\\\hline Embedded & 2.8& 1.2 & projects operating within (is embedded in) a strongly coupled complex of hardware, software, regulations, and operational procedures.\\\hline Semi-Detached & 3.0 & 1.12 & An intermediary mode between organic and embedded. \end{tabular} \end{center}} \caption{Standard COCOMO 81 development modes.}\label{fig:eso} \end{figure} \begin{figure*}{\footnotesize \begin{center} \begin{tabular}{|p{0.35in}|p{6.5in}|}\hline ~\newline ~\newline Data sources&\bd \item[{\em Coc81:}] 63 records in the COCOMO 81 format. Source: \cite[p496-497]{Boehm81}. \newline Download from \url{http://unbox.org/wisp/trunk/cocomo/data/coc81modeTypeLangType.csv}. \item[{\em Nasa93:}] ~~~ 93 NASA records in the COCOMO 81 format. \newline Download from \url{http://unbox.org/wisp/trunk/cocomo/data/nasa93.csv}. \item[{\em CocII:}] 161 records in the COCOMO II format from the COCOMO consortium (co-ordinated by USC). This data is not in the public domain. \ed\\\hline ~\newline ~\newline Data \newline subsets&\bd \item[{\em All:}] selects all records from a particular source; e.g. "coc81\_all''. \item[{\em Category:}] ~~~~~is a NASA-specific designation selecting the type of project; e.g. avionics, data capture, etc. \item[{\em Dev:}] indicates the development methodology; e.g. div.waterfall. \item[{\em DevEnd:}] ~~~shows the last year of the software project. \item[{\em Fg:}] selects either ``$f$'' (flight) or ``$g$'' (ground) software. \item[{\em Kind:}] selects records relating to the development platform; max= mainframe and mic= microprocessor. \item[{\em Lang:}] selects records about different development languages. \item[{\em Project} and {\em center}:]~~~~~~~~~~~~~~~ $nasa93$ designations selecting records relating to where the software was built and the name of the project. \item[{\em Mode=e:}]~\newline selects records relating to the {\em embedded} COCOMO 81 development mode. The different COCOMO 81 development models were described in \fig{eso}. \item[{\em Mode=o: }]~\newline selects COCOMO 81 {\em organic} mode records. \item[{\em Mode=sd: }]~\newline selects COCOMO 81 {\em semi-detached} mode records. \item[{\em Org:}] is a $cocII$ designation showing what organization provided the data. \item[{\em Size:}] is a $cocII$ specific designation grouping the records into (e.g.) those around 100KLOC. \item[{\em Type:}] selects different $coc81$ designations and include ``bus'' (for business application) or ``sys'' (for system software). \item[{\em Year:}] is a $nasa93$ term that selects the development years, grouped into units of five; e.g. 1970,1971,1972,1973,1974 are labeled ``1970''. \ed \\\hline \end{tabular} \end{center}} \caption{Data sets (top) and parts (bottom) of the data used in this study.}\label{fig:parts} \end{figure*} \subsection{Performance Measures} The performance of models generating continuous output can be assessed in many ways, including PRED(30), MMRE, correlation, etc. PRED(30) is a measure calculated from the relative error, or RE, which is the relative size of the difference between the actual and estimated value. One way to view these measures is to say that training data contains records with variables $1,2,3,..,N$ and performance measures add additional new variables $N+1,N+2,...$. The magnitude of the relative error, or MRE, is the absolute value of that relative error: {\small \[ MRE = |predicted - actual|/actual \]}\noindent The mean magnitude of the relative error, or MMRE, is the average percentage of the absolute values of the relative errors over an entire data set. MMRE results were shown in \fig{one} in the {\em mean\% average test error} column. Given $T$ tests, MMRE is calculated as follows: {\small \[MMRE = \frac{100}{T}\sum_i^T\frac{|predicted_i-actual_i|}{actual_i}\]}\noindent PRED(N) reports the average percentage of estimates that were within N\% of the actual values. Given $T$ tests, then: {\small \[ PRED(N) = \frac{100}{T} \sum_i^T\left\{ \begin{array}{l} 1\;if\;MRE_i\;{\le}\;\frac{N}{100}\\ 0\;otherwise \end{array}\right. \]}\noindent For example, a PRED(30)=50\% means that half the estimates are within 30\% of the actual. Another performance measure of a model predicting numeric values is the correlation between predicted and actual values. Correlation ranges from +1 to -1 and a correlation of +1 means that there is a perfect positive linear relationship between variables. Appendix C shows how to calculate correlation. All these performance measures (correlation, MMRE and PRED) address subtly different issues. Overall, PRED measures how {\em well} an effort model performs while MMRE measures {\em poor} performance. A single large mistake can skew the MMREs and not effect the PREDs. Shepperd and Schofield comment that: \begin{quote} MMRE is fairly conservative with a bias against overestimates while PRED(30) will identify those prediction systems that are generally accurate but occasionally wildly inaccurate~\cite[p736]{Shepperd97}. \end{quote} Since they measure different aspects of model performance, COSEEKMO uses combinations of PRED, MMRE, and correlation (using the methods described later in this paper). %\section{Attribute Ranking Driven Greedy Search Feature Subset Selection} \section{COCOMIN} \PARstart{F}{ollowing} the slow but steady success of COSEEKMO, it becomes necessary to find a similarly accurate algorithm which is fast enough to be used by business users in the real world. We decided to begin the search for such an algorithm using a minimal approach, that is, a feature subset selection algorithm that used a greedy search to minimize evaluations of the attribute space. This algorithm was named COCOMIN because it was a minimal approach to feature subset selection on the COCOMO model. Pseudocode for COCOMIN is in \fig{cocomin}. The code can be found at \url{http://unbox.org/wisp/trunk/cocomost/cocomin}. We decided to rank the attributes using correlation and to use the ranked results as the order to grow the attribute set in the case of a forward select greedy search, or prune the set in the case of a backward elimination greedy search. At each step the attribute set is evaluated with the MMRE, Pred(30), deviation, and correlation after using COCOMO regression with the attribute set in question. If the change is considered an improvement then it is kept and the search continues, otherwise it stops. We introduced a stale variable that could be set to allow for the search to continue even without improvement. % An extensive experiment was run to benchmark the %many customizations of the greedy FSS approach against standard COCOMO least squares regression as described in the %pseudocode in \fig{cocomin}. The results are shown in \fig{attrCOC81} and \fig{attrNASA}. For the COC81 dataset, %standard COCOMO least squares regression was found to be far superior. However, for the NASA93 dataset, many %customizations of the greedy feature subset selection had similar or slighty better MMRE and Pred(30) values, and lower deviations. %(refer to background on datasets to explain this). %Early results showed that although quick, this algorithm wasn't very accurate and accurate enough to be an acceptable replacement for even %avoiding feature subset selection entirely. Although it had some slight success with the NASA93 dataset it %failed to provide similar results to COCOMO regression on the COC81 dataset. Early results showed that this algorithm wasn't very accurate. Also, by noticing that the results tended to improve with higher stale values, we reasoned that the attribute space needed more exploration. Finally, with a dataset as small as the COC81 and NASA93 datasets, the importance of heuristics to prune the state space explosion were examined. Next we decided to build an efficient algorithm that evaluated all 32,768 attribute combinations. \begin{figure}[!b] {\scriptsize\begin{alltt} for attribute in rankedList newSet = bestSet + attribute if (search==backward) tmpSet=inverse(newSet) else tmpSet = newSet oldScore = newScore results = LC(train, train, tmpSet) newScore = results.eval if (newScore better than oldScore) bestSet = newSet staleCount = stale else newScore = oldScore staleCount-- if (staleCount < 1) exit for next attribute if (search==backward) bestSet=inverse(bestSet) return bestSet \end{alltt}} \caption{The cocomin algorithm which builds a subset of attributes using a greedy search guided by an attribute ranking. %\caption{This pseudocode outlines an experiment that compared standard COCOMO 81 local calibration %with an approach that ranks the attributes based on correlation, and then builds a subset of %attributes using a greedy search guided by the attribute ranking. }\label{fig:cocomin} \end{figure} \section{COCOMOST} %\begin{figure}[!t] \begin{figure}[!b] {\scriptsize\begin{alltt} attributes = null bestMMRE = LC(train, attributes) bestAttributes = null {\bf for} attributes {\bf in} 2^15 newMMRE = LC(train, attributes) if newMMRE > bestMMRE bestMMRE = newMMRE bestAttributes = attributes return bestAttributes \end{alltt}} \caption{Cocomost performs a complete search over the attribute space and evaluates the attribute sets using the target learner: COCOMO-based local calibration. }\label{fig:cocomost} \end{figure} \PARstart{T}{he} COCOMOST algorithm, as outlined in \fig{cocomost}, uses feature subset selection to prune irrelevant, redundant, noisy, and unreliable attributes from the COCOMO model. The code begins at \url{http://unbox.org/wisp/trunk/cocomost/cocomost}. It executes a complete search of the attribute space, evaluating attribute sets using local calibration. Thus, it is a ``wrapper`` attribute selection technique instead of a ``filter`` because it evaluates using the target learner~\cite{Hall03}. However, it shares several of the advantages of a filter. Unlike most wrappers, COCOMOST is fast enough to search the entire attribute space instead of using heuristics to limit the state space explosion. This introduces the vulnerability that every attribute beyond the 15 used in this study will double COCOMOST's execution time. %To compare the effectiveness of standard local calibration versus feature subset selection with COCOMOST followed by local calibration, we randomly pulled a test set from the training data 30 times and made estimates with each learner as described in \fig{study}. \section{Experimental Design} \PARstart{C}{onsidering} the small amount of data available, we decided to use n-fold cross validation to compare the COCOMIN and COCOMOST learners against standard COCOMO-based local calibration as shown in \fig{compete}. The code for this experiment can be found at \url{http://unbox.org/wisp/trunk/cocomost/compete}. %\fig{compete} shows pseudocode describing this experiment. %\begin{figure}[!t] \begin{figure}[!b] {\scriptsize\begin{alltt} {\bf for} data {\bf in} dataSets num = countRecords(data) {\bf for} i {\bf in} 1 {\bf to} num test = data.i train = data - test attr = (all COCOMO 81 attributes) results = LC(test, train, attr) print variables and results attr = cocomost(train) results = LC(test, train, attr) print variables and results for attr in COCOMO_81_attributes rank correlation using LC(train, train, attr) order += sorted list of ranked attrs for search in "forward backward" for stale in "0 1 2 4 8 16" for eval in "mmre sd_mre pred30 corr" attr=cocomin(test,train,search,stale,eval,order) results = LC(test, train, attrs) print variables and results \end{alltt}} \caption{This experiment benchmarks standard COCOMO-based local calibration against local calibration that uses cocomost to perform feature subset selection, or local calibration that uses each customization of cocomin for feature subset selection. }\label{fig:compete} \end{figure} %\section{Data} %\PARstart{T}{he} training data used in the experiments are the coc81 and nasa93 datasets available at \protect\url{http://unbox.org/wisp/trunk/data}. %The coc81 dataset comes from Boehm~\cite{Boehm81} and the nasa93 dataset contains project metrics from NASA software projects that have been sanitized for public use. %In addition, the coc81 dataset contains 6 stratifications, and the nasa93 dataset contains 11. %The learners were run on all 19 of these datasets to create the experimental results. % comes after %\section{Results} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Quartile Charts of Performance Deltas} One method we used to assess the experimental results was to create quartile charts of the performance deltas of the learners. We prefer quartile charts of performance deltas to other summarization methods for M*N studies. Firstly, they offer a very succinct summary of a large number of experiments. Secondly, they are a {\em non-parametric} display; i.e. they make no assumptions about the underling distribution. Standard practice in data mining is to compare the mean performance of different methods using t-test~\cite{Witten05}. T-tests are a {\em parametric method} that assume that the underling population distribution is a Gaussian. Recent results suggest that there are many statistical issues left to explore regarding how to best to apply those t-tests for summarizing M*N-way studies~\cite{Brouck03}. The performance deltas were computed using simple subtraction, defined as follows. A {\em positive performance delta} for method X means that method X has out-performed some other method in {\em one} comparison. Using performance deltas, we say that the best method is the one that generates the largest performance deltas over {\em all} comparisons. The performance deltas for each method were sorted and displayed as {\em quartile charts}. To generate these charts, the performance deltas for some $method$ were sorted to find the lowest and highest quartile as well as the median value; e.g. \[ {\footnotesize \overbrace{\underbrace{-59}_{min},-19,-19}^{lowest\; quartile},-16,-14,-10,\underbrace{-10}_{median},5,14,39,\overbrace{42,62,\underbrace{69}_{max}}^{highest\; quartile} } \] In a quartile chart, the upper and lower quartiles are marked with black lines; the median is marked with a black dot; and vertical bars are added to mark (i)~the zero point and (ii)~the minimum possible value and (iii)~the maximum possible value (in our case, -100\% and 100\%). The above numbers would therefore be drawn as follows: {\small \[-100\%\; \boxplot{20}{10}{45}{80}{14}\; 100\%\]} %\begin{figure} %{\footnotesize{\bf Pd:} %\input{pd.changes}} %{\footnotesize {\bf NotPf=$100- pf$} %\input{notPf.changes}} %{\footnotesize {\bf Balance} %\input{bal.changes}} %\caption{Performance deltas for $pd,notPf,bal$ using all 38 attributes.}\label{fig:allpd}. %\end{figure} \begin{comment} \begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2} method & median & & & \\\cline{1-2} cocomin\_forward\_0\_correlation\_sd\_mre & 4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\ cocomin\_forward\_0\_correlation\_pred30 & 4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\ cocomin\_forward\_0\_correlation\_mmre & 4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\ cocomin\_forward\_0\_correlation\_correlation & 4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\ cocomin\_forward\_1\_correlation\_pred30 & 4.1 & -100\% & \boxplot{-328}{375}{52}{60}{233}&100\%\\ cocomin\_forward\_1\_correlation\_sd\_mre & 2.2 & -100\% & \boxplot{-328}{375}{51}{58}{235}&100\%\\ cocomin\_forward\_1\_correlation\_mmre & 1.7 & -100\% & \boxplot{-328}{375}{50}{57}{396}&100\%\\ cocomin\_forward\_2\_correlation\_pred30 & 1.5 & -100\% & \boxplot{-254}{300}{50}{57}{188}&100\%\\ cocomin\_backward\_16\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-166}{211}{50}{56}{310}&100\%\\ cocomin\_backward\_8\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-166}{211}{50}{56}{310}&100\%\\ cocomin\_forward\_4\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-198}{243}{50}{56}{190}&100\%\\ cocomin\_backward\_4\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-328}{373}{50}{55}{217}&100\%\\ cocomin\_forward\_4\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\ cocomin\_forward\_1\_correlation\_correlation & 0.3 & -100\% & \boxplot{-280}{326}{50}{56}{262}&100\%\\ cocomin\_forward\_2\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-254}{299}{50}{55}{245}&100\%\\ cocomin\_forward\_16\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\ cocomin\_forward\_8\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\ cocomin\_forward\_8\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-198}{242}{50}{55}{190}&100\%\\ cocomin\_forward\_16\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-198}{242}{50}{55}{190}&100\%\\ cocomin\_backward\_16\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-318}{363}{50}{54}{351}&100\%\\ cocomin\_backward\_2\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-328}{373}{50}{54}{243}&100\%\\ cocomin\_forward\_2\_correlation\_mmre & 0.0 & -100\% & \boxplot{-173}{218}{50}{54}{266}&100\%\\ cocomin\_backward\_8\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-318}{363}{50}{54}{352}&100\%\\ cocomin\_backward\_4\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-318}{363}{50}{53}{352}&100\%\\ cocomin\_backward\_1\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{331}&100\%\\ cocomin\_backward\_1\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-328}{373}{50}{53}{352}&100\%\\ cocomin\_forward\_2\_correlation\_correlation & 0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\ cocomin\_backward\_8\_correlation\_mmre & -0.3 & -100\% & \boxplot{-328}{372}{49}{53}{299}&100\%\\ cocomin\_backward\_16\_correlation\_mmre & -0.5 & -100\% & \boxplot{-225}{270}{49}{53}{299}&100\%\\ cocomin\_forward\_8\_correlation\_correlation & 0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\ cocomin\_forward\_4\_correlation\_correlation & 0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\ cocomin\_forward\_16\_correlation\_correlation & 0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\ cocomin\_backward\_8\_correlation\_correlation & 0.0 & -100\% & \boxplot{-227}{272}{50}{53}{257}&100\%\\ cocomin\_backward\_16\_correlation\_correlation & 0.0 & -100\% & \boxplot{-227}{272}{50}{53}{257}&100\%\\ cocomin\_backward\_0\_correlation\_sd\_mre & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\ cocomin\_backward\_0\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\ cocomin\_backward\_0\_correlation\_mmre & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\ cocomin\_backward\_0\_correlation\_correlation & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\ cocomin\_backward\_2\_correlation\_pred30 & 0.0 & -100\% & \boxplot{-328}{372}{50}{53}{352}&100\%\\ cocomin\_backward\_2\_correlation\_correlation & 0.0 & -100\% & \boxplot{-325}{370}{50}{53}{264}&100\%\\ LC & -1.3 & -100\% & \boxplot{-354}{397}{49}{54}{374}&100\%\\ cocomin\_backward\_4\_correlation\_correlation & 0.0 & -100\% & \boxplot{-311}{356}{50}{53}{257}&100\%\\ cocomin\_backward\_1\_correlation\_mmre & 0.0 & -100\% & \boxplot{-328}{372}{50}{52}{353}&100\%\\ cocomin\_backward\_1\_correlation\_correlation & 0.0 & -100\% & \boxplot{-325}{369}{50}{53}{265}&100\%\\ cocomin\_backward\_4\_correlation\_mmre & -0.2 & -100\% & \boxplot{-328}{372}{49}{53}{299}&100\%\\ cocomin\_forward\_4\_correlation\_mmre & -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\ cocomin\_backward\_2\_correlation\_mmre & -0.6 & -100\% & \boxplot{-328}{372}{49}{52}{353}&100\%\\ cocomin\_forward\_8\_correlation\_mmre & -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\ cocomin\_forward\_16\_correlation\_mmre & -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\ cocomost & -2.0 & -100\% & \boxplot{-354}{397}{49}{53}{264}&100\%\\ \cline{1-2}\end{tabular} \end{comment} %method&median&-808.2\%&\boxplot{50}{0}{50}{53}{0}&808.2\%\\ \begin{comment} \scriptsize \begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2} method&median&&&\\\cline{1-2} cocomin-f-0-sd-mre& 4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-pred30& 4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-mmre& 4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-corr& 4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-1-pred30& 4.1&-808.2\%&\boxplot{3}{46}{52}{53}{28}&808.2\%\\ cocomin-f-1-sd-mre& 2.2&-808.2\%&\boxplot{3}{46}{51}{53}{29}&808.2\%\\ cocomin-f-1-mmre& 1.7&-808.2\%&\boxplot{3}{46}{50}{53}{49}&808.2\%\\ cocomin-f-2-pred30& 1.5&-808.2\%&\boxplot{12}{37}{50}{53}{23}&808.2\%\\ cocomin-b-16-sd-mre& 0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\ cocomin-b-8-sd-mre& 0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\ cocomin-f-4-pred30& 0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-b-4-sd-mre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{26}&808.2\%\\ cocomin-f-4-sd-mre& 0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-1-corr& 0.3&-808.2\%&\boxplot{9}{40}{50}{53}{32}&808.2\%\\ cocomin-f-2-sd-mre& 0.0&-808.2\%&\boxplot{12}{37}{50}{53}{30}&808.2\%\\ cocomin-f-16-sd-mre& 0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-8-sd-mre& 0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-8-pred30& 0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-f-16-pred30& 0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-b-16-pred30& 0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-2-sd-mre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{30}&808.2\%\\ cocomin-f-2-mmre& 0.0&-808.2\%&\boxplot{22}{27}{50}{53}{32}&808.2\%\\ cocomin-b-8-pred30& 0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-4-pred30& 0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-1-sd-mre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{41}&808.2\%\\ cocomin-b-1-pred30& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-f-2-corr& 0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-b-8-mmre& -0.3&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\ cocomin-b-16-mmre& -0.5&-808.2\%&\boxplot{15}{33}{49}{53}{37}&808.2\%\\ cocomin-f-8-corr& 0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-f-4-corr& 0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-f-16-corr& 0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-b-8-corr& 0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\ cocomin-b-16-corr& 0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\ cocomin-b-0-sd-mre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-pred30& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-mmre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-corr& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-2-pred30& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-2-corr& 0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\ LC& -1.3&-808.2\%&\boxplot{0}{49}{49}{53}{46}&808.2\%\\ cocomin-b-4-corr& 0.0&-808.2\%&\boxplot{5}{44}{50}{53}{31}&808.2\%\\ cocomin-b-1-mmre& 0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-1-corr& 0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\ cocomin-b-4-mmre& -0.2&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\ cocomin-f-4-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomin-b-2-mmre& -0.6&-808.2\%&\boxplot{3}{46}{49}{53}{43}&808.2\%\\ cocomin-f-8-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomin-f-16-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomost& -2.0&-808.2\%&\boxplot{0}{49}{49}{53}{32}&808.2\%\\ \cline{1-2}\end{tabular} \end{comment} \scriptsize \begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2} method&median&&&\\\cline{1-2} cocomin-f-0-sd-mre&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-pred30&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-mmre&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-0-corr&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\ cocomin-f-1-pred30&4.1&-808.2\%&\boxplot{3}{46}{52}{53}{28}&808.2\%\\ cocomin-f-1-sd-mre&2.2&-808.2\%&\boxplot{3}{46}{51}{53}{29}&808.2\%\\ cocomin-f-1-mmre&1.7&-808.2\%&\boxplot{3}{46}{50}{53}{49}&808.2\%\\ cocomin-f-2-pred30&1.5&-808.2\%&\boxplot{12}{37}{50}{53}{23}&808.2\%\\ cocomin-b-16-sd-mre&0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\ cocomin-b-8-sd-mre&0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\ cocomin-f-4-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-b-4-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{26}&808.2\%\\ cocomin-f-4-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-1-corr&0.3&-808.2\%&\boxplot{9}{40}{50}{53}{32}&808.2\%\\ cocomin-f-2-sd-mre&0.0&-808.2\%&\boxplot{12}{37}{50}{53}{30}&808.2\%\\ cocomin-f-16-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-8-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\ cocomin-f-8-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-f-16-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\ cocomin-b-16-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-2-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{30}&808.2\%\\ cocomin-f-2-mmre&0.0&-808.2\%&\boxplot{22}{27}{50}{53}{32}&808.2\%\\ cocomin-b-8-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-4-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\ cocomin-b-1-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{41}&808.2\%\\ cocomin-b-1-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-f-2-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-b-8-mmre&-0.3&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\ cocomin-b-16-mmre&-0.5&-808.2\%&\boxplot{15}{33}{49}{53}{37}&808.2\%\\ cocomin-f-8-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-f-4-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-f-16-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\ cocomin-b-8-corr&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\ cocomin-b-16-corr&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\ cocomin-b-0-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-mmre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-0-corr&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-2-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-2-corr&0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\ LC&-1.3&-808.2\%&\boxplot{0}{49}{49}{53}{46}&808.2\%\\ cocomin-b-4-corr&0.0&-808.2\%&\boxplot{5}{44}{50}{53}{31}&808.2\%\\ cocomin-b-1-mmre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\ cocomin-b-1-corr&0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\ cocomin-b-4-mmre&-0.2&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\ cocomin-f-4-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomin-b-2-mmre&-0.6&-808.2\%&\boxplot{3}{46}{49}{53}{43}&808.2\%\\ cocomin-f-8-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomin-f-16-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\ cocomost&-2.0&-808.2\%&\boxplot{0}{49}{49}{53}{32}&808.2\%\\ \cline{1-2}\end{tabular} \normalsize %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Results} %%%%%%%%Deltas%%%%%%%%%%%%%%%%%%% %\begin{figure} %{\footnotesize{\bf all:} %\input{cocomost_all.changes}} %%{\footnotesize {\bf NotPf=$100- pf$} %%\input{notPf.changes}} %%{\footnotesize {\bf Balance} %%\input{bal.changes}} %%\caption{Performance deltas for $pd,notPf,bal$ using all 38 attributes.}\label{fig:allpd}. %\caption{Performance deltas for }\label{fig:allpd} %\end{figure} \begin{comment} \begin{figure}[!b] {\scriptsize\begin{alltt} ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomin_forward_0_correlation_sd_mre, 13.9, 63.0,36015,-757.4, -5.0, 4.9, 24.8,808.2,**********|.********* 2, (= 1), cocomin_forward_0_correlation_pred30, 13.9, 63.0,36015,-757.4, -5.0, 4.9, 24.8,808.2,**********|.********* 3, (= 1), cocomin_forward_0_correlation_mmre, 13.9, 63.0,36015,-757.4, -5.0, 4.9, 24.8,808.2,**********|.********* 4, (= 1), cocomin_forward_0_correlation_correlation, 13.9, 63.0,36015,-757.4, -5.0, 4.9, 24.8,808.2,**********|.********* 5, (< 1), cocomin_forward_1_correlation_pred30, 9.1, 51.5,36015,-757.4, -5.7, 4.1, 20.2,487.7,**********|.********* 6, (< 5), cocomin_forward_1_correlation_sd_mre, 6.2, 48.8,36015,-757.4, -6.7, 2.2, 17.6,487.7,**********|.********* 7, (< 6), cocomin_forward_1_correlation_mmre, 4.8, 48.3,36015,-757.4, -7.3, 1.7, 15.7,808.2,**********|.********* 8, (= 7), cocomin_forward_2_correlation_pred30, 4.6, 44.5,36015,-608.9, -8.0, 1.5, 15.8,393.5,**********|.********* 9, (< 7), cocomin_backward_16_correlation_sd_mre, 3.0, 41.3,36015,-432.3, -8.9, 0.0, 12.4,632.7,**********|********** 10, (= 9), cocomin_backward_8_correlation_sd_mre, 2.9, 41.3,36015,-432.3, -8.9, 0.0, 12.3,632.7,**********|********** 11, (< 9), cocomin_forward_4_correlation_pred30, 1.2, 40.6,36015,-497.4,-10.2, 0.0, 13.1,393.5,*********.|********** 12, (= 11), cocomin_backward_4_correlation_sd_mre, 0.8, 39.8,36015,-756.6, -8.7, 0.0, 10.2,446.1,*********.|********** 13, (= 11), cocomin_forward_4_correlation_sd_mre, 0.7, 40.7,36015,-551.2, -9.5, 0.0, 11.0,418.0,**********|********** 14, (= 11), cocomin_forward_1_correlation_correlation, 0.7, 39.9,36015,-660.9, -8.7, 0.3, 12.0,536.4,**********|********** 15, (= 11), cocomin_forward_2_correlation_sd_mre, 0.6, 41.8,36015,-609.3, -9.4, 0.0, 11.1,502.9,*********.|********** 16, (< 11), cocomin_forward_16_correlation_sd_mre, 0.6, 40.8,36015,-551.2,-10.0, 0.0, 11.1,418.0,**********|********** 17, (= 16), cocomin_forward_8_correlation_sd_mre, 0.1, 40.6,36015,-551.2,-10.3, 0.0, 10.3,418.0,**********|********** 18, (< 16), cocomin_forward_8_correlation_pred30, -0.5, 39.6,36015,-497.4,-11.6, 0.0, 11.8,393.5,*********.|********** 19, (= 18), cocomin_forward_16_correlation_pred30, -0.5, 39.6,36015,-497.4,-11.6, 0.0, 11.8,393.5,*********.|********** 20, (< 18), cocomin_backward_16_correlation_pred30, -1.1, 39.8,36015,-737.3,-10.9, 0.0, 9.3,712.7,*********.|********** 21, (= 20), cocomin_backward_2_correlation_sd_mre, -1.2, 37.9,36015,-756.6, -9.0, 0.0, 9.1,495.8,*********.|********** 22, (= 20), cocomin_forward_2_correlation_mmre, -1.5, 39.8,36015,-447.4, -9.6, 0.0, 9.3,541.6,*********.|********** 23, (= 20), cocomin_backward_8_correlation_pred30, -1.6, 39.6,36015,-737.3,-10.7, 0.0, 8.7,712.7,*********.|********** 24, (< 20), cocomin_backward_4_correlation_pred30, -2.0, 39.7,36015,-737.3,-10.7, 0.0, 7.6,712.7,*********.|********** 25, (= 24), cocomin_backward_1_correlation_sd_mre, -2.1, 40.4,36015,-756.6,-10.7, 0.0, 7.0,670.2,*********.|********** 26, (= 24), cocomin_backward_1_correlation_pred30, -2.3, 40.9,36015,-756.6,-10.4, 0.0, 7.1,712.7,*********.|********** 27, (< 24), cocomin_forward_2_correlation_correlation, -2.6, 35.9,36015,-539.5, -9.4, 0.0, 8.7,572.5,**********|********** 28, (= 27), cocomin_backward_8_correlation_mmre, -2.6, 36.5,36015,-756.6,-11.4, -0.3, 6.7,604.8,*********.|********** 29, (= 27), cocomin_backward_16_correlation_mmre, -2.6, 35.2,36015,-551.7,-11.7, -0.5, 6.7,604.8,*********.|********** 30, (= 27), cocomin_forward_8_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5, 0.0, 8.6,572.5,*********.|********** 31, (= 27), cocomin_forward_4_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5, 0.0, 8.6,572.5,*********.|********** 32, (= 27), cocomin_forward_16_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5, 0.0, 8.6,572.5,*********.|********** 33, (= 27), cocomin_backward_8_correlation_correlation, -2.7, 34.8,36015,-555.0, -9.9, 0.0, 7.9,523.2,**********|********** 34, (= 27),cocomin_backward_16_correlation_correlation, -2.7, 34.8,36015,-555.0, -9.9, 0.0, 7.9,523.2,**********|********** 35, (= 27), cocomin_backward_0_correlation_sd_mre, -2.8, 42.4,36015,-756.6,-10.9, 0.0, 6.4,712.7,*********.|********** 36, (= 27), cocomin_backward_0_correlation_pred30, -2.8, 42.4,36015,-756.6,-10.9, 0.0, 6.4,712.7,*********.|********** 37, (= 27), cocomin_backward_0_correlation_mmre, -2.8, 42.4,36015,-756.6,-10.9, 0.0, 6.4,712.7,*********.|********** 38, (= 27), cocomin_backward_0_correlation_correlation, -2.8, 42.4,36015,-756.6,-10.9, 0.0, 6.4,712.7,*********.|********** 39, (= 27), cocomin_backward_2_correlation_pred30, -2.9, 39.4,36015,-756.6,-10.7, 0.0, 7.3,712.7,*********.|********** 40, (= 27), cocomin_backward_2_correlation_correlation, -3.0, 36.0,36015,-750.2, -9.8, 0.0, 7.9,536.4,**********|********** 41, (= 27), LC, -3.0, 48.8,36015,-808.2,-13.5, -1.3, 8.0,757.4,*********.|********** 42, (< 27), cocomin_backward_4_correlation_correlation, -3.2, 35.7,36015,-722.6, -9.9, 0.0, 7.5,523.2,**********|********** 43, (= 42), cocomin_backward_1_correlation_mmre, -3.4, 41.7,36015,-756.6,-11.3, 0.0, 5.5,712.7,*********.|********** 44, (= 42), cocomin_backward_1_correlation_correlation, -3.6, 37.3,36015,-750.2,-11.3, 0.0, 6.4,536.4,*********.|********** 45, (< 42), cocomin_backward_4_correlation_mmre, -4.0, 36.1,36015,-756.6,-11.2, -0.2, 6.1,604.8,*********.|********** 46, (= 45), cocomin_forward_4_correlation_mmre, -4.2, 38.4,36015,-447.4,-12.0, -0.7, 6.6,541.6,*********.|********** 47, (= 45), cocomin_backward_2_correlation_mmre, -4.3, 38.7,36015,-756.6,-11.3, -0.6, 4.9,712.7,*********.|********** 48, (= 45), cocomin_forward_8_correlation_mmre, -4.5, 38.4,36015,-447.4,-12.3, -0.7, 6.6,541.6,*********.|********** 49, (= 45), cocomin_forward_16_correlation_mmre, -4.5, 38.4,36015,-447.4,-12.3, -0.7, 6.6,541.6,*********.|********** 50, (< 45), cocomost, -5.1, 38.1,36015,-808.2,-13.3, -2.0, 6.7,536.5,*********.|********** \end{alltt}} \caption{Performance deltas for all learners combinations over all 19 datasets collectively. }\label{fig:deltas_all} \end{figure} % 3 learners by 2 coc81,nasa93 \begin{figure}[!b] {\scriptsize\begin{alltt} coc81 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1),cocomin_forward_16_correlation_mmre, 4.3, 40.0,420, -35.9, -4.5, 0.0, 5.5,541.6,......****|********** 2, (= 1), cocomost, 0.5, 30.2,420,-541.6, -5.2, 0.4, 7.2, 71.3,**********|*******... 3, (< 1), LC, -4.8, 30.4,420,-541.6, -7.2, 0.0, 2.1, 35.9,**********|***....... nasa93 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 4.4, 46.4,1050,-313.5, -8.6, 0.7, 11.1,383.1,**********|********** 2, (< 1), cocomost, -2.0, 34.8,1050,-383.1, -8.0, -0.1, 6.8,313.5,**********|********** 3, (= 2),cocomin_forward_16_correlation_mmre, -2.3, 36.7,1050,-370.7, -7.3, 0.0, 7.1,313.5,**********|********** \end{alltt}} \caption{Performance deltas for LC, cocomost, and cocomin over the coc81 and nasa93 datasets. }\label{fig:deltas_byCoc81Nasa93} \end{figure} % 3 learners by each subset \begin{figure}[!b] {\scriptsize\begin{alltt} coc81_all ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 0.7, 8.8,126,-16.9, -5.3, 0.1, 5.2, 23.9,........**|**........ 2, (= 1),cocomin_forward_16_correlation_mmre, -0.4, 6.2,126,-23.9, -0.1, 0.0, 0.0, 16.9,........**|*......... 3, (= 1), LC, -0.4, 6.2,126,-23.9, -0.1, 0.0, 0.0, 16.9,........**|*......... coc81_kind_min ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1),cocomin_forward_16_correlation_mmre, 28.7,117.0,42, -18.4, -8.0, 2.6, 11.8,541.6,........**|********** 2, (= 1), cocomost,-11.9, 84.4,42,-541.6, -3.6, 0.8, 8.5, 22.6,**********|**........ 3, (< 1), LC,-16.8, 83.9,42,-541.6,-10.2, -2.6, 3.6, 18.4,**********|*......... coc81_lang_ftn ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1),cocomin_forward_16_correlation_mmre, 5.2, 22.8,48, -28.3, -3.7, 0.4, 10.4,100.0,.......***|*********. 2, (= 1), LC, 3.9, 20.3,48, -93.0, -3.9, 2.8, 16.6, 34.5,.*********|.**....... 3, (< 1), cocomost, -9.1, 18.3,48,-100.0,-16.5, -7.0, 0.0, 19.7,*********|**......... coc81_mode_e ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 6.0, 18.3,56,-32.9, -7.7, 1.6, 15.7, 61.5,.......***|.*****.... 2, (= 1),cocomin_forward_16_correlation_mmre, 4.6, 17.8,56,-27.9, -5.9, 0.0, 11.2, 71.7,.......***|*******... 3, (< 1), LC,-10.6, 19.6,56,-71.7,-21.6, -8.1, 3.4, 22.8,...******|**......... coc81_kind_max ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 2.6, 11.0,62,-38.6, 0.0, 0.0, 5.9, 25.2,......****|**........ 2, (= 1),cocomin_forward_16_correlation_mmre, 2.0, 12.0,62,-24.4, 0.0, 0.0, 0.0, 43.9,........**|****...... 3, (< 1), LC, -4.6, 10.1,62,-43.9, -5.4, 0.0, 0.0, 7.9,......****|.......... coc81_mode_org ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 6.3, 15.0,46,-31.2, -2.0, 3.9, 10.0, 62.4,.......***|******.... 2, (< 1),cocomin_forward_16_correlation_mmre, -1.5, 17.0,46,-35.9, -7.4, -0.5, 4.4, 72.5,......***.|******.... 3, (= 2), LC, -4.8, 20.0,46,-72.5,-10.0, -2.6, 3.2, 35.9,...*******|***....... coc81_lang_mol ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 6.1, 18.1,40,-32.1, -2.5, 1.6, 11.8, 71.3,.......***|******.... 2, (= 1),cocomin_forward_16_correlation_mmre, 2.7, 17.7,40,-24.1,-11.7, -0.5, 14.2, 49.4,........**|****...... 3, (< 1), LC, -8.7, 19.4,40,-71.3,-15.2, -3.5, 0.6, 20.9,...******.|**........ nasa93_all ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 3.1, 36.7,186,-149.1, -8.9, -1.6, 8.5,195.5,**********|********** 2, (= 1),cocomin_forward_16_correlation_mmre, -0.8, 28.0,186,-195.5, -7.6, 0.0, 7.9,149.1,*********.|********** 3, (= 1), cocomost, -2.3, 28.8,186,-195.5, -7.9, 0.5, 7.8,121.5,*********.|********** nasa93_project_gro ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 8.1, 56.2,46, -46.2,-11.7, 4.3, 10.0,331.2,.....****.|********** 2, (= 1), cocomost, 3.3, 37.6,46,-147.7, -4.9, 2.4, 6.9,183.5,**********|********** 3, (= 1),cocomin_forward_16_correlation_mmre,-11.3, 56.7,46,-331.2, -8.8, -3.5, 3.0, 38.0,**********|***....... nasa93_cat2_missionplanning ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 5.7, 21.9,40,-46.3, -7.2, 4.0, 10.8, 94.0,.....****.|*********. 2, (= 1), LC, 3.0, 25.4,40,-40.9,-15.6, 6.8, 13.6, 78.4,......***..|******... 3, (< 1),cocomin_forward_16_correlation_mmre, -8.7, 24.1,40,-94.0,-12.5, -6.3, 0.9, 40.9,.********|****....... nasa93_cat2_avionicsmonitoring ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1),cocomin_forward_16_correlation_mmre, 9.0, 42.8,60, -25.6, -2.5, 0.8, 7.1,313.5,.......***|********** 2, (= 1), cocomost, 7.3, 44.2,60, -28.1, -3.2, -0.3, 3.5,313.5,.......***|********** 3, (< 1), LC,-16.3, 59.7,60,-313.5, -7.1, -1.5, 2.7, 13.3,**********|*......... nasa93_mode_semidetached ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 11.6, 49.5,138, -53.8, -6.2, 1.4, 15.1,314.9,.....*****|.********* 2, (< 1),cocomin_forward_16_correlation_mmre, -4.0, 36.5,138,-291.1, -7.5, 0.1, 6.5, 53.8,**********|*****..... 3, (= 2), cocomost, -7.6, 36.1,138,-314.9,-10.6, -1.4, 5.5, 38.9,**********|****...... nasa93_project_sts ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 6.4, 31.4,76, -83.0, -4.9, 4.2, 14.7,147.3,..********|********** 2, (= 1), cocomost, -2.1, 25.2,76,-147.3, -8.2, -2.7, 4.8, 83.0,*********.|*******... 3, (< 1),cocomin_forward_16_correlation_mmre, -4.2, 23.1,76,-147.3,-11.4, -1.1, 4.5, 69.2,*********.|******.... nasa93_forg_g ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 1.7, 48.9,160,-262.1, -8.9, -0.1, 9.9,294.6,**********|********** 2, (= 1), cocomost, 0.7, 33.8,160,-201.7, -7.4, 1.8, 8.9,246.5,**********|********** 3, (= 1),cocomin_forward_16_correlation_mmre, -2.4, 39.3,160,-294.6, -9.9, -1.9, 7.0,262.1,**********|********** nasa93_center_5 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 2.4, 36.1,78,-144.5,-10.1, 1.4, 16.1,147.6,**********|.********* 2, (= 1), cocomost, -1.1, 31.5,78,-147.6,-12.4, -4.1, 5.2,144.5,*********.|********** 3, (= 1),cocomin_forward_16_correlation_mmre, -1.3, 26.2,78,-147.6, -5.1, 2.5, 8.0, 89.4,**********|*********. nasa93_year_1975 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 9.4, 40.3,74, -68.4, -6.1, 5.1, 16.8,212.3,...*******.|********* 2, (< 1),cocomin_forward_16_correlation_mmre, -4.1, 29.7,74,-212.3, -7.1, 0.0, 2.4, 58.5,**********|*****..... 3, (= 2), cocomost, -5.3, 29.0,74,-212.3, -7.5, -1.5, 4.3, 68.4,**********|******.... nasa93_year_1980 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 9.7, 59.6,76,-104.8,-12.6, 3.0, 17.9,383.1,**********|.********* 2, (= 1),cocomin_forward_16_correlation_mmre, 2.4, 45.1,76,-236.1, -8.9, 1.9, 11.0,147.0,*********.|********** 3, (< 1), cocomost,-12.2, 54.5,76,-383.1,-15.7, -4.3, 6.8, 72.5,*********.|*******... nasa93_mode_embedded ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), LC, 17.6, 73.7,42, -42.4, -8.0, -3.8, 28.4,370.7,......****|..******** 2, (= 1), cocomost, -4.6, 48.1,42,-270.7, -2.8, 0.0, 5.7,100.0,**********|********** 3, (< 1),cocomin_forward_16_correlation_mmre,-13.0, 61.1,42,-370.7, -6.8, 0.0, 2.8, 42.4,**********|***....... nasa93_center_2 ID,deltas(95%), #rx, mean, sd, n, min, q1,median, q3, max,..........0.......... 1, (= 1), cocomost, 2.1, 21.6,74,-101.4, -5.6, -1.1, 8.2,119.0,**********|********** 2, (= 1),cocomin_forward_16_correlation_mmre, 0.9, 32.5,74,-220.4, -3.2, 3.0, 10.6, 65.5,**********|******.... 3, (= 1), LC, -3.0, 32.8,74, -65.5,-15.7, -4.2, 4.5,220.4,...*****..|********** \end{alltt}} \caption{Performance deltas for LC, cocomost, and cocomin over each of the 19 datasets separately. }\label{fig:deltas_subsets} \end{figure} \end{comment} % info about attributes dropped \begin{figure}[!b] {\scriptsize\begin{alltt} Average # of Attributes Used: 13.6 Average # of Dropped Attributes: 1.4 Percentage of Times Each Attribute is Dropped: rely : 2.4% data : 11.0% cplx : 9.0% time : 0.5% stor : 12.9% virt : 1.4% turn : 64.3% acap : 1.0% aexp : 0.0% pcap : 0.0% vexp : 0.0% lexp : 1.9% modp : 19.0% tool : 17.1% sced : 0.5% \end{alltt}} \caption{Attributes Dropped by COCOMOST for the COC81 data. }\label{fig:attributes_cocomost_coc81} \end{figure} \begin{figure}[!b] {\scriptsize\begin{alltt} Average # of Attributes Used: 8.1 Average # of Dropped Attributes: 6.9 Percentage of Times Each Attribute is Dropped: rely : 34.1% data : 5.3% cplx : 30.9% time : 9.7% stor : 67.0% virt : 30.9% turn : 15.0% acap : 25.0% aexp : 32.6% pcap : 68.0% vexp : 61.5% lexp : 72.2% modp : 75.6% tool : 82.9% sced : 79.4% \end{alltt}} \caption{Attributes Dropped by COCOMOST for the NASA93 data. }\label{fig:attributes_cocomost_nasa93} \end{figure} \begin{figure}[!b] {\scriptsize\begin{alltt} Average # of Attributes Used: 13.6 Average # of Dropped Attributes: 1.4 Percentage of Times Each Attribute is Dropped: rely : 1.0% data : 14.3% cplx : 13.8% time : 10.0% stor : 2.4% virt : 1.9% turn : 36.2% acap : 11.4% aexp : 0.5% pcap : 2.4% vexp : 7.1% lexp : 7.1% modp : 12.4% tool : 13.8% sced : 01.0% \end{alltt}} \caption{Attributes Dropped by COCOMIN for the COC81 data. }\label{fig:attributes_cocomin_coc81} \end{figure} \begin{figure}[!b] {\scriptsize\begin{alltt} Average # of Attributes Used: 7.7 Average # of Dropped Attributes: 7.3 Percentage of Times Each Attribute is Dropped: rely : 36.8% data : 45.5% cplx : 41.0% time : 6.1% stor : 52.2% virt : 21.0% turn : 16.6% acap : 22.5% aexp : 35.2% pcap : 60.2% vexp : 58.7% lexp : 65.3% modp : 93.9% tool : 88.0% sced : 82.5% \end{alltt}} \caption{Attributes Dropped by COCOMIN for the NASA93 data. }\label{fig:attributes_cocomin_nasa93} \end{figure} %%%%%%%%GRAPHS%%%%%%%%%%%%%%%%%%% \begin{figure}[!t] \begin{center} [{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/mmre.pdf}}] \caption{Mean Magnitude of Relative Error} \label{fig:mmre} \end{center} \end{figure} \begin{figure}[!t] \begin{center} [{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/sd.pdf}}] \caption{Standard Deviation of Magnitude of Relative Error} \label{fig:sd} \end{center} \end{figure} \begin{figure}[!t] \begin{center} [{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/pred30.pdf}}] \caption{PRED(30)} \label{fig:pred30} \end{center} \end{figure} \fig{deltas_all} shows the performance deltas of LC, COCOMOST, and each variation of COCOMIN. COCOMOST performs the best but by a slight margin. In addition, some variations of COCOMIN outperformed standard COCOMO-based local calibration. Another depiction of these slight yet clear differences can be seen in \fig{mmre}. Over all 19 datasets, COCOMIN works best when a forward select is used, the Stale factor is set to 16, correlation is used to rank the attributes, and MMRE is used to evaluate improvement. For the rest of this paper we will consider COCOMIN as this variation only. In \fig{deltas_all}, one can also observe the tendency of higher stale values to outperform lower stale values, except with backward select searches. A backward select search with a low stale value would tend to keep most of the attributes and thus be similar to LC which explains this phenomenon. It is clear from \fig{deltas_byCoc81Nasa93} and \fig{deltas_subsets} that the effectiveness of feature subset selection, at least as seen from the COCOMOST and COCOMIN algorithms, is highly dependent on the underlying data. \fig{sd} shows that both COCOMOST and COCOMIN were able to reduce the deviation of the mean relative error. Interestingly, \fig{mmre} and \fig{sd} both show that COCOMOST and COCOMIN tend to help in the worse cases, as the deviation and mmre rise, but don't make improvements on the better predictions. Note that this is consistent with the observation that the effectiveness of feature subset selection is highly dependent on the underlying data, i.e. the learners had a positive impact on the Nasa93 data but not on the Coc81 data. In addition, the effectiveness of the feature subset selectors on the worse estimates but not the better estimates explains the higher Pred30 values for LC seen in \fig{pred30}. Information about how many and which attributes were dropped by COCOMOST can be found in \fig{attributes_cocomost_nasa93} and \fig{attributes_cocomost_coc81}, and \fig{attributes_cocomost_nasa93} and \fig{attributes_cocomost_coc81} for COCOMIN. Note that the attributes from the NASA93 dataset were discarded far more often than those from the COC81 dataset. Finally, in this experiment COCOMOST took about 2 seconds for each execution while COCOMIN took about 27. This is because the core of COCOMOST was written in C++ and optimized while COCOMIN uses lots of bash and gawk calls. Local Calibration took less than a second for each execution. % add info about execution time? %(PULL QUARTILE CHARTS TEXT FROM MENZIES PAPER) %\fig{mmre} shows the mean magnitude of relative error for each of the 19 subsets of data. %Standard local calibration is shown in red and COCOMOST is in green. %The results show similar errors between the learners. %\fig{sd} is similarly structured, except it displays the standard deviation of the error. %These results show that COCOMOST greatly reduces the variance for a large portion of the datasets. \section{Conclusion} \PARstart{T}{he} results show that feature subset selection can improve upon the error and deviation of COCOMO estimates, but this effectiveness is highly dependent on the underlying data. If the effort multipliers aren't properly correlated to the output then it is more likely that a feature subset selector will find improvement by discarding them. In addition, this improvement was observed on the datasets that gave the worst errors and deviations but not the datasets with better error rates and deviations, standard COCOMO showed better Pred30 values due to this effect. In addition, more complete searches of the attribute space were shown to be more effective than limited searches. COCOMOST was found to have the best MMRE of the learners used in the experiment, and was also shown to be effective at lowering the standard devation of the errors which will make it easier to distinguish rival methods. Finallly, while COCOMOST ran quickly in this set up it clearly won't scale well to growing numbers of attributes in the dataset. %add line about execution time? %\PARstart{T}{he} results show the limits of feature subset selection using local calibration as the evaluation criteria. %COCOMOST produced statistically equivalent errors as Boehm's COCOMO model. %Since the attribute space was searched completely, this indicates that more involved evaluation methods or models need to be explored to reduce the error. %However, COCOMOST was able to reduce the variance in the model which will make it easier to distinguish rival methods. %This will provide a stepping stone for further research. %In addition, for this evaluation method, heuristics were found to be less effective and slower than a finely tuned complete search of the attribute space. \section{Future Work} \PARstart{T}{here} is a lot of work to be done in improving software cost estimation models. Some possibilities suggested by this research include: \bi \item Feature Subset Selection using other evaluation methods and learners. \item Expanding upon COCOMOST with more data mining techniques such as bagging. %Expanding upon COCOMOST with more data mining techniques such as bagging(ADD REFERENCE). \item More comparisons of learners using COCOMOST as a ``strawman`` FSS model. \item Due to COCOMOST's fairly quick runtime it could be used by WRAPPER algorithms such as COSEEKMO. %Due to COCOMOST's fairly quick runtime it could be used by WRAPPER algorithms (add ref?) such as COSEEKMO. \ei \bibliographystyle{IEEEtranBST/IEEEtran.bst} \bibliography{myrefs,../../../../../tex/refs} \begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio] {stuff/timbeach}}]{Tim Menzies} is an associate professor at the University of West Virginia and works with NASA on software quality. His recent research concerns modeling and learning, with a particular focus on lightweight modeling methods. He received his PhD in AI and knowledge engineering from the University of New South Wales. Contact him at tim@menzies.us. \end{biography} \begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio] {stuff/dan}}]{Dan Baker} is a graduate student at West Virginia University. He received his Bachelor's degree in Computer Science from West Virginia University in May of 2006. For his senior project he developed the database used in the West Virginia Crime Reduction and Information Management Effort. He is currently working on his Master's degree in Computer Science at West Virginia University while working as a research assistant for Dr Menzies. His research focuses on data mining in data starved domains such as software cost estimation. He is the president of the student chapter of Upsilon Pi Epsilon at West Virginia University. UPE is the first and only international honor society for the Computing Sciences. \end{biography} \appendix {\bf APPENDIX A - COCOMO-I vs COCOMO-II:} In COCOMO II, the exponential COCOMO 81 term $b$ was expanded into the following expression: {\small \begin{equation}\label{eq:coc2} b + 0.01*\sum_jSF_j \end{equation}} \noindent where b is 0.91 in COCOMO II 2000, and $SF_j$ is one of five {\em scale factors} that exponentially influence effort. Other changes in COCOMO II included dropping the development modes of \fig{eso} as well as some modifications to the list of effort multipliers and their associated numeric constants (see appendix E). {\bf APPENDIX B - Calculating Correlation:} Given a test set of size $T$, correlation is calculated as follows: { \[ \begin{array}{l@{~}c@{~}l@{~}l@{~}c@{~}l} \bar{p}&=& \frac{\sum_I^Tpredicted_i}{T}&\bar{a} &=& \frac{\sum_I^Tactual_i}{T}\\ S_{p} &=& \frac{\sum_i^T(predicted_i - \bar{p})^2}{T-1}& S_{a} &=& \frac{\sum_i^T(actual_i - \bar{a})^2}{T-1}\\ S_{pa} &=& \frac{\sum_i^T(predicted_i - \bar{p})(actual_i - \bar{a})}{T-1}\\ corr &=& S_{pa}/\sqrt{S_p*S_a}\\ \end{array} \]} {\bf APPENDIX C - Local Calibration:} This approach assumes that a matrix $D_{i,j}$ holds: \bi \item The natural log of the $KLOC$ estimates; \item The natural log of the actual efforts for projects $i \le $j$ \le t$; \item The natural logarithm of the cost drivers (the scale factors and effort multipliers) at locations $1\le i \le 15$ (for COCOMO 81) or $1 \le i \le 22$ (for COCOMO-II). \ei With those assumptions, Boehm~\cite{Boehm81} shows that for COCOMO 81, the following calculation yields estimates for ``$a$'' and ``$b$'' that minimizes the sum of the squares of residual errors: \begin{equation}\label{eq:ab}{\small \left. \begin{array}{rcl} EAF_i& =& \sum_j^N D_{i,j}\\ a_0 & =& t \\\ a_1 & =& \sum_i^t KLOC_i\\ a_2 & =& \sum_i^t (KLOC_i)^2\\ d_0 & =& \sum_i^t \left(actual_i - EAF_i\right)\\ d_1 & =& \sum_i^t \left((actual_i - EAF_i)*KLOC_i\right)\\ b & =& (a_0d_1 - a_1*d_0)/(a_0a_2 - a_1^2)\\ a_3 & =& (a_2d_0 - a_1d_1)/(a_0a_2 - a_1^2)\\ a & =& e^{a_3} \end{array}\right\}} \end{equation} {\bf APPENDIX D - COCOMO Numerics:} \fig{em} shows the COCOMO 81 $EM_j$ (effort multipliers). The effects of that multiplier on the effort are shown in \fig{effortmults}. Increasing the {\em upper} and {\em lower} groups of variables will {\em decrease} or {\em increase} the effort estimate, respectively. \begin{figure} \begin{center} {\scriptsize \begin{tabular}{l|r@{:~}l|}\cline{2-3} upper: &acap&analysts capability\\ increase &pcap&programmers capability\\ these to &aexp&application experience\\ decrease &modp&modern programming practices\\ effort &tool&use of software tools\\ &vexp&virtual machine experience\\ &lexp&language experience\\\cline{2-3} middle &sced&schedule constraint\\\cline{2-3} lower: &data&data base size\\ decrease &turn&turnaround time\\ these to &virt&machine volatility\\ increase &stor&main memory constraint\\ effort &time&time constraint for cpu\\ &rely&required software reliability\\ &cplx&process complexity\\\cline{2-3} \end{tabular}} \end{center} \caption{{ COCOMO 81 effort multipliers.}}\label{fig:em} \end{figure} \begin{figure} \begin{center} {\scriptsize \begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|} \hline & &very&&&&very&extra\\ &&low&low&nominal&high&high&high\\ \hline upper&ACAP &1.46 &1.19 &1.00 &0.86 &0.71 &\\ (increase&PCAP &1.42 &1.17 &1.00 &0.86 &0.70 &\\ these to&AEXP &1.29 &1.13 &1.00 &0.91 &0.82 &\\ decrease&MODP &1.2 &1.10 &1.00 &0.91 &0.82 &\\ effort)&TOOL &1.24 &1.10 &1.00 &0.91 &0.83 &\\ &VEXP &1.21 &1.10 &1.00 &0.90 & &\\ &LEXP &1.14 &1.07 &1.00 &0.95 & &\\\hline middle&SCED &1.23 &1.08 &1.00 &1.04 &1.10 & \\\hline lower&DATA & & 0.94 &1.00 &1.08 &1.16&\\ (increase&TURN & &0.87 &1.00 &1.07 &1.15 &\\ these to&VIRT & &0.87 &1.00 &1.15 &1.30 &\\ increase&STOR & & &1.00 &1.06 &1.21 &1.56\\ effort)&TIME & & &1.00 &1.11 &1.30 &1.66\\ &RELY &0.75& 0.88& 1.00 & 1.15 & 1.40&\\ &CPLX &0.70 &0.85 &1.00 &1.15 &1.30 &1.65\\ \hline \end{tabular}} \end{center} \caption{{ The precise COCOMO 81 effort multiplier values.}}\label{fig:effortmults} \end{figure} \begin{figure} \begin{center} {\scriptsize \begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|} \hline & &very&&&&very&extra\\ &&low&low&nominal&high&high&high\\ \hline upper &ACAP & 1.2 & 1.1& 1.00 & 0.9& 0.8 & \\ (increase &PCAP & 1.2 & 1.1& 1.00 & 0.9& 0.8 & \\ these to&AEXP & 1.2 & 1.1& 1.00 & 0.9& 0.8 & \\ decrease&MODP & 1.2 & 1.1& 1.00 & 0.9& 0.8 & \\ effort)&TOOL & 1.2 & 1.1& 1.00 & 0.9& 0.8 & \\ &VEXP & 1.2 & 1.1& 1.00 & 0.9& & \\ &LEXP & 1.2 & 1.1& 1.00 & 0.9& &\\\hline middle&SCED & 1.2 & 1.1& 1.00 & 1.1& 1.2 & \\\hline lower&DATA & & 0.9 & 1.00 & 1.1& 1.2 & \\ (increase&TURN & & 0.9 & 1.00 & 1.1& 1.2 & \\ these to&VIRT & & 0.9 & 1.00 & 1.1& 1.2 & \\ increase&STOR & & & 1.00 & 1.1& 1.2 & 1.3\\ effort)&TIME & & & 1.00 & 1.1& 1.2 & 1.3\\ &RELY & 0.8 & 0.9 & 1.00 & 1.1& 1.2 & \\ &CPLX & 0.8 & 0.9 & 1.00 & 1.1& 1.2 & 1.3\\ \hline \end{tabular}} \end{center} \caption{{ Rounded COCOMO 81 effort multiplier values.}}\label{fig:roundedem} \end{figure} \fig{roundedem} shows the COCOMO 81 effort multipliers of \fig{effortmults}, rounded and simplified to two significant figures. \fig{emsf2}, \fig{effortmults2} and \fig{effortmultsrd2} show the COCOMO-II values analogies to \fig{em}, \fig{effortmults} and \fig{roundedem} (respectively). \begin{figure} \begin{center} {\scriptsize \begin{tabular}{l|r@{:~}l|}\cline{2-3} scale &prec & have we done this before?\\ factors &flex & development flexibility \\ (exponentially &resl & any risk resolution activities?\\ decrease &team & team cohesion\\ effort) &pmat & process maturity\\\hline upper &acap & analyst capability\\ (linearly &pcap & programmer capability\\ decrease &pcon & programmer continuity\\ effort) &aexp & analyst experience\\ &pexp & programmer experience\\ <ex & language and tool experience\\ &tool & tool use\\ &site & multiple site development\\ &sced & length of schedule \\\hline lower &rely & required reliability \\ (linearly &data & secondary memory storage requirements\\ increase &cplx & program complexity\\ effort) &ruse & software reuse\\ &docu & documentation requirements\\ &time & runtime pressure\\ &stor & main memory requirements\\ &pvol & platform volatility \\\cline{2-3} \end{tabular}} \end{center} \caption{{ The COCOMO~II scale factors and effort multipliers.}}\label{fig:emsf2} \end{figure} \begin{figure} \begin{center} {\scriptsize \begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|} \hline & &extra &very& & & &very &extra\\ & &low &low &low&nominal&high&high &high\\ \hline scale &prec & &6.20 &4.96 &3.72 &2.48 &1.24 &0.00\\ factors &flex & &5.07 &4.05 &3.04 &2.03 &1.01 &0.00\\ (exponentially &resl & &7.07 &5.65 &4.24 &2.83 &1.41 &0.00\\ decreases &team & &5.48 &4.38 &3.29 &2.19 &1.10 &0.00\\ effort) &pmat & &7.80 &6.24 &4.68 &3.12 &1.56 &0.00\\ \hline upper &acap & &1.42 &1.19 &1.00 &0.85 &0.71 & \\ (linearly &pcap & &1.34 &1.15 &1.00 &0.88 &0.76 & \\ decreases &pcon & &1.29 &1.12 &1.00 &0.90 &0.81 & \\ effort) &aexp & &1.22 &1.10 &1.00 &0.88 &0.81 & \\ &pexp & &1.19 &1.09 &1.00 &0.91 &0.85 & \\ <ex & &1.20 &1.09 &1.00 &0.91 &0.84 & \\ &tool & &1.17 &1.09 &1.00 &0.90 &0.78 & \\ &site & &1.22 &1.09 &1.00 &0.93 &0.86 &0.80\\ &sced & &1.43 &1.14 &1.00 &1.00 &1.00 & \\\hline lower &rely & &0.82 &0.92 &1.00 &1.10 &1.26 & \\ (linearly &data & & &0.90 &1.00 &1.14 &1.28 & \\ increases &cplx & &0.73 &0.87 &1.00 &1.17 &1.34 &1.74\\ effort) &ruse & & &0.95 &1.00 &1.07 &1.15 &1.24\\ &docu & &0.81 &0.91 &1.00 &1.11 &1.23 & \\ &time & & & &1.00 &1.11 &1.29 &1.63\\ &stor & & & &1.00 &1.05 &1.17 &1.46\\ &pvol & & &0.87 &1.00 &1.15 &1.30 & \\ \hline \end{tabular}} \end{center} \caption{{ The precise COCOMO II numerics.}}\label{fig:effortmults2} \end{figure} \begin{figure} \begin{center} {\scriptsize \begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|} \hline & &extra &very& & & &very &extra\\ & &low &low &low&nominal&high&high &high\\ \hline Scale &PREC & &6.3 &5.1 &3.8 &2.5 &1.3 &0\\ Factors &FLEX & &6.3 &5.1 &3.8 &2.5 &1.3 &0\\ &RESL & &6.3 &5.1 &3.8 &2.5 &1.3 &0\\ &TEAM & &6.3 &5.1 &3.8 &2.5 &1.3 &0\\ &PMAT & &6.3 &5.1 &3.8 &2.5 &1.3 &0\\ \hline upper &ACAP & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &PCAP & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &PCON & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &AEXP & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &PEXP & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ <EX & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &TOOL & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ &SITE & &1.3 &1.1 &1.0 &0.9 &0.8 &0.8\\ &SCED & &1.3 &1.1 &1.0 &0.9 &0.8 & \\ \hline lower &RELY & &0.8 &0.9 &1.0 &1.1 &1.3 & \\ &DATA & & &0.9 &1.0 &1.1 &1.3 & \\ &CPLX & &0.8 &0.9 &1.0 &1.1 &1.3 &1.5\\ &RUSE & & &0.9 &1.0 &1.1 &1.3 &1.5\\ &DOCU & &0.8 &0.9 &1.0 &1.1 &1.3 & \\ &TIME & & & &1.0 &1.1 &1.3 &1.5\\ &STOR & & & &1.0 &1.1 &1.3 &1.5\\ &PVOL & & &0.9 &1.0 &1.1 &1.3 & \\ \hline \end{tabular}} \end{center} \caption{{ The rounded COCOMO II numerics.}}\label{fig:effortmultsrd2} \end{figure} \end{document}