% use to find bib's not used
% grep bibitem cocomost.bbl | gawk '{gsub(/bibitem/,"");gsub(/{/,"");gsub(/}/,"");gsub(/\\/,"");print $0}'

\documentclass[journal]{IEEEtran/IEEEtran}

\newenvironment{smallitem}{
   \setlength{\topsep}{0pt}
   \setlength{\partopsep}{0pt}
   \setlength{\parskip}{0pt}
   \begin{itemize}
   \setlength{\leftmargin}{.2in}
   \setlength{\parsep}{0pt}
   \setlength{\parskip}{0pt}
   \setlength{\itemsep}{0pt}}{\end{itemize}}

\newenvironment{smallnums}{
   \setlength{\topsep}{0pt}
   \setlength{\partopsep}{0pt}
   \setlength{\parskip}{0pt}
   \begin{enumerate}
   \setlength{\leftmargin}{.2in}
   \setlength{\parsep}{0pt}
   \setlength{\parskip}{0pt} \setlength{\itemsep}{0pt}}{\end{enumerate}}

\newenvironment{hhh}{
  \begin{hypo}[?]}{\end{hypo}}
\newenvironment{yup}{
  \begin{hypo}[\ding{51}]}{\end{hypo}}
\newenvironment{nupe}{
  \begin{hypo}[\ding{55}]}{\end{hypo}}

% stuff for deltas plots
\newcommand{\boxplot}[5]{\begin{picture}(100,10)
\put(0,0){\line(0,1){8}}
\put(100,0){\line(0,1){8}}
\put(#1,4){\line(1,0){#2}}
\put(#3,4){\circle*{6}}
\put(#4,4){\line(1,0){#5}}
\put(50,0){\line(0,1){8}}
\end{picture}}

\newcommand{\aive}{{\"aive}}
\newcommand{\bi}{\begin{smallitem}}
\newcommand{\ei}{\end{smallitem}}
\newcommand{\be}{\begin{smallnums}}
\newcommand{\ee}{\end{smallnums}}
\newcommand{\bd}{\begin{description}}
\newcommand{\ed}{\end{description}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\hyp}[1]{Hypothesis~\ref{hyp:#1}}


% some very useful LaTeX packages include:
\usepackage{alltt}
\usepackage{cite}      % Written by Donald Arseneau
                        % V1.6 and later of IEEEtran pre-defines the format
                        % of the cite.sty package \cite{} output to follow
                        % that of IEEE. Loading the cite package will
                        % result in citation numbers being automatically
                        % sorted and properly "ranged". i.e.,
                        % [1], [9], [2], [7], [5], [6]
                        % (without using cite.sty)
                        % will become:
                        % [1], [2], [5]--[7], [9] (using cite.sty)
                        % cite.sty's \cite will automatically add leading
                        % space, if needed. Use cite.sty's noadjust option
                        % (cite.sty V3.8 and later) if you want to turn this
                        % off. cite.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/cite/

\usepackage[pdftex]{graphicx}
%\fi
% However, be warned that pdflatex will require graphics to be in PDF
% (not EPS) format and will preclude the use of PostScript based LaTeX
% packages such as psfrag.sty and pstricks.sty. IEEE conferences typically
% allow PDF graphics (and hence pdfLaTeX). However, IEEE journals do not
% (yet) allow image formats other than EPS or TIFF. Therefore, authors of
% journal papers should use traditional LaTeX with EPS graphics.
%
% The path(s) to the graphics files can also be declared: e.g.,
% \graphicspath{{../eps/}{../ps/}}
% if the graphics files are not located in the same directory as the
% .tex file. This can be done in each branch of the conditional above
% (after graphicx is loaded) to handle the EPS and PDF cases separately.
% In this way, full path information will not have to be specified in
% each \includegraphics command.
%
% Note that, when switching from latex to pdflatex and vice-versa, the new
% compiler will have to be run twice to clear some warnings.

\usepackage{subfigure} % Written by Steven Douglas Cochran
                        % This package makes it easy to put subfigures
                        % in your figures. i.e., "figure 1a and 1b"
                        % Docs are in "Using Imported Graphics in LaTeX2e"
                        % by Keith Reckdahl which also documents the graphicx
                        % package (see above). subfigure.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/subfigure/

\usepackage{url}       % Written by Donald Arseneau
                        % Provides better support for handling and breaking
                        % URLs. url.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/other/misc/
                        % Read the url.sty source comments for usage information.

\usepackage{amssymb}
\usepackage{amsmath}   % From the American Mathematical Society
                        % A popular package that provides many helpful commands
                        % for dealing with mathematics. Note that the AMSmath
                        % package sets \interdisplaylinepenalty to 10000 thus
                        % preventing page breaks from occurring within multiline
                        % equations. Use:
\interdisplaylinepenalty=2500
                        % after loading amsmath to restore such page breaks
                        % as IEEEtran.cls normally does. amsmath.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/


\usepackage{pifont}

\usepackage{verbatim}	% for block comment

%\usepackage[dvips]{color}
\begin{document}

\title{Feature Subset Selection Methods for\\COCOMO Based Software Effort Estimation}
\author{Daniel~Baker\thanks{Dan Baker is with the Lane Department of Computer Science,
 West Virginia University and can be reached at \protect\url{danielryanbaker@gmail.com}},Tim~Menzies~\IEEEmembership{Member,~IEEE}% <-this % stops a space
\thanks{Dr. Menzies is with the Lane Department of Computer
Science, West Virginia University and can be reached at \protect\url{tim@menzies.us}}
\thanks{This research was conducted with funds from the NASA
Software Assurance Research Program led by the
NASA IV\&V Facility.
}
\thanks{Manuscript received January 1, 2006; revised XXX YY, 200Z.}}% <-this % stops a space
\markboth{IEEE Transactions on Software Engineering,~Vol.~W, No.~X,~YYY~200Z}{Baker, Menzies, Feature Subset Selection}
\maketitle

\begin{abstract}
This paper demonstrates the results of feature subset selection methods for the COCOMO model.  Two algorithms are
tested, COCOMIN and COCOMOST.  COCOMIN was designed as a `strawman` approach to feature subset selection.  COCOMOST
was designed to efficiently evaluate every subset in the attribute space instead of using heuristics because
the domain of software cost estimation is sufficiently data starved to warrant exhaustive methods. 
\end{abstract}

\section{Introduction}
%\PARstart{I}{n} 2005...(START WITH QUOTE AND REFERENCE ABOUT MONEY LOST FROM INACURRATE SOFTWARE COST ESTIMATION/BUDGETING)
%Currently cost estimation is accomplished using models such as COCOMO~\cite{Boehm81}, (LIST MANY OTHER METHODS WITH REFERENCES).
\PARstart{C}{urrently} cost estimation is accomplished using models such as COCOMO~\cite{Boehm81}.
These models predict the development cost for a new software project based on past project data.
%For an accurate prediction the training data needs quantity, quality, and relevance to the new project(ADD REFERENCES).
For an accurate prediction the training data needs quantity, quality, and relevance to the new project.
Unfortunately this is difficult in practice and estimates are often made using inadequate training data. 
%Consequently, these models are plagued with problems including highly inaccurate predictions(FIND BEST REFERENCE) and the variance problem(INSERT REFERENCE).
Consequently, these models are plagued with problems including highly inaccurate predictions and the variance problem.
%It has been shown that the variance can be reduced by feature subset selection methods that discard irrelevant, redundant, noisy, and unreliable attributes(LOTS OF REF HERE).
It has been shown that the variance can be reduced by feature subset selection methods that discard irrelevant, redundant, noisy, and unreliable attributes.
This paper explores some of these attribute pruning techniques.

%\section{Background: COCOMO and COSEEKMO}
%\PARstart{P}{ull} text from COSEEKMO paper.  First explain COCOMO.  Then justify FSS.  Then explain COSEEKMO.  Basically say that
%COSEEKMO worked but was too slow, so we decided to try the minimal approach (attr - next section).  Given that the
%datasets were small, we decided extensive searches were ok and tried a complete approach instead of heuristics (next section on cocomost).


 \section{Background}

\subsection{COCOMO}

The case study material for this paper uses
 COCOMO-format data.
 COCOMO (the COnstructive COst MOdel)   was originally developed by Barry
Boehm in 1981~\cite{Boehm81} and was extensively revised in
2000~\cite{Boehm00b}.  
%% COCOMO helps software developers
%% reason about the cost and schedule implications of their software
%% decisions such as software investment decisions; setting project budgets
%% and schedules; negotiating cost, schedule, and performance
%% tradeoffs; making software risk management decisions, and making
%% software improvement decisions.
%% One advantage of COCOMO (and this is why we use it)
%% is that unlike  many other costing models such as SLIM or SEER,
%% COCOMO is an {\em
%% open model} with numerous published data~\cite{Boehm81, Boehm00b}.
%COCOMO
%measures effort in calendar months where one month is 152
% hours (and includes development and management hours). 
The core
intuition behind COCOMO-based estimation is that
as a program grows
in size, the development effort  grows exponentially. More specifically:



{\footnotesize
\begin{equation}\label{eq:one}
effort(person months)=a*\left(KLOC^b\right)*\left(\prod_jEM_j\right)
\end{equation}
}
%XXXwhat about repeatability (folks repeating other people's work), straw man.
%also, acceptable if >= cocomo1 stratified results
%competent if pred(30) cocomo stratified results:sp
%good if pred(25)> 83%
Here, $KLOC$
is thousands of  delivered source instructions. $KLOC$ can be
 estimated
directly or via a {\em function point estimation}. Function points
are a
 product of five defined data components (inputs, outputs, inquiries,
files, external interfaces) and 14 weighted environment characteristics
(data comm, performance, reusability, etc.)~\cite{Boehm00b,Capers98}.
A 1,000 line Cobol program would typically implement about 14
function points,
 while a 1,000-line C program would implement about
seven\footnote{\url{http://www.qsm.com/FPGearing.html}}.


% XXX
In \eq{one}, $EM_j$ is one of
{\em effort multipliers} such as {\em cplx} (complexity)
or {\em pcap} (programmer capability).
In order to model the effects of $EM_j$ on development effort,
Boehm
proposed reusing numeric values 
which he generated via
regression on historical data for each value of $EM_i$.

%\begin{figure}[!t] {\footnotesize
%\begin{center}
%\begin{tabular}{r|r|r|p{1.8in}}
%\noindent
%Mode & a & b & notes\\\hline
%Organic & 3.2 & 1.05 & projects from relatively small software teams develop software
%in a highly familiar, in-house environment.\\\hline
%Embedded & 2.8& 1.2 & projects operating within (is embedded in) a strongly
%coupled complex of hardware, software, regulations, and operational
%procedures.\\\hline
%Semi-Detached & 3.0 & 1.12 & An intermediary mode between organic and embedded.
%\end{tabular}
%\end{center}}
%\caption{Standard COCOMO 81 development modes.}\label{fig:eso}
%\end{figure}
%\begin{figure*}{\footnotesize
%\begin{center}
%\begin{tabular}{|p{0.35in}|p{6.5in}|}\hline
%~\newline
%~\newline
%Data sources&\bd
%\item[{\em Coc81:}] 63 records in the
%COCOMO 81 format. Source: \cite[p496-497]{Boehm81}. \newline Download from 
%\url{http://unbox.org/wisp/trunk/cocomo/data/coc81modeTypeLangType.csv}.
%\item[{\em Nasa93:}] ~~~ 93 NASA records in the COCOMO 81 format. \newline 
%Download from \url{http://unbox.org/wisp/trunk/cocomo/data/nasa93.csv}.
%\item[{\em CocII:}] 161  records in the COCOMO II format from the COCOMO consortium
%(co-ordinated by USC). This data is not in the public domain.
%\ed\\\hline
%~\newline
%~\newline
%Data \newline
%subsets&\bd
%\item[{\em All:}] selects all records from a particular source;
%e.g. "coc81\_all''.
%\item[{\em Category:}] ~~~~~is a NASA-specific
% designation selecting the type of project; e.g. avionics, data capture, etc.
%\item[{\em Dev:}] indicates the development methodology; e.g. div.waterfall.
%\item[{\em DevEnd:}] ~~~shows the last year of the software project.
%\item[{\em Fg:}] selects either ``$f$'' (flight) or ``$g$'' (ground) software.
%\item[{\em Kind:}] selects records relating
%to the development platform; max= mainframe and mic= microprocessor.
%\item[{\em Lang:}] selects records about different development languages.
%\item[{\em Project} and {\em center}:]~~~~~~~~~~~~~~~  $nasa93$
% designations selecting records relating to where the software was built and the name
%of the project.
%\item[{\em Mode=e:}]~\newline
%selects records relating to
%the {\em embedded}  COCOMO 81 development mode. The different COCOMO 81 development models were described
%in \fig{eso}.
%\item[{\em Mode=o: }]~\newline
%selects COCOMO 81 {\em organic} mode records.
%\item[{\em Mode=sd: }]~\newline
%selects COCOMO 81 {\em semi-detached} mode records.
%\item[{\em Org:}] is a $cocII$ designation showing  what organization provided the data.
%\item[{\em Size:}] is a $cocII$ specific designation grouping the records into (e.g.) those around 100KLOC.
%\item[{\em Type:}] selects different $coc81$  designations and include ``bus'' (for business application)
%or ``sys'' (for system software).
%\item[{\em Year:}] is a $nasa93$ term that
%selects the development years, grouped into units of five; e.g. 1970,1971,1972,1973,1974
%are  labeled ``1970''.
%\ed
%\\\hline
%\end{tabular}
%\end{center}}
%\caption{Data sets (top) and
%parts (bottom) of the data used in this study.}\label{fig:parts}
%\end{figure*}

In practice, effort data forms exponential distributions. Appendix B describes
methods for using such distributions in effort modeling.


Note that in COCOMO 81,
Boehm identified three common types of software:
{\em embedded, semi-detached},  and {\em organic}.
Each has their
own characteristic ``$a$'' and ``$b$'' (see  \fig{eso}).
COCOMO-II ignores these distinctions. 
This study used data sets in both the COCOMO 81 and COCOMO-II format.
For more on the differences between COCOMO 81
and COCOMO-II, see Appendix A. 

\subsection{Data}\label{sec:data}

The software project data we used in this study came from two sources (see \fig{parts}).
$Coc81$ is the original COCOMO data
used by Boehm to calibrate COCOMO 81.  $Nasa93$ comes from a NASA-wide database
recorded in the COCOMO 81 format. 
This data has been in
the public domain for several years but few have been aware of it.
It can now be found on-line in several places including
the PROMISE (Predictor Models in Software
Engineering) web site\footnote{\url{http://promise.site.uottawa.ca/SERepository/} and 
\url{http://unbox.org/wisp/trunk/cocomo/data}.}.
$Nasa93$
was
originally collected to create a NASA-tuned version of
COCOMO,
funded by the Space
Station Freedom Program.
$Nasa93$ contains data from six  NASA centers including the Jet Propulsion
Laboratory.
Hence,
it covers a very
wide range of software domains, development
processes, languages, and complexity as well as fundamental differences
in culture and business practices between each center. All of these
factors contribute to the large variances observed in this data
set.

When the $nasa93$ data was collected, it was
required that there be multiple interviewers with one person
leading the interview and one or two others recording and checking
documentation.  Each data point was cross-checked with either
official records or via independent subjective inputs from other
project personnel who fulfilled various roles on the project.  After
the data was translated into the COCOMO 81 format, the data was
reviewed with those who originally provided the data.  Once sufficient
data existed the data was analyzed to identify outliers and the
data values were re-verified with the development teams once again
if deemed necessary.  This typically required from two to four trips
to each NASA center.  All of the supporting information was placed
in binders, which we still on occasion reference even today. 

Using Boehm's COCOMO-I ``local calibration`` the $nasa93$ data has been shown to contain large deviations
due to the wide variety of projects in that data set,
and {\em not} poor data collection. Our belief is that $nasa93$ was collected
%and {\em not} poor data collection (ADD REFERENCE TO COSEEKMO PAPER). Our belief is that $nasa93$ was collected
using methods equal to, or better, than standard industrial practice. If so, then
industrial data would suffer from deviations equal to or lager than those seen in the $nasa93$ data.

\begin{figure}[!t] {\footnotesize
\begin{center}
\begin{tabular}{r|r|r|p{1.8in}}
\noindent
Mode & a & b & notes\\\hline
Organic & 3.2 & 1.05 & projects from relatively small software teams develop software
in a highly familiar, in-house environment.\\\hline
Embedded & 2.8& 1.2 & projects operating within (is embedded in) a strongly
coupled complex of hardware, software, regulations, and operational
procedures.\\\hline
Semi-Detached & 3.0 & 1.12 & An intermediary mode between organic and embedded.
\end{tabular}
\end{center}}
\caption{Standard COCOMO 81 development modes.}\label{fig:eso}
\end{figure}
\begin{figure*}{\footnotesize
\begin{center}
\begin{tabular}{|p{0.35in}|p{6.5in}|}\hline
~\newline
~\newline
Data sources&\bd
\item[{\em Coc81:}] 63 records in the
COCOMO 81 format. Source: \cite[p496-497]{Boehm81}. \newline Download from 
\url{http://unbox.org/wisp/trunk/cocomo/data/coc81modeTypeLangType.csv}.
\item[{\em Nasa93:}] ~~~ 93 NASA records in the COCOMO 81 format. \newline 
Download from \url{http://unbox.org/wisp/trunk/cocomo/data/nasa93.csv}.
\item[{\em CocII:}] 161  records in the COCOMO II format from the COCOMO consortium
(co-ordinated by USC). This data is not in the public domain.
\ed\\\hline
~\newline
~\newline
Data \newline
subsets&\bd
\item[{\em All:}] selects all records from a particular source;
e.g. "coc81\_all''.
\item[{\em Category:}] ~~~~~is a NASA-specific
 designation selecting the type of project; e.g. avionics, data capture, etc.
\item[{\em Dev:}] indicates the development methodology; e.g. div.waterfall.
\item[{\em DevEnd:}] ~~~shows the last year of the software project.
\item[{\em Fg:}] selects either ``$f$'' (flight) or ``$g$'' (ground) software.
\item[{\em Kind:}] selects records relating
to the development platform; max= mainframe and mic= microprocessor.
\item[{\em Lang:}] selects records about different development languages.
\item[{\em Project} and {\em center}:]~~~~~~~~~~~~~~~  $nasa93$
 designations selecting records relating to where the software was built and the name
of the project.
\item[{\em Mode=e:}]~\newline
selects records relating to
the {\em embedded}  COCOMO 81 development mode. The different COCOMO 81 development models were described
in \fig{eso}.
\item[{\em Mode=o: }]~\newline
selects COCOMO 81 {\em organic} mode records.
\item[{\em Mode=sd: }]~\newline
selects COCOMO 81 {\em semi-detached} mode records.
\item[{\em Org:}] is a $cocII$ designation showing  what organization provided the data.
\item[{\em Size:}] is a $cocII$ specific designation grouping the records into (e.g.) those around 100KLOC.
\item[{\em Type:}] selects different $coc81$  designations and include ``bus'' (for business application)
or ``sys'' (for system software).
\item[{\em Year:}] is a $nasa93$ term that
selects the development years, grouped into units of five; e.g. 1970,1971,1972,1973,1974
are  labeled ``1970''.
\ed
\\\hline
\end{tabular}
\end{center}}
\caption{Data sets (top) and
parts (bottom) of the data used in this study.}\label{fig:parts}
\end{figure*}


\subsection{Performance Measures}

The performance of  models generating continuous output can be
assessed in many ways, including PRED(30), MMRE, correlation, etc.
PRED(30) is a measure calculated from
the relative error, or RE, which is the relative
size of the difference between the actual and estimated value.
One way to view these measures is to say that training data contains records
with variables $1,2,3,..,N$ and performance measures add additional new variables $N+1,N+2,...$.


The magnitude of the relative error, or MRE, is the absolute value of that relative error:

{\small
\[
MRE = |predicted - actual|/actual
\]}\noindent
The mean magnitude of the relative error, or MMRE, is the average percentage
of the absolute values of the relative errors over an entire data set.
MMRE results were shown in \fig{one} in the {\em mean\% average test error} column.
Given  $T$ tests, MMRE is calculated
as follows:

{\small
\[MMRE = \frac{100}{T}\sum_i^T\frac{|predicted_i-actual_i|}{actual_i}\]}\noindent


PRED(N) reports the average percentage of estimates that were within
N\% of the actual values. Given $T$ tests, then:

{\small
\[
PRED(N) = \frac{100}{T} \sum_i^T\left\{
\begin{array}{l}
1\;if\;MRE_i\;{\le}\;\frac{N}{100}\\
0\;otherwise
\end{array}\right.
\]}\noindent
For example, a PRED(30)=50\% means that half the estimates are within
30\% of the actual.

Another performance measure of a  model predicting numeric values is the correlation
between predicted and actual values.  Correlation
ranges from +1 to -1 and a correlation of +1 means that there is a
perfect positive linear relationship between variables. Appendix C shows how to
 calculate correlation.

All  these performance measures
(correlation, MMRE and PRED) address subtly different issues.
Overall, PRED measures how {\em well} an effort model performs while
MMRE measures  {\em poor} performance.
A single large mistake can skew the MMREs and not effect the PREDs.
Shepperd and Schofield comment that:
\begin{quote}
MMRE is fairly conservative with a bias
against overestimates while PRED(30) will identify those
prediction systems that are generally accurate but occasionally
wildly inaccurate~\cite[p736]{Shepperd97}.
\end{quote}

Since they measure different aspects of model
performance, COSEEKMO  
uses combinations of   PRED, MMRE, and correlation (using the methods
described later in this paper).


%\section{Attribute Ranking Driven Greedy Search Feature Subset Selection}
\section{COCOMIN}
\PARstart{F}{ollowing} the slow but steady success of COSEEKMO, it becomes necessary to find a similarly accurate
algorithm which is fast enough to be used by business users in the real world.  We decided to begin the search for
such an algorithm using a minimal approach, that is, a feature subset selection algorithm that used a greedy search
to minimize evaluations of the attribute space.  This algorithm was named COCOMIN because it was a minimal approach 
to feature subset selection on the COCOMO model.  Pseudocode for COCOMIN is in \fig{cocomin}.  The code can be found at \url{http://unbox.org/wisp/trunk/cocomost/cocomin}. We decided to rank the attributes using correlation and to use the
ranked results as the order to grow the attribute set in the case of a forward select greedy search, or prune the set
in the case of a backward elimination greedy search.  At each step the attribute set is evaluated with
the MMRE, Pred(30), deviation, and correlation after using COCOMO regression with the attribute set in question.
If the change is considered an
improvement then it is kept and the search continues, otherwise it stops.  We introduced a stale variable that could
be set to allow for the search to continue even without improvement.

%  An extensive experiment was run to benchmark the
%many customizations of the greedy FSS approach against standard COCOMO least squares regression as described in the
%pseudocode in \fig{cocomin}.  The results are shown in \fig{attrCOC81} and \fig{attrNASA}.  For the COC81 dataset,
%standard COCOMO least squares regression was found to be far superior.  However, for the NASA93 dataset, many
%customizations of the greedy feature subset selection had similar or slighty better MMRE and Pred(30) values, and lower deviations.
%(refer to background on datasets to explain this).

%Early results showed that although quick, this algorithm wasn't very accurate and  accurate enough to be an acceptable replacement for even
%avoiding feature subset selection entirely.  Although it had some slight success with the NASA93 dataset it
%failed to provide similar results to COCOMO regression on the COC81 dataset.
Early results showed that this algorithm wasn't very accurate.  Also, by 
noticing that the results tended to improve with higher stale values, we reasoned that
the attribute space needed more exploration.  Finally, with a dataset as small as the COC81 and NASA93 datasets,
the importance of heuristics to prune the state space explosion were examined.  Next we decided to build an efficient
algorithm that evaluated all 32,768 attribute combinations.

\begin{figure}[!b]
{\scriptsize\begin{alltt}
for attribute in rankedList
  newSet = bestSet + attribute
  if (search==backward)
    tmpSet=inverse(newSet)
  else
    tmpSet = newSet
  oldScore = newScore
  results = LC(train, train, tmpSet)
  newScore = results.eval
  if (newScore better than oldScore)
    bestSet = newSet
    staleCount = stale
  else
    newScore = oldScore
    staleCount--
  if (staleCount < 1)
    exit for
next attribute
if (search==backward)
  bestSet=inverse(bestSet)
return bestSet
\end{alltt}}
\caption{The cocomin algorithm which builds a subset of attributes using a greedy search guided by an attribute ranking.
%\caption{This pseudocode outlines an experiment that compared standard COCOMO 81 local calibration
%with an approach that ranks the attributes based on correlation, and then builds a subset of
%attributes using a greedy search guided by the attribute ranking.
}\label{fig:cocomin}
\end{figure}


\section{COCOMOST}

%\begin{figure}[!t]
\begin{figure}[!b]
{\scriptsize\begin{alltt}
attributes = null
bestMMRE = LC(train, attributes)
bestAttributes = null
{\bf for} attributes {\bf in} 2^15
  newMMRE = LC(train, attributes)
  if newMMRE > bestMMRE
      bestMMRE = newMMRE
      bestAttributes = attributes
return bestAttributes
\end{alltt}}
\caption{Cocomost performs a complete search over the attribute space and evaluates the attribute
sets using the target learner: COCOMO-based local calibration.
}\label{fig:cocomost}
\end{figure}

\PARstart{T}{he} COCOMOST algorithm, as outlined in \fig{cocomost}, uses feature subset selection to prune irrelevant, redundant, noisy, and unreliable attributes from the COCOMO model. The code begins at \url{http://unbox.org/wisp/trunk/cocomost/cocomost}.
It executes a complete search of the attribute space, evaluating attribute sets using local calibration.
Thus, it is a ``wrapper`` attribute selection technique instead of a ``filter`` because it evaluates using the target learner~\cite{Hall03}.
However, it shares several of the advantages of a filter.  Unlike most wrappers, COCOMOST is fast enough to search the entire attribute space instead of using heuristics to limit the state space explosion.
This introduces the vulnerability that every attribute beyond the 15 used in this study will double COCOMOST's execution time.

%To compare the effectiveness of standard local calibration versus feature subset selection with COCOMOST followed by local calibration, we randomly pulled a test set from the training data 30 times and made estimates with each learner as described in \fig{study}.


\section{Experimental Design}

\PARstart{C}{onsidering} the small amount of data available, we decided to use n-fold cross
validation to compare the COCOMIN and COCOMOST learners against standard COCOMO-based local calibration
as shown in \fig{compete}. The code for this experiment can be found at \url{http://unbox.org/wisp/trunk/cocomost/compete}.
%\fig{compete} shows pseudocode describing this experiment.

%\begin{figure}[!t]
\begin{figure}[!b]
{\scriptsize\begin{alltt}
{\bf for} data {\bf in} dataSets
 num = countRecords(data)
 {\bf for} i {\bf in} 1 {\bf to} num
  test       =  data.i
  train      =  data - test
  attr       =  (all COCOMO 81 attributes)
  results    =  LC(test, train, attr)
  print variables and results
  attr       =  cocomost(train)
  results    =  LC(test, train, attr)
  print variables and results
  for attr in COCOMO_81_attributes
   rank correlation using LC(train, train, attr)
   order += sorted list of ranked attrs
  for search in "forward backward"
   for stale in "0 1 2 4 8 16"
    for eval in "mmre sd_mre pred30 corr"
     attr=cocomin(test,train,search,stale,eval,order)
     results = LC(test, train, attrs)
     print variables and results
      
\end{alltt}}
\caption{This experiment benchmarks standard COCOMO-based local calibration against
local calibration that uses cocomost to perform feature subset selection, or 
local calibration that uses each customization of cocomin for feature subset selection.
}\label{fig:compete}
\end{figure}



%\section{Data}
%\PARstart{T}{he} training data used in the experiments are the coc81 and nasa93 datasets available at \protect\url{http://unbox.org/wisp/trunk/data}.
%The coc81 dataset comes from Boehm~\cite{Boehm81} and the nasa93 dataset contains project metrics from NASA software projects that have been sanitized for public use.
%In addition, the coc81 dataset contains 6 stratifications, and the nasa93 dataset contains 11.
%The learners were run on all 19 of these datasets to create the experimental results.


  
  
 
% comes after 
%\section{Results}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Quartile Charts of Performance Deltas}


One method we used to assess the experimental results was to create quartile charts of the performance
deltas of the learners.  We prefer quartile charts of performance deltas to other summarization
methods for M*N studies. Firstly, they offer a very succinct summary
of a large number of experiments. Secondly, they
are a {\em non-parametric} display; i.e. they make no assumptions
about the underling distribution.  Standard practice in data mining is
to compare the mean performance of different methods using
t-test~\cite{Witten05}. T-tests are a {\em parametric method} that
assume that the underling population distribution is a Gaussian.
Recent results suggest that there are many statistical issues left to
explore regarding how to best to apply those t-tests for summarizing
M*N-way studies~\cite{Brouck03}.

The performance deltas were computed
using simple subtraction,
defined as follows. A {\em positive performance delta} for method X means
that method X has out-performed some other method in {\em one} comparison.
Using performance deltas, we say that the best method is the one
that generates the largest performance deltas over {\em all}
comparisons.

The performance
deltas for each method were sorted and displayed as {\em
quartile charts}.  To generate these charts, the performance
deltas for some $method$ 
were sorted to find the lowest and highest quartile as well as the
median value; e.g.

\[
{\footnotesize
\overbrace{\underbrace{-59}_{min},-19,-19}^{lowest\; quartile},-16,-14,-10,\underbrace{-10}_{median},5,14,39,\overbrace{42,62,\underbrace{69}_{max}}^{highest\; quartile}
}
\]

In a quartile chart,
the upper and lower quartiles are  marked with black lines;
the
median is marked with a black dot; and
vertical bars  are added to
mark (i)~the zero point and (ii)~the minimum possible value
and (iii)~the maximum
possible value (in our case, -100\% and 100\%).
The above numbers
would therefore be drawn as follows:

{\small
\[-100\%\; \boxplot{20}{10}{45}{80}{14}\; 100\%\]}




%\begin{figure}
%{\footnotesize{\bf Pd:}

%\input{pd.changes}}

%{\footnotesize {\bf NotPf=$100- pf$}

%\input{notPf.changes}}

%{\footnotesize {\bf Balance}

%\input{bal.changes}}
%\caption{Performance deltas for $pd,notPf,bal$ using all 38 attributes.}\label{fig:allpd}.
%\end{figure}

\begin{comment}
\begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2}
method & median &  &  & \\\cline{1-2}
cocomin\_forward\_0\_correlation\_sd\_mre &   4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\
cocomin\_forward\_0\_correlation\_pred30 &   4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\
cocomin\_forward\_0\_correlation\_mmre &   4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\
cocomin\_forward\_0\_correlation\_correlation &   4.9 & -100\% & \boxplot{-328}{376}{52}{62}{391}&100\%\\
cocomin\_forward\_1\_correlation\_pred30 &   4.1 & -100\% & \boxplot{-328}{375}{52}{60}{233}&100\%\\
cocomin\_forward\_1\_correlation\_sd\_mre &   2.2 & -100\% & \boxplot{-328}{375}{51}{58}{235}&100\%\\
cocomin\_forward\_1\_correlation\_mmre &   1.7 & -100\% & \boxplot{-328}{375}{50}{57}{396}&100\%\\
cocomin\_forward\_2\_correlation\_pred30 &   1.5 & -100\% & \boxplot{-254}{300}{50}{57}{188}&100\%\\
cocomin\_backward\_16\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-166}{211}{50}{56}{310}&100\%\\
cocomin\_backward\_8\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-166}{211}{50}{56}{310}&100\%\\
cocomin\_forward\_4\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-198}{243}{50}{56}{190}&100\%\\
cocomin\_backward\_4\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-328}{373}{50}{55}{217}&100\%\\
cocomin\_forward\_4\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\
cocomin\_forward\_1\_correlation\_correlation &   0.3 & -100\% & \boxplot{-280}{326}{50}{56}{262}&100\%\\
cocomin\_forward\_2\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-254}{299}{50}{55}{245}&100\%\\
cocomin\_forward\_16\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\
cocomin\_forward\_8\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-225}{270}{50}{55}{203}&100\%\\
cocomin\_forward\_8\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-198}{242}{50}{55}{190}&100\%\\
cocomin\_forward\_16\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-198}{242}{50}{55}{190}&100\%\\
cocomin\_backward\_16\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-318}{363}{50}{54}{351}&100\%\\
cocomin\_backward\_2\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-328}{373}{50}{54}{243}&100\%\\
cocomin\_forward\_2\_correlation\_mmre &   0.0 & -100\% & \boxplot{-173}{218}{50}{54}{266}&100\%\\
cocomin\_backward\_8\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-318}{363}{50}{54}{352}&100\%\\
cocomin\_backward\_4\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-318}{363}{50}{53}{352}&100\%\\
cocomin\_backward\_1\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{331}&100\%\\
cocomin\_backward\_1\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-328}{373}{50}{53}{352}&100\%\\
cocomin\_forward\_2\_correlation\_correlation &   0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\
cocomin\_backward\_8\_correlation\_mmre &  -0.3 & -100\% & \boxplot{-328}{372}{49}{53}{299}&100\%\\
cocomin\_backward\_16\_correlation\_mmre &  -0.5 & -100\% & \boxplot{-225}{270}{49}{53}{299}&100\%\\
cocomin\_forward\_8\_correlation\_correlation &   0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\
cocomin\_forward\_4\_correlation\_correlation &   0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\
cocomin\_forward\_16\_correlation\_correlation &   0.0 & -100\% & \boxplot{-219}{265}{50}{54}{281}&100\%\\
cocomin\_backward\_8\_correlation\_correlation &   0.0 & -100\% & \boxplot{-227}{272}{50}{53}{257}&100\%\\
cocomin\_backward\_16\_correlation\_correlation &   0.0 & -100\% & \boxplot{-227}{272}{50}{53}{257}&100\%\\
cocomin\_backward\_0\_correlation\_sd\_mre &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\
cocomin\_backward\_0\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\
cocomin\_backward\_0\_correlation\_mmre &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\
cocomin\_backward\_0\_correlation\_correlation &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{353}&100\%\\
cocomin\_backward\_2\_correlation\_pred30 &   0.0 & -100\% & \boxplot{-328}{372}{50}{53}{352}&100\%\\
cocomin\_backward\_2\_correlation\_correlation &   0.0 & -100\% & \boxplot{-325}{370}{50}{53}{264}&100\%\\
LC &  -1.3 & -100\% & \boxplot{-354}{397}{49}{54}{374}&100\%\\
cocomin\_backward\_4\_correlation\_correlation &   0.0 & -100\% & \boxplot{-311}{356}{50}{53}{257}&100\%\\
cocomin\_backward\_1\_correlation\_mmre &   0.0 & -100\% & \boxplot{-328}{372}{50}{52}{353}&100\%\\
cocomin\_backward\_1\_correlation\_correlation &   0.0 & -100\% & \boxplot{-325}{369}{50}{53}{265}&100\%\\
cocomin\_backward\_4\_correlation\_mmre &  -0.2 & -100\% & \boxplot{-328}{372}{49}{53}{299}&100\%\\
cocomin\_forward\_4\_correlation\_mmre &  -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\
cocomin\_backward\_2\_correlation\_mmre &  -0.6 & -100\% & \boxplot{-328}{372}{49}{52}{353}&100\%\\
cocomin\_forward\_8\_correlation\_mmre &  -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\
cocomin\_forward\_16\_correlation\_mmre &  -0.7 & -100\% & \boxplot{-173}{217}{49}{53}{267}&100\%\\
cocomost &  -2.0 & -100\% & \boxplot{-354}{397}{49}{53}{264}&100\%\\
\cline{1-2}\end{tabular}
\end{comment}

%method&median&-808.2\%&\boxplot{50}{0}{50}{53}{0}&808.2\%\\

\begin{comment}
\scriptsize
\begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2}
method&median&&&\\\cline{1-2}
cocomin-f-0-sd-mre&  4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-pred30&  4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-mmre&  4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-corr&  4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-1-pred30&  4.1&-808.2\%&\boxplot{3}{46}{52}{53}{28}&808.2\%\\
cocomin-f-1-sd-mre&  2.2&-808.2\%&\boxplot{3}{46}{51}{53}{29}&808.2\%\\
cocomin-f-1-mmre&  1.7&-808.2\%&\boxplot{3}{46}{50}{53}{49}&808.2\%\\
cocomin-f-2-pred30&  1.5&-808.2\%&\boxplot{12}{37}{50}{53}{23}&808.2\%\\
cocomin-b-16-sd-mre&  0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\
cocomin-b-8-sd-mre&  0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\
cocomin-f-4-pred30&  0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-b-4-sd-mre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{26}&808.2\%\\
cocomin-f-4-sd-mre&  0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-1-corr&  0.3&-808.2\%&\boxplot{9}{40}{50}{53}{32}&808.2\%\\
cocomin-f-2-sd-mre&  0.0&-808.2\%&\boxplot{12}{37}{50}{53}{30}&808.2\%\\
cocomin-f-16-sd-mre&  0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-8-sd-mre&  0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-8-pred30&  0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-f-16-pred30&  0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-b-16-pred30&  0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-2-sd-mre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{30}&808.2\%\\
cocomin-f-2-mmre&  0.0&-808.2\%&\boxplot{22}{27}{50}{53}{32}&808.2\%\\
cocomin-b-8-pred30&  0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-4-pred30&  0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-1-sd-mre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{41}&808.2\%\\
cocomin-b-1-pred30&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-f-2-corr&  0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-b-8-mmre& -0.3&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\
cocomin-b-16-mmre& -0.5&-808.2\%&\boxplot{15}{33}{49}{53}{37}&808.2\%\\
cocomin-f-8-corr&  0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-f-4-corr&  0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-f-16-corr&  0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-b-8-corr&  0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\
cocomin-b-16-corr&  0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\
cocomin-b-0-sd-mre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-pred30&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-mmre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-corr&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-2-pred30&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-2-corr&  0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\
LC& -1.3&-808.2\%&\boxplot{0}{49}{49}{53}{46}&808.2\%\\
cocomin-b-4-corr&  0.0&-808.2\%&\boxplot{5}{44}{50}{53}{31}&808.2\%\\
cocomin-b-1-mmre&  0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-1-corr&  0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\
cocomin-b-4-mmre& -0.2&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\
cocomin-f-4-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomin-b-2-mmre& -0.6&-808.2\%&\boxplot{3}{46}{49}{53}{43}&808.2\%\\
cocomin-f-8-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomin-f-16-mmre& -0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomost& -2.0&-808.2\%&\boxplot{0}{49}{49}{53}{32}&808.2\%\\
\cline{1-2}\end{tabular}
\end{comment}

\scriptsize
\begin{tabular}{|rr|r@{~}r@{~}l}\cline{1-2}
method&median&&&\\\cline{1-2}
cocomin-f-0-sd-mre&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-pred30&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-mmre&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-0-corr&4.9&-808.2\%&\boxplot{3}{46}{52}{53}{48}&808.2\%\\
cocomin-f-1-pred30&4.1&-808.2\%&\boxplot{3}{46}{52}{53}{28}&808.2\%\\
cocomin-f-1-sd-mre&2.2&-808.2\%&\boxplot{3}{46}{51}{53}{29}&808.2\%\\
cocomin-f-1-mmre&1.7&-808.2\%&\boxplot{3}{46}{50}{53}{49}&808.2\%\\
cocomin-f-2-pred30&1.5&-808.2\%&\boxplot{12}{37}{50}{53}{23}&808.2\%\\
cocomin-b-16-sd-mre&0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\
cocomin-b-8-sd-mre&0.0&-808.2\%&\boxplot{23}{26}{50}{53}{38}&808.2\%\\
cocomin-f-4-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-b-4-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{26}&808.2\%\\
cocomin-f-4-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-1-corr&0.3&-808.2\%&\boxplot{9}{40}{50}{53}{32}&808.2\%\\
cocomin-f-2-sd-mre&0.0&-808.2\%&\boxplot{12}{37}{50}{53}{30}&808.2\%\\
cocomin-f-16-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-8-sd-mre&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{25}&808.2\%\\
cocomin-f-8-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-f-16-pred30&0.0&-808.2\%&\boxplot{19}{30}{50}{53}{23}&808.2\%\\
cocomin-b-16-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-2-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{30}&808.2\%\\
cocomin-f-2-mmre&0.0&-808.2\%&\boxplot{22}{27}{50}{53}{32}&808.2\%\\
cocomin-b-8-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-4-pred30&0.0&-808.2\%&\boxplot{4}{44}{50}{53}{43}&808.2\%\\
cocomin-b-1-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{41}&808.2\%\\
cocomin-b-1-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-f-2-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-b-8-mmre&-0.3&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\
cocomin-b-16-mmre&-0.5&-808.2\%&\boxplot{15}{33}{49}{53}{37}&808.2\%\\
cocomin-f-8-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-f-4-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-f-16-corr&0.0&-808.2\%&\boxplot{16}{32}{50}{53}{34}&808.2\%\\
cocomin-b-8-corr&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\
cocomin-b-16-corr&0.0&-808.2\%&\boxplot{15}{33}{50}{53}{31}&808.2\%\\
cocomin-b-0-sd-mre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-mmre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-0-corr&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-2-pred30&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-2-corr&0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\
LC&-1.3&-808.2\%&\boxplot{0}{49}{49}{53}{46}&808.2\%\\
cocomin-b-4-corr&0.0&-808.2\%&\boxplot{5}{44}{50}{53}{31}&808.2\%\\
cocomin-b-1-mmre&0.0&-808.2\%&\boxplot{3}{46}{50}{53}{43}&808.2\%\\
cocomin-b-1-corr&0.0&-808.2\%&\boxplot{3}{45}{50}{53}{32}&808.2\%\\
cocomin-b-4-mmre&-0.2&-808.2\%&\boxplot{3}{46}{49}{53}{37}&808.2\%\\
cocomin-f-4-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomin-b-2-mmre&-0.6&-808.2\%&\boxplot{3}{46}{49}{53}{43}&808.2\%\\
cocomin-f-8-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomin-f-16-mmre&-0.7&-808.2\%&\boxplot{22}{26}{49}{53}{33}&808.2\%\\
cocomost&-2.0&-808.2\%&\boxplot{0}{49}{49}{53}{32}&808.2\%\\
\cline{1-2}\end{tabular}


\normalsize


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Results}

%%%%%%%%Deltas%%%%%%%%%%%%%%%%%%%

%\begin{figure}
%{\footnotesize{\bf all:}
%\input{cocomost_all.changes}}
%%{\footnotesize {\bf NotPf=$100- pf$}
%%\input{notPf.changes}}
%%{\footnotesize {\bf Balance}
%%\input{bal.changes}}
%%\caption{Performance deltas for $pd,notPf,bal$ using all 38 attributes.}\label{fig:allpd}.
%\caption{Performance deltas for }\label{fig:allpd}
%\end{figure}

\begin{comment}
\begin{figure}[!b]
{\scriptsize\begin{alltt}

ID,deltas(95%),                                        #rx, mean,   sd,    n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),       cocomin_forward_0_correlation_sd_mre, 13.9, 63.0,36015,-757.4, -5.0,   4.9, 24.8,808.2,**********|.*********
 2,      (= 1),       cocomin_forward_0_correlation_pred30, 13.9, 63.0,36015,-757.4, -5.0,   4.9, 24.8,808.2,**********|.*********
 3,      (= 1),         cocomin_forward_0_correlation_mmre, 13.9, 63.0,36015,-757.4, -5.0,   4.9, 24.8,808.2,**********|.*********
 4,      (= 1),  cocomin_forward_0_correlation_correlation, 13.9, 63.0,36015,-757.4, -5.0,   4.9, 24.8,808.2,**********|.*********
 5,      (< 1),       cocomin_forward_1_correlation_pred30,  9.1, 51.5,36015,-757.4, -5.7,   4.1, 20.2,487.7,**********|.*********
 6,      (< 5),       cocomin_forward_1_correlation_sd_mre,  6.2, 48.8,36015,-757.4, -6.7,   2.2, 17.6,487.7,**********|.*********
 7,      (< 6),         cocomin_forward_1_correlation_mmre,  4.8, 48.3,36015,-757.4, -7.3,   1.7, 15.7,808.2,**********|.*********
 8,      (= 7),       cocomin_forward_2_correlation_pred30,  4.6, 44.5,36015,-608.9, -8.0,   1.5, 15.8,393.5,**********|.*********
 9,      (< 7),     cocomin_backward_16_correlation_sd_mre,  3.0, 41.3,36015,-432.3, -8.9,   0.0, 12.4,632.7,**********|**********
10,      (= 9),      cocomin_backward_8_correlation_sd_mre,  2.9, 41.3,36015,-432.3, -8.9,   0.0, 12.3,632.7,**********|**********
11,      (< 9),       cocomin_forward_4_correlation_pred30,  1.2, 40.6,36015,-497.4,-10.2,   0.0, 13.1,393.5,*********.|**********
12,     (= 11),      cocomin_backward_4_correlation_sd_mre,  0.8, 39.8,36015,-756.6, -8.7,   0.0, 10.2,446.1,*********.|**********
13,     (= 11),       cocomin_forward_4_correlation_sd_mre,  0.7, 40.7,36015,-551.2, -9.5,   0.0, 11.0,418.0,**********|**********
14,     (= 11),  cocomin_forward_1_correlation_correlation,  0.7, 39.9,36015,-660.9, -8.7,   0.3, 12.0,536.4,**********|**********
15,     (= 11),       cocomin_forward_2_correlation_sd_mre,  0.6, 41.8,36015,-609.3, -9.4,   0.0, 11.1,502.9,*********.|**********
16,     (< 11),      cocomin_forward_16_correlation_sd_mre,  0.6, 40.8,36015,-551.2,-10.0,   0.0, 11.1,418.0,**********|**********
17,     (= 16),       cocomin_forward_8_correlation_sd_mre,  0.1, 40.6,36015,-551.2,-10.3,   0.0, 10.3,418.0,**********|**********
18,     (< 16),       cocomin_forward_8_correlation_pred30, -0.5, 39.6,36015,-497.4,-11.6,   0.0, 11.8,393.5,*********.|**********
19,     (= 18),      cocomin_forward_16_correlation_pred30, -0.5, 39.6,36015,-497.4,-11.6,   0.0, 11.8,393.5,*********.|**********
20,     (< 18),     cocomin_backward_16_correlation_pred30, -1.1, 39.8,36015,-737.3,-10.9,   0.0,  9.3,712.7,*********.|**********
21,     (= 20),      cocomin_backward_2_correlation_sd_mre, -1.2, 37.9,36015,-756.6, -9.0,   0.0,  9.1,495.8,*********.|**********
22,     (= 20),         cocomin_forward_2_correlation_mmre, -1.5, 39.8,36015,-447.4, -9.6,   0.0,  9.3,541.6,*********.|**********
23,     (= 20),      cocomin_backward_8_correlation_pred30, -1.6, 39.6,36015,-737.3,-10.7,   0.0,  8.7,712.7,*********.|**********
24,     (< 20),      cocomin_backward_4_correlation_pred30, -2.0, 39.7,36015,-737.3,-10.7,   0.0,  7.6,712.7,*********.|**********
25,     (= 24),      cocomin_backward_1_correlation_sd_mre, -2.1, 40.4,36015,-756.6,-10.7,   0.0,  7.0,670.2,*********.|**********
26,     (= 24),      cocomin_backward_1_correlation_pred30, -2.3, 40.9,36015,-756.6,-10.4,   0.0,  7.1,712.7,*********.|**********
27,     (< 24),  cocomin_forward_2_correlation_correlation, -2.6, 35.9,36015,-539.5, -9.4,   0.0,  8.7,572.5,**********|**********
28,     (= 27),        cocomin_backward_8_correlation_mmre, -2.6, 36.5,36015,-756.6,-11.4,  -0.3,  6.7,604.8,*********.|**********
29,     (= 27),       cocomin_backward_16_correlation_mmre, -2.6, 35.2,36015,-551.7,-11.7,  -0.5,  6.7,604.8,*********.|**********
30,     (= 27),  cocomin_forward_8_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5,   0.0,  8.6,572.5,*********.|**********
31,     (= 27),  cocomin_forward_4_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5,   0.0,  8.6,572.5,*********.|**********
32,     (= 27), cocomin_forward_16_correlation_correlation, -2.7, 36.0,36015,-539.5, -9.5,   0.0,  8.6,572.5,*********.|**********
33,     (= 27), cocomin_backward_8_correlation_correlation, -2.7, 34.8,36015,-555.0, -9.9,   0.0,  7.9,523.2,**********|**********
34,     (= 27),cocomin_backward_16_correlation_correlation, -2.7, 34.8,36015,-555.0, -9.9,   0.0,  7.9,523.2,**********|**********
35,     (= 27),      cocomin_backward_0_correlation_sd_mre, -2.8, 42.4,36015,-756.6,-10.9,   0.0,  6.4,712.7,*********.|**********
36,     (= 27),      cocomin_backward_0_correlation_pred30, -2.8, 42.4,36015,-756.6,-10.9,   0.0,  6.4,712.7,*********.|**********
37,     (= 27),        cocomin_backward_0_correlation_mmre, -2.8, 42.4,36015,-756.6,-10.9,   0.0,  6.4,712.7,*********.|**********
38,     (= 27), cocomin_backward_0_correlation_correlation, -2.8, 42.4,36015,-756.6,-10.9,   0.0,  6.4,712.7,*********.|**********
39,     (= 27),      cocomin_backward_2_correlation_pred30, -2.9, 39.4,36015,-756.6,-10.7,   0.0,  7.3,712.7,*********.|**********
40,     (= 27), cocomin_backward_2_correlation_correlation, -3.0, 36.0,36015,-750.2, -9.8,   0.0,  7.9,536.4,**********|**********
41,     (= 27),                                         LC, -3.0, 48.8,36015,-808.2,-13.5,  -1.3,  8.0,757.4,*********.|**********
42,     (< 27), cocomin_backward_4_correlation_correlation, -3.2, 35.7,36015,-722.6, -9.9,   0.0,  7.5,523.2,**********|**********
43,     (= 42),        cocomin_backward_1_correlation_mmre, -3.4, 41.7,36015,-756.6,-11.3,   0.0,  5.5,712.7,*********.|**********
44,     (= 42), cocomin_backward_1_correlation_correlation, -3.6, 37.3,36015,-750.2,-11.3,   0.0,  6.4,536.4,*********.|**********
45,     (< 42),        cocomin_backward_4_correlation_mmre, -4.0, 36.1,36015,-756.6,-11.2,  -0.2,  6.1,604.8,*********.|**********
46,     (= 45),         cocomin_forward_4_correlation_mmre, -4.2, 38.4,36015,-447.4,-12.0,  -0.7,  6.6,541.6,*********.|**********
47,     (= 45),        cocomin_backward_2_correlation_mmre, -4.3, 38.7,36015,-756.6,-11.3,  -0.6,  4.9,712.7,*********.|**********
48,     (= 45),         cocomin_forward_8_correlation_mmre, -4.5, 38.4,36015,-447.4,-12.3,  -0.7,  6.6,541.6,*********.|**********
49,     (= 45),        cocomin_forward_16_correlation_mmre, -4.5, 38.4,36015,-447.4,-12.3,  -0.7,  6.6,541.6,*********.|**********
50,     (< 45),                                   cocomost, -5.1, 38.1,36015,-808.2,-13.3,  -2.0,  6.7,536.5,*********.|**********

\end{alltt}}
\caption{Performance deltas for all learners combinations over all 19 datasets collectively.
}\label{fig:deltas_all}
\end{figure}

% 3 learners by 2 coc81,nasa93
\begin{figure}[!b]
{\scriptsize\begin{alltt}
coc81
ID,deltas(95%),                                #rx, mean,   sd,  n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),cocomin_forward_16_correlation_mmre,  4.3, 40.0,420, -35.9, -4.5,   0.0,  5.5,541.6,......****|**********
 2,      (= 1),                           cocomost,  0.5, 30.2,420,-541.6, -5.2,   0.4,  7.2, 71.3,**********|*******...
 3,      (< 1),                                 LC, -4.8, 30.4,420,-541.6, -7.2,   0.0,  2.1, 35.9,**********|***.......

nasa93
ID,deltas(95%),                                #rx, mean,   sd,   n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  4.4, 46.4,1050,-313.5, -8.6,   0.7, 11.1,383.1,**********|**********
 2,      (< 1),                           cocomost, -2.0, 34.8,1050,-383.1, -8.0,  -0.1,  6.8,313.5,**********|**********
 3,      (= 2),cocomin_forward_16_correlation_mmre, -2.3, 36.7,1050,-370.7, -7.3,   0.0,  7.1,313.5,**********|**********

\end{alltt}}
\caption{Performance deltas for LC, cocomost, and cocomin over the coc81 and nasa93 datasets.
}\label{fig:deltas_byCoc81Nasa93}
\end{figure}

% 3 learners by each subset
\begin{figure}[!b]
{\scriptsize\begin{alltt}
coc81_all
ID,deltas(95%),                                #rx, mean,   sd,  n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  0.7,  8.8,126,-16.9, -5.3,   0.1,  5.2, 23.9,........**|**........
 2,      (= 1),cocomin_forward_16_correlation_mmre, -0.4,  6.2,126,-23.9, -0.1,   0.0,  0.0, 16.9,........**|*.........
 3,      (= 1),                                 LC, -0.4,  6.2,126,-23.9, -0.1,   0.0,  0.0, 16.9,........**|*.........
 
coc81_kind_min
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),cocomin_forward_16_correlation_mmre, 28.7,117.0,42, -18.4, -8.0,   2.6, 11.8,541.6,........**|**********
 2,      (= 1),                           cocomost,-11.9, 84.4,42,-541.6, -3.6,   0.8,  8.5, 22.6,**********|**........
 3,      (< 1),                                 LC,-16.8, 83.9,42,-541.6,-10.2,  -2.6,  3.6, 18.4,**********|*.........
 
coc81_lang_ftn
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),cocomin_forward_16_correlation_mmre,  5.2, 22.8,48, -28.3, -3.7,   0.4, 10.4,100.0,.......***|*********.
 2,      (= 1),                                 LC,  3.9, 20.3,48, -93.0, -3.9,   2.8, 16.6, 34.5,.*********|.**.......
 3,      (< 1),                           cocomost, -9.1, 18.3,48,-100.0,-16.5,  -7.0,  0.0, 19.7,*********|**.........
 
coc81_mode_e
ID,deltas(95%),                                #rx, mean,   sd, n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  6.0, 18.3,56,-32.9, -7.7,   1.6, 15.7, 61.5,.......***|.*****....
 2,      (= 1),cocomin_forward_16_correlation_mmre,  4.6, 17.8,56,-27.9, -5.9,   0.0, 11.2, 71.7,.......***|*******...
 3,      (< 1),                                 LC,-10.6, 19.6,56,-71.7,-21.6,  -8.1,  3.4, 22.8,...******|**.........
 
coc81_kind_max
ID,deltas(95%),                                #rx, mean,   sd, n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  2.6, 11.0,62,-38.6,  0.0,   0.0,  5.9, 25.2,......****|**........
 2,      (= 1),cocomin_forward_16_correlation_mmre,  2.0, 12.0,62,-24.4,  0.0,   0.0,  0.0, 43.9,........**|****......
 3,      (< 1),                                 LC, -4.6, 10.1,62,-43.9, -5.4,   0.0,  0.0,  7.9,......****|..........
 
coc81_mode_org
ID,deltas(95%),                                #rx, mean,   sd, n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  6.3, 15.0,46,-31.2, -2.0,   3.9, 10.0, 62.4,.......***|******....
 2,      (< 1),cocomin_forward_16_correlation_mmre, -1.5, 17.0,46,-35.9, -7.4,  -0.5,  4.4, 72.5,......***.|******....
 3,      (= 2),                                 LC, -4.8, 20.0,46,-72.5,-10.0,  -2.6,  3.2, 35.9,...*******|***.......
 
coc81_lang_mol
ID,deltas(95%),                                #rx, mean,   sd, n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  6.1, 18.1,40,-32.1, -2.5,   1.6, 11.8, 71.3,.......***|******....
 2,      (= 1),cocomin_forward_16_correlation_mmre,  2.7, 17.7,40,-24.1,-11.7,  -0.5, 14.2, 49.4,........**|****......
 3,      (< 1),                                 LC, -8.7, 19.4,40,-71.3,-15.2,  -3.5,  0.6, 20.9,...******.|**........
 
nasa93_all
ID,deltas(95%),                                #rx, mean,   sd,  n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  3.1, 36.7,186,-149.1, -8.9,  -1.6,  8.5,195.5,**********|**********
 2,      (= 1),cocomin_forward_16_correlation_mmre, -0.8, 28.0,186,-195.5, -7.6,   0.0,  7.9,149.1,*********.|**********
 3,      (= 1),                           cocomost, -2.3, 28.8,186,-195.5, -7.9,   0.5,  7.8,121.5,*********.|**********
 
nasa93_project_gro
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  8.1, 56.2,46, -46.2,-11.7,   4.3, 10.0,331.2,.....****.|**********
 2,      (= 1),                           cocomost,  3.3, 37.6,46,-147.7, -4.9,   2.4,  6.9,183.5,**********|**********
 3,      (= 1),cocomin_forward_16_correlation_mmre,-11.3, 56.7,46,-331.2, -8.8,  -3.5,  3.0, 38.0,**********|***.......
 
nasa93_cat2_missionplanning
ID,deltas(95%),                                #rx, mean,   sd, n,  min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  5.7, 21.9,40,-46.3, -7.2,   4.0, 10.8, 94.0,.....****.|*********.
 2,      (= 1),                                 LC,  3.0, 25.4,40,-40.9,-15.6,   6.8, 13.6, 78.4,......***..|******...
 3,      (< 1),cocomin_forward_16_correlation_mmre, -8.7, 24.1,40,-94.0,-12.5,  -6.3,  0.9, 40.9,.********|****.......
 
nasa93_cat2_avionicsmonitoring
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),cocomin_forward_16_correlation_mmre,  9.0, 42.8,60, -25.6, -2.5,   0.8,  7.1,313.5,.......***|**********
 2,      (= 1),                           cocomost,  7.3, 44.2,60, -28.1, -3.2,  -0.3,  3.5,313.5,.......***|**********
 3,      (< 1),                                 LC,-16.3, 59.7,60,-313.5, -7.1,  -1.5,  2.7, 13.3,**********|*.........
 
nasa93_mode_semidetached
ID,deltas(95%),                                #rx, mean,   sd,  n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC, 11.6, 49.5,138, -53.8, -6.2,   1.4, 15.1,314.9,.....*****|.*********
 2,      (< 1),cocomin_forward_16_correlation_mmre, -4.0, 36.5,138,-291.1, -7.5,   0.1,  6.5, 53.8,**********|*****.....
 3,      (= 2),                           cocomost, -7.6, 36.1,138,-314.9,-10.6,  -1.4,  5.5, 38.9,**********|****......
 
nasa93_project_sts
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  6.4, 31.4,76, -83.0, -4.9,   4.2, 14.7,147.3,..********|**********
 2,      (= 1),                           cocomost, -2.1, 25.2,76,-147.3, -8.2,  -2.7,  4.8, 83.0,*********.|*******...
 3,      (< 1),cocomin_forward_16_correlation_mmre, -4.2, 23.1,76,-147.3,-11.4,  -1.1,  4.5, 69.2,*********.|******....
 
nasa93_forg_g
ID,deltas(95%),                                #rx, mean,   sd,  n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  1.7, 48.9,160,-262.1, -8.9,  -0.1,  9.9,294.6,**********|**********
 2,      (= 1),                           cocomost,  0.7, 33.8,160,-201.7, -7.4,   1.8,  8.9,246.5,**********|**********
 3,      (= 1),cocomin_forward_16_correlation_mmre, -2.4, 39.3,160,-294.6, -9.9,  -1.9,  7.0,262.1,**********|**********
 
nasa93_center_5
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  2.4, 36.1,78,-144.5,-10.1,   1.4, 16.1,147.6,**********|.*********
 2,      (= 1),                           cocomost, -1.1, 31.5,78,-147.6,-12.4,  -4.1,  5.2,144.5,*********.|**********
 3,      (= 1),cocomin_forward_16_correlation_mmre, -1.3, 26.2,78,-147.6, -5.1,   2.5,  8.0, 89.4,**********|*********.
 
nasa93_year_1975
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  9.4, 40.3,74, -68.4, -6.1,   5.1, 16.8,212.3,...*******.|*********
 2,      (< 1),cocomin_forward_16_correlation_mmre, -4.1, 29.7,74,-212.3, -7.1,   0.0,  2.4, 58.5,**********|*****.....
 3,      (= 2),                           cocomost, -5.3, 29.0,74,-212.3, -7.5,  -1.5,  4.3, 68.4,**********|******....
 
nasa93_year_1980
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC,  9.7, 59.6,76,-104.8,-12.6,   3.0, 17.9,383.1,**********|.*********
 2,      (= 1),cocomin_forward_16_correlation_mmre,  2.4, 45.1,76,-236.1, -8.9,   1.9, 11.0,147.0,*********.|**********
 3,      (< 1),                           cocomost,-12.2, 54.5,76,-383.1,-15.7,  -4.3,  6.8, 72.5,*********.|*******...
 
nasa93_mode_embedded
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                                 LC, 17.6, 73.7,42, -42.4, -8.0,  -3.8, 28.4,370.7,......****|..********
 2,      (= 1),                           cocomost, -4.6, 48.1,42,-270.7, -2.8,   0.0,  5.7,100.0,**********|**********
 3,      (< 1),cocomin_forward_16_correlation_mmre,-13.0, 61.1,42,-370.7, -6.8,   0.0,  2.8, 42.4,**********|***.......
 
nasa93_center_2
ID,deltas(95%),                                #rx, mean,   sd, n,   min,   q1,median,   q3,  max,..........0..........
 1,      (= 1),                           cocomost,  2.1, 21.6,74,-101.4, -5.6,  -1.1,  8.2,119.0,**********|**********
 2,      (= 1),cocomin_forward_16_correlation_mmre,  0.9, 32.5,74,-220.4, -3.2,   3.0, 10.6, 65.5,**********|******....
 3,      (= 1),                                 LC, -3.0, 32.8,74, -65.5,-15.7,  -4.2,  4.5,220.4,...*****..|**********
 
\end{alltt}}
\caption{Performance deltas for LC, cocomost, and cocomin over each of the 19 datasets separately.
}\label{fig:deltas_subsets}
\end{figure}
\end{comment}
% info about attributes dropped

\begin{figure}[!b]
{\scriptsize\begin{alltt}
Average # of Attributes Used: 13.6
 
Average # of Dropped Attributes: 1.4
 
Percentage of Times Each Attribute is Dropped:
rely : 2.4%
data : 11.0%
cplx : 9.0%
time : 0.5%
stor : 12.9%
virt : 1.4%
turn : 64.3%
acap : 1.0%
aexp : 0.0%
pcap : 0.0%
vexp : 0.0%
lexp : 1.9%
modp : 19.0%
tool : 17.1%
sced : 0.5%
\end{alltt}}
\caption{Attributes Dropped by COCOMOST for the COC81 data.
}\label{fig:attributes_cocomost_coc81}
\end{figure}

\begin{figure}[!b]
{\scriptsize\begin{alltt}
Average # of Attributes Used: 8.1
 
Average # of Dropped Attributes: 6.9
 
Percentage of Times Each Attribute is Dropped:
rely : 34.1%
data : 5.3%
cplx : 30.9%
time : 9.7%
stor : 67.0%
virt : 30.9%
turn : 15.0%
acap : 25.0%
aexp : 32.6%
pcap : 68.0%
vexp : 61.5%
lexp : 72.2%
modp : 75.6%
tool : 82.9%
sced : 79.4%
\end{alltt}}
\caption{Attributes Dropped by COCOMOST for the NASA93 data.
}\label{fig:attributes_cocomost_nasa93}
\end{figure}

\begin{figure}[!b]
{\scriptsize\begin{alltt}
Average # of Attributes Used: 13.6
 
Average # of Dropped Attributes: 1.4
 
Percentage of Times Each Attribute is Dropped:
rely : 1.0%
data : 14.3%
cplx : 13.8%
time : 10.0%
stor : 2.4%
virt : 1.9%
turn : 36.2%
acap : 11.4%
aexp : 0.5%
pcap : 2.4%
vexp : 7.1%
lexp : 7.1%
modp : 12.4%
tool : 13.8%
sced : 01.0%
\end{alltt}}
\caption{Attributes Dropped by COCOMIN for the COC81 data.
}\label{fig:attributes_cocomin_coc81}
\end{figure}

\begin{figure}[!b]
{\scriptsize\begin{alltt}
Average # of Attributes Used: 7.7
 
Average # of Dropped Attributes: 7.3
 
Percentage of Times Each Attribute is Dropped:
rely : 36.8%
data : 45.5%
cplx : 41.0%
time : 6.1%
stor : 52.2%
virt : 21.0%
turn : 16.6%
acap : 22.5%
aexp : 35.2%
pcap : 60.2%
vexp : 58.7%
lexp : 65.3%
modp : 93.9%
tool : 88.0%
sced : 82.5%
\end{alltt}}
\caption{Attributes Dropped by COCOMIN for the NASA93 data.
}\label{fig:attributes_cocomin_nasa93}
\end{figure}



%%%%%%%%GRAPHS%%%%%%%%%%%%%%%%%%%

\begin{figure}[!t]
\begin{center}
[{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/mmre.pdf}}]
\caption{Mean Magnitude of Relative Error}
\label{fig:mmre}
\end{center}
\end{figure}

\begin{figure}[!t]
\begin{center}
[{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/sd.pdf}}]
\caption{Standard Deviation of Magnitude of Relative Error}
\label{fig:sd}
\end{center}
\end{figure}

\begin{figure}[!t]
\begin{center}
[{\includegraphics[width=2.5in,clip,keepaspectratio]{stuff/pred30.pdf}}]
\caption{PRED(30)}
\label{fig:pred30}
\end{center}
\end{figure}

\fig{deltas_all} shows the performance deltas of LC, COCOMOST, and each variation of COCOMIN.
COCOMOST performs the best but by a slight margin.  In addition, some variations of COCOMIN outperformed
standard COCOMO-based local calibration.  Another depiction of these slight yet clear differences can be
seen in \fig{mmre}.

Over all 19 datasets, COCOMIN works best when a forward select is used, the Stale factor is set to 16,
correlation is used to rank the attributes, and MMRE is used to evaluate improvement.  For the 
rest of this paper we will consider COCOMIN as this variation only.  In \fig{deltas_all}, one can also
observe the tendency of higher stale values to outperform lower stale values, except with backward select searches.
A backward select search with a low stale value would tend to keep most of the attributes and thus be similar
to LC which explains this phenomenon.

It is clear from \fig{deltas_byCoc81Nasa93} and \fig{deltas_subsets} that the effectiveness of feature subset selection,
at least as seen from the COCOMOST and COCOMIN algorithms, is highly dependent on the underlying data.

\fig{sd} shows that both COCOMOST and COCOMIN were able to reduce the deviation of the mean relative error. 
Interestingly, \fig{mmre} and \fig{sd} both show that COCOMOST and COCOMIN tend to help in the worse cases, as
the deviation and mmre rise, but don't make improvements on the better predictions.  Note that this is 
consistent with the observation that the effectiveness of feature subset selection is highly dependent on the underlying
data, i.e. the learners had a positive impact on the Nasa93 data but not on the Coc81 data.  In addition,
the effectiveness of the feature subset selectors on the worse estimates but not the better estimates explains the
higher Pred30 values for LC seen in \fig{pred30}.

Information about how many and which attributes were dropped by COCOMOST can be found in \fig{attributes_cocomost_nasa93} and \fig{attributes_cocomost_coc81},
and \fig{attributes_cocomost_nasa93} and \fig{attributes_cocomost_coc81} for COCOMIN.  Note that the attributes from 
the NASA93 dataset were discarded far more often than those from the COC81 dataset.

Finally, in this experiment COCOMOST took about 2 seconds for each execution while COCOMIN took about 27.  This is because
the core of COCOMOST was written in C++ and optimized while COCOMIN uses lots of bash and gawk calls.  Local Calibration took
less than a second for each execution.

% add info about execution time?

%(PULL QUARTILE CHARTS TEXT FROM MENZIES PAPER)
%\fig{mmre} shows the mean magnitude of relative error for each of the 19 subsets of data.
%Standard local calibration is shown in red and COCOMOST is in green.
%The results show similar errors between the learners.
%\fig{sd} is similarly structured, except it displays the standard deviation of the error.
%These results show that COCOMOST greatly reduces the variance for a large portion of the datasets.

\section{Conclusion}
\PARstart{T}{he} results show that feature subset selection can improve upon the error and deviation of COCOMO estimates,
but this effectiveness is highly dependent on the underlying data.  If the effort multipliers aren't properly
correlated to the output then it is more likely that a feature subset selector will find improvement by discarding them.
In addition, this improvement was observed on the datasets that gave the worst errors and deviations but not the
datasets with better error rates and deviations, standard COCOMO showed better Pred30 values due to this effect.
In addition, more complete searches of the attribute space were shown to be more effective than limited searches.
COCOMOST was found to have the best MMRE of the learners used in the experiment, and was also shown to be effective
at lowering the standard devation of the errors which will make it easier to distinguish rival methods.  Finallly, while
COCOMOST ran quickly in this set up it clearly won't scale well to growing numbers of attributes in the dataset.

%add line about execution time? 

%\PARstart{T}{he} results show the limits of feature subset selection using local calibration as the evaluation criteria.
%COCOMOST produced statistically equivalent errors as Boehm's COCOMO model.
%Since the attribute space was searched completely, this indicates that more involved evaluation methods or models need to be explored to reduce the error.
%However, COCOMOST was able to reduce the variance in the model which will make it easier to distinguish rival methods.
%This will provide a stepping stone for further research.
%In addition, for this evaluation method, heuristics were found to be less effective and slower than a finely tuned complete search of the attribute space.


\section{Future Work}
\PARstart{T}{here} is a lot of work to be done in improving software cost estimation models.  Some possibilities suggested by this research include:
\bi
\item
Feature Subset Selection using other evaluation methods and learners.
\item
Expanding upon COCOMOST with more data mining techniques such as bagging.
%Expanding upon COCOMOST with more data mining techniques such as bagging(ADD REFERENCE).
\item
More comparisons of learners using COCOMOST as a ``strawman`` FSS model.
\item
Due to COCOMOST's fairly quick runtime it could be used by WRAPPER algorithms such as COSEEKMO.
%Due to COCOMOST's fairly quick runtime it could be used by WRAPPER algorithms (add ref?) such as COSEEKMO.
\ei


\bibliographystyle{IEEEtranBST/IEEEtran.bst}
\bibliography{myrefs,../../../../../tex/refs}

\begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio]
{stuff/timbeach}}]{Tim Menzies}
is an associate professor at the University of West Virginia and works with NASA on software quality.
His recent research concerns modeling and learning, with a particular focus on lightweight modeling methods.
He received his PhD in AI and knowledge engineering from the University of New South Wales.
Contact him at tim@menzies.us.
\end{biography}

\begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio]
{stuff/dan}}]{Dan Baker}
is a graduate student at West Virginia University.  He received his Bachelor's degree in Computer Science
from West Virginia University in May of 2006.  For his senior project he developed the database used in the
West Virginia Crime Reduction and Information Management Effort.
He is currently working on his Master's degree in Computer Science at West Virginia University
while working as a research assistant for Dr Menzies.  His research focuses on data mining in data starved domains such as software cost estimation.
He is the president of the student chapter of Upsilon Pi Epsilon at West Virginia University.
UPE is the first and only international honor society for the Computing Sciences.
\end{biography}


\appendix

{\bf APPENDIX A - COCOMO-I vs COCOMO-II:}
In COCOMO II,
the exponential COCOMO 81
term $b$ was expanded into the following expression:

{\small
\begin{equation}\label{eq:coc2}
b + 0.01*\sum_jSF_j
\end{equation}}
\noindent

where b is 0.91 in COCOMO II 2000, and $SF_j$ is one of five {\em
scale factors} that exponentially influence effort.  Other changes in
COCOMO II included dropping the development modes
of \fig{eso} as well as some modifications
to the  list of effort multipliers and
their associated numeric constants (see appendix E).  


{\bf APPENDIX B - Calculating Correlation:}
Given a test set of size $T$, correlation is calculated
as follows:
{
\[
\begin{array}{l@{~}c@{~}l@{~}l@{~}c@{~}l}
\bar{p}&=& \frac{\sum_I^Tpredicted_i}{T}&\bar{a}   &=& \frac{\sum_I^Tactual_i}{T}\\
S_{p}          &=& \frac{\sum_i^T(predicted_i - \bar{p})^2}{T-1}&
S_{a}          &=& \frac{\sum_i^T(actual_i    - \bar{a})^2}{T-1}\\
S_{pa}         &=& \frac{\sum_i^T(predicted_i - \bar{p})(actual_i - \bar{a})}{T-1}\\
corr           &=& S_{pa}/\sqrt{S_p*S_a}\\
\end{array}
\]}


{\bf APPENDIX C - Local Calibration:}
This approach assumes that a  matrix $D_{i,j}$ 
holds:
\bi
\item
The natural log of the
$KLOC$ estimates;
\item
The natural log of  the actual efforts
for projects $i \le $j$ \le t$;
\item
The  natural logarithm of the cost drivers (the scale factors
and effort multipliers) at locations $1\le i \le 15$ (for COCOMO 81)
or $1 \le i \le 22$ (for COCOMO-II).
\ei
With those assumptions, Boehm~\cite{Boehm81} shows that
for COCOMO 81,
the following calculation yields estimates
for ``$a$'' and ``$b$'' that minimizes the sum of the squares of residual
errors:

\begin{equation}\label{eq:ab}{\small
\left.
\begin{array}{rcl}
EAF_i& =& \sum_j^N D_{i,j}\\
a_0  & =& t \\\
a_1  & =& \sum_i^t KLOC_i\\
a_2  & =& \sum_i^t (KLOC_i)^2\\
d_0  & =& \sum_i^t \left(actual_i - EAF_i\right)\\
d_1  & =& \sum_i^t \left((actual_i - EAF_i)*KLOC_i\right)\\
b  & =& (a_0d_1 - a_1*d_0)/(a_0a_2 - a_1^2)\\
a_3    & =& (a_2d_0 - a_1d_1)/(a_0a_2 - a_1^2)\\
a    & =& e^{a_3}
\end{array}\right\}}
\end{equation}



{\bf APPENDIX D -  COCOMO Numerics:}
\fig{em} shows the COCOMO 81 $EM_j$ (effort multipliers).
The effects of that multiplier on the effort are shown in 
 \fig{effortmults}. 
Increasing the {\em upper} and {\em lower} groups of variables will {\em decrease} or {\em 
increase} the effort estimate, respectively.

\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{l|r@{:~}l|}\cline{2-3}
upper:   &acap&analysts capability\\
increase &pcap&programmers capability\\
these to &aexp&application experience\\
decrease &modp&modern programming practices\\
effort   &tool&use of software tools\\
         &vexp&virtual machine experience\\
         &lexp&language experience\\\cline{2-3}
middle   &sced&schedule constraint\\\cline{2-3}
lower:   &data&data base size\\
decrease &turn&turnaround time\\
these to &virt&machine volatility\\
increase &stor&main memory constraint\\
effort   &time&time constraint for cpu\\
         &rely&required software reliability\\
         &cplx&process complexity\\\cline{2-3}
\end{tabular}}
\end{center}
\caption{{
COCOMO 81 effort multipliers.}}\label{fig:em}
\end{figure}


\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|}
    \hline
&    &very&&&&very&extra\\
    &&low&low&nominal&high&high&high\\
    \hline
upper&ACAP   &1.46   &1.19   &1.00   &0.86   &0.71   &\\
(increase&PCAP   &1.42 &1.17   &1.00   &0.86   &0.70 &\\
these to&AEXP   &1.29 &1.13   &1.00   &0.91   &0.82   &\\
decrease&MODP   &1.2  &1.10 &1.00 &0.91 &0.82 &\\
effort)&TOOL   &1.24 &1.10 &1.00 &0.91 &0.83 &\\
&VEXP   &1.21 &1.10 &1.00 &0.90 &  &\\
&LEXP   &1.14 &1.07 &1.00 &0.95 &  &\\\hline
middle&SCED   &1.23 &1.08 &1.00 &1.04 &1.10 &  \\\hline
lower&DATA   &    & 0.94 &1.00 &1.08 &1.16&\\
(increase&TURN   &       &0.87   &1.00   &1.07   &1.15   &\\
these to&VIRT   &       &0.87   &1.00   &1.15   &1.30   &\\
increase&STOR   &       &       &1.00   &1.06   &1.21   &1.56\\
effort)&TIME   &  &    &1.00   &1.11   &1.30   &1.66\\
&RELY   &0.75& 0.88& 1.00 & 1.15 & 1.40&\\
&CPLX   &0.70 &0.85 &1.00 &1.15 &1.30 &1.65\\
    \hline
\end{tabular}}
\end{center}
\caption{{ The precise COCOMO 81 effort multiplier
values.}}\label{fig:effortmults}
\end{figure}

\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|}
    \hline
&    &very&&&&very&extra\\
    &&low&low&nominal&high&high&high\\
    \hline
upper &ACAP           & 1.2    & 1.1& 1.00   & 0.9& 0.8     & \\
(increase &PCAP           & 1.2    & 1.1& 1.00   & 0.9& 0.8     & \\
these to&AEXP           & 1.2    & 1.1& 1.00   & 0.9& 0.8     & \\
decrease&MODP           & 1.2    & 1.1& 1.00   & 0.9& 0.8     & \\
effort)&TOOL           & 1.2    & 1.1& 1.00   & 0.9& 0.8     & \\
&VEXP           & 1.2    & 1.1& 1.00   & 0.9&          & \\
&LEXP           & 1.2    & 1.1& 1.00   & 0.9&          &\\\hline
middle&SCED     & 1.2    & 1.1& 1.00   & 1.1& 1.2     & \\\hline

lower&DATA           &        & 0.9  & 1.00   & 1.1& 1.2     & \\
(increase&TURN           &         & 0.9 & 1.00   & 1.1& 1.2     & \\
these to&VIRT           &         & 0.9 & 1.00   & 1.1& 1.2    & \\
increase&STOR           &         &     & 1.00   & 1.1& 1.2     & 1.3\\
effort)&TIME           &         &     & 1.00   & 1.1& 1.2     & 1.3\\
&RELY      & 0.8    & 0.9  & 1.00   & 1.1& 1.2     & \\
&CPLX           & 0.8    & 0.9  & 1.00   & 1.1& 1.2     & 1.3\\    \hline
\end{tabular}}
\end{center}
\caption{{ Rounded COCOMO 81 effort multiplier
values.}}\label{fig:roundedem}
\end{figure}

\fig{roundedem} shows the COCOMO 81 effort multipliers of \fig{effortmults}, rounded
and simplified to two significant figures.

\fig{emsf2},
\fig{effortmults2} and
\fig{effortmultsrd2} show the COCOMO-II values analogies to
\fig{em},
 \fig{effortmults}
and \fig{roundedem} (respectively).


\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{l|r@{:~}l|}\cline{2-3}
scale   &prec & have we done this before?\\
factors &flex & development flexibility \\
(exponentially        &resl & any risk resolution activities?\\
 decrease       &team &  team cohesion\\
 effort)       &pmat & process maturity\\\hline
upper  &acap & analyst capability\\
(linearly       &pcap & programmer capability\\
 decrease      &pcon & programmer continuity\\
effort)       &aexp &  analyst experience\\
       &pexp &  programmer experience\\
       &ltex &  language and tool experience\\
       &tool &  tool use\\
       &site &  multiple site development\\
       &sced & length of schedule   \\\hline
lower &rely &    required reliability  \\
(linearly      &data &   secondary memory  storage requirements\\
increase      &cplx &  program complexity\\
effort)      &ruse &  software reuse\\
      &docu &   documentation requirements\\
      &time &   runtime pressure\\
      &stor &   main memory requirements\\
     &pvol &    platform volatility  \\\cline{2-3}
\end{tabular}}
\end{center}
\caption{{
The COCOMO~II scale factors and effort multipliers.}}\label{fig:emsf2}
\end{figure}



\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|}
    \hline &    &extra &very&   &       &    &very &extra\\
           &    &low &low &low&nominal&high&high &high\\
    \hline
scale   &prec &   &6.20  &4.96  &3.72  &2.48  &1.24  &0.00\\
factors &flex &   &5.07  &4.05  &3.04  &2.03  &1.01  &0.00\\
(exponentially        &resl &   &7.07  &5.65  &4.24  &2.83  &1.41  &0.00\\
decreases         &team &   &5.48  &4.38  &3.29  &2.19  &1.10  &0.00\\
effort)        &pmat &   &7.80  &6.24  &4.68  &3.12  &1.56  &0.00\\
    \hline
upper  &acap &   &1.42  &1.19  &1.00  &0.85  &0.71  &  \\
(linearly       &pcap &   &1.34  &1.15  &1.00  &0.88  &0.76  &  \\
 decreases      &pcon &   &1.29  &1.12  &1.00  &0.90  &0.81  &  \\
effort)       &aexp &   &1.22  &1.10  &1.00  &0.88  &0.81  &  \\
       &pexp &   &1.19  &1.09  &1.00  &0.91  &0.85  &  \\
       &ltex &   &1.20  &1.09  &1.00  &0.91  &0.84  &  \\
       &tool &   &1.17  &1.09  &1.00  &0.90  &0.78  &  \\
       &site &   &1.22  &1.09  &1.00  &0.93  &0.86  &0.80\\
       &sced &   &1.43  &1.14  &1.00  &1.00  &1.00  &  \\\hline
lower &rely &    &0.82   &0.92  &1.00  &1.10  &1.26  &  \\
(linearly      &data &    &       &0.90  &1.00  &1.14  &1.28  &  \\
 increases    &cplx &    &0.73   &0.87  &1.00  &1.17  &1.34  &1.74\\
 effort)     &ruse &    &       &0.95  &1.00  &1.07  &1.15  &1.24\\
      &docu &    &0.81   &0.91  &1.00  &1.11  &1.23  &  \\
      &time &    &       &      &1.00  &1.11  &1.29  &1.63\\
      &stor &    &       &      &1.00  &1.05  &1.17  &1.46\\
     &pvol &    &       &0.87  &1.00  &1.15  &1.30  &  \\
    \hline
\end{tabular}}
\end{center}
\caption{{ The precise COCOMO II numerics.}}\label{fig:effortmults2}
\end{figure}


\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{|l|c|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|}
    \hline &    &extra &very&   &       &    &very &extra\\
           &    &low &low &low&nominal&high&high &high\\
    \hline
Scale   &PREC &   &6.3  &5.1  &3.8  &2.5  &1.3  &0\\
Factors &FLEX &   &6.3  &5.1  &3.8  &2.5  &1.3  &0\\
        &RESL &   &6.3  &5.1  &3.8  &2.5  &1.3  &0\\
        &TEAM &   &6.3  &5.1  &3.8  &2.5  &1.3  &0\\
        &PMAT &   &6.3  &5.1  &3.8  &2.5  &1.3  &0\\
    \hline
upper  &ACAP &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &PCAP &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &PCON &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &AEXP &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &PEXP &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\

       &LTEX &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &TOOL &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
       &SITE &   &1.3  &1.1  &1.0  &0.9  &0.8  &0.8\\
       &SCED &   &1.3  &1.1  &1.0  &0.9  &0.8  &  \\
    \hline
lower &RELY &    &0.8   &0.9  &1.0  &1.1  &1.3  &  \\
      &DATA &    &       &0.9  &1.0  &1.1  &1.3  &  \\
      &CPLX &    &0.8   &0.9  &1.0  &1.1  &1.3  &1.5\\
      &RUSE &    &       &0.9  &1.0  &1.1  &1.3  &1.5\\
      &DOCU &    &0.8   &0.9  &1.0  &1.1  &1.3  &  \\
      &TIME &    &       &      &1.0  &1.1  &1.3  &1.5\\
      &STOR &    &       &      &1.0  &1.1  &1.3  &1.5\\
      &PVOL &    &       &0.9  &1.0  &1.1  &1.3  &  \\
    \hline
\end{tabular}}
\end{center}
\caption{{ The rounded COCOMO II numerics.}}\label{fig:effortmultsrd2}
\end{figure}

\end{document}
