% THIS IS SIGPROC-SP.TEX - VERSION 3.1
% WORKS WITH V3.2SP OF ACM_PROC_ARTICLE-SP.CLS
% APRIL 2009
%
% It is an example file showing how to use the 'acm_proc_article-sp.cls' V3.2SP
% LaTeX2e document class file for Conference Proceedings submissions.
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V3.2SP) *DOES NOT* produce:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) Page numbering
% ---------------------------------------------------------------------------------------------------------------
% It is an example which *does* use the .bib file (from which the .bbl file
% is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission,
% you need to 'insert'  your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% Questions regarding SIGS should be sent to
% Adrienne Griscti ---> griscti@acm.org
%
% Questions/suggestions regarding the guidelines, .tex and .cls files, etc. to
% Gerald Murray ---> murray@hq.acm.org
%
% For tracking purposes - this is V3.1SP - APRIL 2009

\documentclass[compsoc,conference]{IEEEtran}
\usepackage{bibspacing}
\usepackage{alltt,times,url,cite}

% Different font in captions
\newcommand{\captionfonts}{\sffamily\footnotesize}

\makeatletter  % Allow the use of @ in command names
\long\def\@makecaption#1#2{%
  \vskip\abovecaptionskip
  \sbox\@tempboxa{{\captionfonts #1: #2}}%
  \ifdim \wd\@tempboxa >\hsize
    {\captionfonts #1: #2\par}
  \else
    \hbox to\hsize{\hfil\box\@tempboxa\hfil}%
  \fi
  \vskip\belowcaptionskip}
\makeatother   % Cancel the effect of \makeatletter


\newcommand{\bd}{\begin{description}}
\newcommand{\ed}{\end{description}}
\newcommand{\bi}{\begin{smallitem}}
\newcommand{\ei}{\end{smallitem}}
\newcommand{\be}{\begin{smallenum}}
\newcommand{\ee}{\end{smallenum}}
\newcommand{\tion}[1]{\S\ref{tion:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newenvironment{smallitem}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{itemize}
   \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{itemize}}
\newenvironment{smallenum}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{enumerate}
  \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{enumerate}}

\ifCLASSINFOpdf
   \usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
   \graphicspath{{../pdf/}{../jpeg/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
   \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
  % \usepackage[dvips]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../eps/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  % \DeclareGraphicsExtensions{.eps}
\fi

\begin{document}
\title{How Much Software Quality Investment is Enough? An AI Approach}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than si% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

%\numberofauthors{1} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
Philip Green, Tim Menzies, Oussama El-Rawas, Steven Williams\\
Lane Department of CS\&EE, West Virginia University, USA,\\
\url{tim@menzies.us, deathcheese@yahoo.com, orawas@gmail.com, stevencwilliams@gmail.com}
}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
%\date{30 July 1999}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
SEESAW is a combination of   AI search and software
process models. 
We show here that, for the purposes of 
exploring different ways to configure a software project,
SEESAW out-performs a    variety of other AI search
engines.  Also, SEESAW's recommendations are greatly effected by the 
business context of its use; specifically,
different {\em value functions} result
in very different conclusions.  For example, the automatic defect reduction
tools explored at SE are only relevant to a subset of software developments .
It is therefore vital for researchers to offer a business context within which
they claim their tools would be useful. 
\end{abstract}

\IEEEpeerreviewmaketitle



\section{Introduction}
Infinite quality assurance is infinitely expensive.
A {\em linear} increase in
the confidence $c$ that we have found all faults can take {\em
exponentially} more effort. For example, in black-box testing,
when detecting a one-in-a-thousand
module faults, moving the confidence
from 90\% to 94\% to 98\% takes 2301,
2812, and 3910 black box tests (respectively)\footnote{Proof:
A randomly
selected input to a program will find a fault with probability $x$.
Voas observes~\cite{voas95} that after $n$ random black-box tests,
the chance of the inputs not revealing any fault is $(1-x)^n$.
hence, the chance $c$ of seeing the fault is $1-(1-x)^n$ which can
be rearranged to
 $n(c,x)=\frac{log(1 -
c)}{log(1-x)}$. for example, $n(0.90,10^{-3})=2301$.}.
For another example,
the
state space explosion problem imposes
strict limits on how much a system can be explored
via automatic formal methods~\cite{me00q}.
Lowry et.al.~\cite{lowrey98} and Menzies and Cukic~\cite{me99q,me00y}
offer numerous other examples
where assessment effectiveness is exponential on effort.

Exponential costs increase quickly exhaust finite QA resources.
Hence, in real world software projects, it is important
to know when enough QA is enough. 
Ideally, software process models can be used to model the trade-offs associated with a project. 
However, as shown below,
standard software process models can suffer from {\em tuning instability} that confuses the conclusions
reached from them.

Previously we have reported some success with stochastic AI tools that conducts search
through the space of possible tunings within process 
models~\cite{me07g,me07f,me09a,me09e,me09f}.
This approach explored that option space
to find conclusions about the organization of a software project 
that are stable across the entire
space of possibilities. For example, in two studies~\cite{me07f,me07g} we found that automated
defect removal tools such as those discussed at the ASE were often required to achieve
minimum defects and minimum development effort and minimum development time\footnote{{\em Time}
refers to calendar months required to complete a project while {\em effort} refers to the number
of staff hours within those months. For example, 4 people working for one year takes
$time=12$ months and $effort=48$ months.}. Elsewhere~\cite{me09a,me09e}, we have used this
tool to comparatively assess different proposed changes to a project. 

The above work has raised several issues:
\bi
\item What is the best AI tool to use for this kind of analysis?
Our prior work used two different methods
without assessing if one method is better or worse than the other. 
\item How do business conditions effect the outcome? Our prior analysis
used a generic $value$ function that modeled ``better, faster, cheaper'' (BFC); i.e.
build software with fewer defects, in less time, with less effort.
Organizations that strive for BFC have produced spectacular
failures~\cite{me09f}. Other business goals may be required for other businesses
such as
``get to market faster, without overly compromising on quality''.
\ei
%In this paper, we show that the search tool used in our previous walk~\cite{me07f} performs much worse than
%a new variant of  local search.
%Accordingly, this paper repeats and extends our previous study using different $value$ functions.
%It will be shown that different $value$ functions lead to very different conclusions. 
%In a result that is 
% very different result to our prior work, we show that  is not always true that the automated defect removal tools
%of the kind explored at this conference are indicated for all projects.
%

To address these issues, 
this  paper proceeds as follows. A background section discusses tuning instability
and proposes one approach to avoid it (AI search through the {\em project options} while performing Monte Carlo sampling
of the {\em tuning options}).  We next discuss how our approach (which does not require data
to tune modes) is different to other approaches (that try to tune models to local data).
After that, some details of our models are explored including the space of project options and tuning
options inside a model. That sections ends with a definition of the two $value$ functions used in this study. Various methods for searching for good projects options are then reviewed. This is followed
by experiments confirming that technology we have used in prior papers (SA) performs worse than
a new method called SEESAW.
SEESAW is then applied to descriptions of four NASA systems to produce reductions in defect and
effort estimates by 50 to 73\% (median values). Finally, when we change $value$ functions
inside SEESAW, it becomes very clear how different project goals lead to a selection of different
ways to organize a project. 

\subsection{Contributions of This Paper}
It does not require the technology of this paper to assert that different
developer $value$s lead to different ways to design a software project.
Software construction is a very
diverse process and, clear, one method is not best for
all projects.
Having said that, the next question is ``how to determine the  method(s) that are best suited
to a particular project?''.  Answering this question is complicated by the myriad of uncertainties
surrounding a software project. 
Tools like those discussed here are vital in order to make, and defend, appropriate choices of methods
for particular projects.

Our prior work in this area used either simulated
annealing~\cite{me07g,me07f,me09f,me09e} or a variant of MaxWalkSat
called SEESAW~\cite{me09a}. In one respect, those reports were
incomplete in that they did not test if those algorithms were better
or worse than any other.  Here, we explore and compare half a dozen
AI tools.

Previous studies have explored planning and project uncertainty.
For example, the title of this paper is a variant of Huang \& Boehm's 2006 paper
``How Much Software Quality Investment is Enough: A Value-Based Approach''~\cite{huang06}.
In value-based SE, methods are assessed by their cost/benefits with respect to particular stake holders.
Huang \& Boehm explored  the proposed reliability of a project to find a ``sweat spot''
where a company's market risk exposure is minimized.
While that
work motivated this paper, it only varied one input feature.
Our process models have dozens of  input variables and three (sometimes competing)
output variables (defect, effort, time predictions). Searching  that larger input/output space
requires automatic support such as the AI tools explored here.

For more notes on related work, see \S7.  

\S2 and \S4 of this paper
(but not
\S4.5.2), and the appendix, have appeared 
previously~\cite{me07g,me07f,me09a,me09e,me09f}). The rest of
this paper is new work, including all the experiments
of \S6.

\section{Background}
%Avoiding Tuning Instability}
%Much of our prior work concerned methods to tune general software process models to local
%conditions. Such tuned models can be used to conduct what-if queries to learn best local
%organizations of a project. There are two drawbacks with such tunings:
%\bi
%\item They bias the conclusions of the model to historical events which may not be relevant to the current situation;
%\item As shown below, the results exhibit  large {\em tuning instabilities} and, hence, large variances in the model
%predictions. 
%\ei
One way to observe the uncertainties associated with a software engineering 
project is to study  the prediction variance of 
software engineering process models.
When the variance is large,
it is difficult to use their output as evidence of the value of 
a proposed change to a software project.
For example, 
consider the following simplified COCOMO~\cite{boehm00b} model, 
\begin{equation}\footnotesize\label{eq:simp}
effort = a \cdot LOC^{b +  pmat}\cdot acap  
\end{equation}
Here, $(a,b)$ control the linear and exponential
effects (respectively) on model estimates; 
while
$pmat$ (process maturity) and $acap$ (analyst capability) are project choices
adjusted by managers. 
\eq{simp} contains two features ($acap,pmat$) and a full COCOMO-II model contains 22~\cite{boehm00b}.
Baker~\cite{baker07} reports a study that learned values of $(a,b)$ for a full COCOMO model
using Boehm's local calibration method~\cite{boehm81}
from 30 randomly selected samples of 90\% of the available project data. 
The ranges varied widely:
\begin{equation}\footnotesize\label{eq:ab}
\left( 2.2 \le a \le 9.18 \right) \wedge \left(0.88 \le b \le 1.09\right)
\end{equation} 
Such large variations make it possible to misunderstand the effects of project options.
Suppose some proposed change  doubles productivity but $a$ moves from 9 to 4.5.
The improvement resulting from that change would be obscured by the tuning variance.


We  tried many ways  to tame tuning instability:
\bi
\item
Feature selection to prune spurious details~\cite{chen05}; 
\item
Instance selection to prune irrelevancies~\cite{me06d}; 
\item Extended data collection
\ei
Our results highlight, all too well, the effects of the uncertainties in SE data.
In summary, despite much effort, the variance observed in our models remains very large.
Even 
after applying instance-based learning, the variance in our effort predictions remains very large~\cite{jalali07}.
Feature subset selection has also been disappointing: while it reduces
the median performance variance somewhat (in our experiments, from
150\% to 53\%~\cite{me06d}) the residual error rates are  so large enough that it is hard
to use the predictions of these models as evidence for the value of some proposed approach.
Lastly, further data collection has not proven useful.
Certainly, there is an increase
in
the availability
of
historical data on prior projects\footnote{see 
\url{http://promisedata.org.data/data}
and \url{http://www.isbsg.org/}.
}. However, Kitchenham~\cite{kitch07} cautions that the literature
is contradictory regarding the value of using
data from other companies to learn
local models.

In order to tame prediction variance, we need to understand its source.
The predictions of a model about a software engineering project are altered by 
project variables $P$ and tuning variables $T$:
\begin{equation}\label{eq:pred}\footnotesize
prediction = model(P, T)
\end{equation}
For example, in \eq{simp}, the tuning options $T$ are the range of $(a,b)$ and the 
project 
options  $P$ are the range of $pmat$ and $acap$. 
Based on the definitions of the COCOMO model
we can say that
the range of the project variables are $P=1 \le (pmat,acap) \le 5$. Further, 
given the cone of uncertainty associated with a particular project $p$,
we can identify the subset of the project options $p\subseteq P$ relevant to a particular  project.
For example, a project manager may be unsure of the exact skill level of team members. However,
if she was to assert
``my analysts are better than most'', then $p$ would include $\{acap=4,acap=5\}$.

This approach assumes that the dominate influence on the $prediction$ are the project options $p$.
Under this assumption, 
the predictions can be controlled by
\bi
\item
Constraining $p$ (using some AI tool);
\item
While leaving  $T$ unconstrained (and sampling $t\in T$ using Monte Carlo methods).
\ei 
Specifically, we seek 
a treatment $r_x \subseteq p$ that maximizes the $value$ of a  model's predictions where
{\em value} is a domain-specific function that scores model outputs according to user goals:
\begin{equation}\footnotesize \label{eq:argmin}
\arg\max_x\left( \overbrace{r_x \subseteq p}^{AI\;search}, 
\underbrace{t \subseteq T, value(model(r_x,t))}_{Monte Carlo} \right)
\end{equation}

\section{Novelty}
This approach is somewhat different to standard methods for decision support
for software process models. From~\cite{me08f}, we define standard practice as:
\be
\item
Collect domain knowledge;
\item
Build an initial model based on step 1 including as yet unknown parameters. 
Note that these unknowns represent a range of tuning options. 
\item
Tune model by (e.g.) regression on local data.
\item
Conduct sensitivity analysis on the tuned models
\ee
These four steps can take quite some time and, at least in our experience with more complex
versions of \eq{simp}, may result in models with large tuning instabilities.
The sensitivity analysis, if conducted using gradient descent methods, will not be successful
for our models since the gradients exhibit the very large variances  of \eq{ab}.
Hence, another method that need not wait for time-consuming (and possibly
pointless) local data collection and tuning:
\be
\item Check the literature for software process models where the dominate influence
on predictions are the project options $p$, not the tuning options $T$.
\item Map project options to model input options;
\item Sample those models using AI tools to constrain the project options while
sampling the tuning options with Monte Carlo methods; i.e. \eq{argmin}
\ee
One heuristic for checking the literature in step \#1 is to avoid large over-elaborated models whose
authors have extended the model far beyond what can be supported with the available data.
We suspect a ``Goldilocks'' principle might be appropriate:
\bi
\item
Tiny models offer trite conclusions and are insensitive to important project features. 
\item
 Very large models may need much data collection to constrain the tunings. 
\item In between there may exist some models that are ``just right''; i.e., big enough to 
draw interesting conclusions, but small enough such that the internal tuning vari- 
ance does not dominate the variance results from input project options. 
\ei
We make no claim that {\em all} process models are ``just
right'' and, hence, can be controlled via \eq{argmin}.
Some process models can be quite complex and include:
discrete-event models~\cite{law00,kelton02};
system dynamics models~\cite{hamid91}; 
state-based models~\cite{akhavi93,harel90,martin00};
rule-based programs~\cite{mi90}; 
or  standard programming constructs
such as those used in Little-JIL~\cite{cass00,wise00}.
These rich modeling frameworks
allow the representation of 
detailed insights into an organization. However,  
the effort required
to tune them may be is non-trivial. 

In terms of the Goldilocks principle, we suspect
that many process models may not be  near the ``right size'' and will require
extensive tuning before they can be used for decision making. 
Fortunately, we have found that
the  USC  COCOMO and COQUALMO models~\cite{boehm00a}  are ``just right''.
In all our studies~\cite{me07g,me07f,me09a,me09e,me09f} we have found
that prediction variance can be controlled by only constraining 
project options while letting the tuning variance remaining unchecked.
Hence, we use these  models for our research.
One advantage of these models is that
they are 
fully described in the literature. 
The same can not be said for other commercial models such as PRICE TRUE PLANNING~\cite{park88}, 
SLIM~\cite{putnam92}, or SEER-SEM~\cite{jensen83}.
Also, at least
for the COCOMO effort model, there exist  baseline results~\cite{chulani99}. 
\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{l|r@{:~}l|}\cline{2-3}
scale   &prec & have we done this before?\\
factors &flex & development flexibility \\
(exponentially        &resl & any risk resolution activities?\\
 decrease       &team &  team cohesion\\
 effort\&cost)       &pmat & process maturity\\\hline
upper  &acap & analyst capability\\
(linearly       &pcap & programmer capability\\
 decrease      &pcon & programmer continuity\\
effort\&cost)       &aexp &  analyst experience\\
       &pexp &  programmer experience\\
       &ltex &  language and tool experience\\
       &tool &  tool use\\
       &site &  multiple site development\\
       &sced & length of schedule   \\\hline
lower &rely &    required reliability  \\
(linearly      &data &   secondary memory  storage requirements\\
increase      &cplx &  program complexity\\
effort)\&cost      &ruse &  software reuse\\
      &docu &   documentation requirements\\
      &time &   runtime pressure\\
      &stor &   main memory requirements\\
     &pvol &    platform volatility  \\\cline{2-3}
\end{tabular}}
\end{center}
\caption{{
The COCOMO ``scale factors'' and ``effort multipliers'' change effort
and cost by an exponential and linear amount (respectively). Increasing these
values has the effect described in column one.}}\label{fig:emsf}
\end{figure}
\begin{figure}[!t]
\begin{center}
{\scriptsize
\begin{tabular}{|r@{:~}l|}\hline
aa & automated analysis\\
etat & execution-based testing and tools\\
pr & peer-reviews\\\hline
\end{tabular}}
\end{center}
\caption{{
The COQUALMO defect removal methods. Increasing these values
decreases delivered defects. }}\label{fig:defect}
\end{figure}
\section{Model Details}
COCOMO offers effort and time predictions while COQUALMO offers defect predictions.
Using the models we can represent the
project options $P$ and tuning options $T$ of \eq{pred} as follows.

\subsection{Project Options: $P$}
COCOMO and COQUALMO's features are shown in \fig{emsf} and \fig{defect}.
The features
have a range taken from
\{very low, low, nominal, high, very high, extremely high\} or
\[\{vl=1, l=2, n=3, h=4, vh=5, xh=6\}\]
These feature include manual methods for defect removal. High values
for  peer reviews (or {\em pr}, see
\fig{defect}) denote formal peer group review activities
(participants have well defined and separate roles, the  reviews are guided
by
extensive 
review checklists/ root cause 
analysis; review are a continuous process guided by
statistical control theory~\cite{shull00b}).

COQUALMO also models automatic methods for defect removal.
Chulani~\cite{chulani99a} defines
the top half of 
{\em automated analysis} as:
\bi
\item[4] (high):
intermediate-level module \& 
inter-module code syntax \& 
semantic analysis. 
Simple requirements/design 
view consistency  checking. 
\item[5] (very high):
More elaborate 
requirements/design view 
consistency checking. 
Basic distributed-processing 
and temporal analysis, model 
checking, symbolic execution. 
\item[6] (extremely high):
Formalized\footnote{
Consistency-checkable pre- 
conditions and post-conditions, 
but not necessarily mathematical theorems.} 
specification and 
verification. 
Temporal 
analysis, model checking, 
symbolic execution. 
\ei
The top half of {\em execution-based testing and tools} is:
\bi
\item[4] (high):
Well-defined test sequence 
tailored to organization 
(acceptance / alpha / beta / 
flight / etc.) test. 
Basic test coverage tools, test 
support system. 
\item[5] (very high):
More advanced tools, test 
data preparation, basic test 
oracle support, distributed 
monitoring \& analysis, 
assertion checking. 
Metrics-based test process 
management. 
\item[6] (extremely high):
Highly advanced tools:
oracles, distributed monitoring 
\& analysis, assertion 
checking 
Integration of automated 
analysis and test tools. 
Model-based test process 
management. 
\ei
In the sequel, the following observation will become important:
\fig{emsf} is much longer than \fig{defect}. This reflects a modeling intuition of COCOMO/COQUALMO:
it is better to stop defects going in (using changes to \fig{emsf}) than to try and find them, once they
have been introduced (using \fig{defect}).
\subsection{Tuning Options: $T$}

For COCOMO effort multipliers (the features that that effort effort/cost in a linear manner), the
off-nominal ranges \{vl=1, l=2, h=4, vh=5, xh=6\} 
change the prediction by some ratio.
The nominal range \{n=3\}, however, corresponds to an effort multiplier of 1, causing
no change to the prediction.
Hence, these  ranges  can be modeled as straight lines $y=mx+b$
passing through the point $\{x,y\}$=$\{3,1\}$. Such a line has a y-intercept
of $b=1 - 3m$. Substituting this value of $b$ into $y=mx+b$ yields:
\begin{equation}\label{eq:lineem}\small
\begin{array}{l}
\forall x \in \{1..6\}\; EM_i=m_\alpha(x-3) + 1
\end{array}
\end{equation}
where  $m_\alpha$ is the effect of $\alpha$ on effort/cost.

We can also derive a general equation for the features that influence cost/effort in an exponential manner.
These features do not ``hinge'' around (3,1) but takes the following form:
\begin{equation}\label{eq:linesf}\small
 \forall x \in \{1..6\}\; SF_i=m_\beta(x - 6) 
\end{equation}
where  $m_\beta$ is the effect of factor $i$ on effort/cost.

COQUALMO contains equations of the same syntactic form as \eq{lineem}  and \eq{linesf}, but with different
coefficients.
Using experience for 161 projects~\cite{boehm00b}, we can find the
maximum and minimum values ever assigned to
$m$ for COQUALMO and COCOMO. Hence, to explore
tuning variance (the
$t\in T$ term in \eq{argmin}),
all we need do is select $m$ values at random from the min/max $m$ values
ever seen.
An appendix to this document lists those ranges.
 
\subsection{Case Studies: $p \subseteq P$}
\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.8}
{\scriptsize
\begin{center}
\begin{tabular}{c|lrr|lr}
       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{fixed settings}\\\hline
project $p_i$&feature&low&high&feature&setting\\\hline
&prec&1&2&data&3\\
$p_1$=OSP:&flex&2&5&pvol&2\\
Orbital&resl&1&3&rely&5\\
space&team&2&3&pcap&3\\
plane&pmat&1&4&plex&3\\
&stor&3&5&site&3\\
&ruse&2&4&&\\
&docu&2&4&&\\
&acap&2&3&&\\
&pcon&2&3&&\\
&apex&2&3&&\\
&ltex&2&4&&\\
&tool&2&3&&\\
&sced&1&3&&\\
&cplx&5&6\\
&KSLOC&75&125\\\hline
&prec&3&5&flex&3\\
$p_2$=OSP2 &pmat&4&5&resl&4\\
&docu&3&4&team&3\\
&ltex&2&5&time&3\\
&sced&2&4&stor&3\\
&KSLOC&75&125&data&4\\
&&&&pvol&3\\
&&&&ruse&4\\
&&&&rely&5\\
&&&&acap&4\\
&&&&pcap&3\\
&&&&pcon&3\\
&&&&apex&4\\
&&&&plex&4\\
&&&&tool&5\\
&&&&cplx&4\\
&&&&site&6\\\hline
&rely&3&5&tool&2\\
$p_3$=JPL &data&2&3&sced&3\\
flight&cplx&3&6&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&7&418&&\\\hline
&rely&1&4&tool&2\\
$p_4$=JPL &data&2&3&sced&3\\
ground&cplx&1&4&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&11&392&&\\
\end{tabular}
\end{center}}
\caption{Four case studies.
Numeric values 
$\{1,2,3,4,5,6\}$ map to {\em very low, low, nominal,
high, very high, extra high}.}\label{fig:cases}
\end{figure}
We use $p$ to denote the subset of the project options $p_i \subseteq P$ relevant to particular projects.
The four particular projects $p_1, p_2, p_3, p_4$ used in the case studies of this paper are shown in
in \fig{cases}:
\bi
\item OSP is the GNC (guidance, navigation, and control)
component of NASA's {\em Orbital Space Plane}; 
\item OSP2 is a later version of OSP;
\item Flight and ground systems reflect typical ranges seen at NASA's Jet Propulsion Laboratory. 
\ei
Some of the features in \fig{cases}
are known precisely (see all the features with single {\em fixed settings}).
But  many of the features in \fig{cases} do not
have precise settings
(see all the features that {\em range} from some {\em low} to {\em high}
value).  Sometimes the ranges
are very narrow (e.g., the process maturity of JPL ground software is between 2
and 3), and  sometimes the ranges are very broad.

\fig{cases} does not mention all the  features listed in \fig{emsf}
inputs.  For example, our defect predictor has inputs for
use of {\em automated analysis}, {\em peer
reviews}, and {\em execution-based testing tools}. For all inputs
not mentioned in \fig{cases}, ranges are picked at random from 
 (usually)  1,2,3,4,5.


\subsection{Treatments: $r_x \subset p$}
The task of our tools is to (a) find a {\em minimal}  subset of the project options $p$ that
most improves the model's predictions; and (b) show that that improvement
is {\em stable} across the range of tuning options described in the preceding section.

{\em Stability:} Solutions that comment on all controllables
may be 
{\em brittle}; i.e.
seemingly minor changes result in radically different performance
scores.
Harman~\cite{harman07} comments that understanding the solution 
{\em neighborhood} is an open and pressing issue:
\begin{quote}
``...{\em robustness} may be as important as solution functionality. For 
example, it may be better to locate an area of the search 
space that is rich in fit solutions, rather than identifying an 
even better solution that is surrounded by a set of far less fit 
solutions.''
\end{quote}

{\em Minimality:} 
Standard numeric optimizations usually offer settings to all controllables.
This may
result in needlessly complex recommendations since a repeated empirical
observation is that many model inputs are noisy or correlated in similar ways 
to model outputs~\cite{hall03}. Such noisy or correlated variables can be pruned away to
generate simpler solutions that are easier and quicker to understand.

Elaborate solutions, on the other hand, may be hard to
operationalize.  Proposing settings to all options is inappropriate
when managers have only partial control over those options. The
authors have worked with many government software projects staffed
by teams of contractors and sub-contractors and sub-sub-contractors.
Hence, any software initiative must be closely monitored for deviations from that initiative.
The cost of that monitoring is a function of how many observables must be tracked.
Minimal solutions make minimal changes and hence require minimal tracking.

In continuous domains, there
is much work on feature selection~\cite{miller02} and
techniques like principal component analysis~\cite{Dil84} to reduce
the number of dimensions reported by an analysis.  Comparative
studies report that discrete AI-based methods can do better at
reducing the size of the reported theory~\cite{hall03}.


%
%\subsubsection{Digression}
%
%
%
%Need some way to adjust the level of QA work to the particular project.
%
%XXXX prior work infinitely expensive
%
%In theory, Boehm's value-based software engineering paradigm could be, in theory,
%
%Is automated software engineering {\em mature}? 
%A {\em mature} engineering field understands the costs and benefits of applying technology. 
%In such a field, there exist clear
%guidelines for when to apply more,
%or less, technology.  For example, aircraft engineers may reject a new engine
%technology if its power/weight ratio
%is too low, or if the technology increases assembly time, or overly complicates maintenance.
%
%Recent keynote addresses at ASE have highlighted this issue.
%Speaking at ASE'04, Barry Boehm~\cite{boehm04} argued against technologies being 
%{\em value-neutral}; i.e. that in that different decision on technologies have
%zero  impacts on the business
%value of development software. Rather, we argued for {\em value-based software engineering}
%where corporations adapt their decisions according to their {\em value-added} (or otherwise)
%of the technology.
%
%Speaking at ASE'08, Cliff Jones~\cite{jone08} was more forceful.
%
%manner in which we assess our field
%currently assess new technologies. In arguing {\em value-based software engineering}~\cite{boehm04},
%
%Cliff Jones Formal Methods~\cite{jones08}
%
%
%Our intuitions are some projects benefit more from automated SE than others. Clearly,
%the
%kernel of the flight guidance system that controls the ascent stages of a manned orbital rocket
%during initial lift-off deserves the entire arsenal of automated SE  (e.g. model-checking
%of the original design~\cite{pecheur02},
%AI search methods for learning best control strategies~\cite{me08f}, 
%neural nets
%to monitor for fueling anomalies as the launch approaches~\cite{markosian05}).
%
%But our guidelines are not clear cut. There exist many gray cases. For example, one rule of thumb at NASA
%is that flight systems are more critical than ground systems. Lutz found that the majority of errors
%associated with deep space NASA missions were associated with ground storage of downloaded data.XXX
%One response to this cautionary tale is to require full automated analysis for all software (i.e. the view
%of ).
%
%footnote{Communicating across the solar system is an inherently complex task that consumes scare resources.
%Errors in ground storage
%``back up'' through the software and eventually lead  to errors.
%
%that there are projects that benefit more from 
%The \textit{proceedings} are the records of a conference.
%ACM seeks to give these conference by-products a uniform,
%high-quality appearance.  To do this, ACM has some rigid
%requirements for the format of the proceedings documents: there
%is a specified format (balanced  double columns), a specified
%set of fonts (Aerial or Helvetica and Times Roman) in
%certain specified sizes (for instance, 9 point for body copy),
%a specified live area (18 $\times$ 23.5 cm [7" $\times$ 9.25"]) centered on
%the page, specified size of margins (2.54cm [1"] top and
%bottom and 1.9cm [.75"] left and right; specified column width
%(8.45cm [3.33"]) and gutter size (.083cm [.33"]).
%
%The good news is, with only a handful of manual
%settings\footnote{Two of these, the {\texttt{\char'134 numberofauthors}}
%and {\texttt{\char'134 alignauthor}} commands, you have
%already used; another, {\texttt{\char'134 balancecolumns}}, will
%be used in your very last run of \LaTeX\ to ensure
%balanced column heights on the last page.}, the \LaTeX\ document
%class file handles all of this for you.
%
%The remainder of this document is concerned with showing, in
%the context of an ``actual'' document, the \LaTeX\ commands
%specifically available for denoting the structure of a
%proceedings paper, rather than with giving rigorous descriptions
%or explanations of such commands.
%
%contributions: all our prior work biased by the same evaluation function.
%here we show that the results are highly dependent on the eval function
%
%\subsection{Definitions}\label{sec:inc}
%To 
%recognize when ASE tools are useful,
%we must precisely define what is meant by ``ASE tools''.
%This section describes the definitions used in this paper (and is taken from 
%previous work~\cite{me07f}).
%This definition is based on the process models developed by Boehm et al. at the
%University of Southern California:
%\bi
%\item
%The COQUALMO defect predictor~\cite[p254-268]{boehm00b};
%\item
%The  COCOMO effort/time predictor~\cite[p29-57]{boehm00b}.
%\ei
%(``Effort'' differs from ``time'' as follows. A project that takes 2 people 3 months
%to complete has an ``effort'' of $2*3=6$ months and a ``time'' of 3 calendar months.)
%
%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{l|r@{:~}l|}\cline{2-3}
%scale   &prec & have we done this before?\\
%factors &flex & development flexibility \\
%(exponentially        &resl & any risk resolution activities?\\
% decrease       &team &  team cohesion\\
% effort\&cost)       &pmat & process maturity\\\hline
%upper  &acap & analyst capability\\
%(linearly       &pcap & programmer capability\\
% decrease      &pcon & programmer continuity\\
%effort\&cost)       &aexp &  analyst experience\\
%       &pexp &  programmer experience\\
%       &ltex &  language and tool experience\\
%       &tool &  tool use\\
%       &site &  multiple site development\\
%       &sced & length of schedule   \\\hline
%lower &rely &    required reliability  \\
%(linearly      &data &   secondary memory  storage requirements\\
%increase      &cplx &  program complexity\\
%effort)\&cost      &ruse &  software reuse\\
%      &docu &   documentation requirements\\
%      &time &   runtime pressure\\
%      &stor &   main memory requirements\\
%     &pvol &    platform volatility  \\\cline{2-3}
%\end{tabular}}
%\end{center}
%\caption{{
%The COCOMO ``scale factors'' and ``effort multipliers'' change effort
%and cost by an exponential and linear amount (respectively). Increasing these
%values has the effect described in column one.}}\label{fig:emsf}
%\end{figure}
%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{|r@{:~}l|}\hline
%aa & automated analysis\\
%etat & execution-based testing and tools\\
%pr & peer-reviews\\\hline
%\end{tabular}}
%\end{center}
%\caption{{
%The COQUALMO defect removal methods. Increasing these values
%decreases delivered defects. }}\label{fig:defect}
%\end{figure}
%These models use the project features shown in \fig{emsf} and \fig{defect}.
%All these features
%have the range
%\{very low, low, nominal, high, very high, extremely high\} or
%\[\{vl=1, l=2, n=3, h=4, vh=5, xh=6\}\]
%Note that these features include sociological impacts on development effort/cost and defects
%such as:
%\bd
%\item[{\em team}:] sociology
%within a development team; e.g. its cohesiveness. 
%\item[{\em pcon}:]
%the staff turnover rate within a project.
%\item[{\em acap, pcap}:] analyst and programmer
%capabilities (respectively).
%\ed
%These feature also include manual methods for defect removal. High values
%for  peer reviews (or {\em pr}, see
%\fig{defect}) are achieved by denote formal peer group review activities
%(participants have well defined and separate roles, the  reviews are guided
%by
%extensive 
%review checklists/ root cause 
%analysis; review are a continuous process guided by
%statistical control theory~\cite{shull00b}).
%
%As to technological features, ``ASE tools''
%appear as
%{\em execution-based testing tools} and
%{\em automated analysis}.
%Chulani~\cite{chulani99a} defines
%the top half of 
%{\em automated analysis} as:
%\bi
%\item[4] (high):
%intermediate-level module and 
%inter-module code syntax and 
%semantic analysis. 
%Simple requirements/design 
%view consistency  checking. 
%\item[5] (very high):
%More elaborate 
%requirements/design view 
%consistency checking. 
%Basic distributed-processing 
%and temporal analysis, model 
%checking, symbolic execution. 
%\item[6] (extremely high):
%Formalized\footnote{
%Consistency-checkable pre- 
%conditions and post-conditions, 
%but not necessarily mathematical theorems.} 
%specification and 
%verification. 
%Advanced distributed 
%processing and temporal 
%analysis, model checking, 
%symbolic execution. 
%\ei
%The upper half of {\em execution-based testing and tools} is:
%\bi
%\item[4] (high):
%Well-defined test sequence 
%tailored to organization 
%(acceptance / alpha / beta / 
%flight / etc.) test. 
%Basic test coverage tools, test 
%support system. 
%\item[5] (very high):
%More advanced test tools, test 
%data preparation, basic test 
%oracle support, distributed 
%monitoring and analysis, 
%assertion checking. 
%Metrics-based test process 
%management. 
%\item[6] (extremely high):
%Highly advanced tools for test 
%oracles, distributed monitoring 
%and analysis, assertion 
%checking 
%Integration of automated 
%analysis and test tools. 
%Model-based test process 
%management. 
%\ei
%A  review of recent proceedings of the IEEE ASE conferences suggests
%that 
%a range of five or six in the above features includes
%the kind of
%``ASE tools'' explored at this venue.
%Hence, to recognize use of   ``ASE tools''  we will check our results for
%\[
%\begin{array}{c}
%\mbox{{\em automated analysis}}\in \{5,6\}\;\vee \\
%\mbox{{\em execution-based testing and tools}} \in \{5,6\}
%\end{array}
%\]
%
%
%
%\section{Related Work}
%\subsection{Value-Based SE}
%Boehm~\cite{boehm06} divides value-based SE into
%the following areas. Note that this paper explores the  last area  (planning and control):
%\bi
%\item
%{\em Value-based requirements engineering:} Rather than treating all
%requirements as  equally important, prioritize them by, say, modeling stake holder goals;
%\item {\em Value-based architecting:} 
%further reconciliation of the system objectives 
%with achievable architectural solutions.
%\item {\em
%Value-based verification and validation:}
%techniques for verifying and 
%validating that a software solution satisfies its value objectives; and processes for 
%sequencing and prioritizing V\&V tasks operating as an investing activity;
%\item {\em
%Value-based risk management: } principles and practices for risk 
%identification, analysis, prioritization, and mitigation;
%\item {\em
%Value-based people management:}
%includes stake holder team building and 
%expectations management; managing the project's accommodation of all stake holders¿ 
%value propositions throughout the life cycle.
%\item {\em
%Value-based design and development:} 
%ensure that the 
%system's objectives and value considerations are inherited by the software's design 
%and development practices; e.g. using requirements-to-design traceability analysis~\cite{hayes06}
%\item
%{\em 
%Value-based quality management:}
% e.g. using soft-goal to debate 
%trade-offs between user goals
%designs~\cite{mylo99,chiang03,me02c,me03h}.
%\item {\em
%Value-based planning and control:}
%principles and practices for extending 
%traditional cost, schedule, and product planning and control techniques to include 
%planning and control of the value delivered to stake holders. For example, 
%in this article and elsewhere~\cite{me07g,me08c,me09a,me09e,orrego09,me09f} 
%we use
%AI search tools to find subsets of the available process options to minimize
%development time, effort, and defects.
%\ei
%Boehm also lists
%numerous {\em elements} as a starting 
%points for realizing value-based software engineering. These elements are listed below,
%along with possible methods to achieve each element:
%\be
%\item {\em Benefits realization analysis:} 
%map the  possible outcomes of planned software initiative to
%stake holders' desired outcome(s); document all success-critical assumptions
%and additional work required to realize the desired outcome.
%\item {\em Stake holder value proposition elicitation and reconciliation:} 
%Using the results of element \#1,
%conduct interviews with 
%stakeholders to find additional high-priority assumption. Seek clashes and reconciliations between
%those assumptions.
%\item {\em Business case analysis:}  
%conduct  what-if analysis to compare expected
%costs/revenues before/after the initiative is implemented. 
%Such what-if queries can be implemented using the
%USC defect and cost/effort predictors~\cite{boehm00b}.
%\item {\em Continuous risk and opportunity management:}  
%Repeat the above at regular intervals during a project. For each repeat, finds ways to
%reduce risks as early as possible in the lifecycle. 
%Much of our previous work has explored automatic methods to find such risk-reduction
%methods that reduce predictions of effect/effort/cost~\cite{me02f,me00e,me07f,me06b}.
%\item {\em Value-based monitoring and control:} 
%Monitor the environment for changes the critical assumptions found above.
%\item {\em Change as Opportunity:} 
%Continuously monitor 
%technology and the marketplace to identify and reorient the project to address 
%unanticipated risks and opportunities. Allow projects to use hybrid
%plan-based methods and agile methods when some parts of the system warrants
%a slower/faster pace of change.    For more on such hybrid strategies, see~\cite{boehmturner03,port08}.
%\ee
%Some of the above elements are inherently manual tasks that require extensive interaction with
%business stake holders.
%For example, it would be  inappropriate to automate the first two
%elements since
%their  is to uncover the {\em value proposition} of
%the humans
%involved in the developments. Put simply, the value proposition  
%of a software initiative what the customer gets for their money {\em from the perspective
%of that customer}.
%
%If we can extract a {\em goal function} from the value proposition, then
%it is possible to offer extensive automatic support for the rest of these elements.
%A goal function combines multiple factors from an organization to return a single value. 
%AI search functions can use this goal function to control a search for better opportunities.
%Such an AI search can explore inputs to
%the predictor models used in the business case analysis (element \#3).
%The output of the predictor models can then be scored by the goal function. 
%
%Elements 4,6,7 can all be implemented using AI search that use goal functions and
% business models:
%\bi
%\item If the AI search tools are run on a regular (e.g. daily) schedule, then they
%be used to find risk reduction of new 
%business opportunities for the continuous risk and opportunity management (element \#4);
%\item  
%If the AI search engine finds conditions under which the software initiative is most, or least,
%successful, then these most-critical situations could guide the monitoring process of 
%monitoring and control (element \#5); 
%\item
%If our monitors raise an alert that the business context has changed then we can re-run
%our AI search engines to find better opportunities for the business (element \#6).
%\ei
%
%\subsection{Optimization over Continuous Domains}
%
%This paper uses heuristic AI search that assumes any continuous numeric variable
%will be divided into independent discrete ranges. 
%In this paper, and elsewhere~\cite{me02f,me00e,me07f,me06b}, we show that 
% such methods can be insightful.
%
%In theory, the AI search functionality discussed above might also be achieved by standard
%numeric optimization methods. For example, given a business model containing smooth
%continuous functions, gradient descent methods can 
%find global minimum values of (say) defects/ effort/ cost.
%
%We do not use gradient descent methods for three reasons:  {\em minimality};
%{\em neighborhood reasoning}; 
%and {\em gradient instability}. 
%
% to reduce a large space of variables to a smaller set.
%
% synthesis of smaller sets
%t
%while there exist some numeric methods
%f
%
%Huang.
%
%
%
% AI search engines 
%Much of the above is labor intensive (e.g. \#6, \#7) and automatic support 
%
%In practice, {\em gradient instability} precludes the use of gradient descent methods.
%Gradient descent requires some knowledge of the shape of the function being explored.
%For our business models, that shape is not known with certainty.
%Elsewhere~\cite{me05a}, we have studied  the variance on the coefficients on our prediction models.
%Our models contain linear and exponential influences on predicted values. 
%After converted these models into a linear model (by applying logarithms)
%these models take the form
%\[ \beta_0 + \beta_1x_1 + \beta_2x_2 + ...\]
%A least-squared
%regression (LSR) over such linear models
%searches for $\beta$ coefficients that minimizes the error between predictions and
%actual values seen in some training set.  
%Using historical data on project development effort,
%we applied LSR 30 times, on 66\% of the available training data (selected at random).
%For each repeat, we performed
%{\em parameter pruning} to remove superfluous $\beta_ix_i$ values\footnote{
%Step through all the $x_i$ values removing
%the one with the smallest standardized coefficient until
%no improvement is observed in the estimate of the model error
%given by the Akaike information criterion. For more details, see~\cite{quinlan92a}.}
%\begin{figure}
%\begin{center}
%\includegraphics[width=2.5in]{var.pdf}
%\end{center}
%\caption{Mean and standard deviation of $\beta_i$ coefficients in 30 repeats
%of LSR. This plot only shows the $\frac{10}{16}$ coefficients that survived parameter pruning
%in the majority of repeats; i.e. more than $\frac{15}{30}$ times.
%From~\cite{me05a}.}\label{fig:unstable}
%\end{figure}
%We say that {\em relevant} features are all  $x_i$ that survive
%pruning in the majority of cases (more than $frac{15}{30}$ times).
%
%The mean and standard deviation of the relevant $\beta_i$ coefficients are shown in 
%\fig{unstable}. Note that 
%\fig{unstable} 
%
%Experienced regression modelers such as Boehm propose reducing the tuning problem to that of
%finding just two coefficients. In the COCOMO-I model, the $(a,b)$ tuning variables allow
%for the adjustment of the $(linear,exponential)$ (respectively)  influences on
%effort. Boehm's {\em local calibration} (LC) procedure is a special case of LSR that just returns new coefficients for $(a,b)$.
%Baker~\cite{baker07}
%studied the range of $(a,b)$ learned from thirty 90\% random samples of some NASA data
%The ranges varied widely:
%$\left( 2.2 \le a \le 9.18 \right) \wedge \left(0.88 \le b \le 1.09\right)$ 
%Such large variations make it possible to misunderstand the effects of different software initiatives.
%For example, suppose some initiative productivity but $a$ moves from 9 to 4.5.
%The improvement resulting from that technology would be obscured by the tuning variance of $a$.
%
%We have tried, and failed, to tame this instability without process models using a variety of techniques
%including instance selection (to remove outliers)~\cite{jalali07}, feature subset selection (to remove
%superfluous features)~\cite{chen05}, or more data collection. Our results to date are not compelling. 
%Even after
%applying instance-based learning, the variance in our predictions remains very large~\cite{jalali07}.
%Feature subset selection has also been disappointing: while it reduces
%the median performance variance somewhat (in our experiments, from
%150\% to 53\%~\cite{me06d}) the residual error ranges are  still large enough to confuse a discussion about the relative merits of
%different SE technologies.
%Lastly, further data collection has not proven useful.
%Certainly, there is an increase
%in
%the availability
%of
%historical data on prior projects\footnote{e.g. see \url{http://www.isbsg.org/} and
%\url{http://promisedata.org.data/data}}. However, it is not
%clear if it is advisable to use data from other companies data to learn local effort models~\cite{kitch07}.
%As to collecting more local data, this has proven remarkably difficult. 
%Even after 26 years of trying, Boehm only collected less
%than 200 sample projects for the COCOMO database from his group of industrial affiliates~\cite{me09e}.
%Our experiments with that extended data set reveal similar instabilities to those reported in \fig{unstable}~\cite{me06d}.
%
%XXX and it just doesn't matter (safe when paper)
%
%uncertainty (bayes etc)
%
%AI search, various theorem proving
%
%manual value-based methods (boehm and turner)
%
%\begin{figure}
%\renewcommand{\baselinestretch}{0.6}
%{\scriptsize
%\begin{center}
%\begin{tabular}{c|lrr|lr}
%       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{values}\\\hline
%project&feature&low&high&feature&setting\\\hline
%&prec&1&2&data&3\\
%OSP:&flex&2&5&pvol&2\\
%Orbital&resl&1&3&rely&5\\
%space&team&2&3&pcap&3\\
%plane&pmat&1&4&plex&3\\
%&stor&3&5&site&3\\
%&ruse&2&4&&\\
%&docu&2&4&&\\
%&acap&2&3&&\\
%&pcon&2&3&&\\
%&apex&2&3&&\\
%&ltex&2&4&&\\
%&tool&2&3&&\\
%&sced&1&3&&\\
%&cplx&5&6\\
%&KSLOC&75&125\\\hline
%&prec&3&5&flex&3\\
%OSP2 &pmat&4&5&resl&4\\
%&docu&3&4&team&3\\
%&ltex&2&5&time&3\\
%&sced&2&4&stor&3\\
%&KSLOC&75&125&data&4\\
%&&&&pvol&3\\
%&&&&ruse&4\\
%&&&&rely&5\\
%&&&&acap&4\\
%&&&&pcap&3\\
%&&&&pcon&3\\
%&&&&apex&4\\
%&&&&plex&4\\
%&&&&tool&5\\
%&&&&cplx&4\\
%&&&&site&6\\\hline
%&rely&3&5&tool&2\\
%JPL &data&2&3&sced&3\\
%flight&cplx&3&6&&\\
%software&time&3&4&&\\
%&stor&3&4&&\\
%&acap&3&5&&\\
%&apex&2&5&&\\
%&pcap&3&5&&\\
%&plex&1&4&&\\
%&ltex&1&4&&\\
%&pmat&2&3&&\\
%&KSLOC&7&418&&\\\hline
%&rely&1&4&tool&2\\
%JPL &data&2&3&sced&3\\
%ground&cplx&1&4&&\\
%software&time&3&4&&\\
%&stor&3&4&&\\
%&acap&3&5&&\\
%&apex&2&5&&\\
%&pcap&3&5&&\\
%&plex&1&4&&\\
%&ltex&1&4&&\\
%&pmat&2&3&&\\
%&KSLOC&11&392&&\\
%\end{tabular}
%\end{center}}
%\caption{Four case studies.
%Numeric values 
%$\{1,2,3,4,5,6\}$ map to $\{very low, low, nominal,
%high, very high, extra high\}$.}\label{fig:cases}
%\end{figure}
%\section{Options Within Projects}
%SEESAW ranks project changes using the COCOMO and COQUALMO effort, time,
%and defect predictors. The terminology of those models is reviewed
%in \fig{cparems}.
%
%Using that terminology, \fig{cases} summarizes four NASA case studies:
%\bi
%\item OSP is the GNC (guidance, navigation, and control)
%component of NASA's 1990s {\em Orbital Space Plane}; 
%\item OSP2 is a later version of OSP;
%\item Flight and ground systems reflect typical ranges seen at NASA's Jet Propulsion Laboratory. 
%\ei
%Some of the features in \fig{cases}
%are known precisely (see all the features with single {\em values}).
%But  many of the features in \fig{cases} do not
%have precise values
%(see all the features that {\em range} from some {\em low} to {\em high}
%value).  Sometimes the ranges
%are very narrow (e.g., the process maturity of JPL ground software is between 2
%and 3), and  sometimes the ranges are very broad.
%\fig{cases} does not mention all the  features listed in \fig{cparems}
%inputs.  For example, our defect predictor has inputs for
%use of {\em automated analysis}, {\em peer
%reviews}, and {\em execution-based testing tools}. For all inputs
%not mentioned in \fig{cases}, values are picked at random from the full
%range of \fig{cparems}.
%

\subsection{Value}
If there exist multiple possible treatments,
a $value$ function is required to rank alternative. 
The $value$ function should model the goals of the business users who are making project
decisions about some software development.
At the end of this paper, we compare results from two very different $value$ functions.
\subsubsection{``BFC'' = Better, Faster, Cheaper}
Ideally, software is built with
fewer defects $D$,
using less effort $E$, and in shorter time $T$.
A value function for this goal can be modeled as the Euclidean distance
to minimum effort, time, defects:

{\footnotesize \begin{eqnarray}\footnotesize\label{eq:bfc}
bfc& =& \sqrt{f\bar{T}^2 + c\bar{E}^2 + \left(b\bar{D}\left(1 + 1.8^{rely -3}\right)\right)^2}\\
value_{bfc}& =& \frac{1}{bfc}
\end{eqnarray}}
In the above, $value$ is highest when defects and effort and development
time are lowest. Also, 
$0 \le (b,f,c) \le 1$
represents the business importance of (better, faster, cheaper). 
For this study, we use $b=f=c=1$. In other work, we have explored the effects
of using other $b,f,c$ values~\cite{me09f}. 

In \eq{bfc}, $\bar{T},\bar{E},\bar{D}$
are the time, effort, and defect scores normalized zero to one.
This equation models the business intuition that
defects in 
high reliability systems 
and exponentially more troublesome than in low reliability systems. 
If reliability moves from very low to very hight (1 to 6), the term
$1.8^{rely-3}$ models an function
that (a) is ten times larger for very high than very low reliability systems;
and (b) passes through one at $rely=3$ (so systems with nominal 
reliability do not change the importance of defects).
\subsubsection{``RISK'' = Business Risk}
The BFC value function is somewhat idealistic in that it wants to remove defects An alternate value function comes from the Huang and Boehm~\cite{huang06}.
This alternate value function, which we call ``RISK'' models the situation where a software company must rush  a product
to market, without compromising too much on software quality. 
Based on Huang's Ph.D. dissertation~\cite{huang06a},   we operatioanlize that RISK $value$ functions as follows.

Huang defines business
risk exposure ($RE$) as a combination of  {\em software quality investment risk exposure} ($RE_q$) and 
{\em market share erosion risk exposure} ($RE_m$).  We invert that expression to yield $value_{risk}$ (so a risky project has low value):

{\footnotesize
\begin{eqnarray}
RE& =&RE_q + RE_m\\
value_{risk}&=&\frac{1}{RE}
\end{eqnarray}}
$RE_q$ says higher quality software is superior to lower quality software.  
Therefore, it prioritized quality over time. 
$RE_q$ is composed of two primary components: probability of loss due to unacceptable quality ($P_q(L)$) 
and size of loss due to unacceptable quality ($S_q(L)$).  
$P_q(L)$ is calculated based on defects.  $S_q(L)$ is calculated based on complexity (the COCOMO cplx feature) , reliability (rely), and a cost function.  
$S_c$ is a value from a Pareto valued table based on rely.  We choose the project months estiamte as the basis of  this cost function.  
\begin{equation}\footnotesize
RE_q = P_q(L) * S_q(L)
\end{equation}
\begin{equation}\footnotesize
P_q(L) = \frac{defects}{defects_{vl}}
\end{equation}
\begin{equation}\footnotesize
S_q(L) = 3^{\frac{cplx-3}{2}} \cdot PM \cdot S_c
\end{equation}
$RE_m$ says getting to market faster is better because you get your product out there before everyone else.  It prioritized time over quality.
$RE_m$ is calculated from PM and reliability (rely).
$M_c$ is a value from a exponential valued table based on rely.
\begin{equation}\footnotesize
RE_m = PM \cdot M_c
\end{equation}


\section{Searching for $r_x$}
In order to support stability and minimality, all our search engines
run in at least two phases: a {\em forward select} and a {\em back select} phase.
The  {\em forward select} incrementally grows $r_x$, starting
with the empty set. At each round $i$ in the forward select:
One (or more)
ranges (e.g. $acap=3$) are added to $r_x$.    
The resulting $r_x$ set found at  round $i$ is denoted $r_x^i$.

The forward select terminates when the search engine cannot find more ranges to usefully add to $r_x^i$.
Before termination,
we say that the {\em open} features at round $i$ 
are the features in \fig{emsf} and \fig{defect} not mentioned by any range
in $r_x^i$.
The value  value of $r_x^i$ is assessed by, $N$ times, running the model with:
\be
\item  all of $r_x^i$
\item any $t\in T$, selected
at random
\item any range (at random) for  open features.
\ee
The results of the $N$ simulations at round $i$ are reported as  ($median, spread$):
the 50\% and the (75-50)\% percentile values, respectively. The median value assesses
the value of $r_x^i$ and the spread value assesses the likelihood  of that median
(i.e. how often we can read the median,
given that the open features and the tunings are still being selected at random).

In order to ensure minimality, a  {\em back select} checks if the  final
$r_x$ set can be pruned. 
If the forward select caches the simulation results seen at each round $i$, the back
select can perform statistical tests to see if the results of round $i-1$ are significantly
different to round $i$. If so, then the ranges added at round $i$ are {\em pruned} away and
the back select recurses for $i-1$.
We call the un-pruned ranges the {\em selected} ranges and
the point where pruning stops to the {\em policy point}.

For example, in \fig{run}, the policy point is round 13
so the  treatment returned by our search engines are all the ranges $r_x^i$ for  $1 \le i \le 13$.
The {\em selected} ranges are shown in a table at the bottom of the figure and 
The effects of applying the conjunction of ranges in $r_x^{13}$ can be seen
in the plot:
\bi
\item Defects/KLOC reduced:  350 to 75; 
\item Time reduced:  16 to 10 months;
\item Effort reduced:  170 to 80 staff months;
\ei
Note that, in accordance with Harmon's instructions (quoted above), our results come with an indicator about
their robustness in the face of change.
Business managers may not want to implement all constraints offered by the selected ranges.
In this case, our results allows them to quickly conduct some what-if queries on using on some
portion
of our recommendations. For example, a manager can see that they can halve
their defect rate just by applying  only the first eight recommendations found by our search tools (see
the defect results at round=8 in \fig{run}).


\begin{figure}[!t]
\footnotesize
\begin{center}
\includegraphics[width=2.5in]{single-seesaw-tpts100-all-trio-2-1.pdf}

\begin{tabular}{|p{.46\linewidth}|p{.46\linewidth}|}
\multicolumn{2}{l}{Decisions made from round=1 to round=13: }\\\hline
round0: $r_x = \emptyset$\newline
round1 added \{pmat=3\}\newline
round2: added  \{resl=4\} \newline
round3: added   \{team=5\}\newline
round4: added \{aexp=4\} \newline
round5: added  \{docu=3\}\newline
round6: added  \{plex=4\}
&
round7: added \{rely=3\} \newline
round8: added  \{stor = 3\}\newline 
round9: added \{time = 3\}\newline
round10: added  \{tool = 4\}\newline
round11: added \{sced = 2\}\newline
round12: added \{site = 4\}\newline
round13: added  \{acap = 5\}
\\\hline
\end{tabular}
\end{center}
\caption{Example forward and back select results.}
\label{fig:run}
\end{figure}


\subsection{Alternate Search Methods}
In this paper, we use six different algorithms for implementing the
forward
select:
SA, 
MaxWalkSat,  
SEESAW, 
BEAM, A-STAR and 
ISAMP. 
The results from STAR and SEESAW have 
been reported previously~\cite{me07f,me09a}.
Those reports were incomplete in that they did not
not test if STAR or SEESAW were better or worse
than other methods.


{\em SA:} 
In our initial experiments~\cite{me07f}, we ran forward/back selects as a 
post-processor to  a simulated annealer (SA).
Consider a typical SA run that has explored 10,000 variants on some solution.
A side-effect of that run is 10,000 sets of inputs, each scored with a value
function. Our STAR tool classified the outputs into 10\% {\em best} values and 90\% 
{\em rest}. All the
ranges from all the features
were then sorted into a list of length $R$
according to how more frequent they
appeared in {\em best} than {\em rest}. The forward select was then
called using the first $i$ items $1 \le i \le R$.

{\em MaxWalkSat}  
is a local search algorithm~\cite{selman96}.
Given a random selected treatment,   
MaxWalkSat attempts $num\_changes$ modifications to randomly 
selected features.
Sometimes   (controlled by the $\alpha$ parameter), 
the modification is ``smart'':
for the selected feature, the algorithm chooses the range
that minimizes the value of the current solution.
The rest of the time (i.e. at probability
$1-\alpha$), a random range is chosen for the feature.
Occasionally, MaxWalkSat will reset to a new randomly selected
initial solution and,
after MAX\_RESETS, attempts, the best solution is returned.
Our implementation used $\alpha=0.5$ and MAX\_RESETS=20.

{\em SEESAW} is a variant of MaxWalkSat we first reported in~\cite{me09f}.
When ``smart'' searching the ranges of a feature,
the algorithm exploits the monotonic nature of \eq{lineem}
and \eq{linesf}.  SEESAW ignores all ranges except the minimum
and maximum values for a feature in $p$.   
Like MaxWalkSat, sometimes it makes random choices but the we found
that SEESAW is most effective when the
probability
s random choice is very low ($\alpha=0.05$).

Our remaining algorithms use some variant of tree search. 
Each branch of the tree is a different treatment
(a conjunction of ranges)  of size $i$.
In terms of the forward
select search described above, the  y-axis
statistics of \fig{run} are collected when ever branches
of size $i$ are extended to size $i+1$.

{\em BEAM search} extends a tree options as follows:
each branch forks once for every new option available to that
range.
All the new leaves are sorted by their value
and only the top $N$ ranked branches are marked for further expansion.
For this study we used $N=10$ and the $y$ axis of \fig{run} was reported
using the median values seen in the top $N$ branches. 
 
{\em A-STAR} runs like BEAM, but the 
sort order determined by the sum $f$ (the cost of reaching the current
solution) plus $g$ (a heuristic estimate of the cost to reach the final
solution).  
Also, unlike BEAM, the list of options is not truncated 
so
a termination criterion is needed (we stop the search if the
best solution so far has not improved after $m$ iterations).
For this study, we used \eq{bfc} for $g$ and the percentage of the
features with ranges in the current branch as $f$.

{\em ISAMP} is a fast stochastic iterative sampling method that extends
search branches using randomly selected branches.
The algorithm has proved remarkably effective 
at scheduling problems,
perhaps because it can rapidly 
explore more of the search space~\cite{craw94}.
To avoid exploring low-value regions, our version of ISAMP stores
the worst solution seen so far. If the value of any branch reaches
that worst solution, that the algorithm resets to empty branch. If
a branch runs out of new ranges to add, then the ``worst value''
is slightly decreased to make it solution is  of options
This ensures that consecutive searches do not permanently raise the
worst value to to an overly permissive amount.

\begin{figure}[!b]
\begin{center}
{\footnotesize\begin{tabular}{r|rrr}
algorithm &  Defects &months &time\\\hline
SEESAW    &4  &          4 &     3\\ 
BEAM      & 0 &         3 &     3\\
A-star    &  0&           1&     1\\
SA        &  0&            1&    1\\
MaxWalkSat & 0 & 0& 0\\
ISSAMP   & 0  & 0& 0
\end{tabular}}
\end{center}
\caption{Number of times algorithm found to be top-ranked from 20 repeats
of forward/back selecting over the four case studies of \fig{cases}.}\label{fig:xper}
\end{figure}
\section{Experiments }
\subsection{Comparing Algorithms}

To test these algorithms,  we performed 20 forward selects with each one over the four case studies if \fig{cases}.
Each forward select was followed by the same back-select: using a t-test, round $i$ was pruned if
it is was statistically insignificantly different (95\% confidence) to round $i-1$.
Separate statistics were
collected for the defects/effort/time predictions seen at the policy point in the 20*4 trials.
The {\em top-ranked} algorithm(s) had statistically  different
and lower defects/effort/time predictions that any other algorithm(s). 

\fig{xper} shows how many times each algorithm was 
{\em top-ranked}. Note that the maximum value 
possible is ``4'' (i.e. once for each \fig{cases} case study). In those results, the two stand-out 
worst are MaxWalkSat and ISSAMP and the two stand-out
best algorithms are SEESAW and BEAM.
The performance of these algorithms sometimes ties:
see the time results where both algorithms achieved equal number of top ranks. 
However, we cannot recommend BEAM search for this domain. Firstly, it tends to run 10 times slower than SEESAW . Secondly, SEESAW
performs better than BEAM in some cases (particularly for defects, where it always beats BEAM).

Another interesting feature of \fig{xper} is the great different between MaxWalkSat and SEESAW. As mentioned above, the difference between these two
algorithms is very small- recall that SEESAW assumed that the local search
state space was monotonic so it only explored minimum and maximum
values for each feature. This single domain heuristic
resulted in a dramatic  performance improvement.
This is not a new result:
AI research in the 1980s~\cite{mcluskey87} showed the value of combining
domain-general ``weak'' heuristic algorithms (e.g. A-STAR, BEAM search) with
domain-specific ``strong'' heuristics such as the monotonicity assumption
exploited by SEESAW.  However, even if it is not a new finding,
it should remind researchers not to depend too heavily on off-the-shelf algorithms.
As shown here, a little algorithm tweaking with domain knowledge can go a long way.

\fig{res} shows the  percent reductions achieved by SEESAW on the four case studies of \fig{cases}.
SEESAW's treatments can  reduce  project
defect/time/effort  estimates by 73, 30, and 57\% (respectively).  
That is, even if SEESAW cannot offer a major reduction in the development time, it can advise how
to reduce defects and development effort by over half.
Based in \fig{res}, we would recommend using SEESAW before all the project decisions are made, otherwise
there is very little for it to work with.
For example, SEESAW's worst performance was with the
OSP2 case study. As shown in \fig{cases}, this project has the most amount
of fixed settings so SEESAW only has a small range of options it can explore.


\begin{figure}
\begin{center}\footnotesize
\begin{tabular}{r|rrr}
data set &          defects&	time	&effort\\\hline
flight	&			80&		39		&72		\\
ground	&			85&		38	&	73\\
osp		&			65&		4&		42\\
ops2	&			26&		22&		5\\\hline
median   &          73 &    30&    57
\end{tabular}
\end{center}
\caption{Percent reductions $(1-final/initial)$ achieved by SEESAW on the \fig{cases}
case studies. The $initial$ values come from round $0$ of the forward select.
The $finall$ values come from the policy point.
Note that 
all the initial and final values are statistically significant different
(Mann-Whitney, 95\% confidence).  }\label{fig:res}
\end{figure}


%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{l@{~~}r|r|@{~}r|r|@{~}r|r@{~}r}
%        
%		\multicolumn{2}{c}{~}     &\multicolumn{4}{c}{estimates}\\\cline{3-6}
%		\multicolumn{2}{c}{~}     &\multicolumn{2}{|c|}{A = as is}&\multicolumn{2}{c|}{ B = to be}&\multicolumn{2}{c}{percent change}\\\cline{3-6}
%        &     &mean&spread&mean&spread\\\cline{3-6} 
%        &     &A&$\delta$A&B&$\delta$B
%
%&
%
%$1-\frac{A}{B}$
%
%&
%
%$1-\frac{\delta{A}}{\delta{B}}$\\\hline
%defects&flight&1789&2080&355&824&80&70\\
%&ground&1610&2040&241&300&85&85\\
%&osp&3259&1882&1142&884&65&61\\
%&osp2&306&145&226&140&26&19\\\cline{7-7}
%&    &   &   &   &   &med = 73  &   \\\hline
%
%months&flight&31&8&19&7&39&34\\
%&ground&27&7&16&5&38&37\\
%&osp&32&6&31&5&4&4\\
%&osp2&19&4&15&2&22&28\\\cline{7-7}
%&    &   &   &   &   &med = 30  &   \\\hline
%
%effort&flight&701&678&193&275&72&66\\
%&ground&484&398&132&137&73&70\\
%&osp&1113&425&651&281&42&39\\
%&osp2&179&55&171&54&5&4\\\cline{7-7}
%&    &   &   &   &   &med = 57  &   \\\cline{3-6}
%\end{tabular}}
%\end{center}
%\caption{SEESAW estimates. ``As is'' and ``to be'' denotes estimates before/after applying SEESAW.
%Last two columns show percent reductions. ``Median'' denotes
%the median values and ``spread'' shows the (75-50)\% spread.
%Mann-Whitney tests (at 95\% confidence) report that the ``to be'' values are significantly
%different to the ``as is'' values.}\label{fig:reduce}
%\end{figure}
%
%A premise for our endorsement of SEESAW is that the predictions made by this tool are actually realized.
%In order to test that premise, we conducted the following {\em accuracy experiment} to compare the
%SEESAW's recommendations to those provided by other tools. 
%
%That experiment has three parts. Firstly, using Baker's prior work~\cite{baker07,me09e}, we noted
%the error ranges seen in standard effort estimation. 
%
%In that study, Baker selected 10 projects at random from a log of historical NASA data 
%sets~\footnote{\url{http://promisedata.org/?s=nasa93}}. Then,  30 times, an effort model
%was learned using Boehm's local calibration procedure (a standard technique used in the COCOMO
%community)~\cite{boehm81} using 90\% of the training data (taking care to ignore the test instance).
%The accuracy measure of the predictions from the learned models was expressed as the magnitude of the
%relative error
%$MRE=\frac{abs(predicted-actual)}{actual}$. 
%\begin{figure}
%\begin{center}
%\includegraphics[width=2.4in]{comparesReport_lc.pdf}
%\end{center}
%\caption{MRE results seen with standard effort estimation results. From Baker~\cite{baker07}.
%}\label{fig:mre2}
%\end{figure}
%
%Baker's experiments allows us to calibrate the errors expected from standard effort estimation methods.
%\fig{mre2} shows the results. The maximum MRE error is sometimes very  (300\% to 500\%) while
%the median MRE errors in the 10 case studies are: 
%\[\footnotesize
%median(MRE)= \{25,26,27,28,29,30,31,35,37,38\}\%
%\]
%
%SEESAW's errors are 
%
%  results from \eq{ab} we took historical we collected the errors generated by standard effort estimation methods.
%

%predictions made at the policy point found by SEESAW 
%
%{\em Issues with Mean:} The $experiment=1$ results of \fig{delta} shows the differences between the estimates
%generated {\em with} and {\em without} tuning on local data (the no-tuning estimates came from our search-based tool and the
%tuning-results can from
%on-line versions of the standard USC models\footnote{COCOMO II with Heuristic Risk Assessment: \\http://sunset.usc.edu/research/COCOMOII/expert\_cocomo/expert\_cocomo2000.html}).
%The $experiment=2$ results of that figure show
%the mean errors seen 
%when estimates are generated using conventional methods. Note that the mean differences in the $experiment=1$
%results are small (36\% or less).
%Further,
%the mean effort values of the conventional methods is larger (never less than 42\%).
%That is, \fig{delta} shows no evidence that the differences between our methods and conventional methods
%is inordinately large.
%
%{\em Issues with Variance:}
%Our tools sample across the space of possible tunings. The variance of the conclusions reached in that sample
%could be reduced by the use of local data to pre-tune our models, before we perform any sampling.
%However, it must be noted that the variances of our conclusions are quite small (see \fig{r1} and \fig{delta}).
%In summary, while we believe it may be useful to tune with local data, the overall benefit on reducing the variance or the mean  not be large.
%
%
%\begin{figure}
%\small
%\begin{tabular}{r|rrrrr}
%        &            & \multicolumn{4}{c}{data set}\\\cline{3-6}
%        & experiment & flight        & ground & OSP         & OSP2        \\
%\hline
%defects & 1          & 30 $\pm$ 23   & 28 $\pm$ 33 & 39 $\pm$ 33 & 32 $\pm$ 18 \\
% months & 1          & 26 $\pm$ 17   & 27 $\pm$ 16 & 23 $\pm$ 17 & 12 $\pm$ 7  \\
% effort & 1          & 36 $\pm$ 31   & 34 $\pm$ 31 & 20 $\pm$ 17 & 18 $\pm$ 13 \\
%\hline
% effort & 2          & 41 $\pm$ 40   &        & 44 $\pm$ 63 &  42 $\pm$ 57
%\end{tabular}
%     \caption{Results (Mean $\pm$ standard deviations) from our experiments.
%For an explanation of the method by which this was generated, see \fig{deletahow}.
%}\caption{fig:delta}
%\end{figure}
%\begin{figure}
%\small
%\begin{center}
%\begin{tabular}{|p{.95\linewidth}|}\hline
%The lines labeled $experiment=1$ in \fig{delta} were generated as follows:
%\be
%\item
%Generate 100 {\em projects} based on constraints $C$ by using our search-based tool and score them.
%\item
%Score {\em projects}
%against the standard on-line USC
%models \footnote{COCOMO II with Heuristic Risk Assessment: \\http://sunset.usc.edu/research/COCOMOII/expert\_cocomo/expert\_cocomo2000.html};
%\item
%Combine the {\em estimate} scores from search-based tool with the {\em baseline} scores from standard on-line USC models
%using $delta= \frac{estimate - baseline}{baseline}$. 
%\ee
%The line labeled $experiment=2$ in \fig{delta}
%was generated by taking flight systems/OSP/OSP2 systems
%and estimating them using a COCOMO model tuned to some NASA COCOMO, as follows:
%\bi
%\item
%Using the NASA93 COCOMO data set
%\footnote{\url{http://promisedata.org/?s=nasa93}}
%we tuned a COCOMO model using
%using Boehem's LC method~\cite{boehm81};
%\item
%Generate 100 {\em projects} based on constraints $C$ by using our search-based tool and score them.
%\item Effort estimates for the generated projects were generated using the tuned
%COCOMO model.  
%\item The means and standard deviations for those estimates were recorded in \fig{delta}.
%\ei\\\hline
%\end{tabular}
%\end{center}
%\caption{Generation method for the \fig{exp2} results.}\label{fig:deltahow}
%\end{figure}
%
\begin{figure}
\renewcommand{\baselinestretch}{0.92}
{\scriptsize
\begin{center}
\begin{tabular}{l|@{~}c@{~}|r@{~}|r@{~}|r@{~}|c|@{~}c}
    \multicolumn{2}{c}{~}           & \multicolumn{2}{c}{$value$}           &     &\multicolumn{2}{c}{defect}\\\cline{3-4}
Data   & Range           & F=FBC      &  R=risk   &  $\frac{F}{F + R}$    &\multicolumn{2}{c}{removal}\\\cline{6-7}
&&& & & manual & automatic\\\hline
ground&rely = 4 &  70 &  20 &  77                   &        &           \\
      &aa   = 6 &  70 &  25 &  73                   &        &  hi in F  \\
      &resl = 6 &  65 &  40 &  61                   &        &           \\
      &etat = 1 &  35 &  65 &  35                   &        &  lo in R \\
      &aexp = 5 &  45 &  85 &  34                   &        &           \\
      &pr   = 1 &  35 &  80 &  30                   &  lo in R    &           \\
      &aa   = 1 &  25 &  60 &  29                   &   &lo in R            \\
      &data = 2 &  25 &  70 &  26                   &        &           \\
     &rely  = 1 &  15 &  70 &  17                   &        &\\\hline
flight&rely = 5 &  65 &  25 &  72                   &        &           \\
      &flex = 6 &  80 &  50 &  61                   &        &           \\
      &docu = 1 &  55 &  85 &  39                   &        &           \\
      &site = 6 &  55 &  85 &  39                   &        &           \\
      &resl = 6 &  45 &  70 &  39                   &        &           \\
      &pr   = 1 &  45 &  70 &  39                   & lo in R&           \\
      &pvol = 2 &  45 &  75 &  37                   &        &           \\
      &data = 2 &  35 &  60 &  36                   &        &           \\
      &cplx = 3 &  45 &  90 &  33                   &        &           \\
      &rely = 3 &  15 &  60 &  20                   &        &\\
\hline
OSP   &pmat = 4 &  85 &  45 &  65                   &        &           \\
      &resl = 3 &  45 &  70 &  39                   &        &           \\
      &ruse = 2 &  40 &  65 &  38                   &        &           \\
      &docu = 2 &  25 &  90 &  21                   &        &\\
\hline
OSP2  &sced = 2 & 100 &   0 & 100                   &        &           \\
      &sced = 4 &   0 &  80 &   0                   &        &  \\    
\hline
\end{tabular}
\end{center}}
\caption{Frequency (in percents)
of feature ranges seen in 20 repeats of SEESAW, using two different goal functions: energy and risk.
The last two columns comment on any defect reduction feature. Note shown in this figure
are any feature ranges that occur less than 50\% of the time.
}\label{fig:percent}
\end{figure}
\subsection{Effects of Changing the Value Function}
Our prior ASE report on this work~\cite{me07d} used simulated annealing to find treatments.
That work reported that, in all studied cases, automated defect removal tools were
always found in the learned treatments.

The poor performance of SA in \fig{res} means we must revisit those conclusions, using SEESAW
rather than SA.
\fig{percent} shows the  ranges seem in  SEESAW's treatment (after a back select).
The ``FBC'' and ``risk'' columns show the percent frequency of a range appear when 
SEESAW used our different $value$ functions. 
These experiments were repeats 20 times and only the ranges found in the majority (more than 50\%) of the
trials
are reported.  

The results are divided into our four case studies: ground, flight, OSP, and OSP2.
Within each case study, the results are sorted by the fraction $\frac{FBC}{FBC + risk}$. This
fraction ranges 0 to 100 and:
\bi
\item If close to 100, then a range is selected by FBC more than risk.
\item If close to 0, then a range is selected by risk more than FBC.
\ei
The right-hand columns of \fig{percent} comment on the presence in the treatments of
 manual defect remove methods
(pr= peer reviews) or automatic defect removal methods (aa= automated analysis; etat=execution testing a tools).
In results that differs from our prior report,
\bi
\item
High levels of automatic defect removal methods are only frequently required in
ground systems, and
only when valuing FBC (i.e. when trying to reduce defects {\em and} time {\em and} effort.
\item
More often than note, when defect removal removals are found to be common, they are deprecated.
In ground systems, $etat=1 and pr=1 and aa=1$ are all examples of SEESAW {\em discouraging} rather
than endorsing the use defect removal methods.
\item In three of our four case studies, it is more important to stop defects being introduced than
to use after-the-fact defect removal methods. In ground, OSP, and OPS2, defect removal methods
are are very rare (only pr=1 in flight systems).
\ei
Another important aspect of \fig{percent} is 
that there is no example of \mbox{$F\ge 60 \wedge R \ge 60$} 
(the  only two near exceptions to this rule are \mbox{$F=55 \wedge R=85$} in flight
systems).
That is, 
perceptions of $value$ change everything. If a range is commonly selected by FBC, is usually {\em not}
commonly accepted by risk. 

\section{Related Work}
In our view, uncertainty in software process models is an under-explored area.
For example, the variance of \eq{ab} remained undiscovered for 26 years
until Baker, at our suggestion, looked for it. 

Much of the related work on uncertainty in software engineering uses a Bayesian analysis.
For example,
Pendharkar et.al.~\cite{pendharkar05} demonstrate the utility of Bayes networks in effort
estimation while Fenton and Neil explore Bayes nets and defect prediction~\cite{fenton99}
(but unlike this paper,
neither of these teams links defect models to effort models).
We elect to take a non-Bayesian approach since most of the industrial and government
contractors we work with use parametric models like
COCOMO. 

Other related work is the search-based SE approach advocated by Harmon~\cite{harman04}. 
Search-Based Software Engineering (SBSE) uses optimization techniques
from
operations research and meta-heuristic search (e.g. simulated annealing and genetic algorithms)
to hunt for near
optimal solutions to complex and over-constrained
software engineering problems. 
The SBSE approach can and has been applied to many
problems in software engineering (e.g. requirements engineering~\cite{jalali08})
 but most often in the field of software testing~\cite{andrews07}.
Harmon's writing inspired us try simulated annealing to search the what-ifs in untuned COCOMO
models~\cite{me07f}. 
SA is an widely-used  algorithm, perhaps
due to the simplicity of its implementation and its very low memory
requirements. However \fig{xper} strongly suggests that proponents of SA should try a broader
range of search engines.

As to other search algorithms,
the  six algorithms explored here does not cover the space of
all possible algorithms.
For example,
 Gu et al.~\cite{gu97} 
list hundreds of optimization algorithms and   no
single conference paper can experiment with them all.
However, we make one comment as to why the above
list does not include integer programming  methods. 
Coarfa et
al.~\cite{coarfa00} found that  integer programming-based approaches
ran two an order of magnitude slower than discrete methods like
MaxWalkSat and SEESAW.  
Similar results reported by Gu et.al  where discrete
methods ran 100 times faster than integer programming~\cite{gu97}.




\section{Conclusions}
How much QA is enough? Too narrow a question- how much of anything is enough. Gee, I dunno.
Clearly, different value functions effect the conclusions reached by this kind of analysis.
That is the point of this paper. Tools like SEESAW let analysts explicate their $value$ function
and explore its consequences. 

Is not enough to propose some automated defect reduction tool. 
The lesson of \fig{percent} is that  it is  {\em not} true that an automated tool is usually clearly useful
in all contexts.
Rather,
there needs to be more care taken is specifying the business context within which that tool will be useful.

Previously, it was difficult to specify and reason about the business context of a tool. A large
{\em cone of uncertainty} surrounds many software engineering project decisions. However, given careful
model selection and AI tools like SEESAW,
it is possible to find stable conclusions within the space of possibilities.
%\end{document}  % This is where a 'short' article might terminate

%ACKNOWLEDGMENTS are optional
%
% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{IEEEtran}
{\small
\bibliography{refs}
}  % sigproc.bib is the name of the Bibliography in this case
\appendix
This appendix lists the minimum and maximum $m$ values used
for \eq{lineem} and \eq{linesf}.
In the following $m_\alpha$ and $m_\beta$ denote COCOMO's linear and exponential
influences on effort/cost and 
$m_\gamma$ and $m_\delta$ denote COQUALMO's linear and exponential influences on number of
defects.


Their are two sets of  effort/cost multipliers:
\be
\item
The {\em positive effort EM} features, with slopes $m^+_\alpha$, that are proportional to effort/cost. 
These features are:
cplx,
data,
docu,
pvol,
rely,
ruse,
stor, and
time.
\item
The
{\em negative effort EM} features, with slopes $m^-_{\alpha}$,
are inversely proportional to effort/cost.
These features are 
acap,
apex,
ltex,
pcap,
pcon,
plex,
sced,
site, and
tool.
\ee
Their $m$ ranges, as  seen in 161 projects~\cite{boehm00a}, are:
\begin{equation}
\footnotesize
\left(0.073 \le m^+_\alpha \le 0.21\right) \wedge
\left(-0.178 \le m^-_\alpha \le -0.078\right)
\end{equation}
In the same sample of projects,
the COCOMO effort/cost scale factors (prec,flec, resl, team, pmat) has
the range: 
\begin{equation}\footnotesize
-1.56 \le m_\beta \le -1.014
\end{equation}
Similarly, there are two sets of defect multipliers and scale factors:
\be
\item The {\em positive defect} features have slopes $m^+_\gamma$ and are proportional
to estimated defects:
flex,
data,
ruse,
cplx,
time,
stor, and 
pvol.
\item The {\em negative defect} features, with slopes $m^-_\gamma$, that are inversely proportional
to the estimated defects. These features are 
acap,
pcap,
pcon,
apex,
plex,
ltex,
tool,
site,
sced,
 prec,
resl,
team,
pmat,
rely, and
docu.
\ee
COQUALMO divides into three models describing how defects change in requirements, design,
and coding.
These  tunings options have the range:
\begin{equation}\label{eq:coq}\small
\begin{array}{c}
requirements\left\{
\begin{array}{c}
0 \le m^+_\gamma \le 0.112\\
-0.183 \le m^-_\gamma \le -0.035
\end{array}
\right. \\
~\\
design \left\{
\begin{array}{c}
0 \le m^+_\gamma \le 0.14\\
-0.208 \le m^-_\gamma \le -0.048
\end{array}
\right. \\
~\\
coding \left\{
\begin{array}{c}
0 \le m^+_\gamma \le  0.14\\
-0.19 \le m^-_\gamma \le -0.053
\end{array}
\right.
\end{array}
\end{equation}
The tuning options for  the  defect removal features are:
\begin{equation}\label{eq:linedefrem}\small
\begin{array}{rl}
\forall x \in \{1..6\} &SF_i=m_\delta(x - 1)\\
requirements: 	& 0.08 	\le m_\delta \le 0.14 \\
design: 		& 0.1 	\le m_\delta \le 0.156 \\
coding: 		& 0.11 	\le m_\delta \le 0.176 \\
\end{array}
\end{equation}
where  $m_\delta$ denotes the effect of  $i$ on  defect removal.



% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%
%APPENDICES are optional
%\balancecolumns
%\balancecolumns
% That's all folks!
\end{document}
