% THIS IS SIGPROC-SP.TEX - VERSION 3.1
% WORKS WITH V3.2SP OF ACM_PROC_ARTICLE-SP.CLS
% APRIL 2009
%
% It is an example file showing how to use the 'acm_proc_article-sp.cls' V3.2SP
% LaTeX2e document class file for Conference Proceedings submissions.
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V3.2SP) *DOES NOT* produce:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) Page numbering
% ---------------------------------------------------------------------------------------------------------------
% It is an example which *does* use the .bib file (from which the .bbl file
% is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission,
% you need to 'insert'  your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% Questions regarding SIGS should be sent to
% Adrienne Griscti ---> griscti@acm.org
%
% Questions/suggestions regarding the guidelines, .tex and .cls files, etc. to
% Gerald Murray ---> murray@hq.acm.org
%
% For tracking purposes - this is V3.1SP - APRIL 2009

\documentclass[compsoc,conference]{IEEEtran}
\usepackage{bibspacing}
\usepackage{alltt,times,url,cite}

% Different font in captions
\newcommand{\captionfonts}{\sffamily\footnotesize}

\makeatletter  % Allow the use of @ in command names
\long\def\@makecaption#1#2{%
  \vskip\abovecaptionskip
  \sbox\@tempboxa{{\captionfonts #1: #2}}%
  \ifdim \wd\@tempboxa >\hsize
    {\captionfonts #1: #2\par}
  \else
    \hbox to\hsize{\hfil\box\@tempboxa\hfil}%
  \fi
  \vskip\belowcaptionskip}
\makeatother   % Cancel the effect of \makeatletter


\newcommand{\bd}{\begin{description}}
\newcommand{\ed}{\end{description}}
\newcommand{\bi}{\begin{smallitem}}
\newcommand{\ei}{\end{smallitem}}
\newcommand{\be}{\begin{smallenum}}
\newcommand{\ee}{\end{smallenum}}
\newcommand{\tion}[1]{\S\ref{tion:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newenvironment{smallitem}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{itemize}
   \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{itemize}}
\newenvironment{smallenum}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{enumerate}
  \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{enumerate}}

\ifCLASSINFOpdf
   \usepackage[pdftex]{graphicx}
  % declare the path(s) where your graphic files are
   \graphicspath{{../pdf/}{../jpeg/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
   \DeclareGraphicsExtensions{.pdf,.jpeg,.png}
\else
  % or other class option (dvipsone, dvipdf, if not using dvips). graphicx
  % will default to the driver specified in the system graphics.cfg if no
  % driver is specified.
  % \usepackage[dvips]{graphicx}
  % declare the path(s) where your graphic files are
  % \graphicspath{{../eps/}}
  % and their extensions so you won't have to specify these with
  % every instance of \includegraphics
  % \DeclareGraphicsExtensions{.eps}
\fi

\begin{document}
\title{Understanding the Value of Software Engineering Technologies}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than si% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

%\numberofauthors{1} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
Phillip Green II, Tim Menzies, 
Steven Williams,
Oussama El-Rawas
\\
Lane Department of CS\&EE, West Virginia University, USA,\\
\url{deathcheese@yahoo.com, tim@menzies.us, stevencwilliams@gmail.com, orawas@gmail.com}
}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
%\date{30 July 1999}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
SEESAW combines AI search tools, a Monte Carlo simulator,
and some software process models. 
We show here that, when
selecting technologies for a software project,
SEESAW out-performs a variety of other search
engines.  SEESAW's recommendations are greatly affected by the 
business context of its use.
For example, the automatic defect reduction
tools explored by the ASE community are only relevant to a subset of software projects, and only
according to certain value criteria.
Therefore, when arguing for the value of a particular technology,  
that argument should include a description of the value function of the target
user community.
\end{abstract}

\IEEEpeerreviewmaketitle



\section{Introduction}
There are many software engineering (SE) {\em technologies} that a manager
might apply in the hope of improving their software development project.
Some of these technologies are {\em paper-based} methods like
the checklists proposed by orthogonal defect classification~\cite{chill92}.
Other technologies are
{\em tool-based} such as  using the new generation of functional
programming languages\footnote{\protect\url{http://caml.inria.fr/}} or execution and testing
tools~\cite{andrews07} or automated formal analysis~\cite{holzmann97}.
 Yet other
technologies are more {\em process-based} including  process improvement
initiatives, changing an organization's hiring practices, or a continual
renegotiation of the requirements as part of an agile software
development cycle~\cite{port08}.


SE technologies can be inappropriately applied to projects if a  
project manager do not assess the techology's {\em benefits} against its
associated {\em drawbacks}.
For example, using less-skilled developers is tempting (since there are so many of them),
but the resulting product may be more defect-prone. As a result,  products might get to market
faster but contain too
many defects.

In theory, software process models can be used to model the trade-offs associated with 
different technologies.
However, 
such models can suffer from {\em tuning instability}.
Large instabilities make it hard to recognize important influences on a project.
For example, 
consider the following simplified COCOMO~\cite{boehm00b} model, 
\begin{equation}\footnotesize\label{eq:simp}
effort = a \cdot LOC^{b +  pmat}\cdot acap  
\end{equation}
Here, $(a,b)$ control the linear and exponential
effects (respectively) on model estimates; 
while
$pmat$ (process maturity) and $acap$ (analyst capability) are project choices
adjusted by managers. 
\eq{simp} contains two features ($acap,pmat$) and a full COCOMO-II model contains 22~\cite{boehm00b}.
Baker~\cite{baker07} reports a study that learned values of $(a,b)$ for a full COCOMO model
using Boehm's local calibration method~\cite{boehm81}
from 30 randomly selected samples of 90\% of the available project data. 
The ranges varied widely:
\begin{equation}\footnotesize\label{eq:ab}
\left( 2.2 \le a \le 9.18 \right) \wedge \left(0.88 \le b \le 1.09\right)
\end{equation} 
Such large variations make it possible to misunderstand the effects of project options.
Suppose some proposed technology  doubles productivity, but $a$ moves from 9 to 4.5.
The improvement resulting from that change would be obscured by the tuning variance.


Previously we have reported some success with stochastic AI tools that search
through the space of possible tunings within process 
models~\cite{me07g,me07f,me09a,me09e,me09f}.
This approach returned
conclusions about the project that were
stable across the entire
space of possibilities. For example, in two studies~\cite{me07f,me07g} we found that automated
defect removal tools such as those discussed at the ASE were often required to achieve
minimum defects and minimum development effort and minimum development time\footnote{{\em Time}
refers to calendar months required to complete a project while {\em effort} refers to the number
of staff hours within those months. For example, 4 people working for one year takes
$time=12$ months and $effort=48$ months.}. Elsewhere~\cite{me09a,me09e}, we have used this
tool to comparatively assess different proposed changes to a project. 

The above work has raised several issues:
\bi
\item What is the best tool to use for this kind of analysis?
Our prior work used two different methods
without assessing if one method is better or worse than the other. 
\item How do business conditions affect the outcome? Our prior analysis
used a generic $value$ function that modeled ``better, faster, cheaper'' (BFC); i.e.,
build software with fewer defects, in less time, with less effort.
Organizations that strive for BFC have produced spectacular
failures~\cite{me09f}. Other business goals may be required for other businesses
such as
``get to market faster, without overly compromising on quality''.
\ei
%In this paper, we show that the search tool used in our previous walk~\cite{me07f} performs much worse than
%a new variant of  local search.
%Accordingly, this paper repeats and extends our previous study using different $value$ functions.
%It will be shown that different $value$ functions lead to very different conclusions. 
%In a result that is 
% very different result to our prior work, we show that  is not always true that the automated defect removal tools
%of the kind explored at this conference are indicated for all projects.
%

To address these issues, 
this  paper proceeds as follows. 
After discussing related work, 
we propose one approach to avoid the tuning instability problem 
(AI search through the {\em project options} while performing Monte Carlo sampling
of the {\em tuning options}).  We next discuss how our approach (which does not require data
to tune models) is different from other approaches (that try to tune models to local data).
After that, some details of our models are explored including the space of project options and tuning
options inside a model. That section ends with a definition of the two $value$ functions used in this study. Various methods for searching for good project options are then reviewed. This is followed
by experiments confirming that technology we have used in prior papers (simulated annealing or SA) performs worse than
a new method called SEESAW.
SEESAW is then applied to descriptions of four NASA systems to produce reductions in defect and
effort estimates by 50 to 73\% (median values). Finally, when we change $value$ functions
inside SEESAW, it becomes very clear how different project goals lead to a selection of different
ways to organize a project. 

Note that past publications~\cite{me07g,me07f,me09a,me09e,me09f}  
contain parts of this paper; specifically,
 \S2 and  \S4
(but not
\S4.4.2).
The rest of
this paper is new work, including all the experiments
of \S6.
\subsection{Contributions of This Paper}

It should not require the technology of this paper to assert that
different developer $value$s lead to different technology choices.
Software construction is a very diverse process
and, clearly, one method is not best for all projects.

Having said that, the next question  is how to determine the  technology(ies)
that are best suited to a particular project.  Answering this
question is complicated by the myriad of uncertainties surrounding
a software project.  Tools like those discussed here are vital in
order to make, and defend, appropriate choices of technologies for
particular projects.


Our prior work in this area used either simulated
annealing~\cite{me07g,me07f,me09f,me09e} or a variant of MaxWalkSat
called SEESAW~\cite{me09a}. In one respect, those reports were
incomplete in that they did not test if those algorithms were better
or worse than any other.  Here, we explore and compare half a dozen
AI tools. When the best of these tools is applied to four case studies,
we will see that a project's $value$ function impacts the
the technologies selected for  different projects.  The size of this impact
is somewhat startling. Prior to this research, we have a pre-experimental intuition that concepts of 
$value$ might 
change the organization of  a project. However, we suspected that some things would remain constant
(e.g., condoning the use of execution testing tools). We were wrong.
For the two value functions explored here, if one function approves of $X$ then the other usually
approves of $not\; X$. 
This result that {\em value can change everything} should motivate much future work on the business context of
our tools.

\section{Related Work}
This work was inspired by Barry Boehm's 2004 ASE keynote~\cite{boehm04}
in which he advocated assessing
SE tools by their value to a stake-holder, rather than just via their functionality.
More specifically, one of the value functions used in this report
was developed by Boehm and Huang~\cite{huang06}.

In his description of value-based SE, Boehm favors
continuous optimization methods to  conduct cost/benefit
trade-off studies. Tuning instability can
confuse such continuous optimization methods.
For example,
the results of  gradient descent methods can be misleading if the coefficients on the partial
functions are very uncertain.

In our view, uncertainty in software process models is a very under-explored area.
For example, the variance of \eq{ab} remained undiscovered for 26 years
until Baker, at our suggestion, looked for it. 
Much of the related work on uncertainty in software engineering uses a Bayesian analysis.
For example,
Pendharkar et al.~\cite{pendharkar05} demonstrate the utility of Bayes networks in effort
estimation while Fenton and Neil explore Bayes nets and defect prediction~\cite{fenton99}
(but unlike this paper,
neither of these teams links defect models to effort models).
We elect to take a non-Bayesian approach since most of the industrial and government
contractors we work with use parametric models like
COCOMO. 

Other related work is the search-based SE approach advocated by Harmon~\cite{harman04}. 
Search-Based Software Engineering (SBSE) uses optimization techniques
from
operations research and meta-heuristic search (e.g., simulated annealing and genetic algorithms)
to hunt for near-optimal
solutions to complex and over-constrained
software engineering problems. 
Harmon takes care to distinguish AI search-based methods from those seen in
standard numeric optimizations.
Such optimizers usually offer settings to all controllables.
This may
result in needlessly complex recommendations since a repeated empirical
observation is that many model inputs are noisy or correlated in similar ways 
to model outputs~\cite{hall03}. Such noisy or correlated variables can be pruned away to
generate simpler solutions that are easier and quicker to understand.
In continuous domains, there
is much work on feature selection~\cite{miller02} and
techniques like principal component analysis~\cite{Dil84} to reduce
the number of dimensions reported by an analysis.  Comparative
studies report that discrete AI-based methods can do better at
reducing the size of the reported theory~\cite{hall03}.

The SBSE approach can and has been applied to many
problems in software engineering (e.g., requirements engineering~\cite{jalali08})
 but most often in the field of software testing~\cite{andrews07}.
Harmon's writing inspired us to try simulated annealing to search the what-ifs in untuned COCOMO
models~\cite{me07f}. 
SA is a widely-used  algorithm, perhaps
due to the simplicity of its implementation and its very low memory
requirements. Our results, shown below,  indicate that several other algorithms
out-perform SA (at least, on our models).
This result strongly suggests that proponents of SA should try a broader
range of search engines.

The paper compares SEESAW to five AI search algorithms.
These half a dozen algorithms hardly represent an exhaustive list of possibilities.
For example,
 Gu et al.~\cite{gu97} 
list hundreds of optimization algorithms and   no
single conference paper can experiment with them all.
However, we make one comment as to why the above
list does not include integer programming  methods. 
Coarfa et
al.~\cite{coarfa00} found that  integer programming-based approaches
ran an order of magnitude slower than discrete methods like
SEESAW.  
Similar results were reported by Gu et al.  where discrete
methods ran 100 times faster than integer programming~\cite{gu97}.

Our research grew out of a frustration with standard
methods to reduce tuning variance. Previously, we  have tried reducing that variance in various ways:
\bi
\item
Feature selection to prune spurious details~\cite{chen05}; 
\item
Instance selection to prune irrelevancies~\cite{me06d}; 
\item Extended data collection.
\ei
Despite all that work, the variance observed in our models remains very large.
Even 
the application of techniques such as instance-based learning have failed to reduce variance in our effort predictions~\cite{jalali07}.
Feature subset selection has also been disappointing: while it reduces
the median performance variance somewhat (in our experiments, from
150\% to 53\%~\cite{me06d}), the residual error rates are large enough that it is hard
to use the predictions of these models as evidence for the value of some proposed approach.
Lastly, further data collection has not proven useful.
Certainly, there is an increase
in
the availability
of
historical data on prior projects\footnote{see 
\url{http://promisedata.org.data/data}
and \url{http://www.isbsg.org/}.
}. However, Kitchenham~\cite{kitch07} cautions that the literature
is contradictory regarding the value of using
data from other companies to learn
local models.

Having failed to tame tuning variance, despite years of research, we turned to alternate methods.

\section{Dodging Tuning Variance }
%Avoiding Tuning Instability}
%Much of our prior work concerned methods to tune general software process models to local
%conditions. Such tuned models can be used to conduct what-if queries to learn best local
%organizations of a project. There are two drawbacks with such tunings:
%\bi
%\item They bias the conclusions of the model to historical events which may not be relevant to the current situation;
%\item As shown below, the results exhibit  large {\em tuning instabilities} and, hence, large variances in the model
%predictions. 
%\ei

In order to tame prediction variance, we need to understand its source.
The predictions of a model about a software engineering project are altered by 
project variables $P$ and tuning variables $T$:
\begin{equation}\label{eq:pred}\footnotesize
prediction = model(P, T)
\end{equation}
For example, in \eq{simp}, the tuning options $T$ are the range of $(a,b)$ and the 
project 
options  $P$ are the range of $pmat$ (process maturity) and $acap$ 
(analyst capability). 
Based on the definitions of the COCOMO model
we can say that
the ranges of the project variables are $P=1 \le (pmat,acap) \le 5$. Further, 
given the cone of uncertainty associated with a particular project $p$,
we can identify the subset of the project options $p\subseteq P$ relevant to a particular  project.
For example, a project manager may be unsure of the exact skill level of team members. However,
if she were to assert
``my analysts are better than most'', then $p$ would include $\{acap=4,acap=5\}$.

Our approach assumes that the dominant influences on the $prediction$ are the project options $p$ (and not the
tuning options $T$).
Under this assumption, 
the predictions can be controlled by
\bi
\item
Constraining $p$ (using some AI tool);
\item
While leaving  $T$ unconstrained (and sampling $t\in T$ using Monte Carlo methods).
\ei 
Specifically, we seek 
a treatment $r_x \subseteq p$ that maximizes the $value$ of a  model's predictions where
{\em value} is a domain-specific function that scores model outputs according to user goals:
\begin{equation}\footnotesize \label{eq:argmin}
\arg\max_x\left( \overbrace{r_x \subseteq p}^{AI\;search}, 
\underbrace{t \subseteq T, value(model(r_x,t))}_{Monte\;Carlo} \right)
\end{equation}

This approach is somewhat different from standard methods for decision support
for software process models. From~\cite{me08f}, we define standard practice as:
\be
\item
Collect domain knowledge.
\item
Build an initial model based on step 1 including as yet unknown parameters. 
Note that these unknowns represent a range of tuning options. 
\item
Tune model by (e.g.) regression on local data.
\item
Conduct sensitivity analysis on the tuned models.
\ee
These four steps can take quite some time and, at least in our experience with more complex
versions of \eq{simp}, may result in models with large tuning instabilities.
The sensitivity analysis, if conducted using gradient descent methods, will not be successful
for our models since the gradients exhibit the very large variances  of \eq{ab}.
Hence, another method that need not wait for time-consuming (and possibly
pointless) local data collection and tuning:
\be
\item Check the literature for software process models where the dominant influences
on predictions are the project options $P$, not the tuning options $T$.
\item Map project options to model input options.
\item Sample those models using AI tools to constrain the project options,
while
sampling the tuning options with Monte Carlo methods (i.e., \eq{argmin}).
\ee
One heuristic for checking the literature in step 1 is to avoid overly elaborate models whose
authors may have extended the model far beyond what can be supported with the available data.
We suspect a ``Goldilocks'' principle might be appropriate:
\bi
\item
Tiny models offer trite conclusions and are insensitive to important project features. 
\item
 Very large models may need much data collection to constrain the tunings. 
\item In between there may exist some models that are ``just right''; i.e., big enough to 
draw interesting conclusions, but small enough such that the internal tuning
variance does not dominate the variance results from input project options. 
\ei
We make no claim that {\em all} process models are ``just
right'' and, hence, can be controlled via \eq{argmin}.
Some process models can be quite complex and include:
discrete-event models~\cite{law00,kelton02};
system dynamics models~\cite{hamid91}; 
state-based models~\cite{akhavi93,harel90,martin00};
rule-based programs~\cite{mi90}; 
or  standard programming constructs
such as those used in Little-JIL~\cite{cass00,wise00}.
These rich modeling frameworks
allow the representation of 
detailed insights into an organization. However,  
the effort required
to tune them is non-trivial. 

In terms of the Goldilocks principle, we suspect
that many process models may not be  near the ``right size'' and will require
extensive tuning before they can be used for decision making. 
Fortunately, we have found that
the  USC  COCOMO and COQUALMO models~\cite{boehm00a}  are ``just right''.
In all our studies~\cite{me07g,me07f,me09a,me09e,me09f} we have found
that prediction variance can be controlled by only constraining 
project options while letting the tuning variance remaining unchecked.
Hence, we use these  models for our research.
One advantage of these models is that
they are 
fully described in the literature. 
The same can not be said for other commercial models such as PRICE TRUE PLANNING~\cite{park88}, 
SLIM~\cite{putnam92}, or SEER-SEM~\cite{jensen83}.
Also, at least
for the COCOMO effort model, there exist  baseline results~\cite{chulani99}. 
\begin{figure}
\begin{center}
{\scriptsize
\begin{tabular}{l|r@{:~}l|}\cline{2-3}
scale   &prec & have we done this before?\\
factors &flex & development flexibility \\
(exponentially        &resl & any risk resolution activities?\\
 decrease       &team &  team cohesion\\
 effort\&cost)       &pmat & process maturity\\\hline
upper  &acap & analyst capability\\
(linearly       &pcap & programmer capability\\
 decrease      &pcon & programmer continuity\\
effort\&cost)       &aexp &  analyst experience\\
       &pexp &  programmer experience\\
       &ltex &  language and tool experience\\
       &tool &  tool use\\
       &site &  multiple site development\\
       &sced & length of schedule   \\\hline
lower &rely &    required reliability  \\
(linearly      &data &   secondary memory  storage requirements\\
increase      &cplx &  program complexity\\
effort\&cost)      &ruse &  software reuse\\
      &docu &   documentation requirements\\
      &time &   runtime pressure\\
      &stor &   main memory requirements\\
     &pvol &    platform volatility  \\\cline{2-3}
\end{tabular}}
\end{center}
\caption{{
The COCOMO ``scale factors'' and ``effort multipliers'' change effort
and cost by an exponential and linear amount (respectively). Increasing these
values has the effect described in column one.}}\label{fig:emsf}
\end{figure}
\begin{figure}[!t]
\begin{center}
{\scriptsize
\begin{tabular}{|r@{:~}l|}\hline
aa & automated analysis\\
etat & execution-based testing and tools\\
pr & peer-reviews\\\hline
\end{tabular}}
\end{center}
\caption{{
The COQUALMO defect removal methods. Increasing these values
decreases delivered defects. }}\label{fig:defect}
\end{figure}
\section{Model Details}
COCOMO offers effort and time predictions while COQUALMO offers defect predictions.
Using the models we can represent the
project options $P$ and tuning options $T$ of \eq{pred} as follows.

\subsection{Project Options: $P$}
COCOMO and COQUALMO's features are shown in \fig{emsf} and \fig{defect}.
The features
have a range taken from
\{very low, low, nominal, high, very high, extremely high\} or
\[\{vl=1, l=2, n=3, h=4, vh=5, xh=6\}\]
These features include manual methods for defect removal. High values
for  peer reviews (or {\em pr}, see
\fig{defect}) denote formal peer group review activities
(participants have well defined and separate roles, the  reviews are guided
by
extensive 
review checklists/root cause 
analysis, and reviews are a continuous process guided by
statistical control theory~\cite{shull00b}).

COQUALMO also models automatic methods for defect removal.
Chulani~\cite{chulani99a} defines
the top half of 
{\em automated analysis} as:
\bi
\item[4] (high):
intermediate-level module and 
inter-module code syntax and 
semantic analysis. 
Simple requirements/design 
view consistency  checking. 
\item[5] (very high):
More elaborate 
requirements/design view 
consistency checking. 
Basic distributed-processing 
and temporal analysis, model 
checking, symbolic execution. 
\item[6] (extremely high):
Formalized\footnote{
Consistency-checkable pre- 
conditions and post-conditions, 
but not necessarily mathematical theorems.} 
specification and 
verification. 
Temporal 
analysis, model checking, 
symbolic execution. 
\ei
The top half of {\em execution-based testing and tools} is:
\bi
\item[4] (high):
Well-defined test sequence 
tailored to organization 
(acceptance / alpha / beta / 
flight / etc.) test. 
Basic test coverage tools, test 
support system. 
\item[5] (very high):
More advanced tools, test 
data preparation, basic test 
oracle support, distributed 
monitoring and analysis, 
assertion checking. 
Metrics-based test process 
management. 
\item[6] (extremely high):
Highly advanced tools:
oracles, distributed monitoring 
and analysis, assertion 
checking.
Integration of automated 
analysis and test tools. 
Model-based test process 
management. 
\ei
In the sequel, the following observation will become important:
\fig{emsf} is much longer than \fig{defect}. This reflects a modeling intuition of COCOMO/COQUALMO:
it is better to prevent the introduction of defects (using changes to \fig{emsf}) than to try and find them, once they
have been introduced (using \fig{defect}).
\subsection{Tuning Options: $T$}

For COCOMO effort multipliers (the features that that affect effort/cost in a linear manner), the
off-nominal ranges \{vl=1, l=2, h=4, vh=5, xh=6\} 
change the prediction by some ratio.
The nominal range \{n=3\}, however, corresponds to an effort multiplier of 1, causing
no change to the prediction.
Hence, these  ranges  can be modeled as straight lines $y=mx+b$
passing through the point $(x,y)$=$(3,1)$. Such a line has a y-intercept
of $b=1 - 3m$. Substituting this value of $b$ into $y=mx+b$ yields:
\begin{equation}\label{eq:lineem}\small
\begin{array}{l}
\forall x \in \{1..6\}\; EM_i=m_\alpha(x-3) + 1
\end{array}
\end{equation}
where  $m_\alpha$ is the effect of $\alpha$ on effort/cost.

We can also derive a general equation for the features that influence cost/effort in an exponential manner.
These features do not ``hinge'' around (3,1) but take the following form:
\begin{equation}\label{eq:linesf}\small
 \forall x \in \{1..6\}\; SF_i=m_\beta(x - 6) 
\end{equation}
where  $m_\beta$ is the effect of factor $i$ on effort/cost.

COQUALMO contains equations of the same syntactic form as \eq{lineem}  and \eq{linesf}, but with different
coefficients.
Using experience for 161 projects~\cite{boehm00b}, we can find the
maximum and minimum values ever assigned to
$m$ for COQUALMO and COCOMO. Hence, to explore
tuning variance (the
$t\in T$ term in \eq{argmin}),
all we need to do is select $m$ values at random from the min/max $m$ values
ever seen.
An appendix to this document lists those ranges.
 
\subsection{Case Studies: $p \subseteq P$}
\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.8}
{\scriptsize
\begin{center}
\begin{tabular}{c|lrr|lr}
       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{fixed settings}\\\hline
project $p_i$&feature&low&high&feature&setting\\\hline
&prec&1&2&data&3\\
$p_1$=OSP:&flex&2&5&pvol&2\\
Orbital&resl&1&3&rely&5\\
space&team&2&3&pcap&3\\
plane&pmat&1&4&plex&3\\
&stor&3&5&site&3\\
&ruse&2&4&&\\
&docu&2&4&&\\
&acap&2&3&&\\
&pcon&2&3&&\\
&apex&2&3&&\\
&ltex&2&4&&\\
&tool&2&3&&\\
&sced&1&3&&\\
&cplx&5&6\\
&KSLOC&75&125\\\hline
&prec&3&5&flex&3\\
$p_2$=OSP2 &pmat&4&5&resl&4\\
&docu&3&4&team&3\\
&ltex&2&5&time&3\\
&sced&2&4&stor&3\\
&KSLOC&75&125&data&4\\
&&&&pvol&3\\
&&&&ruse&4\\
&&&&rely&5\\
&&&&acap&4\\
&&&&pcap&3\\
&&&&pcon&3\\
&&&&apex&4\\
&&&&plex&4\\
&&&&tool&5\\
&&&&cplx&4\\
&&&&site&6\\\hline
&rely&3&5&tool&2\\
$p_3$=JPL &data&2&3&sced&3\\
flight&cplx&3&6&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&7&418&&\\\hline
&rely&1&4&tool&2\\
$p_4$=JPL &data&2&3&sced&3\\
ground&cplx&1&4&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&11&392&&\\
\end{tabular}
\end{center}}
\caption{Four case studies.
Numeric values 
$\{1,2,3,4,5,6\}$ map to {\em very low, low, nominal,
high, very high, extra high}.}\label{fig:cases}
\end{figure}
We use $p$ to denote the subset of the project options $p_i \subseteq P$ relevant to particular projects.
The four particular projects $p_1, p_2, p_3, p_4$ used as the case studies of this paper are shown in
in \fig{cases}:
\bi
\item OSP is the GNC (guidance, navigation, and control)
component of NASA's {\em Orbital Space Plane}; 
\item OSP2 is a later version of OSP;
\item Flight and ground systems reflect typical ranges seen at NASA's Jet Propulsion Laboratory. 
\ei
Some of the features in \fig{cases}
are known precisely (see all the features with single {\em fixed settings}).
But  many of the features in \fig{cases} do not
have precise settings
(see all the features that {\em range} from some {\em low} to {\em high}
value).  Sometimes the ranges
are very narrow (e.g., the process maturity of JPL ground software is between 2
and 3), and  sometimes the ranges are very broad.

\fig{cases} does not mention all the  features listed in \fig{emsf}
inputs.  For example, our defect predictor has inputs for
use of {\em automated analysis}, {\em peer
reviews}, and {\em execution-based testing tools}. For all inputs
not mentioned in \fig{cases}, ranges are picked at random from 
 (usually)  \{1, 2, 3, 4, 5\}.


%\subsection{Treatments: $r_x \subset p$}
%The task of our tools is to (a) find a {\em minimal}  subset of the project options $p$ that
%most improves the model's predictions; and (b) show that that improvement
%is {\em stable} across the range of tuning options described in the preceding section.
%
%{\em Stability:} Solutions that comment on all controllables
%may be 
%{\em brittle}; i.e.,
%seemingly minor changes result in radically different performance
%scores.
%Harman~\cite{harman07} comments that understanding the solution 
%{\em neighborhood} is an open and pressing issue:
%\begin{quote}
%``...{\em robustness} may be as important as solution functionality. For 
%example, it may be better to locate an area of the search 
%space that is rich in fit solutions, rather than identifying an 
%even better solution that is surrounded by a set of far less fit 
%solutions.''
%\end{quote}
%
%
%
%\subsubsection{Digression}
%
%
%
%Need some way to adjust the level of QA work to the particular project.
%
%XXXX prior work infinitely expensive
%
%In theory, Boehm's value-based software engineering paradigm could be, in theory,
%
%Is automated software engineering {\em mature}? 
%A {\em mature} engineering field understands the costs and benefits of applying technology. 
%In such a field, there exist clear
%guidelines for when to apply more,
%or less, technology.  For example, aircraft engineers may reject a new engine
%technology if its power/weight ratio
%is too low, or if the technology increases assembly time, or overly complicates maintenance.
%
%Recent keynote addresses at ASE have highlighted this issue.
%Speaking at ASE'04, Barry Boehm~\cite{boehm04} argued against technologies being 
%{\em value-neutral}; i.e. that in that different decision on technologies have
%zero  impacts on the business
%value of development software. Rather, we argued for {\em value-based software engineering}
%where corporations adapt their decisions according to their {\em value-added} (or otherwise)
%of the technology.
%
%Speaking at ASE'08, Cliff Jones~\cite{jone08} was more forceful.
%
%manner in which we assess our field
%currently assess new technologies. In arguing {\em value-based software engineering}~\cite{boehm04},
%
%Cliff Jones Formal Methods~\cite{jones08}
%
%
%Our intuitions are some projects benefit more from automated SE than others. Clearly,
%the
%kernel of the flight guidance system that controls the ascent stages of a manned orbital rocket
%during initial lift-off deserves the entire arsenal of automated SE  (e.g. model-checking
%of the original design~\cite{pecheur02},
%AI search methods for learning best control strategies~\cite{me08f}, 
%neural nets
%to monitor for fueling anomalies as the launch approaches~\cite{markosian05}).
%
%But our guidelines are not clear cut. There exist many gray cases. For example, one rule of thumb at NASA
%is that flight systems are more critical than ground systems. Lutz found that the majority of errors
%associated with deep space NASA missions were associated with ground storage of downloaded data.XXX
%One response to this cautionary tale is to require full automated analysis for all software (i.e. the view
%of ).
%
%footnote{Communicating across the solar system is an inherently complex task that consumes scare resources.
%Errors in ground storage
%``back up'' through the software and eventually lead  to errors.
%
%that there are projects that benefit more from 
%The \textit{proceedings} are the records of a conference.
%ACM seeks to give these conference by-products a uniform,
%high-quality appearance.  To do this, ACM has some rigid
%requirements for the format of the proceedings documents: there
%is a specified format (balanced  double columns), a specified
%set of fonts (Aerial or Helvetica and Times Roman) in
%certain specified sizes (for instance, 9 point for body copy),
%a specified live area (18 $\times$ 23.5 cm [7" $\times$ 9.25"]) centered on
%the page, specified size of margins (2.54cm [1"] top and
%bottom and 1.9cm [.75"] left and right; specified column width
%(8.45cm [3.33"]) and gutter size (.083cm [.33"]).
%
%The good news is, with only a handful of manual
%settings\footnote{Two of these, the {\texttt{\char'134 numberofauthors}}
%and {\texttt{\char'134 alignauthor}} commands, you have
%already used; another, {\texttt{\char'134 balancecolumns}}, will
%be used in your very last run of \LaTeX\ to ensure
%balanced column heights on the last page.}, the \LaTeX\ document
%class file handles all of this for you.
%
%The remainder of this document is concerned with showing, in
%the context of an ``actual'' document, the \LaTeX\ commands
%specifically available for denoting the structure of a
%proceedings paper, rather than with giving rigorous descriptions
%or explanations of such commands.
%
%contributions: all our prior work biased by the same evaluation function.
%here we show that the results are highly dependent on the eval function
%
%\subsection{Definitions}\label{sec:inc}
%To 
%recognize when ASE tools are useful,
%we must precisely define what is meant by ``ASE tools''.
%This section describes the definitions used in this paper (and is taken from 
%previous work~\cite{me07f}).
%This definition is based on the process models developed by Boehm et al. at the
%University of Southern California:
%\bi
%\item
%The COQUALMO defect predictor~\cite[p254-268]{boehm00b};
%\item
%The  COCOMO effort/time predictor~\cite[p29-57]{boehm00b}.
%\ei
%(``Effort'' differs from ``time'' as follows. A project that takes 2 people 3 months
%to complete has an ``effort'' of $2*3=6$ months and a ``time'' of 3 calendar months.)
%
%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{l|r@{:~}l|}\cline{2-3}
%scale   &prec & have we done this before?\\
%factors &flex & development flexibility \\
%(exponentially        &resl & any risk resolution activities?\\
% decrease       &team &  team cohesion\\
% effort\&cost)       &pmat & process maturity\\\hline
%upper  &acap & analyst capability\\
%(linearly       &pcap & programmer capability\\
% decrease      &pcon & programmer continuity\\
%effort\&cost)       &aexp &  analyst experience\\
%       &pexp &  programmer experience\\
%       &ltex &  language and tool experience\\
%       &tool &  tool use\\
%       &site &  multiple site development\\
%       &sced & length of schedule   \\\hline
%lower &rely &    required reliability  \\
%(linearly      &data &   secondary memory  storage requirements\\
%increase      &cplx &  program complexity\\
%effort)\&cost      &ruse &  software reuse\\
%      &docu &   documentation requirements\\
%      &time &   runtime pressure\\
%      &stor &   main memory requirements\\
%     &pvol &    platform volatility  \\\cline{2-3}
%\end{tabular}}
%\end{center}
%\caption{{
%The COCOMO ``scale factors'' and ``effort multipliers'' change effort
%and cost by an exponential and linear amount (respectively). Increasing these
%values has the effect described in column one.}}\label{fig:emsf}
%\end{figure}
%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{|r@{:~}l|}\hline
%aa & automated analysis\\
%etat & execution-based testing and tools\\
%pr & peer-reviews\\\hline
%\end{tabular}}
%\end{center}
%\caption{{
%The COQUALMO defect removal methods. Increasing these values
%decreases delivered defects. }}\label{fig:defect}
%\end{figure}
%These models use the project features shown in \fig{emsf} and \fig{defect}.
%All these features
%have the range
%\{very low, low, nominal, high, very high, extremely high\} or
%\[\{vl=1, l=2, n=3, h=4, vh=5, xh=6\}\]
%Note that these features include sociological impacts on development effort/cost and defects
%such as:
%\bd
%\item[{\em team}:] sociology
%within a development team; e.g. its cohesiveness. 
%\item[{\em pcon}:]
%the staff turnover rate within a project.
%\item[{\em acap, pcap}:] analyst and programmer
%capabilities (respectively).
%\ed
%These feature also include manual methods for defect removal. High values
%for  peer reviews (or {\em pr}, see
%\fig{defect}) are achieved by denote formal peer group review activities
%(participants have well defined and separate roles, the  reviews are guided
%by
%extensive 
%review checklists/ root cause 
%analysis; review are a continuous process guided by
%statistical control theory~\cite{shull00b}).
%
%As to technological features, ``ASE tools''
%appear as
%{\em execution-based testing tools} and
%{\em automated analysis}.
%Chulani~\cite{chulani99a} defines
%the top half of 
%{\em automated analysis} as:
%\bi
%\item[4] (high):
%intermediate-level module and 
%inter-module code syntax and 
%semantic analysis. 
%Simple requirements/design 
%view consistency  checking. 
%\item[5] (very high):
%More elaborate 
%requirements/design view 
%consistency checking. 
%Basic distributed-processing 
%and temporal analysis, model 
%checking, symbolic execution. 
%\item[6] (extremely high):
%Formalized\footnote{
%Consistency-checkable pre- 
%conditions and post-conditions, 
%but not necessarily mathematical theorems.} 
%specification and 
%verification. 
%Advanced distributed 
%processing and temporal 
%analysis, model checking, 
%symbolic execution. 
%\ei
%The upper half of {\em execution-based testing and tools} is:
%\bi
%\item[4] (high):
%Well-defined test sequence 
%tailored to organization 
%(acceptance / alpha / beta / 
%flight / etc.) test. 
%Basic test coverage tools, test 
%support system. 
%\item[5] (very high):
%More advanced test tools, test 
%data preparation, basic test 
%oracle support, distributed 
%monitoring and analysis, 
%assertion checking. 
%Metrics-based test process 
%management. 
%\item[6] (extremely high):
%Highly advanced tools for test 
%oracles, distributed monitoring 
%and analysis, assertion 
%checking 
%Integration of automated 
%analysis and test tools. 
%Model-based test process 
%management. 
%\ei
%A  review of recent proceedings of the IEEE ASE conferences suggests
%that 
%a range of five or six in the above features includes
%the kind of
%``ASE tools'' explored at this venue.
%Hence, to recognize use of   ``ASE tools''  we will check our results for
%\[
%\begin{array}{c}
%\mbox{{\em automated analysis}}\in \{5,6\}\;\vee \\
%\mbox{{\em execution-based testing and tools}} \in \{5,6\}
%\end{array}
%\]
%
%
%
%\section{Related Work}
%\subsection{Value-Based SE}
%Boehm~\cite{boehm06} divides value-based SE into
%the following areas. Note that this paper explores the  last area  (planning and control):
%\bi
%\item
%{\em Value-based requirements engineering:} Rather than treating all
%requirements as  equally important, prioritize them by, say, modeling stake holder goals;
%\item {\em Value-based architecting:} 
%further reconciliation of the system objectives 
%with achievable architectural solutions.
%\item {\em
%Value-based verification and validation:}
%techniques for verifying and 
%validating that a software solution satisfies its value objectives; and processes for 
%sequencing and prioritizing V\&V tasks operating as an investing activity;
%\item {\em
%Value-based risk management: } principles and practices for risk 
%identification, analysis, prioritization, and mitigation;
%\item {\em
%Value-based people management:}
%includes stake holder team building and 
%expectations management; managing the project's accommodation of all stake holders¿ 
%value propositions throughout the life cycle.
%\item {\em
%Value-based design and development:} 
%ensure that the 
%system's objectives and value considerations are inherited by the software's design 
%and development practices; e.g. using requirements-to-design traceability analysis~\cite{hayes06}
%\item
%{\em 
%Value-based quality management:}
% e.g. using soft-goal to debate 
%trade-offs between user goals
%designs~\cite{mylo99,chiang03,me02c,me03h}.
%\item {\em
%Value-based planning and control:}
%principles and practices for extending 
%traditional cost, schedule, and product planning and control techniques to include 
%planning and control of the value delivered to stake holders. For example, 
%in this article and elsewhere~\cite{me07g,me08c,me09a,me09e,orrego09,me09f} 
%we use
%AI search tools to find subsets of the available process options to minimize
%development time, effort, and defects.
%\ei
%Boehm also lists
%numerous {\em elements} as a starting 
%points for realizing value-based software engineering. These elements are listed below,
%along with possible methods to achieve each element:
%\be
%\item {\em Benefits realization analysis:} 
%map the  possible outcomes of planned software initiative to
%stake holders' desired outcome(s); document all success-critical assumptions
%and additional work required to realize the desired outcome.
%\item {\em Stake holder value proposition elicitation and reconciliation:} 
%Using the results of element \#1,
%conduct interviews with 
%stakeholders to find additional high-priority assumption. Seek clashes and reconciliations between
%those assumptions.
%\item {\em Business case analysis:}  
%conduct  what-if analysis to compare expected
%costs/revenues before/after the initiative is implemented. 
%Such what-if queries can be implemented using the
%USC defect and cost/effort predictors~\cite{boehm00b}.
%\item {\em Continuous risk and opportunity management:}  
%Repeat the above at regular intervals during a project. For each repeat, finds ways to
%reduce risks as early as possible in the lifecycle. 
%Much of our previous work has explored automatic methods to find such risk-reduction
%methods that reduce predictions of effect/effort/cost~\cite{me02f,me00e,me07f,me06b}.
%\item {\em Value-based monitoring and control:} 
%Monitor the environment for changes the critical assumptions found above.
%\item {\em Change as Opportunity:} 
%Continuously monitor 
%technology and the marketplace to identify and reorient the project to address 
%unanticipated risks and opportunities. Allow projects to use hybrid
%plan-based methods and agile methods when some parts of the system warrants
%a slower/faster pace of change.    For more on such hybrid strategies, see~\cite{boehmturner03,port08}.
%\ee
%Some of the above elements are inherently manual tasks that require extensive interaction with
%business stake holders.
%For example, it would be  inappropriate to automate the first two
%elements since
%their  is to uncover the {\em value proposition} of
%the humans
%involved in the developments. Put simply, the value proposition  
%of a software initiative what the customer gets for their money {\em from the perspective
%of that customer}.
%
%If we can extract a {\em goal function} from the value proposition, then
%it is possible to offer extensive automatic support for the rest of these elements.
%A goal function combines multiple factors from an organization to return a single value. 
%AI search functions can use this goal function to control a search for better opportunities.
%Such an AI search can explore inputs to
%the predictor models used in the business case analysis (element \#3).
%The output of the predictor models can then be scored by the goal function. 
%
%Elements 4,6,7 can all be implemented using AI search that use goal functions and
% business models:
%\bi
%\item If the AI search tools are run on a regular (e.g. daily) schedule, then they
%be used to find risk reduction of new 
%business opportunities for the continuous risk and opportunity management (element \#4);
%\item  
%If the AI search engine finds conditions under which the software initiative is most, or least,
%successful, then these most-critical situations could guide the monitoring process of 
%monitoring and control (element \#5); 
%\item
%If our monitors raise an alert that the business context has changed then we can re-run
%our AI search engines to find better opportunities for the business (element \#6).
%\ei
%
%\subsection{Optimization over Continuous Domains}
%
%This paper uses heuristic AI search that assumes any continuous numeric variable
%will be divided into independent discrete ranges. 
%In this paper, and elsewhere~\cite{me02f,me00e,me07f,me06b}, we show that 
% such methods can be insightful.
%
%In theory, the AI search functionality discussed above might also be achieved by standard
%numeric optimization methods. For example, given a business model containing smooth
%continuous functions, gradient descent methods can 
%find global minimum values of (say) defects/ effort/ cost.
%
%We do not use gradient descent methods for three reasons:  {\em minimality};
%{\em neighborhood reasoning}; 
%and {\em gradient instability}. 
%
% to reduce a large space of variables to a smaller set.
%
% synthesis of smaller sets
%t
%while there exist some numeric methods
%f
%
%Huang.
%
%
%
% AI search engines 
%Much of the above is labor intensive (e.g. \#6, \#7) and automatic support 
%
%In practice, {\em gradient instability} precludes the use of gradient descent methods.
%Gradient descent requires some knowledge of the shape of the function being explored.
%For our business models, that shape is not known with certainty.
%Elsewhere~\cite{me05a}, we have studied  the variance on the coefficients on our prediction models.
%Our models contain linear and exponential influences on predicted values. 
%After converted these models into a linear model (by applying logarithms)
%these models take the form
%\[ \beta_0 + \beta_1x_1 + \beta_2x_2 + ...\]
%A least-squared
%regression (LSR) over such linear models
%searches for $\beta$ coefficients that minimizes the error between predictions and
%actual values seen in some training set.  
%Using historical data on project development effort,
%we applied LSR 30 times, on 66\% of the available training data (selected at random).
%For each repeat, we performed
%{\em parameter pruning} to remove superfluous $\beta_ix_i$ values\footnote{
%Step through all the $x_i$ values removing
%the one with the smallest standardized coefficient until
%no improvement is observed in the estimate of the model error
%given by the Akaike information criterion. For more details, see~\cite{quinlan92a}.}
%\begin{figure}
%\begin{center}
%\includegraphics[width=2.5in]{var.pdf}
%\end{center}
%\caption{Mean and standard deviation of $\beta_i$ coefficients in 30 repeats
%of LSR. This plot only shows the $\frac{10}{16}$ coefficients that survived parameter pruning
%in the majority of repeats; i.e. more than $\frac{15}{30}$ times.
%From~\cite{me05a}.}\label{fig:unstable}
%\end{figure}
%We say that {\em relevant} features are all  $x_i$ that survive
%pruning in the majority of cases (more than $frac{15}{30}$ times).
%
%The mean and standard deviation of the relevant $\beta_i$ coefficients are shown in 
%\fig{unstable}. Note that 
%\fig{unstable} 
%
%Experienced regression modelers such as Boehm propose reducing the tuning problem to that of
%finding just two coefficients. In the COCOMO-I model, the $(a,b)$ tuning variables allow
%for the adjustment of the $(linear,exponential)$ (respectively)  influences on
%effort. Boehm's {\em local calibration} (LC) procedure is a special case of LSR that just returns new coefficients for $(a,b)$.
%Baker~\cite{baker07}
%studied the range of $(a,b)$ learned from thirty 90\% random samples of some NASA data
%The ranges varied widely:
%$\left( 2.2 \le a \le 9.18 \right) \wedge \left(0.88 \le b \le 1.09\right)$ 
%Such large variations make it possible to misunderstand the effects of different software initiatives.
%For example, suppose some initiative productivity but $a$ moves from 9 to 4.5.
%The improvement resulting from that technology would be obscured by the tuning variance of $a$.
%
%We have tried, and failed, to tame this instability without process models using a variety of techniques
%including instance selection (to remove outliers)~\cite{jalali07}, feature subset selection (to remove
%superfluous features)~\cite{chen05}, or more data collection. Our results to date are not compelling. 
%Even after
%applying instance-based learning, the variance in our predictions remains very large~\cite{jalali07}.
%Feature subset selection has also been disappointing: while it reduces
%the median performance variance somewhat (in our experiments, from
%150\% to 53\%~\cite{me06d}) the residual error ranges are  still large enough to confuse a discussion about the relative merits of
%different SE technologies.
%Lastly, further data collection has not proven useful.
%Certainly, there is an increase
%in
%the availability
%of
%historical data on prior projects\footnote{e.g. see \url{http://www.isbsg.org/} and
%\url{http://promisedata.org.data/data}}. However, it is not
%clear if it is advisable to use data from other companies data to learn local effort models~\cite{kitch07}.
%As to collecting more local data, this has proven remarkably difficult. 
%Even after 26 years of trying, Boehm only collected less
%than 200 sample projects for the COCOMO database from his group of industrial affiliates~\cite{me09e}.
%Our experiments with that extended data set reveal similar instabilities to those reported in \fig{unstable}~\cite{me06d}.
%
%XXX and it just doesn't matter (safe when paper)
%
%uncertainty (bayes etc)
%
%AI search, various theorem proving
%
%manual value-based methods (boehm and turner)
%
%\begin{figure}
%\renewcommand{\baselinestretch}{0.6}
%{\scriptsize
%\begin{center}
%\begin{tabular}{c|lrr|lr}
%       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{values}\\\hline
%project&feature&low&high&feature&setting\\\hline
%&prec&1&2&data&3\\
%OSP:&flex&2&5&pvol&2\\
%Orbital&resl&1&3&rely&5\\
%space&team&2&3&pcap&3\\
%plane&pmat&1&4&plex&3\\
%&stor&3&5&site&3\\
%&ruse&2&4&&\\
%&docu&2&4&&\\
%&acap&2&3&&\\
%&pcon&2&3&&\\
%&apex&2&3&&\\
%&ltex&2&4&&\\
%&tool&2&3&&\\
%&sced&1&3&&\\
%&cplx&5&6\\
%&KSLOC&75&125\\\hline
%&prec&3&5&flex&3\\
%OSP2 &pmat&4&5&resl&4\\
%&docu&3&4&team&3\\
%&ltex&2&5&time&3\\
%&sced&2&4&stor&3\\
%&KSLOC&75&125&data&4\\
%&&&&pvol&3\\
%&&&&ruse&4\\
%&&&&rely&5\\
%&&&&acap&4\\
%&&&&pcap&3\\
%&&&&pcon&3\\
%&&&&apex&4\\
%&&&&plex&4\\
%&&&&tool&5\\
%&&&&cplx&4\\
%&&&&site&6\\\hline
%&rely&3&5&tool&2\\
%JPL &data&2&3&sced&3\\
%flight&cplx&3&6&&\\
%software&time&3&4&&\\
%&stor&3&4&&\\
%&acap&3&5&&\\
%&apex&2&5&&\\
%&pcap&3&5&&\\
%&plex&1&4&&\\
%&ltex&1&4&&\\
%&pmat&2&3&&\\
%&KSLOC&7&418&&\\\hline
%&rely&1&4&tool&2\\
%JPL &data&2&3&sced&3\\
%ground&cplx&1&4&&\\
%software&time&3&4&&\\
%&stor&3&4&&\\
%&acap&3&5&&\\
%&apex&2&5&&\\
%&pcap&3&5&&\\
%&plex&1&4&&\\
%&ltex&1&4&&\\
%&pmat&2&3&&\\
%&KSLOC&11&392&&\\
%\end{tabular}
%\end{center}}
%\caption{Four case studies.
%Numeric values 
%$\{1,2,3,4,5,6\}$ map to $\{very low, low, nominal,
%high, very high, extra high\}$.}\label{fig:cases}
%\end{figure}
%\section{Options Within Projects}
%SEESAW ranks project changes using the COCOMO and COQUALMO effort, time,
%and defect predictors. The terminology of those models is reviewed
%in \fig{cparems}.
%
%Using that terminology, \fig{cases} summarizes four NASA case studies:
%\bi
%\item OSP is the GNC (guidance, navigation, and control)
%component of NASA's 1990s {\em Orbital Space Plane}; 
%\item OSP2 is a later version of OSP;
%\item Flight and ground systems reflect typical ranges seen at NASA's Jet Propulsion Laboratory. 
%\ei
%Some of the features in \fig{cases}
%are known precisely (see all the features with single {\em values}).
%But  many of the features in \fig{cases} do not
%have precise values
%(see all the features that {\em range} from some {\em low} to {\em high}
%value).  Sometimes the ranges
%are very narrow (e.g., the process maturity of JPL ground software is between 2
%and 3), and  sometimes the ranges are very broad.
%\fig{cases} does not mention all the  features listed in \fig{cparems}
%inputs.  For example, our defect predictor has inputs for
%use of {\em automated analysis}, {\em peer
%reviews}, and {\em execution-based testing tools}. For all inputs
%not mentioned in \fig{cases}, values are picked at random from the full
%range of \fig{cparems}.
%

\subsection{Value}
If there exist multiple possible treatments,
a $value$ function is required to rank alternatives. 
The $value$ function should model the goals of the business users who are making project
decisions about some software development.
At the end of this paper, we compare results from two very different $value$ functions.
\subsubsection{``BFC'' = Better, Faster, Cheaper}
Ideally, software is built with
fewer defects $D$,
using less effort $E$, and in shorter time $T$.
A value function for this goal can be modeled as the Euclidean distance
to minimum effort, time, defects:

{\footnotesize \begin{eqnarray}\footnotesize\label{eq:bfc}
bfc& =& \sqrt{f\bar{T}^2 + c\bar{E}^2 + \left(b\bar{D}\left(1 + 1.8^{rely -3}\right)\right)^2}\\
value_{bfc}& =& \frac{1}{bfc}
\end{eqnarray}}
In the above, $value$ is highest when defects and effort and development
time are lowest. Also, 
$0 \le (b,f,c) \le 1$
represents the business importance of (better, faster, cheaper). 
For this study, we use $b=f=c=1$. In other work, we have explored the effects
of using other $b,f,c$ values~\cite{me09f}. 

In \eq{bfc}, $\bar{T},\bar{E},\bar{D}$
are the time, effort, and defect scores normalized zero to one.
\eq{bfc} models the business intuition that
defects in 
high reliability systems 
and exponentially more troublesome than in low reliability systems:
\bi
\item 
If reliability moves from very low to very hight (1 to 6), the term
$1.8^{rely-3}$ models a function
that (a) is ten times larger for very high than very low reliability systems;
and (b) passes through 1 at $rely=3$ (so systems with nominal 
reliability do not change the importance of defects).
\ei
\subsubsection{``XPOS'' = Risk Exposure}
The BFC value function is somewhat idealistic in that it seeks to remove {\em all} defects
by spending {\em less} money on {\em faster} developments.
 An alternate value function comes from Huang and Boehm~\cite{huang06}.
This alternate value function, which we call ``XPOS'', 
models the situation where a software company must rush  a product
to market, without compromising too much on software quality. 
Based on Huang's Ph.D. dissertation~\cite{huang06a},   we operationalize XPOS as follows.

Huang defines business
risk exposure ($RE$) as a combination of  {\em software quality investment risk exposure} ($RE_q$) and 
{\em market share erosion risk exposure} ($RE_m$).  We invert that expression to yield $value_{XPOS}$ (so an
exposed  project has low value):

{\footnotesize
\begin{eqnarray}
RE& =&RE_q + RE_m\\
value_{XPOS}&=&\frac{1}{RE}
\end{eqnarray}}
$RE_q$ values high-quality software and therefore prioritizes quality over time. 
$RE_q$ is composed of two primary components: probability of loss due to unacceptable quality $P_q(L)$
and size of loss due to unacceptable quality $S_q(L)$.  
$P_q(L)$ is calculated based on defects.  $S_q(L)$ is calculated based on complexity (the COCOMO $cplx$ feature), reliability ($rely$), and a cost function.  
$S_c$ is a value from a Pareto-valued table based on $rely$.  We choose the project months estimate as the basis of  this cost function.  
\begin{equation}\footnotesize
RE_q = P_q(L) * S_q(L)
\end{equation}
\begin{equation}\label{eq:dvl}\footnotesize
P_q(L) = \frac{defects}{defects_{vl}}
\end{equation}
\begin{equation}\label{eq:xxx}\footnotesize
S_q(L) = 3^{\frac{cplx-3}{2}} \cdot PM \cdot S_c
\end{equation}

In \eq{dvl},  $defects_{vl}$ is the lower bound on defects for that project.

In \eq{xxx} the $\frac{cplx-3}{2}$ term is similar to the $\bar{D}$ coefficient inside \eq{bfc}:
if complexity changes below or above 3, then it reduces or adds (respectively) to the unacceptable quality risk.
However at $cplx=3$, the multiplier is one (i.e., no effect).

$RE_m$ values a fast time-to-market and therefore prioritizes time over quality.
$RE_m$ is calculated from PM and reliability ($rely$).
$M_c$ is a value from a exponential-valued table based on $rely$.
\begin{equation}\footnotesize
RE_m = PM \cdot M_c
\end{equation}

\section{Searching for $r_x$}
Our search
runs two phases: a {\em forward select} and a {\em back select} phase.
The  {\em forward select} grows $r_x$, starting
with the empty set. At each round $i$ in the forward select
one or more
ranges (e.g., $acap=3$) are added to $r_x$.    
The resulting $r_x$ set found at  round $i$ is denoted $r_x^i$.

The forward select ends when the search engine cannot find more ranges to usefully add to $r_x^i$.
Before termination,
we say that the {\em open} features at round $i$ 
are the features in \fig{emsf} and \fig{defect} not mentioned by any range
in $r_x^i$.
The  value of $r_x^i$ is assessed by running the model $N$ times with:
\be
\item  all of $r_x^i$
\item any $t\in T$, selected
at random
\item any range at random for  $open$ features.
\ee
In order to ensure minimality, a  {\em back select} checks if the  final
$r_x$ set can be pruned. 
If the forward select caches the simulation results seen at each round $i$, the back
select can perform statistical tests to see if the results of round $i-1$ are significantly
different from round $i$. If the difference is {\em not} statistically significant, then the ranges added at round $i$ are {\em pruned} away and
the back select recurses for $i-1$.
We call the un-pruned ranges the {\em selected} ranges and
the point where pruning stops the {\em policy point}.

For example, in \fig{run}, the policy point is round 13 and the decisions made at subsequent rounds
are pruned by the back select. That is,
the  treatments returned by our search engines are all the ranges $r_x^i$ for  $1 \le i \le 13$.
The {\em selected} ranges are shown in a table at the bottom of the figure and 
the effects of applying the conjunction of ranges in $r_x^{13}$ can be seen
by comparing the values at round=0 to round=13:
\bi
\item Defects/KLOC reduced:  350 to 75; 
\item Time reduced:  16 to 10 months;
\item Effort reduced:  170 to 80 staff months.
\ei
%Note that, in accordance with Harmon's instructions (quoted above), our results come with an indicator about
%their robustness in the face of change.
%Business managers may not want to implement all constraints offered by the selected ranges.
%In this case, our results allow them to quickly conduct some what-if queries on using on some
%portion
%of our recommendations. For example, a manager can see that they can halve
%their defect rate just by applying  only the first eight recommendations found by our search tools (see
%the defect results at round=8 in \fig{run}).
%
%
\begin{figure}[!t]
\footnotesize
\begin{center}
\includegraphics[width=2.5in]{single-seesaw-tpts100-all-trio-2-1.pdf}

\begin{tabular}{|p{.46\linewidth}|p{.46\linewidth}|}
\multicolumn{2}{l}{Decisions made from round=1 to round=13: }\\\hline
round0: $r_x = \emptyset$\newline
round1 added \{pmat=3\}\newline
round2: added  \{resl=4\} \newline
round3: added   \{team=5\}\newline
round4: added \{aexp=4\} \newline
round5: added  \{docu=3\}\newline
round6: added  \{plex=4\}
&
round7: added \{rely=3\} \newline
round8: added  \{stor = 3\}\newline 
round9: added \{time = 3\}\newline
round10: added  \{tool = 4\}\newline
round11: added \{sced = 2\}\newline
round12: added \{site = 4\}\newline
round13: added  \{acap = 5\}
\\\hline
\end{tabular}
\end{center}
\caption{Example forward and back select results.}
\label{fig:run}
\end{figure}


\subsection{Alternate Search Methods}
This paper implements the forward select using
SA, 
MaxWalkSat,  
SEESAW, 
BEAM, A-STAR and 
ISAMP. 

{\em SA:} 
In our initial experiments~\cite{me07f}, we ran forward/back selects as a 
post-processor to  a simulated annealer.
Consider a typical SA run that has explored 10,000 variants on some solution.
A side-effect of that run is 10,000 sets of inputs, each scored with a value
function. Our tool ``STAR'' classified the outputs into 10\% {\em best} values and 90\% 
{\em rest}. All the
ranges from all the features
were then sorted into a list of length $R$
according to how much more frequently they
appeared in {\em best} than {\em rest}. The forward select was then
called using the first $i$ items $1 \le i \le R$.

{\em MaxWalkSat}  
is a local search algorithm~\cite{selman96}.
Given a random selected treatment,   
MaxWalkSat attempts $n$ modifications to randomly 
selected features.
Sometimes   (controlled by the $\alpha$ parameter), 
the modification is ``smart'':
for the selected feature, the algorithm chooses the range
that minimizes the value of the current solution.
The rest of the time (i.e., at probability
$1-\alpha$), a random range is chosen for the feature.
Occasionally, MaxWalkSat will reset to a new randomly selected
initial solution and,
after $N$ attempts, the best solution is returned.
Our implementation used $n=50$, $\alpha=0.5$, and $N=10$.

{\em SEESAW} is a variant of MaxWalkSat we first reported in~\cite{me09f}.
While searching the ranges of a feature,
this algorithm exploits the monotonic nature of \eq{lineem}
and \eq{linesf}.  SEESAW ignores all ranges except the minimum
and maximum values for a feature in $p$.   
Like MaxWalkSat, the feature chosen on each iteration
is made randomly.  However, SEESAW has the ability to delay bad decisions until the end of the algorithm
(i.e., decisions where constraining the feature to {\em either} the minimum or maximum value results in a worse solution).
These treatments are then guaranteed to be pruned during the back select.

{\em ISAMP} is a fast stochastic iterative sampling method that extends a treatment using
randomly selected ranges.
The algorithm follows one solution, then resets to try other paths (our implementation resets 20 times).
The algorithm has proved remarkably effective 
at scheduling problems,
perhaps because it can rapidly 
explore more of the search space~\cite{craw94}.
To avoid exploring low-value regions, our version of ISAMP stores
the worst solution seen so far. Any conjunction whose $value$ exceeds that
of the worst solution is abandoned, and the new ``worst value'' is stored. If
a conjunction runs out of new ranges to add, then the ``worst value''
is slightly decreased.
This ensures that consecutive failing searches do not permanently raise the
``worst value'' by an overly permissive amount.


Our remaining algorithms use some variant of tree search. 
Each branch of the tree is a different treatment
(a conjunction of ranges)  of size $i$.
In terms of the forward
select search described above, the  y-axis
statistics of \fig{run} are collected whenever branches
of size $i$ are extended to size $i+1$.

{\em BEAM search} extends branches as follows.
Each branch forks once for every new option available to that
range.
All the new leaves are sorted by their value
and only the top $N$ ranked branches are marked for further expansion.
For this study we used $N=10$ and the $y$ axis of \fig{run} was reported
using the median values seen in the top $N$ branches. 
 
{\em A-STAR} runs like BEAM, but the 
sort order is determined by the sum $f$ (the cost of reaching the current
solution) plus $g$ (a heuristic estimate of the cost to reach the final
solution).  
Also, unlike BEAM, the list of options is not truncated 
so
a termination criterion is needed (we stop the search if the
best solution so far has not improved after $m$ iterations).
For this study, we used \eq{bfc} for $g$ and the percentage of the
features with ranges in the current branch as $f$.

\begin{figure}[!b]
\begin{center}
{\footnotesize\begin{tabular}{r|rrr}
algorithm &  Defects &months &time\\\hline
SEESAW    &4  &          4 &     3\\ 
BEAM      & 0 &         3 &     3\\
A-star    &  0&           1&     1\\
SA        &  0&            1&    1\\
MaxWalkSat & 0 & 0& 0\\
ISSAMP   & 0  & 0& 0
\end{tabular}}
\end{center}
\caption{Number of times algorithm found to be top-ranked from 20 repeats
of forward/back selecting over the four case studies of \fig{cases}.}\label{fig:xper}
\end{figure}
\section{Experiments }
\subsection{Comparing Algorithms}

We ran 20 forward selects on the case studies of \fig{cases} using BFC
followed by back selects
(t-tests  pruned round $i$ if
it was statistically the same at 95\% confidence as round $i-1$).
Separate statistics were
collected for the defects/effort/time predictions seen at the policy point in the 20*4 trials.
The {\em top-ranked} algorithm(s) had statistically  different
and lower defects/effort/time predictions than any other algorithm(s). 

\fig{xper} shows how many times each algorithm was 
{\em top-ranked}. Note that the maximum value 
possible is 4 (i.e., once for each \fig{cases} case study). In those results, the two stand-out 
worst algorithms are MaxWalkSat and ISAMP and the two stand-out
best algorithms are SEESAW and BEAM.
The performance of these two is sometimes equivalent
(e.g., in $time$, both algorithms achieved an equal number of top ranks). 
However, we cannot recommend BEAM search for this domain:
\bi
\item BEAM runs 10 times slower than SEESAW.
\item
SEESAW
performs better than BEAM in some cases (e.g. in  defects, BEAM is never top-ranked).
\ei

Another interesting feature of \fig{xper} is the great difference between MaxWalkSat and SEESAW. As mentioned above, the difference between these two
algorithms is very small.  Recall that SEESAW assumed that the local search
state space was monotonic, so it only explored minimum and maximum
values for each feature. This single domain heuristic
resulted in a dramatic  performance improvement.
This is not a new result:
AI research in the 1980s~\cite{mcluskey87} showed the value of combining
domain-general ``weak'' heuristic algorithms (e.g., A-STAR, BEAM search) with
domain-specific ``strong'' heuristics such as the monotonicity assumption
exploited by SEESAW.  However, even if it is not a new finding,
it should remind researchers not to depend too heavily on off-the-shelf algorithms.
As shown here, a little algorithm tweaking with domain knowledge can go a long way.

\fig{res} shows the  percent reductions achieved by SEESAW on the four case studies of \fig{cases}.
SEESAW's treatments can  reduce  project
defect/time/effort  estimates by 73, 30, and 57\% (respectively).  
That is, even if SEESAW cannot offer a major reduction in the development time, it can advise how
to reduce defects and development effort by over half.
Based on \fig{res}, we would recommend using SEESAW before all the project decisions are made, otherwise
there is very little for it to work with.
For example, SEESAW's worst performance was with the
OSP2 case study. As shown in \fig{cases}, this project has the most amount
of fixed settings, so SEESAW only has a small range of options it can explore.


%\begin{figure}
%\begin{center}
%{\scriptsize
%\begin{tabular}{l@{~~}r|r|@{~}r|r|@{~}r|r@{~}r}
%        
%		\multicolumn{2}{c}{~}     &\multicolumn{4}{c}{estimates}\\\cline{3-6}
%		\multicolumn{2}{c}{~}     &\multicolumn{2}{|c|}{A = as is}&\multicolumn{2}{c|}{ B = to be}&\multicolumn{2}{c}{percent change}\\\cline{3-6}
%        &     &mean&spread&mean&spread\\\cline{3-6} 
%        &     &A&$\delta$A&B&$\delta$B
%
%&
%
%$1-\frac{A}{B}$
%
%&
%
%$1-\frac{\delta{A}}{\delta{B}}$\\\hline
%defects&flight&1789&2080&355&824&80&70\\
%&ground&1610&2040&241&300&85&85\\
%&osp&3259&1882&1142&884&65&61\\
%&osp2&306&145&226&140&26&19\\\cline{7-7}
%&    &   &   &   &   &med = 73  &   \\\hline
%
%months&flight&31&8&19&7&39&34\\
%&ground&27&7&16&5&38&37\\
%&osp&32&6&31&5&4&4\\
%&osp2&19&4&15&2&22&28\\\cline{7-7}
%&    &   &   &   &   &med = 30  &   \\\hline
%
%effort&flight&701&678&193&275&72&66\\
%&ground&484&398&132&137&73&70\\
%&osp&1113&425&651&281&42&39\\
%&osp2&179&55&171&54&5&4\\\cline{7-7}
%&    &   &   &   &   &med = 57  &   \\\cline{3-6}
%\end{tabular}}
%\end{center}
%\caption{SEESAW estimates. ``As is'' and ``to be'' denotes estimates before/after applying SEESAW.
%Last two columns show percent reductions. ``Median'' denotes
%the median values and ``spread'' shows the (75-50)\% spread.
%Mann-Whitney tests (at 95\% confidence) report that the ``to be'' values are significantly
%different to the ``as is'' values.}\label{fig:reduce}
%\end{figure}
%
%A premise for our endorsement of SEESAW is that the predictions made by this tool are actually realized.
%In order to test that premise, we conducted the following {\em accuracy experiment} to compare the
%SEESAW's recommendations to those provided by other tools. 
%
%That experiment has three parts. Firstly, using Baker's prior work~\cite{baker07,me09e}, we noted
%the error ranges seen in standard effort estimation. 
%
%In that study, Baker selected 10 projects at random from a log of historical NASA data 
%sets~\footnote{\url{http://promisedata.org/?s=nasa93}}. Then,  30 times, an effort model
%was learned using Boehm's local calibration procedure (a standard technique used in the COCOMO
%community)~\cite{boehm81} using 90\% of the training data (taking care to ignore the test instance).
%The accuracy measure of the predictions from the learned models was expressed as the magnitude of the
%relative error
%$MRE=\frac{abs(predicted-actual)}{actual}$. 
%\begin{figure}
%\begin{center}
%\includegraphics[width=2.4in]{comparesReport_lc.pdf}
%\end{center}
%\caption{MRE results seen with standard effort estimation results. From Baker~\cite{baker07}.
%}\label{fig:mre2}
%\end{figure}
%
%Baker's experiments allows us to calibrate the errors expected from standard effort estimation methods.
%\fig{mre2} shows the results. The maximum MRE error is sometimes very  (300\% to 500\%) while
%the median MRE errors in the 10 case studies are: 
%\[\footnotesize
%median(MRE)= \{25,26,27,28,29,30,31,35,37,38\}\%
%\]
%
%SEESAW's errors are 
%
%  results from \eq{ab} we took historical we collected the errors generated by standard effort estimation methods.
%

%predictions made at the policy point found by SEESAW 
%
%{\em Issues with Mean:} The $experiment=1$ results of \fig{delta} shows the differences between the estimates
%generated {\em with} and {\em without} tuning on local data (the no-tuning estimates came from our search-based tool and the
%tuning-results can from
%on-line versions of the standard USC models\footnote{COCOMO II with Heuristic Risk Assessment: \\http://sunset.usc.edu/research/COCOMOII/expert\_cocomo/expert\_cocomo2000.html}).
%The $experiment=2$ results of that figure show
%the mean errors seen 
%when estimates are generated using conventional methods. Note that the mean differences in the $experiment=1$
%results are small (36\% or less).
%Further,
%the mean effort values of the conventional methods is larger (never less than 42\%).
%That is, \fig{delta} shows no evidence that the differences between our methods and conventional methods
%is inordinately large.
%
%{\em Issues with Variance:}
%Our tools sample across the space of possible tunings. The variance of the conclusions reached in that sample
%could be reduced by the use of local data to pre-tune our models, before we perform any sampling.
%However, it must be noted that the variances of our conclusions are quite small (see \fig{r1} and \fig{delta}).
%In summary, while we believe it may be useful to tune with local data, the overall benefit on reducing the variance or the mean  not be large.
%
%
%\begin{figure}
%\small
%\begin{tabular}{r|rrrrr}
%        &            & \multicolumn{4}{c}{data set}\\\cline{3-6}
%        & experiment & flight        & ground & OSP         & OSP2        \\
%\hline
%defects & 1          & 30 $\pm$ 23   & 28 $\pm$ 33 & 39 $\pm$ 33 & 32 $\pm$ 18 \\
% months & 1          & 26 $\pm$ 17   & 27 $\pm$ 16 & 23 $\pm$ 17 & 12 $\pm$ 7  \\
% effort & 1          & 36 $\pm$ 31   & 34 $\pm$ 31 & 20 $\pm$ 17 & 18 $\pm$ 13 \\
%\hline
% effort & 2          & 41 $\pm$ 40   &        & 44 $\pm$ 63 &  42 $\pm$ 57
%\end{tabular}
%     \caption{Results (Mean $\pm$ standard deviations) from our experiments.
%For an explanation of the method by which this was generated, see \fig{deletahow}.
%}\caption{fig:delta}
%\end{figure}
%\begin{figure}
%\small
%\begin{center}
%\begin{tabular}{|p{.95\linewidth}|}\hline
%The lines labeled $experiment=1$ in \fig{delta} were generated as follows:
%\be
%\item
%Generate 100 {\em projects} based on constraints $C$ by using our search-based tool and score them.
%\item
%Score {\em projects}
%against the standard on-line USC
%models \footnote{COCOMO II with Heuristic Risk Assessment: \\http://sunset.usc.edu/research/COCOMOII/expert\_cocomo/expert\_cocomo2000.html};
%\item
%Combine the {\em estimate} scores from search-based tool with the {\em baseline} scores from standard on-line USC models
%using $delta= \frac{estimate - baseline}{baseline}$. 
%\ee
%The line labeled $experiment=2$ in \fig{delta}
%was generated by taking flight systems/OSP/OSP2 systems
%and estimating them using a COCOMO model tuned to some NASA COCOMO, as follows:
%\bi
%\item
%Using the NASA93 COCOMO data set
%\footnote{\url{http://promisedata.org/?s=nasa93}}
%we tuned a COCOMO model using
%using Boehem's LC method~\cite{boehm81};
%\item
%Generate 100 {\em projects} based on constraints $C$ by using our search-based tool and score them.
%\item Effort estimates for the generated projects were generated using the tuned
%COCOMO model.  
%\item The means and standard deviations for those estimates were recorded in \fig{delta}.
%\ei\\\hline
%\end{tabular}
%\end{center}
%\caption{Generation method for the \fig{exp2} results.}\label{fig:deltahow}
%\end{figure}
%
\begin{figure}
\begin{center}\footnotesize
\begin{tabular}{r|rrr}
data set &          defects&	time	&effort\\\hline
flight	&			80\%&		39\%		&72\%		\\
ground	&			85\%&		38\%	&	73\%\\
osp		&			65\%&		4\%&		42\%\\
ops2	&			26\%&		22\%&		5\%\\\hline
median   &          73\% &    30\%&    57\%
\end{tabular}
\end{center}
\caption{Percent reductions $(1-final/initial)$ achieved by SEESAW on the \fig{cases}
case studies. The $initial$ values come from round $0$ of the forward select.
The $final$ values come from the policy point.
Note that 
all the initial and final values are statistically different
(Mann-Whitney, 95\% confidence).  }\label{fig:res}
\end{figure}


\subsection{Effects of Changing the Value Function}
Our prior ASE report on this work~\cite{me07d} used simulated annealing to find treatments.
That work reported that, in all studied cases, automated defect removal tools were
always found in the learned treatments.

The poor performance of SA in \fig{res} means we must revisit those conclusions, using SEESAW
rather than SA.
\fig{percent} shows the  ranges seen in  SEESAW's treatment (after a back select).
The BFC and XPOS  columns show the percent frequency of a range appearing when 
SEESAW used our different $value$ functions. 
These experiments were repeated 20 times and only the ranges found in the majority (more than 50\%) of the
trials
are reported.  

The results are divided into our four case studies: ground, flight, OSP, and OSP2.
Within each case study, the results are sorted by the fraction $\frac{BFC}{BFC + XPOS}$. This
fraction ranges 0 to 100 and:
\bi
\item If close to 100, then a range is selected by BFC more than XPOS.
\item If close to 0, then a range is selected by XPOS more than BFC.
\ei
The right-hand columns of \fig{percent} flag the presence of
 manual defect remove methods
($pr$=peer reviews) or automatic defect removal methods ($aa$=automated analysis; $etat$=execution testing tools).
Note that 
high levels of automatic defect removal methods are only frequently required in
ground systems, and
only when valuing BFC.
More usually,   defect removal techniques are {\em not} recommended.
In ground systems, $etat=1$, $pr=1$, and $aa=1$ are all examples of SEESAW {\em discouraging} rather
than endorsing the use defect removal methods.
That is,
in three of our four case studies, it is more important to prevent defect introduction than
to use after-the-fact defect removal methods. In ground, OSP, and OSP2, defect removal methods
are very rare (only $pr=1$ in flight systems).


Another important aspect of \fig{percent} is 
that there is no example of both value functions frequently endorsing the same range.
If a range is commonly selected by BFC, then it is usually {\em not}
commonly accepted by XPOS.  The most dramatic example of this is the OSP2 results of
\fig{percent}: BFC always selects (at 100\%) the low end of a feature ($sced$=2) while
XPOS nearly always selects (at 80\% frequency) the opposite high end of that  feature. 
\begin{figure}
\renewcommand{\baselinestretch}{0.92}
{\scriptsize
\begin{center}
\begin{tabular}{l|@{~}c@{~}|r@{~}|r@{~}|r@{~}|c|@{~}c}
    \multicolumn{2}{c}{~}           & \multicolumn{2}{c}{$value$}           &     &\multicolumn{2}{c}{defect}\\\cline{3-4}
Data   & Range           & B=BFC      &  X=XPOS   &  $\frac{B}{B + X}$    &\multicolumn{2}{c}{removal}\\\cline{6-7}
&&& & & manual & automatic\\\hline
ground&rely = 4 &  70 &  20 &  77                   &        &           \\
      &aa   = 6 &  70 &  25 &  73                   &        &  hi in B  \\
      &resl = 6 &  65 &  40 &  61                   &        &           \\
      &etat = 1 &  35 &  65 &  35                   &        &  lo in X \\
      &aexp = 5 &  45 &  85 &  34                   &        &           \\
      &pr   = 1 &  35 &  80 &  30                   &  lo in X    &           \\
      &aa   = 1 &  25 &  60 &  29                   &   &lo in X            \\
      &data = 2 &  25 &  70 &  26                   &        &           \\
     &rely  = 1 &  15 &  70 &  17                   &        &\\\hline
flight&rely = 5 &  65 &  25 &  72                   &        &           \\
      &flex = 6 &  80 &  50 &  61                   &        &           \\
      &docu = 1 &  55 &  85 &  39                   &        &           \\
      &site = 6 &  55 &  85 &  39                   &        &           \\
      &resl = 6 &  45 &  70 &  39                   &        &           \\
      &pr   = 1 &  45 &  70 &  39                   & lo in X&           \\
      &pvol = 2 &  45 &  75 &  37                   &        &           \\
      &data = 2 &  35 &  60 &  36                   &        &           \\
      &cplx = 3 &  45 &  90 &  33                   &        &           \\
      &rely = 3 &  15 &  60 &  20                   &        &\\
\hline
OSP   &pmat = 4 &  85 &  45 &  65                   &        &           \\
      &resl = 3 &  45 &  70 &  39                   &        &           \\
      &ruse = 2 &  40 &  65 &  38                   &        &           \\
      &docu = 2 &  25 &  90 &  21                   &        &\\
\hline
OSP2  &sced = 2 & 100 &   0 & 100                   &        &           \\
      &sced = 4 &   0 &  80 &   0                   &        &  \\    
\hline
\end{tabular}
\end{center}}
\caption{Frequency (in percents)
of feature ranges seen in 20 repeats of SEESAW, using two different goal functions: BFC and XPOS.
The last two columns comment on any defect reduction feature. Not shown in this figure
are any feature ranges that occur less than 50\% of the time.
}\label{fig:percent}
\end{figure}

In summary, perceptions of $value$ can change everything.  Techniques that seem useful to one kind of 
project/value function may be counter-indicated for another.
One characterization of the \fig{percent} results is that, for some projects, it is preferable
to prevent defects before they arrive (by reorganizing the project) rather than try to remove them
afterwards using (say) peer review, automated analysis, or execution test tools.

\section{Conclusions}
There are many different SE technologies that can be applied to a project. Such technologies
can be defined as conjunctions of inputs to software process models. Technology $X$ is preferred
to technology $Y$ if $X$'s inputs lead to better output scores that $Y$.

Assessing the value of different SE technologies via process models is complicated by  tuning
instabilities.
More many years, we have tried to reduce this problem using a variety of techniques. The
failure of that work has prompted research on alternate methods.
We dodge
the model tuning problem by searching for conclusions that are
stable across the space of possible tunings.
Our tool uses AI to search for different project options.
Each option is scored by a Monte Carlo run that selects tuning options at random
from the space of possible options.
For models where project values (and not tuning values) are the dominant influence on predictions,
this combined method finds critical project options without requiring local tuning data.
This is a very different approach to standard software process
modeling where local data is essential to model tuning. 

This paper certified that our current preferred search engine
(SEESAW) runs as well (or better) than several alternatives. When we 
applied SEESAW  with two different $value$ functions,
the treatments found by SEESAW changed dramatically.
In particular, most projects worked
best when they were organized to stop defects entering the code
base rather than using some automated analysis method to remove
them, after they have been introduced.

This work has significant implications for the ASE community.
Based on this work we argue that it is no longer enough to just propose (say) some automated defect reduction tool. 
Rather,
the value of ASE tools for a software project needs to be carefully assessed with respect to
the core $value$s of that project.


%ACKNOWLEDGMENTS are optional
%
% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{IEEEtran}
{\small
\bibliography{refs}
}  % sigproc.bib is the name of the Bibliography in this case
\appendix
This appendix lists the minimum and maximum $m$ values used
for \eq{lineem} and \eq{linesf}.
In the following, $m_\alpha$ and $m_\beta$ denote COCOMO's linear and exponential
influences on effort/cost, and 
$m_\gamma$ and $m_\delta$ denote COQUALMO's linear and exponential influences on number of
defects.


Their are two sets of  effort/cost multipliers:
\be
\item
The {\em positive effort EM} features, with slopes $m^+_\alpha$, that are proportional to effort/cost. 
These features are:
cplx,
data,
docu,
pvol,
rely,
ruse,
stor, and
time.
\item
The
{\em negative effort EM} features, with slopes $m^-_{\alpha}$,
are inversely proportional to effort/cost.
These features are 
acap,
apex,
ltex,
pcap,
pcon,
plex,
sced,
site, and
tool.
\ee
Their $m$ ranges, as  seen in 161 projects~\cite{boehm00a}, are:
\begin{equation}
\footnotesize
\left(0.073 \le m^+_\alpha \le 0.21\right) \wedge
\left(-0.178 \le m^-_\alpha \le -0.078\right)
\end{equation}
In the same sample of projects,
the COCOMO effort/cost scale factors (prec, flex, resl, team, pmat) have
the range: 
\begin{equation}\footnotesize
-1.56 \le m_\beta \le -1.014
\end{equation}
Similarly, there are two sets of defect multipliers and scale factors:
\be
\item The {\em positive defect} features have slopes $m^+_\gamma$ and are proportional
to estimated defects.  These features are
flex,
data,
ruse,
cplx,
time,
stor, and 
pvol.
\item The {\em negative defect} features, with slopes $m^-_\gamma$, that are inversely proportional
to the estimated defects. These features are 
acap,
pcap,
pcon,
apex,
plex,
ltex,
tool,
site,
sced,
 prec,
resl,
team,
pmat,
rely, and
docu.
\ee
COQUALMO divides into three models describing how defects change in requirements, design,
and coding.
These  tunings options have the range:
\begin{equation}\label{eq:coq}\small
\begin{array}{c}
requirements\left\{
\begin{array}{c}
0 \le m^+_\gamma \le 0.112\\
-0.183 \le m^-_\gamma \le -0.035
\end{array}
\right. \\
~\\
design \left\{
\begin{array}{c}
0 \le m^+_\gamma \le 0.14\\
-0.208 \le m^-_\gamma \le -0.048
\end{array}
\right. \\
~\\
coding \left\{
\begin{array}{c}
0 \le m^+_\gamma \le  0.14\\
-0.19 \le m^-_\gamma \le -0.053
\end{array}
\right.
\end{array}
\end{equation}
The tuning options for  the  defect removal features are:
\begin{equation}\label{eq:linedefrem}\small
\begin{array}{rl}
\forall x \in \{1..6\} &SF_i=m_\delta(x - 1)\\
requirements: 	& 0.08 	\le m_\delta \le 0.14 \\
design: 		& 0.1 	\le m_\delta \le 0.156 \\
coding: 		& 0.11 	\le m_\delta \le 0.176 \\
\end{array}
\end{equation}
where  $m_\delta$ denotes the effect of  $i$ on  defect removal.



% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%
%APPENDICES are optional
%\balancecolumns
%\balancecolumns
% That's all folks!
\end{document}
