% This is "sig-alternate.tex" V1.8 June 2007
% This file should be compiled with V2.3 of "sig-alternate.cls" June 2007
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.3 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.3) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
% \CopyrightYear{2007} will cause 2007 to appear in the copyright line.
% \crdata{0-12345-67-8/90/12} will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
% Technical questions _only_ to
% Gerald Murray (murray@acm.org)
% ===============================================================
%
% For tracking purposes - this is V1.8 - June 2007

\documentclass{sig-alternate}
%\usepackage{times}
\usepackage{url}
\usepackage{cite}
%\newcommand{\boxplot}[5]{~}
\input{boxplot}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newcommand{\be}{\begin{smallenum}}
\newcommand{\ee}{\end{smallenum}}
\newcommand{\tion}[1]{\S\ref{tion:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newenvironment{smallitem}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{itemize}
   \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{itemize}}
\newenvironment{smallenum}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{enumerate}
  \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{enumerate}}

\begin{document}
%
% --- Author Metadata here ---
\conferenceinfo{PROMISE}{'09 Vancouver,  Canada USA}
%\CopyrightYear{2007} % Allows default copyright year (200X) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{Can We Build Software  Faster \underline{and} Better \underline{and} Cheaper?
%\titlenote{For a prior draft, see \url{http://menzies.us/pdf/09bfc.pdf}
%\titlenote{
% This research was conducted at WVU and the Jet Propulsion Laboratory under a NASA sub- 
%contract. Reference herein to any speci�c commercial product, process, 
%or service by trade name, trademark, manufacturer, or otherwise, does not 
%constitute or imply its endorsement by the United States Government.
%}
}%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
% For esthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'esthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{3} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Tim Menzies, Oussama El-Rawas\\
       \affaddr{CS\&EE,  West Virginia University,  USA}\\
       \email{tim@menzies.us, oelrawas@mix.wvu.edu}
% 2nd. author
\alignauthor
Jairus Hihn \\
       \affaddr{Jet Propulsion Laboratory}\\
       \affaddr{California, USA}\\
       \email{jairus.hihn@jpl.nasa.gov}
\alignauthor
Barry Boehm\\
       \affaddr{Computer Science Dept.}\\
 	\affaddr{Uni. S. California, USA}\\
      \email{boehm@sunset.usc.edu}
}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for esthetic reasons that you place these 'additional authors'
% in the \addiytional authors block, viz.
\date{\today}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
``Faster, Better, Cheaper'' (FBC) was a development philosophy
adopted by the NASA administration in the mid to late 1990s. 
that lead to some
some dramatic successes such as Mars Pathfinder as well as a number highly publicized mission failures, such as 
the Mars Climate Orbiter \& Polar Lander.

The general consensus on FBC was 
``Faster, Better, Cheaper? Pick any two''. According to that
view, is impossibly to optimize on all three criteria without
compromising the third.
This paper checks that view using
an AI search tool called STAR.
We show that  FBC is indeed feasible and
produces similar or better results when compared to other methods
However, for FBC to work,
there must be a balanced concern and concentration on the quality
aspects of a project. If not,  ``FBC'' becomes ``CF'' (cheaper and faster)
with the inevitable lose in project quality.
\end{abstract}

% A category with the (minimum) three required fields
\category{B.4.8}{Programming techniques}{Performance}{Modeling and prediction}
\category{I.6.4}{Computing Methodologies }{Model Validation and Analysis}
\keywords{software engineering, predictor models, COCOMO, Faster Better Cheaper, simulated annealing, software processes}

\section{Introduction}
Previously,
PROMISE researchers have  used {\em induction} (summarization of data into a model)
to generate predictor models for software engineering. While useful,
induction 
has several drawbacks:
\bi 
\item  It only explores half the story. As Murray Cantor observed in his PROMISE 2008 keynote, 
once predictive models are generated, they 
are used within some business context. While we need more papers on predictive model {\em generation},  it may be time
to ask the PROMISE community to
write
more papers on predictive model {\em usage}.
\item
Automatic induction assumes the existence of data. 
Many
domains are {\em data starved} where  it is hard to 
obtain local data. 
\item
More generally, we ask the question:
why do we persist on generating  new predictive models all the time? If there is any generality in software engineering
it should be possible to reuse predictive models. We should at least experiment with
this approach rather than always assuming the best predictive model is a new model.
\ei

Hence, we explore predictive
model {\em use} and {\em reuse} rather than predictive model {\em
generation}.  Model reuse is complicated by extreme data starvation:
the {\em tuning variance} problem arises if there is insufficient
data to generate precise tunings.  
Our STAR tool~\cite{elrawas08,me08c,me09a}  uses AI methods
to
conduct large scale ``what-if''
queries over software predictive models.  
If parts of a predictive models are not known with certainty,
STAR seeks stable conclusions within the space of possible values.

This paper applies STAR to assess the infamous ``Faster, Better, Cheaper'' (FBC)
development practices used at NASA in the 1990s. 
FBC was advocated in the 1990s by the then-administrator of NASA, Daniel Goldin, as a method for reducing the 
expenditure of NASA. FBC  was in-line with the direction that the Clinton administration's approach 
of doing more for less. FBC was initially successful: projects that usually cost over a billion were implemented
at $\frac{1}{4}th$ of that cost (e.g. Mars Pathfinder). However,
subsequent 
failures (Mars Climate Orbiter and Polar Lander; the  Columbia Shuttle disaster) 
lead to much criticism of FBC.

STAR is a natural tool for assessing FBC.
The tool combines estimates with utility weights
$\{f,b,c\}$ (short for 
\underline{F}aster, 
\underline{B}etter,
\underline{C}heaper): \begin{equation}\label{eq:score}
 score = \frac{
\sqrt{
f.M^2 +
b.D^2 +
c.E^2  
}}{\sqrt{f + b + c}} \end{equation} 
This $score$ value represents
the Euclidean distance to minimum values
for predictions of development effort ``$E$''; total development months ``$M$'';
and ``D'', the number of delivered defects per KLOC.
 If we normalize the predictions min..max to
0..1 then \eq{score} has the range one to zero and {\em lower} scores
are {\em better}.
STAR searches for the minimal 
set of project changes that most reduces this $score$.

By adjusting the various values of $(f,b,c)$, we can compare the effects
of methodologies that deemphasize different project goals:
\bi
\item
BF =  ``better, faster'' i.e.    $c=0$ and $b=f=1$;
\item
BC = ``better, cheaper''    i.e.       $f=0$ and $b=c=1$;
\item
CF  = ``cheaper, faster''  i.e.   $b=0$ and $f=c=1$;
\item
FBC = ``faster, better, cheaper'' i.e. $b=f=c=1$. 
\ei
This paper reports the results of such a comparison.
After a discussion of related work, we will
review the sad history of ``Better, Faster, Cheaper'' at NASA. 
We will then describe the case
studies will be explore, as well as the STAR systems.  
In the subsequent results section we will offer the 
surprising observation that, contrary to the prevailing wisdom at NASA,
FBC is not necessarily a disastrous development methodology. In fact, in $\frac{7}{9}$
of our studies, 
\begin{quote}{\em
FBC produces results that are within 5\% of the minimum estimates generated by any of BF,BC,CF. 
}\end{quote}
Our results suggest that (a) FBC is in fact a viable development
methodology but (b) 
what really went wrong with FBC at NASA was that it was used as a  front for CF; i.e. built it faster and
cheaper, but deemphasized  quality considerations.

Our more general conclusion is that
we can make
more {\em use} and {\em reuse} of predictive models.
If stable conclusions exist across the tuning space, then
AI tools like STAR can offer conclusions from predictive models, even when their is
insufficient data for tuning.  This is an powerful aspect of AI tools
like STAR since
we often lack the
data required to precisely tune an imported model to a local
domains.  
It is hoped that that this paper will inspire more papers at future
PROMISE conferences that explore not just model {\em generation},
but also the {\em reuse} of those generated model  in new domains.
To facilitate that discussion, we have made all our code openly
available\footnote{\url{http://unbox.org/wisp/tags/STAR/2.1/}}
and we are building a web interface to
an executable version of
STAR\footnote{\url{http://nova.unbox.org}}. That interface will be on-line by
the time of the PROMISE'09 conference.
\section{Related Work}
\fig{options}  places STAR in the context of  related work. 
Predictive models can be build automatically or manually.
Automatically generated predictive models can be automatically validated using many methods including
(e.g.) the cross-validation procedures used in much data mining research~\cite{witten05}.
Manually generated predictive models,
built using human expertise, can be remarkably effective: see  
Fenton's  PROMISE'07 paper reported a validation study  of one such predictive model~\cite{fenton07a}).

As shown top right of \fig{options}, STAR is appropriate
in domains that lack both domain experts and data.
Many domains are data starved (see \fig{starve}) in which case
there is
neither sufficient data
to automatically learn predictive models .

\begin{figure}
\begin{center}
\begin{tabular}{c|p{1.5cm}|p{2.8cm}|p{2.8cm}|}\cline{3-4}
\multicolumn{2}{c|}{~}&              domain experts  & no domain experts\\\cline{2-4}
&data starved\newline
(little or \newline no
data)  
& build predictive models manually              
& reuse predictive models from other sites, without tuning;  sample across range of possible options. 
E.g. STAR.\\\cline{2-4}
& data poor \newline
(small 
amounts
of data)    &  build predictive models manually; tune
               with  
               local data; e.g.~\cite{fenton07a}                   & reuse predictive
models from other sites; tune with local data; E.g. Boehm's local calibration of COCOMO models~\cite{boehm81}.  \\\cline{2-4}


& data rich\newline 
(much data) & \multicolumn{2}{p{5.6cm}|}{build predictive models automatically\newline 
e.g.~\cite{chen05,me05a,me06f,me08d,jiang08}.
} \\\cline{2-4}

\end{tabular}
\end{center}
\caption{Predictive model construction options.}\label{fig:options}
\end{figure}
\begin{figure}
\begin{tabular}{|p{.95\linewidth}|}\hline
\bi
\item
Metrics-guru Norman Fenton study data collection~\cite{fenton96} for many years.
Recently, he has 
despaired of that approach. At a keynote address  PROMISE 2007\footnote{\url{http://promisedata.org/?cat=130}}, Fenton shocked
his audience by saying:
\begin{quote}
{\em ``....much of the current software metrics research is inherently irrelevant to the industrial mix ...
any software metrics program that depends on some extensive metrics collection is doomed to failure.''} 
\end{quote}
\item
The COCOMO experience is similar to that of Fenton. After 26 years of trying, the
COCOMO team has  collected less
than 200 sample projects for the COCOMO database.
Also,
even after two years of effort we were only able to
add 7 records to a NASA-wide software cost metrics repository~\cite{me06f}.
\ei\\\hline
\end{tabular}
\caption{Evidence for data starvation in SE.}\label{fig:starve}
\end{figure}

In the completely opposite circumstance, a domain is data rich. In such domains, automatic
methods can learn predictive models. There are many examples of work in this area 
including~\cite{chen05,me05a,me06f,me08d,jiang08}.  
We recommend automatic methods for data rich domains since manual modeling can be very expensive
(for
example,
Fenton manually built his Bayes nets over a two year period~\cite{fenton07a}).

In the middle case of \fig{options},  small amounts of local data can be used to tune either:
\bi
\item Local
predictive models either built manually; or 
\item Predictive models imported from other sites.
\ei
For example, Boehm et at.~\cite{boehm00b} advocate a certain
functional form for generating  software development effort estimates.  In that form,
the development effort is linear on a set of effort multipliers
$EM_i$ and exponential on  a set of scale factors $SF_j$:
\begin{equation}\label{eq:coc}\small 
\begin{array}{rcl}
effort&=&A \cdot KSLOC^{B + 0.01\cdot\sum_j{\beta_j}SF_j}
 \cdot \prod_i {\alpha_i}EM_i 
\end{array}
\end{equation}
\input{cocmo}
The particular effort multipliers and scale factors recommended by
Boehm et al. are shown in \fig{cparems}.
While Boehm et al offer default values for the \eq{coc} variables, 
linear regression on local data can tune the $\alpha_i,\beta_j$
values to the particulars of a local site.  Also, if there is
insufficient data for a full tuning of $\alpha,\beta$, then a coarse
grain tuning can be achieved by just adjusting the  $A,B$\footnote{We will use uppercase $B$
to denote the COCOMO linear tuning variable of \eq{coc}
and lower $b$ to denote the business
utility associated with defect predictions of \eq{score}}.
 linear
and exponential tuning parameters.

A problem that has been under-explored in the literature is {\em tuning variance}.
In data starved domains, there is insufficient data to produce precise tunings.
For example, 
At PROMISE 2005, we have reported very large tuning variance
in the post-tuning values of  $\alpha$ and $\beta$~\cite{me05b}.
Baker~\cite{baker07} offers a similar finding.
After  thirty 90\% random samples of that data,
the $A,B$ ranges found during tuning were surprisingly wide:
\begin{equation}\label{eq:ab}\small
\left( 2.2 \le A \le 9.18 \right) \wedge \left(0.88 \le B \le 1.09\right) 
\end{equation} 
We are not the only research group to be concerns about tuning variance.
At PROMISE 2007,
Korte \& Port~\cite{korte08} 
explore the variance of automatically learned effort predictors.
They comment that this variance is large enough to confuse  standard methods for assessing different
predictive model generators.

Since 2005~\cite{chen05,me06d}, we have been trying to tame tuning variance.
using feature subset selection (FSS).
However, despite years of work, we now report that
FSS reduces but does not tame the variance 
of $A,B,\alpha,\beta$. 

Having 
failed to tame tuning variance,  we have been exploring a new approach.
The STAR tool~\cite{elrawas08,me08c,me09a}.
that we describe below checks
for stable conclusions within  the space of 
possible tunings. 

\section{``Faster, Better, Cheaper''}
This paper  applies STAR to an analysis of FBC.
In the 1990s, the main approach to implementing FBC within NASA was to down size
projects and reduce their cost and complexity, concentrating on
producing missions in volume. Reducing funding naturally meant that
less verification and testing was possible within budget and schedule
constraints. The reasoning behind this however was to be able to
produce a larger volume of unmanned missions, which would counteract
the expected higher rate of mission failure. This would, optimally,
yield more successful missions as well as more scientific data
produced by these projects. Another focus in this policy was allowing
teams to take acceptable risks in projects to allow for cost
reduction, and possibly using new technology that could reduce cost
while possibly providing more capabilities. This was accompanied
by the new view, being pushed at NASA by Goldin,  that
``it's ok to fail'' \cite{spear00(2)}, which was rather misunderstood.
This new policy was meant to eliminate huge budget missions of the
past, that upon possible failure would yield large losses. Project
cost used to routinely exceed the \$1 billion mark, while the first
FBC project, the Mars Pathfinder, was completed for a fraction of
the cost, netting at about \$270 million \cite{david00}.

Some within NASA, like 30 year veteran Frank Hoban, supported
these policies~\cite{david00} who
viewed these new policies as a
necessary break from traditional policies that were very risk averse.
The additional cost reduction, accompanied by the additional risk,
was to allow for a path to cheap and commercial space flight. Even
given the reduced funding, the Mars Pathfinder mission, along with
other first generation FBC missions, were successes. This fueled
enthusiasm to apply FBC across all of NASA to further reduce
spending per mission
 as well cutting the
work force by one third. FBC was extended to be applied on manned
space missions as well, where funding was also reduced. Coming into
a space shuttle program that was starting to age and in need of
updates, the new policies imposed cuts in funding from 48\% of the
NASA budget to 38\% \cite{key03}, further straining that program.
Further more, a single prime contractor (Lockheed Martin) was used
for missions in another bid to reduce cost and managerial
complexity~\cite{young00, NASAwatch}.

This produced opposition within NASA, where traditionally issues
pertaining to the shuttle were designated LOVC (Loss of Vehicle and
Crew) and given priority over all other issues, including cost.
However the cost cuts and layoffs that ensued damaged morale leading
to a string of early retirements of
veteran scientists, skilled engineers and managers\cite{key03}.

Despite this, additional projects were planned including
Mars Climate Orbiter and Polar Lander. These two
projects were more aggressive implementations of FBC, especially
when it came to the Faster-Cheaper part of those policies. Costs
of the Orbiter and the Lander were brought down to \$125 million
and \$165 million respectively~\cite{tuite07}. 
This was   much less
than the previous Pathfinder mission (which itself cost slightly
less than \$300 million) and a huge reduction from
the previous
Viking Mars missions  (cost about \$935 million in 1974 Dollars, equivalent
to \$3.5 billion in 1997 dollars). 
The success of these missions would've
strengthen FBC within NASA and JPL, and been
seen to break new ground in terms of mission completion with the
reduced staff and budget~\cite{hardin99}.

Both of these missions 
failed. Using a single contractor had weakened quality assurance
and caused loss of vehicle.
These flaws where software issues that could have easily been
rectified if they had been discovered on the ground (e.g.
a failure to convert from imperial to metric units, causing
the loss of the Climate Orbiter~\cite{nasa99}). The Mars Program
Independent Assessment Team Report~\cite{young00} found that these
missions were under-staffed, under-funded by at least 30\%, and too
tightly scheduled.

Elsewhere, across the Atlantic in the UK, another Mars mission to
deliver a lander, designated the Beagle 2, was under way. This
mission was also developed cheaply, applying the same concepts in
design and implementation that NASA was at the time using. The
lander however was declared lost after not establishing contact
after separation from the mars express vehicle~\cite{beagle2}.

One other failure that FBC was blamed for was the Columbia Shuttle
disaster in 2003. This was post-Goldin, at a point where NASA had
realized the excessive cost cutting and staff reducing policies
needed to be changed. After that disaster, critics quickly pointed
the finger to these missions being under funded due to FBC. There
were many calls, especially politically, for throwing FBC ``in the
waste basket"~\cite{cowig03,ifpte03}.

\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.95}
\begin{center}
\scriptsize
\begin{tabular}{c|lrr|lr}
       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{values}\\\hline
project&feature&low&high&feature&setting\\\hline
&prec&1&2&data&3\\
OSP:&flex&2&5&pvol&2\\
Orbital&resl&1&3&rely&5\\
space&team&2&3&pcap&3\\
plane&pmat&1&4&plex&3\\
&stor&3&5&site&3\\
&ruse&2&4&&\\
&docu&2&4&&\\
&acap&2&3&&\\
&pcon&2&3&&\\
&apex&2&3&&\\
&ltex&2&4&&\\
&tool&2&3&&\\
&sced&1&3&&\\
&cplx&5&6\\
&KSLOC&75&125\\\hline
&prec&3&5&flex&3\\
OSP2 &pmat&4&5&resl&4\\
&docu&3&4&team&3\\
&ltex&2&5&time&3\\
&sced&2&4&stor&3\\
&KSLOC&75&125&data&4\\
&&&&pvol&3\\
&&&&ruse&4\\
&&&&rely&5\\
&&&&acap&4\\
&&&&pcap&3\\
&&&&pcon&3\\
&&&&apex&4\\
&&&&plex&4\\
&&&&tool&5\\
&&&&cplx&4\\
&&&&site&6\\\hline
&rely&3&5&tool&2\\
JPL &data&2&3&sced&3\\
flight&cplx&3&6&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&7&418&&\\
\end{tabular}
\end{center}
\caption{Three case studies.
Numeric values 
$\{1,2,3,4,5,6\}$ map to $\{very low, low, nominal,
high, very high, extra high\}$.
The terms in column 2 come from \fig{cparems}.}\label{fig:cases}
\end{figure}\section{Case Studies}
Despite the above, this paper advocates FBC using the case studies of 
\fig{cases}.
These studies represent
the NASA flight software, at increasing levels of specificity:
\bi
\item {\em Flight} is a general description of flight software at NASA's Jet Propulsion Laboratory. 
\item {\em OSP} is a specific flight system: the GNC (guidance, navigation, and control)
component of NASA's 1990s {\em Orbital Space Plane}; 
\item {\em OSP2} is a later version of OSP.
\ei

\fig{cases} describes the details of flight, OSP, and OSP2.
Note that \fig{cases} does not mention all the  features listed in \fig{cparems}
inputs.  For example, our defect predictor has inputs for
use of {\em automated analysis}, {\em peer
reviews}, and {\em execution-based testing tools}. For all inputs
not mentioned in \fig{cases}, values are picked at random from the full
range of \fig{cparems}.

One  aspect to note from \fig{cases} is the number of open options
{\em not} specified in the description of the projects.
Some of the features in \fig{cases}
are known precisely (see all the features with single {\em values}).
But  many of the features in \fig{cases} do not
have precise values
(see all the features that {\em range} from some {\em low} to {\em high}
value).  Sometimes the ranges
are very narrow (e.g., the process maturity of JPL ground software is between 2
and 3), and  sometimes the ranges are very broad.
In fact, our case studies can be ranked
\[
flight > OSP > OSP2
\]
according to the options open to
a  project manager:
\bi
\item
In the case of flight systems, the description is very general and managers
have many options. \item
In the case of OSP2, most of the project options are
pre-determined and project managers have very little opportunity  to effect
the course of a project.
\item  OSP is an early version of OSP2 and, measured in terms of the number
of open options, falls in between flight and OSP2.
\ei 
As we shall, the number of open options in a project will
have an important impact on our results.

\section{STAR}
STAR uses \fig{cases} as the inputs to a Monte Carlo simulation over a set of software models.
STAR contains the COCOMO effort $E$ estimator~\cite{boehm00b}
but also the COCOMO development months $M$ estimator~\cite[p29-57]{boehm00b},
and COQUALMO $D$ defects
estimator~\cite[p254-268]{boehm00b},
These estimator generate the $\{E,M,D\}$ variables used by
 \eq{score} in the introduction.

We base our analysis on COCOMO and COQUALMO for several reasons.
These are mature models which have been
developed, refined, and constrained over a very long period of time.
The range of tuning options
explored by STAR  are taken from 30 years of modeling experience and regression
studies of hundreds of projects~\cite{boehm00a}.
COCOMO and COQUALMO 
have been selected 
and tested by a large community of academic and industrial 
researchers led by Boehm (this large group has meet annually since 1985).
Unlike other models such as PRICE TRUE PLANNING~\cite{park88}, 
SLIM~\cite{putnam92}, or SEER-SEM~\cite{jensen83},
the COCOMO family of models
are fully described in the literature. 
Also, at least
for the effort model, there exist  baseline results~\cite{chulani99}. 
Further,
we work extensively with government agencies writing software.
Amongst those agencies, these models are frequently used to generate
and justify budgets.

But the most important reason we use COCOMO \& COQUALMO is that
the space of possible tunings within these models
is well defined. Hence, it is possible
to explore the space of possible tunings.
Recall from \eq{coc} that the COCOMO model includes
$\{A,B,\alpha,\beta\}$ tuning values.
Many of these variables are shared with the
COQUALMO defect predictor which also
 has a separate set
of tuning variables, which we will call $\gamma$.
Using 26 years of  publications about COCOMO-related models,
we inferred the  minimum and maximum values yet seen  for 
$\{A,B,\alpha,\beta,\gamma\}$. For example,  the $A,B$ min/max values come from \eq{ab}.
We use the variable $T$ to store
the range of possible values for these tuning variables.

STAR runs as follows.
First,  a project $P$ is specified as a set of min/max ranges to the input variables of
STAR's models:
\bi
\item
 If  a variable is known to be exactly $x$, then
then \newline
\mbox{$min=max=x$}. 
\item
Else, if a variable's exact value is not known but the range of possible values
is known, then min/max is set to the smallest and largest value in that range 
of possibilities.
\item
Else, if  a variable's value  is completely unknown  then min/min is set to the full
range of that variable in \fig{cparems}. 
\ei

Second, STAR's simulated annealer\footnote{Simulated annealers randomly
alter part of the some $current$ solution. If this $new$ solution scores better
than the current solution, then $current=new$. Else, at some probability
determined by a temperature variable, the simulated annealer may jump to a sub-optimal
$new$
solution.
Initially the temperature is ``hot'' so the annealer jumps all over the solution
space. Later, the temperature ``cools'' and the annealer reverts to   a
simple hill climbing search that only jumps to new better solutions.
For more details, see ~\cite{kirkpatrick83}.}
seeks constraints on the project options $P$ that most reduce the score of \eq{score}
(for examples of $P$, see \fig{cases}).
A  particular subset of $P'\subseteq P$ is scored by using $P'$
as inputs to the COCOMO and COQUALMO. When those predictive models run, variables are selected
at random from the min/max range of possible tunings $T$ and  project
options $P$. 

In practice,
the majority  of the  variables in $P$ can be removed without effecting the score;
i.e. our predictive models exhibit a {\em keys effect} where a small number of variables control
the rest~\cite{me07d}. Finding that minimal set of variables is very useful for management
since it reveals the {\em least} they need to change in order to {\em most} improve the 
outcome.
Hence, after simulated annealing, STAR takes a third step.

In this third step, a
Bayesian
sensitivity analysis finds the  smallest subset of $P'$ 
that most effects the output. The scores seen during simulated annealing are sorted
into the (10,90)\% (best,rest) results. Members of $P'$ are then ranked by their Bayesian
probability
of appearing in $best$. 
For example, $10,000$ runs of the simulated annealer can be
divided into 
1,000 lowest  {\em best} solutions and 9,000
{\em rest}.  If the  range $rely=vh$ might appears 10 times in the $best$ solutions, but only
5 times in the $rest$ then: \newline
\begin{minipage}{\linewidth}\scriptsize
\begin{eqnarray}\nonumber
E&=& (reply=vh)\\\nonumber
P(best)&=&1000/10000 = 0.1\\\nonumber
P(rest)&=&9000/10000 = 0.9\\\nonumber
freq(E|best)&=&10/1000 = 0.01\\\nonumber
freq(E|rest)&=&5/9000 = 0.00056\\\nonumber
like(best|E)&=&freq(E|best) \cdot P(best)=0.001\\\nonumber
like(rest|E)&=&freq(E|rest) \cdot P(rest)= 0.000504\\\label{eq:ten}
P(best|E)&=& \frac{like(best|E)}{like(best|E) + like(rest|E)}=0.66
\end{eqnarray}
\end{minipage}
\eq{ten} 
is a poor ranking heuristic
since it is distracted by low frequency evidence.
For example, note how
the probability of $E$ belonging to the best class
is moderately high even though its support is very low; i.e. $P(best|E)=0.66$ but
$freq(E|best) = 0.01$. 
To avoid such unreliable low frequency evidence, we augment \eq{ten}
with a support term.
Support should $increase$ as the frequency of a range $increases$,  i.e. 
$like(x|best)$ is a valid support measure.
STAR1 hence ranks ranges via
\begin{equation}\label{eq:ps}\small
P(best|E) * support(best|E) = \frac{like(x|best)^2}{like(x|best) + like(x|rest)}
\end{equation}
\input{r1}
After ranking members of $P'$,
STAR then imposes  the top $i$-th 
ranked items of $P'$ on the predictive model inputs, then running the models 100 times.
This continues until the scores seen using $i+1$ items is not statistically
different to those seen using $i$ (t-tests, 95\% confidence).
STAR returns items $1..i$ of $P'$ as the {\em least} set of project decisions that
{\em most} reduce effort, defects, and development time.
We call these returned items the {\em policy}. 

Note that STAR constraints the project options $P$ but never the tuning options $T$.
That is, the {\em policy} generated by STAR contains 
parts of the project options $P$ that most improve the score, despite variations
in the tunings $T$.
This approach has the advantage that it can reuse
COCOMO models without requiring local tuning data.

Previously~\cite{me08c} we have shown that this approach
(that does not use local tuning)
generates estimates very similar
to those generated by ``LC'' method proposed by Boehm
(that does tune the model to local data)~\cite{boehm81}.  We have explained
this effect as follows. Uncertainty in the project options $P$ and the tuning
options $T$ contribute to uncertainty in the estimates generated by
STAR's models. However, at least for the COCOMO and COQUALMO models
used by STAR, the  uncertainty created by $P$ dominates that of $T$. Hence,
any uncertainty in the output can be tamed by constraining $P$ and not $T$.

\section{Results}
\subsection{Format of Results}
\fig{results} shows the defects, months, and effort estimates seen imposing the {\em policy} learned by STAR:
\bi
\item The results have divisions for  defects, months, and effort.
\item Division are sub-divided into
results for
flight, OSP, and OSP2.
\ei
Within each sub-division,
the rows are sorted by median scores.
The ``Do nothing'' row  comes from  Monte Carlo simulations over the project range $P$,
 without any restrictions.

The {\em rank} results shown in column three  show the results
of a statistical comparison of each sub-division.
Two rows have the same rank 
if there is 
no statistical difference in their distributions. 
We use Mann-Whitney for this comparison for the following reasons:
\bi
\item 
The random nature of
Monte Carlo simulations, the inputs to each run are not paired;
\item Ranked tests make no, possibly inappropriate,
 assumption about normality of the results.
\ei
Each row shows
results from 100 
calls to \eq{score}:
\bi
\item Results in each division are  normalized 0..100, min..max.
\item Each row shows the 25\% to 75\% quartile range of the normalized
scores 
collected during the simulation.
\item The median result is shown as a black dot. 
\ei
All the performance scores (effort, months, defects) get {\em better} when
the observed scores get {\em smaller}; i.e. move over the left. 
\subsection{Observations and Recommendations}
Three aspects of \fig{results} deserve our attention.

Firstly,
it is almost always true that some optimizations on any pair or triple
from ``Faster, Better, Cheaper'' can reduce defects {\em and} months {\em and} effort
from the levels seen in the baseline ``do nothing'' scenario.
This result argues for the use of tools like STAR.

Secondly, STAR is most useful when applied
to projects
with many project options.
As evidence of this, 
note how in \fig{results} that  as we move from flight to OSP to OSP2, the median performance
scores get worse.
In order to explain this effect, we repeat remarks made above:
our case studies are sorted 
in decreasing order of ``number of open options''
(there is
much that can be adjusted within the general description of flight systems;
fewer adjustments options are possible in OSP; and even fewer adjustments are possible in OSP2).
As we {\em decrease} the number of open options, our ability to find ``fixes''
to the current project also decreases. 

Thirdly, the goal of faster {\em and} better {\em and} cheaper is not overly ambitions.
Prior to running these experiments, we believed that
to
optimize for three criteria, it may be sometimes necessary
to accept non-minimal results for one of the criteria. 
However, contrary to our expectations, we observe 
that FBC achieves the best (lowest) median results in the case of:
\bi
\item defects for OSP2 and 
\item months for flight systems and 
\item effort for flight systems 
\ei
In the remaining cases,
FBC is statistically indistinguishable from the best (lowest) median result
in the case of 
\bi
\item defects for flight systems and 
\item effort for OSP
\ei
Lastly, in the remaining cases,
FBC's median is within 5\% of the best (lowest) median result in the case of
\bi
\item defects for OSP
\item months for OSP
\ei
That is, 
FBC achieves minimum (or very close to
minimum)  values.
in $\frac{7}{9}$ of \fig{results}'s division, 
Hence, we endorse STAR's default setting of $b=f=c=1$; i.e. try to optimize on all three criteria.
\section{Conclusion}
Our results show that
FBC can be achieved with minimal
compromises on individual criteria. How can we reconcile this result with 
NASA very negative experience with FBC?

One thing forgotten about FBC is that, usually, it worked.
Despite all the criticism against it,
FBC
successful/partially successful in 136 of the total of 146 missions
launched during the period that Goldin was administrator. This would
be called an overall success if it hadn't been for the largely
publicized failures. That is, FBC was mostly a technical success, but a PR failure~\cite{turner03}.

However, one way to explain the very large failures within the FBC program
is to speculate that, sometimes, FBC was a front for  CF (i.e.. cheaper and faster, as the expense of
quality).
\fig{results} shows the disastrous effects of CF:
\bi
\item In the case of OSP CF's defects where $\frac{62}{9}=689\%$ worse
than the worst policy seen using any of BC, BF, or BFC.
\item In the case of OSP2 CF's defects where $\frac{97}{11}=881\%$ worse
than the worst policy seen using any of BC, BF, or BFC.
\ei
Some of the decisions made under the banner of FBC are questionable; i.e.
staff reductions leading to 
loss in veteran engineers and managers to retirement and 
causing experienced managerial staff to be stretched too thin given
tight scheduling~\cite{young00}. This forced projects to use
inexperienced managers which caused management mix ups and human
error.

Like Spear~\cite{spear00} , we would endorse FBC, but under the condition that it is better managed.
Tony
Spear, a JPL veteran engineer from 1962 to 1998, testified
to the possible effectiveness of FBC.
Despite mentioning problems with FBC
(a fixation on cost, causing cost cuts that
were too much for $2^{nd}$ generation FBC projects),
he recommended {\em not} to discard it. Rather, he argued for a
more focused way of implementing it, concentrating on aspects such
as
building and retaining talent, taking
advantage of advancements in technology such as the Internet, and
advancing methods used in project development and verification~\cite{spear00,spear00(2)}. 

To Spear's recommendation  we would add that when applying FBC, never surrender the quest of ``better''. 
Observe how, in \fig{results}, whenever we optimize for ``Better'' using $b=1$, we always reduce
defects by about an order of magnitude over the baseline ``do nothing'' result. 
The only time that our optimizer does not reduce defects is when the ``better'' utility
is set to zero (see the  CF detect results). 
Hence, we recommend always setting $b=1$.
\newpage
%
%\section{Discussions}
%More importantly than the above are the discussions we have about the models. The following section
%is not a formal analysis. Rather, it summarizes the common dialogues we have
%when we present the above results.  
%
%One reaction we get is {\em over-acceptance}. We have found one audience that are uncritical of our work
%and want to implement our results, immediately. Typically, this audience is formed of managers
%who are nervous of their current practices and aver (overly) eager to change to any other approach.
%
%While this is a gratifying response, we have found that
%members of this audience may  not fully appreciate a important part of the above conclusions; namely that 
%any policy that looks useful in general may not be useful in particular cases. Before implementing
%any of the above, we strongly recommend that users use our tools to find specialized policies that do
%best for their particular site.
%
%Another reaction we get is our conclusions come from untuned COCOMO models and {\em the above policies only
%make sense after local tuning}. In reply, we offer separate comments on the mean and variance of the estimates
%generated by our approach.
%
%{\em Issues with Mean:} The $experiment=1$ results of \fig{delta} shows the differences between the estimates
%generated {\em with} and {\em without} tuning on local data (the no-tuning estimates came from our search-based tool and the
%tuning-results can from
%on-line versions of the standard USC models).
%The $experiment=2$ results of that figure show
%the mean errors seen 
%when estimates are generated using conventional methods. Note that the mean differences in the $experiment=1$
%results are small (36\% or less).
%Further,
%the mean effort values of the conventional methods is larger (never less than 42\%).
%That is, \fig{delta} shows no evidence that the differences between our methods and conventional methods
%is inordinately large.
%
%{\em Issues with Variance:}
%Our tools sample across the space of possible tunings. The variance of the conclusions reached in that sample
%could be reduced by the use of local data to pre-tune our models, before we perform any sampling.
%However, it must be noted that the variances of our conclusions are quite small (see \fig{r1} and \fig{delta}).
%In summary, while we believe it may be useful to tune with local data, the overall benefit on reducing the variance or the mean  not be large.
%
%
%\begin{figure}
%\small
%\begin{tabular}{r|rrrr}
%	            &          & \multicolumn{3}{c}{data set}\\\cline{3-5}
%                &experiment&       flight        & OSP& OSP2\\\hline
%        defects &1&  30 $\pm$ 23   &    39  $\pm$ 33 &32 $\pm$ 18\\
%         months &1&       26 $\pm$ 17   &  23 $\pm$ 17    &  12 $\pm$ 7\\
%        effort  &1&    36 $\pm$ 31   &     20 $\pm$ 17&  18 $\pm$ 13\\\hline
%	     effort &2&        41 $\pm$40    & 44 $\pm$ 63 &   42 $\pm$ 57
%\end{tabular}
%     \caption{Results (Mean $\pm$ standard deviations) from our experiments.
%For an explanation of the method by which this was generated, see \fig{deletahow}.
%}\caption{fig:delta}
%\end{figure}
%\begin{figure}
%\small
%\begin{center}
%\begin{tabular}{|p{.95\linewidth}|}\hline
%The lines labeled $experiment=1$ in \fig{delta} were generated as follows:
%\be
%\item
%Impose the constraints $C$ of \fig{cases}
%on our search-based tool; 
%\item Learn a policy $P$; 
%\item
%Impose the combination of $C\cup P$ on our search-based tool;
%\item
%Generate 100 {\em estimates} (without further searching); 
%\item
%Generate {\em baselines}
%from the  standard on-line USC
%models\footnote{Phil: URLs for those models} 
%by picking values at random from within $C$ and setting any unknown values to "nominal"; 
%\item
%Combine the {\em estimates} from step 4 with the {\em baselines} from step 5
%using $delta= \frac{estimate - baseline}{baseline}$. 
%\ee
%The line labeled $experiment=2$ in \fig{delta}
%was generated by taking flight systems/OSP/OSP2 systems
%and estimating them using a COCOMO model tuned to some NASA COCOMO, as follows:
%\bi
%\item
%Using the NASA93 COCOMO data set\footnote{\url{http://promisedata.org/?s=nasa93}}
%we tuned a COCOMO model using
%using Boehem's LC method~\cite{boehm81};
%\item
%Using the  constraints $C$ of \fig{cases} we generated 100 random projects
%at random consistent with $C$; 
%\item Effort estimates for the generated projects were  generated using the tuned
%COCOMO model.  
%\item The means and standard deviations for those estimates were recorded in \fig{delta}.
%\ei\\\hline
%\end{tabular}
%\end{center}
%\caption{Generation method for the \fig{exp2} results.}\label{fig:deltahow}
%\end{figure}
%\fig{cases} and learned policies  
%A third reaction is that our results are {\em biased by the choice of model}.  The models used above
%accept input from a very limited number of variables (just those listed in \fig{variables}).
%COCOMO's variables are both the strongest and weakest part of the technique. On the one hand, they 
%have been selected and tested by a large community of academic and industrial researchers led by Boehm. 
%This group meets annually at the COCOMO forums (large meetings that have been held annually since 
%1985). On the other hand, these features may not be available in the databases of a local site. Hence, 
%regardless of the potential value added of using a well-researched feature set, those features may not be 
%available. 
%
%In reply, we observe that the whole point of our method is that we do not have training data.
%
% that we do not require training we have found that with minimal training, we can collect data in the form  
%
%COCOMO is crap.
%
%COCOMO is adequate and there are significant problems in generate a better-than-adequate solution (data drought).
%Given the pressing demands of project managers for answers to their control dilemmas, we need to offer the
%best solution we can, today, using the currently available information. The USC models and, in particular, the COCOMO
%model is one of the few examples of a long standing software engineering result in the literature. First proposed
%in 1981, it was extensively revised in the late 1990s.  It is widely used, is well-documented, and there exist
%numerous empirical studies describes its strengths (and flaws).
%
%\section{Discussion}
%The above results seem to contradict the NASA experience. We saw above how FBC was blamed for NASA mishaps.
%Yet our STAR results suggest that FBC is not inherently a bad policy for all projects. 
%

\bibliographystyle{abbrv}
\bibliography{refs,myrefs}
\end{document}
