% This is "sig-alternate.tex" V1.8 June 2007
% This file should be compiled with V2.3 of "sig-alternate.cls" June 2007
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.3 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.3) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
 %\CopyrightYear{2009} will cause 2007 to appear in the copyright line.
% \crdata{978-1-60558-036-4/08/05}% will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
% Technical questions _only_ to
% Gerald Murray (murray@acm.org)
% ===============================================================
%
% For tracking purposes - this is V1.8 - June 2007

\documentclass{sig-alternate}
\usepackage{times}
\usepackage{url}
\usepackage{cite}
\usepackage{colortbl}
%\newcommand{\boxplot}[5]{~}
\input{boxplot}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newcommand{\be}{\begin{smallenum}}
\newcommand{\ee}{\end{smallenum}}
\newcommand{\tion}[1]{\S\ref{tion:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newenvironment{smallitem}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{itemize}
   \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{itemize}}
\newenvironment{smallenum}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{enumerate}
  \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{enumerate}}

\begin{document}
%
% --- Author Metadata here ---
%\conferenceinfo{PROMISE}{'09 Vancouver,  Canada USA}
\conferenceinfo{~}{~}
%\CopyrightYear{2007} % Allows default copyright year (200X) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{Can We Build Software  Faster \underline{and} Better \underline{and} Cheaper?
%\titlenote{For a prior draft, see \url{http://menzies.us/pdf/09bfc.pdf}
%\titlenote{
% This research was conducted at WVU and the Jet Propulsion Laboratory under a NASA sub- 
%contract. Reference herein to any specic commercial product, process, 
%or service by trade name, trademark, manufacturer, or otherwise, does not 
%constitute or imply its endorsement by the United States Government.
%}
}%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
% For esthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'esthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{1} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Tim Menzies, Oussama El-Rawas\\
       \affaddr{CS\&EE,  West Virginia University,  USA}\\
       \email{oelrawas@mix.wvu.edu, tim@menzies.us}
% 2nd. author
%\alignauthor
%Jairus Hihn \\
%       \affaddr{Jet Propulsion Laboratory}\\
%       \affaddr{California, USA}\\
%       \email{jairus.hihn@jpl.nasa.gov}
%\alignauthor
%Barry Boehm\\
%       \affaddr{Computer Science Dept.}\\
% 	\affaddr{Uni. S. California, USA}\\
%      \email{boehm@sunset.usc.edu}
}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for esthetic reasons that you place these 'additional authors'
% in the \addiytional authors block, viz.
\date{\today}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
In this paper we seek to further support the viability of FBC, by analyzing its behavior in best and worst case decision making scenarios and comparing that to the other derivate "you pick two" policies. More generally, we seek to utilize the same stochastic AI tool previously used in order to determine the best and worst case scenarios for certain project test cases, all while circumventing model tuning. We find that our tool, STAR, is able to reliably determine worst case scenarios, without the need for model fine tuning. We also provide further support to the viability of FBC by observing the policy's success and failure in various test cases.

%Based on this Analysis, we seek to draw a broader statement concerning using software engineering models with minuscule amounts of tuning data for decision making in projects.

%In this paper, we will attempt to view and evaluate the potential for failure of FBC on different projects, and comparing that to ``pick any two'' variants of FBC. This will be done using the AI search tool called STAR that we have previously used. We find that FBC is the most rounded approach among all its variants, where it will fail more elegantly than the other variants, while also achieving very good overall performance when it comes to optimizing for defects, development time (months), and effort.
\end{abstract}

% A category with the (minimum) three required fields
%\category{B.4.8}{Programming techniques}{Performance}{Modeling and prediction}
%\category{I.6.4}{Computing Methodologies }{Model Validation and Analysis}
\keywords{software engineering, predictor models, COCOMO, Faster Better Cheaper, simulated annealing, software processes}

\section{Introduction}
%Previously,
%PROMISE researchers have  used {\em induction} (summary of data into a model)
%to generate predictor models for software engineering. While useful,
%induction 
%has several drawbacks:
%\bi 
%\item  It only explores half the story. As Murray Cantor observed in his PROMISE 2008 keynote, 
%once predictive models are generated, they 
%are used within some business context. While we need more papers on predictive model {\em generation},  it may be time
%to ask the PROMISE community to
%write
%more papers on predictive model {\em usage}.
%\item
%Automatic induction assumes the existence of data. 
%Many
%domains are {\em data starved} where  it is hard to 
%obtain local data. 
%\item
%More generally, we ask the question:
%why do we persist on generating  new predictive models all the time? If there is any generality in software engineering
%it should be possible to reuse predictive models. We should at least experiment with
%this approach rather than always assuming the best predictive model is a new model.
%\ei

``Faster, Better, Cheaper'' (FBC) was a development philosophy
adopted by the NASA administration in the mid to late 1990s. 
that lead to some
some dramatic successes such as Mars Pathfinder as well as a number highly publicized mission failures, such as 
the Mars Climate Orbiter \& Polar Lander. It was later on blamed for several project failures at NASA.

Previously \textbf{[INSERT CITATION HERE]} we have explored n using STAR and other AI search methods in 
reusing USC COCOMO based models in the absence of sufficient data for tuning these models 
to the particular projects and project templates that we being studied. In these 
studies we've explored the our ability to make good decisions that would improve
our test project by reducing the effort, defects and development time of a project.

%Hence, we've explored predictive
%model {\em use} and {\em reuse} rather than predictive model {\em
%generation}. We found that model reuse is possible despite the state of extreme data starvation
%that these and other models suffer from.  
%Our STAR tool~\cite{elrawas08,me08c,me09a}  uses AI methods
%to conduct large scale ``what-if''
%queries over software predictive models.  
%If parts of a predictive models are not known with certainty,
%STAR seeks stable conclusions within the space of possible values.

%In previous publications~\cite{me07f}, we had explored the benefits to using STAR for the purpose of presenting a concise strategy to software project managers. We have also previously assessed the infamous ``Faster, Better, Cheaper'' (FBC)
%development practices used at NASA in the 1990s. In this paper, we attempt to explore FBC's potential to failure in different kinds of projects and scenarios, and comparing it to that of the variants of FBC. The more general study would be the ability of AI methods to pinpoint the worst case scenarios for projects when evaluating them with data starved and untuned software engineering models.

FBC was advocated in the 1990s by the then-administrator of NASA, Daniel Goldin, as a method for reducing the 
expenditure of NASA. FBC  was in-line with the direction that the Clinton administration's approach 
of doing more for less. FBC was initially successful: projects that usually cost over a billion were implemented
at $\frac{1}{4}th$ of that cost (e.g. Mars Pathfinder). However,
subsequent 
failures (Mars Climate Orbiter and Polar Lander; the  Columbia Shuttle disaster) 
lead to much criticism of FBC.

This paper reports the results of such a comparison.
After a discussion of related work, we will
review the sad history of ``Better, Faster, Cheaper'' at NASA. 
We will then describe the case
studies, as well as the STAR system.  
%In the subsequent results section we will offer the 
%surprising observation that, contrary to the prevailing wisdom at NASA,
%FBC is not necessarily a disastrous development methodology. In fact, in $\frac{7}{9}$
%of our studies, 
%\begin{quote}{\em
%FBC produces results that are within 5\% of the minimum estimates generated by any of BF,BC,CF. 
%}\end{quote}
%Our results suggest that 
Previous results presented in \cite{menzies09} suggested that 
(a) FBC is in fact a viable development
methodology and (b) suggested that when applied properly, can indeed produce similar or better results
compared to the "you pick two" policies in a variety of project cases. 


%what really went wrong with FBC at NASA was that it was used as a  front for CF; i.e. built it faster and
%cheaper, while almost disregarding quality considerations.

%New results relating to our new tests suggests that FBC fails more elegantly than the other derivate development methods. This is evidenced by how bad policies result in more balanced repercussions across defects, months, and effort. This is compared to how other methods fail badly in one area.

Building on that study, this paper presents this new study of FBC using the Previously used AI tool ``STAR'' \textbf{[INSERT CITATION HERE]}. It is found that, even if FBC is applied properly to a project, project managers need to be weary of making bad decisions. Also, bad decisions that are made while applying FBC affect every aspect of the project, increasing the damage done to a project.  In addition to all that, we show that STAR is able to reliably show which combinations of policies are the worst for a certain project, showing that not only are we capable of getting the best policies, but also the worst of them using stochastic AI methods applied to data starved software engineering models with medium granularity.

Previous papers by Menzies et. al explored the validity of the general consensus on ``Faster, Better, Cheaper? Pick any two''. It was found that ``Faster, Better, Cheaper'' (FBC) is indeed feasible when maintaining a balanced concern and concentration on the quality aspects of a project. This paper extends previous publications by the authors \textbf{[INSERT CITATION HERE]} in several ways:

\bi
\item
\item
\item
\ei

The following sections are organized in the following fashion: 

%Our more general conclusion is that
%we can make
%more {\em use} and {\em reuse} of predictive models.
%If stable conclusions exist across the tuning space, then
%AI tools like STAR can offer conclusions from predictive models, even when their is
%insufficient data for tuning.  This is an powerful aspect of AI tools
%like STAR since
%we often lack the
%data required to precisely tune an imported model to a local
%domains.  
%It is hoped that that this paper will inspire more papers at future
%PROMISE conferences that explore not just model {\em generation},
%but also the {\em reuse} of those generated model  in new domains.
%To facilitate that discussion, we have made all our code openly
%available\footnote{\url{http://unbox.org/wisp/tags/STAR/2.1/}}
%and we are building a web interface to
%an executable version of
%STAR\footnote{\url{http://nova.unbox.org}}. That interface will be on-line by
%the time of the PROMISE'09 conference.

%\textbf{INSERT POSSIBLE CONCLUSION HERE}

\section{``Faster, Better, Cheaper''}\label{tion:FBC}
This paper  applies STAR to an analysis of FBC.
In the 1990s, the main approach to implementing FBC within NASA was to down size
projects and reduce their cost and complexity, concentrating on
producing missions in volume. Reducing funding naturally meant that
less verification and testing was possible within budget and schedule
constraints. The reasoning behind this however was to be able to
produce a larger volume of unmanned missions, which would counteract
the expected higher rate of mission failure. This would, optimally,
yield more successful missions as well as more scientific data
produced by these projects. Another focus in this policy was allowing
teams to take acceptable risks in projects to allow for cost
reduction, and possibly using new technology that could reduce cost
while possibly providing more capabilities. This was accompanied
by the new view, being pushed at NASA by Goldin,  that
``it's ok to fail'' \cite{spear00(2)}, which was rather misunderstood.
This new policy was meant to eliminate huge budget missions of the
past, that upon possible failure would yield large losses. Project
cost used to routinely exceed the \$1 billion mark, while the first
FBC project, the Mars Pathfinder, was completed for a fraction of
the cost, netting at about \$270 million \cite{david00}.

Some within NASA, like 30 year veteran Frank Hoban, supported
these policies~\cite{david00} who
viewed these new policies as a
necessary break from traditional policies that were very risk averse.
The additional cost reduction, accompanied by the additional risk,
was to allow for a path to cheap and commercial space flight. Even
given the reduced funding, the Mars Pathfinder mission, along with
other first generation FBC missions, were successes. This fueled
enthusiasm to apply FBC across all of NASA to further reduce
spending per mission
 as well cutting the
work force by one third. FBC was extended to be applied on manned
space missions as well, where funding was also reduced. Coming into
a space shuttle program that was starting to age and in need of
updates, the new policies imposed cuts in funding from 48\% of the
NASA budget to 38\% \cite{key03}, further straining that program.
Further more, a single prime contractor (Lockheed Martin) was used
for missions in another bid to reduce cost and managerial
complexity~\cite{young00, NASAwatch}.

This produced opposition within NASA, where traditionally issues
pertaining to the shuttle were designated LOVC (Loss of Vehicle and
Crew) and given priority over all other issues, including cost.
However the cost cuts and layoffs that ensued damaged morale leading
to a string of early retirements of
veteran scientists, skilled engineers and managers\cite{key03}.

Despite this, additional projects were planned including
Mars Climate Orbiter and Polar Lander. These two
projects were more aggressive implementations of FBC, especially
when it came to the Faster-Cheaper part of those policies. Costs
of the Orbiter and the Lander were brought down to \$125 million
and \$165 million respectively~\cite{tuite07}. 
This was   much less
than the previous Pathfinder mission (which itself cost slightly
less than \$300 million) and a huge reduction from
the previous
Viking Mars missions  (cost about \$935 million in 1974 Dollars, equivalent
to \$3.5 billion in 1997 dollars). 
The success of these missions would've
strengthen FBC within NASA and JPL, and been
seen to break new ground in terms of mission completion with the
reduced staff and budget~\cite{hardin99}.

Both of these missions 
failed. Using a single contractor had weakened quality assurance
and caused loss of vehicle.
These flaws where software issues that could have easily been
rectified if they had been discovered on the ground (e.g.
a failure to convert from imperial to metric units, causing
the loss of the Climate Orbiter~\cite{nasa99}). The Mars Program
Independent Assessment Team Report~\cite{young00} found that these
missions were under-staffed, under-funded by at least 30\%, and too
tightly scheduled.

Elsewhere, across the Atlantic in the UK, another Mars mission to
deliver a lander, designated the Beagle 2, was under way. This
mission was also developed cheaply, applying the same concepts in
design and implementation that NASA was at the time using. The
lander however was declared lost after not establishing contact
after separation from the mars express vehicle~\cite{beagle2}.

One other failure that FBC was blamed for was the Columbia Shuttle
disaster in 2003. This was post-Goldin, at a point where NASA had
realized the excessive cost cutting and staff reducing policies
needed to be changed. After that disaster, critics quickly pointed
the finger to these missions being under funded due to FBC. There
were many calls, especially politically, for throwing FBC ``in the
waste basket"~\cite{cowig03,ifpte03}. It was these  criticisms that 
turned NASA away from FBC towards policies concentrating on two of the three aspects of FBC.


\section{Background}
%\fig{options}  places STAR in the context of  related work. 
%Predictive models can be build automatically or manually.
%Automatically generated predictive models can be automatically validated using many methods including
%(e.g.) the cross-validation procedures used in much data mining research~\cite{witten05}.
%%Manually generated predictive models,
%%built using human expertise, can be remarkably effective: see  
%%Fenton's  PROMISE'07 paper reported a validation study  of one such predictive model~\cite{fenton07a}).

%As shown top right of \fig{options}, STAR is appropriate
%in domains that lack both domain experts and data.
%Many domains are data starved (see \fig{starve}) in which case
%there is
%neither sufficient data
%to automatically learn predictive models .

%\begin{figure}
%\begin{center}
%\begin{tabular}{c|p{1.5cm}|p{2.8cm}|p{2.8cm}|}\cline{3-4}
%\multicolumn{2}{c|}{~}&              domain experts  & no domain experts\\\cline{2-4}
%&data starved\newline
%(little or \newline no
%data)  
%& build predictive models manually              
%& reuse predictive models from other sites, without tuning;  sample across range of possible options. 
%E.g. STAR.\\\cline{2-4}
%& data poor \newline
%(small 
%amounts
%of data)    &  build predictive models manually; tune
%               with  
%               local data; e.g.~\cite{fenton07a}                   & reuse predictive
%models from other sites; tune with local data; E.g. Boehm's local calibration of COCOMO models~\cite{boehm81}.  \\\cline{2-4}


%& data rich\newline 
%(much data) & \multicolumn{2}{p{5.6cm}|}{build predictive models automatically\newline 
%e.g.~\cite{chen05,me05a,me06f,me08d,jiang08}.
%} \\\cline{2-4}

%\end{tabular}
%\end{center}
%\caption{Predictive model construction options.}\label{fig:options}
%\end{figure}
%\begin{figure}
%\begin{tabular}{|p{.95\linewidth}|}\hline
%\bi
%\item
%Metrics-guru Norman Fenton study data collection~\cite{fenton96} for many years.
%Recently, he has 
%despaired of that approach. At a keynote address  PROMISE 2007\footnote{\url{http://promisedata.org/?cat=130}}, Fenton shocked
%his audience by saying:
%\begin{quote}
%{\em ``....much of the current software metrics research is inherently irrelevant to the industrial mix ...
%any software metrics program that depends on some extensive metrics collection is doomed to failure.''} 
%\end{quote}
%\item
%The COCOMO experience is similar to that of Fenton. After 26 years of trying, the
%COCOMO team has  collected less
%than 200 sample projects for the COCOMO database.
%Also,
%even after two years of effort we were only able to
%add 7 records to a NASA-wide software cost metrics repository~\cite{me06f}.
%\ei\\\hline
%\end{tabular}
%\caption{Evidence for data starvation in SE.}\label{fig:starve}
%\end{figure}

%In the completely opposite circumstance, a domain is data rich. In such domains, automatic
%methods can learn predictive models. There are many examples of work in this area 
%including~\cite{chen05,me05a,me06f,me08d,jiang08}.  
%We recommend automatic methods for data rich domains since manual modeling can be very expensive
%(for
%example,
%Fenton manually built his Bayes nets over a two year period~\cite{fenton07a}).

%In the middle case of \fig{options},  small amounts of local data can be used to tune either:
%\bi
%\item Local
%predictive models either built manually; or 
%\item Predictive models imported from other sites.
%\ei

STAR is a natural tool for assessing policies produced under FBC, since it uses COCOMO based software engineering models that have already been used within NASA. These COCOMO models represent FBC in the following manner: 
\bi
\item \underline{F}aster is represented by the Months model, which estimates the total development months needed for a software project.
\item \underline{B}etter is represented by the Defects model, which estimates the number of delivered defects per KLOC (thousand lines of code).
\item \underline{C}heaper is represented by the Effort model, which estimates the effort that is needed for a software project in person-months, and hence can be used to estimate the cost of the development of the project.
\ei

Note that for all the models, lower is better.
The tool combines estimates with utility weights
$\{f,b,c\}$ (short for 
\underline{F}aster, 
\underline{B}etter,
\underline{C}heaper) using the following formula: \begin{equation}\label{eq:score}
 score = \frac{
\sqrt{
f.M^2 +
b.D^2 +
c.E^2  
}}{\sqrt{f + b + c}} \end{equation} 
This $score$ value represents
the Euclidean distance to the normalized values
of the predictions of development effort ``$E$''; total development months ``$M$'';
and ``$D$'', the number of delivered defects per KLOC. This is the utility function that is used in order to assess any given set of ``policies" that might be presented to be implemented in a given software project.
 Given that we normalize the predictions min..max to
0..1 then \eq{score} has the range one to zero and {\em lower} scores
are {\em better}.
STAR searches for the minimal 
set of project changes that most reduces this $score$.

By adjusting the various values of $(f,b,c)$, we can compare the effects
of methodologies that emphasize different project goals:
\bi
\item
BF =  ``better, faster'' i.e.    $c=0$ and $b=f=1$;
\item
BC = ``better, cheaper''    i.e.       $f=0$ and $b=c=1$;
\item
CF  = ``cheaper, faster''  i.e.   $b=0$ and $f=c=1$;
\item
FBC = ``faster, better, cheaper'' i.e. $b=f=c=1$. 
\ei

For example, Boehm et at.~\cite{boehm00b} advocate a certain
functional form for generating  software development effort estimates.  In that form,
the development effort is linear on a set of effort multipliers
$EM_i$ and exponential on  a set of scale factors $SF_j$:
\begin{equation}\label{eq:coc}\small 
\begin{array}{rcl}
effort&=&A \cdot KSLOC^{B + 0.01\cdot\sum_j{\beta_j}SF_j}
 \cdot \prod_i {\alpha_i}EM_i 
\end{array}
\end{equation}
The particular effort multipliers and scale factors recommended by
Boehm et al. are shown in \fig{cparems}.
While Boehm et al offer default values for the \eq{coc} variables, 
linear regression on local data can tune the $\alpha_i,\beta_j$
values to the particulars of a local site.  Also, if there is
insufficient data for a full tuning of $\alpha,\beta$, then a coarse
grain tuning can be achieved by just adjusting the  $A,B$\footnote{We will use uppercase $B$
to denote the COCOMO linear tuning variable of \eq{coc}
and lower $b$ to denote the business
utility associated with defect predictions of \eq{score}}.
 linear
and exponential tuning parameters.

A problem that has been under-explored in the literature is {\em tuning variance}.
In data starved domains, there is insufficient data to produce precise tunings.
For example, 
At PROMISE 2005, we have reported very large tuning variance
in the post-tuning values of  $\alpha$ and $\beta$~\cite{me05b}.
Baker~\cite{baker07} offers a similar finding.
After  thirty 90\% random samples of that data,
the $A,B$ ranges found during tuning were surprisingly wide:
\begin{equation}\label{eq:ab}\small
\left( 2.2 \le A \le 9.18 \right) \wedge \left(0.88 \le B \le 1.09\right) 
\end{equation} 
We are not the only research group to be concerns about tuning variance.
At PROMISE 2007,
Korte \& Port~\cite{korte08} 
explore the variance of automatically learned effort predictors.
They comment that this variance is large enough to confuse  standard methods for assessing different
predictive model generators.

Since 2005~\cite{chen05,me06d}, we have been trying to reduce tuning variance.
using feature subset selection (FSS).
However, despite years of work, we now report that
FSS reduces but does not tame the variance 
of $A,B,\alpha,\beta$. 

Having 
failed to tame tuning variance,  we have been exploring a new approach.
The STAR tool~\cite{elrawas08,me08c,me09a}.
that we describe below checks
for stable conclusions within  the space of 
possible tunings. 

\section{STAR}
STAR uses \fig{cases} as the inputs to a Monte Carlo simulation over a set of software models.
STAR contains the COCOMO effort $E$ estimator~\cite{boehm00b}
but also the COCOMO development months $M$ estimator~\cite[p29-57]{boehm00b},
and COQUALMO $D$ defects
estimator~\cite[p254-268]{boehm00b},
These estimator generate the $\{E,M,D\}$ variables used by
 \eq{score} in the introduction.

We base our analysis on COCOMO and COQUALMO for several reasons.
These are mature models which have been
developed, refined, and constrained over a very long period of time.
The range of tuning options
explored by STAR  are taken from 30 years of modeling experience and regression
studies of hundreds of projects~\cite{boehm00a}.
COCOMO and COQUALMO 
have been selected 
and tested by a large community of academic and industrial 
researchers led by Boehm (this large group has meet annually since 1985).
Unlike other models such as PRICE TRUE PLANNING~\cite{park88}, 
SLIM~\cite{putnam92}, or SEER-SEM~\cite{jensen83},
the COCOMO family of models
are fully described in the literature. 
Also, at least
for the effort model, there exist  baseline results~\cite{chulani99}. 
Further,
we work extensively with government agencies writing software.
Amongst those agencies, these models are frequently used to generate
and justify budgets.

But the most important reason we use COCOMO \& COQUALMO is that
the space of possible tunings within these models
is well defined. Hence, it is possible
to explore the space of possible tunings.
Recall from \eq{coc} that the COCOMO model includes
$\{A,B,\alpha,\beta\}$ tuning values.
Many of these variables are shared with the
COQUALMO defect predictor which also
 has a separate set
of tuning variables, which we will call $\gamma$.
Using 26 years of  publications about COCOMO-related models,
we inferred the  minimum and maximum values yet seen  for 
$\{A,B,\alpha,\beta,\gamma\}$. For example,  the $A,B$ min/max values come from \eq{ab}.
We use the variable $T$ to store
the range of possible values for these tuning variables.

STAR runs as follows.
First,  a project $P$ is specified as a set of min/max ranges to the input variables of
STAR's models:
\bi
\item
 If  a variable is known to be exactly $x$, then
then \newline
\mbox{$min=max=x$}. 
\item
Else, if a variable's exact value is not known but the range of possible values
is known, then min/max is set to the smallest and largest value in that range 
of possibilities.
\item
Else, if  a variable's value  is completely unknown  then min/min is set to the full
range of that variable in \fig{cparems}. 
\ei

Second, STAR's simulated annealer\footnote{Simulated annealers randomly
alter part of the some $current$ solution. If this $new$ solution scores better
than the current solution, then $current=new$. Else, at some probability
determined by a temperature variable, the simulated annealer may jump to a sub-optimal
$new$
solution.
Initially the temperature is ``hot'' so the annealer jumps all over the solution
space. Later, the temperature ``cools'' and the annealer reverts to   a
simple hill climbing search that only jumps to new better solutions.
For more details, see ~\cite{kirkpatrick83}.}
seeks constraints on the project options $P$ that, normally, most reduce the score of \eq{score}
(for examples of $P$, see \fig{cases}). For finding the worst policies, STAR had to be slightly adapted, where lookied for the policies that most increased \eq{score}.
A  particular subset of $P'\subseteq P$ is scored by using $P'$
as inputs to the COCOMO and COQUALMO. When those predictive models run, variables are selected
at random from the min/max range of possible tunings $T$ and  project
options $P$. 

In practice,
the majority  of the  variables in $P$ can be removed without effecting the score;
i.e. our predictive models exhibit a {\em keys effect} where a small number of variables control
the rest~\cite{me07d}. Finding that minimal set of variables is very useful for management
since it reveals the {\em least} they need to change in order to {\em most} improve the 
outcome.
Hence, after simulated annealing, STAR takes a third step.

In this third step, a
Bayesian
sensitivity analysis finds the  smallest subset of $P'$ 
that most effects the output. The scores seen during simulated annealing are sorted
into the (10,90)\% (best,rest) results. Members of $P'$ are then ranked by their Bayesian
probability
of appearing in $best$. 
For example, $10,000$ runs of the simulated annealer can be
divided into 
1,000 lowest  {\em best} solutions and 9,000
{\em rest}.  If the  range $rely=vh$ might appears 10 times in the $best$ solutions, but only
5 times in the $rest$ then: \newline
\begin{minipage}{\linewidth}\scriptsize
\begin{eqnarray}\nonumber
E&=& (reply=vh)\\\nonumber
Prob(best)&=&1000/10000 = 0.1\\\nonumber
Prob(rest)&=&9000/10000 = 0.9\\\nonumber
freq(E|best)&=&10/1000 = 0.01\\\nonumber
freq(E|rest)&=&5/9000 = 0.00056\\\nonumber
like(best|E)&=&freq(E|best) \cdot Prob(best)=0.001\\\nonumber
like(rest|E)&=&freq(E|rest) \cdot Prob(rest)= 0.000504\\\label{eq:ten}
Prob(best|E)&=& \frac{like(best|E)}{like(best|E) + like(rest|E)}=0.66
\end{eqnarray}
\end{minipage}
\eq{ten} 
is a poor ranking heuristic
since it is distracted by low frequency ($freq$) evidence.
For example, note how
the probability ($Prob$) of $E$ belonging to the best class
is moderately high even though its support is very low; i.e. $Prob(best|E)=0.66$ but
$freq(E|best) = 0.01$. 
To avoid such unreliable low frequency evidence, we augment \eq{ten}
with a support term. In \eq{ps}, likelihood ($like$) is chosen as our support term.
Support should $increase$ as the frequency of a range $increases$,  i.e. 
$like(x|best)$ is a valid support measure since it does exactly so. High support would indicate a higher number of examples that "support" that $E$ can be part of the best set.
STAR1 hence ranks ranges via
\begin{equation}\label{eq:ps}\small
Prob(best|E) * support(best|E) = \frac{like(x|best)^2}{like(x|best) + like(x|rest)}
\end{equation}
After ranking members of $P'$,
STAR imposes  the top $i$-th 
ranked items of $P'$ as model inputs, then runs the models 100 times.
This continues until the scores seen using $i+1$ items is not statistically
different to those seen using $i$ (t-tests, 95\% confidence).
STAR returns items $1..i$ of $P'$ as the {\em least} set of project decisions that
{\em most} reduce effort, defects, and development time.
We call these returned items the {\em policy}. 

Note that STAR constrains the project options $P$ but not the tuning options $T$.
Hence,  STAR's generated {\em policy} contains 
subsets of the project options $P$ that most improve the score, despite variations
in the tunings $T$.
This approach means we can
reuse COCOMO models without using local tuning data. The following is a description that further details the manner in which STAR operates:

\be
\item {\em SAMPLE:}
To sample the ranges from the models,
STAR runs the simulated annealer  $K_1$ times. Note that here, we sample across the ranges of all the attributes. While most of the time we sample randomly across the range, we also have a heuristic optimization called extreme sampling. This form of sampling works in the following manner: for x\% (x is set to 5 by default), STAR samples only the extremums of the attributes.
\item {\em DISCRETIZE:}
The data seen in the $K_1$ samples is then discretized into $D=10$ bins.
Discretization converts a continuous range into a histogram
with $n$ break points $b_1\ldots b_n$ where $\left(\forall i < j : b_i \le b_j\right)$. 
After discretization,
many observations can fall into the same range
between $b_i$ and $b_{i+1}$
at frequency counts $c_i$. This study used equal width discretization; i.e. 
\[\forall i,j:
\left(b_i - b_{i-1}\right) = 
\left(b_j - b_{j-1}\right)\]
\item {\em CLASSIFY:}
The ranges are then classified
into those seen in BEST\% {\em best} or {\em rest}.
\item {\em RANK:}
The ranges are then ranked in increasing order using Support-Based Bayesian Ranking using \eq{ps}.
\item {\em PRUNE:}
Also called the back select stage. STAR runs $K_2$ experiments with the models where the top ranked ranges
$1..X$ ranges are pre-set and the
remaining ranges can be selected at random. 
%In terms of the business case for automated software engineering tools,
%this step is crucial since it would rank new ASE tools alongside 
%other methods. 
\item {\em REPORT:}
STAR returns the $1..X$ settings that optimize the best for the fitness function being used according to the weights applied to effort, defects, development time, and threats. These settings are determined by iterating back from the minimum point achieved towards the first point that is statistically similar to the minimum point. This statistical difference is tested via a standard t-test.
\ee

To run our experiments, we had to apply our engineering judgment to set the parameters. The following are the default values:
\[K_1=10,000, K_2=1,000, D=10, BEST=10\%\]

%\input{stability}
Previously~\cite{me08c} we have shown that this approach
(that does not use local tuning)
generates estimates very similar
to those generated by ``LC'' method proposed by Boehm
(that does tune the model to local data)~\cite{boehm81}.  We have explained
this effect as follows. Uncertainty in the project options $P$ and the tuning
options $T$ contribute to uncertainty in the estimates generated by
STAR's models. However, at least for the COCOMO and COQUALMO models
used by STAR, the  uncertainty created by $P$ dominates that of $T$. Hence,
any uncertainty in the output can be tamed by constraining $P$ and not $T$.

%Note that for determining the worst policies for a certain project, we use a different mode within STAR that modifies the target function to choose the worst policies.


\section{Case Studies and Methods}
%Despite the above, this paper advocates FBC using the case studies of 
%\fig{cases}.
\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.95}
\begin{center}
\scriptsize
\begin{tabular}{c|lrr|lr}
       &\multicolumn{3}{c|}{ranges}      &\multicolumn{2}{c}{values}\\\hline
project&feature&low&high&feature&setting\\\hline
&prec&1&2&data&3\\
OSP:&flex&2&5&pvol&2\\
Orbital&resl&1&3&rely&5\\
space&team&2&3&pcap&3\\
plane&pmat&1&4&plex&3\\
&stor&3&5&site&3\\
&ruse&2&4&&\\
&docu&2&4&&\\
&acap&2&3&&\\
&pcon&2&3&&\\
&apex&2&3&&\\
&ltex&2&4&&\\
&tool&2&3&&\\
&sced&1&3&&\\
&cplx&5&6\\
&KSLOC&75&125\\\hline
&prec&3&5&flex&3\\
OSP2 &pmat&4&5&resl&4\\
&docu&3&4&team&3\\
&ltex&2&5&time&3\\
&sced&2&4&stor&3\\
&KSLOC&75&125&data&4\\
&&&&pvol&3\\
&&&&ruse&4\\
&&&&rely&5\\
&&&&acap&4\\
&&&&pcap&3\\
&&&&pcon&3\\
&&&&apex&4\\
&&&&plex&4\\
&&&&tool&5\\
&&&&cplx&4\\
&&&&site&6\\\hline
&rely&3&5&tool&2\\
JPL &data&2&3&sced&3\\
flight&cplx&3&6&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&pvol&2&4&&\\
&acap&3&5&&\\
&apex&2&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&7&418&&\\\hline
&rely&1&4&tool&2\\
JPL &data&2&3&sced&3\\
ground&cplx&1&4&&\\
software&time&3&4&&\\
&stor&3&4&&\\
&pvol&2&4&&\\
&acap&3&5&&\\
&apex&3&5&&\\
&pcap&3&5&&\\
&plex&1&4&&\\
&ltex&1&4&&\\
&pmat&2&3&&\\
&KSLOC&11&392&&\\
\end{tabular}
\end{center}
\caption{Three case studies.
Numeric values 
$\{1,2,3,4,5,6\}$ map to $\{very low, low, nominal,
high, very high, extra high\}$.
The terms in column 2 come from \fig{cparems}.}\label{fig:cases}
\end{figure}

We use two categories of projects in this paper to study the failure of of FBC. The one category includes three projects of various sizes. Designated \textit{Small}, \textit{Medium}, and \textit{Large}, they corresponded to projects of size ranges [7,13] kloc, [70,130] kloc, and [700,1300] kloc respectively. These projects are open ended and are used to study whether there is an influence of project size purely on our study. Open ended here meaning that there are no project limitations that are predetermined but that the only limitation is the default limits of the models and the size of the projects. 

The second category are the NASA project templates that we've previously used, and are presented in \fig{cases}. 
These studies represent
the NASA software, at increasing levels of specificity:
\bi
\item {\em Flight} is a general description of flight software at NASA's Jet Propulsion Laboratory.
\item {\em Ground} is a general description of ground software at NASA's Jet Propulsion Laboratory. 
\item {\em OSP} is a specific flight system: the GNC (guidance, navigation, and control)
component of NASA's 1990s {\em Orbital Space Plane}; 
\item {\em OSP2} is a later version of OSP.
\ei

\input{policiesbad}
\input{policiesgood}
\input{policiesbadnasa}
\input{policiesgoodnasa}

\fig{cases} describes the details of flight, ground, OSP, and OSP2.
Note that \fig{cases} does not mention all the  features listed in \fig{cparems}
inputs.  For example, our defect predictor has inputs for
use of {\em automated analysis}, {\em peer
reviews}, and {\em execution-based testing tools}. For all inputs
not mentioned in \fig{cases}, values are picked at random from the full
range of \fig{cparems}.

One  aspect to note from \fig{cases} is the number of open options
{\em not} specified in the description of the projects.
Some of the features in \fig{cases}
are known precisely (see all the features with single {\em values}).
But  many of the features in \fig{cases} do not
have precise values
(see all the features that {\em range} from some {\em low} to {\em high}
value).  Sometimes the ranges
are very narrow (e.g., the process maturity of JPL ground software is between 2
and 3), and  sometimes the ranges are very broad.
In fact, our case studies can be ranked
\[
ground \equiv flight > OSP > OSP2
\]
according to the options open to
a  project manager:
\bi
\item
In the case of flight and ground systems, the description is very general and managers have many options. \item
In the case of OSP2, most of the project options are
pre-determined and project managers have very little opportunity  to effect
the course of a project.
\item  OSP is an early version of OSP2 and, measured in terms of the number
of open options, falls in between flight and OSP2.
\ei 
%As we shall, the number of open options in a project will
%have an important impact on our results.

The results of our study are presented in the following sections. We present two separate results:

\be
\item The first of these results show the stability of our AI methods in producing results in this study, in addition to presenting the ability of these policies to control the projects that they are selected for. A summary of the best and worst selected policies are also presented.
\item The second result presents the potential for FBC to succeed or fail and compares that potential with that of other ``you-pick-two'' strategies in the form of quartile charts and Mann-Whitney rankings.
\ee

\section{Results}
\subsection{Stability, Control and Validity}\label{tion:stability}

In order to determine the stability of STAR in choosing the worst policies, three open ended projects, of different sizes, were created. We ran the three of those in STAR using FBC and its "pick two" variants, with each case running ten times to account for the random variation in the results. This was repeated twice: one with the intention of choosing the best policies, and the other the worst policies. This was repeated for our NASA projects as well, except each case was repeated twenty times. Figures \ref{fig:badpol} through \ref{fig:goodpolnasa} present the stability results. These figures present the percentage of times that policies appear in the set of chosen policies, whether looking for the best or the worst policies. 

\input{graphpolgoodex}
\input{graphpolbadex}

The results shown confirm previous results concerning the stability in selecting the best policies, and also suggest that this stability is also available when choosing the worst policies. This is indicated by the large amount of policies that are chosen at a high percentage rate. However, the policy generation is not equally stable across the board. In the case of the open ended projects, the results presented in \fig{badpol} and \fig{goodpol} for the \textit{Small} project seem to be less stable, generating a larger range of policies compared to \textit{Medium} and \textit{Large}. This makes sense in small open ended projects, as the project would be more sensitive to the small details within a project, which is reflected in the models as well. Take the case of  the COCOMO II model: \eq{coc} shows that when the size (KLOC) of the project decreases, the effort multipliers will inherently have a larger relative effect on the effort of a project. On the other hand, if the size was large, the scale factors would have a much larger effect, meaning that the effort multipliers that would be included in the policies need to have a large effect on the project to ``make the grade''.

 In the case of stability for the NASA projects, we see a different stability result in \fig{badpolnasa} and \fig{goodpolnasa}, where the more specific a project is the more restricted and stable are its results, for both searching for good and bad policies. The reason to this is that more specified projects tend have a smaller search space, causing smaller policy sets to created by STAR.

\input{range-summary}

We also need to study whether STAR is able to control projects and guide the to the end that it intends, whether good or bad. \fig{polgoodex} and \fig{polbadex} are examples of the STAR runs, both conducted for large projects with an FBC focused evaluation function. The results from these runs are clumped with the results from other runs in order to obtain clearer overall results. The first corroborates results from previous papers we've published previously (\cite{me07f}), and choses the best policies for reducing overall defects, effort and development time. Note here that since the projects being used are open ended, this causes larger policies to me generated. We can see however that even for such open ended projects, STAR is able to control the project well, demonstrated by the large ``dip'' in the graphs of \fig{polgoodex}. On the other hand, \fig{polbadex} shows the policies that are selected by star to be the worst in one of the runs. 

The purpose of this experiment is determining the policies that need to be avoided, while at the same time showing how bad a project can be mishandled in the worst case scenario. This scenario is demonstrated by the spike that appears in the graphs of \fig{polbadex}. The fact that STAR is able to produce such a clear spike when searching for the worst policies demonstrates that it is producing a set of policies which produce definitive worst results for that project.


%We start out here presenting the most common policies that were chosen by STAR. This is done in order to establish the stability of STAR in choosing both the best and the worst policies available to a project. Using the methodology that has been indicated above to generate policy data, the policies generated in all the runs are concatenated and counted. The most common policies are presented in \fig{goodpol} and \fig{badpol}. Policies that occur more often indicate high certainty that they ought to be avoided if possible. Note that all the policies that occur less than 50\% of the time were not included in the tables. Given the results in the table, we can see how the results that are generated by STAR for Larger projects are more stable than those for smaller projects. This applies for both finding good and bad policies. The reason for this is believed to be that for larger projects, really bad policies have a worst effect on the project than on smaller policies, putting in effect the saying "The bigger they are, the harder they fall".


%Before we delve into the results, we need to show that any results produced by STAR are actually stable despite the stocastic nature of the algortihms used. For this purpose we use two sample projects: one that is highly constrained (OSP) and one that is loosely constrained (flight). Both projects are defined in \fig{cases}. STAR was run ten times for both, and then the policy results produced were compared. Each of the projects were run through STAR ten times while using the \textit{all} strategy, where all the model features are included in the search to produce policies. \fig{flightstab} and \fig{ospstab} below presents the percentage of times that a certain policy appears: the higher the percentage, the more that is indicative of the stability of that policy.


%As we can see, there is a high rate of stability with respect to the policies being produced by STAR for these two sample projects. Any degree of instability that is indicated is a normal occurrence given that the core algorithm used in STAR is a meta-heuristic search algorithm, and also given that we are actively varying the internal parameters of the models used.

%\subsection{Format of Results}


%\fig{results} shows the defects, months, and effort estimates seen imposing the {\em policy} learned by STAR:
%\bi
%\item The results have divisions for  defects, months, and effort.
%\item Division are sub-divided into
%results for
%flight, OSP, and OSP2.
%\ei
%Within each sub-division,
%the rows are sorted by median scores.
%The ``Do nothing'' row  comes from  Monte Carlo simulations over the project range $P$,
% without any restrictions.

%The {\em rank} results shown in column three  show the results
%of a statistical comparison of each sub-division.
%Two rows have the same rank 
%if there is 
%no statistical difference in their distributions. 
%We use Mann-Whitney for this comparison for the following reasons:
%\bi
%\item 
%The random nature of
%Monte Carlo simulations, the inputs to each run are not paired;
%\item Ranked tests make no, possibly inappropriate,
% assumption about normality of the results.
%\ei
%Each row shows
%results from 100 
%calls to \eq{score}:
%\bi
%\item Results in each division are  normalized 0..100, min..max.
%\item Each row shows the 25\% to 75\% quartile range of the normalized
%scores 
%collected during the simulation.
%\item The median result is shown as a black dot. 
%\ei
%All the performance scores (effort, months, defects) get {\em better} when
%the observed scores get {\em smaller}; i.e. move over the left. 


The stability results presented in figures \ref{fig:badpol} to \ref{fig:goodpolnasa} are presented in a different manner in figures \ref{fig:policyrange3} to \ref{fig:policyrange2}. \fig{policyrange3} presents the stable policies fort the open ended projects. The shaded cells indicated the range for the particular attribute in that row, which in this case are the default values indicated in the COCOMO II model. Also ``$\blacktriangle$'' and ``$\vartriangle$'' show the locations of the best and worst stable policy respectively. \fig{policyrange1} and \fig{policyrange2} represent the stable policies for the NASA projects. These stable results have been made to overlap with the results of medium size projects, since most of the NASA projects fall in that category. The light gray cells indicate the model ranges that aren't within the project range, while the dark gray cells indicate the ranges of attributes within the project. ``$\blacktriangle$'' and ``$\vartriangle$'' represent the open ended medium-sized project's stable results, while ``$\blacksquare$'' and ``$\square$'' represent the NASA project's stable results, best and worst respectively.

We ask the question of whether an AI tool that covers the full ranges of attributes is necessary or not. Other works such as the work of \cite{me09a} seem to suggest that the nature of the models in use allow us to simply overlook the ranges of the attributes while simply concentrating on the maxima and minima of the attributes. Looking at \fig{policyrange3}, we can see that there is support for this operating assumption made by SEESAW \cite{me09a, me09e} in that most of the attributes (mainly those attributes that are proportional to COCOMO II Effort) seem to to propagate to either extreme. Note however that there is also counter evidence to that is presented in these tables. The obvious evidence is that fact that there seems to be policies that aren't on the extreme, especially for the medium and small open-ended projects. This seems to suggest that there is a need to explore the full ranges of the attributes in order to obtain the best policies. Another piece of evidence is the fact that there are several attributes that aren't set in the medium and large projects. This indicates that sometimes the whole range of an attribute can be ignored, including the extremes. So not only  is there a need to evaluate the whole of the range to evaluate what value of the attribute is needed, but also to evaluate whether a value needs to be set for that attribute.


\subsection{Success/Failure Potential of FBC}\label{tion:FBCpotential}

In this section, we will present the our observations concerning the potential for FBC to succeed and to fail compared to other "you pick two" policies mentioned in \tion{FBC}. These observations are based on the quartile charts and Mann-Whitney ranking presented in figures \ref{fig:resultsbad} through \ref{fig:resultsgoodnasa}. These charts show the normalized median results, as well as the second and third quartiles, accompanied by the ranking of each of the policy schemes. Some of the results presented here  reiterate results that were mentioned in O. El-Rawas' thesis \cite{elrawas08}.

We Start with the results for applying the best policies presented in \fig{resultsgood} and \fig{resultsgoodnasa}.The former present the quartile charts and rankings for FBC and its derivate policies for the open ended projects, and the latter for the NASA projects. When observing these figures, we can see that FBC does well in general, especially compared to ``do nothing'. ``do nothing'' is a baseline result that is produced for each of the projects without applying any policies to them, and is supposed to represent how the project would end up given its current shape and assuming all future decisions are randomly chosen. The results presented in the aforementioned figures show the FBC rarely out does its derivate policies, however it does manage to stay close in terms of results, ranking close to the top according to the Mann-Whitney rankings, and rarely ranking last behind the ``you pick two'' policies. This supports the conclusion presented in \cite{elrawas08}: ``BFC is a `jack of all trades, master of none'. It works well in most instances, and provides a compromise solution in balancing out model estimates.'' This result in addition suggests that FBC is a viable set of priorities to follow that does not have a catastrophic cost side effects. Furthermore, FBC is capable of avoiding model and project specific catastrophic side effects. Some examples of such side effects include the following:

\bi
\item For the open-ended projects in \fig{resultsgood}, as well as OSP and OSP2 in \fig{resultsgoodnasa}, BC fails badly with regards months (development time), even though it ranks first and has the lowest normalized median estimates for defects in all the cases but one. Infact, BC does worse than ``do nothing'' with OSP2 months, which is to say that development time would've been better off in that case had we done nothing. 
\item For OSP and OSP2 in \fig{resultsgoodnasa}, CF fails badly with defects introduced, despite the fact that it ranks at the top with months and effort in 3 of the 4 projects.
\ei

One interesting occurrence that we notice is that BF doesn't seem to fail catastrophically. In this case, we argue that this is due to the nature of the relationship between the effort and the months models. More specifically, it is due to the fact that effort (or rather the vast majority of the effort calculation) is included in the derivation equations of months. This infers that by reducing months we are subsequently reducing effort, but reducing effort doesn't necessarily reduce months. In other words, BF in this case is acting as a pseudo-FBC due to this inter-model dependence.

Having discussed the results and analysis when applying the best policies, we move on to the case where we apply the worst policies. The reason this is done is to find the worst case scenario when applying a priority policy such as FBC, and the results are presented in \fig{resultsbad} and \fig{resultsbadnasa}. ``do nothing'', as presented previously, is a baseline that is supposed to represent how the project would end up given its current shape and assuming all future decisions are randomly chosen. Since we are applying the worst policies, leaving a project as is will naturally be better than applying such policies. Hence the result where ``do nothing'' does best in the vast majority of cases presented in this paper. By finding the worst scenario for FBC and the ``you pick two'' policies, we try to find the potential damage that following these policies could cause on a software project. The following observations can be made about the figure results:

\bi
\item In the case of CF, CF seems to have the least damage potential of all the policies when it comes to defects. In one case it even has less potential damage pertaining to defects than ``do nothing''. This would make sense since CF has no control over defects of a project. Another such case is effort for OSP2, which can be justified by the fact OSP2 is a very well defined and restricted project, meaning that CF isn't able to control it as much as in the cases of flight and ground for effort.
\item In the case of BC, it exhibits lower damage with regards to months for all the open ended projects, OSP and OSP2. we can use the same argument as CF to justify this, i.e. BC does not deal with or attempt to control the development time of a project.
\item BF doesn't seem to exhibit the behavior of both CF and BC. This can be justified by the same reasoning as above, where the relationship between the effort and months models allows BF to have an effect that is close to that of FBC.
\ei

In general the results for the worst case seem to reflect those for the best case. This seems to infer that exploring this maximum potential damage also tells about the control that a policy has over certain aspects of the project's performance. In addition to this, the results presented here indicate that the failure behavior of FBC is very similar to the derivate policies, ranking mid-pack most of the time. This implies that, despite the larger feature coverage of FBC, the potential for FBC to fail isn't any higher. This reinforces our previous conclusion concerning the viability of FBC, where the use of FBC doesn't seem to threaten a software project with a larger failing potential in the case that bad decisions are made.

%Moving on to the failure potential of FBC, we define this as the worst case scenario while using FBC as compared to the other derivate policies.  To determine this potential, we modified STAR to search for the worst policies to apply to a project. The results of doing this are presented in \fig{resultsbad} and \fig{resultsbadnasa}. The results presented here indicate that the failure behavior of FBC is very similar to the derivate policies, ranking mid-pack most of the time. This implies that, despite the larger feature coverage of FBC, the potential for FBC to fail isn't any higher. Note that ``do nothing'' in this test ranks on the top of every list except one. This also makes sense since by doing nothing, you can't make any bad decisions. 
 
\section{Discussion and Validity}
The results presented in \tion{FBCpotential} of this paper can be argued against in various ways. All the analysis in this paper is done under the assumption that COCOMO and its related models are models that are able to provide us with close estimates, and we truly know the tuning limits of this model. The fact is the limits we impose are based on a limited dataset of projects sourced from a limited set of development environments, mainly NASA. The other issue here that comes up is that these models are in some cases related, bringing the study of prioritizing development time (faster), defects introduction (better), or person-months effort (cheaper) into question since the models that measure these aspects aren't independent of each other. Or perhaps this is saying something more profound: that we cannot solely prioritize one or two aspects of a project while ignoring the third without considering the relationships between the models. Either way, we run into the issue of dealing with models which may or may not accurately represent aspects of a software project

\section{Future Work}
Given the above discussion concerning the validity of the models, one possible future direction could be one where we attempt to work independent of models, and move closer to case base reasoning (CBR) approaches. One way to do this could be using \fig{policyrange1} and \fig{policyrange2} and deriving a distance measure between a given project and one that's in the database with known real world or simulated values. This would allow us to circumvent the process of simulating the given project under certain models. What we have done so far for evaluating a project is to:

\be
\item randomly setting undetermined settings in a project to a value that is within project parameters
\item running a simulation of the project
\item repeating 1 and 2 several thousand times in order to account for random variance,  and obtaining the median values.
\ee

By attempting to change our approach to one closer to CBR, we could break free of our model dependence, while at the same time reducing the amount of computation and verification needed potentially giving us significant speedups, as well as additional dataset exposure.
%We have already presented studies and simulations that have been run on open ended as well as specific projects. Both scenarios have shown stability when it comes to generating results. So what if we chose to use the best results from the open ended projects to evaluate the more specific projects and their health based on the allowed ranges in the specific projects? In that case we would be evaluating the project itself and the decisions that have been taken instead of the policies that need to be taken. This could be very valuable for managers of such projects, providing a speedy manner of judging projects.

%One evaluation method that could be used in this case could be a distance measure between the extrimums of the project, and the best positive policies derived from the open ended projects.


%\input{
%Three aspects of \fig{results} deserve our attention.

%Firstly,
%it is almost always true that some optimizations on any pair or triple
%from ``Faster, Better, Cheaper'' can reduce defects {\em and} months {\em and} effort
%from the levels seen in the baseline ``do nothing'' scenario.
%This result argues for the use of tools like STAR.

%Secondly, STAR is most useful when applied
%to projects
%with many project options.
%As evidence of this, 
%note how in \fig{results} that  as we move from flight to OSP to OSP2, the median performance
%scores get worse.
%In order to explain this effect, we repeat remarks made above:
%our case studies are sorted 
%in decreasing order of ``number of open options''
%(there is
%much that can be adjusted within the general description of flight systems;
%fewer adjustments options are possible in OSP; and even fewer adjustments are possible in OSP2).
%As we {\em decrease} the number of open options, our ability to find ``fixes''
%to the current project also decreases. 

%Thirdly, the goal of faster {\em and} better {\em and} cheaper is not overly ambitions.
%Prior to running these experiments, we believed that
%to
%optimize for three criteria, it may be sometimes necessary
%to accept non-minimal results for one of the criteria. 
%However, contrary to our expectations, we observe 
%that FBC achieves the best (lowest) median results in the case of:
%\bi
%\item defects for OSP2 and 
%\item months for flight systems and 
%\item effort for flight systems 
%\ei
%In the remaining cases,
%FBC is statistically indistinguishable from the best (lowest) median result
%in the case of 
%\bi
%\item defects for flight systems and 
%\item effort for OSP
%\ei
%Lastly, in the remaining cases,
%FBC's median is within 5\% of the best (lowest) median result in the case of
%\bi
%\item defects for OSP
%\item months for OSP
%\ei
%That is, 
%FBC achieves minimum (or very close to
%minimum)  values.
%in $\frac{7}{9}$ of \fig{results}'s division, 
%Hence, we endorse STAR's default setting of $b=f=c=1$; i.e. try to optimize on all three criteria.

\section{Related Work}
Much of the related work on uncertainty in software engineering uses a Bayesian analysis.
For example,
Pendharkar et.al.~\cite{pendharkar05} demonstrate the utility of Bayes networks in effort
estimation while Fenton and Neil explore Bayes nets and defect prediction~\cite{fenton99}
(but unlike this paper,
neither of these teams links defect models to effort models).
We elect to take a non-Bayesian approach since most of the industrial and government
contractors we work with use parametric models like
COCOMO. 

The process simulation community (e.g. Raffo~\cite{raffo05c}) studies
models far more elaborate than COCOMO or COQUALMO. While such models
offer more detailed insight into an organization, the effort required
to tune them is non-trivial. For example, Raffo spent two years
tuning one such model to one particular site~\cite{raffo95}.

Other related work is the search-based SE approach advocated by Harmon~\cite{harman04}. 
Search-Based Software Engineering (SBSE) uses optimization techniques
from
operations research and meta-heuristic search (e.g. simulated annealing and genetic algorithms)
to hunt for near
optimal solutions to complex and over-constrained
software engineering problems. 
The SBSE approach can and has been applied to many
problems in software engineering (e.g. requirements engineering~\cite{jalali08})
 but most often in the field of software testing~\cite{andrews07}.
Harmon's writing inspired us try simulated annealing to search the what-ifs in untuned COCOMO
models~\cite{me07f}. However, we found that SEESAW ran much faster and produced results with far less
variance than simulated annealing. In addition the the above related work, a recent ICSP'09 paper by Orrego et. al \cite{orrego09}
has explored the merits of software reuse in the context of NASA projects using the same USC models that we use in this paper. 
For that study the same stochastic AI methods were used and showed that software reuse doesn't always benefit a project, recommending
a case by case evaluation of the appropriateness of software reuse.

\section{Conclusion}

Having discussed the concept of ``Faster, Better, Cheaper'' and its history, we presented a stochastic AI tool which we used to run several simulation in the absence of proper model tuning. We have shown that the results produced by this tool are stable and reliable, and proceeded to use this tool to find best and worst case scenarios with the use of FBC. 

With the help of quartile charts, we were able to show that FBC is a viable option that doesn't exhibit catastrophic consequences with regards to any of the models used in this paper. We also found that other ``you pick two'' policies exhibit such consequences. In addition, they have varying behavior depending on the aspect of the project being discussed and the project itself, while FBC has a more uniform response.

Our final recommendation is that FBC is indeed a viable option if it is implemented properly, that despite having lost credibility in software development circles.


%``Faster, Better, Cheaper'' (FBC) was a development philosophy
%adopted by the NASA administration in the mid to late 1990s. 
%that lead to some
%some dramatic successes such as Mars Pathfinder as well as a number highly publicized mission failures, such as 
%the Mars Climate Orbiter \& Polar Lander.
%
%Previous papers by Menzies et. al explored the validity of the general consensus on ``Faster, Better, Cheaper? Pick any two''. It was found that  FBC is indeed feasible when maintaining a balanced concern and concentration on the quality aspects of a project. 
%
%In This paper we seek to analyze FBC further, by analyzing its behavior in worst case decision making scenarios and comparing that to the other derivate "you pick two" policies. More generally, we seek to tuilize the same stocastic AI tool previously used in order to determine worst case scenarios for certain project test cases, all while circumventing model tuning. We find that our tool, STAR, is able to reliably determine worst case scenarios, without the need for model fine tuning.




%Our results show that
%FBC can be achieved with minimal
%compromises on individual criteria. How can we reconcile this result with 
%NASA very negative experience with FBC?
%
%One thing forgotten about FBC is that, usually, it worked.
%Despite all the criticism against it,
%FBC
%successful/partially successful in 136 of the total of 146 missions
%launched during the period that Goldin was administrator. This would
%be called an overall success if it hadn't been for the largely
%publicized failures. That is, FBC was mostly a technical success, but a PR failure~\cite{turner03}.
%
%However, one way to explain the very large failures within the FBC program
%is to speculate that, sometimes, FBC was a front for  CF (i.e.. cheaper and faster, as the expense of
%quality).
%\fig{results} shows the disastrous effects of CF:
%\bi
%\item In the case of OSP CF's defects where $\frac{62}{9}=689\%$ worse
%than the worst policy seen using any of BC, BF, or FBC.
%\item In the case of OSP2 CF's defects where $\frac{97}{11}=881\%$ worse
%than the worst policy seen using any of BC, BF, or FBC.
%\ei
%Some of the decisions made under the banner of FBC are questionable; i.e.
%staff reductions leading to 
%loss in veteran engineers and managers to retirement and 
%causing experienced managerial staff to be stretched too thin given
%tight scheduling~\cite{young00}. This forced projects to use
%inexperienced managers which caused management mix ups and human
%error.
%
%Like Spear~\cite{spear00} , we would endorse FBC, but under the condition that it is better managed.
%Tony
%Spear, a JPL veteran engineer from 1962 to 1998, testified
%to the possible effectiveness of FBC.
%Despite mentioning problems with FBC
%(a fixation on cost, causing cost cuts that
%were too much for $2^{nd}$ generation FBC projects),
%he recommended {\em not} to discard it. Rather, he argued for a
%more focused way of implementing it, concentrating on aspects such
%as
%building and retaining talent, taking
%advantage of advancements in technology such as the Internet, and
%advancing methods used in project development and verification~\cite{spear00,spear00(2)}. 
%
%To Spear's recommendation  we would add that when applying FBC, never surrender the quest of ``better''. 
%Observe how, in \fig{results}, whenever we optimize for ``Better'' using $b=1$, we always reduce
%defects by about an order of magnitude over the baseline ``do nothing'' result. 
%The only time that our optimizer does not reduce defects is when the ``better'' utility
%is set to zero (see the  CF detect results). 
%Hence, we recommend always setting $b=1$.

%
%\section{Discussions}
%More importantly than the above are the discussions we have about the models. The following section
%is not a formal analysis. Rather, it summarizes the common dialogues we have
%when we present the above results.  
%
%One reaction we get is {\em over-acceptance}. We have found one audience that are uncritical of our work
%and want to implement our results, immediately. Typically, this audience is formed of managers
%who are nervous of their current practices and aver (overly) eager to change to any other approach.
%
%While this is a gratifying response, we have found that
%members of this audience may  not fully appreciate a important part of the above conclusions; namely that 
%any policy that looks useful in general may not be useful in particular cases. Before implementing
%any of the above, we strongly recommend that users use our tools to find specialized policies that do
%best for their particular site.
%
%Another reaction we get is our conclusions come from untuned COCOMO models and {\em the above policies only
%make sense after local tuning}. In reply, we offer separate comments on the mean and variance of the estimates
%generated by our approach.
%
%{\em Issues with Mean:} The $experiment=1$ results of \fig{delta} shows the differences between the estimates
%generated {\em with} and {\em without} tuning on local data (the no-tuning estimates came from our search-based tool and the
%tuning-results can from
%on-line versions of the standard USC models).
%The $experiment=2$ results of that figure show
%the mean errors seen 
%when estimates are generated using conventional methods. Note that the mean differences in the $experiment=1$
%results are small (36\% or less).
%Further,
%the mean effort values of the conventional methods is larger (never less than 42\%).
%That is, \fig{delta} shows no evidence that the differences between our methods and conventional methods
%is inordinately large.
%
%{\em Issues with Variance:}
%Our tools sample across the space of possible tunings. The variance of the conclusions reached in that sample
%could be reduced by the use of local data to pre-tune our models, before we perform any sampling.
%However, it must be noted that the variances of our conclusions are quite small (see \fig{r1} and \fig{delta}).
%In summary, while we believe it may be useful to tune with local data, the overall benefit on reducing the variance or the mean  not be large.
%
%
%\begin{figure}
%\small
%\begin{tabular}{r|rrrr}
%	            &          & \multicolumn{3}{c}{data set}\\\cline{3-5}
%                &experiment&       flight        & OSP& OSP2\\\hline
%        defects &1&  30 $\pm$ 23   &    39  $\pm$ 33 &32 $\pm$ 18\\
%         months &1&       26 $\pm$ 17   &  23 $\pm$ 17    &  12 $\pm$ 7\\
%        effort  &1&    36 $\pm$ 31   &     20 $\pm$ 17&  18 $\pm$ 13\\\hline
%	     effort &2&        41 $\pm$40    & 44 $\pm$ 63 &   42 $\pm$ 57
%\end{tabular}
%     \caption{Results (Mean $\pm$ standard deviations) from our experiments.
%For an explanation of the method by which this was generated, see \fig{deletahow}.
%}\caption{fig:delta}
%\end{figure}
%\begin{figure}
%\small
%\begin{center}
%\begin{tabular}{|p{.95\linewidth}|}\hline
%The lines labeled $experiment=1$ in \fig{delta} were generated as follows:
%\be
%\item
%Impose the constraints $C$ of \fig{cases}
%on our search-based tool; 
%\item Learn a policy $P$; 
%\item
%Impose the combination of $C\cup P$ on our search-based tool;
%\item
%Generate 100 {\em estimates} (without further searching); 
%\item
%Generate {\em baselines}
%from the  standard on-line USC
%models\footnote{Phil: URLs for those models} 
%by picking values at random from within $C$ and setting any unknown values to "nominal"; 
%\item
%Combine the {\em estimates} from step 4 with the {\em baselines} from step 5
%using $delta= \frac{estimate - baseline}{baseline}$. 
%\ee
%The line labeled $experiment=2$ in \fig{delta}
%was generated by taking flight systems/OSP/OSP2 systems
%and estimating them using a COCOMO model tuned to some NASA COCOMO, as follows:
%\bi
%\item
%Using the NASA93 COCOMO data set\footnote{\url{http://promisedata.org/?s=nasa93}}
%we tuned a COCOMO model using
%using Boehem's LC method~\cite{boehm81};
%\item
%Using the  constraints $C$ of \fig{cases} we generated 100 random projects
%at random consistent with $C$; 
%\item Effort estimates for the generated projects were  generated using the tuned
%COCOMO model.  
%\item The means and standard deviations for those estimates were recorded in \fig{delta}.
%\ei\\\hline
%\end{tabular}
%\end{center}
%\caption{Generation method for the \fig{exp2} results.}\label{fig:deltahow}
%\end{figure}
%\fig{cases} and learned policies  
%A third reaction is that our results are {\em biased by the choice of model}.  The models used above
%accept input from a very limited number of variables (just those listed in \fig{variables}).
%COCOMO's variables are both the strongest and weakest part of the technique. On the one hand, they 
%have been selected and tested by a large community of academic and industrial researchers led by Boehm. 
%This group meets annually at the COCOMO forums (large meetings that have been held annually since 
%1985). On the other hand, these features may not be available in the databases of a local site. Hence, 
%regardless of the potential value added of using a well-researched feature set, those features may not be 
%available. 
%
%In reply, we observe that the whole point of our method is that we do not have training data.
%
% that we do not require training we have found that with minimal training, we can collect data in the form  
%
%COCOMO is crap.
%
%COCOMO is adequate and there are significant problems in generate a better-than-adequate solution (data drought).
%Given the pressing demands of project managers for answers to their control dilemmas, we need to offer the
%best solution we can, today, using the currently available information. The USC models and, in particular, the COCOMO
%model is one of the few examples of a long standing software engineering result in the literature. First proposed
%in 1981, it was extensively revised in the late 1990s.  It is widely used, is well-documented, and there exist
%numerous empirical studies describes its strengths (and flaws).
%
%\section{Discussion}
%The above results seem to contradict the NASA experience. We saw above how FBC was blamed for NASA mishaps.
%Yet our STAR results suggest that FBC is not inherently a bad policy for all projects. 
%
\newpage
\appendix

\bibliographystyle{abbrv}
\bibliography{refs,myrefs}

\input{cocmo}
\newpage
\section{Quartile Charts}

\input{r2}
\input{r4}
\input{r1}
\input{r3}

\end{document}
