
% figures line up nicely.

\section{Results}

\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.7}
\footnotesize
\begin{center}
\begin{tabular}{l|rrrrr}
    &              \multicolumn{5}{c}{percentiles}\\
    data         &        10\%&  30\% &   50\% & 70\%  &  90\%\\\hline
lucene-2.4 &  -2.0 &  -0.9 &  -0.3 &   0.4 &   2.6 \\ 
velocity-1.6 &  -1.4 &  -0.6 &  -0.3 &   0.0 &   1.4 \\ 
xerces-1.0 &  -1.1 &  -0.6 &  -0.3 &   0.7 &   1.0 \\ 
ant-1.4 &  -0.4 &  -0.3 &  -0.2 &  -0.1 &   0.8 \\ 
lucene-2.0 &  -2.0 &  -0.8 &  -0.2 &   0.1 &   1.7 \\ 
poi-1.5 &  -1.6 &  -0.9 &  -0.2 &   0.0 &   2.3 \\ 
synapse-1.2 &  -0.8 &  -0.4 &  -0.2 &   0.0 &   1.0 \\ 
xalan-2.5 &  -0.9 &  -0.5 &  -0.2 &   0.5 &   0.8 \\ 
xalan-2.6 &  -0.9 &  -0.5 &  -0.2 &   0.4 &   1.2 \\ 
xalan-2.7 &  -0.5 &  -0.3 &  -0.2 &   0.1 &   0.7 \\ 
xerces-1.2 &  -0.5 &  -0.2 &  -0.2 &  0.0 &   0.9 \\ 
xerces-1.4 &  -2.2 &  -0.4 &  -0.2 &   0.4 &   0.8 \\ 
ant-1.3 &  -0.5 &  -0.3 &  -0.1 &   0.0 &   0.6 \\ 
ant-1.7 &  -0.8 &  -0.3 &  -0.1 &   0.0 &   0.8 \\ 
ivy-2.0 &  -0.3 &  -0.1 &  -0.1 &   0.0 &   0.3 \\ 
lucene-2.2 &  -2.5 &  -0.8 &  -0.1 &   0.4 &   2.0 \\ 
poi-2.0 &  -0.2 &  -0.1 &  -0.1 &  -0.1 &   0.7 \\ 
poi-3.0 &  -1.1 &  -0.4 &  -0.1 &   0.0 &   1.0 \\ 
synapse-1.0 &  -0.3 &  -0.2 &  -0.1 &   0.0 &   0.0 \\ 
synapse-1.1 &  -0.7 &  -0.4 &  -0.1 &   0.0 &   0.9 \\ 
velocity-1.5 &  -1.5 &  -0.7 &  -0.1 &   0.3 &   1.5 \\ 
xalan-2.4 &  -0.5 &  -0.2 &  -0.1 &   0.0 &   0.7 \\ 
ant-1.5 &  -0.3 &  -0.1 &  0.0 &   0.0 &   0.3 \\ 
ant-1.6 &  -0.8 &  -0.3 &  0.0 &   0.0 &   0.9 \\ 
ivy-1.4 &  -0.2 &  -0.1 &  0.0 &   0.0 &   0.0 \\ 
jedit-3.2 &  -2.3 &  -0.8 &   0.0 &   0.0 &   1.8 \\ 
jedit-4.0 &  -1.3 &  -0.5 &   0.0 &   0.0 &   1.0 \\ 
jedit-4.1 &  -1.1 &  -0.4 &  0.0 &   0.0 &   1.0 \\ 
jedit-4.2 &  -0.6 &  -0.2 &  0.0 &   0.0 &   0.3 \\ 
jedit-4.3 &  -0.1 &  0.0 &  0.0 &  0.0 &   0.0 \\ 
poi-2.5 &  -1.3 &  -0.6 &  0.0 &   0.7 &   0.9 \\ 
velocity-1.4 &  -1.2 &  -0.2 &  0.0 &   0.1 &   1.0 \\ 
xerces-1.3 &  -1.4 &  -0.2 &  0.0 &   0.0 &   0.7  
\end{tabular}
\end{center}
\caption{Percentile distributions of $actual - predicted$ number of defects per class. For example,
the median (50th percentile) value of $actual - predicted$ is -0.3 to 0.}\label{fig:ar}
\end{figure}
\newcommand{\wwidth}{2mm}
\begin{figure}
\includegraphics[width=3.5in]{compare.pdf}
\caption{Performance results expressed as a ratio
of the $optimal$ policy. Data sets are sorted according the the $activity$ results.
Median values for $baseline$, $activity$ are 91\%, 96\% of $optimal$, respectively.
}\label{fig:pm}
\end{figure}
%\begin{figure*}
%{\small
%\begin{center}
%\begin{tabular}{r|rrrrrrrr}
%&  &  &      &   &  &   & Clustering \\
%& &     OutDegree  &  Getters    &   Setters & NoM & InDegree  & Coefficient \\
%data set&  $\beta_0$  &   $\beta_1$   &   $\beta_2$  &  $\beta_3$    &   $\beta_4$ &$\beta_5$ &$\beta_6$  \\\hline
%
%ant-1.3  &-0.1351&0.0406&&-0.0183&&&&  \\
%ant-1.4  &0.1459&0.0167&&0.0218&-0.0133&&&  \\
%ant-1.5  &-0.0631&0.0135&-0.0325&0.0293&&&&  \\
%ant-1.6  &-0.043&0.0466&-0.0454&-0.02&0.0486&&&  \\
%ant-1.7  &-0.0564&0.046&-0.0562&-0.0283&0.0546&&&  \\
%ivy-1.4  &-0.0066&0.0044&-0.0355&&0.0115&&&  \\
%ivy-2.0  &-0.0386&0.013&&-0.0226&0.0087&&&  \\
%jedit-3.2  &-1.0808&0.1099&0.4024&0.6921&&0.0522&&  \\
%jedit-4.0  &-0.631&0.0441&0.3699&&&0.0532&&  \\
%jedit-4.1  &-0.5152&0.057&0.1562&&&0.0378&&  \\
%jedit-4.2  &-0.2777&0.0187&0.1275&&&0.0195&&  \\
%jedit-4.3  &-0.0052&0.0016&0.0143&&-0.0007&&&  \\
%lucene-2.0  &-0.7125&0.1277&&0.5153&0.0907&0.06&&  \\
%lucene-2.2  &0.5616&0.0553&&1.0931&0.1688&&&  \\
%lucene-2.4  &0.5217&0.0392&0.1497&-0.5903&0.1572&0.0675&&  \\
%poi-1.5  &0.261&0.0593&0.1426&-0.3501&0.1113&&-1.5802&  \\
%poi-2.0  &0.006&0.0047&&-0.0168&0.0087&&&  \\
%poi-2.5  &-0.014&0.0331&0.1165&-0.1283&0.0387&&&  \\
%poi-3.0  &-0.1698&0.0528&0.1648&-0.1918&0.0537&&&  \\
%synapse-1.0  &0.025&0.022&&&&&&  \\
%synapse-1.1  &-0.0713&0.0515&&0.0951&-0.0279&&&  \\
%synapse-1.2  &-0.1312&0.0565&0.0771&&-0.0207&0.0154&&  \\
%velocity-1.4  &0.8674&0.0198&0.0741&&-0.0104&0.0131&&  \\
%velocity-1.5  &0.2875&0.1333&0.3349&-0.1941&&0.0258&0.8271&  \\
%velocity-1.6  &0.2705&0.0814&0.1209&-0.218&0.0358&&0.9503&  \\
%xalan-2.4  &-0.0722&0.0267&&&0.0075&&&  \\
%xalan-2.5  &0.208&0.0301&0.0242&0.0426&&&-0.3072&  \\
%xalan-2.6  &0.4958&0.0175&-0.0403&0.0707&0.02&&-0.5815&  \\
%xalan-2.7  &1.2681&0.0155&-0.0453&0.0963&0.0042&&-0.4668&  \\
%xerces-1.0  &0.1916&&0.2417&&0.0278&&&  \\
%xerces-1.2  &-0.2139&0.0204&0.0298&-0.0397&&0.0129&0.6292&  \\
%xerces-1.3  &-0.2716&0.1534&-0.0646&&&&&  \\
%xerces-1.4  &-0.1181&0.141&0.2912&-0.4426&0.0865&0.0478&&  \\\hline
%min&-1.0808&0.0016&-0.0646&-0.5903&-0.0279&0.0129&-1.5802&\\
%median&-0.0386&0.0399&0.0968&-0.01915&0.0239&0.0378&-0.3072\\
%max&1.2681&0.1534&0.4024&1.0931&0.1688&0.0675&0.9503\\\hline
%\%used&100\%&97\%&73\%&67\%&67\%&33\%&21\%&
%\end{tabular}
%\end{center}}
%\caption{LSR results for 
%$defects = \beta_0 + \left(\sum^6_i\beta_ix_i\right)$.
%Rows are sorted alphabetically.
%Columns are sorted according to how often a feature appears in the learned equation (see the last row).
%The bottom rows for \emph{min}, \emph{median}, and \emph{max} report a wide range in the
%learned coefficients for the popularity-based defect predictors.
%}\label{fig:eq}
%\end{figure*}
%


\subsection{Sanity Check}

\fig{ar} shows the distribution of $actual - predicted$ defects for our classes where $actual$ comes from 
historical logs and $predicted$ comes from the $C$ predictions seen in our 10-way.
This result is our \emph{sanity check}: if the $actual - predicted$ values were large, then we would doubt the value
of activity-based defect prediction. 
Note that, in the median case (shown in the middle 50\% column), the predictions are very close to actuals (-0 to -0.3).
Since our estimates are close to actuals,  we may continue.

\subsection{$Baseline$ and $Activity$ vs. $Optimal$}

\fig{pm} shows the ratio of the $optimal$ policy achieved with the $activity$ policy (the green curve)
and the $baseline$ policy (the red curve).
These curves are statistically significantly different (Wilcoxon, 95\% confidence).
For both curves, the result are expressed in as a ratio of the $optimal$ policy that uses historical knowledge to determine the number of defects in each class.

We observe that the results of the $baseline$ policy are  far more erratic
than for the $activity$ policy. The $spread$ of a distribution
is the difference between the   75\% and 25\%th percentile range.
The spread  of the 
values in \fig{pm} are:
\bi
\item $Activity:$ 98 - 91 = 7,
\item $Baseline:$ 95 - 82 = 13.
\ei
That is, the results of the $activity$ policy are more predictable (fall
into a narrower range).
On the other hand, the results from $baseline$ 
can spread nearly twice as
far.

Not only are the $activity$ results more predictable, they out-perform 
the $baseline$ policy.
The median value of the red $baseline$ policy results (\ie, inspecting the code 
based on increasing class size) is 91\% of $optimal$.  Note that
$baseline$ is rarely any little better than $activity$,
and often it is much worse:
\bi
\item
When $baseline$ out-performs $activity$ (in only
$\frac{3}{33}$ of our comparisons) it does so only by a small margin.
\item
In the $\frac{30}{33}$ data sets where $baseline$ does
worse than $activity$, sometimes it does much worse (see the
velocity-1.5 and velo\-city-1.6 results which fall to 70\% of optimal).
\ei

On the other hand, the median value of the $activity$ policies results
are 96\%, which is  within 4\% of $optimal$.
Further, the top ten results of $activity$ all score 100\% of $optimal$
(see the right-hand side
of the green curve in \fig{pm}). 
That is,  for the purposes of optimizing inspection, there is little
to no room for improvement on top of the activity-based measures.

Hence, we strongly recommend the $activity$ policy.

\subsection{Summary}

Our key observations in this study are as follows:

\bi
\item
According to \fig{ar}, activity-based measures combined with linear regression lead to defect predictors with low error
rates in open-source object-oriented systems.
\item
According to \fig{pm}, for the task of inspection optimization, 
activity-based defect prediction works significantly better
than the baseline and  
very close to the optimum.
\ei

