% This is "sig-alternate.tex" V1.9 April 2009
% This file should be compiled with V2.4 of "sig-alternate.cls" April 2009
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.4 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.4) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
% \CopyrightYear{2007} will cause 2007 to appear in the copyright line.
% \crdata{0-12345-67-8/90/12} will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
% Technical questions _only_ to
% Gerald Murray (murray@hq.acm.org)
% ===============================================================
%
% For tracking purposes - this is V1.9 - April 2009

\documentclass{sig-alternate}
\usepackage{cite,times}
\usepackage[usenames,dvipsnames]{color}
\usepackage{url}
\usepackage{tim/tricks}
\usepackage{raj/tricks}
\usepackage{markus/tricks}

\begin{document}
%
% --- Author Metadata here ---
\conferenceinfo{MSR}{'11 Honolulu,  Hawaii USA}
%\CopyrightYear{2007} % Allows default copyright year (20XX) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{``Popularity'' is (almost) a Perfect Predictor for Defects}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{2} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Markus Lumpe, Rajesh Vasa\\
       \affaddr{Faculty of ICT}\\
%       \affaddr{Faculty of Information \& Communication Technologies}\\
       \affaddr{Swinburne University of Technology}\\
       \affaddr{Hawthorn, Australia}\\
       \email{\{mlumpe,rvasa\}@swin.edu.au}
\alignauthor
Tim Menzies\\
       \affaddr{Lane Department of Com. Sci. \& Elec. Eng.}\\
%       \affaddr{Lane Department of Computer Science \& Electrical Engineering}\\
       \affaddr{West Virginia University}\\
       \affaddr{Morgantown, USA}\\
       \email{tim@menzies.us}
}

% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
\date{26 Jan 2011}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
BACKGROUND:
Recent
research has shown the value of social metrics for defect prediction. 
But many  repositories lack the
developer information required from a social
metrics analysis.
\\
AIM:
Do we need to redesign our repositories, throwing away all
the old data that lacks social metrics?
\\
METHOD:
We  evaluate social pattern metrics inferred from repositories
that only contain static code attributes.
The ``popularity'' (i.e. frequency of use by developers) of a class is inferred
from 
repositories that lack details about individual developers.
The value of these popularity measures  is assessed in context of 
``inspection optimization'' (reading the fewest lines of code to find the most defects).
\\
RESULTS:
For the task of inspection
optimization
these popularity measures perform 
as well
(usually, within 4\%) 
of the theoretical upper bound on the performance  
of any set of measures.
\\
CONCLUSION:
Defect prediction research about 
static code measures has not been superseded by
social metrics.
There is still much to be discovered in repositories
that lack detailed developer information; e.g.
popularity is
(almost) a perfect predictor for defects. 
\\
FUTURE DIRECTIONS:
The value of theses popularity measures should be tested for
other tasks (i.e. not
just
inspection optimization) and other systems
(i.e. not just
open-source, object-oriented JAVA programs). 
Also, it would be insightful to evaluate other measures of social patterns
inferred
from repositories that lack developer information.
\end{abstract}

% A category with the (minimum) three required fields
%A category including the fourth, optional field follows...
\category{D.2.8}{Software Engineering}{Metrics}[complexity measures, performance measures]

\input{Intro}
\input{Popular}

\input{Prediction}

\input{InspectionOptimization}
%\input{markus/InspectionOptimizationOld}

\section{Popularity and Inspection Optimization}
To assess the value of the measures in \fig{metrics}, we use them to
build defect predictors. Then, after applying them to
33 open source JAVA projects from the Helix porject, we report the
their 
$\frac{learning}{oracle}$ ratio.
As shown below, the median values for that ratio will be 96\%, i.e. very close to the theoretical
upper bound possible for any defect predictor for the the task of inspection optimization.

\input{raj/data}


%
%\begin{figure*}
%\noindent
%\includegraphics[width=2in]{img/ant-13.pdf}        
%\includegraphics[width=2in]{img/ant-14.pdf}     
%\includegraphics[width=2in]{img/ant-15.pdf}     
%
%\noindent
%\includegraphics[width=2in]{img/ant-16.pdf}       
%\includegraphics[width=2in]{img/ant-17.pdf} 
%\includegraphics[width=2in]{img/lucene-20.pdf} 
%
%\noindent
%\includegraphics[width=2in]{img/lucene-22.pdf}
%\includegraphics[width=2in]{img/lucene-24.pdf}
%\includegraphics[width=2in]{img/xalan-24.pdf}
%
%\noindent
%\includegraphics[width=2in]{img/xalan-25.pdf}
%\includegraphics[width=2in]{img/xalan-26.pdf}
%\includegraphics[width=2in]{img/xalan-27.pdf}
%\caption{Results}
%\end{figure*}
%
\section{Results}

\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.7}
\small
\begin{center}
\begin{tabular}{l|rrrrr}
    &              \multicolumn{5}{c}{percentiles}\\
    data         &        10\%&  30\% &   50\% & 70\%  &  90\%\\\hline
lucene-2.4 &  -2.0 &  -0.9 &  -0.3 &   0.4 &   2.6 \\ 
velocity-1.6 &  -1.4 &  -0.6 &  -0.3 &   0.0 &   1.4 \\ 
xerces-1.0 &  -1.1 &  -0.6 &  -0.3 &   0.7 &   1.0 \\ 
ant-1.4 &  -0.4 &  -0.3 &  -0.2 &  -0.1 &   0.8 \\ 
lucene-2.0 &  -2.0 &  -0.8 &  -0.2 &   0.1 &   1.7 \\ 
poi-1.5 &  -1.6 &  -0.9 &  -0.2 &   0.0 &   2.3 \\ 
synapse-1.2 &  -0.8 &  -0.4 &  -0.2 &   0.0 &   1.0 \\ 
xalan-2.5 &  -0.9 &  -0.5 &  -0.2 &   0.5 &   0.8 \\ 
xalan-2.6 &  -0.9 &  -0.5 &  -0.2 &   0.4 &   1.2 \\ 
xalan-2.7 &  -0.5 &  -0.3 &  -0.2 &   0.1 &   0.7 \\ 
xerces-1.2 &  -0.5 &  -0.2 &  -0.2 &  0.0 &   0.9 \\ 
xerces-1.4 &  -2.2 &  -0.4 &  -0.2 &   0.4 &   0.8 \\ 
ant-1.3 &  -0.5 &  -0.3 &  -0.1 &   0.0 &   0.6 \\ 
ant-1.7 &  -0.8 &  -0.3 &  -0.1 &   0.0 &   0.8 \\ 
ivy-2.0 &  -0.3 &  -0.1 &  -0.1 &   0.0 &   0.3 \\ 
lucene-2.2 &  -2.5 &  -0.8 &  -0.1 &   0.4 &   2.0 \\ 
poi-2.0 &  -0.2 &  -0.1 &  -0.1 &  -0.1 &   0.7 \\ 
poi-3.0 &  -1.1 &  -0.4 &  -0.1 &   0.0 &   1.0 \\ 
synapse-1.0 &  -0.3 &  -0.2 &  -0.1 &   0.0 &   0.0 \\ 
synapse-1.1 &  -0.7 &  -0.4 &  -0.1 &   0.0 &   0.9 \\ 
velocity-1.5 &  -1.5 &  -0.7 &  -0.1 &   0.3 &   1.5 \\ 
xalan-2.4 &  -0.5 &  -0.2 &  -0.1 &   0.0 &   0.7 \\ 
ant-1.5 &  -0.3 &  -0.1 &  0.0 &   0.0 &   0.3 \\ 
ant-1.6 &  -0.8 &  -0.3 &  0.0 &   0.0 &   0.9 \\ 
ivy-1.4 &  -0.2 &  -0.1 &  0.0 &   0.0 &   0.0 \\ 
jedit-3.2 &  -2.3 &  -0.8 &   0.0 &   0.0 &   1.8 \\ 
jedit-4.0 &  -1.3 &  -0.5 &   0.0 &   0.0 &   1.0 \\ 
jedit-4.1 &  -1.1 &  -0.4 &  0.0 &   0.0 &   1.0 \\ 
jedit-4.2 &  -0.6 &  -0.2 &  0.0 &   0.0 &   0.3 \\ 
jedit-4.3 &  -0.1 &  0.0 &  0.0 &  0.0 &   0.0 \\ 
poi-2.5 &  -1.3 &  -0.6 &  0.0 &   0.7 &   0.9 \\ 
velocity-1.4 &  -1.2 &  -0.2 &  0.0 &   0.1 &   1.0 \\ 
xerces-1.3 &  -1.4 &  -0.2 &  0.0 &   0.0 &   0.7  
\end{tabular}
\end{center}
\caption{Percentile distributions of $actual - predicted$ number of defects per class. For example,
the median (50th percentile) value of $actual - predicted$ is -0.3 to 0.}\label{fig:ar}
\end{figure}

\subsection{Sanity Check}

\fig{ar} shows the distribution of $actual - predicted$ defects
for our classes where $actual$ comes from historical logs and $predicted$ comes from
the $C$ predictions seen in our 10-way:
\bi
\item
This result is our {\em sanity check}: if the $actual - predicted$ values were large, then we would doubt the value
of popularity-based defect prediction. 
\item
Note that, in the median case (shown in the middle 50\% column), the predictions are very close to actuals (-0 to -0.4).
\item
Since our estimates are close to actuals,  we may continue.
\ei
\subsection{$\frac{learned}{optimal}$ Results}

\fig{pm} shows the effectiveness of the popularity measures,
expressed as $\frac{learned}{optimal}$.  Recall that this was generated as follows:
\bi
\item Use the LSR predictions over the popularity measures to  guess the number of defects in some test set.
\item Sort the classes in that test set  the number of defects per class and the size of class
(favoring the smaler classes with most predicted defects) using \eq{score}.
\ei
Note
the median value of that figure: our popularity-based metrics perform
within 4\% of the optimal. Hence, we say that not only do popularity-based
measures generator
 good predictors, but also that  the addition of the standard social metrics
(e.g. containing detailed developer knowledge) would add very little extra value.
\newcommand{\wwidth}{2mm}
\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.7}
\small
\begin{center}
\begin{tabular}{lrlr}
synapse-1.0 & 74 & \textcolor[named]{Gray}{\rule{24mm}{\wwidth}}\\
ant-1.3 & 77 & \textcolor[named]{Gray}{\rule{25mm}{\wwidth}}\\
poi-2.0 & 78 & \textcolor[named]{Gray}{\rule{25mm}{\wwidth}}\\
jedit-4.3 & 81 & \textcolor[named]{Gray}{\rule{26mm}{\wwidth}}\\
ivy-2.0 & 85 & \textcolor[named]{Gray}{\rule{28mm}{\wwidth}}\\
ivy-1.4 & 89 & \textcolor[named]{Gray}{\rule{29mm}{\wwidth}}\\
ant-1.5 & 90 & \textcolor[named]{Gray}{\rule{29mm}{\wwidth}}\\
velocity-1.6 & 91 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
synapse-1.1 & 92 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
poi-3.0 & 93 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
ant-1.7 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
lucene-2.0 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
velocity-1.5 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
xerces-1.3 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
ant-1.4 & 95 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
poi-1.5 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
poi-2.5 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
xerces-1.4 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
synapse-1.2 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.4 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xerces-1.0 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xerces-1.2 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.5 & 98 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.6 & 98 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
lucene-2.4 & 99 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.7 & 99 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
ant-1.6 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-3.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.0 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.1 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
lucene-2.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
velocity-1.4 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}
\end{tabular}
\end{center}
\caption{The median $\frac{learned}{optimal}\%$ result is 96\%
(at poi-2.5).}\label{fig:pm}
\end{figure}


\begin{figure*}[!t]
{\small
\begin{center}
\begin{tabular}{r|rrrrrrrr}
&  &  &      &   &  &   & Clustering \\
& &     OutDegree  &  Getters    &   Setters &NoM &InDegree  & Coefficient \\
data set&  $\beta_0$  &   $\beta_1$   &   $\beta_2$  &  $\beta_3$    &   $\beta_4$ &$\beta_5$ &$\beta_6$  \\\hline

ant-1.3  &-0.1351&0.0406&&-0.0183&&&&  \\
ant-1.4  &0.1459&0.0167&&0.0218&-0.0133&&&  \\
ant-1.5  &-0.0631&0.0135&-0.0325&0.0293&&&&  \\
ant-1.6  &-0.043&0.0466&-0.0454&-0.02&0.0486&&&  \\
ant-1.7  &-0.0564&0.046&-0.0562&-0.0283&0.0546&&&  \\
ivy-1.4  &-0.0066&0.0044&-0.0355&&0.0115&&&  \\
ivy-2.0  &-0.0386&0.013&&-0.0226&0.0087&&&  \\
jedit-3.2  &-1.0808&0.1099&0.4024&0.6921&&0.0522&&  \\
jedit-4.0  &-0.631&0.0441&0.3699&&&0.0532&&  \\
jedit-4.1  &-0.5152&0.057&0.1562&&&0.0378&&  \\
jedit-4.2  &-0.2777&0.0187&0.1275&&&0.0195&&  \\
jedit-4.3  &-0.0052&0.0016&0.0143&&-0.0007&&&  \\
lucene-2.0  &-0.7125&0.1277&&0.5153&0.0907&0.06&&  \\
lucene-2.2  &0.5616&0.0553&&1.0931&0.1688&&&  \\
lucene-2.4  &0.5217&0.0392&0.1497&-0.5903&0.1572&0.0675&&  \\
poi-1.5  &0.261&0.0593&0.1426&-0.3501&0.1113&&-1.5802&  \\
poi-2.0  &0.006&0.0047&&-0.0168&0.0087&&&  \\
poi-2.5  &-0.014&0.0331&0.1165&-0.1283&0.0387&&&  \\
poi-3.0  &-0.1698&0.0528&0.1648&-0.1918&0.0537&&&  \\
synapse-1.0  &0.025&0.022&&&&&&  \\
synapse-1.1  &-0.0713&0.0515&&0.0951&-0.0279&&&  \\
synapse-1.2  &-0.1312&0.0565&0.0771&&-0.0207&0.0154&&  \\
velocity-1.4  &0.8674&0.0198&0.0741&&-0.0104&0.0131&&  \\
velocity-1.5  &0.2875&0.1333&0.3349&-0.1941&&0.0258&0.8271&  \\
velocity-1.6  &0.2705&0.0814&0.1209&-0.218&0.0358&&0.9503&  \\
xalan-2.4  &-0.0722&0.0267&&&0.0075&&&  \\
xalan-2.5  &0.208&0.0301&0.0242&0.0426&&&-0.3072&  \\
xalan-2.6  &0.4958&0.0175&-0.0403&0.0707&0.02&&-0.5815&  \\
xalan-2.7  &1.2681&0.0155&-0.0453&0.0963&0.0042&&-0.4668&  \\
xerces-1.0  &0.1916&&0.2417&&0.0278&&&  \\
xerces-1.2  &-0.2139&0.0204&0.0298&-0.0397&&0.0129&0.6292&  \\
xerces-1.3  &-0.2716&0.1534&-0.0646&&&&&  \\
xerces-1.4  &-0.1181&0.141&0.2912&-0.4426&0.0865&0.0478&&  \\\hline
min&-1.0808&0.0016&-0.0646&-0.5903&-0.0279&0.0129&-1.5802&\\
median&-0.0386&0.0399&0.0968&-0.01915&0.0239&0.0378&-0.3072\\
max&1.2681&0.1534&0.4024&1.0931&0.1688&0.0675&0.9503\\\hline
\%used&100\%&97\%&73\%&67\%&67\%&33\%&21\%&




\end{tabular}
\end{center}}
\caption{LSR results for 
$defects = \beta_0 + \left(\sum^7_i\beta_ix_i\right)$.
Rows are sorted alphabetically.
Columns are sorted according to how ofter a feature appears in the learned equation (see the last row).
The bottom rows for {\em min}, {\em median}, and {\em max} report a wide range in the
learned co-effecients for the popularity-based defect predictors.
}\label{fig:eq}
\end{figure*}
\subsection{What Popularity Measures are ``Best''?}
\fig{eq} shows the co-efficients on the
linear regression
equations learned over the popularity measures. 
Recall that the 
linear regression tool we used (from WEKA 3.7.3~\cite{witten05}) includes
a greedy back-select that removes those features that do not degrade
the overall performance of the equation.
Observe that there are many blank cells in \fig{eq}:
\bi
\item
The models learned from different data sets used different
popularity measures.
\item
No popularity measure was used in all data sets (max 97\% of the data, often much less).
\ei
The bottom rows in that figure report the {\em min}, {\em median}, and {\em max} co-effecient
values learned in this study.
The range in these values was very large (e.g. $-1.08 \le \beta_0 \le 1.27$).
Combining the results of
\fig{pm} and \fig{eq}, we say that:
\bi
\item
Some combination of popularity-based measure was very useful in nearly all
the  data sets
studied here.
\item
The exact effect of particular measures varied from data set to data set.
\ei
\subsection{Summary}
\bi
\item
According to \fig{ar}, popularity-measures plus linear regression lead to defect predictors with low error
rates in open-source OO systems.
\item
According to \fig{pm}, for the task of inspection optimization,
popularity-based defect prediction works very close to the optimum.
\item
However, 
\fig{eq} shows that 
that while multiple popularity measures are useful, no particular singularity measure is always most effective.
Hence,
we  recommend againts focusing on particular popularity measures.
Rather,
we recommend applying them all.
\ei
%\section{Discussion}
%The value of popularity-based metrics has been partially reported in the literature.
%Turhan et. al~\cite{turhan08aa}  explored defect prediction based on call-graphs and found
%that such call-graph measures reduced the false alarm rate in the predictions. 
%However, their call-graph measures are a small subset  with some of \fig{metrics}. Hence, it
%is not surprising that they 
%missed the full value of popularity.
%
%Neverthless, the results here are quite unequivocal- popularity is a strong predictor for software defects.
%Hence, we need to explain why 
%this effect has not been reported before (not by Turhan et. al;  not in our previous research;  and not, as far we
%know, by any other
%research in this field).
%
%One explanation is that {\em popularity is so simple}  that it escaped the attention of other researchers.
%The literature is full of many complex methods for data mining such as support-vector machines,
%random forests, and tabu search to tune the parameters of a genetic algorithm.
%In this era of increasing learner complexity, something as easy as  linear regression on
%the popularity measures may have been discounted before being explored.
%
%Another explanation is that {\em popularity is a novel concept}. The starting point for this research
%was the observation in the Helix data that most classes stabilize very early in their lifecycle
%while a very small number of popular classes garner the most attention by developers. As discussed
%above in \tion{pop}, this is not the standard picture of the lifecycle of a class. To us, this observation
%was so unique that it prompted the question ``does the amount a class is used by developers predict
%for system defects'' (i.e. this study). However, without that initial surprising observation, we would
%not have conducted the study reported in this paper.
%
%Yet another explanation is that {\em popularity was buried}. When we look we look at the attributes
%we have used 
%in previous studies (e.g.~\cite{me07b}), we can see some overlap between those attributes and those
%of \fig{metrics}. Witten~\cite{witten05} and Wagner~\cite{wagner07aa} and Miller~\cite{miller02}
%offer a theoretical analysis discussing how an excess of attributes
%containing multiple strong predictors for the target class can confuse learning. For example, Wagner
%and Miller note that in a  model comprising $N$ variables, any noise in  variable $N_i$ adds
%to the noise in the output variables. Hence, even though some popularity measures exist in a
%data set, noise from the other variables may drown out the popularity effect.
%
%
%\subsection{very good}
%
%very good. compare to ~\cite{me10a}, must better.
%\subsection{very simple}
%note the absence of complex machinery: SVM, feature selection, random forests, etc.
%
%Instead, a simple theory of human behavior (usage predicts for defects) plus a simple
%data miner (linear regression) lead to predictions of very high competency (median results,
%in \fig{pm}, within 3\% of optimal). Interesting... perhaps there has been too much
%AI and not enough human intelligence in prior research on defect prediction.
%\subsection{ANT}
%Worst results in ANT. Why?
%\section{Future Work}
%Could apply the \cite{me10a} tools to here. But given the minimal room for improvement
%(usually, less than 3\%), why bother?
%
%More defect data sets.
%
\newtheorem{future}{Future work}

%ACKNOWLEDGMENTS are optional
\section{Validity \& Future Work}
{\em Internal validity:} Apart for joining from the PROMISE data sets (for defect counts) to the Helix data sets
(for the popularity measures), we   did not pre-process the datasets in any way. This was done to
repository replication of our results. 

{\em Construct validity:} 
We have made the case above that the measures of \fig{metrics} reflect the ``popularity''
of different classes; i.e. how often a developer  will use or extend the services of a class\footnote{rajesh: is this pop?}.
This case has not been tested.
Hence:
\begin{future}
Participant observation of developers to see what classes they inspect
as part of their work. 
\end{future}


{\em External validity:} The datasets used in this study represent real-world use, collected in
real-world  projects.
Measured in terms of number of data sets, this paper is one of the largest defect prediction studies
that we are aware of. Nevertheless, there is a clear bias in our sample: open-sourced object-oriented JAVA.
Hence:

\begin{future}
Test the  validity of our conclusions to close-sourced or non-OO or non-JAVA projects.
\end{future}

{\em Conclusion validity:} This empirical study was performed using 10-fold cross-validation for statistically reli- 
able results (recommended in~\cite{witten05}).  The performance of classifiers is compared using the criteria
proposed by Arisholm \& Briand~\cite{arisholm06}; i.e.  area under  the \%LOC-vs-PD
curves. This 
is only one way to evaluate these results. Other ways include:
\be
\item 
Counting number of files inspected, rather than the total LOC,  as done by (\eg) Weyuker and Ostrand~\cite{ostrand04,weyeuker08}.
\item 
Precision, as advocated by (\eg) Zhang and Zhang~\cite{zhang07} (but depreciated by Menzies et. al~\cite{me07e});
\item
Area under the curve of the PD-vs-PF curves, as used by (\eg) Lessmann et. al~\cite{lessmann09};
\ee
It is an open-issue if our conclusions  holds for other
criteria such as the three listed above. However, if we view predictor generation as {\em a search procedure},
and the evaluation criteria as a {\em objective function}, then it follows that:
\bi
\item
Changes to the objective function
will bias the results of the learning procedure. 
\item
We should not expect that what words for one criteria will always work for another.
\ei
Hence, we take great care to {\em only} state our conclusions in terms of \%LOC-vs-PD. For the purposes
of finding the most defects after inspecting the fewest lines of code,  the ``popularity'' measures
of \fig{metrics} are nearly perfect (median results, within 96\% of optimum).

Having said that, clearly:
\begin{future}
Explore
the value of popularity for other evaluation criteria.
\end{future}

\input{Related}
\section{Discussion}
The results here are quite unequivocal- popularity is a strong predictor for software defects.
Hence, we need to explain why 
this effect has not been reported before (not in our previous research;  and not, as far we
know, by any other
research in this field).

The value of popularity-based metrics has been partially reported in the literature.
Turhan et. al~\cite{turhan08aa}  explored defect prediction based on call-graphs and found
that such call-graph measures reduced the false alarm rate in the predictions. 
However, their call-graph measures are a small subset  with some of \fig{metrics}. Hence, it
is not surprising that they 
missed the full value of popularity.

As to the reason that other researchers have missed the popularity effect,
we offer three explanations.
The first is that is that {\em popularity is so simple}  that it escaped the attention of other researchers.
The literature is full of many complex methods for data mining such as support-vector machines,
random forests, and tabu search to tune the parameters of a genetic algorithm.
In this era of increasing learner complexity, something as easy as  linear regression on
the popularity measures may have been discounted before being explored.

Secondly, we note that {\em popularity is a novel concept}. The starting point for this research
was the observation in the Helix data that most classes stabilize very early in their lifecycle
while a very small number of popular classes garner the most attention by developers. As discussed
above in \tion{pop}, this is not the standard picture of the lifecycle of a class. To us, this observation
was so unique that it prompted the question ``does the amount a class is used by developers predict
for system defects'' (i.e. this study). However, without that initial surprising observation, we would
not have conducted the study reported in this paper.

Thirdly, it is possible that {\em popularity was buried} under other effects. When we look we look at the attributes
we have used 
in previous studies (e.g.~\cite{me07b}), we can see some overlap between those attributes and those
of \fig{metrics}. Witten~\cite{witten05} and Wagner~\cite{wagner07aa} and Miller~\cite{miller02}
offer a theoretical analysis discussing how an excess of attributes
containing multiple strong predictors for the target class can confuse learning. For example, Wagner
and Miller note that in a  model comprising $N$ variables, any noise in  variable $N_i$ adds
to the noise in the output variables. Hence, even though some popularity measures exist in a
data set, noise from the other variables may drown out the popularity effect.

\section{Acknowledgments}
 
This work was conducted at Swinburne University and West Virginia
University with partial funding from (1)~the New Zealand Foundation for
Research, Science and Technology; (2)~the United States
National Science Foundation, CISE grant 71608561; and (3)~a research
subcontract with the Qatar Unviersity NPRP 09-1205-2-470.

% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{abbrv}
\bibliography{oldrefs,timrefs,rajrefs,markusrefs}  % sigproc.bib is the name of the Bibliography in this case
% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%
%APPENDICES are optional
%\balancecolumns
%Appendix A
%\balancecolumns % GM June 2007
% That's all folks!
\end{document}
%	research.
%
%
%\newtheorem{future}{Future work}
%
%%ACKNOWLEDGMENTS are optional
%\section{Validity \& Future Work}
%{\em Internal validity:} Apart for joining from the PROMISE data sets (for defect counts) to the Helix data sets
%(for the popularity measures), we   did not pre-process the datasets in any way. This was done to
%repository replication of our results. 
%
%{\em Construct validity:} 
%We have made the case above that the measures of \fig{metrics} reflect the ``popularity''
%of different classes; i.e. how often a developer  will use or extend the services of a class\footnote{rajesh: is this pop?}.
%This case has not been tested.
%Hence:
%\begin{future}
%Participant observation of developers to see what classes they inspect
%as part of their work. 
%\end{future}
%
%
%{\em External validity:} The datasets used in this study represent real-world use, collected in
%real-world  projects.
%Measured in terms of number of data sets, this paper is one of the largest defect prediction studies
%that we are aware of. Nevertheless, there is a clear bias in our sample: open-sourced object-oriented JAVA.
%Hence:
%
%\begin{future}
%Test the  validity of our conclusions to close-sourced or non-OO or non-JAVA projects.
%\end{future}
%
%{\em Conclusion validity:} This empirical study was performed using 10-fold cross-validation for statistically reli- 
%able results (recommended in~\cite{witten05}).  The performance of classifiers is compared using the criteria
%proposed by Arisholm \& Briand~\cite{arisholm06}; i.e.  area under  the \%LOC-vs-PD
%curves. This 
%is only one way to evaluate these results. Other ways include:
%\be
%\item 
%Counting number of files inspected, rather than the total LOC,  as done by (\eg) Weyuker and Ostrand~\cite{ostrand04,weyeuker08}.
%\item 
%Precision, as advocated by (\eg) Zhang and Zhang~\cite{zhang07} (but depreciated by Menzies et. al~\cite{me07e});
%\item
%Area under the curve of the PD-vs-PF curves, as used by (\eg) Lessmann et. al~\cite{lessmann09};
%\ee
%It is an open-issue if our conclusions  holds for other
%criteria such as the three listed above. However, if we view predictor generation as {\em a search procedure},
%and the evaluation criteria as a {\em objective function}, then it follows that:
%\bi
%\item
%Changes to the objective function
%will bias the results of the learning procedure. 
%\item
%We should not expect that what words for one criteria will always work for another.
%\ei
%Hence, we take great care to {\em only} state our conclusions in terms of \%LOC-vs-PD. For the purposes
%of finding the most defects after inspecting the fewest lines of code,  the ``popularity'' measures
%of \fig{metrics} are nearly perfect (median results, within 96\% of optimum).
%
%Having said that, clearly:
%\begin{future}
%Explore
%the value of popularity for other evaluation criteria.
%\end{future}
%
%\input{Related}
%\section{Acknowledgments}
% 
%This work was conducted at Swinburne University and West Virginia
%University with partial funding from (1)~the New Zealand Foundation for
%Research, Science and Technology; (2)~the United States
%National Science Foundation, CISE grant 71608561; and (3)~a research
%subcontract with the Qatar Unviersity NPRP 09-1205-2-470.
%
%% The following two commands are all you need in the
%% initial runs of your .tex file to
%% produce the bibliography for the citations in your paper.
%\bibliographystyle{abbrv}
%\bibliography{oldrefs,timrefs,rajrefs,markusrefs}  % sigproc.bib is the name of the Bibliography in this case
%% You must have a proper ".bib" file
%%  and remember to run:
%% latex bibtex latex latex
%% to resolve all references
%%
%% ACM needs 'a single self-contained file'!
%%
%%APPENDICES are optional
%%\balancecolumns
%%Appendix A
%%\balancecolumns % GM June 2007
%% That's all folks!
%\end{document}
