% This is "sig-alternate.tex" V1.9 April 2009
% This file should be compiled with V2.4 of "sig-alternate.cls" April 2009
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.4 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.4) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
% \CopyrightYear{2007} will cause 2007 to appear in the copyright line.
% \crdata{0-12345-67-8/90/12} will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
	% Technical questions _only_ to
	% Gerald Murray (murray@hq.acm.org)
% ===============================================================
%
% For tracking purposes - this is V1.9 - April 2009

\documentclass{sig-alternate}
\usepackage{cite,times}
\usepackage[usenames,dvipsnames]{color}
\usepackage{url}
\usepackage{tim/tricks}
\usepackage{raj/tricks}
\usepackage{markus/tricks}

\begin{document}
%
% --- Author Metadata here ---
\conferenceinfo{MSR}{'11 Honolulu,  Hawaii USA}
%\CopyrightYear{2007} % Allows default copyright year (20XX) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{``Popularity'' is (almost) a Perfect Predictor for Defects}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{2} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Markus Lumpe, Rajesh Vasa\\
       \affaddr{Faculty of ICT}\\
%       \affaddr{Faculty of Information \& Communication Technologies}\\
       \affaddr{Swinburne University of Technology}\\
       \affaddr{Hawthorn, Australia}\\
       \email{\{mlumpe,rvasa\}@swin.edu.au}
\alignauthor
Tim Menzies\\
       \affaddr{Lane Department of Com. Sci. \& Elec. Eng.}\\
%       \affaddr{Lane Department of Computer Science \& Electrical Engineering}\\
       \affaddr{West Virginia University}\\
       \affaddr{Morgantown, USA}\\
       \email{tim@menzies.us}
}

% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
\date{26 Jan 2011}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
BACKGROUND:
Recent
research has shown the value of social metrics for defect prediction. 
But many  repositories lack the
developer information required from a social
metrics analysis.
\\
AIM:
Do we need to redesign our repositories, throwing away all
the old data that lacks social metrics?
\\
METHOD:
We  evaluate social pattern metrics inferred from repositories
that only contain static code attributes.
The ``popularity'' (i.e. frequency of use by developers) of a class is inferred
from 
repositories that lack details about individual developers.
The value of these popularity measures  is assessed in context of 
``inspection optimization'' (reading the fewest lines of code to find the most defects).
\\
RESULTS:
For the task of inspection
optimization
these popularity measures perform 
as well
(usually, within 4\%) 
of the theoretical upper bound on the performance  
of any set of measures.
\\
CONCLUSION:
Defect prediction research about 
static code measures has not been superseded by
social metrics.
There is still much to be discovered in repositories
that lack detailed developer information; e.g.
popularity is
(almost) a perfect predictor for defects. 
\\
FUTURE DIRECTIONS:
The value of theses popularity measures should be tested for
other tasks (i.e. not
just
inspection optimization) and other systems
(i.e. not just
open-source, object-oriented JAVA programs). 
Also, it would be insightful to evaluate other measures of social patterns
inferred
from repositories that lack developer information.
\end{abstract}

% A category with the (minimum) three required fields
%A category including the fourth, optional field follows...
\category{D.2.8}{Software Engineering}{Metrics}[complexity measures, performance measures]

\input{Intro}
\input{Popular}

\input{Prediction}

\input{InspectionOptimization}
%\input{markus/InspectionOptimizationOld}

\section{Popularity and Inspection Optimization}
To assess the value of the measures in \fig{metrics}, we use them to
build defect predictors. Then, after applying them to
33 open source JAVA projects from the Helix porject, we report the
their 
$\frac{learning}{oracle}$ ratio.
As shown below, the median values for that ratio will be 96\%, i.e. very close to the theoretical
upper bound possible for any defect predictor for the the task of inspection optimization.

\input{raj/data}


%
%\begin{figure*}
%\noindent
%\includegraphics[width=2in]{img/ant-13.pdf}        
%\includegraphics[width=2in]{img/ant-14.pdf}     
%\includegraphics[width=2in]{img/ant-15.pdf}     
%
%\noindent
%\includegraphics[width=2in]{img/ant-16.pdf}       
%\includegraphics[width=2in]{img/ant-17.pdf} 
%\includegraphics[width=2in]{img/lucene-20.pdf} 
%
%\noindent
%\includegraphics[width=2in]{img/lucene-22.pdf}
%\includegraphics[width=2in]{img/lucene-24.pdf}
%\includegraphics[width=2in]{img/xalan-24.pdf}
%
%\noindent
%\includegraphics[width=2in]{img/xalan-25.pdf}
%\includegraphics[width=2in]{img/xalan-26.pdf}
%\includegraphics[width=2in]{img/xalan-27.pdf}
%\caption{Results}
%\end{figure*}
%
\section{Results}

\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.7}
\small
\begin{center}
\begin{tabular}{l|rrrrr}
    &              \multicolumn{5}{c}{percentiles}\\
    data         &        10\%&  30\% &   50\% & 70\%  &  90\%\\\hline
lucene-2.4 &  -2.0 &  -0.9 &  -0.3 &   0.4 &   2.6 \\ 
velocity-1.6 &  -1.4 &  -0.6 &  -0.3 &   0.0 &   1.4 \\ 
xerces-1.0 &  -1.1 &  -0.6 &  -0.3 &   0.7 &   1.0 \\ 
ant-1.4 &  -0.4 &  -0.3 &  -0.2 &  -0.1 &   0.8 \\ 
lucene-2.0 &  -2.0 &  -0.8 &  -0.2 &   0.1 &   1.7 \\ 
poi-1.5 &  -1.6 &  -0.9 &  -0.2 &   0.0 &   2.3 \\ 
synapse-1.2 &  -0.8 &  -0.4 &  -0.2 &   0.0 &   1.0 \\ 
xalan-2.5 &  -0.9 &  -0.5 &  -0.2 &   0.5 &   0.8 \\ 
xalan-2.6 &  -0.9 &  -0.5 &  -0.2 &   0.4 &   1.2 \\ 
xalan-2.7 &  -0.5 &  -0.3 &  -0.2 &   0.1 &   0.7 \\ 
xerces-1.2 &  -0.5 &  -0.2 &  -0.2 &  0.0 &   0.9 \\ 
xerces-1.4 &  -2.2 &  -0.4 &  -0.2 &   0.4 &   0.8 \\ 
ant-1.3 &  -0.5 &  -0.3 &  -0.1 &   0.0 &   0.6 \\ 
ant-1.7 &  -0.8 &  -0.3 &  -0.1 &   0.0 &   0.8 \\ 
ivy-2.0 &  -0.3 &  -0.1 &  -0.1 &   0.0 &   0.3 \\ 
lucene-2.2 &  -2.5 &  -0.8 &  -0.1 &   0.4 &   2.0 \\ 
poi-2.0 &  -0.2 &  -0.1 &  -0.1 &  -0.1 &   0.7 \\ 
poi-3.0 &  -1.1 &  -0.4 &  -0.1 &   0.0 &   1.0 \\ 
synapse-1.0 &  -0.3 &  -0.2 &  -0.1 &   0.0 &   0.0 \\ 
synapse-1.1 &  -0.7 &  -0.4 &  -0.1 &   0.0 &   0.9 \\ 
velocity-1.5 &  -1.5 &  -0.7 &  -0.1 &   0.3 &   1.5 \\ 
xalan-2.4 &  -0.5 &  -0.2 &  -0.1 &   0.0 &   0.7 \\ 
ant-1.5 &  -0.3 &  -0.1 &  0.0 &   0.0 &   0.3 \\ 
ant-1.6 &  -0.8 &  -0.3 &  0.0 &   0.0 &   0.9 \\ 
ivy-1.4 &  -0.2 &  -0.1 &  0.0 &   0.0 &   0.0 \\ 
jedit-3.2 &  -2.3 &  -0.8 &   0.0 &   0.0 &   1.8 \\ 
jedit-4.0 &  -1.3 &  -0.5 &   0.0 &   0.0 &   1.0 \\ 
jedit-4.1 &  -1.1 &  -0.4 &  0.0 &   0.0 &   1.0 \\ 
jedit-4.2 &  -0.6 &  -0.2 &  0.0 &   0.0 &   0.3 \\ 
jedit-4.3 &  -0.1 &  0.0 &  0.0 &  0.0 &   0.0 \\ 
poi-2.5 &  -1.3 &  -0.6 &  0.0 &   0.7 &   0.9 \\ 
velocity-1.4 &  -1.2 &  -0.2 &  0.0 &   0.1 &   1.0 \\ 
xerces-1.3 &  -1.4 &  -0.2 &  0.0 &   0.0 &   0.7  
\end{tabular}
\end{center}
\caption{Percentile distributions of $actual - predicted$ number of defects per class. For example,
the median (50th percentile) value of $actual - predicted$ is -0.3 to 0.}\label{fig:ar}
	\end{figure}

\subsection{Sanity Check}

\fig{ar} shows the distribution of $actual - predicted$ defects
for our classes where $actual$ comes from historical logs and $predicted$ comes from
the $C$ predictions seen in our 10-way:
\bi
\item
This result is our {\em sanity check}: if the $actual - predicted$ values were large, then we would doubt the value
of popularity-based defect prediction. 
\item
Note that, in the median case (shown in the middle 50\% column), the predictions are very close to actuals (-0 to -0.4).
\item
Since our estimates are close to actuals,  we may continue.
\ei
\subsection{$\frac{learned}{optimal}$ Results}

\fig{pm} shows the effectiveness of the popularity measures,
expressed as $\frac{learned}{optimal}$.  Recall that this was generated as follows:
\bi
\item Use the LSR predictions over the popularity measures to  guess the number of defects in some test set.
\item Sort the classes in that test set  the number of defects per class and the size of class
(favoring the smaler classes with most predicted defects) using \eq{score}.
\ei
Note
the median value of that figure: our popularity-based metrics perform
within 4\% of the optimal. Hence, we say that not only do popularity-based
measures generator
 good predictors, but also that  the addition of the standard social metrics
(e.g. containing detailed developer knowledge) would add very little extra value.
\newcommand{\wwidth}{2mm}
\begin{figure}[!t]
\renewcommand{\baselinestretch}{0.7}
\small
\begin{center}
\begin{tabular}{lrlr}
synapse-1.0 & 74 & \textcolor[named]{Gray}{\rule{24mm}{\wwidth}}\\
ant-1.3 & 77 & \textcolor[named]{Gray}{\rule{25mm}{\wwidth}}\\
poi-2.0 & 78 & \textcolor[named]{Gray}{\rule{25mm}{\wwidth}}\\
jedit-4.3 & 81 & \textcolor[named]{Gray}{\rule{26mm}{\wwidth}}\\
ivy-2.0 & 85 & \textcolor[named]{Gray}{\rule{28mm}{\wwidth}}\\
ivy-1.4 & 89 & \textcolor[named]{Gray}{\rule{29mm}{\wwidth}}\\
ant-1.5 & 90 & \textcolor[named]{Gray}{\rule{29mm}{\wwidth}}\\
velocity-1.6 & 91 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
synapse-1.1 & 92 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
poi-3.0 & 93 & \textcolor[named]{Gray}{\rule{30mm}{\wwidth}}\\
ant-1.7 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
lucene-2.0 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
velocity-1.5 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
xerces-1.3 & 94 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
ant-1.4 & 95 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
poi-1.5 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
poi-2.5 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
xerces-1.4 & 96 & \textcolor[named]{Gray}{\rule{31mm}{\wwidth}}\\
synapse-1.2 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.4 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xerces-1.0 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xerces-1.2 & 97 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.5 & 98 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.6 & 98 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
lucene-2.4 & 99 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
xalan-2.7 & 99 & \textcolor[named]{Gray}{\rule{32mm}{\wwidth}}\\
ant-1.6 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-3.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.0 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.1 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
jedit-4.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
lucene-2.2 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}\\
velocity-1.4 & 100 & \textcolor[named]{Gray}{\rule{33mm}{\wwidth}}
\end{tabular}
\end{center}
\caption{The median $\frac{learned}{optimal}\%$ result is 96\%
(at poi-2.5).}\label{fig:pm}
\end{figure}


\begin{figure*}[!t]
{\small
\begin{center}
\begin{tabular}{r|rrrrrrrr}
&  &  &      &   &  &   & Clustering \\
& &     OutDegree  &  Getters    &   Setters &NoM &InDegree  & Coefficient \\
data set&  $\beta_0$  &   $\beta_1$   &   $\beta_2$  &  $\beta_3$    &   $\beta_4$ &$\beta_5$ &$\beta_6$  \\\hline

ant-1.3  &-0.1351&0.0406&&-0.0183&&&&  \\
ant-1.4  &0.1459&0.0167&&0.0218&-0.0133&&&  \\
ant-1.5  &-0.0631&0.0135&-0.0325&0.0293&&&&  \\
ant-1.6  &-0.043&0.0466&-0.0454&-0.02&0.0486&&&  \\
ant-1.7  &-0.0564&0.046&-0.0562&-0.0283&0.0546&&&  \\
ivy-1.4  &-0.0066&0.0044&-0.0355&&0.0115&&&  \\
ivy-2.0  &-0.0386&0.013&&-0.0226&0.0087&&&  \\
jedit-3.2  &-1.0808&0.1099&0.4024&0.6921&&0.0522&&  \\
jedit-4.0  &-0.631&0.0441&0.3699&&&0.0532&&  \\
jedit-4.1  &-0.5152&0.057&0.1562&&&0.0378&&  \\
jedit-4.2  &-0.2777&0.0187&0.1275&&&0.0195&&  \\
jedit-4.3  &-0.0052&0.0016&0.0143&&-0.0007&&&  \\
lucene-2.0  &-0.7125&0.1277&&0.5153&0.0907&0.06&&  \\
lucene-2.2  &0.5616&0.0553&&1.0931&0.1688&&&  \\
lucene-2.4  &0.5217&0.0392&0.1497&-0.5903&0.1572&0.0675&&  \\
poi-1.5  &0.261&0.0593&0.1426&-0.3501&0.1113&&-1.5802&  \\
poi-2.0  &0.006&0.0047&&-0.0168&0.0087&&&  \\
poi-2.5  &-0.014&0.0331&0.1165&-0.1283&0.0387&&&  \\
poi-3.0  &-0.1698&0.0528&0.1648&-0.1918&0.0537&&&  \\
synapse-1.0  &0.025&0.022&&&&&&  \\
synapse-1.1  &-0.0713&0.0515&&0.0951&-0.0279&&&  \\
synapse-1.2  &-0.1312&0.0565&0.0771&&-0.0207&0.0154&&  \\
velocity-1.4  &0.8674&0.0198&0.0741&&-0.0104&0.0131&&  \\
velocity-1.5  &0.2875&0.1333&0.3349&-0.1941&&0.0258&0.8271&  \\
velocity-1.6  &0.2705&0.0814&0.1209&-0.218&0.0358&&0.9503&  \\
xalan-2.4  &-0.0722&0.0267&&&0.0075&&&  \\
xalan-2.5  &0.208&0.0301&0.0242&0.0426&&&-0.3072&  \\
xalan-2.6  &0.4958&0.0175&-0.0403&0.0707&0.02&&-0.5815&  \\
xalan-2.7  &1.2681&0.0155&-0.0453&0.0963&0.0042&&-0.4668&  \\
xerces-1.0  &0.1916&&0.2417&&0.0278&&&  \\
xerces-1.2  &-0.2139&0.0204&0.0298&-0.0397&&0.0129&0.6292&  \\
xerces-1.3  &-0.2716&0.1534&-0.0646&&&&&  \\
xerces-1.4  &-0.1181&0.141&0.2912&-0.4426&0.0865&0.0478&&  \\\hline
min&-1.0808&0.0016&-0.0646&-0.5903&-0.0279&0.0129&-1.5802&\\
median&-0.0386&0.0399&0.0968&-0.01915&0.0239&0.0378&-0.3072\\
max&1.2681&0.1534&0.4024&1.0931&0.1688&0.0675&0.9503\\\hline
\%used&100\%&97\%&73\%&67\%&67\%&33\%&21\%&




\end{tabular}
\end{center}}
\caption{LSR results for 
$defects = \beta_0 + \left(\sum^7_i\beta_ix_i\right)$.
Rows are sorted alphabetically.
Columns are sorted according to how ofter a feature appears in the learned equation (see the last row).
The bottom rows for {\em min}, {\em median}, and {\em max} report a wide range in the
learned co-effecients for the popularity-based defect predictors.
}\label{fig:eq}
\end{figure*}
\subsection{What Popularity Measures are ``Best''?}
\fig{eq} shows the co-efficients on the
linear regression
equations learned over the popularity measures. 
Recall that the 
linear regression tool we used (from WEKA 3.7.3~\cite{witten05}) includes
a greedy back-select that removes those features that do not degrade
the overall performance of the equation.
Observe that there are many blank cells in \fig{eq}:
\bi
\item
The models learned from different data sets used different
popularity measures.
\item
No popularity measure was used in all data sets (max 97\% of the data, often much less).
\ei
The bottom rows in that figure report the {\em min}, {\em median}, and {\em max} co-effecient
values learned in this study.
The range in these values was very large (e.g. $-1.08 \le \beta_0 \le 1.27$).
Combining the results of
\fig{pm} and \fig{eq}, we say that:
\bi
\item
Some combination of popularity-based measure was very useful in nearly all
the  data sets
studied here.
\item
The exact effect of particular measures varied from data set to data set.
\ei
\subsection{Summary}
\bi
\item
According to \fig{ar}, popularity-measures plus linear regression lead to defect predictors with low error
rates in open-source OO systems.
\item
According to \fig{pm}, for the task of inspection optimization,
popularity-based defect prediction works very close to the optimum.
\item
However, 
\fig{eq} shows that 
that while multiple popularity measures are useful, no particular singularity measure is always most effective.
Hence,
we  recommend againts focusing on particular popularity measures.
Rather,
we recommend applying them all.
\ei

%\subsection{very good}
%
%very good. compare to ~\cite{me10a}, must better.
%\subsection{very simple}
%note the absence of complex machinery: SVM, feature selection, random forests, etc.
%
%Instead, a simple theory of human behavior (usage predicts for defects) plus a simple
%data miner (linear regression) lead to predictions of very high competency (median results,
%in \fig{pm}, within 3\% of optimal). Interesting... perhaps there has been too much
%AI and not enough human intelligence in prior research on defect prediction.
%\subsection{ANT}
%Worst results in ANT. Why?
%\section{Future Work}
%Could apply the \cite{me10a} tools to here. But given the minimal room for improvement
%(usually, less than 3\%), why bother?
%
%More defect data sets.
%


