\section{Strange Tales of Defect Prediction}
\frame { \frametitle{Ceiling Effects}
\scriptsize
\bi
\item
	An inherent upper bound on the performance of our data miners
\item Have yet to improve our mid-2006 defect predictors: ~\cite{me07b}
\ei

\begin{columns}
\begin{column}{2in}
\includegraphics[height=1.5in]{jiang.png}

~\\ \cite{jiang07}: $\frac{6}{9}$ ``best''
\end{column}
\begin{column}{2in}
\includegraphics[height=1.5in]{all.pdf}

~\\ \cite{lessmann09}: $\frac{14}{19}$ ``best'' 
\end{column}
\end{columns}
}
\frame { \frametitle{More Ceiling Effects (Incremental Learning)}
\bi
\scriptsize \item Randomized order of data; learn from ``X''; test on next 100.

\includegraphics[angle=90,height=.7in]{incrementalKC1.pdf}
\includegraphics[angle=90,height=.7in]{incrementalPC1.pdf}
\item
Little or no improvement after learning from 100 examples.
\item
Micro-sampling: $N$ defective + $N$ non-defective, $N \in \{25, 50, 75, ...\} $

\includegraphics[angle=90,height=.7in]{underKC1.pdf}
\includegraphics[angle=90,height=.7in]{underPC1.pdf}
\item Little or no improvement after learning from 50 examples.
\ei

}
%\frame { \frametitle{Micro Sampling (1 of 2) }
%\bi\scriptsize
%\item
%Under-sampling: throw away the majority class: builds data sets twice the size of the minority class
%\item
%Over-sampling: repeat copies of the majority class- builds data sets twice the size of the majority class.
%\ei
%\begin{center}
%{\tiny\begin{tabular}{cllrc} 
%\multicolumn{3}{c}{~}& \multicolumn{1}{c}{2nd quartile, median }\\
%rank&treatment&median& \multicolumn{1}{c}{and 3rd quartile}\\\hline
%1&NB/ none & 74.6& \boxplot{21.9}{67.7}{74.6}{14}{100}\\ 
%1&NB/ under& 74.1& \boxplot{19.9}{67.1}{74.1}{15}{100}\\
%1&j48/ under& 73.6& \boxplot{21.6}{64.8}{73.6}{18}{100}\\ 
%2&NB/ over  &62.5&\boxplot{17.5}{42}{62.5}{30}{100}\\
%3&j48/ over &  45.6&\boxplot{0}{29.3}{45.6}{27}{100}\\
%4&j48/ none & 42.3&\boxplot{0}{29.3}{42.3}{25}{100}\\\hline
%&& & \multicolumn{1}{c}{50\%} 
%\end{tabular}
%
%NB= N\aive Bayes ; j48 = JAVA port of C4.5
%}
%\end{center}
%\bi
%\scriptsize 
%\item Under-sampling:  (a)~improves C4.5; (b)~does not damge N\aive Bayes
%}
