
\begin{figure}
\renewcommand{\baselinestretch}{0.5}
\begin{center}
{\scriptsize\begin{tabular}{l@{~}l@{~}l@{~}r@{~}r@{~}r@{~}r@{~}r@{~}c}
& & &\multicolumn{5}{c}{quartiles}  &\\\cline{4-8}
& rank & & 0 & 25 & 50 & 75 & 100 &\\\hline

CM1 &1&   CC&98&98&98&98&98&\boxplot{  98.0}{0}{  98.0}{  98.0}{0}\\
&2& NN&76&82&82&84&84&\boxplot{  76.0}{6}{  82.0}{  84.0}{0}\\
&3&   WC&20&60&80&80&100&\boxplot{  20.0}{40}{  80.0}{  80.0}{20}\\\hline


MW1&1&   CC&90&90&90&90&90&\boxplot{  90.0}{0}{  90.0}{  90.0}{0}\\
&2& NN&68&68&68&68&71&\boxplot{  68.0}{0}{  68.0}{  68.0}{3}\\
&2&   WC&0&50&50&75&100&\boxplot{   0.0}{50}{  50.0}{  75.0}{25}\\\hline

PC1&1&   CC&99&99&99&99&99&\boxplot{  99.0}{0}{  99.0}{  99.0}{0}\\
&2& NN&74&77&77&77&78&\boxplot{  74.0}{3}{  77.0}{  77.0}{1}\\
&3&   WC&38&62&62&75&100&\boxplot{  38.0}{24}{  62.0}{  75.0}{25}\\\hline

%AR4&1&   CC&35&90&100&100&100&\boxplot{  35.0}{55}{ 100.0}{ 100.0}{0}\\
%&2& NN&65&65&70&70&70&\boxplot{  65.0}{0}{  70.0}{  70.0}{0}\\
%&3&   WC&35&40&40&40&45&\boxplot{  35.0}{5}{  40.0}{  40.0}{5}\\\hline

%AR3&1&   CC&75&88&88&88&88&\boxplot{  75.0}{13}{  88.0}{  88.0}{0}\\
%&1& NN&88&88&88&88&88&\boxplot{  88.0}{0}{  88.0}{  88.0}{0}\\
%&1&   WC&88&88&88&88&88&\boxplot{  88.0}{0}{  88.0}{  88.0}{0}\\\hline

%AR5&1&   CC&88&100&100&100&100&\boxplot{  88.0}{12}{ 100.0}{ 100.0}{0}\\
%&1& NN&100&100&100&100&100&\boxplot{ 100.0}{0}{ 100.0}{ 100.0}{0}\\
%&1&   WC&88&100&100&100&100&\boxplot{  88.0}{12}{ 100.0}{ 100.0}{0}\\\hline
\multicolumn{8}{c}{~}&50\%
\end{tabular}}
\end{center}
\caption{
 Experiment \#2 PD results where
$NN_{pd} \ge WC_{pd}$. 
Rankings computed via Mann-Whitney (95\% 
confidence) comparing each row to all other rows.}\label{fig:pd1}
\end{figure}
\begin{figure}
\begin{center}
\renewcommand{\baselinestretch}{0.5}
{\scriptsize\begin{tabular}{l@{~}l@{~}l@{~}r@{~}r@{~}r@{~}r@{~}r@{~}c}

& & &\multicolumn{5}{c}{quartiles}  &\\\cline{4-8}
& rank & & 0 & 25 & 50 & 75 & 100 &\\\hline

KC1 &1&   CC&94&94&94&94&94&\boxplot{  94.0}{0}{  94.0}{  94.0}{0}\\
&2&   WC&64&76&82&85&94&\boxplot{  64.0}{12}{  82.0}{  85.0}{9}\\
&3& NN&60&64&65&66&69&\boxplot{  60.0}{4}{  65.0}{  66.0}{3}\\\hline

KC2&1&   CC&94&94&94&94&94&\boxplot{  94.0}{0}{  94.0}{  94.0}{0}\\
&2&   WC&45&73&82&91&100&\boxplot{  45.0}{28}{  82.0}{  91.0}{9}\\
&2& NN&77&78&79&79&80&\boxplot{  77.0}{1}{  79.0}{  79.0}{1}\\\hline

KC3&1&   CC&81&81&81&84&84&\boxplot{  81.0}{0}{  81.0}{  84.0}{0}\\
&2&   WC&20&60&80&100&100&\boxplot{  20.0}{40}{  80.0}{ 100.0}{0}\\
&3& NN&60&63&65&67&70&\boxplot{  60.0}{3}{  65.0}{  67.0}{3}\\\hline

MC2&1&   CC&83&83&83&83&85&\boxplot{  83.0}{0}{  83.0}{  83.0}{2}\\
&2&   WC&17&50&67&83&100&\boxplot{  17.0}{33}{  67.0}{  83.0}{17}\\
&3& NN&56&56&56&56&58&\boxplot{  56.0}{0}{  56.0}{  56.0}{2}\\\hline
\multicolumn{8}{c}{~}&50\%
\end{tabular}}
\end{center}
\caption{
Experiment \#2 PD results where
$NN_{pd} < WC_{pd}$. 
}\label{fig:pd2}
\end{figure}
\begin{figure}
\begin{center}
\renewcommand{\baselinestretch}{0.5}
{\scriptsize\begin{tabular}{l@{~}l@{~}l@{~}r@{~}r@{~}r@{~}r@{~}r@{~}c}

& & &\multicolumn{5}{c}{quartiles}  &\\\cline{4-8}
& rank & & 0 & 25 & 50 & 75 & 100 &\\\hline

KC1&1& NN&22&23&24&25&27&\boxplot{  22.0}{1}{  24.0}{  25.0}{2}\\
&2&   WC&26&32&35&37&43&\boxplot{  26.0}{6}{  35.0}{  37.0}{6}\\
&3&   CC&59&60&60&60&60&\boxplot{  59.0}{1}{  60.0}{  60.0}{0}\\\hline

KC2&1& NN&24&25&25&25&27&\boxplot{  24.0}{1}{  25.0}{  25.0}{2}\\
&1&   WC&10&21&26&31&40&\boxplot{  10.0}{11}{  26.0}{  31.0}{9}\\
&2&   CC&67&67&67&67&67&\boxplot{  67.0}{0}{  67.0}{  67.0}{0}\\\hline

KC3&1& NN&17&18&18&19&20&\boxplot{  17.0}{1}{  18.0}{  19.0}{1}\\
&2&   WC&7&17&21&26&31&\boxplot{   7.0}{10}{  21.0}{  26.0}{5}\\
&3&   CC&26&27&27&27&27&\boxplot{  26.0}{1}{  27.0}{  27.0}{0}\\\hline

MC2&1& NN&29&30&31&32&35&\boxplot{  29.0}{1}{  31.0}{  32.0}{3}\\
&2&   WC&0&27&36&45&73&\boxplot{   0.0}{27}{  36.0}{  45.0}{28}\\
&3&   CC&71&71&71&71&71&\boxplot{  71.0}{0}{  71.0}{  71.0}{0}\\\hline
%AR3&1& NN&38&38&38&38&40&\boxplot{  38.0}{0}{  38.0}{  38.0}{2}\\
%&2&   WC&40&40&40&40&42&\boxplot{  40.0}{0}{  40.0}{  40.0}{2}\\
%&3&   CC&38&55&56&60&80&\boxplot{  38.0}{17}{  56.0}{  60.0}{20}\\\hline

%AR5&1& NN&21&25&25&25&32&\boxplot{  21.0}{4}{  25.0}{  25.0}{7}\\
%&2&   WC&29&29&29&29&29&\boxplot{  29.0}{0}{  29.0}{  29.0}{0}\\
%&3&   CC&29&46&50&50&79&\boxplot{  29.0}{17}{  50.0}{  50.0}{29}\\\hline
\multicolumn{8}{c}{~}&50\%
\end{tabular}}
\end{center}
\caption{Experiments \#2 PF results 
where $NN_{pf} \le WC_{pf}$.}\label{fig:pf1}
\end{figure}

\begin{figure}
\begin{center}
\renewcommand{\baselinestretch}{0.5}
{\scriptsize\begin{tabular}{l@{~}l@{~}l@{~}r@{~}r@{~}r@{~}r@{~}r@{~}c}
& & &\multicolumn{5}{c}{quartiles}  &\\\cline{4-8}
& rank & & 0 & 25 & 50 & 75 & 100 &\\\hline
CM1&1&   WC&16&29&33&38&49&\boxplot{  16.0}{13}{  33.0}{  38.0}{11}\\
&2& NN&40&43&44&45&46&\boxplot{  40.0}{3}{  44.0}{  45.0}{1}\\
&3&   CC&90&91&91&91&93&\boxplot{  90.0}{1}{  91.0}{  91.0}{2}\\\hline

MW1&1&   WC&8&21&26&29&47&\boxplot{   8.0}{13}{  26.0}{  29.0}{18}\\
&2& NN&30&32&33&33&36&\boxplot{  30.0}{2}{  33.0}{  33.0}{3}\\
&3&   CC&67&68&68&69&70&\boxplot{  67.0}{1}{  68.0}{  69.0}{1}\\\hline

PC1&1&   WC&16&24&28&31&40&\boxplot{  16.0}{8}{  28.0}{  31.0}{9}\\
&2& NN&45&48&48&49&53&\boxplot{  45.0}{3}{  48.0}{  49.0}{4}\\
&3&   CC&94&94&94&94&94&\boxplot{  94.0}{0}{  94.0}{  94.0}{0}\\\hline

%AR4&1&   WC&3&3&3&5&7&\boxplot{   3.0}{0}{   3.0}{   5.0}{2}\\
%&2& NN&24&25&25&25&28&\boxplot{  24.0}{1}{  25.0}{  25.0}{3}\\
%&3&   CC&6&47&62&67&78&\boxplot{   6.0}{41}{  62.0}{  67.0}{11}\\\hline
\multicolumn{8}{c}{~}&50\%
\end{tabular}}
\end{center}
\caption{  Experiment \#2 PF results where $NN_{pf} > WC_{pf}$.  }\label{fig:pf2}
\end{figure}
Showing that NN is significantly better than CC, 
the ideal result would be that NN can be used as a surrogate for local WC data.
This, in turn, would mean that 
developers could avoid the tedious and expensive work of local data collection. We now investigate the
relation between NN and WC. 


If NN out-performed WC then two observations would appear:
\begin{itemize}
\item {\em Observation1:} NN would have $pd$ values above or equal to WC's $pd$.
The examples displaying $Observation1$ are shown in \fig{pd1}.
\item {\em Observation2:} NN would have $pf$ values below or equal to WC's $pf$.
The examples displaying $Observation2$ are shown in \fig{pf1}.
\end{itemize}
Please note that the conjunction of
$Observation1$ and $Observation2$ is uncommon.
%\begin{itemize}
%\item Only two data sets
%are found in both \fig{pd1} and \fig{pf1} (AR3 and AR5).
%\end{itemize}
In fact, our results
suggest that $Observation1$ and $Observation2$ are somewhat mutually exclusive:
\begin{itemize}
\item
As shown in Figures \ref{fig:pd1} and \ref{fig:pf2}, the examples where $NN$ increases
the probability of detection are also those where it increases the probability of false alarms.
\end{itemize}
Hence, we cannot recommend NN as a replacement for WC.
Nevertheless, if local WC data are not available, 
then we would recommend processing foreign CC data with NN.
%\item
%NN dramatically reduces the variance in the predictions:
%In all the Experiment \#2 results, the NN results cluster tightly around the median
%value. The same is not true for the other methods: e.g. MC2's WC results in \fig{pf2}
%range from 0 to 73\%. Such reduction variance is useful for 
%In all cases, NN dramatically reduces the high false alarm rates associated with
%the use of cross-company data.
%Often that reduction  halves
%the false alarm range. For example, in MW1, the false alarm rate drops
%between CC to NN from 68\% to 33\%.



%However, the use of local WC data is still endorsed by these results:
%\begin{itemize}
%\item Combining Figures \fig{ref:pd2} and \fig{ref:pd2a},
%we see that NN leads to predictors with higher probability of detection than
%local WC data  in $\frac{6}{10}$ of our examples.
%\item Of the remaining
%\item NN can lead to defect predictors with a 
%\item Nearly half the time (the $\frac{4}{10}$ examples of \fig{pf2}),
%using local WC data produces false alarm rates much lower than NN.
%\item
%In the remaining 
%examples (the $\frac{6}{10}$ examples of \fig{pf2}),
%while NN resulted in lower (i.e. better) false alarm rates than using
%local WC data, the $pf$ difference between WC and NN is not large.
%\ei

