\begin{table*}[!h]
\begin{center}
{\tiny
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | c |}\hline
\multicolumn{11}{|c|}{Rules for flawless with 2 bins}\\\hline
Training & Testing & A & B & C & D & Pd & Pf & Prec & Acc & Rec \\\hline

ANT-A & ANT-B & 40 & 138 & 0 & 0 & 0.0 & 100.0 & 0.0 & 22.47 & \$MOA=0-0\\
ANT-AB & ANT-C & 29 & 169 & 3 & 92 & 35.25 & 90.62 & 96.84 & 41.3 & \$DAM=1-1\\
ANT-ABC & ANT-D & 92 & 259 & 0 & 0 & 0.0 & 100.0 & 0.0 & 26.21 & \$MOA=0-0\\
ANT-ABCD & ANT-E & 166 & 579 & 0 & 0 & 0.0 & 100.0 & 0.0 & 22.28 & \$MOA=0-0\\\hline

CAMEL-A & CAMEL-B & 216 & 392 & 0 & 0 & 0.0 & 100.0 & 0.0 & 35.53 & \$NOC=0-0\\
CAMEL-AB & CAMEL-C & 145 & 727 & 0 & 0 & 0.0 & 100.0 & 0.0 & 16.63 & \$NOC=0-0\\
CAMEL-ABC & CAMEL-D & 188 & 777 & 0 & 0 & 0.0 & 100.0 & 0.0 & 19.48 & \$MOA=0-0\\\hline

FORREST-AB & FORREST-C & 2 & 30 & 0 & 0 & 0.0 & 100.0 & 0.0 & 6.25 & \$MOA=0-0\\\hline

JEDIT-A & JEDIT-B & 75 & 231 & 0 & 0 & 0.0 & 100.0 & 0.0 & 24.51 & \$MOA=0-0\\
JEDIT-AB & JEDIT-C & 79 & 233 & 0 & 0 & 0.0 & 100.0 & 0.0 & 25.32 & \$MOA=0-0\\
JEDIT-ABC & JEDIT-D & 48 & 319 & 0 & 0 & 0.0 & 100.0 & 0.0 & 13.08 & \$MOA=0-0\\
JEDIT-ABCD & JEDIT-E & 11 & 481 & 0 & 0 & 0.0 & 100.0 & 0.0 & 2.236 & \$MOA=0-0\\\hline

LOG4J-A & LOG4J-B & 37 & 72 & 0 & 0 & 0.0 & 100.0 & 0.0 & 33.94 & \$DAM=0-0\\
LOG4J-AB & LOG4J-C & 189 & 16 & 0 & 0 & 0.0 & 100.0 & 0.0 & 92.2 & \$DAM=0-0\\\hline

LUCENE-A & LUCENE-B & 28 & 43 & 116 & 60 & 58.25 & 19.44 & 34.09 & 35.63 & \$LCOM3=1-2\\
LUCENE-AB & LUCENE-C & 41 & 49 & 162 & 88 & 64.23 & 20.2 & 35.2 & 37.94 & \$LCOM3=1-2\\\hline

PBEANS-A & PBEANS-B & 1 & 1 & 40 & 9 & 90.0 & 2.439 & 18.37 & 19.61 & \$MAX\_CC=1-27\\\hline

POI-A & POI-B & 75 & 9 & 202 & 28 & 75.68 & 27.08 & 12.17 & 32.8 & \$LCOM3=1-2\\
POI-AB & POI-C & 2 & 1 & 135 & 247 & 99.6 & 1.46 & 64.66 & 64.68 & \$AVG\_CC=1-14.2222\\
POI-ABC & POI-D & 0 & 1 & 161 & 280 & 99.64 & 0.0 & 63.49 & 63.35 & \$AVG\_CC=1-14.2222\\\hline

PROP-A & PROP-B & 6 & 20 & 2425 & 20563 & 99.9 & .2468 & 89.45 & 89.38 & \$AVG\_CC=1-22.0833\\
PROP-AB & PROP-C & 8 & 16 & 1172 & 9078 & 99.82 & 0.678 & 88.57 & 88.44 & \$AVG\_CC=1-25.5\\
PROP-ABC & PROP-D & 2 & 6 & 838 & 7872 & 99.92 & .2381 & 90.38 & 90.32 & \$AVG\_CC=1-26.2\\
PROP-ABCD & PROP-E & 26 & 308 & 1273 & 6909 & 95.73 & 2.002 & 84.44 & 81.43 & \$DAM=0-0\$DAM=1-1\\
PROP-ABCDE & PROP-F & 47 & 334 & 19 & 260 & 43.77 & 71.21 & 93.19 & 46.52 & \$DAM=0-0\$DAM=1-1\\\hline

SYNAPSE-A & SYNAPSE-B & 60 & 162 & 0 & 0 & 0.0 & 100.0 & 0.0 & 27.03 & \$MOA=0-0\\
SYNAPSE-AB & SYNAPSE-C & 86 & 170 & 0 & 0 & 0.0 & 100.0 & 0.0 & 33.59 & \$MOA=0-0\\\hline

VELOCITY-A & VELOCITY-B & 0 & 2 & 72 & 140 & 98.59 & 0.0 & 66.04 & 65.42 & \$AVG\_CC=1-10.5426\\
VELOCITY-AB & VELOCITY-C & 1 & 1 & 150 & 77 & 98.72 & .6623 & 33.92 & 34.06 & \$AVG\_CC=1-12.8222\\\hline

XALAN-A & XALAN-B & 387 & 416 & 0 & 0 & 0.0 & 100.0 & 0.0 & 48.19 & \$MOA=0-0\\
XALAN-AB & XALAN-C & 411 & 474 & 0 & 0 & 0.0 & 100.0 & 0.0 & 46.44 & \$MOA=0-0\\
XALAN-ABC & XALAN-D & 898 & 11 & 0 & 0 & 0.0 & 100.0 & 0.0 & 98.79 & \$MOA=0-0\\\hline

XERCES-A & XERCES-B & 239 & 40 & 130 & 31 & 43.66 & 64.77 & 19.25 & 61.36 & \$LCOM3=1-2\\
XERCES-AB & XERCES-C & 269 & 13 & 115 & 56 & 81.16 & 70.05 & 32.75 & 71.74 & \$LCOM3=1-2\\
XERCES-ABC & XERCES-D & 128 & 248 & 23 & 189 & 43.25 & 84.77 & 89.15 & 53.91 & \$LCOM3=1-2\\\hline

\end{tabular}}
\end{center}
\caption{Rules for flawless software, discretized to two bins.}
\label{fig:2_bin_flawless}
\end{table*}

\begin{table*}[!h]
\begin{center}
{\tiny
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | c |}\hline
\multicolumn{11}{|c|}{Rules for flawless with 3 bins}\\\hline
Training & Testing & A & B & C & D & Pd & Pf & Prec & Acc & Rec \\\hline

ANT-A & ANT-B & 40 & 138 & 0 & 0 & 0.0 & 100.0 & 0.0 & 22.47 & \$NOC=0-0\\
ANT-AB & ANT-C & 32 & 261 & 0 & 0 & 0.0 & 100.0 & 0.0 & 10.92 & \$NOC=0-0\\
ANT-ABC & ANT-D & 92 & 259 & 0 & 0 & 0.0 & 100.0 & 0.0 & 26.21 & \$MOA=0-0\\
ANT-ABCD & ANT-E & 166 & 579 & 0 & 0 & 0.0 & 100.0 & 0.0 & 22.28 & \$MOA=0-0\\\hline

CAMEL-A & CAMEL-B & 216 & 392 & 0 & 0 & 0.0 & 100.0 & 0.0 & 35.53 & \$NOC=0-0\\
CAMEL-AB & CAMEL-C & 145 & 727 & 0 & 0 & 0.0 & 100.0 & 0.0 & 16.63 & \$NOC=0-0\\
CAMEL-ABC & CAMEL-D & 188 & 777 & 0 & 0 & 0.0 & 100.0 & 0.0 & 19.48 & \$NOC=0-0\\\hline

FORREST-AB & FORREST-C & 2 & 30 & 0 & 0 & 0.0 & 100.0 & 0.0 & 6.25 & \$MOA=0-0\\\hline

JEDIT-A & JEDIT-B & 75 & 231 & 0 & 0 & 0.0 & 100.0 & 0.0 & 24.51 & \$MOA=0-0\\
JEDIT-AB & JEDIT-C & 79 & 233 & 0 & 0 & 0.0 & 100.0 & 0.0 & 25.32 & \$MOA=0-0\\
JEDIT-ABC & JEDIT-D & 48 & 319 & 0 & 0 & 0.0 & 100.0 & 0.0 & 13.08 & \$MOA=0-0\\
JEDIT-ABCD & JEDIT-E & 11 & 481 & 0 & 0 & 0.0 & 100.0 & 0.0 & 2.236 & \$MOA=0-0\\\hline

LOG4J-A & LOG4J-B & 37 & 72 & 0 & 0 & 0.0 & 100.0 & 0.0 & 33.94 & \$DAM=0-0\\
LOG4J-AB & LOG4J-C & 189 & 16 & 0 & 0 & 0.0 & 100.0 & 0.0 & 92.2 & \$DAM=0-0\\\hline

LUCENE-A & LUCENE-B & 144 & 103 & 0 & 0 & 0.0 & 100.0 & 0.0 & 58.3 & \$MOA=0-0\\
LUCENE-AB & LUCENE-C & 203 & 137 & 0 & 0 & 0.0 & 100.0 & 0.0 & 59.71 & \$NOC=0-0\\\hline

PBEANS-A & PBEANS-B & 1 & 1 & 40 & 9 & 90.0 & 2.439 & 18.37 & 19.61 & \$MAX\_CC=1-27\\\hline

POI-A & POI-B & 226 & 29 & 51 & 8 & 21.62 & 81.59 & 13.56 & 74.52 & \$LCOM3=1-1.5\\
POI-AB & POI-C & 3 & 1 & 134 & 247 & 99.6 & 2.19 & 64.83 & 64.94 & \$AVG\_CC=1-14.2222\\
POI-ABC & POI-D & 114 & 245 & 47 & 36 & 12.81 & 70.81 & 43.37 & 33.94 & \$LCOM3=1-1.5\\\hline

PROP-A & PROP-B & 31 & 100 & 2400 & 20483 & 99.51 & 1.275 & 89.51 & 89.14 & \$AVG\_CC=1-22.0833\\
PROP-AB & PROP-C & 15 & 43 & 1165 & 9051 & 99.53 & 1.271 & 88.6 & 88.24 & \$AVG\_CC=1-25.5\\
PROP-ABC & PROP-D & 3 & 13 & 837 & 7865 & 99.83 & .3571 & 90.38 & 90.25 & \$AVG\_CC=1-26.2\\
PROP-ABCD & PROP-E & 25 & 55 & 1274 & 7162 & 99.24 & 1.925 & 84.9 & 84.39 & \$AVG\_CC=1-30.5\\
PROP-ABCDE & PROP-F & 2 & 11 & 64 & 583 & 98.15 & 3.03 & 90.11 & 88.64 & \$AVG\_CC=1-30.5\\\hline

SYNAPSE-A & SYNAPSE-B & 60 & 162 & 0 & 0 & 0.0 & 100.0 & 0.0 & 27.03 & \$MOA=0-0\\
SYNAPSE-AB & SYNAPSE-C & 86 & 170 & 0 & 0 & 0.0 & 100.0 & 0.0 & 33.59 & \$MOA=0-0\\\hline

VELOCITY-A & VELOCITY-B & 1 & 4 & 71 & 138 & 97.18 & 1.389 & 66.03 & 64.95 & \$AVG\_CC=1-10.5426\\
VELOCITY-AB & VELOCITY-C & 1 & 1 & 150 & 77 & 98.72 & .6623 & 33.92 & 34.06 & \$AVG\_CC=1-12.8222\\\hline

XALAN-A & XALAN-B & 387 & 416 & 0 & 0 & 0.0 & 100.0 & 0.0 & 48.19 & \$NOC=0-0\\
XALAN-AB & XALAN-C & 411 & 474 & 0 & 0 & 0.0 & 100.0 & 0.0 & 46.44 & \$NOC=0-0\\
XALAN-ABC & XALAN-D & 898 & 11 & 0 & 0 & 0.0 & 100.0 & 0.0 & 98.79 & \$NOC=0-0\\\hline

XERCES-A & XERCES-B & 369 & 71 & 0 & 0 & 0.0 & 100.0 & 0.0 & 83.86 & \$IC=0-0\\
XERCES-AB & XERCES-C & 384 & 69 & 0 & 0 & 0.0 & 100.0 & 0.0 & 84.77 & \$IC=0-0\\
XERCES-ABC & XERCES-D & 138 & 365 & 13 & 72 & 16.48 & 91.39 & 84.71 & 35.71 & \$LCOM3=1-1.5\\\hline

\end{tabular}}
\end{center}
\caption{Rules for flawless software, discretized to three bins.}
\label{fig:3_bin_flawless}
\end{table*}

\begin{table*}[!h]
\begin{center}
{\tiny
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | c |}\hline
\multicolumn{11}{|c|}{Rules for defective with 2 bins}\\\hline
Training & Testing & A & B & C & D & Pd & Pf & Prec & Acc & Rec \\\hline

ANT-A & ANT-B & 4 & 5 & 134 & 35 & 87.5 & 2.899 & 20.71 & 21.91 & \$CE=1-21\\
ANT-AB & ANT-C & 74 & 0 & 187 & 32 & 100.0 & 28.35 & 14.61 & 36.18 & \$LCOM3=1-2\\
ANT-ABC & ANT-D & 77 & 8 & 182 & 84 & 91.3 & 29.73 & 31.58 & 45.87 & \$LCOM3=1-2\\
ANT-ABCD & ANT-E & 194 & 18 & 385 & 148 & 89.16 & 33.51 & 27.77 & 45.91 & \$LCOM3=1-2\\\hline

CAMEL-A & CAMEL-B & 142 & 80 & 250 & 136 & 62.96 & 36.22 & 35.23 & 45.72 & \$LCOM3=1-2\\
CAMEL-AB & CAMEL-C & 84 & 24 & 643 & 121 & 83.45 & 11.55 & 15.84 & 23.51 & \$DIT=1-6\\
CAMEL-ABC & CAMEL-D & 100 & 21 & 677 & 167 & 88.83 & 12.87 & 19.79 & 27.67 & \$DIT=1-6\\\hline

FORREST-AB & FORREST-C & 30 & 2 & 0 & 0 & 0.0 & 100.0 & 0.0 & 93.75 & \$NOC=0-0\\\hline

JEDIT-A & JEDIT-B & 84 & 6 & 147 & 69 & 92.0 & 36.36 & 31.94 & 50.0 & \$LCOM3=1-2\\
JEDIT-AB & JEDIT-C & 110 & 64 & 123 & 15 & 18.99 & 47.21 & 10.87 & 40.06 & \$DAM=1-1\\
JEDIT-ABC & JEDIT-D & 156 & 35 & 163 & 13 & 27.08 & 48.9 & 7.386 & 46.05 & \$DAM=1-1\\
JEDIT-ABCD & JEDIT-E & 160 & 4 & 321 & 7 & 63.64 & 33.26 & 2.134 & 33.94 & \$LCOM3=1-2\\\hline

LOG4J-A & LOG4J-B & 23 & 4 & 49 & 33 & 89.19 & 31.94 & 40.24 & 51.38 & \$LCOM3=1-2\\
LOG4J-AB & LOG4J-C & 6 & 51 & 10 & 138 & 73.02 & 37.5 & 93.24 & 70.24 & \$LCOM3=1-2\\\hline

LUCENE-A & LUCENE-B & 1 & 1 & 102 & 143 & 99.31 & .9709 & 58.37 & 58.3 & \$AVG\_CC=1-13.7727\\
LUCENE-AB & LUCENE-C & 1 & 0 & 136 & 203 & 100.0 & .7299 & 59.88 & 60.0 & \$AVG\_CC=1-14.0455\\\hline

PBEANS-A & PBEANS-B & 10 & 41 & 0 & 0 & 0.0 & 100.0 & 0.0 & 19.61 & \$MOA=0-0\\\hline

POI-A & POI-B & 37 & 277 & 0 & 0 & 0.0 & 100.0 & 0.0 & 11.78 & \$MFA=0-0\\
POI-AB & POI-C & 248 & 137 & 0 & 0 & 0.0 & 100.0 & 0.0 & 64.42 & \$IC=0-0\\
POI-ABC & POI-D & 281 & 161 & 0 & 0 & 0.0 & 100.0 & 0.0 & 63.57 & \$IC=0-0\\\hline

PROP-A & PROP-B & 5615 & 265 & 14968 & 2166 & 89.1 & 27.28 & 12.64 & 33.81 & \$DIT=1-7\\
PROP-AB & PROP-C & 3672 & 332 & 5422 & 848 & 71.86 & 40.38 & 13.52 & 43.99 & \$DIT=1-8\\
PROP-ABC & PROP-D & 1295 & 111 & 6583 & 729 & 86.79 & 16.44 & 9.97 & 23.22 & \$DIT=1-8\\
PROP-ABCD & PROP-E & 2319 & 269 & 4898 & 1030 & 79.29 & 32.13 & 17.38 & 39.33 & \$DIT=1-9\\
PROP-ABCDE & PROP-F & 15 & 0 & 579 & 66 & 100.0 & 2.525 & 10.23 & 12.27 & \$DIT=1-9\\\hline

SYNAPSE-A & SYNAPSE-B & 162 & 60 & 0 & 0 & 0.0 & 100.0 & 0.0 & 72.97 & \$NOC=0-0\\
SYNAPSE-AB & SYNAPSE-C & 170 & 86 & 0 & 0 & 0.0 & 100.0 & 0.0 & 66.41 & \$NOC=0-0\\\hline

VELOCITY-A & VELOCITY-B & 142 & 72 & 0 & 0 & 0.0 & 100.0 & 0.0 & 66.36 & \$MOA=0-0\\
VELOCITY-AB & VELOCITY-C & 25 & 82 & 53 & 69 & 45.7 & 32.05 & 56.56 & 41.05 & \$LCOM3=1-2\\\hline

XALAN-A & XALAN-B & 194 & 140 & 222 & 247 & 63.82 & 46.63 & 52.67 & 54.92 & \$LCOM3=1-2\\
XALAN-AB & XALAN-C & 244 & 132 & 230 & 279 & 67.88 & 51.48 & 54.81 & 59.1 & \$LCOM3=1-2\\
XALAN-ABC & XALAN-D & 0 & 1 & 11 & 897 & 99.89 & 0.0 & 98.79 & 98.68 & \$AVG\_CC=1-22\\\hline

XERCES-A & XERCES-B & 0 & 4 & 71 & 365 & 98.92 & 0.0 & 83.72 & 82.95 & \$AVG\_CC=1-4.8\\
XERCES-AB & XERCES-C & 1 & 3 & 68 & 381 & 99.22 & 1.449 & 84.86 & 84.33 & \$AVG\_CC=1-13.3585\\
XERCES-ABC & XERCES-D & 4 & 0 & 433 & 151 & 100.0 & .9153 & 25.86 & 26.36 & \$AVG\_CC=1-13.3585\\\hline

\end{tabular}}
\end{center}
\caption{Rules for defective software, discretized to two bins.}
\label{fig:2_bin_defective}
\end{table*}

\begin{table*}[!h]
\begin{center}
{\tiny
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | c |}\hline
\multicolumn{11}{|c|}{Rules for defective with 3 bins}\\\hline
Training & Testing & A & B & C & D & Pd & Pf & Prec & Acc & Rec \\\hline

ANT-A & ANT-B & 14 & 14 & 124 & 26 & 65.0 & 10.14 & 17.33 & 22.47 & \$CE=1-21\\
ANT-AB & ANT-C & 23 & 15 & 238 & 17 & 53.12 & 8.812 & 6.667 & 13.65 & \$CE=1-28\\
ANT-ABC & ANT-D & 24 & 29 & 235 & 63 & 68.48 & 9.266 & 21.14 & 24.79 & \$CE=1-35\\
ANT-ABCD & ANT-E & 27 & 51 & 552 & 115 & 69.28 & 4.663 & 17.24 & 19.06 & \$CE=1-35\\\hline

CAMEL-A & CAMEL-B & 4 & 9 & 388 & 207 & 95.83 & 1.02 & 34.79 & 34.7 & \$MAX\_CC=1-26\\
CAMEL-AB & CAMEL-C & 228 & 50 & 499 & 95 & 65.52 & 31.36 & 15.99 & 37.04 & \$DIT=1-6\\
CAMEL-ABC & CAMEL-D & 241 & 60 & 536 & 128 & 68.09 & 31.02 & 19.28 & 38.24 & \$DIT=1-6\\\hline

FORREST-AB & FORREST-C & 30 & 2 & 0 & 0 & 0.0 & 100.0 & 0.0 & 93.75 & \$NOC=0-0\\\hline

JEDIT-A & JEDIT-B & 166 & 53 & 65 & 22 & 29.33 & 71.86 & 25.29 & 61.44 & \$LCOM3=1-1.8\\
JEDIT-AB & JEDIT-C & 116 & 65 & 117 & 14 & 17.72 & 49.79 & 10.69 & 41.67 & \$DAM=1-1\\
JEDIT-ABC & JEDIT-D & 220 & 42 & 99 & 6 & 12.5 & 68.97 & 5.714 & 61.58 & \$LCOM3=1-1.8\\
JEDIT-ABCD & JEDIT-E & 344 & 10 & 137 & 1 & 9.091 & 71.52 & .7246 & 70.12 & \$LCOM3=1-1.8\\\hline

LOG4J-A & LOG4J-B & 45 & 30 & 27 & 7 & 18.92 & 62.5 & 20.59 & 47.71 & \$LCOM3=1-1.3333334\\
LOG4J-AB & LOG4J-C & 12 & 138 & 4 & 51 & 26.98 & 75.0 & 92.73 & 30.73 & \$LCOM3=1-1.3333334\\\hline

LUCENE-A & LUCENE-B & 6 & 17 & 97 & 127 & 88.19 & 5.825 & 56.7 & 53.85 & \$CE=1-29\\
LUCENE-AB & LUCENE-C & 3 & 2 & 134 & 201 & 99.01 & 2.19 & 60.0 & 60.0 & \$AVG\_CC=1-14.0455\\\hline

PBEANS-A & PBEANS-B & 10 & 41 & 0 & 0 & 0.0 & 100.0 & 0.0 & 19.61 & \$MOA=0-0\\\hline

POI-A & POI-B & 37 & 277 & 0 & 0 & 0.0 & 100.0 & 0.0 & 11.78 & \$MOA=0-0\\
POI-AB & POI-C & 248 & 137 & 0 & 0 & 0.0 & 100.0 & 0.0 & 64.42 & \$MOA=0-0\\
POI-ABC & POI-D & 281 & 161 & 0 & 0 & 0.0 & 100.0 & 0.0 & 63.57 & \$MOA=0-0\\\hline

PROP-A & PROP-B & 9808 & 817 & 10775 & 1614 & 66.39 & 47.65 & 13.03 & 49.63 & \$DIT=1-7\\
PROP-AB & PROP-C & 6385 & 845 & 2709 & 335 & 28.39 & 70.21 & 11.01 & 65.41 & \$DIT=1-8\\
PROP-ABC & PROP-D & 3025 & 335 & 4853 & 505 & 60.12 & 38.4 & 9.425 & 40.49 & \$DIT=1-8\\
PROP-ABCD & PROP-E & 5691 & 1096 & 1526 & 203 & 15.63 & 78.86 & 11.74 & 69.21 & \$DIT=1-9\\
PROP-ABCDE & PROP-F & 27 & 0 & 567 & 66 & 100.0 & 4.545 & 10.43 & 14.09 & \$DIT=1-9\\\hline

SYNAPSE-A & SYNAPSE-B & 101 & 35 & 61 & 25 & 41.67 & 62.35 & 29.07 & 56.76 & \$LCOM3=1-1.5\\
SYNAPSE-AB & SYNAPSE-C & 170 & 86 & 0 & 0 & 0.0 & 100.0 & 0.0 & 66.41 & \$NOC=0-0\\\hline

VELOCITY-A & VELOCITY-B & 142 & 72 & 0 & 0 & 0.0 & 100.0 & 0.0 & 66.36 & \$NOC=0-0\\
VELOCITY-AB & VELOCITY-C & 78 & 151 & 0 & 0 & 0.0 & 100.0 & 0.0 & 34.06 & \$MOA=0-0\\\hline

XALAN-A & XALAN-B & 0 & 2 & 416 & 385 & 99.48 & 0.0 & 48.06 & 47.95 & \$AVG\_CC=1-22\\
XALAN-AB & XALAN-C & 0 & 2 & 474 & 409 & 99.51 & 0.0 & 46.32 & 46.21 & \$AVG\_CC=1-22\\
XALAN-ABC & XALAN-D & 0 & 2 & 11 & 896 & 99.78 & 0.0 & 98.79 & 98.57 & \$AVG\_CC=1-22\\\hline

XERCES-A & XERCES-B & 0 & 2 & 71 & 367 & 99.46 & 0.0 & 83.79 & 83.41 & \$MAX\_CC=1-31\\
XERCES-AB & XERCES-C & 1 & 13 & 68 & 371 & 96.61 & 1.449 & 84.51 & 82.12 & \$AVG\_CC=1-13.3585\\
XERCES-ABC & XERCES-D & 14 & 0 & 423 & 151 & 100.0 & 3.204 & 26.31 & 28.06 & \$AVG\_CC=1-13.3585\\\hline

\end{tabular}}
\end{center}
\caption{Rules for defective software, discretized to three bins.}
\label{fig:3_bin_defective}
\end{table*}

The results we obtained are presented in tables one through four on the following pages. Besides our results, each table lists a number of values other than the Pd and Pf discussed previously. These values are used in our confusion matrix computations for Pd and Pf. A, B, C, and D each represent the number of true positives, false positives, false negatives, and true negatives respectively. Prec and Acc, respectively precision and accuracy, as well as Pd and Pf, are computed using the following equations:

\bi
\item
$acc=\dfrac{A+D}{A+B+C+D}$
\\
\item
$prec=\dfrac{D}{C+D}$
\\
\item
$pd=\dfrac{D}{B+D}$
\\
\item
$pf=\dfrac{C}{A+C}$
\ei

The tables we obtained from our testing can quickly give us insight into the datasets we used. What we look at most closely are the Pd and Pf values and not so much Pred and Acc. The precision and accuracy values are not of much interest due to the relative rarity of our target class. What we like to see for each row in our tables is as high a Pd and as low a Pf as possible. This combination of Pd and Pf values represents a predictor with high probability of detecting the class we are trying to predict for and a low probability of falsely identifying said class. The general conclusion outlined by these results is that predicting for defective classes is a rather dataset-dependent procedure. The following sections go into further details concerning several aspects of our results.

\subsection{Difference in flawless/defective results}
An obvious difference one notices when looking at the results is that for some reason {\W} is a lot more successful in predicting for defective classes rather than flawless ones. The flawless tables entire top half consists of nothing but bad rules. This might seem odd at first glance, however we believe this is normal operation in this particular case and that {\W} itself is not at fault in any way. This conclusion comes from our previous experience with the datasets in question. We know from previous experiments that our datasets contained a much higher population of flawless historical instances than defective ones. We believe that the projects for which {\W} failed at predicting for flawless classes simply have too many flawless instances. One can imagine a situation where {\W} is unable to come up with a successful flawless predictor after being presented with a very large number of flawless instances whose metrics values are highly variant. This issue remains an open avenue for future research.

Besides the ratio of flawless to defective historical instances in the datasets, we believe the way {\W} currently scores rules is partly responsible for the difference in flawless/defective results.

\subsection{Difference in two/three bin results}
We have to consider different numbers of discretization bins in order to further validate our results. Looking at our results, one immediately notices that the rules obtained for individual projects are generally different for different numbers of bins. What is not immediately clear is that for some projects, the rules we obtain are the same. Predicting for defective classes, the only project that exhibits this property is prop. Predicting for flawless classes, we have two projects that show the same property: pbeans and velocity. Another, even less obvious, observation is that for some projects, the rules are indeed different for different bin numbers, but not drastically different. Some examples of this are camel, log4j, and xerxes, all predicting for defective classes. The only other detail left to examine with respect to different numbers of bins is whether or not Pd and Pf values are affected. Our results show that indeed they are, but not in any seemingly predictable way. For example, when predicting for defective classes, ant, camel, and prop have significantly better values for Pd and Pf with two bins as opposed to three whereas xalan does much better with three. These results suggest some sort of correlation between the individual project and the existence of an optimal number of bins for that project. What exactly would determine this number of bins remains a topic for research.

\subsection{Stubborn datasets}
There are several datasets that simply did not result in any meaningful conclusion. Some of these datasets consist of only one version of a project, however not all of them do. Forrest and synapse, for example, did not give us any results for either flawless or defective with either two or three bins. Others work fine when predicting for one of the classes but not the other. One such example is poi which comes up with solid results predicting for flawless classes and nothing but junk when predicting for defective classes. These odd cases are discussed further in section 5.2.

\begin{table}[!h]
\begin{center}
{\tiny
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | c |}\hline
\multicolumn{3}{|c|}{Top 8 rules, train on xerxes-ab, test on xerxes-c, 5 times}\\\hline

Pd & Pf & Rec\\
99.22 & 1.449 & \$AVG\_CC=1-13.3585\\
0.0 & 100.0 & \$DAM=0-0\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$DAM=0-0\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$CAM=0.37068966-0.6\\
62.24 & 46.38 & \$MFA=1-1\\
61.98 & 46.38 & \$AVG\_CC=1-13.3585\$MFA=1-1\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\$DAM=0-0\\\hline

Pd & Pf & Rec\\
99.22 & 1.449 & \$AVG\_CC=1-13.3585\\
0.0 & 100.0 & \$DAM=0-0\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$DAM=0-0\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$CAM=0.37068966-0.6\\
62.24 & 46.38 & \$MFA=1-1\\
61.98 & 46.38 & \$AVG\_CC=1-13.3585\$MFA=1-1\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\$DAM=0-0\\\hline

Pd & Pf & Rec\\
99.22 & 1.449 & \$AVG\_CC=1-13.3585\\
0.0 & 100.0 & \$DAM=0-0\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$DAM=0-0\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$CAM=0.37068966-0.6\\
62.24 & 46.38 & \$MFA=1-1\\
99.22 & 10.14 & \$AVG\_CC=1-13.3585\$IC=1-4\\
99.22 & 4.348 & \$CBM=1-22\\\hline

Pd & Pf & Rec\\
99.22 & 1.449 & \$AVG\_CC=1-13.3585\\
0.0 & 100.0 & \$DAM=0-0\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$DAM=0-0\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$CAM=0.37068966-0.6\\
62.24 & 46.38 & \$MFA=1-1\\
61.98 & 46.38 & \$AVG\_CC=1-13.3585\$MFA=1-1\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\$DAM=0-0\\\hline

Pd & Pf & Rec\\
99.22 & 1.449 & \$AVG\_CC=1-13.3585\\
0.0 & 100.0 & \$DAM=0-0\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$DAM=0-0\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\\
0.0 & 100.0 & \$AVG\_CC=1-13.3585\$CAM=0.37068966-0.6\\
62.24 & 46.38 & \$MFA=1-1\\
0.0 & 100.0 & \$CAM=0.37068966-0.6\$DAM=0-0\\
98.44 & 5.797 & \$AVG\_CC=1-13.3585\$CBM=1-22\\\hline

\end{tabular}}
\end{center}
\caption{These are the top 8 rules for 5 runs training on xerxes-ab and testing on xerxes-c with two bins and predicting for defective classes.}
\label{fig:stability-xerxes}
\end{table}

\subsection{Stability}
One important point we must consider is the stability of {\W}'s results over a number of runs. Table 5 contains the top eight rules inside {\W}'s stack after training on xerxes-ab and testing on xerxes-c. The runs were performed using two discretization bins and are predicting for defective classes. We can immediately see that the rules at the top of the stack rarely change, in fact the very top ones never seem to deviate. These outcomes strongly point to the inherent stability within {\W}'s algorithm and further help to validate the rest of our results.
