% This is "sig-alternate.tex" V1.8 June 2007
% This file should be compiled with V2.3 of "sig-alternate.cls" June 2007
%
% This example file demonstrates the use of the 'sig-alternate.cls'
% V2.3 LaTeX2e document class file. It is for those submitting
% articles to ACM Conference Proceedings WHO DO NOT WISH TO
% STRICTLY ADHERE TO THE SIGS (PUBS-BOARD-ENDORSED) STYLE.
% The 'sig-alternate.cls' file will produce a similar-looking,
% albeit, 'tighter' paper resulting in, invariably, fewer pages.
%
% ----------------------------------------------------------------------------------------------------------------
% This .tex file (and associated .cls V2.3) produces:
%       1) The Permission Statement
%       2) The Conference (location) Info information
%       3) The Copyright Line with ACM data
%       4) NO page numbers
%
% as against the acm_proc_article-sp.cls file which
% DOES NOT produce 1) thru' 3) above.
%
% Using 'sig-alternate.cls' you have control, however, from within
% the source .tex file, over both the CopyrightYear
% (defaulted to 200X) and the ACM Copyright Data
% (defaulted to X-XXXXX-XX-X/XX/XX).
% e.g.
% \CopyrightYear{2007} will cause 2007 to appear in the copyright line.
% \crdata{0-12345-67-8/90/12} will cause 0-12345-67-8/90/12 to appear in the copyright line.
%
% ---------------------------------------------------------------------------------------------------------------
% This .tex source is an example which *does* use
% the .bib file (from which the .bbl file % is produced).
% REMEMBER HOWEVER: After having produced the .bbl file,
% and prior to final submission, you *NEED* to 'insert'
% your .bbl file into your source .tex file so as to provide
% ONE 'self-contained' source file.
%
% ================= IF YOU HAVE QUESTIONS =======================
% Questions regarding the SIGS styles, SIGS policies and
% procedures, Conferences etc. should be sent to
% Adrienne Griscti (griscti@acm.org)
%
% Technical questions _only_ to
% Gerald Murray (murray@acm.org)
% ===============================================================
%
% For tracking purposes - this is V1.8 - June 2007

\pdfpagewidth=8.5in
\pdfpageheight=11in
\documentclass{sig-alt-full}
\clubpenalty=10000
\widowpenalty = 10000

\usepackage{times}
\usepackage{multirow}
\usepackage{multicol}
\usepackage{graphicx}
\usepackage{url}


\newenvironment{smallitem}{
   \setlength{\topsep}{0pt}
   \setlength{\partopsep}{0pt}
   \setlength{\parskip}{0pt}
   \begin{itemize}
   \setlength{\leftmargin}{.2in}
   \setlength{\parsep}{0pt}
   \setlength{\parskip}{0pt}
   \setlength{\itemsep}{0pt}}{\end{itemize}}

\newenvironment{smallnums}{
   \setlength{\topsep}{0pt}
   \setlength{\partopsep}{0pt}
   \setlength{\parskip}{0pt}
   \begin{enumerate}
   \setlength{\leftmargin}{.2in}
   \setlength{\parsep}{0pt}
   \setlength{\parskip}{0pt} \setlength{\itemsep}{0pt}}{\end{enumerate}}

\newcommand{\bi}{\begin{smallitem}}
\newcommand{\ei}{\end{smallitem}}
\newcommand{\be}{\begin{enumerate}}
\newcommand{\ee}{\end{enumerate}}




\begin{document}
%
% --- Author Metadata here ---
\conferenceinfo{WOODSTOCK}{'97 El Paso, Texas USA}
%\CopyrightYear{2007} % Allows default copyright year (200X) to be over-ridden - IF NEED BE.
%\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{Does transformation help?}
%\subtitle{[Does transformation help?]}
%\textit{Author's Guide to Preparing ACM SIG Proceedings Using
%\LaTeX$2_\epsilon$\ and BibTeX} at
%\texttt{www.acm.org/eaddress.htm}}}
%
% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. We just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{1} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor
Yue Jiang,Bojan Cukic,Tim Menzies\\
       \affaddr{Dept. of LCSEE}\\
       \affaddr{West Virginia University}\\
       \affaddr{Morgantown,WV,USA}\\
       \email{cukic,yue@csee.wvu.edu;tim@menzies.us}
% 2nd. author
%\alignauthor
%G.K.M. Tobin\titlenote{The secretary disavows
%any knowledge of this author's actions.}\\
%       \affaddr{Institute for Clarity in Documentation}\\
%       \affaddr{P.O. Box 1212}\\
%       \affaddr{Dublin, Ohio 43017-6221}\\
%       \email{webmaster@marysville-ohio.com}
% 3rd. author
%\alignauthor Lars Th{\o}rv{\"a}ld\titlenote{This author is the
%one who did all the really hard work.}\\
%       \affaddr{The Th{\o}rv{\"a}ld Group}\\
%       \affaddr{1 Th{\o}rv{\"a}ld Circle}\\
%       \affaddr{Hekla, Iceland}\\
%       \email{larst@affiliation.org}
%\and  % use '\and' if you need 'another row' of author names
% 4th. author
%\alignauthor Lawrence P. Leipuner\\
%       \affaddr{Brookhaven Laboratories}\\
%       \affaddr{Brookhaven National Lab}\\
%       \affaddr{P.O. Box 5000}\\
%       \email{lleipuner@researchlabs.org}
% 5th. author
%\alignauthor Sean Fogarty\\
%       \affaddr{NASA Ames Research Center}\\
%       \affaddr{Moffett Field}\\
%       \affaddr{California 94035}\\
%       \email{fogartys@amesres.org}
% 6th. author
%\alignauthor Charles Palmer\\
%       \affaddr{Palmer Research Laboratories}\\
%       \affaddr{8600 Datapoint Drive}\\
%       \affaddr{San Antonio, Texas 78229}\\
%       \email{cpalmer@prl.com}
}
% There's nothing stopping you putting the seventh, eighth, etc.
% author on the opening page (as the 'third row') but we ask,
% for aesthetic reasons that you place these 'additional authors'
% in the \additional authors block, viz.
%\additionalauthors{Additional authors: John Smith (The Th{\o}rv{\"a}ld Group,
%email: {\texttt{jsmith@affiliation.org}}) and Julius P.~Kumquat
%(The Kumquat Consortium, email: {\texttt{jpkumquat@consortium.net}}).}
%\date{30 July 1999}
% Just remember to make sure that the TOTAL number of authors
% is the number that will appear on the first page PLUS the
% number that will appear in the \additionalauthors section.

\maketitle
\begin{abstract}
Data preprocessing(or transformation) plays an important role in data mining and machine learning. In this study, we investigate the effect of 4 different preprocessing methods to fault-proneness prediction by using 9 datasets from NASA Metrics Data Programs (MDP) and 10 machine learners. From our experiments,we found that log transformation do not affect most learners' performance except bagging classifier. However, discretization affect the performance of many different learners. Learners such as random forest perform better in original and log transformation data. Learners such as boosting and NaiveBayes perform significantly better on discretized data. After comparing the performance of different learners in different transformation methods, we found that the one which has the best performance is random forest in original and log transformation data. Therefore,when conducting transformation on the data, in order to gain better performance for a learner, we had better based on the specific learner instead of using universal transformation for all kinds of learners.
\end{abstract}

% A category with the (minimum) three required fields
%\category{H.4}{Information Systems Applications}{Miscellaneous}
%A category including the fourth, optional field follows...
%\category{D.2.8}{Software Engineering}{Metrics}[complexity measures, performance measures]

%\terms{Delphi theory}

%\keywords{ACM proceedings, \LaTeX, text tagging}

\section{Introduction}
Data mining is the process of finding useful information from datasets. Dataset is generally contain noise which affect the performance of a machine learner. Hence, many preprocessing (transformation) methods are proposed to carried out before the actual learning process. Normalization~\cite{Faraway02}, linear and non-linear transformation~\cite{Faraway02}, feature subset selection~\cite{Guyon06}, principal component analysis~\cite{Jolliffe02}, and discretization~\cite{Dougherty95} are often conducted before prediction.

In this study, we compare four different preprocessing methods in $9$ NASA MDP datasets across $10$ different machine learners. These four different preprocessing methods are the original data ($none$), log transformation data ($log$), discretized data ($nom$), and discretized on the log transformation data ($log\&nom$). From our experiments,we found that log transformation do not affect most learners' performance.  However, discretization affect the performance of many different learners. Learners such as random forest perform better in original ($none$) and log transformation ($log$) data. More specifically, Random forest performs better in original data with more number of individual data values. Bagging has the best performance in log transformation data ($log$). Discretization data is better for boosting, NaiveBayes,logistic and IBk machine learners. Random forest in the original has the best performance across all kinds of learners in these four kinds of transformation methods.

\section{Experiment Setup}
In this study, we use $10$ by $10$ way cross-validation ($10x10$ CV) to generate Receiver Operating Characteristic (ROC) curves through all possible threshold. The Area Under the ROC curve, referred to as $AUC$, is used as the performance measurement to evaluate different learners.

Cross-validation is the statistical practice of partitioning a
sample of data into two subsets: training and testing subset.  We
use 10 by 10 way cross-validation in all experiments.
$90\%$ of data is randomly assigned to the training subset and the
remaining $10\%$ of data is used for testing. The data is randomly
divided into $10$ fixed bins of equal size. We leave one bin to act as
test data and the other $9$ bins is used to train the learners.  This
procedure is repeated $10$ times.

ROC curves provide an intuitive
way to compare the classification performance of different metrics.
An ROC curve is a plot of the Probability of Detection ($pd$) as a
function of the Probability of False alarm ($pf$) across all the
possible experimental threshold settings. AUC is a numeric
performance evaluation measure directly associated with an ROC
curve. It is very common to use AUC to compare the performance of
different classifiers. In this study, we use AUC as a measurement of the performance of learners.

A box and whisker diagram,referred as a boxplot diagram, graphically
depicts numerical data distributions using five first order
statistics: the smallest observation, lower quartile (Q1), median,
upper quartile (Q3), and the largest observation.  The box is
constructed based on the interquartile range (IQR) from Q1 to Q3.
The line inside the box depicts the median which follows the central
tendency. The whiskers indicate the smallest observation and the
largest observation. Boxplot diagrams are shown in this study to visually depict the learners performance too.

\subsection{Used DataSet and Preprocessing Methods}
There are $9$ datasets shown in Table~\ref{tab:dataset} from Metrics Data Programs (MDP)~\cite{mdp} are used. In this study, we predict whether a module is fault-prone or not instead of predicting how many faults a module has.

\begin{table*}
\centering
\caption{Datasets used in this study}
\label{tab:dataset}
\begin{tabular}{|c|c|c|c|l|c|}
\hline Data	&     mod.\#  & 	faulty mod. \#	& \% faulty &	Note & language\\
\hline CM1 &	505&	81&	16.04\%&	Spacecraft instrument&C\\
\hline KC1&	2107&	293&	13.9\%	&storage management for receiving/processing ground data&C++\\
\hline KC3&	458&	29&	6.3\%&	Storage management for ground data&Java\\
\hline KC4&	125&	60&	48\%&	a ground-based subscription server&Perl\\
\hline PC1&	1107&	73&	6.59\%&	flight software from an earth orbiting satellite&C\\
\hline PC3&	1563&	163&	10.43\%&	Flight software for earth orbiting satellite&C\\
\hline PC4&	1458&	178&	12.21\%&	Flight software for earth orbiting satellite&C\\
\hline MW1&	403&	27&	6.7\%&	a zero gravity experiment related to combustion	&C\\
\hline MC2&	161&	52&	32.30\%&	a video guidance system&C++\\
\hline\end{tabular}
\end{table*}

Four kinds of preprocessing methods are listed as follows.
\be
  \item \textbf{none:} This is the original data set without applying any preprocessing methods. All the independent variables in MDP are continuous attributes.
  \item \textbf{log:}   The original continuous independent variable values are transformed by taking mathematical log operation. To avoid numerical errors with $\ln(0)$,  all numbers under 0.000001 are replaced with $\ln(0.000001)$. This transformation method is reported effective transformation in~\cite{menzies07}.
  \item \textbf{nom:}  The original independent variable continuous values are discretized to nominal values by using Fayyad and Irani's minimum description length (MDL) method (Weka default discretization method)~\cite{Dougherty95, Witten05}. We discuss this method in detail below.
  \item \textbf{log\&nom:} The log-transformed data are discretized to nominal values using Fayyad and Irani's discretization method.
\ee

\textbf{Fayyad and Irani's method} : We use the recursive minimal entropy discretization method proposed by Fayyad and Irani~\cite{Fayyad93}. It is a supervised, non-parametric discretization method. It uses the class information entropy to partition bin boundaries. The criterion to stop is the minimum description length which do not need the user intervention. For example, assume there is a dataset with $S$ instances, a variable $A$, and a partition boundary $T$, there are two classes ($S_1$, $S_2$) for the dataset, the class entropy is denoted as \emph{E(A,T;S)}:
   $E(A,T;S)=\frac{|S_1|}{|S|}E(S_1)+\frac{|S_2|}{|S|}E(S_2)$                (1)\\
It creates a tree of discretization from top to down. The branch of the partition is recursively discretized and evaluated independently. Thus, in some branches, the continuous values with relatively high entropy will be partitioned very finely, while others that have relatively low entropy will be partitioned coarsely.
\begin{table}
\centering
\caption{The number of distinct values for an attribute of 4 preprocessing methods in average.}
\label{tab:attribNum}
\begin{tabular}{|c|c|c|c|c|c|}
\hline dataset&	none&	Log&	nom&	log\&nom&	attrib.\#\\
\hline cm1	&63.27	&63.27	&1.81&	1.78&	37\\
\hline kc1	&68.38&	68.38&	3.1&	3.1 &	21\\
\hline kc3	&51.46&	51.46&	1.9&	1.9	&39\\
\hline kc4	&34.77&	34.77&	1.69&	1.69&	13\\
\hline pc1	&69.84&	69.84&	1.68&	1.65&	37\\
\hline pc3	&72.54&	72.54&	2.11&	2.11&	37\\
\hline pc4	&64.89&	64.89&	2.22&	2.22&	37\\
\hline mw1	&53.14&	53.14&	1.68&	1.65&	37\\
\hline mc2	&51.85&	51.85&	1.64&	1.62&	39\\
\hline ave.	&58.90&	58.90&	1.98&	1.97&	33\\
\hline
\end{tabular}
\end{table}

Table~\ref{tab:attribNum} shows the average number of distinct values for an attributes for the MDP datasets we study. For example, in the first cell, the number is ``63.27". It means that in CM1 data, under the preprocessing method of ``none" (the original data set), there are $63.27$ number of distinct values per attribute in average. From Table~\ref{tab:attribNum}, we find that:
\begin{itemize}
  \item The average number of distinct values for $none$ and $log$ are the same.
  \item The average numbers of distinct values for $none$ and $log$ are larger than that of $nom$ and $log\&nom$.
  \item The average number of distinct values for $nom$ and $log\&nom$ are similar.
\end{itemize}
%For example, in CM1 project, the average number of distinct values per attribute is $63.27$ while that of $nom$ and %$log\&nom$ are approximately $1.81$ and $1.78$ respectively..

\subsection{Different Learners}
Table~\ref{tab:learners} shows the learners we used in this study from Weka~\cite{Witten05}.
\begin{table}
\centering
\caption{Machine learners used.}
\label{tab:learners}
\begin{tabular}{|c|l|l|}
\hline &	learner&	Abbrev.\\
\hline 1&	Random Forest&	rf\\
\hline 2&	Bagging&	bag\\
\hline 3&	Logistic regression&	lgi\\
\hline 4&	Boosting&	bst\\
\hline 5&	Naivebayes& 	nb\\
\hline 6&	Jrip&	jrip\\
\hline 7&	IBk&	IBk\\
\hline 8&	J48&	j48\\
\hline 9&	Decorate&	dec\\
\hline 10&	AODE&	aode\\
\hline
\end{tabular}
\end{table}

Random Forest (rf)~\cite{Breiman01} is a decision  tree-based classifier demonstrated
to have good performance in software engineering studies by Guo $et$
$al$~\cite{guo04}. As implied from its name, it builds a ``forest"
of decision trees. The trees are constructed using the following
strategy: \bi \item{ The root node of each tree contains a bootstrap
sample data of the same size as the original data. Each tree has a
different bootstrap sample.}
\item{At each node, a subset of variables are randomly selected from
all the input variables to split the node and the best split is
adopted.} \item{Each tree is grown to the largest extent possible
without pruning.} \item{When all trees in the forest are built, new
instances are fitted to all the trees and a voting process takes
place. The forest selects the classification with the most votes as
the prediction of new instance(s).} \ei

NaiveBayes (nb) ``naively" assumes data independence. This
assumption may be considered overly simplistic in real life
application scenarios. However, in software engineering data sets
it's performance is surprisingly good. Naive Bayes classifiers have
been used extensively in fault-proneness prediction, for example in
~\cite{menzies07}.

Bagging (bag) stands for bootstrap  aggregating.  It relies on an
ensemble of different models. The training data is resampled from
the original data set. According to Witten and
Frank~\cite{Witten05}, bagging typically performs better than single
method models and almost never significantly worse.

Boosting (bst)  combines multiple models by explicitly seeking
models that complement one another. First, it is similar to bagging
in using voting for classification or averaging for numeric
prediction. Like bagging, boosting combines the models of the same
type. However, boosting is iterative. ``Whereas in bagging
individual models are built separately, in boosting each new model
is influenced by the performance of those built previously. Boosting
encourages new models to become experts for instances handled
incorrectly by earlier ones."~\cite{Witten05}.

Logistic regression(lgi) is a classification scheme  which uses
mathematical logistic regression functions.  The most popular models
are generalized linear models. J48 is a Weka~\cite{Witten05} implementation of J.R. Quinlan C4.5~\cite{Quinlan93} decision tree algorithm.

Decorate is a Weka classifier that builds ensembles of diverse classifiers by using specially generated artificial training examples~\cite{Witten05}. According to~\cite{Witten05}, it outperforms bagging and random forest.  It outperforms boosting on small training sets and rivals it on larger training sets~\cite{Witten05}. Decorate classifier are only run on data of \emph{none} and \emph{log} two preprocessing methods because of the reason that it does not allow an attribute with only one distinct value, which is not a rarely case when a continuous variable is discretized. %; 2) it is slow to run.

``AODE (averaged, one-dependence estimators) is one of Bayesian method that averages
over a space of alternative Bayesian models that have weaker independence assumptions than Naive Bayes. The algorithm may yield more accurate classification than Naive Bayes on datasets with nonindependent attributes." ~\cite{Witten05}. AODE is only able to classify nominal data. So, we only compare it with others on the nominal data (on \emph{nom} and \emph{log\&nom}).

All the learners are run in their default parameter except random forest with 500 trees because in the original random forest algorithm proposed by Breidman~\cite{Breiman01}, the default random forest algorithm is build with $500$ trees.  The first $8$ learners are run on all $4$ proprecessing methods of $9$ datasets. Decorate only runs on \emph{none} and \emph{log} two preprocessing methods and AODE only run on \emph{nom} and \emph{log\&nom} these two preprocessing methods. Thus, in a specific preprocessing method of a data, there are $9$ learners run against it.


\subsection{Statistical test hypotheses}
The most popular method used to evaluate a classifier's performance
on a data set is based on $10$ by $10$ ways cross-validation
($10\times10$ CV). The $10\times10$ CV results in $10$ individual values of
AUC. These $10$ values are usually similar to each other, given that
they come from the same population after randomization.  With only
$10$ values it is difficult to say whether the values follow the
normal distribution (i.e., indicate that they obey the central limit
theorem). Therefore, using parametric statistical methods which
assume a normally distributed population to compare the performance
of classifiers may not be justified.  A prudent approach calls for
the use of nonparametric methods.  The loss of efficiency caused by
using nonparametric tests is typically
marginal~\cite{conover99,siegel56}.

In ~\cite{demsar06}, Demsar overviewed the theoretical work on
statistical tests for the comparison of multiple classifiers over
multiple data sets. He recommended the Wilcoxon signed rank test for
the comparison of two classifiers and the Friedman test with the
corresponding post-hoc tests when the comparison includes more than
two classifiers. The Wilcoxon signed rank test and the Friedman's test are
nonparametric counterparts for paired $t-test$ and analysis of
variance (ANOVA) parametric methods, respectively. Demsar advocates
these tests largely due to the fact that nonparametric procedures
make less stringent demands on the data. However, two issues need
attention. First, nonparametric tests do not utilize all the
information available. The actual data values (in our case, for
example, AUCs) are not used in the test procedure. Instead, the
signs or ranks of the observations are used. Therefore,
nonparametric procedures will be more powerful than their parametric
counterparts, when justifiably used.  The second point is that
signed rank tests are constructed for the null hypothesis that the
difference of the performance measure is symmetrically distributed.
For non-symmetric distributions, this test can lead to a wrong
conclusion.

All statistical tests conducted here utilize the routines provided
in the statistical package R~\cite{R05}. Based on Demsar's
recommendation, we use the Friedman test to evaluate whether there
is a difference in the performance amongst the models developed from
the $9$ different algorithms over the $9$ data sets. Provided that the
Friedman test indicates statistically significant difference, we
need pairwise comparisons of models to determine which model
performs the best for each data set.

Assume we are looking  at two different learners in a
dataset, $A$ and $B$,in order to derive which learner has a better performance, the appropriate statistical test hypotheses are:
\\
\emph{$H_0$:  There is no difference in the performance of the models which use group A or group B metrics;}\\
\emph{$H_1$:  The performance of the group A model is better than the performance of group B model;}\\
\emph{$H_2$:  The performance of the group A model is worse than the performance of group B model.}\\

First, using the $95\%$ confidence interval, we test whether two learners
have the same performance using $H_0$. The p-value greater than 0.05 in this case indicates no
difference in the performance of group $A$ and $B$ models. In such a
case, further tests of hypotheses $H_1$ and $H_2$  are not necessary
since $H_0$ is the correct one. Otherwise, we test $H_1$. If the
p-value of $H_1$ is less than $0.05$, then $H_1$ is accepted.
Otherwise, if $H_1$ is rejected $H_2$ will be tested. After
conducting these three hypothesis tests, the relationship in the
performance of group A and B models will be clear on the given data
set: if $H_0$: $A=B$, if $H_1$: $A>B$, if $H_2$: $A<B$.

Recall that, in a specific preprocessing method of a data, there are $9$ learners' performance needed to be compared. In order to get the overall performances of $9$ learners across $9$ datasets in a specific preprocessing method, we count the number of wins($>$), losses($<$), and ties ($=$) over pairwise comparisons. For a specific learner in a preprocessing of a dataset, there are $8+7+6+5+4+3+2+1=36$ comparisons. When a learner $A$ is compared to a learner $B$, learner $B$ is compared to learner $A$ as well. Thus, totally, there are $36*2=72$ comparisons.

\section{Experimental Result}
Since the results of the Mann-Whitney and the Wilcoxon two statistical nonparametric tests do not vary too much, in the following we only show the number of wins,losses and ties of the Wilcoxon test.
\subsection{Original Data}
\begin{table}
\centering
\caption{Comparison result in original data($none$).}
\label{tab:originalWLT}
\begin{tabular}{|l|l|c|c|c|}
\hline Learner&	prep&	ties&	wins&	losses\\
\hline rf&	none&	5&	67&	0\\
\hline bag&	none&	12&	48&	12\\
\hline bst&	none&	9&	41&	22\\
\hline lgi&	none&	10&	38&	24\\
\hline dec&	none&	12&	35&	25\\
\hline nb&	none&	11&	32&	29\\
\hline IBk&	none&	6&	12&	54\\
\hline j48&	none&	6&	11&	55\\
\hline jrip&	none&	5&	2&	65\\
\hline
\end{tabular}
\end{table}

\begin{figure}
\centering
\psfig{file=none.ps,height=3in, width=3in,}
\caption{Boxplot diagrams in the original data.}\label{fig:none}
\end{figure}

\begin{table}
\centering
\caption{The trend in original ($none$) dataset}
\label{tab:originalWLTtrend}
\begin{tabular}{|l|l|}
\hline dataset&	Order of log transformation\\
\hline general&	rf >bag>bst>lgi>dec>nb>IBk>J48>jrip\\
\hline CM1&	rf >bst>lgi>nb>bag>dec>IBk>J48=jrip\\
\hline KC1&	rf>bag>lgi>nb>bst>dec>IBk>J48>jrip\\
\hline KC3&	rf>nb=bag=bst>lgi=dec>IBk>J48=jrip\\
\hline KC4&	rf=bag =dec>J48=bst>lgi>jrip=nb>IBk\\
\hline PC1&	rf >bag>bst>lgi=dec>nb>IBk=J48>jrip\\
\hline PC3&	rf>lgi>bag=bst>nb=dec>IBk=J48>jrip\\
\hline PC4&	rf>bag=bst=dec>lgi>nb>J48>IBk=jrip\\
\hline MW1&	rf=bst>nb=bag>log=dec>IBk>jrip>J48\\
\hline MC2&	rf>lgi = dec>IBk=nb=bag>J48>bst>jrip\\
\hline
\end{tabular}
\end{table}

Table~\ref{tab:originalWLT} shows the comparison result from the original data ($none$) listed in the decreasing order of the number of wins. We can tell that random forest (rf) is the best learner in $none$ preprocessing data set from Table~\ref{tab:originalWLT} which wins $67$ out of $72$ cases, ties with other learners in $5$ cases, and never lose in any case to other learners. Figure~\ref{fig:none} shows the boxplot diagrams of these $9$ learners over $9$ datasets. Table~\ref{tab:originalWLTtrend} shows the trend of $9$ learners in each dataset.
The following observations can be drawn from the above:
\be
\item  Random Forest has the best performance among $9$ learners with $67$ wins, 5 ties and none loss.
\item  Bagging, logistic and boosting are not bad in the original data.
\item IBk, j48, and Jrip do not have good enough performance comparing to other classifiers in original dataset.
\ee
\subsection{Log transformation}
\begin{table}
\centering
\caption{Comparison result of log transformation.}
\label{tab:logWLTtrend}
\begin{tabular}{|l|l|c|c|c|}
\hline Learner&	prep&	ties&	wins&	losses\\
\hline rf&	log&	7&	65&	0\\
\hline bag&	log&	14&	46&	12\\
\hline bst&	log&	13&	38&	21\\
\hline lgi&	log&	10&	37&	25\\
\hline nb&	log&	11&	37&	24\\
\hline dec&	log&	9&	34&	29\\
\hline IBk&	log&	6&	17&	49\\
\hline j48&	log&	5&	9&	58\\
\hline jrip&	log&	3&	2&	67\\
\hline
\end{tabular}
\end{table}

%\begin{figure}
%\centering
%\psfig{file=none.ps,height=3in, width=3in,}
%\caption{Mean and variance in the original data.}\label{fig:none}
%\end{figure}
\begin{figure}
\centering
\psfig{file=log.ps,height=3in, width=3in,}
\caption{Boxplot diagrams of log transformation ($log$) data.}\label{fig:log}
\end{figure}

\begin{table}
\centering
\caption{The trend of log transformation ($log$)}
\label{tab:originalWLTtrend}
\begin{tabular}{|l|l|}
\hline dataset&	Order of log transformation\\
\hline General&	Rf>bag>bst>nb>Dec>IBk>j48=jrip\\
\hline CM1&	Rf>lgi>nb>bag>bst>Dec>IBk>J48=jrip\\
\hline KC1&	Rf>bag>lgi>bst>nb=Dec>IBk>j48=jrip\\
\hline KC3&	Rf>nb>bag=bst>IBk=lgi=Dec>j48>jrip\\
\hline KC4&	Rf=nb=bag=Dec>bst>j48>jrip=lgi>IBk\\
\hline PC1&	Rf>bag>lgi=bst>Dec>nb>IBk=j48>jrip\\
\hline PC3&	Rf>lgi=bag=bst>nb>Dec>IBk>j48>jrip\\
\hline PC4&	Rf>lgi>bag>bst=Dec>nb>j48>IBk=jrip\\
\hline MW1&	Rf=bst>nb=bag>Dec>lgi>IBk>jrip>j48\\
\hline MC2&	Rf=IBk=Dec>nb>lgi=bag>j48=bst>jrip\\
\hline
\end{tabular}
\end{table}

From Table~\ref{tab:logWLTtrend}, Table~\ref{tab:originalWLTtrend}, and Figure~\ref{fig:log}, we can see that the performance of log transformation ($log$) is similar to that of the original dataset ($none$) in terms of the number of wins, losses and tie, boxplot diagrams and overall trend of individual dataset. Similar observations can be made in $log$ as that of $none$ method.

\subsection{Discretized data}
\begin{table}
\centering
\caption{Comparison result of discretized data ($nom$).}
\label{tab:nomWLT}
\begin{tabular}{|l|l|c|c|c|}
\hline Learner&	prep&	ties&	wins&	losses\\
\hline bst&	nom&	7&	57&	8\\
\hline nb&	nom&	6&	50&	16\\
\hline aode&	nom&	5&	49&	18\\
\hline lgi&	nom&	10&	39&	23\\
\hline bag&	nom&	15&	34&	23\\
\hline rf&	nom&	13&	29&	30\\
\hline IBk&	nom&	3&	23&	46\\
\hline j48&	nom&	6&	7&	59\\
\hline jrip&	nom&	3&	2&	67\\
\hline
\end{tabular}
\end{table}

\begin{table}
\centering
\caption{The trend of discretized data($nom$).}
\label{tab:nomWLTtrend}
\begin{tabular}{|l|l|}
\hline dataset&	Order of discretized data\\
\hline General&	bst>nb>aode>lgi>bag>rf>IBk>j48>jrip\\
\hline CM1&	bst >nb>aode>IBk>lgi>rf>bag>j48=jrip\\
\hline KC1&	nb=bag>bst=aode>lgi>j48=rf>IBk>jrip\\
\hline KC3&	nb>aode>bag=bst>lgi>rf>j48>IBk\\
\hline KC4&	nb>lgi=bst>aode>rf=bag>IBk>j48>jrip\\
\hline PC1&	bst>lgi=aode>IBk>nb=rf=bag>jrip>j48\\
\hline PC3&	bst>lgi=rf=bag>aode>nb>IBk>jrip>j48\\
\hline PC4&	lgi=bst>rf=bag>aode>j48=nb>IBk>jrip\\
\hline MW1&	bst>nb=aode>lgi=rf>IBk=bag>j48=jrip\\
\hline MC2&	nb>aode>IBk>rf>bag=bst>j48=lgi>jrip\\
\hline
\end{tabular}
\end{table}

\begin{figure}
\centering
\psfig{file=nom.ps,height=3in, width=3in,}
\caption{Boxplot in the discretized data($nom$).}\label{fig:nom}
\end{figure}

Table~\ref{tab:nomWLT} shows the number of wins, losses, and ties in discretization data in the decreasing order of the number of wins. Table~\ref{tab:nomWLTtrend} shows the general trend of each learner's performance in individual data. Figure~\ref{fig:nom} shows the boxplot diagrams of discretization data. From these tables and figure, we have these observations:
\be
\item  Boosting is the best learner on discretized data.
\item  NaiveBayes, AODE, logistic, and bag are not bad.
\item  In a result consistent with prior work~\cite{Dougherty95}, we observe here
that Naivebayes performs much better on discretized data than that of in $none$ and $log$ methods.
\item  The performance of Random Forest is decreased in discretized data.
\item  IBk, j48, jrip still remain three worst learners in discretized data.
\ee

\subsection{Discretized on log data}
\begin{table}
\centering
\caption{Comparison result of discretization on $log$ data (log\&nom).}
\label{tab:logandnomWLT}
\begin{tabular}{|l|l|c|c|c|}
\hline Learner&	prep&	ties&	wins&	losses\\
\hline bst&	log\&nom&	6&	58&	8\\
\hline nb&	log\&nom&	4&	52&	16\\
\hline aode&	log\&nom&	5&	48&	19\\
\hline lgi&	log\&nom&	11&	39&	22\\
\hline bag&	log\&nom&	13&	36&	23\\
\hline rf&	log\&nom&	13&	31&	28\\
\hline IBk&	log\&nom&	6&	17&	49\\
\hline j48&	log\&nom&	5&	7&	60\\
\hline jrip&	log\&nom&	3&	3&	66\\
\hline
\end{tabular}
\end{table}

\begin{figure}
\centering
\psfig{file=logandnom.ps,height=3in, width=3in,}
\caption{Boxplot in $log\&nom$.}\label{fig:logandnom}
\end{figure}

\begin{table}
\centering
\caption{The trend of discretized on log transformation ($log\&nom$).}
\label{tab:logandnomWLTtrend}
\begin{tabular}{|l|l|}
\hline dataset&	Order of discretized data\\
\hline General&	bst>nb>aode>lgi>bag>rf>IBk>j48>jrip\\
\hline CM1&	bst>nb>aode>lgi>IBk>rf>IBk>j48>jrip\\
\hline KC1&	nb=bag>bst=aode>lgi>j48=rf>IBk>jrip\\
\hline KC3&	nb>bag=aode>bst>lgi>j48=rf>IBk>jrip\\
\hline KC4&	nb>lgi=bst>aode>rf=bag>IBk>j48=jrip\\
\hline PC1&	bst>lgi=aode>nb>rf>bag>IBk>jrip>j48\\
\hline PC3&	bst>lgi=rf=bag>aode>nb>IBk>jrip>j48\\
\hline PC4&	lgi=bst>rf=bag>aode>j48>nb>IBk>jrip\\
\hline MW1&	bst>nb>aode>lgi=rf>IBk=bag>jrip>j48\\
\hline MC2&	nb>aode>IBk=rf=bag=bst>lgi>j48=jrip\\
\hline
\end{tabular}
\end{table}

Table~\ref{tab:logandnomWLT} shows the number of wins,losses,and ties in discretization data in the decreasing order of the number of wins. Table~\ref{tab:logandnomWLTtrend} shows the general trend of each learner's performance in individual data. Figure~\ref{fig:logandnom} shows the boxplot diagrams of discretization data. From these tables and figure, we see that all the observations we have in $nom$ method are hold in this $log\&nom$ method.

\subsection{Comparison between Random Forest and Boosting}
We perform the Wilcoxon nonparametric statistical test to compare the performance between the Random Forest in the form of $none$, $log$ preprocessing data and that of boosting in the form of $nom$ and $log\&nom$ data.  The $p$ value of $H_0$ is $0.00000001510363 (<0.05)$ which suggests a significance difference between these two learners. The $p$ value of $H_1$ is $0.000000007.551813 (<0.05)$ which support that the performance of random forest in $none$ and $log$ is better than that of boosting in $nom$ and $log\&nom$. We performs the Mann-Whitney test which also gives the same result although with different $p$ value. Figure~\ref{fig:bestCom} shows the boxplot diagrams of them. Thus, comparing two best performance learners in different preprocessing methods, random forest of $none$ and $log$ beats boosting of $nom$ and $log\&nom$ methods.
%Table~\ref{tab:bestCom} shows the mean and variance of random forest and boosting in their best preprocessing methods.
 %The other point is that the %run time is required more for random forest than that of boosting.  If in some time %constraint and memory and cpu %constraint situation, discretized data using boosting is a good choice.
%\begin{table}
%\centering
%\caption{Mean and variance of random forest and boosting.}
%\label{tab:bestCom}
%\begin{tabular}{|l|l|l|l|}
%\hline learner&prep &		meanAUC&	var\\
%\hline rf&	log&	0.827733&	0.004106\\
%\hline rf&	none&	0.829074&	0.004079\\
%\hline bst&	log\&nom&	0.813569&	0.003304\\
%\hline bst&	nom&	0.812349&	0.003459\\
%\hline
%\end{tabular}
%\end{table}
\begin{figure}
\centering
\psfig{file=bestComparison.ps,height=3in, width=3in,}
\caption{Comparison of best learners in 4 preprocessing methods.}\label{fig:bestCom}
\end{figure}

\subsection{Comparison among different preprocessing methods}
\begin{table}
\centering
\caption{Comparison among the best learners in 4 preprocessing methods}
\label{tab:comPrep}
\begin{tabular}{|l|l|}
\hline rf	& none=log>nom=log\&nom\\
\hline bag & log>nom=none=log\&nom\\
\hline bst	& none=log<nom=log\&nom\\
\hline IBk&	none<log<nom=log\&nom\\
\hline nb & none=log<nom=log\&nom\\
\hline lgi &	none=log<nom=log\&nom\\
\hline j48 &	none=log=log\&nom=nom\\
\hline jrip	& none=log=nom=log\&nom\\
\hline dec	& none=log\\
\hline aode	& nom=log\&nom\\
\hline
\end{tabular}
\end{table}
We perform the Wilcoxon nonparametric statistical test to compare the performance of $4$ different preprocessing methods in the MDP dataset. Table~\ref{tab:comPrep} shows the comparison result. Different learners perform differently in different preprocessing methods. We have the following observations:
\be
\item Boosting, NaiveBayes, IBk, and logistic have better performance in discretization data.
\item J48 and jrip has the same performance in $4$ different preprocessing methods.
\item Decorate has the same performance in the original and log transformation data.
\item AODE has the same performance in two discretized data.
\item Random forest's performance is better in $none$ and $log$ preprocessing methods than that of discretization data ($nom$ and $log\&nom$)
\item Bagging is the only learner which has better performance in log transformation than that of other preprocessing methods.
\ee

\section{Conclusions}
This paper investigates the effect of four preprocessing methods on prediction of software fault-proneness quality models based on NASA MDP dataset. We have the following observations.
\be
\item Random forest in the original has the best performance across any other kinds of learners in any kinds of transformation methods. Its performance is decreased in discretized data.
\item Boosting is the best learner among all learners in discretized data.
\item NaiveBayes' performance is greatly improved in discretized data which confirms prior results in~\cite{Dougherty95}.
\item IBk, J48, and jrip probably are not right learners to be considered in MDP dataset because they are all ranked in the worst three learners in all 4 preprocessing methods.
\item Log transformation do not affect predictive models performance except on learner bagging. 
\ee
Therefore, in the different situation, in order to gain better predictive result, we should choose a preprocessing method based on a specific learner instead of using universal transformation for all kinds of learners. 
%Random forest performs better in original data with more number of individual data values. Bagging has the best %performance in log transformation data. Discretization data is better for boosting, NaiveBayes,logistic and IBk machine %learners.

%\end{document}  % This is where a 'short' article might terminate

%ACKNOWLEDGMENTS are optional
%\section{Acknowledgments}
%This section is optional; it is a location for you
%to acknowledge grants, funding, editing assistance and
%what have you.  In the present case, for example, the
%authors would like to thank Gerald Murray of ACM for
%his help in codifying this \textit{Author's Guide}
%and the \textbf{.cls} and \textbf{.tex} files that it describes.

%
% The following two commands are all you need in the
% initial runs of your .tex file to
% produce the bibliography for the citations in your paper.
\bibliographystyle{abbrv}
\bibliography{pf}
%\bibliography{sigproc}  % sigproc.bib is the name of the Bibliography in this case
% You must have a proper ".bib" file
%  and remember to run:
% latex bibtex latex latex
% to resolve all references
%
% ACM needs 'a single self-contained file'!
%
%APPENDICES are optional
%\balancecolumns
%\balancecolumns % GM June 2007
% That's all folks!
\end{document}
