\documentclass{sig-alternate}
\usepackage{url} 
\usepackage{graphicx} 
%\usepackage{subfig}
\usepackage{verbatim}
\usepackage{fancyvrb}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{times}
%\usepackage{timtricks}
\usepackage{cite}
%\newcommand{\bi}{\begin{smallitem}}
%\newcommand{\ei}{\end{smallitem}}
%\newcommand{\be}{\begin{smallenum}}
%\newcommand{\ee}{\end{smallenum}}
\newcommand{\bi}{\begin{smallitem}}
\newcommand{\ei}{\end{smallitem}}
\newcommand{\be}{\begin{smallenum}}
\newcommand{\ee}{\end{smallenum}}
\newcommand{\tion}[1]{\S\ref{tion:#1}}
\newcommand{\eq}[1]{Equation~\ref{eq:#1}}
\newcommand{\fig}[1]{Figure~\ref{fig:#1}}
\newenvironment{smallitem}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{itemize}
   \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{itemize}}
\newenvironment{smallenum}
 {\setlength{\topsep}{0pt}
  \setlength{\partopsep}{0pt}
  \setlength{\parskip}{0pt}
  \begin{enumerate}
  \setlength{\leftmargin}{.2in}
  \setlength{\parsep}{0pt}
  \setlength{\parskip}{0pt}
  \setlength{\itemsep}{0pt}}
 {\end{enumerate}}

\begin{document}


\CopyrightYear{2007} % Allows default copyright year (200X) to be over-ridden - IF NEED BE.
\crdata{0-12345-67-8/90/01}  % Allows default copyright data (0-89791-88-6/97/05) to be over-ridden - IF NEED BE.
% --- End of Author Metadata ---

\title{When to Use Data from Other Projects for Effort Estimation}


% You need the command \numberofauthors to handle the 'placement
% and alignment' of the authors beneath the title.
%
% For aesthetic reasons, we recommend 'three authors at a time'
% i.e. three 'name/affiliation blocks' be placed beneath the title.
%
% NOTE: You are NOT restricted in how many 'rows' of
% "name/affiliations" may appear. we just ask that you restrict
% the number of 'columns' to three.
%
% Because of the available 'opening page real-estate'
% we ask you to refrain from putting more than six authors
% (two rows with three columns) beneath the article title.
% More than six makes the first-page appear very cluttered indeed.
%
% Use the \alignauthor commands to handle the names
% and affiliations for an 'aesthetic maximum' of six authors.
% Add names, affiliations, addresses for
% the seventh etc. author(s) as the argument for the
% \additionalauthors command.
% These 'additional authors' will be output/set for you
% without further effort on your part as the last section in
% the body of your article BEFORE References or any Appendices.

\numberofauthors{3} %  in this sample file, there are a *total*
% of EIGHT authors. SIX appear on the 'first-page' (for formatting
% reasons) and the remaining two appear in the \additionalauthors section.
%
\author{
\alignauthor
Ekrem Kocaguneli, Gregory Gay\\Tim Menzies\\
       \affaddr{LCSEE, WVU, USA}\\
       \email{ekocagun@mix.wvu.edu, greg@greggay.com, tim@menzies.us}
% 2nd. author
\alignauthor Ye Yang\\
       \affaddr{Institute of Software}\\
	   \affaddr{Chinese Academy of Sciences}\\       
       \affaddr{Beijing, People's Republic of China }\\
       \email{ye@itechs.iscas.ac.cn}
      % 3rd. author
\alignauthor Jacky W. Keung\\
       \affaddr{School of Computer Science and Engineering}\\
	   \affaddr{University of New South Wales}\\       
       \affaddr{Sydney, Australia}\\
       \email{jacky.keung@nicta.com.au}       
}

\conferenceinfo{ASE}{'10 Antwerp, Belgium}
\maketitle
\begin{abstract}
%  ~\\\textit{\textbf{Background}}: Collecting the project data
%  required for quality prediction is a time-consuming and expensive
%  process.  Therefore, it is tempting {\em not} to collect that data
%  within one company and, instead, make quality predictions using data
%  collected from other companies.  However, the reliability of (e.g.)
%  effort estimates based on {\em cross}-company data is questionable.
%  \\\textit{\textbf{Aim}:} we hypothesis that {\em data irrelevancies}
%is the root cause of unreliable {\em cross}-company effort estimation.
%we therefore tested if {\em relevancy filtering} can
%increase the reliability of  {\em cross}-company data.
%  \\\textit{\textbf{Method}:} 
%we test the effects of relevancy
%  filtering on three effort estimation datasets and
%  their subsets.   \\\textit{\textbf{Results}:} In seven of our nine experiments
%  (\textit{3 datasets * 3 subsets = 9 treatments}), after applying
%relevancy filtering, the estimates using
%  {\em cross}-company data were statistically the same as those generated using
% within-company data.
%  \\\textit{\textbf{Conclusion}:} 
%Reliable estimates can be generated from {\em cross}-company data,
%provided a relevancy filter has been applied prior to prediction.
Collecting the data required for quality prediction {\em within}
a development team is time-consuming
and expensive.  An alternative to make 
predictions using data that {\em crosses}  from other projects
or even other
companies.
We show that with/without {\em relevancy filtering}, imported data
performs the same/worse (respectively)  than using local data. 
Therefore, we recommend the use of relevancy filtering whenever
generating estimates using data from another project.
\end{abstract}

%A category including the fourth, optional field follows...
\category{D.2.9}{Software Engineering}{Effort Estimation}

\keywords{Effort estimation, data mining, cross, within}

\section{Introduction}
When  data is scarce {\em within} one project, it is tempting to use
data imported from other projects.  Such
{\em cross}-project data exist; for example the PROMISE repository
\cite{Boetticher2007} offers a dozen effort estimation data sets
for public access.

A  recent survey
paper has evaluated {\em within} or {\em cross} data for effort
estimation~\cite{Kitchenham2007}. They concluded that they
could not make a conclusion; that the current findings are contradictory
about the relative merits of {\em within} or {\em cross} data.

In other work~\cite{Turhan2009}, we have shown that it is
acceptable to
use {\em cross} data sources for defect prediction, providing that data has been  pre-processed by some
sort of {\em relevancy filtering}.
Given a large training set, such relevancy filters select a small subset relevant to the current test case.
Such filtering removes training instances that create noise in the estimation
process, leaving a body of data that, in theory, follows the principle
of locality.

The success of relevancy filtering for defect prediction prompts us to
apply it to effort estimation. 
To the best of our knowledge, this is
the first exploration in the effort estimation community of the
effects of relevancy filtering when applied to {\em cross} and {\em within}
project data.
We show that
{\em cross} data
can usually attain estimation accuracies just as high as those of {\em within} data, provided that a relevancy filter
is applied to the data, prior to making estimates.
%
%\section{Background}
%
%\subsection{Experts vs. Models}
%
%We can group software effort estimation studies into~\cite{Shepperd2007}: Expert judgment and model-based techniques.
%
%Expert judgment methods are one of the most widely used estimation methods\cite{Jorgensen2004}.
%Application of expert judgment may be either explicit (following a method like Delphi\cite{Boehm2000}) or implicit (informal discussions among 
%such experts).
%One problem with expert-based estimates is that they may fall victim to competing interests in the sense that a faulty estimation of a 
%senior expert may be taken over the more accurate estimation made by a junior expert {\em within} the same company.
%Another problem, indicated by Jorgensen et. al., is the poor capability of humans to improve their own expert judgment\cite{Jorgensen2009}. 
%
%Model-based techniques are the methods generated by using algorithmic and parametric approaches or by induced prediction systems.
%The former approach is the adaptation of an expert-proposed model to local data.
%A well known example to such an approach is Boehm's COCOMO method\cite{Boehm1981}.
%The latter approach is useful in the case where local data does not conform to the specifications of the expert's method.
%A few examples of induced prediction systems are linear regression, neural nets, model trees and analogies \cite{Menzies2006, Kadoda2001}.
%All of these systems are built on inherent assumptions.
%In the case where data violates such assumptions, patches are applied. An example of a patch is
%taking the logarithm of exponential distributions before linear regression \cite{Boehm1981, Kitchenham2009}.
%However, choosing the appropriate patch again requires qualified experts.
%
%Different organizations may choose to use an expert judgment, a model-based approach, or some combination of the two in different settings.
%However, the goal of any estimation model is a common one - to attain high estimation accuracy.
%On the other hand, estimation accuracy does not depend only on the choice of estimation model.
%Another critical factor is the possession of detailed and accurate historical data that has been carefully brought up to date.
% 
%\subsection{Within/Cross Effort Estimation}
%
%When a model-based technique is chosen for estimation, historical data is required
%to learn the inner details of those models.
%Organizations may choose to collect and maintain information concerning their own past projects in a dataset - this
%is referred to as a {\em within} dataset.
%Previous studies have suggested using such locality-specific past data for accurate estimates \cite{kemerer87, Kitchenham1984}.
%However, collection of {\em within} data comes with a cost.
%As experts on the field, Kitchenham and Mendes have reported three problems likely to occur when an organization wishes to rely on 
%local  data \cite{Mendes2004, Mendes2005, Kitchenham2007}:
%
%\bi
%\item[1] The time required to accumulate enough local data may be prohibitive.
%\item[2] By the time that the local dataset is large enough, the technologies employed by the local projects 
%may have changed and old projects may have become obsolete.
%\item[3] For collecting local data in a consistent manner, care is necessary.
%\ei
%
%On top of the before mentioned reasons, based on our personal experience, we can say that in the result-oriented environment of 
%today's corporations, the time that must be spent collecting data (and the inevitable postponement of tangible results) 
%is likely to decrease the enthusiasm of managers for estimation practices.
%
%The problems regarding local data collection have motivated researchers to search for alternative solutions like the use of {\em cross}  datasets 
%(that is, datasets containing data from several different companies) \cite{Mendes2004, Mendes2005}.   
%Although software effort estimation is a relatively young research field, the collective effort of many contributors have enabled the
%building of considerable repositories of software effort datasets.
%For example, in the PROMISE data repository alone\footnote{http://promisedata.org}, there are currently twelve effort prediction datasets 
%that are available for public access \cite{Boetticher2007}.
%However, the use of {\em cross} data is not a silver-bullet solution and comes with its own set of 
%problems\cite{Mendes2004, Mendes2005, Kitchenham2007}:
%\bi
%\item[1)] It is even more difficult to ensure the consistency of data collection across multiple organizations.
%\item[2)] Process and practices differ across companies, leading to differing trends in the data.
%\item[3)] In the absence of a proper sampling strategy, it is difficult to make sure that randomly-selected {\em cross} projects truly 
%form a random sample of the defined population.
%\ei
%
%In a systematic review on the issue, Kitchenham and Mendes tried to find a consensus
%in the literature about the relative benefits of {\em cross} vs {\em within} data for effort 
%estimation~\cite{Kitchenham06asystematic, Kitchenham2007}. Their reluctant conclusion
%was that no such conclusion exists.
%Among the ten studies in their review, only seven of them showed independent evidence 
%in their comparisons of accuracies between {\em cross} and {\em within} prediction models.
%Amongst those seven valid studies, three studies found that {\em cross} performance was not significantly worse than {\em within}.
%The remaining four found that {\em within} models significantly outperformed {\em cross} models\cite{Kitchenham06asystematic}.
%Therefore, the findings on {\em cross} vs. {\em within} data usage are currently inconclusive. 
%
%\subsection{Within/Cross Defect Prediction}
%
%Using {\em cross} data for the purposes of prediction is not limited to software effort estimation.
%The subject has also been addressed in the software defect prediction domain.
%Turhan et. al. questioned the applicability of {\em cross} data in defect prediction and found that {\em cross} 
%data can be successfully used \cite{Turhan2009}.
%{\em Cross} data, taken \textit{``as is''}, is only useful under relatively limited conditions.
%However, {\em cross} data filtered for irrelevancies through the use of a nearest-neighbor technique has yielded surprisingly 
%high accuracy results\cite{Turhan2009}.
%
%Zimmermann et. al. have addressed the same question at the individual project level and have come up with similar conclusions \cite{Zimmermann2009}.
%They agree that  building {\em cross}-prediction models is a serious challenge.
%They also report  using the {\em cross} data {\em as is} 
%does not lead to useful predictions.
%In order to increase these accuracy values, Zimmermann et. al. propose an approach that selects projects for {\em cross} 
%predictors \cite{Zimmermann2009}. This approach can be regarded as an expert-guided relevancy filtering.
%Taking these findings in defect prediction domain as an impetus, we would like to explore the applicability and performance of 
%applying a {\em relevancy filtering} technique to {\em cross} data in the effort estimation domain.
%
\section{Relevancy Filtering}
\label{section:relevancy-filtering}
Our relevancy filter  extends standard analogy-based estimation methods (which we call ABE0).
ABE0
generates estimates for a test project by gathering evidence from the
effort values of similar projects in some training set.
By analyzing the previous research of experts like Shepperd et. al.\cite{Shepperd1996}, 
Mendes et. al.\cite{Mendes2003} and Li et. al.\cite{Li2009} on the field of analogy-based estimation, we can come up with a baseline technique:
\bi
\item Build a training data from rows of past projects;
\item The columns of this set are composed of \textit{independent} variables (the features that define projects) and a \textit{dependent} 
variable (the recorded effort value). 
\item Decide on how many similar projects (\textit{analogies}) to use when examining a new test instance
, i.e. \textit{k}-values.
\item For each test instance, select \textit{k} analogies from training.
\bi
\item While selecting analogies, use a similarity measure (such as the Euclidean distance of features).
\item Before calculating similarity, apply a scaling measure on independent features to equalize their influence on this similarity measure.
\ei
\item Use the effort values of the \textit{k} nearest analogies to calculate an effort estimate.
\ei
We can refer to this baseline framework as ABE0.
For the similarity measure, we use Euclidean distance:
\begin{equation}\label{equ:euclid}\small
Distance = \sqrt{\sum_{i=1}^{n}{(x_{i} - y_{i})^{2}}}
\end{equation}
ABE0 returns the median of the efforts in the \textit{k} nearest analogies. 

%Using cross-company data in an \textit{as is} fashion has been shown to have a negative effect on accuracy values \cite{Zimmermann2009}.
%However, the use of a relevancy filtering method has been shown to increase the accuracy of cross-company models in software defect prediction 
%domain \cite{Turhan2009, Zimmermann2009}. 
%Although different researchers have used different relevancy filters (Turhan et. al. use a nearest neighbor filter, whereas the filtering
%employed by Zimmermann et. al is closer to guided expert judgment), it is obvious that filtering for relevant data is helpful when 
%attempting to increase accuracy in cross-company models in the defect prediction domain.
%
%we have described an analogy-based estimation baseline (ABE0) in Section \ref{subsection:ABE0}.
%For our relevancy filtering, we will use a form of ABE0 in which the selection of analogies will be based on the calculated performance variance.

Our relevancy filter  is a small variant of ABE0. It is a two-pass system.
Pass 1 removes  the training instances implicated in poor decisions;
pass 2 selects those instances nearest the test instance.

In pass 1, the training projects are used to generate a binary
tree.  The leaves of this binary tree are
formed by the individual training projects, which are then greedily
clustered in tuples to form the parent levels.  This binary tree,
which we will call BT1, is then traversed upwards from the root to
height level one (one higher level than the leaves).  The variance
of the effort values in each sub-tree (the performance variance)
is then recorded and normalized to a 0-1 interval.  Pass one prunes all sub-trees with a 
variance greater than $10\%$
of the maximum variance seen in any tree.  

%A random relevancy filtering is applied on these normalized variances as follows:
%\bi
%\item Generate a random number from a normal distribution.
%\item Prune nodes with a normalized performance variance above the random value.
%\ei
The leaves of the remaining sub-trees are the {\em survivors} of pass one.
These move to pass 2 where the
survivors are used to build a second binary tree
(called BT2).  BT2 is generated and traversed by test instances
in the same fashion as
BT1.  This time, while traversing the tree, instead of storing the
variances of sub-trees, we use the variance as a decision criterion.
If the variance of the current tree is larger than its sub-trees,
then continue to move down the subtree; otherwise, stop moving and
select the instances in the current tree as the relevant instances
and adapt them for estimation.  
Relevancy filtering is visualized in 
Figure \ref{fig:filtering}.

\begin{figure}
\begin{center}
\includegraphics[width=0.35\textwidth]{filtering1.png}
\end{center}
\caption{Two pass relevancy filtering. 
Each tree BT1 and BT2 are binary cluster trees.
The red sub-tree is pruned in pass one due to high variance. The remaining subtrees (shown in
green) form the right-hand tree. In pass two, test instances
start at the root of this tree and traverse to the nearest child (and so on, recursively).
While the sub-tree variance continues to decrease, the traversal continues.
Estimates are generated from the median of the instances of the right-hand-side
sub-tree with lowest variance.
}\label{fig:filtering}
\end{figure}

This filter is similar to the NN-filter
used by Turhan et.al.\cite{Turhan2009}, except that there is no
need to pre-specify the number of analogies $k$ to be used for estimation.
Each
test instance selects its own relevant analogies by traversing
to different sub-trees of BT2.

For a detailed discussion on the rationale behind this filter, see~\cite{koc10}.
All we need to say here is that this filter is known to
generate low errors for ABE0-style effort
estimation~\cite{koc10}. Hence, it is a suitable tool for the rest of this study.
%\begin{figure*}[!t]
%\centering
%{\small  
%\begin{tabular}{l|rrl|l}  
% \textbf{} & \textbf{} & \textbf{} & \textbf{}&\multicolumn{1}{c}{\textbf{Historical Effort Data}}\\
% \textbf{Dataset} & \textbf{Features} & \textbf{Projects} & \textbf{Content}&\textbf{Units}\\ \hline
%  Cocomo81 &17&63& NASA projects& months\\
%  \hspace{6 mm}Coc81o &17&24& Cocomo81 organic projects&months\\ 
%  \hspace{6 mm}Coc81e &17&28& Cocomo81 embedded projects&months\\ 
%  \hspace{6 mm}Coc81s &17&11& Cocomo81 semidetached projects&months\\   
%  Nasa93 &17&93& NASA projects&months\\ 
%  \hspace{6 mm}Nasa93c1 &17&12& Nasa93 projects from center 1&months\\ 
%  \hspace{6 mm}Nasa93c2 &17&37& Nasa93 projects from center 2&months\\ 
%  \hspace{6 mm}Nasa93c5 &17&39& Nasa93 projects from center 5&months\\ 
%  Desharnais &12&81& Canadian software projects&hours\\
%  \hspace{6 mm}DesL1 &11&46& Desharnais projects developed with language 1&hours\\ 
%  \hspace{6 mm}DesL2 &11&25& Desharnais projects developed with language 2&hours\\
%  \hspace{6 mm}DesL3 &11&10& Desharnais projects developed with language 3&hours  
%\end{tabular}}
%\caption{Nine treatments = three subsets of three data sets. 
%Indentation in column one denotes that indented dataset is a subset of another dataset.
%}\label{fig:data}
%\end{figure*}

\section{Methodology}

In our research, we have used subsets of three commonly-used datasets in software effort estimation research: Nasa93, the original 
Cocomo81 \cite{Boehm1981}, and Desharnais\cite{Desharnais1989}.

We will denote the subsets of Nasa93 as Nasa93c1, Nasa93c2 and Nasa93c5.
Nasa93c1, Nasa93c2 and Nasa93c5 contain projects from different NASA development centers around the United States 
(denoted as development centers 1, 2 and 5 in the complete dataset).
In a similar fashion, subsets of Cocomo81 will be denoted as Coc81o, Coc81e and Coc81s (for
organic, embedded, and semidetached).
Finally, the
Desharnais dataset is split into three different subsets:
DesL1, DesL2 and DesL3 (languages 1, 2 and 3 respectively).
Since each of these subsets have certain common criteria (the development center, development mode, or development language), 
each subset will be treated as a separate {\em within} dataset.
All of the datasets used in this research are available in PROMISE data repository \cite{Boetticher2007}.

For each of the three main datasets (Nasa93, Cocomo81 and Desharnais) in our research, we have conducted {\em within} and {\em cross} 
experiments.
Each subset became a {\em within} dataset that contains projects sharing the particular characteristics
of a single development firm.
%
%\begin{figure*}[ht!]
%\begin{center}
%\includegraphics[width=0.8\textwidth]{ccexperiments.png}
%\end{center}
%\caption{In {\em cross}-company experiments for subsets $X_1$, $X_2$ and $X_3$ we have 3 treatments. At each treatment, one subset is selected as test set and the remaining two become the {\em cross}-company data, i.e. train set. After filtering, median of the remaining instances from the train set become our estimate.}\label{fig:ccexperiments}
%\end{figure*}
%\begin{figure}[!b]
%\renewcommand{\baselinestretch}{1}
%\begin{algorithmic} \scriptsize
%\STATE \textit{$win_{i}$} = 0, \textit{$tie_{i}$} = 0, \textit{$loss_{i}$} = 0
%\STATE \textit{$win_{j}$} = 0, \textit{$tie_{j}$} = 0, \textit{$loss_{j}$} = 0
%\IF{MANN-WHITNEY(\textit{$MRE's_{i}$}, \textit{$MRE's_{i}$}) says they are the same}
%	\STATE \textit{$tie_{i}$} = \textit{$tie_{i}$} + 1;
%	\STATE \textit{$tie_{j}$} = \textit{$tie_{j}$} + 1;
%\ELSE
%	\IF{median(\textit{$MRE's_{i}$}) $<$ median(\textit{$MRE's_{j}$})}
%		\STATE \textit{$win_{i}$} = \textit{$win_{i}$} + 1
%		\STATE \textit{$loss_{j}$} = \textit{$loss_{j}$} + 1
%	\ELSE
%		\STATE \textit{$win_{j}$} = \textit{$win_{j}$} + 1
%		\STATE \textit{$loss_{i}$} = \textit{$loss_{i}$} + 1
%	\ENDIF
%\ENDIF
%\end{algorithmic}
%\caption{Pseudocode for Win-Tie-Loss Calculation Between Method \textit{i} and \textit{j}}
%\label{FigureWinTieLossPseudocode}
%\end{figure}
\begin{figure*}[!t]
%\renewcommand{\baselinestretch}{.8}
\centering
\small
  \begin{tabular}{l | l  l  l | c  c  c }   
  \textbf{Data set} &\textbf{Train Set} & \textbf{Test Set} & \textbf{Method} & \textbf{Win} & \textbf{Tie} & \textbf{Loss} \\ \hline			
Nasa93	&	Nasa93c1	&	Leave-one-out test instance	&	{\em Within}	& 1	&   	&  \\
	&	Nasa93c2 and Nasa93c5	&	Nasa93c1	&	{\em Cross}			& 	&   	& 1	\\\cline{2-7}
	&	Nasa93c2	&	Leave-one-out test instance	&	{\em Within}		& 1	&		&  \\
	&	Nasa93c1 and Nasa93c5	&	Nasa93c2	&	{\em Cross}			& 	&		& 1	\\\cline{2-7}
	&	Nasa93c5	&	Leave-one-out test instance	&	{\em Within}		& 	&	1	& 	\\
    &	Nasa93c1 and Nasa93c2	&	Nasa93c5	&	{\em Cross}			& 	&	1	& 	\\	\hline
Cocomo81	&	Coc81o	&	Leave-one-out test instance	&	{\em Within}	& 1 &		& 	\\	
	&	Coc81e and Coc81s	&	Coc81o	&	{\em Cross}					& 	&		& 1	\\\cline{2-7}
	&	Coc81e	&	Leave-one-out test instance	&	{\em Within}			& 	&	1	&  \\
	&	Coc81o and Coc81s	&	Coc81e	&	{\em Cross}					& 	&	1	& 	\\\cline{2-7}
	&	Coc81s	&	Leave-one-out test instance	&	{\em Within}			&  &	1	& 	\\
	&	Coc81o and Coc81e	&	Coc81s	&	{\em Cross}					& 	&	1	& \\	\hline
Desharnais	&	DesL1	&	Leave-one-out test instance	&	{\em Within}	& 1	&		& 	\\
	&	DesL2 and DesL3	&	DesL1	&	{\em Cross}						& 	&		& 1	\\\cline{2-7}
	&	DesL2	&	Leave-one-out test instance	&	{\em Within}			& 	&	1	& 	\\	
	&	DesL1 and DesL3	&	DesL2	&	{\em Cross}						& 	&	1	& 	\\\cline{2-7}
	&	DesL3	&	Leave-one-out test instance	&	{\em Within}			& 1 &		& 	\\
	&	DesL1 and DesL2	&	DesL3	&	{\em Cross}						& 	&		& 1	\\
	\end{tabular}
\caption{MRE win-tie-loss results without relevancy filtering.
Every odd and even line is a pair of experiments.
In each pair, there is a {\em within} and a {\em cross} experiment. 
In {\em cross} experiment, a linear regression model is built on {\em cross} data and tested on the {\em within} data. 
In {\em within} experiment, the test instance is selected with leave-one-out, 
and a linear regression model is built on the remaining instances and tested on the selected test instance.
A ``1'' denotes which item in the pair won, lost or tied.}\label{fig:wltNoFiltering}
\end{figure*}



To understand the {\em within} and {\em cross} data formation, assume that a dataset $X$ with its three subsets $X_1$, $X_2$ and $X_3$ 
is under consideration.
For {\em within} experiments, the relevancy filtering described in Section \ref{section:relevancy-filtering} is applied on each one 
of $X_1$, $X_2$ and $X_3$ separately and the median of the filtered project instances in the training set is stored as the effort estimate 
for the test instance.
For the separation of training and testing sets, the leave-one-out method is used.
Leave-one-out selects one instance out of a dataset of $n$ instances as the test instance and uses the remaining $n-1$ instances as the training set.

For the {\em cross} experiments, one of $X_1$, $X_2$ or $X_3$ is chosen as the test set and the combination of the remaining two forms 
the {\em cross} dataset for training.
This time, the relevancy filtering is applied on the {\em cross} dataset, and the estimations for projects in the test set are stored.

Each of the {\em within} and {\em cross} experiments are repeated twenty times in order to remove any bias that would otherwise 
be brought by a particular test and training set combination. 

In order to compare the performance of {\em within} and {\em cross} datasets,
we have used two measures: the magnitude of relative error (MRE) and win-tie-loss values generated by a statistical rank-sum test.
\begin{equation} \small
MRE = \frac{|actual_{i} - predicted_{i}|}{actual_{i}}
\label{EquationMRE} 
\end{equation}
Using a Mann-Whitney test (95\%), we checked how often one treatment won/lost/tied with the others. Here, a ``tie''
means
that they are not
statistically significant different.
If statistically different,  then the method with a lower median MRE score gets one more ``win'' and the other method gets one more ``loss''.
\section{Results}

In our experiment, we analyzed \textit{3 datasets * 3 subsets = 9
treatments}.  
We evaluated {\em cross} and {\em within} performances of each
particular dataset, subject to statistical tests, with and without relevancy filtering.


%\subsection{Preliminary Results}
%
%Based on the earlier results (as well as the related literature reviews\cite{Turhan2009, Zimmermann2009, Mendes2004, Mendes2005, Kitchenham06asystematic, Kitchenham2007}), 
%we were quite aware of the fact that the usage of cross-company versus {\em within}-company data for effort estimation purposes is a 
%controversial issue.
%Therefore, before going into the details of our results, we took the time to observe the behavior of {\em cross}-company projects 
%from different datasets when subject to a relevancy filter.
%
%In Figure \ref{fig:srsizeNasa93}, we can clearly see the effect of relevancy filtering for the Nasa93 dataset.
%The instances of the {\em cross}-company dataset that come from different centers are not favored over one another.
%On the contrary, some instances were commonly selected from every center that formed a {\em cross}-company treatment.
%In the second column of Figure \ref{fig:srsizeNasa93}, we can see how many instances were selected on average 
%from each center in each {\em cross}-company treatment.
%While we could note that the relevancy filter chooses certain projects from every center, we were unable to observe any definitive patterns.
%
%\begin{figure}[th!]
%\renewcommand{\baselinestretch}{.8}
%\centering
%\footnotesize
%  \begin{tabular}{ l | c  c  c }   
%  \textbf{{\em {\em cross}}-Companies} & \multicolumn{3}{c}{\textbf{Selected}} \\\hline	
%    &		&		&		\\
%	&		Nasa93c1	&	Nasa93c2	&	Nasa93c5	\\
%Nasa93c2 and Nasa93c5	&		0	&	2.35	&	1.17	\\
%Nasa93c1 and Nasa93c5	&		1.41	&	0	&	1.79	\\
%Nasa93c1 and Nasa93c2	&		2.36	&	1.08	&	0	\\
%  &		&		&		\\
%  	\end{tabular}
%\caption{After relevancy filtering on {\em cross}-company datasets, the distribution of \textit{selected} instances do not come from a particular center of Nasa93.}
%\label{fig:srsizeNasa93}
%\end{figure}
%
%Note that in Figure \ref{fig:srsizeNasa93}, each row has one center whose number of selected instances is zero.
%This is due to the fact that one particular center is always kept as the separate test set. Hence, no instances from that center 
%can be selected for that row.
%
%Figure \ref{fig:srsizeCoc81} shows the average number of projects selected from each subset in the {\em cross}-company datasets of Cocomo81.
%The same relevancy filtering effect that appeared in Nasa93 can also be observed on Cocomo81.
%As we can see in Figure \ref{fig:srsizeCoc81}, relevancy filtering selects instances from all of the subsets forming a particular 
%{\em cross}-company treatment.
%For the last treatment (where Coc81o and Coc81e are the {\em cross}-company firms) in Figure \ref{fig:srsizeCoc81}, we see that the effect of 
%relevancy filtering for selecting instances from each subset is quite limited. The average number of instances chosen from Coc81o is 
%2.75 whereas the same number for Coc81e is 0.25.
%
%\begin{figure}[th!]
%\renewcommand{\baselinestretch}{.8}
%\centering
%\footnotesize
%  \begin{tabular}{ l | c  c  c }   
%  \textbf{{\em {\em cross}}-Companies} & \multicolumn{3}{c}{\textbf{Selected}} \\\hline	
%    &		&		&		\\
%	&		Coc81o	&	Coc81e	&	Coc81s	\\
%Coc81e and Coc81s	&			0	&	1.98	&	1.29	\\
%Coc81o and Coc81s	&			3.57	&	0	&	1.36	\\
%Coc81o and Coc81e	&			2.75	&	0.25	&	0	\\
%  &		&		&		\\
%  	\end{tabular}
%\caption{Effect of relevancy filtering on Cocomo81 {\em cross}-companies is similar to those of Nasa93. Instances are selected from all subsets.}
%\label{fig:srsizeCoc81}
%\end{figure}
%
%Figure \ref{fig:srsizeDesharnais} shows the same effect for Desharnais that we observed for Nasa93 and Cocomo81 in Figures \ref{fig:srsizeNasa93} and
%\ref{fig:srsizeCoc81}. That is, the selection of instances is never limited to a single subset of the {\em cross}-company treatment.
%However, if we look at row two (DesL1 and DesL3) of Figure \ref{fig:srsizeDesharnais}, we can see that this effect is again extremely limited -
%an average of 2.05 instances are chosen from DesL1, while only 0.1 instances are chosen from DesL2.
%This same situation was encountered for one of the Cocomo81 treatments.
%Therefore, we would suggest that there is a hidden influence of certain data features on the selection of instances from {\em cross}-company datasets.
%The task of discovering such features and defining their influences is a potential basis for future work.
%
%\begin{figure}[th!]
%\renewcommand{\baselinestretch}{.8}
%\centering
%\footnotesize
%  \begin{tabular}{ l | c  c  c }   
%  \textbf{{\em {\em cross}}-Companies} & \multicolumn{3}{c}{\textbf{Selected}} \\\hline	
%    &		&		&		\\
%	&		DesL1	&	DesL2	&	DesL3	\\
%DesL2 and DesL3	&	0	&	2.27	&	0.65 \\
%DesL1 and DesL3	&	2.05	&	0	&	0.1 \\
%DesL1 and DesL2	&	3.17	&	1.59	&	0 \\
%
%  &		&		&		\\
%  	\end{tabular}
%\caption{Similar trend of relevancy filtering continues for Desharnais dataset as well. Selected instances come from all datasets of {\em cross}-company datasets.}
%\label{fig:srsizeDesharnais}
%\end{figure}
%
%As a result of our preliminary research, we have seen that relevancy filtering, in the majority of cases, does not favor any single subset 
%of a {\em cross}-company dataset. 
%On the contrary, the instances selected as relevant (i.e. the instances in the SR tree of Figure \ref{fig:filtering}) come from all of the 
%"companies" that compose a single {\em cross}-company dataset.
%A summary of our preliminary results is given in Figure \ref{fig:srsize}.
%we can observe from this figure that, for each {\em cross}-company dataset, there are instances from every subset being selected by the relevancy filter.
%From this, it can be deduced that - although the definition of {\em within}-company data is based on single dimensions such as geographical proximity 
%of development teams (centers 1, 2 and 5 of Nasa93), mode of development (organic, embedded and semi-detached modes of Cocomo81) or 
%particular languages used to develop projects (languages 1, 2 and 3 of Desharnais) - it is actually the individual independent variables 
%that have an influence on the selection of projects as relevant to a particular test instance. 
%
\subsection{Without Relevancy Filtering}
\label{none}

In
this first experiment, we have 9 treatments 
and for each treatment we observe the estimation
performances when {\em within} and {\em cross} datasets are
used.  For this purpose we used a linear regression model. Two-pass filtering was not applied. 

In
{\em cross} experiments, for each data set, we selected one of the 3 subsets as the
test set and the remaining two as the train set.  We then built a
linear regression model on the {\em cross} data and applied this
model on the test set.  For the {\em within} dataset we also used
a linear regression model.  The test case selection for {\em within}
experiment is performed in accordance with leave-one-out method,
which picks up one of the instances in the dataset as the test set
and uses the remaining instances as the train set.  The linear
regression model that is built on the train set is then tested on
the single test instance. 
After applying linear regression model on {\em within} and {\em cross} datasets, we calculated the win-tie-loss values for each treatment.
%One caution before commenting on the results is that in Figure \ref{fig:wltNoFiltering} the sum of win-tie-loss values do not add up to 20 unlike previous cases.
%The reason is that we can make only a single run in that scenario, since we will end up with the same linear model in different runs.

As shown in  Figure \ref{fig:wltNoFiltering}, we see that in a minority
of cases ($\frac{4}{9}$, see
Nasa93c5, Coc81e, Coc81s and DesL2), {\em cross} and {\em within} data
perform just as well as each other.
In the majority case ($\frac{5}{9}$, see Nasa93c1, Nasa93c2, Coc81o, DesL1,
DesL3),  {\em within} performed better than {\em cross} data.
That is, in the absence of relevancy
filtering, the {\em within} datasets yield significantly lower
MRE values in majority of  cases.
%
%
\subsection{With Relevancy Filtering}
\label{with}

This section shows that for each data set, the application of relevancy filtering reverses the conclusion
of the previous section; i.e. the {\em cross} data becomes useful for estimating the local project.
  
Figure \ref{fig:wltNasa93}, shows the the win-tie-loss values for the subsets of Nasa93.
The greedy clustering algorithm of the two pass relevancy filtering uses some non-determinism (when breaking ties
between instances of similar distances), so we repeat these experiments twenty times. 

This results  shows us that, in all three treatments, the $tie$ values are quite high.
This indicates that, for 
at least 75\% of the tests, there is no statistical difference between filtered {\em cross} and {\em within} results.
In short, for Nasa93, the performance of {\em cross} data (filtered for relevancy) is indistinguishable from the performance of 
{\em within} data.

\begin{figure}[th!]
\renewcommand{\baselinestretch}{.8}
\centering
\footnotesize
  \begin{tabular}{l | l | c  c  c }   
  \textbf{Dataset} & \textbf{Method} & \textbf{Win} & \textbf{Tie} & \textbf{Loss} \\ \hline
  & & & \\			
Nasa93c1				&	within	& 3	&	    15	&	     2 \\
Nasa93c2 and Nasa93c5	&	cross	& 2	&	    15	&	     3	\\
 & & & \\	\hline
   & & & \\	
Nasa93c2				&	within	&	3	&	17	&	0 \\
Nasa93c1 and Nasa93c5	&	cross	&	0	&	17	&	3	\\
 & & & \\	\hline
   & & & \\	
Nasa93c5				&	within	&	1	&	19	&	0	\\
Nasa93c1 and Nasa93c2	&	cross	&	0	&	19	&	1	\\
	\end{tabular}
\caption{MRE win-tie-loss values for Nasa93 from 20 {\em randomized assessments}. In all treatments $tie$ values are quite high. For Nasa93, the performance of {\em cross} data is mostly same as {\em within} data.}
\label{fig:wltNasa93}
\end{figure}

Figure \ref{fig:wltCocomo81} shows the win-tie-loss values for the subsets of Cocomo81.
In two out of the three treatments the $tie$ values are 19, which tells us that for these treatments, {\em within} and {\em cross} performance
are almost identical.
However, the first treatment shows a preference for {\em within} data on thirteen of the twenty tests.

\begin{figure}[th!]
\renewcommand{\baselinestretch}{.8}
\centering
\footnotesize
  \begin{tabular}{l | l | c  c  c }   
  \textbf{Dataset} & \textbf{Method} & \textbf{Win} & \textbf{Tie} & \textbf{Loss} \\ \hline
  & & & \\			
Coc81o				&	within		&	13	&	7	&	0	\\
Coc81e and Coc81s	&	cross		&	0	&	7	&	13	\\
 & & & \\	\hline
   & & & \\	
Coc81e				&	within		&	1	&	19	&	0	\\
Coc81o and Coc81s	&	cross		&	0	&	19	&	1	\\
 & & & \\	\hline
   & & & \\	
Coc81s				&	within		&	0	&	20	&	0	\\
Coc81o and Coc81e	&	cross		& 	0	&	20	&	0	\\
\end{tabular}
\caption{MRE win-tie-loss values for Cocomo81 from 20 {\em randomized assessments}. In 2 treatments {\em cross}
data  is the same as the {\em within} data. However, in the case of Coc81o, {\em within} outperforms {\em cross} data.}
\label{fig:wltCocomo81}
\end{figure}

The win-tie-loss values for subsets of Desharnais are given in Figure \ref{fig:wltDesharnais}.
The derived results for the Desharnais subsets are similar to those of Cocomo81 treatments:
Two out of the three treatments show identical $tie$ values of 19, which again suggests that the performance of filtered {\em cross} datasets is
statistically identical to {\em within} datasets.
However, in one of the treatments, {\em within} outperforms {\em cross} on sixteen of the twenty trials.

\begin{figure}[th!]
\renewcommand{\baselinestretch}{.8}
\centering
\footnotesize
  \begin{tabular}{l | l | c  c  c }   
  \textbf{Dataset} & \textbf{Method} & \textbf{Win} & \textbf{Tie} & \textbf{Loss} \\ \hline
  & & & \\			
DesL1	&	within				&	1	&	19	&	0	\\
DesL2 and DesL3	&	cross		& 	0	&	19	&	1	\\
 & & & \\	\hline
   & & & \\	
DesL2	&	within &	1	&	19	&	0	\\
DesL1 and DesL3	&	cross 		&	0	&	19	&	1	\\
 & & & \\	\hline
   & & & \\	
DesL3	&	within 				&	16	&	4	&	0	\\
DesL1 and DesL2	&	cross 		&	0	&	4	&	16	\\
		\end{tabular}
\caption{MRE win-tie-loss values for Desharnais from 20 {\em randomized assessments}. In the case of DesL3 the
{\em within} data  is much better than the {\em cross} data. For other treatments, {\em within} and {\em cross}
data are statistically the same.}
\label{fig:wltDesharnais}
\end{figure}


In summary, with relevancy filtering, in the majority case
($\frac{7}{9}$ treatments) the {\em cross} data performs as well as the
{\em within} data for effort estimation.  There are only two treatments,
DesL3 and Coc81o, where {\em within} performance was significantly better
than {\em cross} performance.  A possible explanation for those two
scenarios may be hidden in the dataset size or in the quality of
the {\em within} datasets, but the currently-available information makes
it difficult to suggest any conclusive reason for the situation.

%\begin{figure*}[th!]
%\renewcommand{\baselinestretch}{.8}
%\centering
%\footnotesize
%  \begin{tabular}{l | l  l  l | c  c  c }   
%  \textbf{Data set} &\textbf{Train Set} & \textbf{Test Set} & \textbf{Method} & \textbf{Win} & \textbf{Tie} & \textbf{Loss} \\ \hline			
%Nasa93	&	Nasa93c1	&	Leave-one-out test instance	&	{\em within}	& 3	&   15	& 2 \\
%	&	Nasa93c2 and Nasa93c5	&	Nasa93c1	&	{\em Cross}			& 2	&   15	& 3	\\
%	&	Nasa93c2	&	Leave-one-out test instance	&	{\em Within}		& 3	&	17	& 0 \\
%	&	Nasa93c1 and Nasa93c5	&	Nasa93c2	&	{\em Cross}			& 0	&	17	& 3	\\
%	&	Nasa93c5	&	Leave-one-out test instance	&	{\em Within}		& 1	&	19	& 0	\\
%    &	Nasa93c1 and Nasa93c2	&	Nasa93c5	&	{\em Cross}			& 0	&	19	& 1	\\	\hline
%Cocomo81	&	Coc81o	&	Leave-one-out test instance	&	{\em Within}	& 13&	7	& 0	\\	
%	&	Coc81e and Coc81s	&	Coc81o	&	{\em Cross}					& 0	&	7	& 13	\\
%	&	Coc81e	&	Leave-one-out test instance	&	{\em Within}			& 1	&	19	& 0 \\
%	&	Coc81o and Coc81s	&	Coc81e	&	{\em Cross}					& 0	&	19	& 1	\\
%	&	Coc81s	&	Leave-one-out test instance	&	{\em Within}			& 0 &	20	& 0	\\
%	&	Coc81o and Coc81e	&	Coc81s	&	{\em Cross}					& 0	&	20	& 0\\	\hline
%Desharnais	&	DesL1	&	Leave-one-out test instance	&	{\em Within}	& 1	&	19	& 0	\\
%	&	DesL2 and DesL3	&	DesL1	&	{\em Cross}						& 0	&	19	& 1	\\
%	&	DesL2	&	Leave-one-out test instance	&	{\em Within}			& 1	&	19	& 0	\\	
%	&	DesL1 and DesL3	&	DesL2	&	{\em Cross}						& 0	&	19	& 1	\\
%	&	DesL3	&	Leave-one-out test instance	&	{\em Within}			& 16&	4	&	0	\\
%	&	DesL1 and DesL2	&	DesL3	&	{\em Cross}						& 0	&	4	&	16	\\
%	\end{tabular}
%\caption{MRE win-loss-tie results from 20 {\em randomized assessments}.
%Comparisons are between within and {\em cross} performance of each subset of every dataset.}\label{fig:wlt}
%\end{figure*}

\begin{figure*}[th!]
%\renewcommand{\baselinestretch}{.8}
\centering
%\small
  \begin{tabular}{ l | l | c  c  c }   
  \textbf{{\em Cross} Dataset} & \textbf{Test Set}  & \multicolumn{3}{c}{\textbf{Instances selected in BT2}} \\\hline	
    &		&		&		&		\\
	&		&	From Nasa93c1	&	From Nasa93c2	&	From Nasa93c5	\\
Nasa93c2 and Nasa93c5	&	Nasa93c1	&	0	&	2.4	&	1.2	\\
Nasa93c1 and Nasa93c5	&	Nasa93c2	&	1.4	&	0	&	1.8	\\
Nasa93c1 and Nasa93c2	&	Nasa93c5	&	2.4	&	1.1	&	0	\\
  &		&		&		&		\\\hline
  &		&		&		&		\\									
	&		&	From Coc81o	&	From Coc81e	&	From Coc81s	\\
Coc81e and Coc81s	&	Coc81o	&	0	&	2.0	&	1.3	\\
Coc81o and Coc81s	&	Coc81e	&	3.6	&	0	&	1.4	\\
Coc81o and Coc81e	&	Coc81s	&	2.8	&	0.3&	0	\\
  &		&		&		&		\\\hline
		  &		&		&		&		\\							
	&		&	From DesL1	&	From DesL2	&	From DesL3	\\
DesL2 and DesL3	&	DesL1	&	0	&	2.3	&	0.7	\\
DesL1 and DesL3	&	DesL2	&	2.1	&	0	&	0.1	\\
DesL1 and DesL2	&	DesL3	&	3.2	&	1.6	&	0	\\
	\end{tabular}
\caption{Mean number of instances used for estimation
after filtering in 20 runs. {\em Cross} datasets are combinations of two {\em within} datasets tested on another {\em within} dataset. 
}
\label{fig:srsize}
\end{figure*}
\newpage
\section{Discussion}
\fig{srsize} shows the mean number of
instances used for analogy-based estimation by our two-pass relevancy
filtering algorithm.  
Surprisingly, the number of selected analogies is very small: mean value around 3.
Further, while exceptions exist, the selected analogies come from multiple other
projects. For example, for the Nasa93 dataset,  the data relevant to center c1
came $\frac{2}{3}$-rds and
$\frac{1}{3}$-rd from  centers c2 and c5 (respectively).

This suggests that we should revisit what we mean by {\em within} and {\em cross}:
\bi
\item
Effort
estimation functions on a theory of {\em locality}; i.e.  new projects 
follow similar practices to historical projects and should require a
similar amount of effort.  As Chen et. al. \cite{chen09} have shown,
inconsistencies in data collection across multiple companies create
locality-specific biases in {\em cross} data sets.  Such biases
result in an unacceptable amount of variance in the effort
calculations.
\item
\fig{srsize} is saying that {\em similar} projects may not come from the same geographical
location. Our relevance filters hunted out handfuls of {\em similar} projects from other
subsets. Perhaps software differs on many dimensions (such as  where it was written),
and the most important dimensions for finding {\em similar} projects may not be
mere geography. 
\ei
Given the complex multi-dimensional nature of the software
creation process, the geographical dimension may be less important
than other factors.  The most similar software to what you
are writing now may not be in the next office. Rather, it may be
in an office on the other side of the world.

Going forward, we would like to learn exactly why an instance is
deemed ``relevant" or ``irrelevant" by our filter.
In other words, we would like to know exactly which
features are most influential when assigning relevancy.  The ability
to identify these exact dimensions would make the selection of
appropriate projects easier for any institution that uses {\em cross}
data.
It would also lead to (a)~more accurate filtering techniques;
and (b)~a better understanding of the structure of software projects including
where to find data most relevant to some current project.


%
%\section{Threats to Validity}
%
%The ideal case for satisfying internal validity would be to learn a theory from a past situation and then to apply the theory to a new setting.
%However, general software effort studies make use of commonly-explored past datasets such as the ones used in this research
%(Nasa93, Cocomo81, and Desharnais).
%This issue of internal validity threaten all effort studies using only past data.
%We can mitigate this problem by simulating the behavior of a learned theory in new settings.
%In our study, we make use of the leave-one-out method for all treatments to address such internal validity issues. 
%Leave-one-out selection enables us to separate the training and test sets completely in each experiment.
%
%Another issue is our choice of learning algorithms.
%In the above, we have apparently compared the results of linear regression (in \S~\ref{none}) to analogy-based
%estimation (in \S~\ref{with}). Is such a comparison valid?  We believe so since what was constant in the comparison was
%the {\em cross} vs {\em within} data used in the analysis. Only the learner was changed, and changed in a very specific way.
%For the purposes of this discussion, the essential feature of the linear regression study was that it trained from all available
%{\em cross} data. The second study, on the other hand,  explored what happens when the training  set was restricted to just instances
%close by the test instance. We considered conducting a third study where we applied relevancy filtering as a pre-processor to linear regression.
%In that third study, we would have used relevancy filtering to
%select the training instances for linear regression. However,
%\fig{srsize} showed that the number of selected instances selected
%in this way would be around three- a number that is far too small for effective regression.
%
%A further  issue requiring discussion is the two passes used by our relevancy filter. Perhaps that it is an over-elaboration, and a standard
%simple one-pass Euclidean nearest neighbor algorithm might suffice. 
%This issue has been addressed elsewhere~\cite{koc10}: our two-pass system produces much lower errors than the standard
%nearest neighbor algorithm. Hence, we recommend it for analogy-based effort estimation.
%%XXX need to think more about "local"
%
%Lastly, one frequently asked question about this work relates to the 
%``{\em cross}'' nature of the subsets of our data.
%Is it fair to describe NASA projects from different centers as ``{\em cross}''?
%When writing about defect data sets from NASA,
%Zimmerman et al. comment:
%\begin{quote}
%{\em For their study Turhan et al.~\cite{Turhan2009} analyzed 12 NASA projects 
%(mostly in C++) which they considered cross-company because 
%they were all developed by contractors under the umbrella of 
%NASA. However, all projects had to follow stringent ISO-9001 
%industrial practices imposed by NASA, so it is unclear to what 
%extent the data can be actually considered cross-company. }~\cite{Zimmermann2009}
%\end{quote}
%In reply, we assert that NASA is a much more diverse organization
%than is commonly appreciated. NASA software is written by layers
%of contractors working for different companies around the nation.
%Arguing that all  ISO-compliant organizations are the same is like
%saying the a CMM level3 weapons manufacturer is writing the same
%software as a CMM level3  finance company. It is true that rigid
%hardware requirements on flight systems forces a conformity in the
%final software. However the methods used to create that software
%can be radically different at different contractor organizations.
%Also, very little of our NASA data is from flight systems (13 out
%out 93 records spread evenly across the three centers used to form the NASA subsets). The bulk of our data 
%come from ground systems which are much less constrained in nature than flight systems.
%In any case, we have empirical support for our claim that all our subsets come
%from truly different sources. 
%In the results presented above:
%\bi
%\item
%Without relevancy filtering,
%for the purposes of estimating the current project,
%the {\em cross}-subset data is
%usually {\em useless}.
%\item
%With relevancy filtering,  the differences disappear. Predictions using the {\em cross}-subset
%data is usually {\em useful}.
%\ei
%Those results are enough to recommend the use of relevancy filtering 
%when processing data from other projects.
%
%%XXX if you must use cross, then apply relevancy filtering
%\section{Conclusion}
%With these results, we can now address the questions that we proposed to guide this research.
%
%\textbf{RQ1) What is the effect of selecting particular {\em cross} projects on the estimation performance?}
%
%We have seen that, in the majority case, {\em cross} datasets filtered for relevancy have performed as well as 
%{\em within} datasets.
%Therefore, we can state that selecting particular {\em cross} projects, guided by a relevancy filter,
%has a positive effect on the estimation performance of analogy-based estimation.
%
%\textbf{RQ2) Is there evidence that {\em cross} data can yield accuracy values equal to that of {\em within} data?}
%
%For seven out of the nine treatments used in our experiments, filtered {\em cross} data has performed equally to 
%{\em within} data. 
%Our experiments have yielded statistical evidence that {\em cross} data can be used to obtain high accuracy values.
%However, for {\em cross} data to attain such values, we cannot use the data \textit{``as is"}.
%Rather, we must use a relevancy filtering method to remove the instances that cause a high performance variance.
%
%\textbf{RQ3) Do the characteristics of particular datasets influence the {\em within} versus {\em cross} performance?}
%
%In two treatments, we have observed that {\em within} datasets significantly outperformed {\em cross} datasets.
%Therefore, we cannot say that the raw characteristics of particular datasets have no influence on  estimation performance.
%However, the exact reason behind this is not yet obvious.
%In our case, the two treatments that yielded higher {\em within} performance were datasets with 24 and 10 projects respectively. 
%Based on dataset size alone, it is difficult to observe any connection between number of projects and estimation performance.
%One potential reason could be the data quality of each set.
%However, we do not yet have any evidence to back such claims.
%
%\section{Future Work}
%
\section*{Acknowledgments}
This work was partially
supported by the National Natural Science Foundation of China under Grant Nos. 60873072 and 90718042.
\bibliographystyle{abbrv}
\bibliography{myref}  % sigproc.bib is the name of the Bibliography in this case

\end{document}
