%\documentclass[12pt,draftscls,onecolumn,journal]{IEEEtran}
\documentclass[journal]{IEEEtran}
%\renewcommand{\baselinestretch}{1.67}
\usepackage{alltt}
\usepackage{bar}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage{cite}      % Written by Donald Arseneau
                        % V1.6 and later of IEEEtran pre-defines the format
                        % of the cite.sty package \cite{} output to follow
                        % that of IEEE. Loading the cite package will
                        % result in citation numbers being automatically
                        % sorted and properly "ranged". i.e.,
                        % [1], [9], [2], [7], [5], [6]
                        % (without using cite.sty)
                        % will become:
                        % [1], [2], [5]--[7], [9] (using cite.sty)
                        % cite.sty's \cite will automatically add leading
                        % space, if needed. Use cite.sty's no adjust option
                        % (cite.sty V3.8 and later) if you want to turn this
                        % off. cite.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/cite/

\usepackage{graphicx}  % Written by David Carlisle and Sebastian Rahtz
                        % Required if you want graphics, photos, etc.
                        % graphicx.sty is already installed on most LaTeX
                        % systems. The latest version and documentation can
                        % be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/required/graphics/
                        % Another good source of documentation is "Using
                        % Imported Graphics in LaTeX2e" by Keith Reckdahl
                        % which can be found as esplatex.ps and epslatex.pdf
                        % at: http://www.ctan.org/tex-archive/info/

\usepackage{subfigure} % Written by Steven Douglas Cochran
                        % This package makes it easy to put sub figures
                        % in your figures. i.e., "figure 1a and 1b"
                        % Docs are in "Using Imported Graphics in LaTeX2e"
                        % by Keith Reckdahl which also documents the graphicx
                        % package (see above). sub figure.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/subfigure/

\usepackage{url}       % Written by Donald Arseneau
                        % Provides better support for handling and breaking
                        % URLs. url.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/other/misc/
                        % Read the url.sty source comments for usage information.

%\usepackage{stfloats}  % Written by Sigitas Tolusis
                        % Gives LaTeX2e the ability to do double column
                        % floats at the bottom of the page as well as the top.
                        % (e.g., "\begin{figure*}[!b]" is not normally
                        % possible in LaTeX2e). This is an invasive package
                        % which rewrites many portions of the LaTeX2e output
                        % routines. It may not work with other packages that
                        % modify the LaTeX2e output routine and/or with other
                        % versions of LaTeX. The latest version and
                        % documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/sttools/
                        % Documentation is contained in the stfloats.sty
                        % comments as well as in the presfull.pdf file.
                        % Do not use the stfloats baseline float ability as
                        % IEEE does not allow \baselineskip to stretch.
                        % Authors submitting work to the IEEE should note
                        % that IEEE rarely uses double column equations and
                        % that authors should try to avoid such use.
                        % Do not be tempted to use the cited.sty or
                        % midfloat.sty package (by the same author) as IEEE
                        % does not format its papers in such ways.

\usepackage{amsmath}   % From the American Mathematical Society
                        % A popular package that provides many helpful commands
                        % for dealing with mathematics. Note that the AMSmath
                        % package sets \interdisplaylinepenalty to 10000 thus
                        % preventing page breaks from occurring within multiline
                        % equations. Use:
%\interdisplaylinepenalty=2500
                        % after loading amsmath to restore such page breaks
                        % as IEEEtran.cls normally does. amsmath.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/



% Other popular packages for formatting tables and equations include:

\usepackage{array}
% Change the following for any hyphenation issues.  These are just examples. (Not actually used yet.)
\hyphenation{op-tical net-works semi-conduc-tor}
\input{../jim/tricks.sty}
\input{../tim/tricks.sty}
\input{../jeremy/tricks.sty}
\begin{document}
%
% paper title
\title{A Data Miner for Searching Model-Based Software }
%
%
% author names and IEEE memberships
% note positions of commas and non breaking spaces ( ~ ) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
\author{
Tim~Menzies,~\IEEEmembership{Member,~IEEE,}
\thanks{Dr. Menzies is with the Lane Department of Computer
Science, West Virginia University and can be reached at \protect\url{tim@menzies.us}},
James~Kiper,~\IEEEmembership{Member,~IEEE,} \thanks{Dr. Kiper is with the
Computer Science and Systems Analysis Department,
School of Engineering \& Applied Science, Miami University, and can be reached
at \protect\url{kiperjd@muohio.edu}} 
Jeremy Greenwald\thanks{Mr. Greenwald is with
Computer Science, Portland State University and can be reached
at \protect\url{jegreen@cecs.pdx.edu}},  Ying Hu\thanks{Ms. Hu is a software
designer in Vancouver, British Columbia and can be reached at 
\protect\url{huying_ca.@yahoo.com}}, 
David Raffo, \IEEEmembership{Member,~IEEE}\thanks{Dr. Raffo is a Associate Professor in the School of Business
Administration, Portland State University, and can be reached at \protect\url{raffod@sba.pdx.edu}},
and Siri-on Setamanit\thanks{Ms. Setamanit is a graduate student at Portland State University
and can be reached at \protect\url{sirion@pdx.edu}}\thanks{The research described in this paper
was carried out at Miami University, 
West Virginia University and Portland State University under
contracts and sub-contracts with NASA's Software Assurance Research Program.
Reference herein to any specific commercial product, process, or service
by trademark, manufacturer, or otherwise, does not constitute or imply
its endorsement by the United States Government.
}\thanks{Manuscript received January 1, 2007.
Download an earlier draft from
\protect\url{http://menzies.us/pdf/07exploit.pdf}
}}% <-this % stops a space\thanks{Contact information: kiperjd@muohio.edu, timm@cs.pdx.edu, jegreen@cs.pdx.edu .}}
% note the % following the last \IEEEmembership and also the first \thanks - 
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
% 
% \author{....lastname \thanks{...} \thanks{...} }
%                     ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "A\textbf{} \textbf{}B" will typeset as "A B" not "AB". If you want
% "AB" then you have to do: "A\textbf{}\textbf{}B"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.
%
% The paper headers
%\markboth{Journal of \LaTeX\ Class Files,~Vol.~1, No.~11,~November~2002}{Shell \MakeLowercase{\textit{et al.}}: Bare Demo of IEEEtran.cls for Journals}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the two side option.
% 
% *** Note that you probably will NOT want to include the author's name in ***
% *** the headers of peer review papers.                                   ***

% If you want to put a publisher's ID mark on the page
% (can leave text blank if you just want to see how the
% text height on the first page will be reduced by IEEE)
%\pubid{0000--0000/00\$00.00~\copyright~2002 IEEE}

% use only for invited papers
%\specialpapernotice{(Invited Paper)}

% make the title area
\maketitle


\begin{abstract}
\input{../jim/abstract}
\end{abstract}

\begin{keywords}
model-driven software engineering, contrast set learning, treatment learning
\end{keywords}
% Note that keywords are not normally used for peer review papers.

% For peer review papers, you can put extra information on the cover
% page as needed:
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
%
% For peer review papers, inserts a page break and creates the second title.
% Will be ignored for other modes.
\IEEEpeerreviewmaketitle


\input{../jim/jimOne}
\input{../tim/general}
\input{../tim/tar2}
\input{../tim/examples}
%\input{../tim/related}

\noindent
In all the  case studies explored by TAR2,
the same three observations were made:
\bi
\item Treatment learning can find very small treatments,
even for seemingly complex models;
\item
These
treatments can be far smaller that models generated by standard data miners.
\item 
Despite uncertainties or variabilities in the model, TAR2 was able
to find effective treatments that selected for preferred model output
(but the less uncertainty or model variability, the smaller the variance
in TAR2's predicted output for the treated model).
\ei

%\section{Related Work}
%
%Elsewhere~\cite{me03l} we have discussed the connection of treatment learning to
%soft computing methods such as fuzzy logic, genetic algorithms, and
%neural nets. 
%Approximate,
%heuristically generated models would be adequate to control
%models if those models contain clumps and collars.
%That is,
%the widespread presence of clumps and collars
%would explain the success of soft computing. 
%
%Treatment learner offers coarse-grained heuristic
%controllers for models with a wide range of internal parameter settings.
%{\em Automatic formal methods},
%on the other hand, rigorously explore the structure of
%small models
%that are precisely specified (albeit with some non-determinism).
%Hence they are tools to be used for {\em different tasks} in the
%life cycle of a model.
%
%Early in the software life cycle, engineers are concerned with
%exploring a large space of possible models or possible model
%configurations. Model-based tools needed for such {\em option assessment}
%(e.g. TAR2) may be very different from model-based tools for {\em
%fine-tuning} and verifying a final model such as SPIN~\cite{spinbook}.
%Certainly, for mission and safety critical models, it is essential
%that such rare but dangerous bugs be located and fixed. However,
%while much of the model remains under debate, the rigorous
%search of, say, SPIN
%can be rendered obsolete by the next revision to the model.
%
\section{Conclusion}\label{conclusions}

\PARstart{U}nderstanding model configuration options means understanding how
input choices affect output scores. That understanding is complex
for a certain class of {\em hard models}; i.e. those with high
dimensionality models that are non-linear, non-continuous and built
in domains with much noise or other uncertainties, and where managers
have limited control over all model inputs.  

Hard modeling problems may defeat standard methods.
Visualization can't handle very large dimensionality.  Analytical
methods such as an eigenvector study offer spurious results if the
parameters of the variables are uncertain. Other standard automatic
methods may be defeated by ``cliffs'' in non-linear models where the
association between inputs and outputs changes abruptly.
Data mining methods can handle non-linear models and scale to very large
dimensionality. Sadly, data mining techniques like neural nets, genetic algorithms,
C4.5 and CART  can yield models that are incomprehensible to humans.

TAR2 is a special kind of data miner that produces very succinct
output. It assumes that
within models there exists a small number of key variables that control 
the rest.
There is much evidence for this assumption.
The mathematics of clumps and collars promises that
models
naturally contain structures that
greatly restrict the space
of possible model behaviors. 
TAR2 is a data miner
designed to exploit such collars and clumps.
%\begin{wrapfigure}{r}{1.5in}
%\begin{center}
%\includegraphics[width=1.5in]{ai.pdf}
%\end{center}
%\caption{Humans assisted by AI.}\label{fig:cycle}
%\end{wrapfigure}
It is a minimal contrast set learner that returns a ``treatment'';
i.e. a minimal, most influential
set of deltas between different classes of outcome.
The case studies in this paper show that
a minimal list of the differences between concepts can
be {\em much smaller} than a detailed description of all aspects of a concept.
For models where TAR2 can generate  succinct summaries,
its algorithm can significantly improved searched-based methods of data mining.
%Human designers could avoid wasting time on superfluous details in that
%models could be iteratively built by humans, 
%then pruned by minimal contrast set learning. 
%In this way, 
%human-based design intelligence could be augmented in a cost-effective manner via
%artificial intelligence.
%

TAR2 addresses the hard modeling problems (discussed in \tion{easyhard}) 
as follows:

\bi
\item
TAR2 offers minimal constraints on the input space  and tracks the effects
of those constraints, while letting all the other variables vary randomly.
Hence, its proposed solutions are not brittle to changes outside the treatments. 
\item
Since it only references a subset of the model inputs, it is a dimensionality
reduction tool. In this report,
we offered examples where TAR2 reasoned over
100-variable inputs spaces. Elsewhere, we have run it on data sets with over 250 variables. In all cases seen to date, it reduces those spaces to a handful of variables.
\item
Such small solutions are easier to explain and audit than solutions using
all model inputs. 
\item
Further, when managers do not have the budget or authority to control all model
input variables,
TAR2 can offer them 
a minimal set which they can use to focus their resources.
\item
TAR2 has been applied to models with large amounts of noise. For example,
in a TAR2-style
analysis, we often 
explore what happens to the solutions when the variance
on the model variables increases.  Such studies return statements of the form
``the solutions offered by this analysis hold for variances up to the following
critical threshold values, after which we do not know how to control this domain''.
\ei

Treatment learning is not indicated
for low-dimensionality
linear continuous models built in domains that have no noise or other
uncertainties, and where managers have full control over all model inputs.
Other reasons not to use our tools include when where there is no
need to explain or audit models, where reducing model variance is
not valuable, and there exists budgets for building and using
maximal models.

As to further work,
there is no reason to polarize the SE modeling field into ``traditional methods''
vs ``treatment learning''. Much could be achieved by combining the two techniques.
For example, many of the methods described in \tion{easyhard} suffered
from the curse of dimensionality. 
TAR2 could be used as a fast dimensionality reduction tool
that could focus a data visualization environment or a sensitivity
analysis on the parts of the inputs space that are most crucial.
Ideally, that focusing need not wait till the simulation terminates.
In {\em incremental treatment learning},
TAR2 offers feedback during a simulation run into order the guide
the simulator into regions of interest. Before TAR2 can be deployed
in that manner, it must be optimized so that it can run fast enough
to keep up with the simulator. Currently, we are exploring stochastic
methods for that optimization.



%
%Alternatives to TAR2
%include In a {\em sensitivity} analysis, the key factors that most
%influence a model are isolated. Also, recommended settings for
%those key factors are generated. We take care to distinguish
%sensitivity analysis from traditional optimization methods. In our
%experience, the real systems we deal with are so complex that they
%do not always fit into (e.g.) a linear optimization framework.
%Studying  data grown from simulators lets us investigate complex,
%non-linear systems using a variety of data driven distributions.
%These models can capture complex feedback and rework loops which
%are not possible for traditional optimization methods.  Our
%experience is that simulation models can look at processes in
%detail as well as at a high level of abstraction which is where
%the more analytic models must reside. Finally, simulation models
%can capture multiple performance measures not able to be explored
%using these optimization formulations.  This is not to say that
%traditional optimization models are not useful. For certain
%questions, traditional optimization formulations provide the best
%fit for the question that is trying to be answered.   However, for
%many questions, the standard optimization models are not the best
%choice and something like {\em treatment learning} may be more
%useful.
%


% use section* for acknowledgment
%\section*{Acknowledgment}
% optional entry into table of contents (if used)
%\addcontentsline{toc}{section}{Acknowledgment}
%The authors would like to thank...

% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}

% references section
% NOTE: BibTeX documentation can be easily obtained at:
% http://www.ctan.org/tex-archive/biblio/bibtex/contrib/doc/

% can use a bibliography generated by BibTeX as a .bbl file
% standard IEEE bibliography style from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/supported/IEEEtran/bibtex
\bibliographystyle{IEEEtran.bst}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{../../../tex/refs.bib}
\bibliography{refs,miamiRefs,../jim/refs,../tim/refs,../jeremy/refs}
\input{../tim/bio}
\input{../jim/bio}
% if you will not have a photo at all:
%\newpage
\input{../jeremy/bio}
\begin{biographynophoto}{Ying Hu}
is a software designer working in Vancouver, British Vancouver.
Formerly a graduate student at School of Electrical Engineering
at the University of British Columbia,
Ms Hu implemented TAR2.
\end{biographynophoto} 
%\begin{figure*}
%\begin{minipage}{.45\linewidth}
\begin{biography}[{\includegraphics[width=0.8in,clip,keepaspectratio]
{../one/raffo}}]{David Raffo}
is an Associate Professor of Computer Science and
   Business Administration at Portland State University and a Principal
   Consultant at Quantel, Inc.  Raffo completed his Ph.D. at Carnegie
   Mellon University.  His research interests include: Strategic Process
   Design, Financial Analysis of Systems Engineering Decisions, Process
   Simulation, and Value Based Systems Engineering.  Dr. Raffo has over
   forty refereed publications  and
   is co-Editor-in-Chief of the international journal of Software
   Process: Improvement and Practice.  
%He is a Visiting Scientist at the
%   Software Engineering Institute and a Research Member of the
%   International Process Research Consortium (IPRC).  Prior professional
%   experience includes programming as well as managing software
%   development and consulting projects at Arthur D. Little, Inc.  Dr.
%   Raffo teaches courses in Software Process Improvement, Software
%   Process Modeling and Simulation, and Systems Analysis and Design.
\end{biography} 
%\end{minipage}\hfill\begin{minipage}{.45\linewidth}
%\vspace{-55pt}
\begin{biography}[{\includegraphics[width=0.8in,clip,keepaspectratio]
{../one/noon}}]{Siri-on Setamanit} is currently a Consultant at
Quantel, Inc., a Portland, Oregon based firm specializing in Process
Simulation and Modeling tools and services.  Ms. Setamanit's research
interests include: Software Process Modeling and Simulation, Global
Software Development processes, and Supply and Logistics Management.
Ms. Setamanit received a M.S. and Ph.D. (2007) in Systems Science from
Portland State University, an MBA from the University of Oregon,
and a BBA from Chulalongkorn University in Bangkok, Thailand.
\end{biography}
%\end{minipage}
%\end{figure*}
% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
%\begin{thebibliography}{1}
%
%\bibitem{IEEEhowto:kopka}
%H.~Kopka and P.~W. Daly, \emph{A Guide to {\LaTeX}}, 3rd~ed.\hskip 1em plus
%  0.5em minus 0.4em\relax Harlow, England: Addison-Wesley, 1999.
%
%\end{thebibliography}
%
% biography section
% 
% If you have an EPS/PDF photo (graphicx package needed) extra braces are
% needed around the contents of the optional argument to biography to prevent
% the LaTeX parser from getting confused when it sees the complicated
% \includegraphics command within an optional argument. (You could create
% your own custom macro containing the \includegraphics command to make things
% simpler here.)
%\begin{biography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{mshell}}]{Michael Shell}
% where an .eps filename suffix will be assumed under latex, and a .pdf suffix
% will be assumed for pdflatex; or if you just want to reserve a space for
% a photo:
%\vfill

% Can be used to pull up biographies so that the bottom of the last one
% is flush with the other column.
%\enlargethispage{-5in}

% that's all folks
\end{document}


