\documentclass[journal]{IEEEtran}
\usepackage{alltt}
\usepackage{bar}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage{cite}      % Written by Donald Arseneau
                        % V1.6 and later of IEEEtran pre-defines the format
                        % of the cite.sty package \cite{} output to follow
                        % that of IEEE. Loading the cite package will
                        % result in citation numbers being automatically
                        % sorted and properly "ranged". i.e.,
                        % [1], [9], [2], [7], [5], [6]
                        % (without using cite.sty)
                        % will become:
                        % [1], [2], [5]--[7], [9] (using cite.sty)
                        % cite.sty's \cite will automatically add leading
                        % space, if needed. Use cite.sty's no adjust option
                        % (cite.sty V3.8 and later) if you want to turn this
                        % off. cite.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/cite/

\usepackage{graphicx}  % Written by David Carlisle and Sebastian Rahtz
                        % Required if you want graphics, photos, etc.
                        % graphicx.sty is already installed on most LaTeX
                        % systems. The latest version and documentation can
                        % be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/required/graphics/
                        % Another good source of documentation is "Using
                        % Imported Graphics in LaTeX2e" by Keith Reckdahl
                        % which can be found as esplatex.ps and epslatex.pdf
                        % at: http://www.ctan.org/tex-archive/info/

\usepackage{subfigure} % Written by Steven Douglas Cochran
                        % This package makes it easy to put sub figures
                        % in your figures. i.e., "figure 1a and 1b"
                        % Docs are in "Using Imported Graphics in LaTeX2e"
                        % by Keith Reckdahl which also documents the graphicx
                        % package (see above). sub figure.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/subfigure/

\usepackage{url}       % Written by Donald Arseneau
                        % Provides better support for handling and breaking
                        % URLs. url.sty is already installed on most LaTeX
                        % systems. The latest version can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/other/misc/
                        % Read the url.sty source comments for usage information.

%\usepackage{stfloats}  % Written by Sigitas Tolusis
                        % Gives LaTeX2e the ability to do double column
                        % floats at the bottom of the page as well as the top.
                        % (e.g., "\begin{figure*}[!b]" is not normally
                        % possible in LaTeX2e). This is an invasive package
                        % which rewrites many portions of the LaTeX2e output
                        % routines. It may not work with other packages that
                        % modify the LaTeX2e output routine and/or with other
                        % versions of LaTeX. The latest version and
                        % documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/contrib/supported/sttools/
                        % Documentation is contained in the stfloats.sty
                        % comments as well as in the presfull.pdf file.
                        % Do not use the stfloats baseline float ability as
                        % IEEE does not allow \baselineskip to stretch.
                        % Authors submitting work to the IEEE should note
                        % that IEEE rarely uses double column equations and
                        % that authors should try to avoid such use.
                        % Do not be tempted to use the cited.sty or
                        % midfloat.sty package (by the same author) as IEEE
                        % does not format its papers in such ways.

\usepackage{amsmath}   % From the American Mathematical Society
                        % A popular package that provides many helpful commands
                        % for dealing with mathematics. Note that the AMSmath
                        % package sets \interdisplaylinepenalty to 10000 thus
                        % preventing page breaks from occurring within multiline
                        % equations. Use:
%\interdisplaylinepenalty=2500
                        % after loading amsmath to restore such page breaks
                        % as IEEEtran.cls normally does. amsmath.sty is already
                        % installed on most LaTeX systems. The latest version
                        % and documentation can be obtained at:
                        % http://www.ctan.org/tex-archive/macros/latex/required/amslatex/math/



% Other popular packages for formatting tables and equations include:

\usepackage{array}
% Change the following for any hyphenation issues.  These are just examples. (Not actually used yet.)
\hyphenation{op-tical net-works semi-conduc-tor}
\input{../jim/tricks.sty}
\input{../tim/tricks.sty}
\input{../jeremy/tricks.sty}
\begin{document}
%
% paper title
\title{Data Mining for Model-Based Engineering}
%
%
% author names and IEEE memberships
% note positions of commas and non breaking spaces ( ~ ) LaTeX will not break
% a structure at a ~ so this keeps an author's name from being broken across
% two lines.
% use \thanks{} to gain access to the first footnote area
% a separate \thanks must be used for each paragraph as LaTeX2e's \thanks
% was not built to handle multiple paragraphs
\author{James~Kiper,~\IEEEmembership{Member,~IEEE,} \thanks{Dr. Kiper is with the
Computer Science and Systems Analysis Department,
School of Engineering \& Applied Science, Miami University, and can be reached
at \protect\url{kiperjd@muohio.edu}} Tim~Menzies,~\IEEEmembership{Member,~IEEE,}
\thanks{Dr. Menzies is with the Lane Department of Computer
Science, West Virginia University and can be reached at \protect\url{tim@menzies.us}},
Jeremy Greenwald,\thanks{Mr. Greenwald is with
Computer Science, Portland State University and can be reached
at \protect\url{jegreen@cecs.pdx.edu}}, \newline Ying Hu\thanks{Ms. Hu is a software
designer in Vancouver, British Columbia and can be reached at 
\protect\url{huying_ca.@yahoo.com}}, 
David Raffo, \IEEEmembership{Member,~IEEE}\thanks{Dr. Raffo is a Associate Professor in the School of Business
Administration, Portland State University, and can be reached at \protect\url{raffod@sba.pdx.edu}},
and Siri-on Setamanit\thanks{Ms. Setamanit is a graduate student at Portland State University
and can be reached at \protect\url{sirion@pdx.edu}}\thanks{The research described in this paper
was carried out at Miami University, 
West Virginia University and Portland State University under
contracts and sub-contracts with NASA's Software Assurance Research Program.
Reference herein to any specific commercial product, process, or service
by trademark, manufacturer, or otherwise, does not constitute or imply
its endorsement by the United States Government.
}\thanks{Manuscript received January 1, 2007.
Download an earlier draft from
\protect\url{http://menzies.us/pdf/07exploit.pdf}
}}% <-this % stops a space\thanks{Contact information: kiperjd@muohio.edu, timm@cs.pdx.edu, jegreen@cs.pdx.edu .}}
% note the % following the last \IEEEmembership and also the first \thanks - 
% these prevent an unwanted space from occurring between the last author name
% and the end of the author line. i.e., if you had this:
% 
% \author{....lastname \thanks{...} \thanks{...} }
%                     ^------------^------------^----Do not want these spaces!
%
% a space would be appended to the last name and could cause every name on that
% line to be shifted left slightly. This is one of those "LaTeX things". For
% instance, "A\textbf{} \textbf{}B" will typeset as "A B" not "AB". If you want
% "AB" then you have to do: "A\textbf{}\textbf{}B"
% \thanks is no different in this regard, so shield the last } of each \thanks
% that ends a line with a % and do not let a space in before the next \thanks.
% Spaces after \IEEEmembership other than the last one are OK (and needed) as
% you are supposed to have spaces between the names. For what it is worth,
% this is a minor point as most people would not even notice if the said evil
% space somehow managed to creep in.
%
% The paper headers
%\markboth{Journal of \LaTeX\ Class Files,~Vol.~1, No.~11,~November~2002}{Shell \MakeLowercase{\textit{et al.}}: Bare Demo of IEEEtran.cls for Journals}
% The only time the second header will appear is for the odd numbered pages
% after the title page when using the two side option.
% 
% *** Note that you probably will NOT want to include the author's name in ***
% *** the headers of peer review papers.                                   ***

% If you want to put a publisher's ID mark on the page
% (can leave text blank if you just want to see how the
% text height on the first page will be reduced by IEEE)
%\pubid{0000--0000/00\$00.00~\copyright~2002 IEEE}

% use only for invited papers
%\specialpapernotice{(Invited Paper)}

% make the title area
\maketitle


\begin{abstract}
\input{../jim/abstract}
\end{abstract}

\begin{keywords}
model-driven software engineering, contrast set learning, treatment learning
\end{keywords}
% Note that keywords are not normally used for peer review papers.

% For peer review papers, you can put extra information on the cover
% page as needed:
% \begin{center} \bfseries EDICS Category: 3-BBND \end{center}
%
% For peer review papers, inserts a page break and creates the second title.
% Will be ignored for other modes.
\IEEEpeerreviewmaketitle


\input{../jim/jimOne}
\input{../tim/general}
\input{../tim/tar2}
\input{../tim/examples}
%\input{../tim/related}

\noindent
In all the  case studies explored by TAR2,
the same three observations were made:
\bi
\item Treatment learning can find very small treatments,
even for seemingly complex models;
\item
These
treatments can be far smaller that models generated by standard data miners.
\item 
Despite uncertainties or variabilities in the model, TAR2 was able
to find effective treatments that selected for preferred model output
(but the less uncertainty or model variability, the smaller the variance
in TAR2's predicted output for the treated model).
\ei

\section{Related Work}

Elsewhere~\cite{me03l} we have discussed the connection of treatment learning to
soft computing methods such as fuzzy logic, genetic algorithms, and
neural nets. 
Approximate,
heuristically generated models would be adequate to control
models if those models contain clumps and collars.
That is,
the widespread presence of clumps and collars
would explain the success of soft computing. 

Treatment learner shares the same goals, but uses different methods,
to {\em sensitivity analysis}~\cite{saltelli00} that isolates the
key factors that most influence a model
(also, recommended settings
for those key factors are generated).  
A canonical sensitivity analysis method might be to compute
eigenvector of a linear system in order to understand its long-term
temporal behaviour.
However, for models where the collar
and clump assumption holds, a detailed exploration of the nuances
of the model's states may yield little more information than
TAR2's treatments. 

Treatment learning would be preferred to numerical
sensitivity analysis
when the goal is a succinct policy statement that can be quickly
explained to business users. Also, treatment learning might be
a useful pre-processor to sensivitiy analysis in order
to focus the sensitivity analysis on the
 parts of the model that are most crucial.
On the other hand,
TAR2 is not the tool of
choice when seeking intricate optimizations of complex non-linear
systems with massive feedback.
For such optimizations, standard statistical sensitivity
methods might be preferred to treatment learning.

It is insightful to contrast treatment learning with automatic
formal methods.
Treatment learner offers coarse-grained heuristic
controllers for models with a wide range of internal parameter settings.
Automatic formal methods,
on the other hand, rigorously explore the structure of
for small models
that are precisely specified (albeit with some non-determinism).
Hence they are tools to be used for {\em different tasks} in the
life cycle of a model.

Early in the software life cycle, engineers are concerned with
exploring a large space of possible models or possible model
configurations. Model-based tools needed for such {\em option assessment}
(e.g. TAR2) may be very different from model-based tools for {\em
fine-tuning} and verifying a final model such as SPIN~\cite{spinbook}.
Certainly, for mission and safety critical models, it is essential
that such rare but dangerous bugs be located and fixed. However,
while much of the model remains under debate, the rigorous
search of, say, SPIN
can be rendered obsolete by the next revision to the model.




\section{Conclusion}\label{conclusions}

Controlling some models may be much 
simpler than previously thought.
Models are written by people; people have cognitive limits; therefore
models written by people aren't always  complex.
Also, regardless of who writes a model,
the mathmematics of clumps and collars promises that
models
naturally contain structures that
greatly restrict the space
of possible model behaviors. 

It is possible to design data miners to exploit such collars and clumps.
%\begin{wrapfigure}{r}{1.5in}
%\begin{center}
%\includegraphics[width=1.5in]{ai.pdf}
%\end{center}
%\caption{Humans assisted by AI.}\label{fig:cycle}
%\end{wrapfigure}
The case studies in this paper show that
a minimal list of the differences between concepts can
be {\em much smaller} than a detailed description of all aspects of a concept.
For models where TAR2 can generate  succinct summaries,
its algorithm can significantly improved searched-based methods of data mining.
Human designers could avoid wasting time on superfluous details in that
models could be iteratively built by humans, 
then pruned by minimal contrast set learning. 
In this way, 
human-based design intelligence could be augmented in a cost-effective manner via
artificial intelligence.

Treatment learning will fail when models don't contain clumps and collars.
Such models can't be controlled by a small number of key variables-- in which case
TAR2's tiny rules will never be useful.
However, this paper has presented  theoretical and empirical evidence to suggest that
collars and clumps can be expected in many models.
Hence, 
treatment learning should be applicable to much of model-based SE,
particular when large numbers of
configuration possibilities are being discussed.



%
%Alternatives to TAR2
%include In a {\em sensitivity} analysis, the key factors that most
%influence a model are isolated. Also, recommended settings for
%those key factors are generated. We take care to distinguish
%sensitivity analysis from traditional optimization methods. In our
%experience, the real systems we deal with are so complex that they
%do not always fit into (e.g.) a linear optimization framework.
%Studying  data grown from simulators lets us investigate complex,
%non-linear systems using a variety of data driven distributions.
%These models can capture complex feedback and rework loops which
%are not possible for traditional optimization methods.  Our
%experience is that simulation models can look at processes in
%detail as well as at a high level of abstraction which is where
%the more analytic models must reside. Finally, simulation models
%can capture multiple performance measures not able to be explored
%using these optimization formulations.  This is not to say that
%traditional optimization models are not useful. For certain
%questions, traditional optimization formulations provide the best
%fit for the question that is trying to be answered.   However, for
%many questions, the standard optimization models are not the best
%choice and something like {\em treatment learning} may be more
%useful.
%


% use section* for acknowledgment
%\section*{Acknowledgment}
% optional entry into table of contents (if used)
%\addcontentsline{toc}{section}{Acknowledgment}
%The authors would like to thank...

% trigger a \newpage just before the given reference
% number - used to balance the columns on the last page
% adjust value as needed - may need to be readjusted if
% the document is modified later
%\IEEEtriggeratref{8}
% The "triggered" command can be changed if desired:
%\IEEEtriggercmd{\enlargethispage{-5in}}

% references section
% NOTE: BibTeX documentation can be easily obtained at:
% http://www.ctan.org/tex-archive/biblio/bibtex/contrib/doc/

% can use a bibliography generated by BibTeX as a .bbl file
% standard IEEE bibliography style from:
% http://www.ctan.org/tex-archive/macros/latex/contrib/supported/IEEEtran/bibtex
\bibliographystyle{IEEEtran.bst}
% argument is your BibTeX string definitions and bibliography database(s)
%\bibliography{../../../tex/refs.bib}

\bibliography{refs,miamiRefs,../jim/refs,../tim/refs,../jeremy/refs}
\appendix
\section{Appendix}

\fig{cart} and \fig{lsr} show the decision trees learned from
the example of \tion{cmm}.
\section{About the authors}

% if you will not have a photo at all:
\input{../tim/bio}

\input{../jim/bio}
% insert where needed to balance the two columns on the last page
%\newpage

\input{../jeremy/bio}




\begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio]
{../one/hing}}]{Ying Hu}
is a software designer working in Vancouver, British Vancouver.
Formerly a graduate student at School of Electrical Engineering
at the University of British Columbia,
Ms Hu implemented TAR2.
\end{biography} 

\begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio]
{../one/raffo}}]{David Raffo}
is an Associate Professor at Portland State
University with a  a joint appointments in the School Business
Administration 
and in the College of Engineering and Computer Science 
Dr. Raffo's
research interests include: software process design, financial
analysis of software engineering decisions, process simulation, and
value based software engineering. Dr. Raffo is
the co-editor-in-chief of the International Journal of Software Process:
Improvement and Practice. \end{biography} 



\begin{biography}[{\includegraphics[width=1in,clip,keepaspectratio]
{../one/siri}}]{Siri-on Setamanit}
is a graduate student at Portland State University, working on software
process modeling.
\end{biography} 


\begin{figure*}[!t]
\begin{alltt}\scriptsize
policy=NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF <= 0.5 : 
|   policy=NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF <= 0.5 : 
|  |   policy=WNNN,NWNN,FNNW,FNNB,BNNN,NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,
              NNWF,BBBB,FFFF,NNNF,BFFF <= 0.5 : 
|  |  |   _detCap_8 <= 0.431 : 6940 
|  |  |   _detCap_8 >  0.431 : 8180 
|  |   policy=WNNN,NWNN,FNNW,FNNB,BNNN,NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,
              NNWF,BBBB,FFFF,NNNF,BFFF >  0.5 : 
|  |  |   _detCap_8 <= 0.491 : 
|  |  |  |   _inspDur_10 <= 94 : 8780 
|  |  |  |   _inspDur_10 >  94 : 7860 
|  |  |   _detCap_8 >  0.491 : 9320 
|   policy=NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF >  0.5 : 
|  |   policy=BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF <= 0.5 : 9750 
|  |   policy=BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF >  0.5 : 10300 
policy=NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF >  0.5 : 
|   policy=NNWF,BBBB,FFFF,NNNF,BFFF <= 0.5 : 
|  |   _detCap_38 <= 0.418 : 10700 
|  |   _detCap_38 >  0.418 : 11400 
|   policy=NNWF,BBBB,FFFF,NNNF,BFFF >  0.5 : 
|  |   _corErr_50 <= 155 : 11900 
|  |   _corErr_50 >  155 : 
|  |  |   policy=NNNF,BFFF <= 0.5 : 12400 
|  |  |   policy=NNNF,BFFF >  0.5 : 
|  |  |  |   _detCap_38 <= 0.519 : 12500 
|  |  |  |   _detCap_38 >  0.519 : 13600 
\end{alltt}
\caption{A regression tree learned by CART.
In this figure, and \protect\fig{lsr},
policies are denoted as a four-part sequence
describing the inspection policy at each phase.
For example, ``FFFN'' denotes 
using
no inspections for code but
full Fagan inspections for all earlier phases.
}\label{fig:cart}


\end{figure*}

\begin{figure*}[!t]
\begin{alltt}\scriptsize
utility = 7370
 + 1220policy=WNNN,NWNN,FNNW,FNNB,BNNN,NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,
              NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 258policy=FNNB,BNNN,NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW, FNNF,
             NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 640policy=NNWN,NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 505policy=NNNW,WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 - 190policy=WNNW,BNNW,WNNB,BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 206policy=BNNB,WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 - 177policy=WWWW,NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 890policy=NNNB,NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 - 296policy=NNWW,FNNF,NNWB,NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 545policy=NNFN,BNNF,WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 640policy=WNNF,NNWF,BBBB,FFFF,NNNF,BFFF
 + 459policy=NNWF,BBBB,FFFF,NNNF,BFFF - 143policy=BBBB,FFFF,NNNF,BFFF
 + 835policy=NNNF,BFFF - 142_spec_2=n,b + 27_spec_2=b + 3690_detCap_8
 - 135_inspEff_9 - 7320_inspDur_10 - 64400_corErr_11 - 760_detCap_18
 - 134_inspEff_19 - 7310_inspDur_20 - 64300_corErr_21 - 9100_detCap_28
 - 134_inspEff_29 - 7320_inspDur_30 - 64300_corErr_31 + 6170_detCap_38
 - 134_inspEff_39 - 7320_inspDur_40 - 64400_corErr_41
+ 21500_inspEff_48 + 1.17e6_inspDur_49 + 64400_corErr_50
\end{alltt}
\caption{Linear regression.}\label{fig:lsr}
\end{figure*}

% <OR> manually copy in the resultant .bbl file
% set second argument of \begin to the number of references
% (used to reserve space for the reference number labels box)
%\begin{thebibliography}{1}
%
%\bibitem{IEEEhowto:kopka}
%H.~Kopka and P.~W. Daly, \emph{A Guide to {\LaTeX}}, 3rd~ed.\hskip 1em plus
%  0.5em minus 0.4em\relax Harlow, England: Addison-Wesley, 1999.
%
%\end{thebibliography}
%
% biography section
% 
% If you have an EPS/PDF photo (graphicx package needed) extra braces are
% needed around the contents of the optional argument to biography to prevent
% the LaTeX parser from getting confused when it sees the complicated
% \includegraphics command within an optional argument. (You could create
% your own custom macro containing the \includegraphics command to make things
% simpler here.)
%\begin{biography}[{\includegraphics[width=1in,height=1.25in,clip,keepaspectratio]{mshell}}]{Michael Shell}
% where an .eps filename suffix will be assumed under latex, and a .pdf suffix
% will be assumed for pdflatex; or if you just want to reserve a space for
% a photo:

%\vfill

% Can be used to pull up biographies so that the bottom of the last one
% is flush with the other column.
%\enlargethispage{-5in}

% that's all folks
\end{document}


