\newcommand{\registryfunctionstable}[1]{
\begin{table}[h]
\centering
\scriptsize
\begin{tabular}{ll}
Action & Function \\
\hline
%Register & register-#1\index{registry!#1!register-#1}\\
%Register with parameters & register-#1-by-parameters\index{registry!#1!register-#1-by-parameters}\\
Lookup   & lookup-#1\index{registry!#1!lookup-#1}\\
List IDs & all-#1-ids\index{registry!#1!all-#1-ids}\\
List All & all-#1s\index{registry!#1!all-#1s}\\
\end{tabular}
\caption{Registry functions for #1}
\label{tab:registry-functions-#1}
\end{table}}

\section{NOVA-USER}\label{subsec:user-nova-user}
\glossary{name={NOVA-USER},description={Package from which users should interact with \nova.}}
When using \nova, users should start in the \textit{NOVA-USER} package. See \subsec{running-nova}.  
It imports all of the necessary interfaces for \nova.

\section{Registries}\label{subsec:user-registry}\index{registry}
To ease accessing different component instances in \nova{}, registeries are used.
Registries define a handful of functions used for adding, viewing, 
and retrieving elements in a registry.\footnote{For full registry details, see \subsec{programmer-registry}.}
As a standard user, only two functions of a registry will be used:
\begin{description}
\item[all-\textit{REGISTRYNAME}-ids] returns a list of \textit{ids}.
Each \textit{id} identifies a different instance of the \textit{REGISTRYNAME}
that has been registered.  See \code{example:all-search-engine-ids}.
\item[lookup-\textit{REGISTRYNAME}] returns the registered instance of \textit{REGISTRYNAME}
for the specified key.  See \code{example:lookup-search-engine}.
\end{description}

\begin{lstlisting}[language=nova,caption=Example: all-search-engine-ids,label=code:example:all-search-engine-ids]
NOVA-USER> (all-search-engine-ids)
(NOVA.IMPL.SEARCH-ENGINES:SEESAW-D NOVA.IMPL.SEARCH-ENGINES:SEESAW-R
 NOVA.IMPL.SEARCH-ENGINES:SA NOVA.IMPL.SEARCH-ENGINES:KEYS NONE)
NOVA-USER>
\end{lstlisting}

\begin{lstlisting}[language=nova,caption=Example: lookup-search-engine,label=code:example:lookup-search-engine]
NOVA-USER> (lookup-search-engine 'nova.impl.search-engines:sa)
#S(SEARCH-ENGINE-DESCRIPTOR
   :NAME ``sa''
   :FUNCTION #<CLOSURE NOVA.IMPL.SEARCH-ENGINES:SA>)
T
NOVA-USER>
\end{lstlisting}

\section{Attributes}\label{subsec:user-attributes}\index{attribute|textbf}
\glossary{name={Attribute},description={A piece of data used by a model to make estimations.}}
Each model used in \nova{} defines attributes\index{attribute}.
Each attribute\index{attribute} has a default range of values associated with it.
When attributes are used with-in \nova{}, values are randomly generated based upon the specified default range 
and any additinal constraints\index{constraint} \subsec{user-constraints}.

\subsection{Constraints}\label{subsec:user-constraints}\index{constraints|textbf}
\glossary{name={Constraint},description={Limitation imposed on to possible values of an attribute.}}
Depending on the scenario\index{scenario}, values/ranges of certain attributes
may need to be modified from their default value/ranges.
This modification is handled through constraints\index{constraints}.
Constraints\index{constraints} are implemented as a list of lists.
The inner list has the attribute\index{attribute} as the first element.
Any elements after the first specifies the value or range of values
the attribute is constrained to be.

\section{Scoring Methods}\label{subsec:user-scoring-methods}\index{scoring method!user|textbf}
Software models are used for their ability to estimate certain outcomes: time to completion, cost, bugs, costs, \ldots.
In \nova{}, these estimations are represented as \textit{scoring methods}\index{scoring method}.  
\textit{Scoring methods} produce scores based on attributes values of a model.  
Generally, models will have multiple \textit{scoring-methods}

\registryfunctionstable{scoring-method}


\section{\nova~Learning}\label{subsec:nova-learning}\index{NOVA!learning@\nova~Learning|textbf}
\glossary{name={\nova~Learning},description={TODO}}

\nova~Learning makes suggestions on how to improve a project team based upon scenario, search-engine, evaluation-method, find-best-state-fn.
\nova~Learning generates an initial state based on the scenario.  Next the search engine is run with the initial state and evaluation-method.
A result is then created using the path from the search engine and find-best-state-fn.

\subsection{Modifiable Attributes}\label{subsec:user-modifiable-attributes}\index{modifiable attributes|textbf}
\glossary{name={Modifiable Attribute},description={An attribute whose values can be constrained during learning.}}
Models usually define many attributes.  However, not all of the attributes\index{attribute} are modifiable.
Modifiable attributes\index{modifiable-attributes} are simply a list of attributes that are allowed to be ``modified''.  ``Modified''
is simply allowing the value/ranges of a particular attribute to be constrained\index{constraints} \subsec{user-constraints}.


\subsection{Scenarios}\label{subsec:user-scenario}\index{scenario}
A scenario represents a project team over an undefined period of time. 
It combines of a case study\index{case study} and a policy\index{policy}.

\subsubsection{Case Studies}\label{subsec:user-case-studies}\index{case study}
\glossary{name={Case Study},description={A named list of constraints used to represent a project team.}}
A case study is a named list of constraints\index{constraint} used to represent a project team's capability.

\paragraph{Example}
Attribute ``mexp'' represents the experience of a project team in terms of average years of experience (1 - 30).
Company Y is inexperienced and different permuations of people result in a range of ``mexp'' being 1 to 5.
Specifing this in a constraint for \nova~Learning will result in more accurate suggestions.

\registryfunctionstable{case-study}

\subsubsection{Policies}\label{subsec:user-policies}\index{policy|textbf}
\glossary{name={Policy},description={A named list of modifiable attributes used to represent buisness practices or practicality.}}
A policy is list of modifiable attributes used to represent buisness practices or practicality.

\paragraph{Example}
As in the example for case study, Attribute ``mexp'' represents the experience of a project team.
If \nova~Learning is being used to generate suggestions for improving a single project, 
it doesn't make sense for \nova~Learning to waste time suggesting to improve experience, 
because over the course of a single project, it isn't practical.
To prevent this, the policy wouldn't list ``mexp'' as a modifiable attribute.

\registryfunctionstable{policy}

\subsection{Search Engines}\label{subsec:user-search-engines}\index{search engines!user|textbf}
Search engines\index{search engine} are used by \textit{nova-learn}~\code{nova-learn}\index{NOVA!nova-learn} to create paths.
A path is a list of states traversed by the search engine with the goal of improving the \textit{evaluation score}.
\glossary{name={Path},description={A list of states, where each state was created based on the ``best'' decision available by a search engine.}}

\registryfunctionstable{search-engine}

\subsection{Evaluation Methods}\label{subsec:user-evaluation-methods}\index{evaluation method!user|textbf}
\glossary{name={Evaluation Method},description={A special function used to guide search engines.}}
Evaluation methods\index{evaluation method} are similar to scoring methods\index{scoring method} \subsec{user-scoring-methods}.
Evaluation methods for a model are usually comprised of some or all of the scoring methods of that model.
An evaluation method is used by search engines\index{search engine} \subsec{user-search-engines} to compare states.

\registryfunctionstable{evaluation-method}

\subsection{find-best-state-fn}\label{subsec:user-find-best-state-fn}\index{find-best-state-fn}
find-best-state-fn is used to find the best state in the path.  
In general, the states in a path continue until a ``best state'' and the get worse again.  
How to find the ``best state'' within a path is delegated to find-best-state-fn.

\subsection{nova-learn}\label{subsec:user-nova-learn}\index{NOVA!nova-learn}
\nova~Learning's interface function is \textit{nova-learn}~\code{nova-learn}\index{nova-learn|textbf}.
It accepts three required parameters:
\textit{evaluation-method-descriptor}, \textit{search-engine-descriptor}, \textit{find-best-state-fn}
and two keyed parameters:
\textit{constraints}, \textit{modifiable-attributes}.

\textit{evaluation-method-descriptor} specifies the
evalution method~\subsec{user-evaluation-methods} to be used during the searching.
Similiarly, \textit{search-engine-descriptor} specifies the
search engine~\subsec{user-search-engines} used for searching.
Searching generates a path that is turned into results by
\textit{nova-learn}\index{nova-learn}.  The best-state found in the path is determined by
appling \textit{find-best-state-fn} to the path.

Any constraints\index{constraints} specifed in \textit{constraints} will be applied
before nova-learn\index{nova-learn} starts searching.
Specifing attributes\index{attribute} via \textit{modifiable-attributes} will limit
the attributes that are modified during searching in nova-learn\index{nova-learn}.

\lstinputlisting[language=lisp,caption=nova-learn,label=code:nova-learn,firstline=19,lastline=40]{code/nova/apps/learning/learning.lisp}


\section{\nova~Simulation}\label{subsec:nova-simulating}\index{NOVA!simulation@\nova~Simulation|textbf}
\glossary{name={\nova~Simulation},description={TODO}}
\glossary{name={Project},description={A list of model attributes with values}}

\nova~Simulate is used to generate random ``projects''.  
A ``project'' is a list of model attributes with values.
The values are randomly generated based on the specified constraints.
\nova~Simulate will also score each project if a list of scoring functions is specified.

\subsection{nova-simulate}\label{subsec:nova-simulate}\index{NOVA!nova-simulate|textbf}
\nova~Simulation's interface function is \textit{nova-simulate}~\code{nova-simulate}\index{nova-simulate|textbf}.
It accepts three keyed parameters: \textit{runs}, \textit{scoring-methods}, \textit{constraints}.
\textit{runs} specifies the number of projects to simulate.  \textit{scoring-methods} is a list of scoring-methods~\subsec{user-scoring-methods}.
Each simulated project will be scored with each scoring-method specified in \textit{score-methods}.  The scores calculated will be added to
the project.  \textit{constraints} allows you to specify constraints~\subsec{user-constraints}\index{constraints} to be applied before projects are simulated.

\lstinputlisting[language=lisp,caption=nova-simulate,label=code:nova-simulate,firstline=19,lastline=33]{code/nova/apps/simulation/simulation.lisp}

\section{Experiments}\label{subsec:user-experiments}\index{experiments}
Experiments are comprised of scenarios\index{experiments!scenario} \subsec{user-experiment-scenarios},
treatments\index{experiments!treatment} \subsec{user-experiment-treatments},
and scoring-methods\index{scoring-method} \subsec{user-scoring-methods}.

Each object type, except Reports,  mentioned in this section (including experiments) has a registery.  They are defined
using \text{defregistry}\index{macros!defregistry}.

\registryfunctionstable{experiments}

\subsection{Scenarios}\label{subsec:user-experiment-scenarios}\index{experiments!scenario}
\glossary{name={Scenario},description={Combination of attribute constraints and which attributes may be modified to simulate an actual working scenario. }}
Scenarios are concepts used to describe a project scenario. 

For a given scenario, certain model attributes may have constraints
while other attributes aren't modifiable.
Hence, scenarios combine a case-study\index{case-study} \subsec{user-case-studies},
and a policy\index{policy} \subsec{user-policies}.  Scenarios also require a name to be used
only for referring to the scenario.

\registryfunctionstable{experiment-scenario}

\subsection{Treatments}\label{subsec:user-experiment-treatments}\index{experiments!treatment}
\glossary{name={Treatment},description={A action to improve scores for a scenario.  The default implemention combines an evaluation method, search-engine, and find-best-state-method.}}
Treatments are what is applied to scenarios to produce learned improvments.
At its core, treatments uses \textit{nova-learn}~\code{nova-learn}\index{NOVA!nova-learn}.
The slots on treatment are used to as inputs for \textit{nova-learn}.
Treatments are comprised of a name,
an evaluation-method\index{evaluation-method} \subsec{user-evaluation-methods},
a search-engine\index{search-engine} \subsec{user-search-engines},
and a find-best-state-method\index{find-best-state-method} \subsec{user-find-best-state-methods}.
\footnote{These are only the standard slots.  See \subsec{programmer-experiment-treatments} for further details.}

\registryfunctionstable{experiment-treatment}


\subsection{Reports}\label{subsec:user-experiment-reports}\index{experiments!report}
Reports are generated using report-generators.\index{report-generator}
Report generators need varying parameters.  This made registries for them
not practical.  This is different from the other objects spoken about in this chapter.
Therefore there is no register for report-generators.  To obtain a report generator,
you will need to use different \textit{generate-*-report-generator} functions.

Currently, \nova support two types of report-generators: \textit{experiment-results-report-generator} \subsec{results-report-generator}
and \textit{experiment-data-report-generator} \subsec{data-report-generator}.  Each type used different information when generating
a report.  \textit{experiment-results-report-generator} uses the \textit{results} returned from
\textit{nova-learn}\index{nova-learn} \subsec{nova-learning}, while \textit{experiment-data-report-generator} uses the
scores returned from \textit{nova-simulate}\index{nova-simulate} \subsec{nova-simulating}.


\subsubsection{results-report-generator}\label{subsec:results-report-generator}

\textit{results-report-generators} are passed results from \textit{nova-learn}\index{nova-learn} \subsec{nova-learning}
when they are called.  Only the data in results and any initializing parameters to the generator can
be used to generate the report.

\paragraph{feature-consistency-results-report-generator}

The report generated by \textit{feature-consistency-results-report-generator} is a table of every
\text{feature}\index{feature} that occured during the experiment.  A \textit{feature}\index{feature|textbf}
is an \textit{attribute}\index{attribute}. Along with each \textit{feature} is count and percent.
There is a new table for each \textit{treatment}\index{treatment} and tables are grouped by \textit{scenario}\index{scenario}.

\begin{lstlisting}[language=nova,caption=Example: generate-feature-consistency-results-report-generator,label=code:example:generate-feature-consistency-results-report-generator]
NOVA-USER> (generate-feature-consistency-results-report-generator)
#S(NOVA.EXPERIMENTS::EXPERIMENT-RESULTS-REPORT-GENERATOR
   :REPORT-NAME ``feature-consistency''
   :FN #<FUNCTION (LAMBDA #) {BB23E5D}>)
NOVA-USER>
\end{lstlisting}

\input{samples/feature-consistency-results-report-generator-report}

\paragraph{range-consistency-results-report-generator}

The report generated by \textit{range-consistency-results-report-generator} is a table of every
\text{range}\index{range} that occured during the experiment.  A \textit{range}\index{range|textbf}
is an \textit{attribute}\index{attribute} with value[s]. Along with each \textit{range} is count and percent.
There is a new table for each \textit{treatment}\index{treatment} and tables are grouped by \textit{scenario}\index{scenario}.

\begin{lstlisting}[language=nova,caption=Example: generate-range-consistency-results-report-generator,label=code:example:generate-range-consistency-results-report-generator]
NOVA-USER> (generate-range-consistency-results-report-generator)
#S(NOVA.EXPERIMENTS::EXPERIMENT-RESULTS-REPORT-GENERATOR
   :REPORT-NAME ``range-consistency''
   :FN #<FUNCTION (LAMBDA #) {AFBB68D}>)
NOVA-USER>
\end{lstlisting}

\input{samples/range-consistency-results-report-generator-report}


\subsubsection{data-report-generator}\label{subsec:data-report-generator}

\textit{data-report-generators} are passed a list of \textit{experiment-data}\index{experiment-data}
from \textit{nova-simulate}\index{nova-simulate} \subsec{nova-simulating}
when they are called.  Each \textit{experiment-data}\index{experiment-data} instance
came from a different \textit{scoring-method}\index{scoring-method} defined in the experiment.
Only the list of experiment-data and any initializing parameters to the generator can
be used to generate the report.

\paragraph{mann-whitney-experiment-data-report-generator}

Reports generated by \textit{mann-whitney-experiment-data-report-generator} are first grouped
by \textit{scoring-method}\index{scoring-method}, then \textit{scenario}\index{scenario}.
Each row shows a \textit{treatment}\index{treatment} with its win-loss record versus the
other \textit{treatments}\index{treatment}.  Following the win-loss record is the quartile
chart and the raw values of the quartiles.

\begin{lstlisting}[language=nova,caption=Example: generate-mann-whitney-experiment-data-report-generator,label=code:example:generate-mann-whitney-experiment-data-report-generator]
NOVA-USER> (generate-mann-whitney-experiment-data-report-generator 95 :combine-fn #'mean)
#S(EXPERIMENT-DATA-REPORT-GENERATOR
   :REPORT-NAME ``mann-whitney-95''
   :FN #<CLOSURE (LAMBDA #) {B0E9395}>)
NOVA-USER>
\end{lstlisting}

\input{samples/mann-whitney-experiment-data-report-generator-report}

\subsection{Running Experiments}\label{subsec:run-experiment}\index{experiments!run}

\textit{run-experiment}\index{run-experiment|textbf} is used to run an experiment.
It needs two required parameters (\textit{experiment}\index{experiment} and \textit{report-generators}\index{report-generator})
and two keyed parameters (\textit{repeats} and \textit{iterations}).  \textit{experiment} is the experiment to be run and
\textit{report-generators} is a list of report generators to be applied to the results of the experiment.
\textit{Repeats} is how many times a search is done (number of results in result-report) 
and \textit{iterations} is how many projects are simulated (number of data lists in data-report).

%TODO talk about keyed parameters used to augment report names and locations

\begin{lstlisting}[language=nova,caption=Example: run-experiment,label=code:example:run-experiment]
NOVA-USER> (run-experiment (lookup-experiment 'default) (list (generate-mann-whitney-experiment-data-report-generator 95)))
default
CompanyZ-all-energy-sa-minscore..........
CompanyZ-all-energy-sa-minscore-by-ttest..........
CompanyZ-all-none..........
CompanyY-all-energy-sa-minscore..........
CompanyY-all-energy-sa-minscore-by-ttest..........
CompanyY-all-none..........
CompanyX-all-energy-sa-minscore..........
CompanyX-all-energy-sa-minscore-by-ttest..........
CompanyX-all-none..........
default-ALL-energy-sa-minscore..........
default-ALL-energy-sa-minscore-by-ttest..........
default-ALL-none..........
...(Report Removed for Brevity)...
NOVA-USER>
\end{lstlisting}\index{run-experiment}
