\documentclass[]{beamer}
% Class options include: notes, notesonly, handout, trans,
%                        hidesubsections, shadesubsections,
%                        inrow, blue, red, grey, brown

% Theme for beamer presentation.
\usepackage{Warsaw} 
\usepackage{wrapfig}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newcounter{over}
\newcounter{max}
\newcommand{\fred}[2]{\setcounter{over}{#2}\addtocounter{over}{#1}}
\newcommand{\boxplot}[5]{%
\scalebox{0.5}{\textcolor{white}{\setcounter{max}{#4}\addtocounter{max}{#5}\fred{#4}{-#2}}%
\begin{picture}(100,10)%
\put(0,0){\line(0,1){8}}%
\put(100,0){\line(0,1){8}}%
%\put(#1,4){\circle*{1}}%
%\put(\themax,4){\circle*{1}}%
\put(#2,4){\line(1,0){\theover}}%
\put(#3,4){\circle*{8}}%
%\put(#4,4){\line(1,0){#5}}%
\put(50,0){\line(0,1){8}}%
\end{picture}}
} 

\setcounter{tocdepth}{1}

\AtBeginSection[]
{
   \begin{frame}
       \frametitle{Outline}
       \tableofcontents[currentsection]
   \end{frame}
}

% Other themes include: beamerthemebars, beamerthemelined, 
%                       beamerthemetree, beamerthemetreebars  

\title{Ourmine: An Open Source Data Mining Toolkit}    % Enter your title between curly braces
\author{Adam Nelson}           % Enter your name between curly braces
\institute{Masters Defense\newline
Committee: Tim Menzies, Ph.D; Tim McGraw, Ph.D; Frances VanScoy, Ph.D\newline
West Virginia University, LCSEE}      % Enter your institute name between curly braces
\date{\today}                    % Enter the date or \today between curly braces
%\includegraphics[width=1in]{logo}
\begin{document}

% Creates title page of slide show using above information
\begin{frame}
  \titlepage
\end{frame}
                           % typeset with the notes or notesonly class options

% Creates table of contents slide incorporating
% all \section and \subsection commands

%\begin{frame}
%  \tableofcontents
%\end{frame}


%%%%%%%%%%%%% PROBLEM SLIDES %%%%%%%%%%%%%%%%%

\section{Background}

\subsection{Data Sharing}

\begin{frame}
	\frametitle{Data Sharing}
		
		\begin{center}
			\includegraphics[width=2in]{promiselogo.png}
		\end{center}
		
		Since 2006, the PROMISE~\cite{promise-repo} community has addressed data sharing. This is important because
						
		\bi
			\item researchers can use standardized data
			\item the data is freely available
			\item experimentation is encouraged
		\ei			 		  	
\end{frame}

\subsection{Experiment Sharing}

\begin{frame}
	\frametitle{Experiment Sharing}
	
		But in 2009, Gay et al.~\cite{gay09} explored a step beyond this. After $data$ 
		sharing comes $experiment$ sharing.\newline 
		
		Experiment sharing 
		
		\bi
			\item allows researchers to confirm prior results
			\item lets authors publish entire experiments alongside results
			\item increases understandability of experiments
			\item gives way to modifications of experiments
		\ei
			 		  	
\end{frame}


\subsection{How?}

\begin{frame}
	\frametitle{How?}
							
	Repeatable experiments in publications require a textual environment capable of not only running data mining algorithms, but also performing analyses and report generation. \newline
	
	To demonstrate Ourmine for this use:
	\begin{center}
	%\begin{block}{Note:}	
	%	Completeness is mandatory.
	%\end{block}
		{\tiny
		\tableofcontents}
		\end{center}
							  	
\end{frame}

%%%%%%%%%%%%%% END PROBLEM SLIDES %%%%%%%%%%%%%%%%

%%%%%%%%%%%%% RELATED WORK SLIDES %%%%%%%%%%%%%%%%%

\section{Related Work}

\subsection{KNIME}

\begin{frame}
	\frametitle{KNIME}
	
		\begin{center}
			\includegraphics[height=1.0in]{knime.png}
		\end{center}
	
		The KNIME~\cite{knimetool}(Konstanz Information Miner) is a visual-based modular data 
		exploration environment. It incorporates
		
		\bi
			\item over 100 processing nodes for various tasks (data preprocessing, mining and analysis)
			\item personalized workflows via the {\em workflow editor}
			\item editable nodes within each workflow		
		\ei
\end{frame}

\subsection{RapidMiner}

\begin{frame}
	\frametitle{RapidMiner}
		RapidMiner~\cite{rapidminer-tool} is a popular data mining system available worldwide. 
		
		\bi
			\item Provides a graphical interface as well as a custom scripting environment
			\item Uses workflows to build streams of execution
			\item Allows data to be imported from databases
		\ei
		
		\begin{block}{Important!}
			Only commercially available editions allow integration into closed-source software
		\end{block}
\end{frame}

\subsection{Weka}

\begin{frame}
	\frametitle{Weka}
		
		Weka~\cite{weka-tool}(the Waikato Environment for Knowledge Analysis) is an extremely popular, 
		open source toolset. 
		
		\bi
			\item Written in Java. Any modifications to source requires knowledge of Java and rebuilding.
			\item Offers:
				\bi
					\item {\em Knowledge Flow}: construction of experiments through interconnected nodes
					\item {\em Explorer}: allows quick exection of data mining algorithms, data visualization, etc.
					\item {\em Experimenter}: builds experiments using a standard GUI (drop-down boxes, text fields, etc.)
					\item {\em CLI}: Weka's command-line interface. Gives standard interaction but through text commands.
				\ei	
		\ei
\end{frame}

\subsection{Others}

\begin{frame}
	\frametitle{Others}

	Other important open source toolkits include:
	
	\bi
		\item Orange~\cite{orange-tool} - data mining through visual programming, or Python scripting
		\item ADaM~\cite{adamtool}(Algorithm Development and Mining System) - hundreds of components included as executables and Python modules
		\item Gnome Data Mining Tools~\cite{gnome-data-mine} - requires Gnome and Python. Operates using command-driven GUIs.
		\item Rattle~\cite{rattle-tool} (the R Analytical Tool To Learn Easily) - attempts to teach "R" through the use of its data mining  GUIs
	\ei	
	
\end{frame}

\subsection{Textual vs. Visual Appraoches}

\begin{frame}
	\frametitle{Textual vs. Visual Appraoches}
	
	According to Menzies~\cite{me00v}, typical VPLs seen in many standard toolkits are 
	
	\bi
		\item adequate for beginning operations
		\item ...but {\em discourage} further experimentation because
		
		\bi
			\item developers spend too much time supporting the environment - i.e., never actually data mining
		\ei
	\ei
	
	Rithoff et al.~\cite{ritthoff01} found that 
	
	\bi
		\item Real-world applications are more intricate than running one algorithm
		\item These applications require intricate {\em combinations} of data miners, preprocessors, report generators	
		\item Widely used tools (e.g. WEKA) do not support rapid generations of these combinations
	\ei 
		  
\end{frame}

%%%%%%%%%%%%% END RELATED WORK SLIDES %%%%%%%%%%%%%%%%%



%%%%%%%%%%%%% GOAL SLIDES %%%%%%%%%%%%%%%%%

\section{The Goal}

\subsection{The Goal}

\begin{frame}
	\frametitle{The Goal}
	
		\begin{center}
			\includegraphics[width=1.5in]{target.jpg}
		\end{center}
	
	
To develop a data mining toolkit that promotes experiment sharing {\em in addition} to
				
		\bi
			\item modularity
			\item self-containment
			\item brevity of code
			\item availability at no cost
		\ei
		  	
\end{frame}

\subsection{Modularity}

\begin{frame}
	\frametitle{Modularity}
		\begin{center}
			\includegraphics[width=2.5in]{modules.pdf}
		\end{center}				
		A modular environment makes extending and modifying code easy.												  	
\end{frame}

\subsection{Self-containment}

\begin{frame}
	\frametitle{Self-containment}
		
		Required modules should be included with the environment upon download:
		
		\bi
			\item data sets
			\item learners
			\item experiment utilities (e.g., cross-validation, evaluation)
		\ei
													  	
\end{frame}

\subsection{Brevity of Code}

\begin{frame}
	\frametitle{Brevity of Code}
		
		Publishable code should be short and sweet; large Java or C++ files are 
		much more difficult to inspect and learn from.\newline
		
			\begin{block}{Example}
				\begin{center}
					\includegraphics[height=1.5in]{brevitycode.png}
				\end{center}
			\end{block}

\end{frame}

\subsection{Availability at No Cost}

\begin{frame}
	\frametitle{Availability at No Cost}
		
		Open source environments 
		
		\bi
			\item advocate modification to the source code for customization/improvements
			\item reduces "risk factor" of trying the new software unlike other tools (e.g. Matlab)
			\item beneficial changes can be shared among all users of the environment
		\ei
															  	
\end{frame}


%%%%%%%%%%%%% END GOAL SLIDES %%%%%%%%%%%%%%%%%



%%%%%%%%%%%%% OUTLINE SLIDES %%%%%%%%%%%%%%%%%

%\section{Outline}

%\begin{frame}
%	\frametitle{Outline}
%		Outline
%\end{frame}

%%%%%%%%%%%%% END OUTLINE SLIDES %%%%%%%%%%%%%%%%%



%%%%%%%%%%%%% OURMINE SLIDES %%%%%%%%%%%%%%%%%

\section{Ourmine}

\subsection{What is Ourmine?}

\begin{frame}
	\frametitle{What is Ourmine?}
		  		
	Ourmine is a data mining scripting environment developed at WVU. 
	
	\bi
		\item Designed to assist graduate data mining students
		\item In 2009 was mostly rebuilt to be {\em modular}
		\item Has tools written in BASH, GAWK, JAVA, PERL
		\item Uses shell scripting as the "glue" between command-line APIs
		\item Comes with standard data sets, machine learning algorithms, evaluation 
		functions, etc.
		\item Available for free from a publicly available repository
	\ei
	
	  
\end{frame}

\subsection{Why a Scripting Environment?}

\begin{frame}
	\frametitle{Why a Scripting Environment?}
	
	Shell scripting has many advantages:
	
	\bi
		\item allows publishable experiments to be constructed
		\item requires limited resources
		\item executes on any UNIX-based OS\newline
				
		\begin{block}{Note}
			Tool-specific operations can be limiting.
		\end{block}
		
	\ei	
	  
\end{frame}


%\subsection{Building a Simple Experiment}

%\begin{frame}
%	\frametitle{Building a Simple Experiment}
		  		
%  	Building a Simple Experiment
  
%\end{frame}

%\subsection{Evaluating Experimental Results}

%\begin{frame}
%	\frametitle{Evaluating Experimental Results}
		  		
%  	Evaluating Experimental Results
  
%\end{frame}

\subsection{Important Functions}

\begin{frame}
	\frametitle{Important Functions}
	
	\bi	  		
		\item \url{MakeTrainAndTest} - builds training and testing sets separately
		\item \url{abcd} - evaluates classification performance ($accuracy$,$pd$,$pf$,$precision$,$balance$)
		\item \url{winLossTie} - allows statisical rankings of measures
		\item \url{quartile} - provides ASCII/numeric representations of quartile charts
		\item \url{show} - prints function code
		\item \url{funs} - lists available functions
		\item \url{gotwant} - extracts predicted and actual classes
	\ei		
		  		  
\end{frame}

\subsection{Installing and Running Ourmine}

\begin{frame}
	\frametitle{Installing and Running Ourmine}
		  		
	Ourmine is an open source toolkit licensed under GPL 3.0. It can
	be downloaded and installed from \url{http://code.google.com/p/ourmine}. \newline
	
	Once there, follow the instructions to get started using Ourmine.
  
\end{frame}

%%%%%%%%%%%%% END OURMINE SLIDES %%%%%%%%%%%%%%%%%


%%%%%%%%%%%%% EXPERIMENT SLIDES %%%%%%%%%%%%%%%%%

\section{Experiments Using Ourmine}

\begin{frame}
	\frametitle{Experiments Using Ourmine}
	
	To prove Ourmine as a tool usable in both industry and research, experiments 
	were conducted using it:
	
	\bi
		\item Commissioning a Learner
		\item Cross-Company Defect Prediction using Relevancy Filtering 
		%\item Predicting Student Retention
		\item Component vs. Whole-based Defect Prediction
		\item Analyzing the Scalability of Clustering Text Documents
	\ei
		  		  
\end{frame}

\subsection{Commissioning a Learner}
	

\begin{frame}
	\frametitle{Commissioning a Learner}
		  		
	
	In a recent journal paper under review~\cite{ourmine-10}, reviewers requested proof of Ourmine's 
	use in industry. \newline
	
	
	\begin{block}{Problem}
	
		Many factors can lead to better results in software defect prediction:
		
		\bi
			\item Learners?
			\item Discretize the data?
			\item Data size?
		\ei
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Commissioning a Learner}
	
	\begin{block}{Problem}	
		Choosing these is important
		
		\bi			
			\item Some treatments perform better on certain data sets
			\item As software is released in stages, it's important to know how {\em early} learners can be applied 
		\ei
		
	\end{block}
	
	\begin{block}{Purpose}
		To demonstrate, for practicioners, how learners/discretized data/data sizes can be selected for use at a local site.
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Commissioning a Learner}
	
	\begin{block}{Setup I}
		Using seven PROMISE defect data sets (CM1, KC1, KC2, KC3, MC2, MW1, PC1), 
		determine the best combinations of learners/discretizers to use.\newline
		
		Evaluate using $pd$ and $pf$ in a 10X10-way cross-validation. 
		
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Results 1a}

	
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pd}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Treatment& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&J48 + discretization& 20& 66& 99&  \boxplot{0}{20}{66}{99}{7}  \\
	1&ADTree + discretization& 25& 66& 97&  \boxplot{0}{25}{66}{97}{7}  \\
	2&NB + discretization& 63& 73& 82&  \boxplot{0}{63}{73}{82}{7}   \\
	2&One-R + discretization& 13& 56& 99&  \boxplot{0}{13}{56}{99}{7}  \\
	3&J48& 29& 82& 96& \boxplot{0}{29}{82}{96}{7} \\
	3&NB& 40& 79& 91& \boxplot{0}{40}{79}{91}{7}  \\
	4&ADTree& 21& 83& 97& \boxplot{0}{21}{83}{97}{7} \\
	4&One-R& 17& 83& 97& \boxplot{0}{17}{83}{97}{7} \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}
	\label{fig:pds2}
	\end{figure}
	
	
\end{frame}

\begin{frame}
	\frametitle{Results 1b}

	
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pf}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Treatment& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&One-R + discretization& 0& 10& 86& \boxplot{0}{0}{10}{86}{7}  \\
	1&ADTree + discretization& 2& 11& 75& \boxplot{0}{2}{11}{75}{7} \\
	1&J48 + discretization& 1& 11& 80& \boxplot{0}{1}{11}{80}{7} \\
	2&NB& 9& 19& 60&  \boxplot{0}{9}{19}{60}{7}  \\
	2&J48& 4& 33& 72& \boxplot{0}{4}{33}{72}{7} \\
	3&NB + discretization& 12& 25& 33&  \boxplot{0}{12}{25}{33}{7}   \\
	3&ADTree& 3& 37& 79&  \boxplot{0}{3}{37}{79}{7}  \\
	4&One-R& 3& 44& 83&  \boxplot{0}{3}{44}{83}{7}  \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}	
	\label{fig:pfs2}
	\end{figure}	
	
\end{frame}

\begin{frame}
	\frametitle{Commissioning a Learner}
	
	\begin{block}{Setup II}

		Using the same data sets, find the minimal number of training examples required.\newline
		
		Using the winning method above, incrementally select instances from $N$ = 100, 200 
		to 1000. $Train$ = 90\% * $N$ and $Test$ = 100, both selected at random.
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Results 2a}

	
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pd}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Training Size& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&600& 32& 88& 99&  \boxplot{0}{32}{88}{99}{7}  \\
	1&700& 30& 86& 98&  \boxplot{0}{30}{86}{98}{7}   \\
	1&900& 32& 86& 98&  \boxplot{0}{32}{86}{98}{7}  \\
	1&800& 31& 85& 98&  \boxplot{0}{31}{85}{98}{7}  \\
	1&500& 30& 84& 99& \boxplot{0}{30}{84}{99}{7} \\
	1&1000& 30& 84& 99& \boxplot{0}{30}{84}{99}{7}  \\
	2&300& 31& 85& 98& \boxplot{0}{31}{85}{98}{7} \\ 
	2&400& 29& 84& 99& \boxplot{0}{29}{84}{99}{7} \\
	3&200& 29& 83& 98& \boxplot{0}{29}{83}{98}{7} \\
	3&100& 29& 77& 96& \boxplot{0}{29}{77}{96}{7} \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}	
	\label{fig:pds3}
	\end{figure}	
	
\end{frame}

\begin{frame}
	\frametitle{Results 2b}

	
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pf}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Training Size& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&600& 1& 12& 68&  \boxplot{0}{1}{12}{68}{7}  \\
	1&700& 1& 14& 70&  \boxplot{0}{1}{14}{70}{7}  \\
	1&900& 1& 14& 68& \boxplot{0}{1}{14}{68}{7}  \\
	1&1000& 1& 14& 69&  \boxplot{0}{1}{14}{69}{7}  \\
	1&500& 1& 15& 69&  \boxplot{0}{1}{15}{69}{7}   \\
	1&800& 1& 15& 69& \boxplot{0}{1}{15}{69}{7} \\
	1&300& 2& 15& 69& \boxplot{0}{2}{15}{69}{7} \\ 
	1&400& 1& 15& 71& \boxplot{0}{1}{15}{71}{7} \\
	2&200& 2& 17& 71& \boxplot{0}{2}{17}{71}{7} \\
	3&100& 4& 23& 71& \boxplot{0}{4}{23}{71}{7} \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}
\label{fig:pfs3}
\end{figure}
	
\end{frame}

\begin{frame}
	\frametitle{Conclusion}
	
	The best method is ADTree + discretization.
	
	\bi
		\item Given by the best Mann-Whitney ranking and medians for $pd$ and $pf$.		
	\ei
	
	The smallest/best size of data to train on is:
	
	\bi
		\item 600 instances 
		\item However, 300 is nearly as good
		
		\bi
			\item Loss in $pd$ ranking of 1	and 3\% decrease in medians over 600 instances
			\item $Pf$ ranking is {\em identical}, and medians increased by another 3\%.
		\ei
	\ei 
	
\end{frame}


\subsection{Cross-Company Defect Prediction using Relevancy Filtering}

\begin{frame}

	\frametitle{Cross-Company Defect Prediction using Relevancy Filtering}
	
	When companies want to determine the quality of their software, they can use
	
	\bi
		\item Within Company data (WC)
		\item Cross Company data (CC)
	\ei
			
	\begin{block}{Problem}
	
		Adequate local data is expensive to acquire and may be impossible for newer companies. Instead, freely available, 
		public defect data can be used. 
						
	\end{block}
	
\end{frame}

\begin{frame}

	\frametitle{Cross-Company Defect Prediction using Relevancy Filtering}
	
	In 2008, Turhan et al.~\cite{turhan08} determined that by using {\em relevancy filtering} 
	on cross-company training data, results are highly benefited.\newline
	
	The authors found that
	
	\bi
		\item in a 10-way cross-validation, CC predictions weren't useful because of high false alarm rates
		\item however, with relevency filtered training data, CC results were nearly as good as those using WC 
		training data.
	\ei
	
\end{frame}

\begin{frame}

	\frametitle{Cross-Company Defect Prediction using Relevancy Filtering}
	
	\begin{block}{Purpose}
		To verify Turhan et al.'s relevancy filtering results. This experiment can also be found in~\cite{ourmine-10}.
	\end{block}
	
	\begin{block}{Setup}
	
		Using seven {\em combined} defect data sets (CM1, KC1, KC2, KC3, MC2, MW1, PC1).\newline
		
		To test relevancy filtering, 		
		\bi
			\item a k-NN filter was used 
			\item duplicate instances were removed, remaining instances became the new training set
		\ei									
	\end{block}
	
	
\end{frame}



\begin{frame}
	\frametitle{Results 1a}
	
	\begin{center}
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pd}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Treatment& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&WC (local data) + filter& 66& 73& 80&  \boxplot{0}{66}{73}{80}{3}  \\
	2&CC (imported data) + filter& 57& 71& 83&  \boxplot{0}{57}{71}{83}{5}  \\
	2&WC (local data)  & 59& 69& 83&  \boxplot{0}{59}{69}{83}{7}  \\
	3&CC (imported data)  & 49& 66& 87&  \boxplot{0}{49}{66}{87}{7}   \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}
	\end{figure}
	\end{center}

\end{frame}

\begin{frame}
	\frametitle{Results 1b}
	
	\begin{figure}
	\small
	\begin{center}
	\begin{tabular}{l@{~}|l@{~}|r@{~}r@{~}@{~}r@{~}|c}
	%\multicolumn{2}{c}{~}&\multicolumn{5}{c}{quartiles}\\\cline{3-5}
	%&min& & median & &max\\\cline{3-5}
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{pf}&2nd quartile\\
	\multicolumn{2}{c}{~}&\multicolumn{3}{c}{percentiles}&median,\\\cline{3-5}
	Rank&Treatment& 25\%& 50\% & 75\%&3rd quartile\\\hline
	1&WC (local data) + filter & 20& 27& 34&  \boxplot{0}{20}{27}{34}{8}  \\
	2&WC (local data)  & 17& 30& 41&  \boxplot{0}{17}{30}{41}{7}  \\
	3&CC (imported data) + filter& 17& 29& 43&  \boxplot{0}{17}{29}{43}{3}  \\
	3&CC (imported data)  & 13& 34& 51&  \boxplot{0}{13}{34}{51}{5} \\\hline
	\multicolumn{5}{c}{~}&~~~~~0~~~~~~~~50~~~~100
	\end{tabular}
	\end{center}
\end{figure}

\end{frame}

\begin{frame}
	\frametitle{Conclusion}
	
	In this experiment, we find that:
	
	\bi
		\item filtered WC always outperforms CC
		\item however, CC + {\em relevancy filtering} performs nearly as well as WC
	\ei
	
	Thus, 
		
	\bi	
		\item When local data is available, that data should be used to build defect predictors 
		\item If needed, imported data can be used to build defect predictors when using relevancy filtering 		
	\ei
	
\end{frame}

%\subsection{Predicting Student Retention}

%\begin{frame}
%	\frametitle{Predicting Student Retention}
	
%	Institutions of higher education are afflicted by low student retention. According to Druzdel et al.~\cite{druzdzel}, 
%	almost 45\% of first-year freshmen do not return for the following year. \newline
	
%	\begin{block}{Problem}
%		Low student retention has negative effects on institutions, as well as decreased career opportunities for individuals. 
%	\end{block} 
	
%\end{frame}

%\begin{frame}
%	\frametitle{Predicting Student Retention}
	
%	\begin{block}{Purpose}
%		To determine actionable attributes crucial to the successful prediction of student retention.
%	\end{block}
	
%	\begin{block}{Setup}
%		The experiment began with many options:
		
%		\bi
%			\item 7 Learners - Naive Bayes, C4.5, One-R, Zero-R, ADTrees, Bayesian Network, RBFN
%			\item 4 Feature Subset Selectors - Correlation-Based Feature Selection, Information Gain, Chi-squared, One-R
%			\item Number of attributes to select (i.e. 5, 10, 15, 20 ... 100)			
%		\ei
		
%	\end{block}
		
%\end{frame}

%\begin{frame}
%	\frametitle{The Problem with Variance}
	
%	As analysis of the experimental results began, variance became a problem
	
%	\bi
%		\item Many High PD/Low PF treatments contained high variances (75th - 25th percentile)
%	\ei

%	\begin{block}{Solution}
%		Visualize results based on performance measures {\em and} variance.
%	\end{block}	
			
%\end{frame}

%\begin{frame}
%	\frametitle{Visualizing the Results}
	
%	After visualization, it was concluded that:
	
%	\bi
%		\item first year retention prediction barely met the baseline
%		\item second year retention prediction was slightly above the baseline
%		\item third year retention prediction yielded significantly better results than the baseline
%	\ei	
	
%	The number of attributes became clear
	
%	\bi
%		\item any attribute sizes above 60 yielded high variance
%		\item 20 - 60 attributes resulted in high performances/lower variance
%	\ei
				
%\end{frame}


%\begin{frame}
%	\frametitle{Results}
	
%	Focusing on 20-60 attributes, Mann-Whitney rankings found that
	
%	\bi
%		\item the best learner was Bayesian Network
%		\item the best FSS was One-R
%		\item the best number of attributes was 30
%	\ei
									
%\end{frame}

%\begin{frame}
%	\frametitle{Results}
		
	%\begin{center}
%		\includegraphics[height=2.5in]{attr-ranks.pdf}
	%\end{center}
										
%\end{frame}

%\begin{frame}
%	\frametitle{Conclusion}		
	
%	In this study, a 20\% increase in prediction was made over the baseline. This degree of success 
%	has not been seen in our review of the literature.\newline
	
%	\begin{block}{Food for Thought}
%		According to these results, the socioeconomic status of a student's family is crucial 
%	to third-year retention.	
%	\end{block}
										
%\end{frame}

\subsection{Component vs. Whole-based Defect Prediction}

\begin{frame}
	\frametitle{Component vs. Whole-based Defect Prediction}
	
	\begin{block}{Problem}
		Despite efforts to improve software defect predictors through new algorithms 
		(i.e. classifiers), results have reached a ceiling.
	\end{block}
	
	\begin{block}{Purpose}
		To develop a new method of treating the {\em data itself} to improve performance.
	\end{block}
	
\end{frame}

\begin{frame}
	\frametitle{What's a Component?}
	
	Hongyu Zhang, Tsinghua University, noted that most software defects occur in small numbers of components.
	
	\bi	
		\item Components are portions of software that encapsulate certain functionality (e.g. a specific tool in a GUI).\newline	
		\item Components contain 1 or more modules. Modules are functions/methods, depending on the programming language.
	\ei
		
\end{frame}

\begin{frame}
	\frametitle{Component vs. Whole-based Defect Prediction}
	
	\begin{block}{Setup}
		Using five NASA defect data sets (CM1, KC1, MC1, PC1, PC3), training instances were selected as follows:
			\bi
				\item Pick components whose number of defective modules exceeded the median number of 
				defective modules across {\em all} components.
				\item From those components, extract modules for training. This was the {\em dense} treatment.
			\ei		
	\end{block}							
\end{frame}

\begin{frame}

	\frametitle{Distribution of Defects}
	\begin{center}
		\includegraphics*[viewport=50 50 400 300, width=3in]{KC1.pdf}
	\end{center}
	
\end{frame}

\begin{frame}[fragile]
	\frametitle{Component vs. Whole-based Defect Prediction}
	
	\begin{block}{Setup cont.}
		
		The experiment was conducted as follows:
		
{\small
\begin{verbatim}
1 For run = 1 to 10
2   For each dense component C in data set D
3     Train = C
4     Test  = D - C
5     For bin = 1 to 10
6        Test'  = 10% of Test (picked at random)
7        Train' = 90% of Train (picked at random)
8        Naive Bayes (Train', Test')
9     end bin
10   end component
11 end run
\end{verbatim}}
		
			
	\end{block}							
\end{frame}

\begin{frame}
	\frametitle{Component vs. Whole-based Defect Prediction}

	\begin{block}{Setup cont.}
	
		Other methods have been proposed~\cite{menziesgay08} to alter the class distribution of defect training data:
		
			\bi
				\item Under-sampling: randomly minimize the non-target class to equal the size of the target class
				\item Over-sampling: randomly maximize the target class to equal the size of the non-target class
			\ei				
		
		For rigor, in this experiment, comparisons are made between all four treatments: over/under sampling, 
		all/dense component learning.
	\end{block}

\end{frame}

\begin{frame}
	\frametitle{Results per Data set}
	\begin{center}
		\includegraphics[width=4.2in]{datatable.png}
	\end{center}
\end{frame}


\begin{frame}
	\frametitle{Summary Table}
	\begin{center}
	\tiny
	\begin{tabular}{|c|r|c|c|c|c|}\hline
	data & performance & all        & dense      & over     & under\\
	set & measure      & components & components & sampling & sampling\\\hline
	CM1&    precision&  0&  0& 0& 0\\
    &   recall&         +&  -& -& -\\
    &   pf&         -&  +& -& -\\\hline
	KC1&    precision&  0&  0& 0& 0\\
    &   recall&         0&  0& 0& 0\\
    &   pf&         0&  0& 0& 0\\\hline
	MC1&    precision&  0&  0&  0&  0\\
    &   recall&         -&  -& 0& 0\\
    &   pf&         -& -& 0& 0\\\hline
	PC1&    precision&  -&  +& -& -\\
    &   recall&         -&  +& -& -\\
    &   pf&         -&  +& -& -\\\hline
	PC3&    precision&  0&  0&  0&  0\\
    &   recall&         -&  -& 0& 0\\
    &   pf&         -&  +& -& -\\\hline\hline
	summary& +&         1& 5& 0& 0\\
       &0 &         6& 6& 9& 9\\
       &- &        8&  4& 6& 6\\\hline
\end{tabular}
\end{center}

\end{frame}

\begin{frame}
	\frametitle{Conclusion}
	
	Focusing on defect dense regions yields 
	
	\bi
		\item better predictors when considering {\em recall}, {\em pf} and {\em precision}.
		\item smaller, more managable data sets.
	\ei
	
	\begin{block}{Future Work}
		Applying training on defect-dense components in a cross-company experiment. Example: 
		\bi
			\item combine all training data's (CC) components
			\item extract modules as per the previously discussed filter
			\item train predictors using these modules
		\ei
	\end{block}
	
\end{frame}

\subsection{Analyzing the Scalability of Clustering Text Documents}

\begin{frame}
	\frametitle{Analyzing the Scalability of Clustering Text Documents}
	
	Document clustering is widely used for document organization, topic extraction, search result categorization, etc.\newline
	
	\begin{block}{Problem}
		Rigorous clustering is slow. 
		\bi
			\item K-means: Uses $K*I*D$ document comparisons {\em plus} centroid update computations. 
		\ei
	\end{block}
	
	\begin{block}{Purpose}
			To analyze the scalability and, thus, usability of faster heuristic methods	to 
			cluster and reduce the dimensions (terms) of the data.	
	\end{block}
	
\end{frame}


\begin{frame}[fragile]
	\frametitle{Analyzing the Scalability of Clustering Text Documents}
	
	\begin{block}{Setup}
		Data sets used in this experiment:
		
		\bi
			\item EXPRESS schemas: AP-203, AP-214
			\item Text mining datasets: BBC, Reuters, The Guardian (multi-view text datasets), 20 Newsgroup subsets:
			sb-3-2, sb-8-2, ss-3-2, sl-8-2%\footnotetext[1]{\url{http://mlg.ucd.ie/datasets}}.
		\ei
	\end{block}
\end{frame}

\begin{frame}[fragile]
	\frametitle{Analyzing the Scalability of Clustering Text Documents}
	
	\begin{block}{Setup cont.}
		Clustering: K-means vs. GenIc~\cite{genic04} and Canopy Clustering~\cite{canopies00}
		
		\bi
			\item GenIc 
			{\small
				\begin{verbatim}
				1  Select original candidate centers randomly
				2  For each point p in the stream of points
				3    Find the nearest candidate centroid to p
				4    Move the nearest centroid toward p 
				5    Increment its weight
				6    When n, 2n, 3n, ... instances have been seen
				7  Compute the centroid's probability of survival
				8  If low, kill it and replace it. Set its weight to 1.
				9  If high, keep it and go to the next generation.
				10 Candidate centers can be used in final clusters. 			
				\end{verbatim}
			}
			
		\ei		
	\end{block}
	
\end{frame}


\begin{frame}[fragile]
	\frametitle{Analyzing the Scalability of Clustering Text Documents}
	
	\begin{block}{Setup cont.}		
		\bi
			\item Canopy clustering
			{\small
				\begin{verbatim}
					1  Each "canopy" has two distance thresholds, T1/T2
					2  Randomly select a point from the data
					3  Use a cheap distance measure create a canopy 
				    with that point as a centroid. Canopy members
				     are within T2
					4  Expensive distance measures are only used within 
					the same canopies
					5  Points inside of T1 are excluded from becoming 
					other canopy members
				\end{verbatim}
			}
			
		\ei		
	\end{block}
	
\end{frame}


\begin{frame}[fragile]
	\frametitle{Analyzing the Scalability of Clustering Text Documents}

	\begin{block}{Setup cont.}		
		\bi
			\item Dimensionality Reduction
			\bi
				\item TF*Idf (Term Frequency * Inverse Document Frequency) 
					For each term $t$ in document $D_j$
						\begin{equation} 
							Tf*df(t,D_j) = \frac{tf(t_i,D_j)}{|D_j|}log(\frac{|D|}{df(t_i)})
						\end{equation}
					
				\item PCA (Principal Component Analysis) - Find axes of the data such that variation 
					is maximized (i.e. lack of variation in an axis suggests this is a less important 
					dimension)					
							\bi
								\item Find eigenvalues and eigenvectors of the covariance matrix 
							\ei				
			\ei
		\ei		
	\end{block}
	
\end{frame}

\begin{frame}[fragile]
	\frametitle{Analyzing the Scalability of Clustering Text Documents}

	\begin{block}{Setup cont.}		
		
		Evaluation of the clustering/reduction methods was done through
		
		\bi
			\item Runtime of clustering/reducing the data
			\item $Gain = intra$-$similarity - inter$-$similarity$
			\item Cluster intra-similarity
				\bi
					\item For each document $d$ in cluster $C$, find the similarity between $d$ and all documents 
					belonging to $C$ [maximized]					
				\ei
			\item Cluster inter-similarity
				\bi
					\item For each document $d$ in cluster $C$, find the similarity between $d$ and all documents
					belonging to all other clusters. [minimized]									
				\ei
		\ei
		
	\end{block}
	
\end{frame}

\begin{frame}[fragile]
	\frametitle{Results}

	\begin{center}
		\includegraphics[width=4in]{simgraph.pdf}
	\end{center}	
\end{frame}


\begin{frame}[fragile]
	\frametitle{Conclusion}
	
	By examining runtimes and similarity gain, it was shown that
		
	\bi
		\item Faster clustering algorithms are competitive because of their $scalability$ 
		(GenIc vs. K-means)
		\item Faster dimensionality reduction methods sometimes outperform more rigorous 
		ones on text mining data sets (Tf-IDF vs. PCA).
	\ei
	
	In summary, faster heuristics can lead to adequate results on some text mining data sets.
		
\end{frame}



%%%%%%%%%%%%% END EXPERIMENT SLIDES %%%%%%%%%%%%%%%%%

%%%%%%%%%%%%% CONCLUSION SLIDES %%%%%%%%%%%%%%%%%

\section{Conclusion}
\begin{frame}
	\frametitle{Conclusion}
Ourmine lends itself to being a competitive toolkit for data mining because

\bi
	\item it's freely available
	\item it remains optimal for publishing experiments through printable scripts representing experiments
	\item it supports any language/libraries with a command-line API
	\item its use builds multi-functional knowledge
	\item the environment is functional as a teaching tool
		\bi
			\item novices can learn data mining concepts through simple scripts
		\ei 
	\item it's capable of complex experiments
\ei
\end{frame}


\begin{frame}
	\frametitle{Conclusion}
	
	Most existing, free environments suffer from 
	
	\bi
		\item lack of experiment sharing abilities (must publish {\em instructions} to build an experiment, \underline{not} 
		the experiment itself!)
		\item conformity to standards (custom scripting languages)
		\item complexity
		\item language restriction (Java, C++, Python, etc.)
	\ei
	
\end{frame}
%%%%%%%%%%%%% END CONCLUSION WORK SLIDES %%%%%%%%%%%%%%%%%




%%%%%%%%%%%%% FUTURE WORK SLIDES %%%%%%%%%%%%%%%%%

\section{Future Work}

\begin{frame}[fragile]
	\frametitle{Future Work}
		
	\begin{block}{Note}
		Ourmine is far from complete! 
	\end{block}
	
	Possible future work could include:
	
	\bi
		\item Regulated contributions from the open source community
		\item Rapid syntax for less complex experiments. 
		
		\bi
			\item Example: \begin{verbatim} 
				data="weather mushroom iris"
				learners="nb j48"
				_crossval=10x10
				$data->$learners
				\end{verbatim}								
		\ei		
		\item Automated plotting/visualization of results 
		
	\ei
	
\end{frame}

%%%%%%%%%%%%% END FUTURE WORK SLIDES %%%%%%%%%%%%%%%%%




%%%%%%%%%%%%% QUESTIONS SLIDES %%%%%%%%%%%%%%%%%
\section{}

\begin{frame}
	\frametitle{Questions/Comments?}
\end{frame}

%%%%%%%%%%%%% QUESTIONS SLIDES %%%%%%%%%%%%%%%%%

%%%%%%%%%%%%% REFS SLIDES %%%%%%%%%%%%%%%%%


%\begin{frame}
%	\frametitle{References}
	\bibliographystyle{thesis}
	{\tiny
	\bibliography{refs}
	}
%\end{frame}

\begin{frame}
	Images sources:\newline
	\small
	Slide 7: \url{http://www.knime.org/documentation/gettingstarted}
	Slide 13: \url{http://www.freedigitalphotos.net/images/view_photog.php?photogid=987}
\end{frame}

%%%%%%%%%%%%% REFS SLIDES %%%%%%%%%%%%%%%%%


\end{document}
