%\documentclass[journal,compsoc]{IEEEtran} \documentclass[onecolumn,12pt,journal,compsoc]{IEEEtran}\renewcommand{\baselinestretch}{1.41} \usepackage[table]{xcolor} % *** CITATION PACKAGES *** % \ifCLASSOPTIONcompsoc % IEEE Computer Society needs nocompress option % requires cite.sty v4.0 or later (November 2003) % \usepackage[nocompress]{cite} \else % normal IEEE % \usepackage{cite} \fi \usepackage{cite,url} \ifCLASSINFOpdf \usepackage[pdftex]{graphicx} % declare the path(s) where your graphic files are \graphicspath{{/plots/}} % and their extensions so you won't have to specify these with % every instance of \includegraphics \DeclareGraphicsExtensions{.pdf,.jpeg,.png, .eps} \else % or other class option (dvipsone, dvipdf, if not using dvips). graphicx % will default to the driver specified in the system graphics.cfg if no % driver is specified. % \usepackage[dvips]{graphicx} % declare the path(s) where your graphic files are % \graphicspath{{../eps/}} % and their extensions so you won't have to specify these with % every instance of \includegraphics % \DeclareGraphicsExtensions{.eps} \fi \usepackage{graphicx} \usepackage{subfigure} \usepackage{verbatim} \usepackage{fancyvrb} \usepackage{algorithm} \usepackage{algorithmic} \usepackage{timtricks} \usepackage{multirow} \usepackage{rotating} \usepackage{times} \usepackage{cite} \usepackage{float} \usepackage{hyperref} %\restylefloat{figure} % correct bad hyphenation here \hyphenation{op-tical net-works semi-conduc-tor} \begin{document} \section{Bias and Variance Problem Between LOO and 3Way} \subsection{Bias-Variance with and Without Statistical Test} Every plot in this section for each dataset is shown with no statistical test and a statistical test applied. On the plots only those methods are shown, whose bias/variance values are statistically different to one another. When we apply a statistical test (Wilcoxon) to compare the results of 90 methods on each dataset, we see that only a few (sometimes none) of the methods attain statistical difference. Most of the time, whether subject to LOO or 3Way, methods produce statistically same estimations. \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/cocomo81.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatcocomo81.png}} \caption{cocomo81 Bias-Var for LOO and 3-Way} \label{fig:coc81-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/cocomo81o.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatcocomo81o.png}} \caption{cocomo81o Bias-Var for LOO and 3-Way} \label{fig:coc81o-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/cocomo81s.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatcocomo81s.png}} \caption{cocomo81s Bias-Var for LOO and 3-Way} \label{fig:coc81s-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/cocomo81e.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatcocomo81e.png}} \caption{cocomo81e Bias-Var for LOO and 3-Way} \label{fig:coc81e-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/nasa93.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatnasa93.png}} \caption{nasa93 Bias-Var for LOO and 3-Way} \label{fig:nasa93-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/nasa93_center_1.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatnasa93_center_1.png}} \caption{nasa93\_center\_1 Bias-Var for LOO and 3-Way} \label{fig:nasa93c1-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/nasa93_center_2.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatnasa93_center_2.png}} \caption{nasa93\_center\_2 Bias-Var for LOO and 3-Way} \label{fig:nasa93c2-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/nasa93_center_5.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatnasa93_center_5.png}} \caption{nasa93\_center\_5 Bias-Var for LOO and 3-Way} \label{fig:nasa93c5-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/desharnais.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatdesharnais.png}} \caption{desharnais Bias-Var for LOO and 3-Way} \label{fig:desharnais-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/desharnaisL1.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatdesharnaisL1.png}} \caption{desharnaisL1 Bias-Var for LOO and 3-Way} \label{fig:desharnaisL1-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/desharnaisL2.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatdesharnaisL2.png}} \caption{desharnaisL2 Bias-Var for LOO and 3-Way} \label{fig:desharnaisL2-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/desharnaisL3.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatdesharnaisL3.png}} \caption{desharnaisL3 Bias-Var for LOO and 3-Way} \label{fig:desharnaisL3-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/sdr.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatsdr.png}} \caption{sdr Bias-Var for LOO and 3-Way} \label{fig:sdr-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/albrecht.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatalbrecht.png}} \caption{albrecht Bias-Var for LOO and 3-Way} \label{fig:albrecht-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/finnish.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatfinnish.png}} \caption{finnish Bias-Var for LOO and 3-Way} \label{fig:finnish-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/kemerer.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatkemerer.png}} \caption{kemerer Bias-Var for LOO and 3-Way} \label{fig:kemerer-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/maxwell.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatmaxwell.png}} \caption{maxwell Bias-Var for LOO and 3-Way} \label{fig:maxwell-bias-var} \end{figure*} \begin{figure*}[th!] \centering \subfigure[Without Stat. Test]{\label{fig:withoutStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/miyazaki94.png}} \subfigure[With Stat. Test]{\label{fig:withStat}\includegraphics[width=0.48\textwidth]{../biasVarPlots/withStatmiyazaki94.png}} \caption{miyazaki94 Bias-Var for LOO and 3-Way} \label{fig:miyazaki94-bias-var} \end{figure*} \clearpage \section{Jiggle Problem: How much does a method deviate?} Our aim is to find the deviations of methods between two testing strategies (LOO and CV). When doing that our fix ordering is the ICSE submission, fig.4, where ordering was based on loss-values-ordering across all datasets and all error measures subject to LOO. There are two dimensions to that problem: \begin{enumerate} \item How much does a method deviate across datasets for different error measures? \item How much variation per method do we have across error measures when we fix the dataset? \end{enumerate} For both problems, there are intermediary analysis and summary analysis: \begin{itemize} \item \textbf{Intermediary:} What is rank delta (absolute rank change) for each method? \item \textbf{Intermediary:} What is the rank-order for each method when we keep icse order fixed in one axis and plot new order in the other axis? \item \textbf{Summary:} What is the occurrence count of a method in top/bottom-x across? \end{itemize} \subsection{Deviation Across Datasets for Each Error Measure} To answer the first item, I searched for top-16 and bottom-16 methods in all datasets for each error measure separately. Whenever a method appeared in top/bottom-16 according to an error measure I incremented its occurrence value by 1. Since we have 19 datasets, a method may have the highest number of 19. The complete list of occurrence numbers (\textbf{summary}) is \href{http://unbox.org/wisp/var/ekrem/resultsVariance/Code-v3(stability)/plots/icsePlots/top-bottom-16-occurrences.xls}{here} and the rank-delta-plots (\textbf{intermediary}) are \href{http://unbox.org/wisp/var/ekrem/resultsVariance/Code-v3(stability)/plots/absRanks/}{here}. The formula for rank-delta plots is: \textit{abs(my Index acc. to win/loss/win-loss - myIcseOrder)}. Note in summary plots that 3-Way jiggles mostly behave like an envolope for LOO jiggles, i.e. 3-Way mostly has higher variation w.r.t. icse sorting. The rank-order is another way of looking at stability of ranking. On rank-order plots, the x-axis is the methods in order of Icse, whereas y-axis is the order of corresponding methods according to win/loss or win-loss count under current error measure. Rank-orders are \href{http://unbox.org/wisp/var/ekrem/resultsVariance/Code-v3(stability)/plots/orderedRanks/}{here}. \subsection{Deviation Across Error Measures for Each Dataset} Another dimension to the jiggle problem is the deviations across error measures for every dataset. The absolute rank-order for that dimension can be found \href{http://unbox.org/wisp/var/ekrem/resultsVariance/Code-v3(stability)/plots/absRanks/perDataset/}{here}. It is difficult to observe a general trend between LOO and 3-Way on dataset level when absolute rank deltas are concerned. \clearpage \section{ICSE Plots} \begin{figure*}[th!] \centering \subfigure[For LOO]{\label{fig:winLOO}\includegraphics[width=0.48\textwidth]{icsePlots/winValuesLOO.pdf}} \subfigure[For 3Way]{\label{fig:win3Way}\includegraphics[width=0.48\textwidth]{icsePlots/winValues3Way.pdf}} \caption{Win Percentages} \label{fig:win-perc} \end{figure*} \begin{figure*}[th!] \centering \subfigure[For LOO]{\label{fig:lossLOO}\includegraphics[width=0.48\textwidth]{icsePlots/lossValuesLOO.pdf}} \subfigure[For 3Way]{\label{fig:loss3Way}\includegraphics[width=0.48\textwidth]{icsePlots/lossValues3Way.pdf}} \caption{Loss Percentages} \label{fig:win-perc} \end{figure*} \begin{figure*}[th!] \centering \subfigure[For LOO]{\label{fig:win-lossLOO}\includegraphics[width=0.48\textwidth]{icsePlots/winMinusLossLOO.pdf}} \subfigure[For 3Way]{\label{fig:win-loss3Way}\includegraphics[width=0.48\textwidth]{icsePlots/winMinusLoss3Way.pdf}} \caption{Win-Loss Percentages} \label{fig:win-perc} \end{figure*} \end{document}