% define datasets of experiment
myDatasets = {
    'cocomo81o','cocomo81s','cocomo81','cocomo81e',...
    'desharnaisL1','desharnais','desharnaisL2',...
    'desharnaisL3',...
    'nasa93','nasa93_center_1','nasa93_center_2','nasa93_center_5',...
    'sdr'...
    'albrecht',...
    'finnish','kemerer','maxwell','miyazaki94','telecom1',...
    %'china'
    };

% for 3Way and LOO, define 90 by 3 matrices for each error measure
% 90 by 3 matrices will keep the appearance of methods on top 16
% first column is appearance w.r.t. win values, second is w.r.t loss
% and the third one is w.r.t. (win-loss)
stabilityMAR3Way = zeros(90,3);
stabilityMMRE3Way = zeros(90,3);
stabilityMMER3Way = zeros(90,3);
stabilityMBRE3Way = zeros(90,3);
stabilityMIBRE3Way = zeros(90,3);
stabilityMDMRE3Way = zeros(90,3);
stabilityPRED3Way = zeros(90,3);

stabilityMARLOO = zeros(90,3);
stabilityMMRELOO = zeros(90,3);
stabilityMMERLOO = zeros(90,3);
stabilityMBRELOO = zeros(90,3);
stabilityMIBRELOO = zeros(90,3);
stabilityMDMRELOO = zeros(90,3);
stabilityPREDLOO = zeros(90,3);

datasetSize = size(myDatasets,2);
for datasetCounter = 1:datasetSize
    
    % load win and loss values
    eval(['load(''workspaces\3Way\',char(myDatasets(datasetCounter)),'.mat'');']);
    winValues3Way = winValues';
    lossValues3Way = lossValues';
    winMinusLossValues3Way = (winValues-lossValues)';
    % now get sorted indices for win, loss and (win-loss)
    
    %%%% MAR 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(1,:), 'descend');
    stabilityMAR3Way(winIndices(1:16),1) = stabilityMAR3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(1,:), 'ascend');
    stabilityMAR3Way(lossIndices(1:16),2) = stabilityMAR3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(1,:), 'descend');
    stabilityMAR3Way(winMinusLossIndices(1:16),3) = stabilityMAR3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(2,:), 'descend');
    stabilityMMRE3Way(winIndices(1:16),1) = stabilityMMRE3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(2,:), 'ascend');
    stabilityMMRE3Way(lossIndices(1:16),2) = stabilityMMRE3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(2,:), 'descend');
    stabilityMMRE3Way(winMinusLossIndices(1:16),3) = stabilityMMRE3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MMER 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(3,:), 'descend');
    stabilityMMER3Way(winIndices(1:16),1) = stabilityMMER3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(3,:), 'ascend');
    stabilityMMER3Way(lossIndices(1:16),2) = stabilityMMER3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(3,:), 'descend');
    stabilityMMER3Way(winMinusLossIndices(1:16),3) = stabilityMMER3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(4,:), 'descend');
    stabilityMBRE3Way(winIndices(1:16),1) = stabilityMBRE3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(4,:), 'ascend');
    stabilityMBRE3Way(lossIndices(1:16),2) = stabilityMBRE3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(4,:), 'descend');
    stabilityMBRE3Way(winMinusLossIndices(1:16),3) = stabilityMBRE3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MIBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(5,:), 'descend');
    stabilityMIBRE3Way(winIndices(1:16),1) = stabilityMIBRE3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(5,:), 'ascend');
    stabilityMIBRE3Way(lossIndices(1:16),2) = stabilityMIBRE3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(5,:), 'descend');
    stabilityMIBRE3Way(winMinusLossIndices(1:16),3) = stabilityMIBRE3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MDMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(6,:), 'descend');
    stabilityMDMRE3Way(winIndices(1:16),1) = stabilityMDMRE3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(6,:), 'ascend');
    stabilityMDMRE3Way(lossIndices(1:16),2) = stabilityMDMRE3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(6,:), 'descend');
    stabilityMDMRE3Way(winMinusLossIndices(1:16),3) = stabilityMDMRE3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% PRED 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValues3Way(7,:), 'descend');
    stabilityPRED3Way(winIndices(1:16),1) = stabilityPRED3Way(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValues3Way(7,:), 'ascend');
    stabilityPRED3Way(lossIndices(1:16),2) = stabilityPRED3Way(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValues3Way(7,:), 'descend');
    stabilityPRED3Way(winMinusLossIndices(1:16),3) = stabilityPRED3Way(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    %%%%%% NOW START DOING THE STABILITY CALCULATIONS %%%%%%
    %%%%%% FOR LOO BELOW BASICALL DO THE SAME STUFF AGAIN %%
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    % load win and loss values
    eval(['load(''workspaces\LOO\',char(myDatasets(datasetCounter)),'.mat'');']);
    winValuesLOO = winValues';
    lossValuesLOO = lossValues';
    winMinusLossValuesLOO = (winValues-lossValues)';
    % now get sorted indices for win, loss and (win-loss)
    
    %%%% MAR LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(1,:), 'descend');
    stabilityMARLOO(winIndices(1:16),1) = stabilityMARLOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(1,:), 'ascend');
    stabilityMARLOO(lossIndices(1:16),2) = stabilityMARLOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(1,:), 'descend');
    stabilityMARLOO(winMinusLossIndices(1:16),3) = stabilityMARLOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MMRE LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(2,:), 'descend');
    stabilityMMRELOO(winIndices(1:16),1) = stabilityMMRELOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(2,:), 'ascend');
    stabilityMMRELOO(lossIndices(1:16),2) = stabilityMMRELOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(2,:), 'descend');
    stabilityMMRELOO(winMinusLossIndices(1:16),3) = stabilityMMRELOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MMER LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(3,:), 'descend');
    stabilityMMERLOO(winIndices(1:16),1) = stabilityMMERLOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(3,:), 'ascend');
    stabilityMMERLOO(lossIndices(1:16),2) = stabilityMMERLOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(3,:), 'descend');
    stabilityMMERLOO(winMinusLossIndices(1:16),3) = stabilityMMERLOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MBRE LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(4,:), 'descend');
    stabilityMBRELOO(winIndices(1:16),1) = stabilityMBRELOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(4,:), 'ascend');
    stabilityMBRELOO(lossIndices(1:16),2) = stabilityMBRELOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(4,:), 'descend');
    stabilityMBRELOO(winMinusLossIndices(1:16),3) = stabilityMBRELOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MIBRE LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(5,:), 'descend');
    stabilityMIBRELOO(winIndices(1:16),1) = stabilityMIBRELOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(5,:), 'ascend');
    stabilityMIBRELOO(lossIndices(1:16),2) = stabilityMIBRELOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(5,:), 'descend');
    stabilityMIBRELOO(winMinusLossIndices(1:16),3) = stabilityMIBRELOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% MDMRE LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(6,:), 'descend');
    stabilityMDMRELOO(winIndices(1:16),1) = stabilityMDMRELOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(6,:), 'ascend');
    stabilityMDMRELOO(lossIndices(1:16),2) = stabilityMDMRELOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(6,:), 'descend');
    stabilityMDMRELOO(winMinusLossIndices(1:16),3) = stabilityMDMRELOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    
    %%%% PRED LOO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    [C winIndices] = sort(winValuesLOO(7,:), 'descend');
    stabilityPREDLOO(winIndices(1:16),1) = stabilityPREDLOO(winIndices(1:16),1) + 1; 
    [C lossIndices] = sort(lossValuesLOO(7,:), 'ascend');
    stabilityPREDLOO(lossIndices(1:16),2) = stabilityPREDLOO(lossIndices(1:16),2) + 1; 
    [C winMinusLossIndices] = sort(winMinusLossValuesLOO(7,:), 'descend');
    stabilityPREDLOO(winMinusLossIndices(1:16),3) = stabilityPREDLOO(winMinusLossIndices(1:16),3) + 1; 
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
end

% write the results into excel, where
% each sheet corresponds to an error measure
xlswrite('plots/stability.xls',stabilityMAR3Way, 'MAR');
xlswrite('plots/stability.xls',stabilityMMRE3Way, 'MMRE');
xlswrite('plots/stability.xls',stabilityMMER3Way, 'MMER');
xlswrite('plots/stability.xls',stabilityMBRE3Way, 'MBRE');
xlswrite('plots/stability.xls',stabilityMIBRE3Way, 'MIBRE');
xlswrite('plots/stability.xls',stabilityMDMRE3Way, 'MDMRE');
xlswrite('plots/stability.xls',stabilityPRED3Way, 'PRED25');

xlswrite('plots/stability.xls',stabilityMARLOO, 'MAR','E1:G90');
xlswrite('plots/stability.xls',stabilityMMRELOO, 'MMRE','E1:G90');
xlswrite('plots/stability.xls',stabilityMMERLOO, 'MMER','E1:G90');
xlswrite('plots/stability.xls',stabilityMBRELOO, 'MBRE','E1:G90');
xlswrite('plots/stability.xls',stabilityMIBRELOO, 'MIBRE','E1:G90');
xlswrite('plots/stability.xls',stabilityMDMRELOO, 'MDMRE','E1:G90');
xlswrite('plots/stability.xls',stabilityPREDLOO, 'PRED25','E1:G90');

% and now plot for each error measure
h=figure;
subplot(3,1,1); hold on; plot(stabilityMAR3Way(:,1),'r');plot(stabilityMARLOO(:,1),'g'); hold off;
title('Win');ylabel('MAR');legend('LOO','3Way');
subplot(3,1,2); hold on; plot(stabilityMAR3Way(:,2),'r');plot(stabilityMARLOO(:,2),'g'); hold off;
title('Loss');ylabel('MAR');legend('LOO','3Way');
subplot(3,1,3); hold on; plot(stabilityMAR3Way(:,3),'r');plot(stabilityMARLOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MAR');legend('LOO','3Way');
eval(['print(h,''-dpng'', ''plots\MAR.png'');']);
eval(['print(h,''-dpdf'', ''plots\MAR.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMMRE3Way(:,1),'r');plot(stabilityMMRELOO(:,1),'g'); hold off;
title('Win');ylabel('MMRE');legend('LOO','3Way');
subplot(3,1,2); hold on; plot(stabilityMMRE3Way(:,2),'r');plot(stabilityMMRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MMRE');legend('LOO','3Way');
subplot(3,1,3); hold on; plot(stabilityMMRE3Way(:,3),'r');plot(stabilityMMRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MMRE');legend('LOO','3Way');
eval(['print(h,''-dpng'', ''plots\MMRE.png'');']);
eval(['print(h,''-dpdf'', ''plots\MMRE.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMMER3Way(:,1),'r');plot(stabilityMMERLOO(:,1),'g'); hold off;
title('Win');ylabel('MMER');legend('LOO','3Way');
subplot(3,1,2); hold on; plot(stabilityMMER3Way(:,2),'r');plot(stabilityMMERLOO(:,2),'g'); hold off;
title('Loss');ylabel('MMER');legend('LOO','3Way');
subplot(3,1,3); hold on; plot(stabilityMMER3Way(:,3),'r');plot(stabilityMMERLOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MMER');legend('LOO','3Way');
eval(['print(h,''-dpng'', ''plots\MMER.png'');']);
eval(['print(h,''-dpdf'', ''plots\MMER.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMBRE3Way(:,1),'r');plot(stabilityMBRELOO(:,1),'g'); hold off;
title('Win');ylabel('MBRE');legend('LOO','3Way');
subplot(3,1,2); hold on; plot(stabilityMBRE3Way(:,2),'r');plot(stabilityMBRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MBRE');legend('LOO','3Way');
subplot(3,1,3); hold on; plot(stabilityMBRE3Way(:,3),'r');plot(stabilityMBRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MBRE');legend('LOO','3Way');
eval(['print(h,''-dpng'', ''plots\MBRE.png'');']);
eval(['print(h,''-dpdf'', ''plots\MBRE.pdf'');']);
close all;