clear;clc;
biasVarFile = fopen('biasVar.txt', 'w');
fclose(biasVarFile);

% define datasets of experiment
myDatasets = {
    'cocomo81','cocomo81o','cocomo81e','cocomo81s',...
    'nasa93','nasa93_center_1','nasa93_center_2','nasa93_center_5',...
    'desharnais','desharnaisL1','desharnaisL2',...
    'desharnaisL3',...
    'sdr'...
    'albrecht',...
    'finnish','kemerer','maxwell','miyazaki94','telecom1',...
    'china'
    };

datasetSize = size(myDatasets,2);
% below variables keep track of percentage methods which are statistically
% different from one another
biasPercentages = -1 * ones(datasetSize,3);
varPercentages = -1 * ones(datasetSize,3);
% below variables keep percentage values
myQuartile3Way10WayLOO = [];
myQuartile3Way = [];
myQuartile10Way = [];
myQuartileLOO = [];

for datasetCounter = 1:datasetSize
    
    % convert the order to icse
    orderInIcse = [41	51	3	4	17	20	27	36	48	65	80	5	6	67	66	12	39	86	81	89	60	61	78	76	59	42	87	74	82	30	31	79	73	55	23	88	63	75	44	45	68	72	69	47	85	49	77	32	33	64	70	50	52	83	40	58	1	2	54	71	26	34	84	37	57	7	8	24	16	11	13	56	35	90	21	22	15	19	62	43	46	38	25	9	10	29	28 18 14 53];
    
    eval(['load(''workspaces\10way\',char(myDatasets(datasetCounter)),'.mat'');']);
    actuals10Way = actual10by10values;
    algorithmResults10Way = algorithmResults(orderInIcse,:);
    biasValues10Way = -1*ones(size(algorithmResults10Way,1),1);
    varValues10Way = -1*ones(size(algorithmResults10Way,1),1);
    
    eval(['load(''workspaces\3way\',char(myDatasets(datasetCounter)),'.mat'');']);
    actuals3Way = actual10by3values;
    algorithmResults3Way = algorithmResults(orderInIcse,:);
    biasValues3Way = -1*ones(size(algorithmResults3Way,1),1);
    varValues3Way = -1*ones(size(algorithmResults3Way,1),1);
    
    eval(['load(''workspaces\LOO\',char(myDatasets(datasetCounter)),'.mat'');']);
%     actualsLOO = actualsToSave';
    actualsLOO = pseudoActuals';
    algorithmResultsLOO = algorithmResults(orderInIcse,:);
    biasValuesLOO = -1*ones(size(algorithmResults3Way,1),1);
    varValuesLOO = -1*ones(size(algorithmResults3Way,1),1);
    
    % below variables keep track of statistical difference
    % there are 3 columns, because we have 3 sampling methods to compare
    % 1st column: LOO vs. 3Way difference
    % 2nd column: LOO vs. 10Way difference
    % 3rd column: 3Way vs. 10Way difference
    biasDifference = -1 * ones(size(varValues3Way,1),3);
    varDifference = -1 * ones(size(varValues3Way,1),3);
    
    % calculate bias and variance
    labels = {};
    labelsBias = {}; % this one keeps the labels of statistically different guys
    labelsVar = {}; % this one keeps the labels of statistically different guys
    for i = 1:size(varValues3Way,1)
        labels =[labels i];
        fHatMean10Way = mean(algorithmResults10Way(i,:));
        biasSeparate10Way = ((fHatMean10Way - actuals10Way).^2);
        biasValues10Way(i) = mean(biasSeparate10Way);
        varSeparate10Way = ((fHatMean10Way - algorithmResults10Way(i,:)).^2);
        varValues10Way(i) = mean(varSeparate10Way);
        
        algorithmResults3Way(i,(isnan(algorithmResults3Way(i,:)))) = 0;
        fHatMean3Way = mean(algorithmResults3Way(i,:));
        biasSeparate3Way = ((fHatMean3Way - actuals3Way).^2);
        biasValues3Way(i) = mean(biasSeparate3Way);
        varSeparate3Way = ((fHatMean3Way - algorithmResults3Way(i,:)).^2);
        varValues3Way(i) = mean(varSeparate3Way);
        
        fHatMeanLOO = mean(algorithmResultsLOO(i,:));
        biasSeparateLOO = ((fHatMeanLOO - actualsLOO).^2);
        biasValuesLOO(i) = mean(biasSeparateLOO);
        varSeparateLOO = ((fHatMeanLOO - algorithmResultsLOO(i,:)).^2);
        varValuesLOO(i) = mean(varSeparateLOO);
        
        % if there is a statistical difference in bias then mark it with a 1
        [P,H] = ranksum(biasSeparate3Way,biasSeparateLOO);
        if H == 0   % if they are the same
            biasDifference(i,1) = 0;
        else
            biasDifference(i,1) = 1;
        end
        [P,H] = ranksum(biasSeparate10Way,biasSeparateLOO);
        if H == 0   % if they are the same
            biasDifference(i,2) = 0;
        else
            biasDifference(i,2) = 1;
        end
        [P,H] = ranksum(biasSeparate3Way,biasSeparate10Way);
        if H == 0   % if they are the same
            biasDifference(i,3) = 0;
        else
            biasDifference(i,3) = 1;
        end
        
        % if there is a statistical difference in bias then mark it with a 1
        [P,H] = ranksum(varSeparate3Way,varSeparateLOO);
        if H == 0   % if they are the same
            varDifference(i,1) = 0;
        else
            varDifference(i,1) = 1;
            labelsVar =[labelsBias i];
        end
        [P,H] = ranksum(varSeparate10Way,varSeparateLOO);
        if H == 0   % if they are the same
            varDifference(i,2) = 0;
        else
            varDifference(i,2) = 1;
        end
        [P,H] = ranksum(varSeparate3Way,varSeparate10Way);
        if H == 0   % if they are the same
            varDifference(i,3) = 0;
        else
            varDifference(i,3) = 1;
        end
    end
    
    % now print the significant difference between different methods
    biasPercentages(datasetCounter,1) = (100 * size(find(biasDifference(:,1) == 1),1)) / size(biasDifference(:,1),1);
    biasPercentages(datasetCounter,2) = (100 * size(find(biasDifference(:,2) == 1),1)) / size(biasDifference(:,2),1);
    biasPercentages(datasetCounter,3) = (100 * size(find(biasDifference(:,3) == 1),1)) / size(biasDifference(:,3),1);
    varPercentages(datasetCounter,1) = (100 * size(find(varDifference(:,1) == 1),1)) / size(varDifference(:,1),1);
    varPercentages(datasetCounter,2) = (100 * size(find(varDifference(:,2) == 1),1)) / size(varDifference(:,2),1);
    varPercentages(datasetCounter,3) = (100 * size(find(varDifference(:,3) == 1),1)) / size(varDifference(:,3),1);
    
    % now start plotting all bias-var plots without statistical check
    h = figure;hold on;
    set(gca,'FontSize',28,'FontName','Arial');
    plot(log(biasValues3Way), log(varValues3Way),'*g','MarkerSize',12);
    plot(log(biasValues10Way), log(varValues10Way),'ob','MarkerSize',12);
    plot(log(biasValuesLOO), log(varValuesLOO),'+r','MarkerSize',12);
    myQuartile3Way = [min(log(biasValues3Way)) prctile(log(biasValues3Way),25) median(log(biasValues3Way)) prctile(log(biasValues3Way),75) max(log(biasValues3Way)) ...
        min(log(varValues3Way)) prctile(log(varValues3Way),25) median(log(varValues3Way)) prctile(log(varValues3Way),75) max(log(varValues3Way))];
    myQuartile10Way = [min(log(biasValues10Way)) prctile(log(biasValues10Way),25) median(log(biasValues10Way)) prctile(log(biasValues10Way),75) max(log(biasValues10Way)) ...
        min(log(varValues10Way)) prctile(log(varValues10Way),25) median(log(varValues10Way)) prctile(log(varValues10Way),75) max(log(varValues10Way))];
    myQuartileLOO = [min(log(biasValuesLOO)) prctile(log(biasValuesLOO),25) median(log(biasValuesLOO)) prctile(log(biasValuesLOO),75) max(log(biasValuesLOO)) ...
        min(log(varValuesLOO)) prctile(log(varValuesLOO),25) median(log(varValuesLOO)) prctile(log(varValuesLOO),75) max(log(varValuesLOO))];
    myQuartile3Way10WayLOO = [myQuartile3Way10WayLOO;myQuartile3Way;myQuartile10Way;myQuartileLOO];

    
%     myQuartile3Way = [min(log(biasValues3Way)) myPerc(log(biasValues3Way),25) median(log(biasValues3Way)) myPerc(log(biasValues3Way),75) max(log(biasValues3Way)) ...
%         min(log(varValues3Way)) myPerc(log(varValues3Way),25) median(log(varValues3Way)) myPerc(log(varValues3Way),75) max(log(varValues3Way))];
%     myQuartile10Way = [min(log(biasValues10Way)) myPerc(log(biasValues10Way),25) median(log(biasValues10Way)) myPerc(log(biasValues10Way),75) max(log(biasValues10Way)) ...
%         min(log(varValues10Way)) myPerc(log(varValues10Way),25) median(log(varValues10Way)) myPerc(log(varValues10Way),75) max(log(varValues10Way))];
%     myQuartileLOO = [min(log(biasValuesLOO)) myPerc(log(biasValuesLOO),25) median(log(biasValuesLOO)) myPerc(log(biasValuesLOO),75) max(log(biasValuesLOO)) ...
%         min(log(varValuesLOO)) myPerc(log(varValuesLOO),25) median(log(varValuesLOO)) myPerc(log(varValuesLOO),75) max(log(varValuesLOO))];
%     myQuartile3Way10WayLOO = [myQuartile3Way10WayLOO;myQuartile3Way;myQuartile10Way;myQuartileLOO];



%     axis([0 20 0 20]);
%     text(log(biasValues3Way),log(varValues3Way), labels,'EdgeColor','green','LineStyle',':','FontSize',16);
    
%     text(log(biasValuesLOO),log(varValuesLOO), labels,'EdgeColor','red','LineStyle','-.','FontSize',16);
    xlabel('Logged Bias Values');
    ylabel('Logged Variance Values');
%     title([char(myDatasets(datasetCounter)) ', ' num2str(size(biasValues3Way,1)) ' Methods Shown.' ...
%         ' Green=3Way, Red=LOO and Blue=10Way']);
    eval(['print(h,''-dpng'', ''biasVarPlots\',char(myDatasets(datasetCounter)),'.png'');']);
    legend('3Way','10Way','LOO');
    hold off;close all;
    %     % then start plotting all bias-var plots this time only those that are
    %     % statistically different
    %     h = figure;hold on;
    %     set(gca,'FontSize',20);
    %     biasValues3Way = biasValues3Way(find(biasDifference == 1));
    %     varValues3Way = varValues3Way(find(varDifference == 1));
    %     biasValuesLOO = biasValuesLOO(find(biasDifference == 1));
    %     varValuesLOO = varValuesLOO(find(varDifference == 1));
    %     plot(log(biasValues3Way), log(varValues3Way),'.');
    %     axis([0 20 0 20]);
    %     text(log(biasValues3Way),log(varValues3Way), labelsStats,'EdgeColor','green','LineStyle',':','FontSize',16);
    %     plot(log(biasValuesLOO), log(varValuesLOO),'.');
    %     text(log(biasValuesLOO),log(varValuesLOO), labelsStats,'EdgeColor','red','LineStyle','-.','FontSize',16);
    %     xlabel('Logged Bias Values');
    %     ylabel('Logged Variance Values');
    %     title([char(myDatasets(datasetCounter)) ', ' num2str(size(biasValues3Way,1)) ' Methods Shown.' ...
    %         ' Green=3Way and Red=LOO.']);
    %     eval(['print(h,''-dpng'', ''biasVarPlots\withStat',char(myDatasets(datasetCounter)),'.png'');']);
    %     hold off;close all;
    biasVarFile = fopen('biasVar.txt', 'a');
    fprintf(biasVarFile, [char(myDatasets(datasetCounter)) '\n']);
    fprintf(biasVarFile,  'biasValues3Way');
    if size(labelsBias,2) > 0
        for tmpCounter = 1:(size(labelsBias,2)-1)
            fprintf(biasVarFile,  [',' num2str((biasValues3Way(cell2mat(labelsBias(tmpCounter)))))]);
        end
        fprintf(biasVarFile,  [',' num2str((biasValues3Way(cell2mat(labelsBias(size(labelsBias,2)))))) '\n']);
        
        for tmpCounter = 1:(size(labelsBias,2)-1)
            fprintf(biasVarFile,  [',' num2str((biasValues10Way(cell2mat(labelsBias(tmpCounter)))))]);
        end
        fprintf(biasVarFile,  [',' num2str((biasValues10Way(cell2mat(labelsBias(size(labelsBias,2)))))) '\n']);
        
        fprintf(biasVarFile,  'biasValuesLOO');
        for tmpCounter = 1:(size(labelsBias,2)-1)
            fprintf(biasVarFile,  [',' num2str(biasValuesLOO(cell2mat(labelsBias(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(biasValuesLOO(cell2mat(labelsBias(size(labelsBias,2))))) '\n']);
    else
        fprintf(biasVarFile,  '\nbiasValuesLOO\n');
    end
    
    fprintf(biasVarFile,  'varValues3Way');
    if size(labelsVar,2) > 0
        for tmpCounter = 1:(size(labelsVar,2)-1)
            fprintf(biasVarFile,  [',' num2str(varValues3Way(cell2mat(labelsVar(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(varValues3Way(cell2mat(labelsVar(size(labelsVar,2))))) '\n']);
        
        for tmpCounter = 1:(size(labelsVar,2)-1)
            fprintf(biasVarFile,  [',' num2str(varValues10Way(cell2mat(labelsVar(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(varValues10Way(cell2mat(labelsVar(size(labelsVar,2))))) '\n']);
        
        fprintf(biasVarFile,  'varValuesLOO');
        for tmpCounter = 1:(size(labelsVar,2)-1)
            fprintf(biasVarFile,  [',' num2str(varValuesLOO(cell2mat(labelsVar(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(varValuesLOO(cell2mat(labelsVar(size(labelsVar,2))))) '\n\n']);
    else
        fprintf(biasVarFile,  '\nvarValuesLOO\n');
    end
end

% now write bias variance difference percentages
xlswrite('biasVarianceDiffPerc', [biasPercentages varPercentages]);
fclose(biasVarFile);