% the difference between myMainFile2 and myMainFile
% is that this version shows all active and randActive labelings
% plus the best label count = stop if no improvement in 3 consecutive runs
% in terms of mdmre
% or if the mre value is less than 0.25

% clearn workspace and command window
clear;clc;

% define how many times the experiments are to be repeated
maxExperimentCount = 100;
minManyLabels = 3;
infinitesimalChange = 0.01;

% store the stopping percentages
allStoppingPercentages = [];
% store all instances with E(1) > 0
greaterThan1Popular = [];
% keep dataset sizes
myDatasetInstSizes = [];

% define datasets of experiment
myDatasets = {
    'cocomo81',...
    'cocomo81e',...
    'cocomo81o',...
    'cocomo81s',...
    'desharnais',...
    'desharnaisL1',...
    'desharnaisL2',...
    'desharnaisL3',...
    'nasa93',...
    'nasa93_center_1',...
    'nasa93_center_2',...
    'nasa93_center_5',...
    'sdr',...
    'albrecht',...
    'finnish',...
    'kemerer',...
    'maxwell',...
    'miyazaki94',...
    'telecom1',...
%     'china',...
%     'chinaResource1',...
%     'chinaResource2',...
%     'chinaResource3',...
%     'chinaResource4',...
    };

% below variables keep win,tie,loss for 4 learners: active, passive, rand,
% cart subject to 7 error measures: mmre, mdmre, pred25, mar, mbre, mibre
marWinTieLoss = [];
mbreWinTieLoss = [];
mibreWinTieLoss = [];
mmreWinTieLoss = [];
mdmreWinTieLoss = [];
pred25WinTieLoss = [];
mmerWinTieLoss = [];


% open a file to write the reasons
reasonsFile = fopen('plots\stoppingReasons.txt','a');


% for each dataset in the study
datasetSize = size(myDatasets,2);
for datasetCounter = 1:datasetSize
    
    eval(['load ',char(myDatasets(datasetCounter)),'.csv;']);
    eval(['nowDataset = ',char(myDatasets(datasetCounter)),';']);
    eval(['clear ',char(myDatasets(datasetCounter)),';']);
    clc;
    myDatasets(datasetCounter)  % print out the dataset name
    
    % below variable is needed for stopping conditions
    maxLabelCounter = [];
    
    % get the row and column number (without dependent variable)
    n = size(nowDataset,1);
    p = size(nowDataset,2) - 1; % exclude the effort value, i.e. dependent var.
    myDatasetInstSizes = [myDatasetInstSizes;n];
    
    % log the dataset
    logNowDataset = [log(nowDataset(:,1:p)), nowDataset(:,(p+1))];
    
    % also keep a normalized version of the dataset
    normNowDataset = myNormalizer(nowDataset);
    
    % define test set size; the remaining instances will be the train
    testSize = ceil(n/10);
    
    % define overall actuals and predictions
    myOverallTestActuals = [];
    myOverallPassivePredictions = [];
    myOverallActivePredictions = [];
    myOverallRandomPredictions = [];
    myOverallCartPredictions = [];
    datasetActuals = logNowDataset(:,p+1);
    
    % calculate the index matrix that is the sorted distance matrix
    [tmpMatrix myDistMatrix] = calculateMyDistanceMatrix(logNowDataset);
        
    for experimentCounter = 1:maxExperimentCount
        
        % pick up testSize-many instances at random
        trainIndices = 1:n;
        testIndices = [];
        for testPicker = 1:testSize
            tmpRnd = randi([1 size(trainIndices,2)],1,1);
            testIndices = [testIndices,trainIndices(tmpRnd)];
            trainIndices(tmpRnd) = [];
        end
        
        testActuals = logNowDataset(testIndices,p+1)'; % store actual effort values
        myOverallTestActuals = [myOverallTestActuals,testActuals];
        trainActuals = logNowDataset(trainIndices,p+1)';
        
        % now start getting the predictions from active and normal k
        % first get the predictions of k=1 with all effort known
        predictionsPassive = -1 * ones(1,testSize);
        predictionsCart = -1 * ones(1,testSize);
        for i = 1:testSize
            relatedIndices = tmpMatrix(:,testIndices(i));
            relatedIndices = relatedIndices(trainIndices);
            [C closestGuyIndex] = min(relatedIndices);
            predictionsPassive(i) = trainActuals(closestGuyIndex);
        end
        % store passive prediction
        myOverallPassivePredictions = [myOverallPassivePredictions,predictionsPassive];
        
        % now get the predictions for k=1, with different numbers of labeled
        % instances from 1 to n-1
        [popIndex maxLabelsToAsk] = findPopularity(tmpMatrix); % order instances w.r.t. popularity
        maxLabelCounter = [maxLabelCounter maxLabelsToAsk];
        popIndex = popIndex(trainIndices);    % get pop indices of train instances
        predictionsActive = [];
        predictionsRandom = [];
        
        % before finding the stopping point, we ask n-1 labels where n is
        % dataset size
        howManyLabels = size(trainActuals,2);
        % in below for-loop, make active1nn predictions as well as
        % activeRand1nn predictions
        for i = 1:howManyLabels
            tmpPredictionsActive = activePredict(datasetActuals,tmpMatrix,popIndex,i,testIndices,myDistMatrix);
            predictionsActive = [predictionsActive;tmpPredictionsActive];
            tmpPredictionsRandom = activeRandomPredict(datasetActuals,tmpMatrix,popIndex,i,testIndices,myDistMatrix);
            predictionsRandom = [predictionsRandom;tmpPredictionsRandom];
        end
        myOverallActivePredictions = [myOverallActivePredictions,predictionsActive];
        myOverallRandomPredictions = [myOverallRandomPredictions,predictionsRandom];
        
        % and now apply the CART algorithm and get its predictions
        tmpCartPredictions = myCARTOn(normNowDataset(testIndices,:),normNowDataset(trainIndices,:));
        myOverallCartPredictions = [myOverallCartPredictions,tmpCartPredictions];
    end
    
    
    % now we will plot the results w.r.t. different error measures
    
    % firstly calculate the mdmre's
    mdmrePassive = ones(1,howManyLabels) * median(abs(myOverallTestActuals - myOverallPassivePredictions)./myOverallTestActuals);
    mdmreActive = -1 * ones(1,howManyLabels);
    mdmreRandom = -1 * ones(1,howManyLabels);
    mdmreCart = ones(1,howManyLabels) * median(abs(myOverallTestActuals - myOverallCartPredictions)./myOverallTestActuals);
    for i = 1:howManyLabels
        mdmreActive(i) = median(abs(myOverallTestActuals - myOverallActivePredictions(i,:))./myOverallTestActuals);
    end
    for i = 1:howManyLabels
        mdmreRandom(i) = median(abs(myOverallTestActuals - myOverallRandomPredictions(i,:))./myOverallTestActuals);
    end
    
    % find the stopping point: how many labels would be enough
    % rule 1 is: stop if mdmre<0.25
    % rule 2 is: look back 3 instances before you move, 
    % rule 3: stop if it is a zero-popularity instance
    % if there is no improvement in the last 3 runs, then stop
    % so we start with at least 4 labels
    lookBackAmount = 4;
    stopReason = 'none';
    % store popular instances 
    greaterThan1Popular = [greaterThan1Popular max(maxLabelCounter)];
    
    for stoppingPoint = lookBackAmount:howManyLabels 
        mdmresToConsider = mdmreActive((stoppingPoint-(lookBackAmount-1)):stoppingPoint);
        [maxMdmreVal maxMdmreInd] = max(mdmresToConsider);
        if mdmreActive(stoppingPoint) < 0.25
            stopReason = 'lessThan0.25';
            break;
%         elseif mdmreActive(stoppingPoint) <= mdmrePassive(1)
%             stopReason = 'mdmreActiveWentBelowPassive';
%             break;
        elseif (abs(mdmreActive(stoppingPoint) - mdmreActive(stoppingPoint-1)) <= infinitesimalChange) ...
                && (abs(mdmreActive(stoppingPoint) - mdmreActive(stoppingPoint-2)) <= infinitesimalChange) ...
                && (abs(mdmreActive(stoppingPoint) - mdmreActive(stoppingPoint-3)) <= infinitesimalChange)
            stopReason = 'infinitesimalChange';
            break;
        elseif (maxMdmreInd == stoppingPoint) 
%                 && (size(find((mdmresToConsider(1)-mdmresToConsider(2:size(mdmresToConsider,2)))==0),2)==0)% meaning that there was no mdmre decrease
            stopReason = 'No MdMRE Decrease';
            break;
        elseif    max(maxLabelCounter) == stoppingPoint % encountered a zero-popularity instance
            
            stopReason = 'Encountered Zero-Pop Instance';
            break;
        else
            % no condition met yet, so continue
            if stoppingPoint == howManyLabels
                stopReason = 'No Stop Condition Met';
                break;
            end
        end
    end
    
    % store the current stopping point
    allStoppingPercentages = [allStoppingPercentages,(stoppingPoint*100/n)];
    
    % check statistical difference and see if active and passive mre's are
    % statistically different
    passiveMREs = (abs(myOverallTestActuals - myOverallPassivePredictions)./myOverallTestActuals);
    activeMREs = (abs(myOverallTestActuals - myOverallActivePredictions(stoppingPoint,:))./myOverallTestActuals);
    [p myH] = ranksum(passiveMREs, activeMREs);
    
    % calculate the win-tie-loss values of all the performance measures
    % first MAR
    [tmpWin,tmpTie,tmpLoss] = myMAR(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      marWinTieLoss = [marWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then MBRE
      [tmpWin,tmpTie,tmpLoss] = myMBRE(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      mbreWinTieLoss = [mbreWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then MIBRE
      [tmpWin,tmpTie,tmpLoss] = myMIBRE(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      mibreWinTieLoss = [mibreWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then MMRE
      [tmpWin,tmpTie,tmpLoss] = myMMRE(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
          myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      mmreWinTieLoss = [mmreWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then MdMRE
      [tmpWin,tmpTie,tmpLoss] = myMDMRE(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      mdmreWinTieLoss = [mdmreWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then pred25
      [tmpWin,tmpTie,tmpLoss] = myPred25(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      pred25WinTieLoss = [pred25WinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      % then MMER
      [tmpWin,tmpTie,tmpLoss] = myMMER(myOverallTestActuals', ...
        [myOverallPassivePredictions; ...
          myOverallActivePredictions(stoppingPoint,:); ...
%           myOverallRandomPredictions(stoppingPoint,:); ...
          myOverallCartPredictions], ...
          [1;2;3]);
      mmerWinTieLoss = [mmerWinTieLoss;[tmpWin,tmpTie,tmpLoss]];
      
    
      
    close all;
    h = figure;
%     hold on;
%     axis([1 howManyLabels ...
%         (min([log(mdmrePassive) log(mdmreActive) log(mdmreRandom) log(mdmreCart)])-0.2) ...
%         (max([log(mdmrePassive) log(mdmreActive) log(mdmreRandom) log(mdmreCart)])+0.2) ]);
%     semilogy(1:howManyLabels,(mdmrePassive),'o-'...
%         ,1:howManyLabels,(mdmreActive),'p-'...
%         ,1:howManyLabels,(mdmreRandom),'s-'...
%         ,1:howManyLabels,(mdmreCart),'d-'...
%         ,'LineWidth',1.5...
%         ,'MarkerSize',8);
    axis([1 howManyLabels ...
        (min([(mdmrePassive) (mdmreActive) (mdmreRandom) (mdmreCart)])-0.2) ...
        (max([(mdmrePassive) (mdmreActive) (mdmreRandom) (mdmreCart)])+0.2) ]);
    stoppingPointArray1 = [(min([log10(mdmrePassive) log10(mdmreActive) log10(mdmreRandom) log10(mdmreCart)])-0.2) ...
        :0.01:(max([log10(mdmrePassive) log10(mdmreActive) log10(mdmreRandom) log10(mdmreCart)])+ 0.2)];
    stoppingPointArray2 = stoppingPoint * ones(1,size(stoppingPointArray1,2));
    plot(1:howManyLabels,log10(mdmrePassive),'o-'...
        ,1:howManyLabels,log10(mdmreActive),'p-'...
        ,1:howManyLabels,log10(mdmreRandom),'s-'...
        ,1:howManyLabels,log10(mdmreCart),'d-'...
        ,stoppingPointArray2,stoppingPointArray1,'-' ...
        ,'LineWidth',1.5...
        ,'MarkerSize',8);
    legend('Passive 1NN','Active 1NN','Random 1NN','Norm-Cart/On',['Stop Point=' num2str(stoppingPoint)]);
    xlabel('Number of labeled instances');ylabel('MdMRE');
    title([char(myDatasets(datasetCounter)) ', Significance = ' num2str(myH) ', StopReason: ' stopReason]);
    eval(['print(h,''-dpng'', ''plots\',char(myDatasets(datasetCounter)),'.png'');']);
%     hold off;
    close all;
    
    % now write the stuff to csv files for gnuplot
    csvwrite(['lib\mdmre_' char(myDatasets(datasetCounter)) '.csv'] ...
        ,[[1:1:howManyLabels]',mdmrePassive',mdmreActive',mdmreRandom',mdmreCart' ...
        ,(myH*ones(howManyLabels,1)) ...
        , [0.2,ones(1,howManyLabels-2),1.5]' ...
        ,(stoppingPoint*ones(howManyLabels,1)) ...
        ,((stoppingPoint*100/size(nowDataset,1))*ones(howManyLabels,1))]);
    fprintf(reasonsFile,[char(myDatasets(datasetCounter)) ', ' stopReason '\n']);
    
end
fclose(reasonsFile);
%write win,tie,loss tables into an excel
xlswrite('winTieLossResults',marWinTieLoss,'MAR');
xlswrite('winTieLossResults',mbreWinTieLoss,'MBRE');
xlswrite('winTieLossResults',mibreWinTieLoss,'MIBRE');
xlswrite('winTieLossResults',mmreWinTieLoss,'MMRE');
xlswrite('winTieLossResults',mdmreWinTieLoss,'MDMRE');
xlswrite('winTieLossResults',pred25WinTieLoss,'PRED25');
xlswrite('winTieLossResults',mmerWinTieLoss,'MMER');


