% define datasets of experiment
myDatasets = {
    'cocomo81o','cocomo81s','cocomo81','cocomo81e',...
    'desharnaisL1','desharnais','desharnaisL2',...
    'desharnaisL3',...
    'nasa93','nasa93_center_1','nasa93_center_2','nasa93_center_5',...
    'sdr'...
    'albrecht',...
    'finnish','kemerer','maxwell','miyazaki94','telecom1',...
    'china'
    };

% define pre-processors of experiment
myPreProcessors = {'none','log',...
    'freq3bin','freq5bin','width3bin','width5bin',...
    'norm',...
    'SWReg','PCA','SFS'};

% define algorithms of experiment
myAlgorithms = {'SWReg','SLReg',...
    'CART-On','CART-Off',...
    'PLSR','PCR',...
    '1NN','ABE0',...
    'NNet',...
    'MoE'};

% define error-measures of experiment
myErrorMeasures = {'MAR','MMRE','MMER','MBRE','MIBRE','MDMRE','Pred25'};

% define how many random runs you want for each dataset
randomRunSize = size(myAlgorithms,2) * size(myPreProcessors,2);

% store size of each of the above variables
datasetSize = size(myDatasets,2);
preProcessorSize = size(myPreProcessors,2);
errorMeasureSize = size(myErrorMeasures,2);
algorithmSize = size(myAlgorithms,2);

% delete previous files by opening and closing same files with write
% permission
separateLossValues = fopen('separateLossValues.txt','w');
separateWinValues = fopen('separateWinValues.txt','w');
sumLossValues = fopen('sumLossValues.txt','w');
sumWinValues = fopen('sumWinValues.txt','w');
mdmreValues = fopen('mdmreValues.txt','w');
fclose(separateLossValues);
fclose(separateWinValues);
fclose(sumLossValues);
fclose(sumWinValues);
fclose(mdmreValues);

for i = 1:size(myErrorMeasures,2)
    eval(['myErrorMeasure' num2str(i) 'LossValues = fopen(''myErrorMeasure' num2str(i) 'LossValues.txt'', ''w'');']);
    eval(['fclose(myErrorMeasure' num2str(i) 'LossValues);']);
    eval(['myErrorMeasure' num2str(i) 'WinValues = fopen(''myErrorMeasure' num2str(i) 'WinValues.txt'', ''w'');']);
    eval(['fclose(myErrorMeasure' num2str(i) 'WinValues);']);
end

% define variables for randomSize methods
fixedPreProcessors = cell(randomRunSize,1) ;
fixedAlgorithms = cell(randomRunSize,1);
% settle the ten different algorithm&pre-processor combinations
% randNum1 = randi([1 preProcessorSize],1,randomRunSize);   % random num. for pre-proc.
% randNum2 = randi([1 algorithmSize],1,randomRunSize);   % random num. for algorithms

% above code is for random mappings between methods and pre-processors
% below code is somewhat hard coded mappings
randNum1 = [];
randNum2 = [];
for i = 1:size(myPreProcessors,2)
    for j = 1:size(myAlgorithms,2)
        randNum1 = [randNum1,i];
        randNum2 = [randNum2,j];
    end
end


for runIndex = 1:randomRunSize
    % randomly pick up pre-processor
    fixedPreProcessors(runIndex) = (myPreProcessors(randNum1(runIndex)));
    % randomly pick up an algorithm
    fixedAlgorithms(runIndex) = (myAlgorithms(randNum2(runIndex)));
end

% write current dataset and error measure to output files
sumLossValues=fopen('sumLossValues.txt','a' );
fprintf(sumLossValues, 'SUM OF LOSSES','%s');
sumWinValues=fopen('sumWinValues.txt','a' );
fprintf(sumWinValues, 'SUM OF WINS','%s');
mdmreValues=fopen('mdmreValues.txt','a' );
fprintf(mdmreValues, 'MdMRE Values','%s');

% write the selected methods
for runIndex = 1:randomRunSize
    % write pre-processor
    fprintf(separateLossValues, [',' char(fixedPreProcessors(runIndex))]);
    % write algorithm
    fprintf(separateLossValues, ['-' char(fixedAlgorithms(runIndex))]);
    % write pre-processor
    fprintf(separateWinValues, [',' char(fixedPreProcessors(runIndex))]);
    % write algorithm
    fprintf(separateWinValues, ['-' char(fixedAlgorithms(runIndex))]);
    % write pre-processor
    fprintf(mdmreValues, [',' char(fixedPreProcessors(runIndex))]);
    % write algorithm
    fprintf(mdmreValues, ['-' char(fixedAlgorithms(runIndex))]);
end
fprintf(sumLossValues, '\n');    % append a new line at the end of the line
fprintf(sumWinValues, '\n');    % append a new line at the end of the line
fprintf(mdmreValues, '\n');    % append a new line at the end of the line
fclose(sumLossValues);    % close file
fclose(sumWinValues);    % close file
fclose(mdmreValues);    % close file


% for each dataset in the study
for datasetCounter = 1:datasetSize
    
    eval(['load ',char(myDatasets(datasetCounter)),'.csv;']);
    eval(['dataset = ',char(myDatasets(datasetCounter)),';']);
    eval(['clear ',char(myDatasets(datasetCounter)),';']);
    clc;
    myDatasets(datasetCounter)  % print out the dataset name
    
    % define a variable to keep the loss values
    lossValues = -1 * ones(errorMeasureSize,randomRunSize);
    % define a variable to keep the win values
    winValues = -1 * ones(errorMeasureSize,randomRunSize);
    
    % write current dataset and error measure to output files
    separateLossValues=fopen('separateLossValues.txt','a' );
    fprintf(separateLossValues, char(myDatasets(datasetCounter)),'%s');
    separateWinValues=fopen('separateWinValues.txt','a' );
    fprintf(separateWinValues, char(myDatasets(datasetCounter)),'%s');
    
    
    % write the selected methods
    for runIndex = 1:randomRunSize
        % write pre-processor
        fprintf(separateLossValues, [',' char(fixedPreProcessors(runIndex))]);
        % write algorithm
        fprintf(separateLossValues, ['-' char(fixedAlgorithms(runIndex))]);
        % write pre-processor
        fprintf(separateWinValues, [',' char(fixedPreProcessors(runIndex))]);
        % write algorithm
        fprintf(separateWinValues, ['-' char(fixedAlgorithms(runIndex))]);
    end
    fprintf(separateLossValues, '\n');    % append a new line at the end of the line
    fprintf(separateWinValues, '\n');    % append a new line at the end of the line
    
    % close files
    fclose(separateLossValues);
    fclose(separateWinValues);
    
    % for each error measure
    for errorCounter = 1: size(myErrorMeasures,2)
        currentErrorMeasure = char(myErrorMeasures(errorCounter));
        % open file to append
        separateLossValues=fopen('separateLossValues.txt','a' );        % write error measure to file
        separateWinValues=fopen('separateWinValues.txt','a' );        % write error measure to file
        
        fprintf(separateLossValues,char(myErrorMeasures(errorCounter)));
        fprintf(separateWinValues,char(myErrorMeasures(errorCounter)));
        % close files
        fclose(separateLossValues);
        fclose(separateWinValues);
        
        % define a variable to keep the results of each method
        algorithmResults = -1 * ones(randomRunSize, size(dataset,1));
        
        for runIndex = 1:(randomRunSize)
            % reload dataset to get rid of previous pre-processor effect
            eval(['load ',char(myDatasets(datasetCounter)),'.csv;']);
            eval(['dataset = ',char(myDatasets(datasetCounter)),';']);
            eval(['clear ',char(myDatasets(datasetCounter)),';']);
            
            % pick up the pre-processor on the queue
            currentPreProcessor = char(fixedPreProcessors(runIndex));
            % apply the pre-processor and get the new dataset
            dataset = applyPreProcess(dataset, currentPreProcessor);
            
            % run the next algorithm
            currentAlgorithm = char(fixedAlgorithms(runIndex));
            if strcmp(currentAlgorithm, 'MoE') == 1
                % finally get the mixture of experts as the median of all the other
                % algorithm results (below code line does that)
                algorithmResults(runIndex,:)= median(algorithmResults((runIndex-9):(runIndex-1),:));
                
            else
                %                 algorithmResults(runIndex,:) = algorithmRunner(dataset, currentAlgorithm, currentErrorMeasure, currentPreProcessor, 0); % 0 means without GAC pruning
                algorithmResults(runIndex,:) = algorithmRunner(dataset, currentAlgorithm, currentErrorMeasure, currentPreProcessor, 1); % 1 means without GAC pruning
            end
        end
        
        % at this point we have the predictions of all the algorithms
        % now we can compare these results and store their loss values
        [lossValues(errorCounter,:), winValues(errorCounter,:)] = orderAlgorithms(algorithmResults,dataset(:,size(dataset,2)),currentErrorMeasure,fixedAlgorithms, fixedPreProcessors, myDatasets(datasetCounter));
        
        % open file to append and write the win and loss values for current
        % error measure
        separateLossValues=fopen('separateLossValues.txt','a' );        % write error measure to file
        separateWinValues=fopen('separateWinValues.txt','a' );        % write error measure to file
        
        for printCounter = 1:size(lossValues,2)
            fprintf(separateLossValues,[', ' num2str(lossValues(errorCounter,printCounter))]);
            fprintf(separateWinValues,[', ' num2str(winValues(errorCounter,printCounter))]);
        end
        fprintf(separateLossValues, '\n');
        fprintf(separateWinValues, '\n');
        % close files
        fclose(separateLossValues);
        fclose(separateWinValues);
        
        errorMeasureLossValues(errorCounter,datasetCounter,:) = lossValues(errorCounter,:);
        errorMeasureWinValues(errorCounter,datasetCounter,:) = winValues(errorCounter,:);
        
    end
    
    % save algorithm results
    eval([char(myDatasets(datasetCounter)) 'Results = algorithmResults;']);
    
    % open file to append
    separateLossValues=fopen('separateLossValues.txt','a' );        % write sum to file
    separateWinValues=fopen('separateWinValues.txt','a' );        % write sum to file
    fprintf(separateLossValues,'\n\n');  % put a blank line for the next dataset
    fprintf(separateWinValues,'\n\n');  % put a blank line for the next dataset
    % close files
    fclose(separateLossValues);
    fclose(separateWinValues);
    
    %%%% now write the sum of win and loss values to related files
    % write current dataset and error measure to output files
    sumLossValues=fopen('sumLossValues.txt','a' );
    fprintf(sumLossValues, char(myDatasets(datasetCounter)),'%s');
    sumWinValues=fopen('sumWinValues.txt','a' );
    fprintf(sumWinValues, char(myDatasets(datasetCounter)),'%s');
    
    sums = sum(lossValues);
    for tempCounter = 1:randomRunSize
        fprintf(sumLossValues, [',' num2str(sums(tempCounter))]);
    end
    fprintf(sumLossValues,'\n');  % put a blank line for the next dataset
    
    sums = sum(winValues);
    for tempCounter = 1:randomRunSize
        fprintf(sumWinValues, [',' num2str(sums(tempCounter))]);
    end
    fprintf(sumWinValues,'\n');  % put a blank line for the next dataset
    
    % close files
    fclose(sumLossValues);
    fclose(sumWinValues);
end

% open the separate loss and win counts for different error measures
for errorCounter = 1:size(myErrorMeasures,2)
    % firstly open the required files
    eval(['myErrorMeasure' num2str(errorCounter) 'LossValues = fopen(''myErrorMeasure' num2str(errorCounter) 'LossValues.txt'', ''a'');']);
    eval(['myErrorMeasure' num2str(errorCounter) 'WinValues = fopen(''myErrorMeasure' num2str(errorCounter) 'WinValues.txt'', ''a'');']);
    
    %     eval(['fprintf([''' char(myErrorMeasures(errorCounter)) ''' ], myErrorMeasure' num2str(errorCounter) 'LossValues);']);
    %     eval(['fprintf([''' char(myErrorMeasures(errorCounter)) ''' ], myErrorMeasure' num2str(errorCounter) 'WinValues);']);
    
    for datasetCounter = 1:datasetSize
        eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'LossValues,'''...
            char(myDatasets(datasetCounter)) ...
            ''');']);
        eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'WinValues,'''...
            char(myDatasets(datasetCounter)) ...
            ''');']);
        for instanceCounter = 1:size(errorMeasureLossValues(errorCounter,datasetCounter,:),3)
            eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'LossValues,'''...
                [ ',' num2str(errorMeasureLossValues(errorCounter,datasetCounter,instanceCounter))]   ...
                ''');']);
            eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'WinValues,'''...
                [ ',' num2str(errorMeasureWinValues(errorCounter,datasetCounter,instanceCounter))]   ...
                ''');']);
        end
        eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'LossValues,'''...
            '\n' ...
            ''');']);  % add new line at the end of every dataset
        eval(['fprintf(myErrorMeasure' num2str(errorCounter) 'WinValues,'''...
            '\n' ...
            ''');']);  % add new line at the end of every dataset
    end
    
    % lastly close the opened files
    eval(['fclose(myErrorMeasure' num2str(errorCounter) 'LossValues);']);
    eval(['fclose(myErrorMeasure' num2str(errorCounter) 'WinValues);']);
end


