% function AnalyzeParameterTuning

clear
t_90margin_df4 = 2.1318;

%% Plotting the AUC comparisons
numFeaActiveBasis = 40;
% run 1~5: 10 positive training examples
count = 1;
auc10_before = zeros(1,5);
auc10_afterAdaboost = zeros(1,5);
auc10_afterSVM = zeros(1,5);
auc10_afterMNR = zeros(1,5);
for iRun = 6:10
    load(sprintf('out_run%d.mat',iRun),'testAUC');
    auc10_before(count) = testAUC(numFeaActiveBasis);
    load(sprintf('adjusted_adaboost_run%d.mat',iRun),'testAUC');
    auc10_afterAdaboost(count) = testAUC;
    load(sprintf('adjusted_svm_run%d.mat',iRun),'testAUC');
    auc10_afterSVM(count) = testAUC;
    load(sprintf('adjusted_lr_run%d.mat',iRun),'testAUC');
    auc10_afterMNR(count) = testAUC;
    count = count + 1;
end

% run 6~10: 20 positive training examples
count = 1; 
auc20_before = zeros(1,5);
auc20_afterAdaboost = zeros(1,5);
auc20_afterSVM = zeros(1,5);
auc20_afterMNR = zeros(1,5);
for iRun = 11:15
    load(sprintf('out_run%d.mat',iRun),'testAUC');
    auc20_before(count) = testAUC(numFeaActiveBasis);
    load(sprintf('adjusted_adaboost_run%d.mat',iRun),'testAUC');
    auc20_afterAdaboost(count) = testAUC;
    load(sprintf('adjusted_svm_run%d.mat',iRun),'testAUC');
    auc20_afterSVM(count) = testAUC;
    load(sprintf('adjusted_lr_run%d.mat',iRun),'testAUC');
    auc20_afterMNR(count) = testAUC;
    count = count + 1;
end

% run 11~15: 40 positive training examples
count = 1;
auc40_before = zeros(1,5);
auc40_afterAdaboost = zeros(1,5);
auc40_afterSVM = zeros(1,5);
auc40_afterMNR = zeros(1,5);
for iRun = 16:20
    load(sprintf('out_run%d.mat',iRun),'testAUC');
    auc40_before(count) = testAUC(numFeaActiveBasis);
    load(sprintf('adjusted_adaboost_run%d.mat',iRun),'testAUC');
    auc40_afterAdaboost(count) = testAUC;
    load(sprintf('adjusted_svm_run%d.mat',iRun),'testAUC');
    auc40_afterSVM(count) = testAUC;
    load(sprintf('adjusted_lr_run%d.mat',iRun),'testAUC');
    auc40_afterMNR(count) = testAUC;
    count = count + 1;
end

% run 16~20: 80 positive training examples
count = 1; 
auc80_before = zeros(1,5);
auc80_afterAdaboost = zeros(1,5);
auc80_afterSVM = zeros(1,5);
auc80_afterMNR = zeros(1,5);
for iRun = 21:25
    load(sprintf('out_run%d.mat',iRun),'testAUC');
    auc80_before(count) = testAUC(numFeaActiveBasis);
    load(sprintf('adjusted_adaboost_run%d.mat',iRun),'testAUC');
    auc80_afterAdaboost(count) = testAUC;
    load(sprintf('adjusted_svm_run%d.mat',iRun),'testAUC');
    auc80_afterSVM(count) = testAUC;
    load(sprintf('adjusted_lr_run%d.mat',iRun),'testAUC');
    auc80_afterMNR(count) = testAUC;
    count = count + 1;
end


% run 21~25: 5 positive training examples
count = 1; 
auc5_before = zeros(1,5);
auc5_afterAdaboost = zeros(1,5);
auc5_afterSVM = zeros(1,5);
auc5_afterMNR = zeros(1,5);
for iRun = 1:5
    load(sprintf('out_run%d.mat',iRun),'testAUC');
    auc5_before(count) = testAUC(numFeaActiveBasis);
    load(sprintf('adjusted_adaboost_run%d.mat',iRun),'testAUC');
    auc5_afterAdaboost(count) = testAUC;
    load(sprintf('adjusted_svm_run%d.mat',iRun),'testAUC');
    auc5_afterSVM(count) = testAUC;
    load(sprintf('adjusted_lr_run%d.mat',iRun),'testAUC');
    auc5_afterMNR(count) = testAUC;
    count = count + 1;
end

curve1 = [mean(auc5_before), mean(auc10_before), mean(auc20_before), mean(auc40_before), mean(auc80_before)];
err1 = [std(auc5_before), std(auc10_before), std(auc20_before), std(auc40_before), std(auc80_before)];
err1 = err1 * t_90margin_df4 / sqrt(5);
curve2 = [mean(auc5_afterAdaboost), mean(auc10_afterAdaboost), mean(auc20_afterAdaboost), mean(auc40_afterAdaboost), mean(auc80_afterAdaboost)];
err2 = [std(auc5_afterAdaboost), std(auc10_afterAdaboost), std(auc20_afterAdaboost), std(auc40_afterAdaboost), std(auc80_afterAdaboost)];
err2 = err2 * t_90margin_df4 / sqrt(5);
curve3 = [mean(auc5_afterSVM), mean(auc10_afterSVM), mean(auc20_afterSVM), mean(auc40_afterSVM), mean(auc80_afterSVM)];
err3 = [std(auc5_afterSVM), std(auc10_afterSVM), std(auc20_afterSVM), std(auc40_afterSVM), std(auc80_afterSVM)];
err3 = err3 * t_90margin_df4 / sqrt(5);
curve4 = [mean(auc5_afterMNR), mean(auc10_afterMNR), mean(auc20_afterMNR), mean(auc40_afterMNR), mean(auc80_afterMNR)];
err4 = [std(auc5_afterMNR), std(auc10_afterMNR), std(auc20_afterMNR), std(auc40_afterMNR), std(auc80_afterMNR)];
err4 = err4 * t_90margin_df4 / sqrt(5);


% Plot the three curves of Test AUC
%   with 90% confidence bounds
figure;
h1 = errorbar([5,10,20,40,80],curve1,err1,'b-s','LineWidth',3); hold on 
h2 = errorbar([5,10,20,40,80],curve2,err2,'r-.','LineWidth',3);
h3 = errorbar([5,10,20,40,80],curve3,err3,'k--','LineWidth',3);
h4 = errorbar([5,10,20,40,80],curve4,err4,'y','LineWidth',3);

set(gca,'FontSize',12);
set(gca,'ylim',[0.99 1]);
set(gca,'xlim',[0 85]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h1 h2 h3 h4],{'active basis','active basis + AdaBoost','active basis + SVM','active basis + logistic regression'},'FontSize',14,0);
title('discriminative adjustments','FontSize',18);
set(gcf, 'PaperPositionMode', 'auto');
print -depsc analyze_adjustment_all.eps
print -dpng analyze_adjustment_all.png

% plot only the mean AUC (without the error bounds)
%{
figure;
hold on;
h1 = plot([5,10,20,40,80],curve1,'b-s','LineWidth',3);
h2 = plot([5,10,20,40,80],curve2,'r-.','LineWidth',3);
set(gcf, 'PaperPositionMode', 'auto');
set(gca,'FontSize',12);
set(gca,'ylim',[0.95 1]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h1 h2],{'before adjustment','after adjustment'},'FontSize',16,0);
title('before and after adjusting feature weights','FontSize',18);
print -depsc analyze_adjustment_simple.eps
print -dpng analyze_adjustment_simple.png
%}



