% function analysisCrossValidation
clear
t_90margin_df4 = 2.1318;

%% Plotting the AUC comparisons
numFeaActiveBasis = 40;
% run 1~5: 10 positive training examples
count = 1; 
auc10_adaboost = zeros(1,5);
auc10_eigenface = zeros(1,5);
auc10_activebasisAdapt =zeros(1,5);
for iRun = 1:5
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc10_adaboost(count) = testAUC(end);
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc10_eigenface(count) = testAUC(2);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc10_activebasisAdapt(count) = testAUC(numFeaActiveBasis);
    count = count + 1;
end

% run 6~10: 20 positive training examples
count = 1; 
auc20_adaboost = zeros(1,5);
auc20_eigenface = zeros(1,5);
auc20_activebasisAdapt =zeros(1,5);
for iRun = 6:10
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc20_adaboost(count) = testAUC(end);
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc20_eigenface(count) = testAUC(2);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc20_activebasisAdapt(count) = testAUC(numFeaActiveBasis);
    count = count + 1;
end

% run 11~15: 40 positive training examples
count = 1; 
auc40_adaboost = zeros(1,5);
auc40_eigenface = zeros(1,5);
auc40_adaboostMax = zeros(1,5);
auc40_activebasisAdapt =zeros(1,5);
for iRun = 11:15
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc40_adaboost(count) = testAUC(end);
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc40_eigenface(count) = testAUC(2);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc40_activebasisAdapt(count) = testAUC(numFeaActiveBasis);
    count = count + 1;
end

% run 16~20: 80 positive training examples
count = 1; 
auc80_adaboost = zeros(1,5);
auc80_eigenface = zeros(1,5);
auc80_activebasisAdapt =zeros(1,5);
for iRun = 16:20
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc80_adaboost(count) = testAUC(end);
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc80_eigenface(count) = testAUC(2);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc80_activebasisAdapt(count) = testAUC(numFeaActiveBasis);
    count = count + 1;
end


% run 21~25: 5 positive training examples
count = 1; 
auc5_adaboost = zeros(1,5);
auc5_eigenface = zeros(1,5);
auc5_activebasisAdapt =zeros(1,5);
for iRun = 21:25
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc5_adaboost(count) = testAUC(end);
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc5_eigenface(count) = testAUC(2);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc5_activebasisAdapt(count) = testAUC(numFeaActiveBasis);
    count = count + 1;
end

curve1 = [mean(auc5_activebasisAdapt), mean(auc10_activebasisAdapt), mean(auc20_activebasisAdapt), mean(auc40_activebasisAdapt), mean(auc80_activebasisAdapt)];
err1 = [std(auc5_activebasisAdapt), std(auc10_activebasisAdapt), std(auc20_activebasisAdapt), std(auc40_activebasisAdapt), std(auc80_activebasisAdapt)];
err1 = err1 * t_90margin_df4 / sqrt(5);
curve2 = [mean(auc5_adaboost), mean(auc10_adaboost), mean(auc20_adaboost), mean(auc40_adaboost), mean(auc80_adaboost)];
err2 = [std(auc5_adaboost), std(auc10_adaboost), std(auc20_adaboost), std(auc40_adaboost), std(auc80_adaboost)];
err2 = err2 * t_90margin_df4 / sqrt(5);
curve3 = [mean(auc5_eigenface), mean(auc10_eigenface), mean(auc20_eigenface), mean(auc40_eigenface), mean(auc80_eigenface)];
err3 = [std(auc5_eigenface), std(auc10_eigenface), std(auc20_eigenface), std(auc40_eigenface), std(auc80_eigenface)];
err3 = err3 * t_90margin_df4 / sqrt(5);

% Plot the three curves of Test AUC
%   with 90% confidence bounds
figure;

h1 = errorbar([5,10,20,40,80],curve1,err1,'b-s','LineWidth',3);hold on 
h2 = errorbar([5,10,20,40,80],curve2,err2,'r-.','LineWidth',3);
h3 = errorbar([5,10,20,40,80],curve3,err3,'k-','LineWidth',2);


set(gca,'FontSize',12);
set(gca,'ylim',[0.85 1]);
set(gca,'xlim',[0 85]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h1 h2 h3],{'Active Basis Adaptive','Adaboost on MAX1','PCA'},'FontSize',16,0);
title('AUC vs. the number of positive training examples','FontSize',18);
set(gcf, 'PaperPositionMode', 'auto');
print -depsc auc_comparison_new.eps
print -dpng auc_comparison_new.png

% plot only the mean AUC (without the error bounds)
figure;
hold on;
h1 = plot([5,10,20,40,80],curve1,'b-s','LineWidth',3);
h2 = plot([5,10,20,40,80],curve2,'r-.','LineWidth',3);
h3 = plot([5,10,20,40,80],curve3,'k-','LineWidth',3);
set(gcf, 'PaperPositionMode', 'auto');
set(gca,'FontSize',12);
set(gca,'ylim',[0.85 1]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h1,h2,h3],{'Active Basis Adaptive','Adaboost on MAX1','PCA'},'Location','SouthEast','FontSize',16);
title('AUC vs. the number of positive training examples','FontSize',18);
print -depsc auc_comparison_mean_new.eps
print -dpng auc_comparison_mean_new.png


% *****************************************************************
%% Plotting AUC against #feature
maxNF = 80; % maximum # of features

auc40_activebasis = zeros(5,maxNF);
count = 1;
for iRun = 11:15
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc40_activebasis(count,:) = testAUC(:);
    count = count + 1;
end
curve14 = mean(auc40_activebasis,1);
err14 = zeros(1,maxNF);
for iF = 1:maxNF
    err14(iF) = std(auc40_activebasis(:,iF));
end
err14 = err14 * t_90margin_df4 / sqrt(5);

auc5_activebasis = zeros(5,maxNF);
count = 1;
for iRun = 21:25
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc5_activebasis(count,:) = testAUC(:);
    count = count + 1;
end
curve17 = mean(auc5_activebasis,1);
err17 = zeros(1,maxNF);
for iF = 1:maxNF
    err17(iF) = std(auc5_activebasis(:,iF));
end
err17 = err17 * t_90margin_df4 / sqrt(5);

figure;
errorbar(1:3:maxNF,curve14(1:3:maxNF),err14(1:3:maxNF),'b-','LineWidth',2); hold on
errorbar(1:3:maxNF,curve17(1:3:maxNF),err17(1:3:maxNF),'r-.','LineWidth',3);
set(gca,'FontSize',12);
set(gca,'ylim',[0.8 1]);
set(gcf, 'PaperPositionMode', 'auto');
set(get(gca,'XLabel'),'String','# basis elements','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend({'40 examples', '5 examples'},'Location','SouthEast','FontSize',20);
title('AUC vs the number of basis elements','FontSize',18);
print -depsc auc_features.eps
print -dpng auc_features.png

% only plot the mean curve
h = figure;
hold on;
plot(1:3:maxNF,curve14(1:3:maxNF),'k-s','LineWidth',2);
plot(1:3:maxNF,curve17(1:3:maxNF),'k-.','LineWidth',3);
set(gca,'FontSize',12);
set(gca,'ylim',[0.8 1]);
set(gcf, 'PaperPositionMode', 'auto');
set(get(gca,'XLabel'),'String','# basis elements','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend({'40 examples','5 examples'},'Location','SouthEast','FontSize',20);
title('AUC vs the number of basis elements','FontSize',18);
print -depsc auc_features_mean.eps
print -dpng auc_features_mean.png


