% function analysisCrossValidation
clear
t_90margin_df4 = 2.1318;

%% Plotting the AUC comparisons

% run 1~5: 5 positive training examples
count = 1;
auc5_adaboost = zeros(1,5);
auc5_pca = zeros(1,5);
auc5_adaboostMax = zeros(1,5);
auc5_activebasisAdapt =zeros(1,5);
for iRun = 1:5
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc5_pca(count) = testAUC(2);
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc5_adaboostMax(count) = testAUC(end);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc5_activebasisAdapt(count) = testAUC(end);
    count = count + 1;
end

% run 6~10: 10 positive training examples
count = 1; 
auc10_adaboost = zeros(1,5);
auc10_pca = zeros(1,5);
auc10_adaboostMax = zeros(1,5);
auc10_activebasisAdapt =zeros(1,5);
for iRun = 6:10
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc10_pca(count) = testAUC(2);
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc10_adaboostMax(count) = testAUC(end);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc10_activebasisAdapt(count) = testAUC(end);
    count = count + 1;
end

% run 11~15: 20 positive training examples
count = 1; 
auc20_adaboost = zeros(1,5);
auc20_pca = zeros(1,5);
auc20_adaboostMax = zeros(1,5);
auc20_activebasisAdapt =zeros(1,5);
for iRun = 11:15
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc20_pca(count) = testAUC(2);
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc20_adaboostMax(count) = testAUC(end);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc20_activebasisAdapt(count) = testAUC(end);
    count = count + 1;
end

% run 16~20: 40 positive training examples
count = 1; 
auc40_adaboost = zeros(1,5);
auc40_pca = zeros(1,5);
auc40_adaboostMax = zeros(1,5);
auc40_activebasisAdapt =zeros(1,5);
for iRun = 16:20
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc40_pca(count) = testAUC(2);
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc40_adaboostMax(count) = testAUC(end);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc40_activebasisAdapt(count) = testAUC(end);
    count = count + 1;
end

% run 21~25: 80 positive training examples
count = 1; 
auc80_adaboost = zeros(1,5);
auc80_pca = zeros(1,5);
auc80_adaboostMax = zeros(1,5);
auc80_activebasisAdapt =zeros(1,5);
for iRun = 21:25
    load(sprintf('pca/pca_result_run%d.mat',iRun),'testAUC');
    auc80_pca(count) = testAUC(2);
    load(sprintf('adaboostMAXC/out_adaboost_run%d.mat',iRun),'testAUC');
    auc80_adaboostMax(count) = testAUC(end);
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc80_activebasisAdapt(count) = testAUC(end);
    count = count + 1;
end

curve3 = [mean(auc5_pca), mean(auc10_pca), mean(auc20_pca), mean(auc40_pca), mean(auc80_pca)];
err3 = [std(auc5_pca), std(auc10_pca), std(auc20_pca), std(auc40_pca), std(auc80_pca)];
err3 = err3 * t_90margin_df4 / sqrt(5);
curve4 = [mean(auc5_adaboostMax), mean(auc10_adaboostMax), mean(auc20_adaboostMax), mean(auc40_adaboostMax), mean(auc80_adaboostMax)];
err4 = [std(auc5_adaboostMax), std(auc10_adaboostMax), std(auc20_adaboostMax), std(auc40_adaboostMax), std(auc80_adaboostMax)];
err4 = err4 * t_90margin_df4 / sqrt(5);
err4(1:2) = 0;
curve5 = [mean(auc5_activebasisAdapt), mean(auc10_activebasisAdapt), mean(auc20_activebasisAdapt), mean(auc40_activebasisAdapt), mean(auc80_activebasisAdapt)];
err5 = [std(auc5_activebasisAdapt), std(auc10_activebasisAdapt), std(auc20_activebasisAdapt), std(auc40_activebasisAdapt), std(auc80_activebasisAdapt)];
err5 = err5 * t_90margin_df4 / sqrt(5);

% Plot the three curves of Test AUC
%   with 95% confidence bounds
figure;
handle_curve5 = errorbar(([5,10,20,40,80]),curve5,err5,'b-s','LineWidth',2); hold on
handle_curve3 = errorbar(([5,10,20,40,80]),curve3,err3,'k-','LineWidth',3);
handle_curve4 = errorbar(([5,10,20,40,80]),curve4,err4,'r-.','LineWidth',3);

set(gca,'FontSize',12);
set(gca,'xlim',[0 (81)]);
set(gca,'ylim',[0.985 1]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([handle_curve5 handle_curve3 handle_curve4], {'Active Basis Adaptive','PCA','Adaboost on Max1'},'FontSize',16,0);
title('AUC vs. the number of positive training examples','FontSize',18);
set(gcf, 'PaperPositionMode', 'auto');
print -depsc auc_comparison.eps
print -dpng auc_comparison.png


% plot only the mean AUC (without the error bounds)
figure;
hold on;
h5 = plot([5,10,20,40,80],curve5,'b-s','LineWidth',3);
h3 = plot([5,10,20,40,80],curve3,'k-','LineWidth',3);
h4 = plot([5,10,20,40,80],curve4,'r-.','LineWidth',3);
set(gcf, 'PaperPositionMode', 'auto');
set(gca,'FontSize',12);
set(gca,'xlim',[0 81]);
set(gca,'ylim',[0.98 1]);
set(get(gca,'XLabel'),'String','# training positives','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h5,h3,h4],{'Active Basis Adaptive','PCA','Adaboost on MAX1'},'Location','SouthEast','FontSize',20);
title('AUC vs. the number of positive training examples','FontSize',18);
print -depsc auc_comparison_mean.eps
print -dpng auc_comparison_mean.png


% *****************************************************************
%% Plotting AUC against #feature
maxNF = 150; % maximum # of features

auc40_activebasis = zeros(5,maxNF);
count = 1;
for iRun = 16:20
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc40_activebasis(count,:) = testAUC(:);
    count = count + 1;
end
curve14 = mean(auc40_activebasis,1);
err14 = zeros(1,maxNF);
for iF = 1:maxNF
    err14(iF) = std(auc40_activebasis(:,iF));
end
err14 = err14 * t_90margin_df4 / sqrt(5);

auc5_activebasis = zeros(5,maxNF);
count = 1;
for iRun = 1:5
    load(sprintf('sigmoidAdaptive/out_run%d.mat',iRun),'testAUC');
    auc5_activebasis(count,:) = testAUC(:);
    count = count + 1;
end
curve17 = mean(auc5_activebasis,1);
err17 = zeros(1,maxNF);
for iF = 1:maxNF
    err17(iF) = std(auc5_activebasis(:,iF));
end
err17 = err17 * t_90margin_df4 / sqrt(5);

figure;
h14 = errorbar(1:3:maxNF,curve14(1:3:maxNF),err14(1:3:maxNF),'b-','LineWidth',2); hold on
h17 = errorbar(1:3:maxNF,curve17(1:3:maxNF),err17(1:3:maxNF),'r-.','LineWidth',3);
set(gca,'FontSize',12);
set(gca,'xlim',[-.1 81]);
set(gca,'ylim',[0.95 1]);
set(gcf, 'PaperPositionMode', 'auto');
set(get(gca,'XLabel'),'String','# basis elements','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend([h14 h17],{'40 examples','5 examples'},'Location','SouthEast','FontSize',20);
title('AUC vs the number of basis elements','FontSize',18);
print -depsc auc_features.eps
print -dpng auc_features.png

% only plot the mean curve
h = figure;
hold on;
plot(1:maxNF,curve14,'b-','LineWidth',3);
plot(1:maxNF,curve17,'r-.','LineWidth',3);
set(gca,'FontSize',12);
set(gca,'ylim',[0.95 1]);
set(gcf, 'PaperPositionMode', 'auto');
set(get(gca,'XLabel'),'String','# basis elements','FontSize',18);
set(get(gca,'YLabel'),'String','testing AUC','FontSize',18);
legend({'40 examples','5 examples'},'Location','SouthEast','FontSize',20);
title('AUC vs the number of basis elements','FontSize',18);
print -depsc auc_features_mean.eps
print -dpng auc_features_mean.png

