%LOSO  Leave_One_Set_Out crossvalidation
%
%		[E,C,D] = LOSO(A,CLASSF,LABLISTNAME)
%		[E,C,D] = LOSO(A,CLASSF,SET_LABELS)
%		[E,C,D] = LOSO(A,CLASSF,SET_LABELS,SET_LABLIST)
%
% INPUT
%   A            Dataset
%   CLASSF       Untrained classifier
%   LABLISTNAME  Name of label list in case of multiple labeling
%   SET_LABELS   Set of labels for objects in A
%   SET_LABLIST  Order and selection of labels in SET_LABELS
%
% OUTPUT
%   E            Classification error
%   C            Array with numbers of erroneaously clasified objects
%                per label (vertically) and per class (horizontally)
%   D            Classification matrix of classified objects
%                (order may be different from A)
%
% DESCRIPTION
% In crossvalidation it may be desired that sets of corresponding objects
% (e.g. pixels from the same image) are all together in the training set or
% in the test set. This might be enabled by adding an additional labeling to
% the dataset A (see ADDLABELS) corresponding to the sets and running LOSO
% with the corresponding LABLISTNAME.
% Alternatively, the set labels may be supplied in the call. In SET_LABLIST
% a ranking of the used labels can be supplied that will be used in C.
% In case SET_LABLIST does not contain all set labels used in SET_LABELS
% LOSO will only test the set labels given in SET_LABLIST and thereby 
% perform areduced crosvalidation.
%
% The reported error E identical to E = sum(C)./classsizes(D)*getprior(A)';
% By confmat(D) a confusion matrix can be visualised.
%
% SEE ALSO (<a href="http://37steps.com/prtools">PRTools Guide</a>)
% DATASETS, MAPPINGS, TESTC, CONFMAT

% Copyright: R.P.W. Duin, r.p.w.duin@37steps.com
% Faculty EWI, Delft University of Technology
% P.O. Box 5031, 2600 GA Delft, The Netherlands

function [e,err,d] = loso(a,classf,set_lab,set_lablist)

if nargin < 4, testfun = testc; end

[curn,curlist] = curlablist(a);  % present lablist number
if nargin == 3 & size(set_lab,1) == 1
	if (isstr(set_lab) & strcmp(set_lab,curlist)) | set_lab == curn
		error('The desired LOO set should differ from the present lablist')
	end
	lablistname = set_lab;
	b = changelablist(a,lablistname);
elseif nargin == 3
	b = addlabels(a,set_lab); 
else
	b = addlablist(a,set_lablist);
	b = setnlab(b,renumlab(set_lab,set_lablist));
end
b = setlablist(b);  % throw out empty sets

nset = getsize(b,3);  
S = [1:nset 0];
c = getsize(a,3);
s = sprintf('Testing %i sets: ',nset);
prwaitbar(nset,s);
err = zeros(nset,c);
d = [];
N = 0;
for j=1:nset
	prwaitbar(nset,j,[s int2str(j)]);
	T = S;
	T(j) = [];
	x = changelablist(seldat(b,T),curlist);
	y = changelablist(seldat(b,j),curlist); 
	N = N+size(y,1);
	if ~isempty(y)
		dd = y*(x*classf);
		dd = prdataset(dd);
		[ee,err(j,:)] = testd(dd);
		d = [d;dd];
	end
end
prwaitbar(0);
e = sum(err)./classsizes(d);
e = e*getprior(a)';