diff --git a/CSRKDApredict.m b/CSRKDApredict.m new file mode 100644 index 0000000..3f9698c --- /dev/null +++ b/CSRKDApredict.m @@ -0,0 +1,71 @@ +function [accuracy,predictlabel,elapse] = CSRKDApredict(fea, gnd, model) +% SRKDApredict: Spectral Regression Kernel Discriminant Analysis Prediction +% SRKDApredict use SRKDA as a classifier. It used the nearest +% center rule in the SRKDA subspace for classification. +% +% [predictlabel,accuracy,elapse] = SRKDApredict(fea, gnd, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% gnd - Label vector of fea. +% model - model trained by SRKDAtrain.m +% +% Output: +% +% accuracy - classification accuracy +% predictlabel - predict label for fea +% elapse - running time. +% +% Examples: +% +% +% See also SRKDAtrain, KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [2] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [3] V. Sindhwani, P. Niyogi, M. Belkin, "Beyond the Point Cloud: from +% Transductive to Semi-supervised Learning", ICML 2005. +% +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 8000; % You can change this number based on your memory. + + + +nTrain = size(model.Landmark,1); +nTest = size(fea,1); +nBlock = ceil(MAX_MATRIX_SIZE*MAX_MATRIX_SIZE/nTrain); +Embed_Test = zeros(nTest,size(model.projection,2)); +for i = 1:ceil(nTest/nBlock) + if i == ceil(nTest/nBlock) + smpIdx = (i-1)*nBlock+1:nTest; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.Landmark,model.options); + Embed_Test(smpIdx,:) = KTest*model.projection; + clear KTest; +end + +D = EuDist2(Embed_Test,model.ClassCenter,0); +[dump, idx] = min(D,[],2); +predictlabel = model.ClassLabel(idx); + +accuracy = 1 - length(find(predictlabel-gnd))/nTest; + + + + + diff --git a/CSRKDAtrain.m b/CSRKDAtrain.m new file mode 100644 index 0000000..4619574 --- /dev/null +++ b/CSRKDAtrain.m @@ -0,0 +1,157 @@ +function [model] = CSRKDAtrain(feaLabel, gnd, options, feaTrain, Landmark) +% SRKDAtrain: Training Spectral Regression Kernel Discriminant Analysis +% +% [model] = SRKDAtrain(feaLabel, gnd) +% [model] = SRKDAtrain(feaLabel, gnd, options) +% [model] = SRKDAtrain(feaLabel, gnd, options, feaTrain) +% +% Input: +% +% feaLabel - data matrix. Each row is a data point. +% gnd - Label vector of feaLabel. +% feaTrain - data matrix. This input is optional. If provided, +% SRKDA will be performed in a semi-supervised way. +% feaTrain will be the training data without label. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% KernelType - Choices are: +% 'Gaussian' - e^{-(|x-y|^2)/2t^2} +% 'Polynomial' - (x'*y)^d +% 'PolyPlus' - (x'*y+1)^d +% 'Linear' - x'*y +% +% t - parameter for Gaussian +% d - parameter for Poly +% +% ReguAlpha - regularization paramter for regression +% Default 0.01 +% +% The following fields are only useful when feaTrain is provided. +% +% ReguBeta - Paramter for manifold regularizer +% Default 1 +% Fields for W - Please see ConstructW.m for detailed options. +% +% LaplacianNorm = 0 | 1 (0 for un-normalized and 1 for +% normalized graph laplacian) +% Default: 0 +% LaplacianDegree - power of the graph Laplacian to use as +% the graph regularizer +% Default: 1 +% +% +% +% +% Output: +% model - used for SRKDApredict.m +% +% +% Examples: +% +% +% +% See also KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [2] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [3] V. Sindhwani, P. Niyogi, M. Belkin, "Beyond the Point Cloud: from +% Transductive to Semi-supervised Learning", ICML 2005. +% +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +if ~exist('options','var') + options = []; +end + +if ~isfield(options,'KernelType') + options.ReguAlpha = 'Gaussian'; +end + +if ~isfield(options,'t') + nSmp = size(feaLabel,1); + idx=randperm(nSmp); + if nSmp > 3000 + D = EuDist2(feaLabel(idx(1:3000),:)); + else + D = EuDist2(feaLabel); + end + options.t = mean(mean(D)); +end + +options.ReguType = 'Ridge'; +if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.01; +end + +if ~isfield(options,'BasisNum') + options.BasisNum = 500; +end + +if ~isfield(options,'MaxIter') + options.MaxIter = 10; +end + +model.options = options; + +nSmp = size(feaLabel,1); + +ClassLabel = unique(gnd); +model.ClassLabel = ClassLabel; +nClass = length(ClassLabel); + +% Response Generation +rand('state',0); +Y = rand(nClass,nClass); +Z = zeros(nSmp,nClass); +for i=1:nClass + idx = find(gnd==ClassLabel(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); +end +Z(:,1) = ones(nSmp,1); +[Y,R] = qr(Z,0); +Y(:,1) = []; + +if exist('Landmark','var') + model.Landmark = Landmark; +else + if exist('feaTrain','var') + fea = [feaLabel;feaTrain]; + if size(fea,1) < options.BasisNum + error('The data is too small, use SRKDA directly!'); + end + [dump, model.Landmark] = litekmeans(fea, options.BasisNum, 'MaxIter', options.MaxIter); + else + if size(feaLabel,1) < options.BasisNum + error('The data is too small, use SRKDA directly!'); + end + [dump, model.Landmark] = litekmeans(feaLabel, options.BasisNum, 'MaxIter', options.MaxIter); + end +end + +K = constructKernel(feaLabel,model.Landmark,options); +options.RemoveMean = 1; +model.projection = SR(options, Y, K); +Embed_Train = K*model.projection; + +ClassCenter = zeros(nClass,size(Embed_Train,2)); +for i = 1:nClass + feaTmp = Embed_Train(gnd == ClassLabel(i),:); + ClassCenter(i,:) = mean(feaTmp,1); +end +model.ClassCenter = ClassCenter; + + + + + diff --git a/EMR.m b/EMR.m new file mode 100644 index 0000000..4094819 --- /dev/null +++ b/EMR.m @@ -0,0 +1,126 @@ +function [score, model] = EMR(data,y0,opts) +% [score, model] = EMR(data,y0,opts): Efficient Manifold Ranking +% Input: +% - data: the data matrix of size nSmp x nFea, where each row is a sample +% point +% - y0: the initial query vector, e.g., query item =1 and the other all 0; +% +% opts: options for this algorithm +% - p: the number of landmarks picked (default 1000) +% - r: the number of nearest landmarks for representation (default 5) +% - a: weight in manifold ranking, score = (I - aS)^(-1)y, default 0.99 +% - mode: landmark selection method, currently support +% - 'kmeans': use centers of clusters generated by kmeans (default) +% - 'random': use randomly sampled points from the original +% data set +% The following parameters are effective ONLY in mode 'kmeans' +% - kmNumRep: the number of replicates for initial kmeans (default 1) +% - kmMaxIter: the maximum number of iterations for initial kmeans (default 5) +% +% Output: +% - score: the ranking scores for each point +% - model: the learned model for out-of-sample retrieval +% +% Usage: +% +% See: http://www.zjucadcg.cn/dengcai/Data/ReproduceExp.html#EMR +% +%Reference: +% +% Bin Xu, Jiajun Bu, Chun Chen, Deng Cai, Xiaofei He, Wei Liu, Jiebo +% Luo, "Efficient Manifold Ranking for Image Retrieval",in Proceeding of +% the 34th International ACM SIGIR Conference on Research and +% Development in Information Retrieval (SIGIR), 2011, pp. 525-534. +% +% version 2.0 --Feb./2012 +% version 1.0 --Sep./2010 +% +% Written by Bin Xu (binxu986 AT gmail.com) +% Deng Cai (dengcai AT gmail.com) + + +% Set and parse parameters +if (~exist('opts','var')) + opts = []; +end + +p = 1000; +if isfield(opts,'p') + p = opts.p; +end + +r = 5; +if isfield(opts,'r') + r = opts.r; +end + +a = 0.99; +if isfield(opts,'a') + a = opts.a; +end + +mode = 'kmeans'; +if isfield(opts,'mode') + mode = opts.mode; +end + +nSmp =size(data,1); + +% Landmark selection +if strcmp(mode,'kmeans') + kmMaxIter = 5; + if isfield(opts,'kmMaxIter') + kmMaxIter = opts.kmMaxIter; + end + kmNumRep = 1; + if isfield(opts,'kmNumRep') + kmNumRep = opts.kmNumRep; + end + [dump,landmarks]=litekmeans(data,p,'MaxIter',kmMaxIter,'Replicates',kmNumRep); + clear kmMaxIter kmNumRep +elseif strcmp(mode,'random') + indSmp = randperm(nSmp); + landmarks = data(indSmp(1:p),:); + clear indSmp +else + error('mode does not support!'); +end + +model.landmarks = landmarks; +model.a = a; +model.r = r; + +% Z construction +D = EuDist2(data,landmarks); +dump = zeros(nSmp,r); +idx = dump; +for i = 1:r + [dump(:,i),idx(:,i)] = min(D,[],2); + temp = (idx(:,i)-1)*nSmp+[1:nSmp]'; + D(temp) = 1e100; +end +dump = bsxfun(@rdivide,dump,dump(:,r)); +dump = 0.75 * (1 - dump.^2); +Gsdx = dump; +Gidx = repmat([1:nSmp]',1,r); +Gjdx = idx; +Z=sparse(Gidx(:),Gjdx(:),Gsdx(:),nSmp,p); + +model.Z = Z'; + +% Efficient Ranking +feaSum = full(sum(Z,1)); +D = Z*feaSum'; +D = max(D, 1e-12); +D = D.^(-.5); +H = spdiags(D,0,nSmp,nSmp)*Z; + +C = speye(p); +A = H'*H-(1/a)*C; + +tmp = H'*y0; +tmp = A\tmp; +score = y0 - H*tmp; + + + diff --git a/EMRtest.m b/EMRtest.m new file mode 100644 index 0000000..43463c6 --- /dev/null +++ b/EMRtest.m @@ -0,0 +1,69 @@ +function [score] = EMRtest(x,model) +% [score] = EMRtest(x,model): Efficient Manifold Ranking for out-of-sample +% retrieval +% Input: +% - x: the query point, row vector. +% - model: the model learned by EMR +% +% Output: +% - score: the ranking scores for each point in the database +% +% Usage: +% +% See: http://www.zjucadcg.cn/dengcai/Data/Examples.html#EMR +% +%Reference: +% +% Bin Xu, Jiajun Bu, Chun Chen, Deng Cai, Xiaofei He, Wei Liu, Jiebo +% Luo, "Efficient Manifold Ranking for Image Retrieval",in Proceeding of +% the 34th International ACM SIGIR Conference on Research and +% Development in Information Retrieval (SIGIR), 2011, pp. 525-534. +% +% version 2.0 --Feb./2012 +% version 1.0 --Sep./2010 +% +% Written by Bin Xu (binxu986 AT gmail.com) +% Deng Cai (dengcai AT gmail.com) + + + + +r = model.r; +a = model.a; +p = size(model.landmarks,1); + + +% Z construction +D = EuDist2(x,model.landmarks); +[dump,idx] = sort(D); +dump = dump(1:r)/dump(r); +dump = 0.75 * (1 - dump.^2); +z = sparse(idx(1:r),1,dump,p,1); + +Z = [model.Z z]; +Z = Z'; + +nSmp =size(Z,1); + + +y0 = zeros(nSmp,1); +y0(end) = 1; + +% Efficient Ranking +feaSum = full(sum(Z,1)); +D = Z*feaSum'; +D = max(D, 1e-12); +D = D.^(-.5); +H = spdiags(D,0,nSmp,nSmp)*Z; + +C = speye(p); +A = H'*H-(1/a)*C; + +tmp = H'*y0; +tmp = A\tmp; +score = y0 - H*tmp; + +score(end) = []; + + + diff --git a/Eigenmap.m b/Eigenmap.m new file mode 100644 index 0000000..c2643c9 --- /dev/null +++ b/Eigenmap.m @@ -0,0 +1,117 @@ +function [Y, eigvalue] = Eigenmap(W, ReducedDim, bEigs) +%function [Y, eigvalue] = Eigenmap(W, ReducedDim, bEigs) +% +% W - the affinity matrix. +% ReducedDim - the dimensionality of the reduced subspace. +% bEigs - whether to use eigs to speed up. If not +% specified, this function will automatically +% decide based on the size of W. +% +% version 2.1 --November/2011 +% version 2.0 --May/2009 +% version 1.5 --Dec./2005 +% version 1.0 --Aug./2003 +% +% Written by Deng Cai (dengcai AT gmail.com) + +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + + +[row,col] = size(W); +if row ~= col + error('W must square matrix!!'); +end + +nSmp = row; + +if ~exist('ReducedDim','var') + ReducedDim = 10; +end +ReducedDim = min(ReducedDim+1,row); + +D_mhalf = full(sum(W,2).^-.5); +D_mhalfMatrix = spdiags(D_mhalf,0,nSmp,nSmp); +W = D_mhalfMatrix*W*D_mhalfMatrix; + +W = max(W,W'); + + +dimMatrix = size(W,2); +if ~exist('bEigs','var') + if (dimMatrix > MAX_MATRIX_SIZE && ReducedDim < dimMatrix*EIGVECTOR_RATIO) + bEigs = 1; + else + bEigs = 0; + end +end + +if bEigs + option = struct('disp',0); + [Y, eigvalue] = eigs(W,ReducedDim,'la',option); + eigvalue = diag(eigvalue); +else + [Y, eigvalue] = eig(full(W)); + eigvalue = diag(eigvalue); + + [junk, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + Y = Y(:,index); + if ReducedDim < length(eigvalue) + Y = Y(:, 1:ReducedDim); + eigvalue = eigvalue(1:ReducedDim); + end +end + + +eigIdx = find(abs(eigvalue) < 1e-6); +eigvalue (eigIdx) = []; +Y (:,eigIdx) = []; + +nGotDim = length(eigvalue); + +idx = 1; +while(abs(eigvalue(idx)-1) < 1e-12) + idx = idx + 1; + if idx > nGotDim + break; + end +end +idx = idx - 1; + +if(idx > 1) % more than one eigenvector of 1 eigenvalue + u = zeros(size(Y,1),idx); + + d_m = 1./D_mhalf; + cc = 1/norm(d_m); + u(:,1) = cc./D_mhalf; + + bDone = 0; + for i = 1:idx + if abs(Y(:,i)' * u(:,1) - 1) < 1e-14 + Y(:,i) = Y(:,1); + Y(:,1) = u(:,1); + bDone = 1; + end + end + + if ~bDone + for i = 2:idx + u(:,i) = Y(:,i); + for j= 1:i-1 + u(:,i) = u(:,i) - (u(:,j)' * Y(:,i))*u(:,j); + end + u(:,i) = u(:,i)/norm(u(:,i)); + end + Y(:,1:idx) = u; + end +end + +Y = D_mhalfMatrix*Y; + +Y(:,1) = []; +eigvalue(1) = []; + + + + diff --git a/EuDist2.m b/EuDist2.m new file mode 100644 index 0000000..6ff2374 --- /dev/null +++ b/EuDist2.m @@ -0,0 +1,60 @@ +function D = EuDist2(fea_a,fea_b,bSqrt) +%EUDIST2 Efficiently Compute the Euclidean Distance Matrix by Exploring the +%Matlab matrix operations. +% +% D = EuDist(fea_a,fea_b) +% fea_a: nSample_a * nFeature +% fea_b: nSample_b * nFeature +% D: nSample_a * nSample_a +% or nSample_a * nSample_b +% +% Examples: +% +% a = rand(500,10); +% b = rand(1000,10); +% +% A = EuDist2(a); % A: 500*500 +% D = EuDist2(a,b); % D: 500*1000 +% +% version 2.1 --November/2011 +% version 2.0 --May/2009 +% version 1.0 --November/2005 +% +% Written by Deng Cai (dengcai AT gmail.com) + + +if ~exist('bSqrt','var') + bSqrt = 1; +end + +if (~exist('fea_b','var')) || isempty(fea_b) + aa = sum(fea_a.*fea_a,2); + ab = fea_a*fea_a'; + + if issparse(aa) + aa = full(aa); + end + + D = bsxfun(@plus,aa,aa') - 2*ab; + D(D<0) = 0; + if bSqrt + D = sqrt(D); + end + D = max(D,D'); +else + aa = sum(fea_a.*fea_a,2); + bb = sum(fea_b.*fea_b,2); + ab = fea_a*fea_b'; + + if issparse(aa) + aa = full(aa); + bb = full(bb); + end + + D = bsxfun(@plus,aa,bb') - 2*ab; + D(D<0) = 0; + if bSqrt + D = sqrt(D); + end +end + diff --git a/GNMF.m b/GNMF.m new file mode 100644 index 0000000..3983924 --- /dev/null +++ b/GNMF.m @@ -0,0 +1,97 @@ +function [U_final, V_final, nIter_final, objhistory_final] = GNMF(X, k, W, options, U, V) +% Graph regularized Non-negative Matrix Factorization (GNMF) +% +% where +% X +% Notation: +% X ... (mFea x nSmp) data matrix +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% options.alpha ... the regularization parameter. +% [default: 100] +% alpha = 0, GNMF boils down to the ordinary NMF. +% +% +% You only need to provide the above four inputs. +% +% X = U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Xiaoyun Wu, and Jiawei Han. "Non-negative +% Matrix Factorization on Manifold", Proc. 2008 Int. Conf. on Data Mining +% (ICDM'08), Pisa, Italy, Dec. 2008. +% +% [2] Deng Cai, Xiaofei He, Jiawei Han, Thomas Huang. "Graph Regularized +% Non-negative Matrix Factorization for Data Representation", IEEE +% Transactions on Pattern Analysis and Machine Intelligence, , Vol. 33, No. +% 8, pp. 1548-1560, 2011. +% +% +% version 2.0 --April/2009 +% version 1.0 --April/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if min(min(X)) < 0 + error('Input should be nonnegative!'); +end + +if ~isfield(options,'error') + options.error = 1e-5; +end +if ~isfield(options, 'maxIter') + options.maxIter = []; +end + +if ~isfield(options,'nRepeat') + options.nRepeat = 10; +end + +if ~isfield(options,'minIter') + options.minIter = 30; +end + +if ~isfield(options,'meanFitRatio') + options.meanFitRatio = 0.1; +end + +if ~isfield(options,'alpha') + options.alpha = 100; +end + +nSmp = size(X,2); + +if isfield(options,'alpha_nSmp') && options.alpha_nSmp + options.alpha = options.alpha*nSmp; +end + +if isfield(options,'weight') && strcmpi(options.weight,'NCW') + feaSum = full(sum(X,2)); + D_half = X'*feaSum; + X = X*spdiags(D_half.^-.5,0,nSmp,nSmp); +end + +if ~isfield(options,'Optimization') + options.Optimization = 'Multiplicative'; +end + +if ~exist('U','var') + U = []; + V = []; +end + +switch lower(options.Optimization) + case {lower('Multiplicative')} + [U_final, V_final, nIter_final, objhistory_final] = GNMF_Multi(X, k, W, options, U, V); + otherwise + error('optimization method does not exist!'); +end + + + + \ No newline at end of file diff --git a/GNMF_KL.m b/GNMF_KL.m new file mode 100644 index 0000000..1f17829 --- /dev/null +++ b/GNMF_KL.m @@ -0,0 +1,135 @@ +function [U_final, V_final, nIter_final, objhistory_final] = GNMF_KL(X, k, W, options, U, V) +% Graph regularized Non-negative Matrix Factorization with Divergence Formulation +% Locality Preserving Non-negative Matrix Factorization (LPNMF) +% +% where +% X +% Notation: +% X ... (mFea x nSmp) data matrix +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% options.alpha: manifold regularization prameter (default 100) +% if alpha=0, GNMF_KL boils down to the ordinary +% NMF with KL divergence. Please see [1][2] for details. +% +% You only need to provide the above four inputs. +% +% X = U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Xuanhui Wang, Hujun Bao, and Jiawei Han. +% "Locality Preserving Nonnegative Matrix Factorization", Proc. 2009 Int. +% Joint Conf. on Arti_cial Intelligence (IJCAI-09), Pasadena, CA, July 2009. +% +% [2] Deng Cai, Xiaofei He, Jiawei Han, Thomas Huang. "Graph Regularized +% Non-negative Matrix Factorization for Data Representation", IEEE +% Transactions on Pattern Analysis and Machine Intelligence, , Vol. 33, No. +% 8, pp. 1548-1560, 2011. +% +% +% +% version 3.0 --Jan/2012 +% version 2.0 --April/2009 +% version 1.0 --April/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +ZERO_OFFSET = 1e-200; + +nSmp = size(X,2); +if min(min(X)) < 0 + error('Input should be nonnegative!'); +end + +if ~isfield(options,'error') + options.error = 1e-5; +end +if ~isfield(options, 'maxIter') + options.maxIter = []; +end +if ~isfield(options,'nRepeat') + options.nRepeat = 10; +end + +if ~isfield(options,'minIter') + options.minIter = 10; +end + +if ~isfield(options,'meanFitRatio') + options.meanFitRatio = 0.1; +end + +if ~isfield(options,'alpha') + options.alpha = 100; +end + +if isfield(options,'alpha_nSmp') && options.alpha_nSmp + options.alpha = options.alpha*nSmp; +end + +if ~isfield(options,'kmeansInit') + options.kmeansInit = 1; +end + + +if ~exist('U','var') + U = []; + V = []; +end + +NCWeight = []; +if isfield(options,'weight') && strcmpi(options.weight,'NCW') + feaSum = full(sum(X,2)); + NCWeight = (X'*feaSum).^-1; + tmpNCWeight = NCWeight; +else + tmpNCWeight = ones(nSmp,1); +end +if issparse(X) + nz = nnz(X); + nzk = nz*k; + [idx,jdx,vdx] = find(X); + if isempty(NCWeight) + Cons = sum(vdx.*log(vdx) - vdx); + else + Cons = sum(NCWeight(jdx).*(vdx.*log(vdx) - vdx)); + end + ldx = sub2ind(size(X),idx,jdx); + + options.nz = nz; + options.nzk = nzk; + options.idx = idx; + options.jdx = jdx; + options.vdx = vdx; + options.ldx = ldx; +else + Y = X + ZERO_OFFSET; + if isempty(NCWeight) + Cons = sum(sum(Y.*log(Y)-Y)); + else + Cons = sum(NCWeight'.*sum(Y.*log(Y)-Y,1)); + end + clear Y; +end +options.NCWeight = NCWeight; +options.tmpNCWeight = tmpNCWeight; +options.Cons = Cons; + +if ~isfield(options,'Optimization') + options.Optimization = 'Multiplicative'; +end + + +switch lower(options.Optimization) + case {lower('Multiplicative')} + [U_final, V_final, nIter_final, objhistory_final] = GNMF_KL_Multi(X, k, W, options, U, V); + otherwise + error('optimization method does not exist!'); +end + diff --git a/GNMF_KL_Multi.m b/GNMF_KL_Multi.m new file mode 100644 index 0000000..64d74c3 --- /dev/null +++ b/GNMF_KL_Multi.m @@ -0,0 +1,442 @@ +function [U_final, V_final, nIter_final, objhistory_final] = GNMF_KL_Multi(X, k, W, options, U, V) +% Graph regularized Non-negative Matrix Factorization with Divergence Formulation +% Locality Preserving Non-negative Matrix Factorization (LPNMF) +% +% where +% X +% Notation: +% X ... (mFea x nSmp) data matrix +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% options.alpha: manifold regularization prameter (default 100) +% if alpha=0, GNMF_KL boils down to the ordinary +% NMF with KL divergence. Please see [1][2] for details. +% +% You only need to provide the above four inputs. +% +% X = U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Xuanhui Wang, Hujun Bao, and Jiawei Han. +% "Locality Preserving Nonnegative Matrix Factorization", Proc. 2009 Int. +% Joint Conf. on Arti_cial Intelligence (IJCAI-09), Pasadena, CA, July 2009. +% +% [2] Deng Cai, Xiaofei He, Jiawei Han, Thomas Huang. "Graph Regularized +% Non-negative Matrix Factorization for Data Representation", IEEE +% Transactions on Pattern Analysis and Machine Intelligence, , Vol. 33, No. +% 8, pp. 1548-1560, 2011. +% +% +% +% version 3.0 --Jan/2012 +% version 2.0 --April/2009 +% version 1.0 --April/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +differror = options.error; +maxIter = options.maxIter; +nRepeat = options.nRepeat; +minIter = options.minIter - 1; +if ~isempty(maxIter) && maxIter < minIter+1 + minIter = maxIter-1; +end +meanFitRatio = options.meanFitRatio; +kmeansInit = options.kmeansInit; + +alpha = options.alpha; + + +NCWeight = options.NCWeight; +tmpNCWeight = options.tmpNCWeight; +Cons = options.Cons; +if issparse(X) + nz = options.nz; + nzk = options.nzk; + idx = options.idx; + jdx = options.jdx; + vdx = options.vdx; + ldx = options.ldx; +end + +Norm = 2; +NormV = 0; + +[mFea,nSmp]=size(X); + +if alpha > 0 + DCol = full(sum(W,2)); + D = spdiags(DCol,0,speye(size(W,1))); + L = D - W; + if isfield(options,'NormW') && options.NormW + D_mhalf = DCol.^-.5; + + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + L = (tmpD_mhalf.*L).*tmpD_mhalf'; + clear D_mhalf tmpD_mhalf; + end + L = alpha*L; + L = max(L, L'); +else + L = []; +end + +selectInit = 1; +if isempty(U) + if kmeansInit + [U,V] = NMF_init(X, k); + else + U = abs(rand(mFea,k)); + V = abs(rand(nSmp,k)); + end +else + nRepeat = 1; +end +[U,V] = NormalizeUV(U, V, NormV, Norm); + +if nRepeat == 1 + selectInit = 0; + minIter = 0; + if isempty(maxIter) + if issparse(X) + [obj_NMFhistory, obj_Laphistory] = CalculateObjSparse(Cons, jdx, vdx, ldx, U, V, L, NCWeight); + else + [obj_NMFhistory, obj_Laphistory] = CalculateObj(Cons, X, U, V, L, NCWeight); + end + objhistory = obj_NMFhistory + obj_Laphistory; + meanFit = objhistory*10; + end +end + +maxM = 62500000; +mn = numel(X); +if issparse(X) + nBlockNZ = floor(maxM/(k*2)); +end + +tryNo = 0; +nIter = 0; +while tryNo < nRepeat + tryNo = tryNo+1; + maxErr = 1; + while(maxErr > differror) + if issparse(X) + % ===================== update V ======================== + if nzk < maxM + Y = sum(U(idx,:).*V(jdx,:),2); + else + Y = zeros(size(vdx)); + for i = 1:ceil(nz/nBlockNZ) + if i == ceil(nz/nBlockNZ) + smpIdx = (i-1)*nBlockNZ+1:nz; + else + smpIdx = (i-1)*nBlockNZ+1:i*nBlockNZ; + end + Y(smpIdx) = sum(U(idx(smpIdx),:).*V(jdx(smpIdx),:),2); + end + end + Y = vdx./max(Y,1e-10); + Y = sparse(idx,jdx,Y,mFea,nSmp); + Y = (U'*Y)'; + + if alpha > 0 + if isempty(NCWeight) + Y = V.*Y; + else + Y = repmat(NCWeight,1,k).*V.*Y; + end + sumU = max(sum(U,1),1e-10); + + for i = 1:k + tmpL = L; + tmpD = tmpNCWeight*sumU(i); + tmpL = tmpL+spdiags(tmpD,0,nSmp,nSmp); + V(:,i) = tmpL\Y(:,i); + end + else + sumU = max(sum(U,1),1e-10); + Y = V.*Y; + V = Y./repmat(sumU,nSmp,1); + end + + % ===================== update U ======================== + if nzk < maxM + Y = sum(U(idx,:).*V(jdx,:),2); + else + Y = zeros(size(vdx)); + for i = 1:ceil(nz/nBlockNZ) + if i == ceil(nz/nBlockNZ) + smpIdx = (i-1)*nBlockNZ+1:nz; + else + smpIdx = (i-1)*nBlockNZ+1:i*nBlockNZ; + end + Y(smpIdx) = sum(U(idx(smpIdx),:).*V(jdx(smpIdx),:),2); + end + end + + Y = vdx./max(Y,1e-10); + Y = sparse(idx,jdx,Y,mFea,nSmp); + if isempty(NCWeight) + Y = Y*V; + sumV = max(sum(V,1),1e-10); + else + wV = repmat(NCWeight,1,k).*V; + Y = Y*wV; + sumV = max(sum(wV,1),1e-10); + clear wV; + end + U = U.*(Y./repmat(sumV,mFea,1)); + else + if mn < maxM + % ===================== update V ======================== + Y = U*V'; + Y = X./max(Y,1e-10); + Y = Y'*U; + + if alpha > 0 + if isempty(NCWeight) + Y = V.*Y; + else + Y = repmat(NCWeight,1,k).*V.*Y; + end + sumU = max(sum(U,1),1e-10); + + for i = 1:k + tmpL = L; + tmpD = tmpNCWeight*sumU(i); + tmpL = tmpL+spdiags(tmpD,0,nSmp,nSmp); + V(:,i) = tmpL\Y(:,i); + end + else + sumU = max(sum(U,1),1e-10); + Y = V.*Y; + V = Y./repmat(sumU,nSmp,1); + end + + % ===================== update U ======================== + Y = U*V'; + Y = X./max(Y,1e-10); + if isempty(NCWeight) + Y = Y*V; + sumV = max(sum(V,1),1e-10); + else + wV = repmat(NCWeight,1,k).*V; + Y = Y*wV; + sumV = max(sum(wV,1),1e-10); + clear wV; + end + U = U.*(Y./repmat(sumV,mFea,1)); + else + error('Not implemented!'); + end + end + clear Y sumU sumV; + + nIter = nIter + 1; + if nIter > minIter + if selectInit + if issparse(X) + [obj_NMFhistory, obj_Laphistory] = CalculateObjSparse(Cons, jdx, vdx, ldx, U, V, L, NCWeight); + else + [obj_NMFhistory, obj_Laphistory] = CalculateObj(Cons, X, U, V, L, NCWeight); + end + objhistory = obj_NMFhistory + obj_Laphistory; + maxErr = 0; + else + if isempty(maxIter) + if issparse(X) + [obj_NMF, obj_Lap] = CalculateObjSparse(Cons, jdx, vdx, ldx, U, V, L, NCWeight); + else + [obj_NMF, obj_Lap] = CalculateObj(Cons, X, U, V, L, NCWeight); + end + newobj = obj_NMF + obj_Lap; + + obj_NMFhistory = [obj_NMFhistory obj_NMF]; %#ok + obj_Laphistory = [obj_Laphistory obj_Lap]; %#ok + + objhistory = [objhistory newobj]; %#ok + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*newobj; + maxErr = (meanFit-newobj)/meanFit; + else + maxErr = 1; + if nIter >= maxIter + maxErr = 0; + objhistory = 0; + end + end + end + end + end + + if tryNo == 1 + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + else + if objhistory(end) < objhistory_final(end) + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + end + end + + if selectInit + if tryNo < nRepeat + if kmeansInit + [U,V] = NMF_init(X, k); + else + U = abs(rand(mFea,k)); + V = abs(rand(nSmp,k)); + end + [U,V] = NormalizeUV(U, V, NormV, Norm); + nIter = 0; + else + tryNo = tryNo - 1; + nIter = minIter + 1; + selectInit = 0; + U = U_final; + V = V_final; + objhistory = objhistory_final; + meanFit = objhistory*10; + end + end +end + +[U_final,V_final] = NormalizeUV(U_final, V_final, NormV, Norm); + +%======================================================= +function [obj_NMF, obj_Lap] = CalculateObjSparse(Cons, jdx, vdx, ldx, U, V, L, NCWeight) + ZERO_OFFSET = 1e-200; + + maxM = 62500000; + mFea = size(U,1); + nSmp = size(V,1); + mn = mFea*nSmp; + nBlock = floor(maxM/(mFea*2)); + + if mn < maxM + Y = U*V'; + if isempty(NCWeight) + obj_NMF = sum(sum(Y)) - sum(vdx.*log(Y(ldx)+ZERO_OFFSET)); % 14p (p << mn) + else + obj_NMF = sum(NCWeight'.*sum(Y,1)) - sum(NCWeight(jdx).*(vdx.*log(Y(ldx)+ZERO_OFFSET))); % 14p (p << mn) + end + else + obj_NMF = 0; + ldxRemain = ldx; + vdxStart = 1; + for i = 1:ceil(nSmp/nBlock) + if i == ceil(nSmp/nBlock) + smpIdx = (i-1)*nBlock+1:nSmp; + ldxNow = ldxRemain; + vdxNow = vdxStart:length(vdx); + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + ldxLast = find(ldxRemain <= mFea*i*nBlock, 1 ,'last'); + ldxNow = ldxRemain(1:ldxLast); + ldxRemain = ldxRemain(ldxLast+1:end); + vdxNow = vdxStart:(vdxStart+ldxLast-1); + vdxStart = vdxStart+ldxLast; + end + Y = U*V(smpIdx,:)'; + if isempty(NCWeight) + obj_NMF = obj_NMF + sum(sum(Y)) - sum(vdx(vdxNow).*log(Y(ldxNow-mFea*(i-1)*nBlock)+ZERO_OFFSET)); + else + obj_NMF = obj_NMF + sum(NCWeight(smpIdx)'.*sum(Y,1)) - sum(NCWeight(jdx(vdxNow)).*(vdx(vdxNow).*log(Y(ldxNow-mFea*(i-1)*nBlock)+ZERO_OFFSET))); + end + end + end + obj_NMF = obj_NMF + Cons; + + Y = log(V + ZERO_OFFSET); + if isempty(L) + obj_Lap = 0; + else + obj_Lap = sum(sum((L*Y).*V)); + end + +%======================================================= +function [obj_NMF, obj_Lap] = CalculateObj(Cons, X, U, V, L, NCWeight) + ZERO_OFFSET = 1e-200; + [mFea, nSmp] = size(X); + maxM = 62500000; + mn = numel(X); + nBlock = floor(maxM/(mFea*2)); + + if mn < maxM + Y = U*V'+ZERO_OFFSET; + if isempty(NCWeight) + obj_NMF = sum(sum(Y - X.*log(Y))); % 14mn + else + obj_NMF = sum(NCWeight'.*sum(Y - X.*log(Y),1)); % 14mn + end + else + obj_NMF = 0; + for i = 1:ceil(nSmp/nBlock) + if i == ceil(nSmp/nBlock) + smpIdx = (i-1)*nBlock+1:nSmp; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + Y = U*V(smpIdx,:)'+ZERO_OFFSET; + if isempty(NCWeight) + obj_NMF = obj_NMF + sum(sum(Y - X(:,smpIdx).*log(Y))); + else + obj_NMF = obj_NMF + sum(NCWeight(smpIdx)'.*sum(Y - X(:,smpIdx).*log(Y),1)); + end + end + end + obj_NMF = obj_NMF + Cons; + + Y = log(V + ZERO_OFFSET); + if isempty(L) + obj_Lap = 0; + else + obj_Lap = sum(sum((L*Y).*V)); + end + +%======================================================= +function [U, V] = NormalizeUV(U, V, NormV, Norm) + K = size(U,2); + if Norm == 2 + if NormV + norms = max(1e-15,sqrt(sum(V.^2,1)))'; + V = V*spdiags(norms.^-1,0,K,K); + U = U*spdiags(norms,0,K,K); + else + norms = max(1e-15,sqrt(sum(U.^2,1)))'; + U = U*spdiags(norms.^-1,0,K,K); + V = V*spdiags(norms,0,K,K); + end + else + if NormV + norms = max(1e-15,sum(abs(V),1))'; + V = V*spdiags(norms.^-1,0,K,K); + U = U*spdiags(norms,0,K,K); + else + norms = max(1e-15,sum(abs(U),1))'; + U = U*spdiags(norms.^-1,0,K,K); + V = V*spdiags(norms,0,K,K); + end + end + + + +function [U,V] = NMF_init(X, k) + [label, center] = litekmeans(X',k,'maxIter',10); + center = max(0,center); + U = center'; + UTU = U'*U; + UTU = max(UTU,UTU'); + UTX = U'*X; + V = max(0,UTU\UTX); + V = V'; + + \ No newline at end of file diff --git a/GNMF_Multi.m b/GNMF_Multi.m new file mode 100644 index 0000000..c9a788c --- /dev/null +++ b/GNMF_Multi.m @@ -0,0 +1,281 @@ +function [U_final, V_final, nIter_final, objhistory_final] = GNMF_Multi(X, k, W, options, U, V) +% Graph regularized Non-negative Matrix Factorization (GNMF) with +% multiplicative update +% +% where +% X +% Notation: +% X ... (mFea x nSmp) data matrix +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% +% You only need to provide the above four inputs. +% +% X = U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Xiaoyun Wu, and Jiawei Han. "Non-negative +% Matrix Factorization on Manifold", Proc. 2008 Int. Conf. on Data Mining +% (ICDM'08), Pisa, Italy, Dec. 2008. +% +% [2] Deng Cai, Xiaofei He, Jiawei Han, Thomas Huang. "Graph Regularized +% Non-negative Matrix Factorization for Data Representation", IEEE +% Transactions on Pattern Analysis and Machine Intelligence, , Vol. 33, No. +% 8, pp. 1548-1560, 2011. +% +% +% version 2.1 --Dec./2011 +% version 2.0 --April/2009 +% version 1.0 --April/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +differror = options.error; +maxIter = options.maxIter; +nRepeat = options.nRepeat; +minIter = options.minIter - 1; +if ~isempty(maxIter) && maxIter < minIter + minIter = maxIter; +end +meanFitRatio = options.meanFitRatio; + +alpha = options.alpha; + +Norm = 2; +NormV = 0; + +[mFea,nSmp]=size(X); + +if alpha > 0 + W = alpha*W; + DCol = full(sum(W,2)); + D = spdiags(DCol,0,nSmp,nSmp); + L = D - W; + if isfield(options,'NormW') && options.NormW + D_mhalf = spdiags(DCol.^-.5,0,nSmp,nSmp) ; + L = D_mhalf*L*D_mhalf; + end +else + L = []; +end + +selectInit = 1; +if isempty(U) + U = abs(rand(mFea,k)); + V = abs(rand(nSmp,k)); +else + nRepeat = 1; +end + +[U,V] = NormalizeUV(U, V, NormV, Norm); +if nRepeat == 1 + selectInit = 0; + minIter = 0; + if isempty(maxIter) + objhistory = CalculateObj(X, U, V, L); + meanFit = objhistory*10; + else + if isfield(options,'Converge') && options.Converge + objhistory = CalculateObj(X, U, V, L); + end + end +else + if isfield(options,'Converge') && options.Converge + error('Not implemented!'); + end +end + + + +tryNo = 0; +nIter = 0; +while tryNo < nRepeat + tryNo = tryNo+1; + maxErr = 1; + while(maxErr > differror) + % ===================== update V ======================== + XU = X'*U; % mnk or pk (p< 0 + WV = W*V; + DV = D*V; + + XU = XU + WV; + VUU = VUU + DV; + end + + V = V.*(XU./max(VUU,1e-10)); + + % ===================== update U ======================== + XV = X*V; % mnk or pk (p< minIter + if selectInit + objhistory = CalculateObj(X, U, V, L); + maxErr = 0; + else + if isempty(maxIter) + newobj = CalculateObj(X, U, V, L); + objhistory = [objhistory newobj]; %#ok + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*newobj; + maxErr = (meanFit-newobj)/meanFit; + else + if isfield(options,'Converge') && options.Converge + newobj = CalculateObj(X, U, V, L); + objhistory = [objhistory newobj]; %#ok + end + maxErr = 1; + if nIter >= maxIter + maxErr = 0; + if isfield(options,'Converge') && options.Converge + else + objhistory = 0; + end + end + end + end + end + end + + if tryNo == 1 + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + else + if objhistory(end) < objhistory_final(end) + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + end + end + + if selectInit + if tryNo < nRepeat + %re-start + U = abs(rand(mFea,k)); + V = abs(rand(nSmp,k)); + + [U,V] = NormalizeUV(U, V, NormV, Norm); + nIter = 0; + else + tryNo = tryNo - 1; + nIter = minIter+1; + selectInit = 0; + U = U_final; + V = V_final; + objhistory = objhistory_final; + meanFit = objhistory*10; + end + end +end + +[U_final,V_final] = NormalizeUV(U_final, V_final, NormV, Norm); + + +%========================================================================== + +function [obj, dV] = CalculateObj(X, U, V, L, deltaVU, dVordU) + MAXARRAY = 500*1024*1024/8; % 500M. You can modify this number based on your machine's computational power. + if ~exist('deltaVU','var') + deltaVU = 0; + end + if ~exist('dVordU','var') + dVordU = 1; + end + dV = []; + nSmp = size(X,2); + mn = numel(X); + nBlock = ceil(mn/MAXARRAY); + + if mn < MAXARRAY + dX = U*V'-X; + obj_NMF = sum(sum(dX.^2)); + if deltaVU + if dVordU + dV = dX'*U + L*V; + else + dV = dX*V; + end + end + else + obj_NMF = 0; + if deltaVU + if dVordU + dV = zeros(size(V)); + else + dV = zeros(size(U)); + end + end + PatchSize = ceil(nSmp/nBlock); + for i = 1:nBlock + if i*PatchSize > nSmp + smpIdx = (i-1)*PatchSize+1:nSmp; + else + smpIdx = (i-1)*PatchSize+1:i*PatchSize; + end + dX = U*V(smpIdx,:)'-X(:,smpIdx); + obj_NMF = obj_NMF + sum(sum(dX.^2)); + if deltaVU + if dVordU + dV(smpIdx,:) = dX'*U; + else + dV = dU+dX*V(smpIdx,:); + end + end + end + if deltaVU + if dVordU + dV = dV + L*V; + end + end + end + if isempty(L) + obj_Lap = 0; + else + obj_Lap = sum(sum((V'*L).*V')); + end + obj = obj_NMF+obj_Lap; + + + + + +function [U, V] = NormalizeUV(U, V, NormV, Norm) + K = size(U,2); + if Norm == 2 + if NormV + norms = max(1e-15,sqrt(sum(V.^2,1)))'; + V = V*spdiags(norms.^-1,0,K,K); + U = U*spdiags(norms,0,K,K); + else + norms = max(1e-15,sqrt(sum(U.^2,1)))'; + U = U*spdiags(norms.^-1,0,K,K); + V = V*spdiags(norms,0,K,K); + end + else + if NormV + norms = max(1e-15,sum(abs(V),1))'; + V = V*spdiags(norms.^-1,0,K,K); + U = U*spdiags(norms,0,K,K); + else + norms = max(1e-15,sum(abs(U),1))'; + U = U*spdiags(norms.^-1,0,K,K); + V = V*spdiags(norms,0,K,K); + end + end + + \ No newline at end of file diff --git a/GenSpatialSmoothRegularizer.m b/GenSpatialSmoothRegularizer.m new file mode 100644 index 0000000..c86084d --- /dev/null +++ b/GenSpatialSmoothRegularizer.m @@ -0,0 +1,41 @@ +function [R] = GenSpatialSmoothRegularizer(nRow,nCol) +% function [R] = GenSpatialSmoothRegularizer(nRow,nCol) +% Usage: +% R = GenSpatialSmoothRegularizer(nRow,nCol) +% +% nRow,nCol : dimensions of the images. +% +% R : the spatially smooth regularizer +% +% References: +% [1] Deng Cai, Xiaofei He, Yuxiao Hu, Jiawei Han and Thomas Huang, +% "Learning a Spatially Smooth Subspace for Face Recognition", CVPR'07. +% +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai2 AT gmail.com) +% + + +B = ones(nRow,3); +B(:,2) = -2; +B(1,2) = -1; +B(end,2) = -1; +D = spdiags(B,[-1,0,1],nRow,nRow); +I = speye(nCol); + +M = kron(D,I); + +B = ones(nCol,3); +B(:,2) = -2; +B(1,2) = -1; +B(end,2) = -1; +D = spdiags(B,[-1,0,1],nCol,nCol); +I = speye(nRow); + +M2 = kron(I,D); + +M = M+M2; +R = M'*M; + +R = max(R,R'); diff --git a/GenTwoNoisyCircle.m b/GenTwoNoisyCircle.m new file mode 100644 index 0000000..dc0ab07 --- /dev/null +++ b/GenTwoNoisyCircle.m @@ -0,0 +1,24 @@ +function [fea, gnd] = GenTwoNoisyCircle() +% [fea, gnd] = GenTwoNoisyCircle(N) +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +N = 200; + +rA = ones(1,N)*2+0.5*rand(1,N); +rB = ones(1,N)+0.5*rand(1,N); + +thetaPos = pi*(2.*[1:N]./N); +zeroA = [0,0]; +zeroB = [0,0]; + +feaA = [rA.*cos(thetaPos)+zeroA(1); rA.*sin(thetaPos)+zeroA(2)]'; %' +feaB = [rB.*cos(thetaPos)+zeroB(1); rB.*sin(thetaPos)+zeroB(2)]'; %' + +fea = [feaB;feaA]; +gnd = [ones(N,1);2*ones(N,1)]; + diff --git a/GraphSC.m b/GraphSC.m new file mode 100644 index 0000000..27518f0 --- /dev/null +++ b/GraphSC.m @@ -0,0 +1,145 @@ +function [B S stat] = GraphSC(X, W, num_bases, alpha, beta, num_iters, Binit, pars) +% Graph regularized sparse coding algorithms +% +% minimize_B,S 0.5*||X - B*S||^2 + alpha*Tr(SLS') + beta*sum(abs(S(:))) +% subject to ||B(:,j)||_2 <= l2norm, forall j=1...size(S,1) +% +% Notation: +% X: data matrix, each column is a sample vector +% W: affinity graph matrix +% num_bases: number of bases +% alpha: Laplician parameter +% beta: sparsity penalty parameter +% num_iters: number of iteration +% Binit: initial B matrix +% pars: additional parameters to specify (see the code) +% +% This code is modified from the codes provided by Honglak Lee, Alexis +% Battle, Rajat Raina, and Andrew Y. Ng in the following paper: +% 'Efficient Sparse Codig Algorithms', Honglak Lee, Alexis Battle, Rajat Raina, Andrew Y. Ng, +% Advances in Neural Information Processing Systems (NIPS) 19, 2007 +% +% References: +% [1] Miao Zheng, Jiajun Bu, Chun Chen, Can Wang, Lijun Zhang, Guang Qiu, Deng Cai. +% "Graph Regularized Sparse Coding for Image Representation", +% IEEE Transactions on Image Processing, Vol. 20, No. 5, pp. 1327-1336, 2011. +% +% Version1.0 -- Nov/2009 +% Version2.0 -- Jan/2012 +% Written by Miao Zheng +% + +diff = 1e-7; + +pars.mFea = size(X,1); +pars.nSmp = size(X,2); +pars.num_bases = num_bases; +pars.num_iters = num_iters; +pars.beta = beta; +pars.noise_var = 1; +pars.sigma = 1; +pars.VAR_basis = 1; + + +% Sparsity parameters +if ~isfield(pars,'tol') + pars.tol = 0.005; +end + +% initialize basis +if ~exist('Binit','var') || isempty(Binit) + B = rand(pars.mFea,pars.num_bases)-0.5; + B = B - repmat(mean(B,1), size(B,1),1); + B = B*diag(1./sqrt(sum(B.*B))); +else + disp('Using Binit...'); + B = Binit; +end; + + +% initialize t only if it does not exist +t=0; +% statistics variable +stat= []; +stat.fobj_avg = []; +stat.fresidue_avg = []; +stat.fsparsity_avg = []; +stat.var_tot = []; +stat.svar_tot = []; +stat.elapsed_time=0; + + +% Construct the K-NN Graph +if isempty(W) + W = constructW(X'); +end +DCol = full(sum(W,2)); +D = spdiags(DCol,0,speye(size(W,1))); +L = D - W; + + +% optimization loop +while t < pars.num_iters + t=t+1; + start_time= cputime; + + stat.fobj_total=0; + stat.fresidue_total=0; + stat.fsparsity_total=0; + stat.flaplacian_total = 0; + stat.var_tot=0; + stat.svar_tot=0; + + % learn coefficients (conjugate gradient) + if t ==1 + S= learn_coefficients(B, X, alpha, pars.beta/pars.sigma*pars.noise_var,L); + else + S= learn_coefficients(B, X, alpha, pars.beta/pars.sigma*pars.noise_var, L, S); + end + S(isnan(S))=0; + + % get objective + [fobj, fresidue, fsparsity, flaplacian] = getObjective(B, S, X, alpha, L, pars.noise_var, pars.beta, pars.sigma); + + stat.fobj_total = stat.fobj_total + fobj; + stat.flaplacian_total = stat.flaplacian_total + flaplacian; + stat.fresidue_total = stat.fresidue_total + fresidue; + stat.fsparsity_total = stat.fsparsity_total + fsparsity; + stat.var_tot = stat.var_tot + sum(sum(S.^2,1))/size(S,1); + + % update basis + B = learn_basis(X, S, pars.VAR_basis); + + % get statistics + stat.fobj_avg(t) = stat.fobj_total / pars.nSmp; + stat.fresidue_avg(t) = stat.fresidue_total / pars.nSmp; + stat.fsparsity_avg(t) = stat.fsparsity_total / pars.nSmp; + stat.flaplacian_avg(t) = stat.flaplacian_total / pars.nSmp; + stat.var_avg(t) = stat.var_tot / pars.nSmp; + stat.svar_avg(t) = stat.svar_tot / pars.nSmp; + stat.elapsed_time(t) = cputime - start_time; + + + if t>199 + if(stat.fobj_avg(t-1) - stat.fobj_avg(t)0, The distance between two nodes +% in the same class will be smaller than +% two nodes have diff. labels +% The label information 'gnd' should be +% provided. +% +% k - The number of neighbors. +% Default k = 5; +% gnd - The parameter needed under 'Supervised' +% NeighborMode. Colunm vector of the label +% information for each data point. +% +% Please see LGE.m for other options. +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The eigvalue of LPP eigen-problem. sorted from +% smallest to largest. +% elapse - Time spent on different steps +% +% +% Examples: +% +% +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.k = 0; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% [eigvector, eigvalue] = IsoP(options, fea); +% Y = fea*eigvector; +% +% +% +% See also LPP, LGE +% +%Reference: +% +% Deng Cai, Xiaofei He, and Jiawei Han, "Isometric Projection", +% Twenty-Second Conference on Artificial Intelligence (AAAI-07), 2007 +% +% Deng Cai, Xiaofei He and Jiawei Han, "Isometric Projection", Technical +% report, Computer Science Department, UIUC, UIUCDCS-R-2006-2747, July 2006 +% +% Joshua B. Tenenbaum, Vin de Silva, and John C. Langford. "A Global +% Geometric Framework for Nonlinear Dimensionality Reduction", Science, +% v.290 no.5500 , Dec.22, 2000. pp.2319-2323. +% +% +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.1 --May/2006 +% version 1.0 --Nov/2005 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) + +INFratio = 1000; + + +if (~exist('options','var')) + options = []; +end + +if ~isfield(options,'NeighborMode') + options.NeighborMode = 'KNN'; +end + +if ~isfield(options,'k') + options.k = 5; +end + +nSmp = size(data,1); + +if options.k >= nSmp + error('k is too large!'); +end + + +if options.k <= 0 % Always supervised! + if ~isfield(options,'gnd') + error('gnd should be provided!'); + end + if length(options.gnd) ~= nSmp + error('gnd and data mismatch!'); + end + + Label = unique(options.gnd); + nLabel = length(Label); + + G = zeros(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D = EuDist2(data(classIdx,:),[],1); + G(classIdx,classIdx) = D; + end + maxD = max(max(G)); + INF = maxD*INFratio; % effectively infinite distance + + D = INF*ones(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D(classIdx,classIdx) = G(classIdx,classIdx); + end + + clear G +else + switch lower(options.NeighborMode) + case {lower('KNN')} + D = EuDist2(data); + maxD = max(max(D)); + INF = maxD*INFratio; % effectively infinite distance + + [dump,iidx] = sort(D,2); + iidx = iidx(:,(2+options.k):end); + for i=1:nSmp + D(i,iidx(i,:)) = 0; + end + D = max(D,D'); + + D = sparse(D); + D = dijkstra(D, 1:nSmp); + + D = reshape(D,nSmp*nSmp,1); + infIdx = find(D==inf); + if ~isempty(infIdx) + D(infIdx) = INF; + end + D = reshape(D,nSmp,nSmp); + + case {lower('Supervised')} + if ~isfield(options,'gnd') + error('gnd should be provided!'); + end + if length(options.gnd) ~= nSmp + error('gnd and data mismatch!'); + end + + Label = unique(options.gnd); + nLabel = length(Label); + + + G = zeros(nSmp,nSmp); + maxD = 0; + for idx=1:nLabel + classIdx = find(options.gnd==Label(idx)); + nSmpClass = length(classIdx); + D = EuDist2(data(classIdx,:),[],1); + if maxD < max(max(D)) + maxD = max(max(D)); + end + if options.k >= nSmpClass + G(classIdx,classIdx) = D; + else + [dump,iidx] = sort(D,2); + iidx = iidx(:,(2+options.k):end); + for i=1:nSmpClass + D(i,iidx(i,:)) = 0; + end + D = max(D,D'); + D = sparse(D); + D = dijkstra(D, 1:nSmpClass); + G(classIdx,classIdx) = D; + end + end + + INF = maxD*INFratio; % effectively infinite distance + + D = INF*ones(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D(classIdx,classIdx) = G(classIdx,classIdx); + end + clear G + + otherwise + error('NeighborMode does not exist!'); + end +end + + +S = D.^2; +sumS = sum(S); +H = sumS'*ones(1,nSmp)/nSmp; +TauDg = -.5*(S - H - H' + sum(sumS)/(nSmp^2)); + +TauDg = max(TauDg,TauDg'); + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + + +[eigvector, eigvalue] = LGE(TauDg, [], options, data); + + +eigIdx = find(eigvalue < 1e-3); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + diff --git a/KDA.m b/KDA.m new file mode 100644 index 0000000..d01100c --- /dev/null +++ b/KDA.m @@ -0,0 +1,191 @@ +function [eigvector, eigvalue] = KDA(options,gnd,data) +% KDA: Kernel Discriminant Analysis +% +% [eigvector, eigvalue] = KDA(options, gnd, data) +% +% Input: +% data - +% if options.Kernel = 0 +% Data matrix. Each row vector of fea is a data +% point. +% if options.Kernel = 1 +% Kernel matrix. +% +% gnd - Colunm vector of the label information for each +% data point. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Kernel - 1: data is actually the kernel matrix. +% 0: ordinary data matrix. +% Default: 0 +% +% Regu - 1: regularized solution, +% a* = argmax (a'KWKa)/(a'KKa+ReguAlpha*I) +% 0: solve the sinularity problem by SVD +% Default: 0 +% +% ReguAlpha - The regularization parameter. Valid +% when Regu==1. Default value is 0.1. +% +% Please see constructKernel.m for other Kernel options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% eigvalue - The sorted eigvalue of LDA eigen-problem. +% elapse - Time spent on different steps +% +% Examples: +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options.KernelType = 'Gaussian'; +% options.t = 1; +% [eigvector, eigvalue] = KDA(gnd, options, fea); +% +% feaTest = rand(3,10); +% Ktest = constructKernel(feaTest,fea,options) +% Y = Ktest*eigvector; +% +% +% +% See also KSR, KLPP, KGE +% +% NOTE: +% In paper [2], we present an efficient approach to solve the optimization +% problem in KDA. We named this approach as Kernel Spectral Regression +% (KSR). I strongly recommend using KSR instead of this KDA algorithm. +% +%Reference: +% +% [1] G. Baudat, F. Anouar, “Generalized +% Discriminant Analysis Using a Kernel Approach", Neural Computation, +% 12:2385-2404, 2000. +% +% [2] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [3] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Dec/2011 +% version 2.0 --August/2007 +% version 1.0 --April/2005 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) +% + +if (~exist('options','var')) + options = []; +end + +if ~isfield(options,'Regu') || ~options.Regu + bPCA = 1; +else + bPCA = 0; + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.01; + end +end + + +if isfield(options,'Kernel') && options.Kernel + K = data; + K = max(K,K'); +else + K = constructKernel(data,[],options); +end +clear data; + + + + +% ====== Initialization +nSmp = size(K,1); +if length(gnd) ~= nSmp + error('gnd and data mismatch!'); +end + +classLabel = unique(gnd); +nClass = length(classLabel); +Dim = nClass - 1; + +K_orig = K; + +sumK = sum(K,2); +H = repmat(sumK./nSmp,1,nSmp); +K = K - H - H' + sum(sumK)/(nSmp^2); +K = max(K,K'); +clear H; + +%====================================== +% SVD +%====================================== + +if bPCA + + [U,D] = eig(K); + D = diag(D); + + maxEigValue = max(abs(D)); + eigIdx = find(abs(D)/maxEigValue < 1e-6); + if length(eigIdx) < 1 + [dump,eigIdx] = min(D); + end + D (eigIdx) = []; + U (:,eigIdx) = []; + + Hb = zeros(nClass,size(U,2)); + for i = 1:nClass, + index = find(gnd==classLabel(i)); + classMean = mean(U(index,:),1); + Hb (i,:) = sqrt(length(index))*classMean; + end + + [dumpVec,eigvalue,eigvector] = svd(Hb,'econ'); + eigvalue = diag(eigvalue); + + if length(eigvalue) > Dim + eigvalue = eigvalue(1:Dim); + eigvector = eigvector(:,1:Dim); + end + + eigvector = (U.*repmat((D.^-1)',nSmp,1))*eigvector; + +else + + Hb = zeros(nClass,nSmp); + for i = 1:nClass, + index = find(gnd==classLabel(i)); + classMean = mean(K(index,:),1); + Hb (i,:) = sqrt(length(index))*classMean; + end + B = Hb'*Hb; + T = K*K; + + for i=1:size(T,1) + T(i,i) = T(i,i) + options.ReguAlpha; + end + + B = double(B); + T = double(T); + + B = max(B,B'); + T = max(T,T'); + + option = struct('disp',0); + [eigvector, eigvalue] = eigs(B,T,Dim,'la',option); + eigvalue = diag(eigvalue); +end + + +tmpNorm = sqrt(sum((eigvector'*K_orig).*eigvector',2)); +eigvector = eigvector./repmat(tmpNorm',size(eigvector,1),1); + + + + diff --git a/KGE.m b/KGE.m new file mode 100644 index 0000000..518550a --- /dev/null +++ b/KGE.m @@ -0,0 +1,235 @@ +function [eigvector, eigvalue] = KGE(W, D, options, data) +% KGE: Kernel Graph Embedding +% +% [eigvector, eigvalue] = KGE(W, D, options, data) +% +% Input: +% data - +% if options.Kernel = 0 +% Data matrix. Each row vector of fea is a data +% point. +% if options.Kernel = 1 +% Kernel matrix. +% W - Affinity graph matrix. +% D - Constraint graph matrix. +% KGE solves the optimization problem of +% a* = argmax (a'KWKa)/(a'KDKa) +% Default: D = I +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Kernel - 1: data is actually the kernel matrix. +% 0: ordinary data matrix. +% Default: 0 +% +% ReducedDim - The dimensionality of the reduced +% subspace. If 0, all the dimensions +% will be kept. Default is 30. +% +% Regu - 1: regularized solution, +% a* = argmax (a'KWKa)/(a'KDKa+ReguAlpha*I) +% 0: solve the sinularity problem by SVD +% Default: 0 +% +% ReguAlpha - The regularization parameter. Valid +% when Regu==1. Default value is 0.1. +% +% Please see constructKernel.m for other Kernel options. +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% eigvalue - The sorted eigvalue of the eigen-problem. +% elapse - Time spent on different steps +% +% +% Examples: +% +% See also KernelLPP, KDA, constructKernel. +% +% +%Reference: +% +% 1. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for Efficient +% Regularized Subspace Learning", IEEE International Conference on +% Computer Vision (ICCV), Rio de Janeiro, Brazil, Oct. 2007. +% +% 2. Deng Cai, Xiaofei He, Yuxiao Hu, Jiawei Han, and Thomas Huang, +% "Learning a Spatially Smooth Subspace for Face Recognition", CVPR'2007 +% +% 3 Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% 4. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Dec/2011 +% version 2.0 --July/2007 +% version 1.0 --Sep/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + +if (~exist('options','var')) + options = []; +end + +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +else + ReducedDim = 30; +end + +if ~isfield(options,'Regu') || ~options.Regu + bPCA = 1; +else + bPCA = 0; + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.01; + end +end + + +bD = 1; +if ~exist('D','var') || isempty(D) + bD = 0; +end + +if isfield(options,'Kernel') && options.Kernel + K = data; +else + K = constructKernel(data,[],options); +end +clear data; + + +nSmp = size(K,1); +if size(W,1) ~= nSmp + error('W and data mismatch!'); +end +if bD && (size(D,1) ~= nSmp) + error('D and data mismatch!'); +end + + + + +% K_orig = K; + +sumK = sum(K,2); +H = repmat(sumK./nSmp,1,nSmp); +K = K - H - H' + sum(sumK)/(nSmp^2); +K = max(K,K'); +clear H; + +%====================================== +% SVD +%====================================== + +if bPCA + [eigvector_PCA, eigvalue_PCA] = eig(K); + eigvalue_PCA = diag(eigvalue_PCA); + + maxEigValue = max(abs(eigvalue_PCA)); + eigIdx = find(eigvalue_PCA/maxEigValue < 1e-6); + if length(eigIdx) < 1 + [dump,eigIdx] = min(eigvalue_PCA); + end + eigvalue_PCA(eigIdx) = []; + eigvector_PCA(:,eigIdx) = []; + + K = eigvector_PCA; + clear eigvector_PCA + + if bD + DPrime = K*D*K; + DPrime = max(DPrime,DPrime'); + end +else + if bD + DPrime = K*D*K; + else + DPrime = K*K; + end + + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + + DPrime = max(DPrime,DPrime'); +end + +WPrime = K*W*K; +WPrime = max(WPrime,WPrime'); + + + +%====================================== +% Generalized Eigen +%====================================== + + +dimMatrix = size(WPrime,2); + +if ReducedDim > dimMatrix + ReducedDim = dimMatrix; +end + +if isfield(options,'bEigs') + bEigs = options.bEigs; +else + if (dimMatrix > MAX_MATRIX_SIZE) && (ReducedDim < dimMatrix*EIGVECTOR_RATIO) + bEigs = 1; + else + bEigs = 0; + end +end + + +if bEigs + %disp('use eigs to speed up!'); + option = struct('disp',0); + if bPCA && ~bD + [eigvector, eigvalue] = eigs(WPrime,ReducedDim,'la',option); + else + [eigvector, eigvalue] = eigs(WPrime,DPrime,ReducedDim,'la',option); + end + eigvalue = diag(eigvalue); +else + if bPCA && ~bD + [eigvector, eigvalue] = eig(WPrime); + else + [eigvector, eigvalue] = eig(WPrime,DPrime); + end + eigvalue = diag(eigvalue); + + [dump, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + eigvector = eigvector(:,index); + + if ReducedDim < size(eigvector,2) + eigvector = eigvector(:, 1:ReducedDim); + eigvalue = eigvalue(1:ReducedDim); + end +end + +if bPCA + eigvalue_PCA = eigvalue_PCA.^-1; + eigvector = K*(repmat(eigvalue_PCA,1,length(eigvalue)).*eigvector); +end + +% tmpNorm = sqrt(sum((eigvector'*K_orig).*eigvector',2)); +tmpNorm = sqrt(sum((eigvector'*K).*eigvector',2)); +eigvector = eigvector./repmat(tmpNorm',size(eigvector,1),1); + + + + diff --git a/KLPP.m b/KLPP.m new file mode 100644 index 0000000..e488280 --- /dev/null +++ b/KLPP.m @@ -0,0 +1,125 @@ +function [eigvector, eigvalue] = KLPP(W, options, data) +% KLPP: Kernel Locality Preserving Projections +% +% [eigvector, eigvalue] = KLPP(W, options, data) +% +% Input: +% data - +% if options.Kernel = 0 +% Data matrix. Each row vector of fea is a data +% point. +% if options.Kernel = 1 +% Kernel matrix. +% W - Affinity matrix. You can either call "constructW" +% to construct the W, or construct it by yourself. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Please see KGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% eigvalue - The sorted eigvalue of LPP eigen-problem. +% elapse - Time spent on different steps +% +% +% Examples: +% +%----------------------------------------------------------------- +% fea = rand(50,10); +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% +% options.Regu = 0; +% [eigvector, eigvalue] = KLPP(W, options, fea); +% +% feaTest = rand(5,10); +% Ktest = constructKernel(feaTest,fea,options) +% Y = Ktest*eigvector; +% +%----------------------------------------------------------------- +% fea = rand(50,10); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% options.bLDA = 1; +% W = constructW(fea,options); +% +% options.KernelType = 'Gaussian'; +% options.t = 1; +% options.Regu = 1; +% options.ReguAlpha = 0.01; +% [eigvector, eigvalue] = KLPP(W, options, fea); +% +% feaTest = rand(5,10); +% Ktest = constructKernel(feaTest,fea,options) +% Y = Ktest*eigvector; +%-----------------------------------------------------------------% +% +% +% See also constructW, KGE, KDA, KSR +% +% NOTE: +% In paper [3], we present an efficient approach to solve the optimization +% problem in KLPP. We named this approach as Kernel Spectral Regression +% (KSR). I strongly recommend using KSR instead of this KLPP algorithm. +% +%Reference: +% [1] Xiaofei He, and Partha Niyogi, "Locality Preserving Projections" +% Advances in Neural Information Processing Systems 16 (NIPS 2003), +% Vancouver, Canada, 2003. +% +% [2] Xiaofei He, "Locality Preserving Projections" +% PhD's thesis, Computer Science Department, The University of Chicago, +% 2005. +% +% [3] Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for +% Dimensionality Reduction", Department of Computer Science +% Technical Report No. 2856, University of Illinois at Urbana-Champaign +% (UIUCDCS-R-2007-2856), May 2007. +% +% version 2.0 --May/2007 +% version 1.0 --April/2004 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) +% + + +if (~exist('options','var')) + options = []; +end + +if isfield(options,'Kernel') && options.Kernel + K = data; + clear data; +else + K = constructKernel(data,[],options); +end + +nSmp = size(K,1); +D = full(sum(W,2)); +if isfield(options,'Regu') && options.Regu + options.ReguAlpha = options.ReguAlpha*sum(D)/length(D); +end +D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + +options.Kernel = 1; +[eigvector, eigvalue] = KGE(W, D, options, K); + +eigIdx = find(eigvalue < 1e-3); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + + diff --git a/KPCA.m b/KPCA.m new file mode 100644 index 0000000..906062b --- /dev/null +++ b/KPCA.m @@ -0,0 +1,114 @@ +function [eigvector, eigvalue] = KPCA(data, options) +%KPCA Kernel Principal Component Analysis +% +% Usage: +% [eigvector, eigvalue] = KPCA(data, options) +% +% Input: +% data - +% if options.Kernel = 0 +% Data matrix. Each row vector of fea is a data +% point. +% if options.Kernel = 1 +% Kernel matrix. +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% Kernel - 1: data is actually the kernel matrix. +% 0: ordinary data matrix. +% Default: 0 +% +% Please see constructKernel.m for other Kernel options. +% +% ReducedDim - The dimensionality of the reduced subspace. If 0, +% all the dimensions will be kept. Default is 30. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% eigvalue - The sorted eigvalue of PCA eigen-problem. +% +% Examples: +% options.KernelType = 'Gaussian'; +% options.t = 1; +% options.ReducedDim = 4; +% fea = rand(7,10); +% [eigvector,eigvalue] = KPCA(fea,options); +% feaTest = rand(3,10); +% Ktest = constructKernel(feaTest,fea,options) +% Y = Ktest*eigvector; +% +%Reference: +% +% Bernhard Schölkopf, Alexander Smola, Klaus-Robert Müller, “Nonlinear +% Component Analysis as a Kernel Eigenvalue Problem", Neural Computation, +% 10:1299-1319, 1998. +% +% +% version 1.1 --Dec./2011 +% version 1.0 --April/2005 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + + +if isfield(options,'Kernel') && options.Kernel + K = data; +else + K = constructKernel(data,[],options); +end +clear data; + +nSmp = size(K,1); +if (ReducedDim > nSmp) || (ReducedDim <=0) + ReducedDim = nSmp; +end + +sumK = sum(K,2); +H = repmat(sumK./nSmp,1,nSmp); +K = K - H - H' + sum(sumK)/(nSmp^2); +K = max(K,K'); +clear H; + +if nSmp > MAX_MATRIX_SIZE && ReducedDim < nSmp*EIGVECTOR_RATIO + % using eigs to speed up! + option = struct('disp',0); + [eigvector, eigvalue] = eigs(K,ReducedDim,'la',option); + eigvalue = diag(eigvalue); +else + [eigvector, eigvalue] = eig(K); + eigvalue = diag(eigvalue); + + [dump, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + eigvector = eigvector(:,index); +end + + +if ReducedDim < length(eigvalue) + eigvalue = eigvalue(1:ReducedDim); + eigvector = eigvector(:, 1:ReducedDim); +end + +maxEigValue = max(abs(eigvalue)); +eigIdx = find(abs(eigvalue)/maxEigValue < 1e-6); +eigvalue (eigIdx) = []; +eigvector (:,eigIdx) = []; + +for i=1:length(eigvalue) % normalizing eigenvector + eigvector(:,i)=eigvector(:,i)/sqrt(eigvalue(i)); +end; + + + + diff --git a/KSR.m b/KSR.m new file mode 100644 index 0000000..40bcbfd --- /dev/null +++ b/KSR.m @@ -0,0 +1,213 @@ +function [eigvector, LassoCardi] = KSR(options,Responses,K) +% KSR: Kernel Spectral Regression +% +% [eigvector, LassoCardi] = KSR(options,Responses,K) +% +% Input: +% K - Kernel matrix. +% Responses - response vectors. Each column is a response vector +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReguType - 'Ridge': Tikhonov regularization +% L2-norm regularizer +% 'Lasso': L1-norm regularizer +% 'RidgeLasso': Combine Ridge and Lasso +% Default: 'Ridge' +% +% 'Lasso' and 'RidgeLasso' will produce +% sparse solution [See Ref 2,3] +% +% ReguAlpha - The regularization parameter. +% Default value is 0.01. +% +% RidgeAlpha - Only useful if ReguType is 'RidgeLasso', +% 'ReguAlpha' will be the +% regularization parameter for L1-penalty +% 'RidgeAlpha' will be the +% regularization parameter for L2-penalty +% Default value is 0.001. +% +% LASSOway - 'LARs': use LARs to solve the +% LASSO problem. You need to +% specify the cardinality +% requirement in LassoCardi. +% +% 'SLEP': use SLEP to solve the +% LASSO problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% +% If 'Lasso' or 'RidgeLasso' regularization is +% used and 'LARs' is choosed to solve the +% problem, the output eigvector will be a cell, +% each element in the cell will be an eigenvector. +% +% LassoCardi - Only useful when ReguType is 'Lasso' and 'RidgeLasso' +% and LASSOway is 'LARs' +% +% +% Examples: +% +% See KSR_caller.m +% +%Reference: +% +% 1. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% 2. Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% 3. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Approach for Sparse Subspace Learning", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% +% version 3.0 --Jan/2011 +% version 2.0 --Aug/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.001; +end + +LassoCardi = 1; +switch lower(options.ReguType) + case {lower('Ridge')} + + case {lower('Lasso')} + options.RidgeAlpha = 0; + options.ReguType = 'RidgeLasso'; + + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + if strcmpi(options.LASSOway,'LARs') + if isfield(options,'LassoCardi') + LassoCardi = options.LassoCardi; + else + LassoCardi = 10:10:50; + end + LassoCardi(LassoCardi>size(K,1)) = []; + else + if options.ReguAlpha >= 1 + error('ReguAlpha should be a ratio in (0, 1)!'); + end + end + case {lower('RidgeLasso')} + if ~isfield(options,'RidgeAlpha') + options.RidgeAlpha = 0.001; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + if strcmpi(options.LASSOway,'LARs') + if isfield(options,'LassoCardi') + LassoCardi = options.LassoCardi; + else + LassoCardi = 10:10:50; + end + LassoCardi(LassoCardi>size(K,1)) = []; + else + if options.ReguAlpha >= 1 + error('ReguAlpha should be a ratio in (0, 1)!'); + end + end + otherwise + error('ReguType does not exist!'); +end + + +switch lower(options.ReguType) + case {lower('Ridge')} + if options.ReguAlpha > 0 + for i=1:size(K,1) + K(i,i) = K(i,i) + options.ReguAlpha; + end + end + + R = chol(K); + eigvector = R\(R'\Responses); + + tmpNorm = sqrt(sum((eigvector'*K).*eigvector',2)); + eigvector = eigvector./repmat(tmpNorm',size(eigvector,1),1); + case {lower('RidgeLasso')} + nVector = size(Responses,2); + switch lower(options.LASSOway) + case {lower('LARs')} + if options.RidgeAlpha > 0 + for i=1:size(K,1) + K(i,i) = K(i,i) + options.RidgeAlpha; + end + end + Gram = K'*K; + Gram = max(Gram,Gram'); + eigvector = cell(nVector,1); + for i = 1:nVector + eigvector_T = lars(K, Responses(:,i),'lasso', -(max(LassoCardi)+5),1,Gram,LassoCardi); + eigvector{i} = eigvector_T; + end + case {lower('SLEP')} + eigvector = zeros(size(K,2),nVector); + opts=[]; + opts.rFlag=1; % the input parameter 'ReguAlpha' is a ratio in (0, 1) + opts.init = 2; + if options.RidgeAlpha > 0 + opts.rsL2=options.RidgeAlpha; + end + for i = 1:nVector + eigvector(:,i) = LeastR(K, Responses(:,i), options.ReguAlpha, opts); + end + tmpNorm = sqrt(sum((eigvector'*K).*eigvector',2)); + eigvector = eigvector./repmat(tmpNorm',size(eigvector,1),1); + otherwise + error('Method does not exist!'); + end + otherwise + error('ReguType does not exist!'); +end + + +if strcmpi(options.ReguType,'RidgeLasso') && strcmpi(options.LASSOway,'LARs') + eigvectorAll = eigvector; + eigvector = cell(length(LassoCardi),1); + + for i = 1:length(eigvectorAll) + eigvector_T = full(eigvectorAll{i}); + [dump,tn] = size(eigvector_T); + tCar = zeros(tn,1); + for k = 1:tn + tCar(k) = length(find(eigvector_T(:,k))); + end + + for cardidx = 1:length(LassoCardi) + ratio = LassoCardi(cardidx); + iMin = find(tCar == ratio); + if isempty(iMin) + error('Card dose not exist!'); + end + tmpEigvec = eigvector_T(:,iMin(end))/sqrt(eigvector_T(:,iMin(end))'*K*eigvector_T(:,iMin(end))); + eigvector{cardidx} = [eigvector{cardidx} tmpEigvec]; + end + end +end + + + + + diff --git a/KSR_caller.m b/KSR_caller.m new file mode 100644 index 0000000..37095bd --- /dev/null +++ b/KSR_caller.m @@ -0,0 +1,268 @@ +function [eigvector, LassoCardi] = KSR_caller(options, data) +% KSR: Kernel Spectral Regression +% +% [eigvector, LassoCardi] = KSR_caller(options, data) +% +% Input: +% data - data matrix. Each row vector of data is a +% sample vector. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Kernel - 1: data is actually the kernel matrix. +% 0: ordinary data matrix. +% Default: 0 +% +% gnd - Colunm vector of the label information +% for each data point. +% If gnd is provided, SR will give the +% SRDA solution [See Ref 7,8] +% +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If gnd is not provided, W is required and +% SR will give the RLPI (RLPP) solution +% [See Ref 1] +% +% ReducedDim - The number of dimensions. If gnd is +% provided, ReducedDim=c-1 where c is the number +% of classes. Default ReducedDim = 30. +% +% Please see KSR.m for other options. +% Please see constructKernel.m for other Kernel options. +% +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = K(x,:)*eigvector +% will be the embedding result of x. +% K(x,:) = [K(x1,x),K(x2,x),...K(xm,x)] +% +% If 'Lasso' or 'RidgeLasso' regularization is +% used and 'LARs' is choosed to solve the +% problem, the output eigvector will be a cell, +% each element in the cell will be an eigenvector. +% +% LassoCardi - Only useful when ReguType is 'Lasso' and 'RidgeLasso' +% and LASSOway is 'LARs' +% +% +% +% +%=================================================================== +% Examples: +% +% (Supervised case with L2-norm (ridge) regularizer, SR-KDA) +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.gnd = gnd; +% options.ReguAlpha = 0.01; +% options.ReguType = 'Ridge'; +% +% options.KernelType = 'Gaussian'; +% options.t = 5; +% +% Ktrain = constructKernel(fea,[],options); +% options.Kernel = 1; +% [eigvector] = KSR_caller(options, Ktrain); +% +% Ytrain = Ktrain*eigvector; % Ytrain is training samples in the KSR subspace +% +% feaTest = rand(3,70); +% Ktest = constructKernel(feaTest,fea,options); +% Ytest = Ktest*eigvector; % Ytest is test samples in the KSR subspace +% +%------------------------------------------------------------------- +% (Unsupervised case with L2-norm (ridge) regularizer, SR-KLPP) +% +% fea = rand(50,70); +% options = []; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% +% options = []; +% options.W = W; +% options.ReguType = 'Ridge'; +% options.ReguAlpha = 0.01; +% options.ReducedDim = 10; +% +% options.KernelType = 'Gaussian'; +% options.t = 5; +% +% Ktrain = constructKernel(fea,[],options); +% options.Kernel = 1; +% [eigvector] = KSR_caller(options, Ktrain); +% +% feaTest = rand(3,70); +% Ktest = constructKernel(feaTest,fea,options); +% Y = Ktest*eigvector; % Y is samples in the KSR subspace +% +%------------------------------------------------------------------- +% (Supervised case with L1-norm (Lasso) regularizer, SR-SparseKDA) +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.gnd = gnd; +% options.ReguType = 'RidgeLasso'; +% +% options.KernelType = 'Gaussian'; +% options.t = 5; +% Ktrain = constructKernel(fea,[],options); +% options.Kernel = 1; +% +% feaTest = rand(3,70); +% Ktest = constructKernel(feaTest,fea,options); +% +% ---- use SLEP ---- Please see http://www.public.asu.edu/~jye02/Software/SLEP/ for details on SLEP. +% options.ReguAlpha = 0.3; +% options.LASSOway = 'SLEP'; +% eigvector = KSR_caller(options, Ktrain); +% +% Ytrain = Ktrain*eigvector; % Ytrain is training samples in the sparse KSR subspace +% Ytest = Ktest*eigvector; % Ytest is test samples in the sparse KSR subspace +% +% ---- use LARs ---- +% options.RidgeAlpha = 0.001; +% options.LASSOway = 'LARs'; +% options.LassoCardi = [10:5:40]; +% [eigvectorAll,LassoCardi] = KSR_caller(options, Ktrain); +% +% for i = 1:length(LassoCardi) +% eigvector = eigvectorAll{i}; %projective functions with cardinality LassoCardi(i) +% Ytrain = Ktrain*eigvector; % Ytrain is training samples in the sparse KSR subspace +% Ytest = Ktest*eigvector; % Ytest is test samples in the sparse KSR subspace +% end +% +%------------------------------------------------------------------- +% (Unsupervised case with L2-norm (ridge) regularizer, SR-SparseKLPP) +% +% fea = rand(50,70); +% options = []; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% +% options = []; +% options.W = W; +% options.ReguType = 'RidgeLasso'; +% options.ReducedDim = 10; +% +% options.KernelType = 'Gaussian'; +% options.t = 5; +% Ktrain = constructKernel(fea,[],options); +% options.Kernel = 1; +% +% feaTest = rand(3,70); +% Ktest = constructKernel(feaTest,fea,options); +% +% ---- use SLEP ---- Please see http://www.public.asu.edu/~jye02/Software/SLEP/ for details on SLEP. +% options.ReguAlpha = 0.3; +% options.LASSOway = 'SLEP'; +% eigvector = KSR_caller(options, Ktrain); +% +% Ytrain = Ktrain*eigvector; % Ytrain is training samples in the sparse KSR subspace +% Ytest = Ktest*eigvector; % Ytest is test samples in the sparse KSR subspace +% +% ---- use LARs ---- +% options.RidgeAlpha = 0.001; +% options.LASSOway = 'LARs'; +% options.LassoCardi = [10:5:40]; +% [eigvectorAll,LassoCardi] = KSR_caller(options, Ktrain); +% +% for i = 1:length(LassoCardi) +% eigvector = eigvectorAll{i}; %projective functions with cardinality LassoCardi(i) +% Ytrain = Ktrain*eigvector; % Ytrain is training samples in the sparse KSR subspace +% Ytest = Ktest*eigvector; % Ytest is test samples in the sparse KSR subspace +% end +% +%=================================================================== +% +%Reference: +% +% 1. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% 2. Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% 3. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Approach for Sparse Subspace Learning", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% +% version 3.0 --Jan/2011 +% version 2.0 --Aug/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + +if isfield(options,'Kernel') && options.Kernel + K = data; + clear data; + K = max(K,K'); +else + K = constructKernel(data,[],options); +end + +nSmp = size(K,1); +if (ReducedDim > nSmp) || (ReducedDim <=0) + ReducedDim = nSmp; +end + + +bSup = 0; +if isfield(options,'gnd') + gnd = options.gnd; + bSup = 1; + if length(gnd) ~= nSmp + error('Label vector wrong!'); + end +else + if ~isfield(options,'W') || (size(options.W,1) ~= nSmp) + error('Graph Error!'); + end + W = options.W; +end + + + +if bSup + Label = unique(gnd); + nClass = length(Label); + + rand('state',0); + Y = rand(nClass,nClass); + Z = zeros(nSmp,nClass); + for i=1:nClass + idx = find(gnd==Label(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); + end + Z(:,1) = ones(nSmp,1); + [Y,R] = qr(Z,0); + Y(:,1) = []; +else + Y = Eigenmap(W,ReducedDim); +end + + +[eigvector, LassoCardi] = KSR(options, Y, K); + + + diff --git a/LCCF.m b/LCCF.m new file mode 100644 index 0000000..9650b99 --- /dev/null +++ b/LCCF.m @@ -0,0 +1,82 @@ +function [U_final, V_final, nIter_final, objhistory_final] = LCCF(X, k, W, options, U, V) +% Locally Consistant Concept Factorization (LCCF) +% +% where +% X +% Notation: +% X ... (mFea x nSmp) data matrix +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% +% You only need to provide the above four inputs. +% +% X = X*U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Jiawei Han, "Locally Consistent Concept +% Factorization for Document Clustering", IEEE Transactions on Knowledge +% and Data Engineering, Vol. 23, No. 6, pp. 902-913, 2011. +% +% +% version 2.0 --April/2010 +% version 1.0 --Dec./2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +nSmp=size(X,2); + +if ~isfield(options,'error') + options.error = 1e-5; +end +if ~isfield(options, 'maxIter') + options.maxIter = []; +end + +if ~isfield(options,'nRepeat') + options.nRepeat = 10; +end + +if ~isfield(options,'minIter') + options.minIter = 30; +end + +if ~isfield(options,'meanFitRatio') + options.meanFitRatio = 0.1; +end + +if ~isfield(options,'alpha') + options.alpha = 100; +end + +if isfield(options,'alpha_nSmp') && options.alpha_nSmp + options.alpha = options.alpha*nSmp; +end + +if ~exist('U','var') + U = []; + V = []; +end + +K = constructKernel(X',[],options); + +if isfield(options,'weight') && strcmpi(options.weight,'NCW') + D_mhalf = sum(K,2).^-.5; + D_mhalf = spdiags(D_mhalf,0,nSmp,nSmp); + K = D_mhalf*K*D_mhalf; +end + +if ~isfield(options,'Optimization') + options.Optimization = 'Multiplicative'; +end + +switch lower(options.Optimization) + case {lower('Multiplicative')} + [U_final, V_final, nIter_final,objhistory_final] = LCCF_Multi(K, k, W, options, U, V); + otherwise + error('optimization method does not exist!'); +end diff --git a/LCCF_Multi.m b/LCCF_Multi.m new file mode 100644 index 0000000..f9fa48a --- /dev/null +++ b/LCCF_Multi.m @@ -0,0 +1,246 @@ +function [U_final, V_final, nIter_final, objhistory_final] = LCCF_Multi(K, k, W, options, U, V) +% Locally Consistant Concept Factorization (LCCF) +% +% where +% X +% Notation: +% K ... (nSmp x nSmp) kernel matrix +% k ... number of hidden factors +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% +% You only need to provide the above four inputs. +% +% X = X*U*V' +% +% References: +% [1] Deng Cai, Xiaofei He, Jiawei Han, "Locally Consistent Concept +% Factorization for Document Clustering", IEEE Transactions on Knowledge +% and Data Engineering, Vol. 23, No. 6, pp. 902-913, 2011. +% +% +% version 2.0 --April/2010 +% version 1.0 --Dec./2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +differror = options.error; +maxIter = options.maxIter; +nRepeat = options.nRepeat; +minIterOrig = options.minIter; +minIter = minIterOrig-1; +meanFitRatio = options.meanFitRatio; + +alpha = options.alpha; + +Norm = 2; +NormV = 1; + +nSmp = size(K,1); + +if alpha < 0 + alpha = 0; +end +W = alpha*W; +DCol = full(sum(W,2)); +D = spdiags(DCol,0,speye(size(W,1))); +L = D - W; +if isfield(options,'NormW') && options.NormW + D_mhalf = DCol.^-.5; + + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + L = (tmpD_mhalf.*L).*tmpD_mhalf'; + clear D_mhalf tmpD_mhalf; + + L = max(L, L'); +end + + +if isempty(U) + U = abs(rand(nSmp,k)); + V = abs(rand(nSmp,k)); +else + nRepeat = 1; +end +[U,V] = NormalizeUV(K, U, V, NormV, Norm); + +selectInit = 1; +if nRepeat == 1 + selectInit = 0; + minIterOrig = 0; + minIter = 0; + if isempty(maxIter) + [obj_NMFhistory, obj_Laphistory] = CalculateObj(K, U, V, L); + objhistory = obj_NMFhistory + obj_Laphistory; + meanFit = objhistory*10; + else + if isfield(options,'Converge') && options.Converge + [obj_NMFhistory, obj_Laphistory] = CalculateObj(K, U, V, L); + objhistory = obj_NMFhistory + obj_Laphistory; + end + end +else + if isfield(options,'Converge') && options.Converge + error('Not implemented!'); + end +end + +tryNo = 0; +while tryNo < nRepeat + tmp_T = cputime; + tryNo = tryNo+1; + nIter = 0; + maxErr = 1; + while(maxErr > differror) + % ===================== update V ======================== + KU = K*U; % n^2k + UKU = U'*KU; % nk^2 + VUKU = V*UKU; % nk^2 + + if alpha > 0 + WV = W*V; + DV = repmat(DCol,1,k).*V; + + KU = KU + WV; + VUKU = VUKU + DV; + end + + V = V.*(KU./max(VUKU,1e-10)); + clear WV DV KU UKU VUKU; + % ===================== update U ======================== + KV = K*V; % n^2k + VV = V'*V; % nk^2 + KUVV = K*U*VV; % n^2k + + U = U.*(KV./max(KUVV,1e-10)); + clear KV VV KUVV; + + nIter = nIter + 1; + if nIter > minIter + [U,V] = NormalizeUV(K, U, V, NormV, Norm); + if selectInit + [obj_NMFhistory, obj_Laphistory] = CalculateObj(K, U, V, L); + objhistory = obj_NMFhistory + obj_Laphistory; + maxErr = 0; + else + if isempty(maxIter) + [obj_NMF, obj_Lap] = CalculateObj(K, U, V, L); + newobj = obj_NMF + obj_Lap; + objhistory = [objhistory newobj]; %#ok + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*newobj; + maxErr = (meanFit-newobj)/meanFit; + else + if isfield(options,'Converge') && options.Converge + [obj_NMF, obj_Lap] = CalculateObj(K, U, V, L); + newobj = obj_NMF + obj_Lap; + objhistory = [objhistory newobj]; %#ok + end + maxErr = 1; + if nIter >= maxIter + maxErr = 0; + if isfield(options,'Converge') && options.Converge + else + objhistory = 0; + end + end + end + end + end + end + + + if tryNo == 1 + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + else + if objhistory(end) < objhistory_final(end) + U_final = U; + V_final = V; + nIter_final = nIter; + objhistory_final = objhistory; + end + end + + if selectInit + if tryNo < nRepeat + %re-start + U = abs(rand(nSmp,k)); + V = abs(rand(nSmp,k)); + + [U,V] = NormalizeUV(K, U, V, NormV, Norm); + else + tryNo = tryNo - 1; + minIter = 0; + selectInit = 0; + U = U_final; + V = V_final; + objhistory = objhistory_final; + meanFit = objhistory*10; + end + end +end + +nIter_final = nIter_final + minIterOrig; + +Norm = 2; +NormV = 0; + +[U_final,V_final] = NormalizeUV(K, U_final, V_final, NormV, Norm); + + + +%========================================================================== + +function [obj_NMF, obj_Lap, dV] = CalculateObj(K, U, V, L, deltaVU, dVordU) +if ~exist('deltaVU','var') + deltaVU = 0; +end +if ~exist('dVordU','var') + dVordU = 1; +end +dV = []; + +UK = U'*K; % n^2k +UKU = UK*U; % nk^2 +VUK = V*UK; % n^2k +VV = V'*V; % nk^2 +obj_NMF = sum(diag(K))-2*sum(diag(VUK))+sum(sum(UKU.*VV)); +if deltaVU + if dVordU + dV = V*UKU - UK'+ L*V; %nk^2 + else + dV = (VUK-K)'*V; %n^2k + end +end +obj_Lap = sum(sum((L*V).*V)); + + +function [U, V] = NormalizeUV(K, U, V, NormV, Norm) +k = size(U,2); +if Norm == 2 + if NormV + norms = max(1e-15,sqrt(sum(V.^2,1)))'; + V = V*spdiags(norms.^-1,0,k,k); + U = U*spdiags(norms,0,k,k); + else + norms = max(1e-15,sqrt(sum(U.*(K*U),1)))'; + U = U*spdiags(norms.^-1,0,k,k); + V = V*spdiags(norms,0,k,k); + end +else + if NormV + norms = max(1e-15,sum(abs(V),1))'; + V = V*spdiags(norms.^-1,0,k,k); + U = U*spdiags(norms,0,k,k); + else + norms = max(1e-15,sum(U.*(K*U),1))'; + U = U*spdiags(norms.^-1,0,k,k); + V = V*spdiags(norms,0,k,k); + end +end + diff --git a/LCGMM.m b/LCGMM.m new file mode 100644 index 0000000..7b33b8b --- /dev/null +++ b/LCGMM.m @@ -0,0 +1,294 @@ +function [pkx_final, cluster_final, LogL_final] = LCGMM(X, k, W, options) +% Local Consistent Gaussian Mixture Model (LCGMM) +% +% where +% X +% Notation: +% X ... (nSmp x mFea) observed data matrix +% nSmp ... number of samples +% mFea ... number of features +% +% K ... number of clusters +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% +% You only need to provide the above four inputs. +% +% pkx ... P(z|x) +% R ... covariance matrix +% mu ... mean +% +% +% References: +% [1] Jialu Liu, Deng Cai, Xiaofei He, "Gaussian Mixture Model with +% Local Consistency", AAAI 2010. +% [2] Jialu Liu, "Notes on Local Consistent Gaussian Mixture Model(LCGMM)", +% Online available at http://relau.com/jialuliu/technical_notes/LCGMM.pdf +% [accessed 27-Dec-2011]. +% +% version 2.0 --Dec/2011 +% version 1.0 --Dec/2009 +% +% Written by Jialu Liu (remenberl AT gmail.com) +% Deng Cai (dengcai AT gmail.com) +% +ZERO_OFFSET = 1e-200; + +differror = 1e-7; +if isfield(options,'error') + differror = options.error; +end + +lambda = 0.1; +if isfield(options,'lambda') + lambda = options.lambda; +end + +nRepeat = 5; +if isfield(options,'nRepeat') + nRepeat = options.nRepeat; +end + +maxIter = 100; +if isfield(options,'maxIter') + maxIter = options.maxIter; +end + +minIterOrig = 5; +if isfield(options,'minIter') + minIterOrig = options.minIter; +end +minIter = minIterOrig-1; + +meanFitRatio = 0.1; +if isfield(options,'meanFitRatio') + meanFitRatio = options.meanFitRatio; +end +meanFitControl = 1; +if isfield(options,'meanFitControl') + meanFitControl = options.meanFitControl; +end + + +if ~isfield(options,'InitWay') + options.InitWay = 'kmeans'; +end + +show = 0; +if isfield(options,'show') + show = options.show; +end + +debug = 0; +if isfield(options,'debug') + debug = options.debug; +end + + + +% init mixture +[nSmp mFea] = size(X); + +if lambda > 0 + DCol = full(sum(W,2)); + D = spdiags(DCol,0,nSmp,nSmp); + L = D - W; +end + +[cluster,pkx,LogL] = GMM_init(X,k,options); +meanFit = LogL/10; + + +tryNo = 0; +selectInit = 1; +nIter = 0; +while tryNo < nRepeat + tryNo = tryNo+1; + maxErr = 1; + retry = 0; + while(maxErr > differror || maxErr==-1) + % EM iteration + alertFlag = 0; + for kidx=1:k + % compute pi + cluster(kidx).pb = sum(pkx(:,kidx)); + if cluster(kidx).pb < 1e-20 + retry = 1; + break; + end + % compute Tk + if lambda > 0 + Tk = 1 - lambda * sum(bsxfun(@minus, pkx(:,kidx), pkx(:,kidx)') .* W, 2) ./ (pkx(:,kidx) + ZERO_OFFSET); + else + Tk = ones(nSmp, 1); + end + + if min(Tk) < 0 && alertFlag == 0 && debug == 1 + alertFlag = 1; + disp('The covariance matrix might not be positive semidefinate since lambda is too big such that some value of Tik is negative.'); + end + + % compute mean + cluster(kidx).mu = ((pkx(:,kidx) .* Tk)' * X) / cluster(kidx).pb; + + % compute covariance matrix + Y1 = X-repmat(cluster(kidx).mu,nSmp,1); + Y2 = bsxfun(@times,sqrt(pkx(:,kidx) .* Tk), Y1); + R = (Y2'*Y2) /cluster(kidx).pb; + %for elemetns in Tk which are negative + Y3 = bsxfun(@times,sqrt(pkx(:,kidx) .* (-(Tk<0) .* Tk)), Y1); + R = R - 2 * (Y3'*Y3) /cluster(kidx).pb; + + % + clear Y2; + R = max(R,R'); + + + detR = det(R); + if detR <= 0 + retry = 1; + break; + end + + cluster(kidx).cov = R; + const = -(mFea*log(2*pi) + log(detR))/2; + + Y2 = R\Y1'; + Y2 = -Y2'/2; + pkx(:,kidx) = dot(Y1,Y2,2)+const; + clear Y1 Y2 R; + end + + if retry + break; + end + + llmax=max(pkx,[],2); + pkx =exp( pkx-repmat(llmax,1,k) ); + pkx = pkx.*repmat([cluster(:).pb],nSmp,1); + ss = sum(pkx,2); + llnew = sum(log(ss)+llmax); + pkx = pkx./(repmat(ss,1,k)); + + % compute new likelihood + if lambda > 0 + llnew = llnew - sum(sum((log(pkx' + ZERO_OFFSET) * L).* pkx')) * lambda / 2; + end + + LogL = [LogL llnew]; + % + nIter = nIter + 1; + + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*llnew; + maxErr = (llnew-meanFit)/meanFit; + if show + if length(LogL) > 1 + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(LogL(end)),' deltaLogL: ',num2str(LogL(end)-LogL(end-1)),' maxErr:',num2str(maxErr)]); + else + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(LogL(end)),' maxErr:',num2str(maxErr)]); + end + end + if nRepeat > 1 && selectInit + maxErr = 1; + end + if ~meanFitControl + maxErr = 1; + end + if nIter > minIter + if selectInit + maxErr = 0; + else + if nIter >= maxIter + maxErr = 0; + end + end + end + end + + if retry && ~(tryNo == nRepeat && nIter >= nIter_final) + tryNo = tryNo - 1; + [cluster,pkx,LogL] = GMM_init(X,k,options); + meanFit = LogL/10; + nIter = 0; + continue; + end + + if tryNo == 1 + pkx_final = pkx; + cluster_final = cluster; + LogL_final = LogL; + nIter_final = nIter; + else + if LogL(end) > LogL_final(end) + pkx_final = pkx; + cluster_final = cluster; + LogL_final = LogL; + nIter_final = nIter; + end + end + + if selectInit + if tryNo < nRepeat + [cluster,pkx,LogL] = GMM_init(X,k,options); + meanFit = LogL/10; + nIter = 0; + else + tryNo = tryNo - 1; + selectInit = 0; + pkx = pkx_final; + cluster = cluster_final; + LogL = LogL_final; + nIter = nIter_final; + meanFit = LogL(end)/10; + end + end +end + + + +function [cluster,pkx,llnew] = GMM_init(X,k,options) + ZERO_OFFSET = 1e-200; + [nSmp, mFea] = size(X); + if strcmpi(options.InitWay,'kmeans') + kmeansres = litekmeans(X,k,'maxIter',10); + residx = unique(kmeansres); + for kidx=1:k + smpidx = kmeansres==residx(kidx); + cluster(kidx).mu = mean(X(smpidx,:),1); + cluster(kidx).pb = 1/k; + end + else + permutation = randperm(nSmp); + nSmpClass = floor(nSmp/k); + for kidx=1:k + cluster(kidx).mu = mean(X(permutation((kidx-1)*nSmpClass+1:kidx*nSmpClass),:),1); + cluster(kidx).pb = 1/k; + end + end + + R = (nSmp-1)*cov(X)/nSmp; + R = max(R,R'); + + % EM iteration + + pkx=zeros(nSmp,k); + detR = det(R); + if detR <= 0 + error('The covariance matrix is not positive definite. Use PCA to reduce the dimensions first!'); + end + const = -(mFea*log(2*pi) +log(detR))/2; + for kidx=1:k + Y1=X-repmat(cluster(kidx).mu,nSmp,1); + Y2 = R\Y1'; + Y2 = -Y2'/2; + pkx(:,kidx) = dot(Y1,Y2,2)+const; + clear Y1 Y2; + end + clear R; + llmax=max(pkx,[],2); + pkx =exp( pkx-repmat(llmax,1,k) ); + pkx = pkx.*repmat([cluster(:).pb],nSmp,1); + ss = sum(pkx,2); + llnew = sum(log(ss)+llmax); + pkx = pkx./(repmat(ss,1,k)); diff --git a/LDA.m b/LDA.m new file mode 100644 index 0000000..32f7633 --- /dev/null +++ b/LDA.m @@ -0,0 +1,280 @@ +function [eigvector, eigvalue] = LDA(gnd,options,data) +% LDA: Linear Discriminant Analysis +% +% [eigvector, eigvalue] = LDA(gnd, options, data) +% +% Input: +% data - Data matrix. Each row vector of fea is a data point. +% gnd - Colunm vector of the label information for each +% data point. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Regu - 1: regularized solution, +% a* = argmax (a'X'WXa)/(a'X'Xa+ReguAlpha*I) +% 0: solve the sinularity problem by SVD +% Default: 0 +% +% ReguAlpha - The regularization parameter. Valid +% when Regu==1. Default value is 0.1. +% +% ReguType - 'Ridge': Tikhonov regularization +% 'Custom': User provided +% regularization matrix +% Default: 'Ridge' +% regularizerR - (nFea x nFea) regularization +% matrix which should be provided +% if ReguType is 'Custom'. nFea is +% the feature number of data +% matrix +% Fisherface - 1: Fisherface approach +% PCARatio = nSmp - nClass +% Default: 0 +% +% PCARatio - The percentage of principal +% component kept in the PCA +% step. The percentage is +% calculated based on the +% eigenvalue. Default is 1 +% (100%, all the non-zero +% eigenvalues will be kept. +% If PCARatio > 1, the PCA step +% will keep exactly PCARatio principle +% components (does not exceed the +% exact number of non-zero components). +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of LDA eigen-problem. +% elapse - Time spent on different steps +% +% Examples: +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.Fisherface = 1; +% [eigvector, eigvalue] = LDA(gnd, options, fea); +% Y = fea*eigvector; +% +% +% See also LPP, constructW, LGE +% +% +% +%Reference: +% +% Deng Cai, Xiaofei He, Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis", IEEE Transactions on Knowledge and +% Data Engineering, 2007. +% +% Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Dec/2011 +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.1 --Feb/2006 +% version 1.0 --April/2004 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + + +if (~exist('options','var')) + options = []; +end + +if ~isfield(options,'Regu') || ~options.Regu + bPCA = 1; + if ~isfield(options,'PCARatio') + options.PCARatio = 1; + end +else + bPCA = 0; + if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; + end + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end +end + +% ====== Initialization +[nSmp,nFea] = size(data); +if length(gnd) ~= nSmp + error('gnd and data mismatch!'); +end + +classLabel = unique(gnd); +nClass = length(classLabel); +Dim = nClass - 1; + +if bPCA && isfield(options,'Fisherface') && options.Fisherface + options.PCARatio = nSmp - nClass; +end + + +if issparse(data) + data = full(data); +end +sampleMean = mean(data,1); +data = (data - repmat(sampleMean,nSmp,1)); + + + +bChol = 0; +if bPCA && (nSmp > nFea+1) && (options.PCARatio >= 1) + DPrime = data'*data; + DPrime = max(DPrime,DPrime'); + [R,p] = chol(DPrime); + + if p == 0 + bPCA = 0; + bChol = 1; + end +end + + +%====================================== +% SVD +%====================================== +if bPCA + [U, S, V] = mySVD(data); + [U, S, V]=CutonRatio(U,S,V,options); + eigvalue_PCA = full(diag(S)); + + data = U; + eigvector_PCA = V*spdiags(eigvalue_PCA.^-1,0,length(eigvalue_PCA),length(eigvalue_PCA)); +else + if ~bChol + DPrime = data'*data; + +% options.ReguAlpha = nSmp*options.ReguAlpha; + + switch lower(options.ReguType) + case {lower('Ridge')} + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + case {lower('Tensor')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + case {lower('Custom')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + otherwise + error('ReguType does not exist!'); + end + + DPrime = max(DPrime,DPrime'); + end +end + + +[nSmp,nFea] = size(data); + +Hb = zeros(nClass,nFea); +for i = 1:nClass, + index = find(gnd==classLabel(i)); + classMean = mean(data(index,:),1); + Hb (i,:) = sqrt(length(index))*classMean; +end + + +if bPCA + [dumpVec,eigvalue,eigvector] = svd(Hb,'econ'); + + eigvalue = diag(eigvalue); + eigIdx = find(eigvalue < 1e-3); + eigvalue(eigIdx) = []; + eigvector(:,eigIdx) = []; + + eigvalue = eigvalue.^2; + eigvector = eigvector_PCA*eigvector; +else + WPrime = Hb'*Hb; + WPrime = max(WPrime,WPrime'); + + dimMatrix = size(WPrime,2); + if Dim > dimMatrix + Dim = dimMatrix; + end + + + if isfield(options,'bEigs') + bEigs = options.bEigs; + else + if (dimMatrix > MAX_MATRIX_SIZE) && (ReducedDim < dimMatrix*EIGVECTOR_RATIO) + bEigs = 1; + else + bEigs = 0; + end + end + + if bEigs + %disp('use eigs to speed up!'); + option = struct('disp',0); + if bChol + option.cholB = 1; + [eigvector, eigvalue] = eigs(WPrime,R,Dim,'la',option); + else + [eigvector, eigvalue] = eigs(WPrime,DPrime,Dim,'la',option); + end + eigvalue = diag(eigvalue); + else + [eigvector, eigvalue] = eig(WPrime,DPrime); + eigvalue = diag(eigvalue); + + [junk, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + eigvector = eigvector(:,index); + + if Dim < size(eigvector,2) + eigvector = eigvector(:, 1:Dim); + eigvalue = eigvalue(1:Dim); + end + end +end + +for i = 1:size(eigvector,2) + eigvector(:,i) = eigvector(:,i)./norm(eigvector(:,i)); +end + + + + + + + +function [U, S, V]=CutonRatio(U,S,V,options) + if ~isfield(options, 'PCARatio') + options.PCARatio = 1; + end + + eigvalue_PCA = full(diag(S)); + if options.PCARatio > 1 + idx = options.PCARatio; + if idx < length(eigvalue_PCA) + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end + elseif options.PCARatio < 1 + sumEig = sum(eigvalue_PCA); + sumEig = sumEig*options.PCARatio; + sumNow = 0; + for idx = 1:length(eigvalue_PCA) + sumNow = sumNow + eigvalue_PCA(idx); + if sumNow >= sumEig + break; + end + end + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end diff --git a/LGE.m b/LGE.m new file mode 100644 index 0000000..a1d7237 --- /dev/null +++ b/LGE.m @@ -0,0 +1,296 @@ +function [eigvector, eigvalue] = LGE(W, D, options, data) +% LGE: Linear Graph Embedding +% +% [eigvector, eigvalue] = LGE(W, D, options, data) +% +% Input: +% data - data matrix. Each row vector of data is a +% sample vector. +% W - Affinity graph matrix. +% D - Constraint graph matrix. +% LGE solves the optimization problem of +% a* = argmax (a'data'WXa)/(a'data'DXa) +% Default: D = I +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReducedDim - The dimensionality of the reduced +% subspace. If 0, all the dimensions +% will be kept. Default is 30. +% +% Regu - 1: regularized solution, +% a* = argmax (a'X'WXa)/(a'X'DXa+ReguAlpha*I) +% 0: solve the sinularity problem by SVD (PCA) +% Default: 0 +% +% ReguAlpha - The regularization parameter. Valid +% when Regu==1. Default value is 0.1. +% +% ReguType - 'Ridge': Tikhonov regularization +% 'Custom': User provided +% regularization matrix +% Default: 'Ridge' +% regularizerR - (nFea x nFea) regularization +% matrix which should be provided +% if ReguType is 'Custom'. nFea is +% the feature number of data +% matrix +% +% PCARatio - The percentage of principal +% component kept in the PCA +% step. The percentage is +% calculated based on the +% eigenvalue. Default is 1 +% (100%, all the non-zero +% eigenvalues will be kept. +% If PCARatio > 1, the PCA step +% will keep exactly PCARatio principle +% components (does not exceed the +% exact number of non-zero components). +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% sample vector (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of the eigen-problem. +% elapse - Time spent on different steps +% +% +% +% Examples: +% +% See also LPP, NPE, IsoProjection, LSDA. +% +% +%Reference: +% +% 1. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for Efficient +% Regularized Subspace Learning", IEEE International Conference on +% Computer Vision (ICCV), Rio de Janeiro, Brazil, Oct. 2007. +% +% 2. Deng Cai, Xiaofei He, Yuxiao Hu, Jiawei Han, and Thomas Huang, +% "Learning a Spatially Smooth Subspace for Face Recognition", CVPR'2007 +% +% 3. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Subspace Learning Framework for Content-Based Image Retrieval", ACM +% Multimedia 2007, Augsburg, Germany, Sep. 2007. +% +% 4. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Dec/2011 +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.0 --Sep/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + +if (~exist('options','var')) + options = []; +end + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + +if ~isfield(options,'Regu') || ~options.Regu + bPCA = 1; + if ~isfield(options,'PCARatio') + options.PCARatio = 1; + end +else + bPCA = 0; + if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; + end + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end +end + +bD = 1; +if ~exist('D','var') || isempty(D) + bD = 0; +end + + +[nSmp,nFea] = size(data); +if size(W,1) ~= nSmp + error('W and data mismatch!'); +end +if bD && (size(D,1) ~= nSmp) + error('D and data mismatch!'); +end + +bChol = 0; +if bPCA && (nSmp > nFea) && (options.PCARatio >= 1) + if bD + DPrime = data'*D*data; + else + DPrime = data'*data; + end + DPrime = full(DPrime); + DPrime = max(DPrime,DPrime'); + [R,p] = chol(DPrime); + if p == 0 + bPCA = 0; + bChol = 1; + end +end + +%====================================== +% SVD +%====================================== + +if bPCA + [U, S, V] = mySVD(data); + [U, S, V]=CutonRatio(U,S,V,options); + eigvalue_PCA = full(diag(S)); + if bD + data = U*S; + eigvector_PCA = V; + + DPrime = data'*D*data; + DPrime = max(DPrime,DPrime'); + else + data = U; + eigvector_PCA = V*spdiags(eigvalue_PCA.^-1,0,length(eigvalue_PCA),length(eigvalue_PCA)); + end +else + if ~bChol + if bD + DPrime = data'*D*data; + else + DPrime = data'*data; + end + + switch lower(options.ReguType) + case {lower('Ridge')} + if options.ReguAlpha > 0 + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + end + case {lower('Tensor')} + if options.ReguAlpha > 0 + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + end + case {lower('Custom')} + if options.ReguAlpha > 0 + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + end + otherwise + error('ReguType does not exist!'); + end + + DPrime = max(DPrime,DPrime'); + end +end + +WPrime = data'*W*data; +WPrime = max(WPrime,WPrime'); + + + +%====================================== +% Generalized Eigen +%====================================== + +dimMatrix = size(WPrime,2); + +if ReducedDim > dimMatrix + ReducedDim = dimMatrix; +end + + +if isfield(options,'bEigs') + bEigs = options.bEigs; +else + if (dimMatrix > MAX_MATRIX_SIZE) && (ReducedDim < dimMatrix*EIGVECTOR_RATIO) + bEigs = 1; + else + bEigs = 0; + end +end + + +if bEigs + %disp('use eigs to speed up!'); + option = struct('disp',0); + if bPCA && ~bD + [eigvector, eigvalue] = eigs(WPrime,ReducedDim,'la',option); + else + if bChol + option.cholB = 1; + [eigvector, eigvalue] = eigs(WPrime,R,ReducedDim,'la',option); + else + [eigvector, eigvalue] = eigs(WPrime,DPrime,ReducedDim,'la',option); + end + end + eigvalue = diag(eigvalue); +else + if bPCA && ~bD + [eigvector, eigvalue] = eig(WPrime); + else + [eigvector, eigvalue] = eig(WPrime,DPrime); + end + eigvalue = diag(eigvalue); + + [junk, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + eigvector = eigvector(:,index); + + if ReducedDim < size(eigvector,2) + eigvector = eigvector(:, 1:ReducedDim); + eigvalue = eigvalue(1:ReducedDim); + end +end + + +if bPCA + eigvector = eigvector_PCA*eigvector; +end + +for i = 1:size(eigvector,2) + eigvector(:,i) = eigvector(:,i)./norm(eigvector(:,i)); +end + + + + + +function [U, S, V]=CutonRatio(U,S,V,options) + if ~isfield(options, 'PCARatio') + options.PCARatio = 1; + end + + eigvalue_PCA = full(diag(S)); + if options.PCARatio > 1 + idx = options.PCARatio; + if idx < length(eigvalue_PCA) + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end + elseif options.PCARatio < 1 + sumEig = sum(eigvalue_PCA); + sumEig = sumEig*options.PCARatio; + sumNow = 0; + for idx = 1:length(eigvalue_PCA) + sumNow = sumNow + eigvalue_PCA(idx); + if sumNow >= sumEig + break; + end + end + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end diff --git a/LPP.m b/LPP.m new file mode 100644 index 0000000..f52ec38 --- /dev/null +++ b/LPP.m @@ -0,0 +1,155 @@ +function [eigvector, eigvalue] = LPP(W, options, data) +% LPP: Locality Preserving Projections +% +% [eigvector, eigvalue] = LPP(W, options, data) +% +% Input: +% data - Data matrix. Each row vector of fea is a data point. +% W - Affinity matrix. You can either call "constructW" +% to construct the W, or construct it by yourself. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Please see LGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of LPP eigen-problem. +% +% +% Examples: +% +% fea = rand(50,70); +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% options.PCARatio = 0.99 +% [eigvector, eigvalue] = LPP(W, options, fea); +% Y = fea*eigvector; +% +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% options.bLDA = 1; +% W = constructW(fea,options); +% options.PCARatio = 1; +% [eigvector, eigvalue] = LPP(W, options, fea); +% Y = fea*eigvector; +% +% +% Note: After applying some simple algebra, the smallest eigenvalue problem: +% data^T*L*data = \lemda data^T*D*data +% is equivalent to the largest eigenvalue problem: +% data^T*W*data = \beta data^T*D*data +% where L=D-W; \lemda= 1 - \beta. +% Thus, the smallest eigenvalue problem can be transformed to a largest +% eigenvalue problem. Such tricks are adopted in this code for the +% consideration of calculation precision of Matlab. +% +% +% See also constructW, LGE +% +%Reference: +% Xiaofei He, and Partha Niyogi, "Locality Preserving Projections" +% Advances in Neural Information Processing Systems 16 (NIPS 2003), +% Vancouver, Canada, 2003. +% +% Xiaofei He, Shuicheng Yan, Yuxiao Hu, Partha Niyogi, and Hong-Jiang +% Zhang, "Face Recognition Using Laplacianfaces", IEEE PAMI, Vol. 27, No. +% 3, Mar. 2005. +% +% Deng Cai, Xiaofei He and Jiawei Han, "Document Clustering Using +% Locality Preserving Indexing" IEEE TKDE, Dec. 2005. +% +% Deng Cai, Xiaofei He and Jiawei Han, "Using Graph Model for Face Analysis", +% Technical Report, UIUCDCS-R-2005-2636, UIUC, Sept. 2005 +% +% Xiaofei He, "Locality Preserving Projections" +% PhD's thesis, Computer Science Department, The University of Chicago, +% 2005. +% +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.1 --Feb/2006 +% version 1.0 --April/2004 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) +% + + +if (~exist('options','var')) + options = []; +end + +[nSmp,nFea] = size(data); +if size(W,1) ~= nSmp + error('W and data mismatch!'); +end + + +%========================== +% If data is too large, the following centering codes can be commented +% options.keepMean = 1; +%========================== +if isfield(options,'keepMean') && options.keepMean + ; +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + + + + +D = full(sum(W,2)); + + +if ~isfield(options,'Regu') || ~options.Regu + DToPowerHalf = D.^.5; + D_mhalf = DToPowerHalf.^-1; + + if nSmp < 5000 + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + W = (tmpD_mhalf.*W).*tmpD_mhalf'; + clear tmpD_mhalf; + else + [i_idx,j_idx,v_idx] = find(W); + v1_idx = zeros(size(v_idx)); + for i=1:length(v_idx) + v1_idx(i) = v_idx(i)*D_mhalf(i_idx(i))*D_mhalf(j_idx(i)); + end + W = sparse(i_idx,j_idx,v1_idx); + clear i_idx j_idx v_idx v1_idx + end + W = max(W,W'); + + data = repmat(DToPowerHalf,1,nFea).*data; + [eigvector, eigvalue] = LGE(W, [], options, data); +else + options.ReguAlpha = options.ReguAlpha*sum(D)/length(D); + + D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + [eigvector, eigvalue] = LGE(W, D, options, data); +end + + +eigIdx = find(eigvalue < 1e-3); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + diff --git a/LSC.m b/LSC.m new file mode 100644 index 0000000..d349cab --- /dev/null +++ b/LSC.m @@ -0,0 +1,130 @@ +function label = LSC(data,k,opts) +% label = LSC(data,k,opts): Landmark-based Spectral Clustering +% Input: +% - data: the data matrix of size nSmp x nFea, where each row is a sample +% point +% - k: the number of clusters +% opts: options for this algorithm +% - p: the number of landmarks picked (default 1000) +% - r: the number of nearest landmarks for representation (default 5) +% - numRep: the number of replicates for the final kmeans (default 10) +% - maxIter: the maximum number of iterations for final kmeans (default 100) +% - mode: landmark selection method, currently support +% - 'kmeans': use centers of clusters generated by kmeans (default) +% - 'random': use randomly sampled points from the original +% data set +% The following parameters are effective ONLY in mode 'kmeans' +% - kmNumRep: the number of replicates for initial kmeans (default 1) +% - kmMaxIter: the maximum number of iterations for initial kmeans (default 5) +% Output: +% - label: the cluster assignment for each point +% Requre: +% litekmeans.m +% Usage: +% data = rand([100,50]); +% label = LSC(data,10); +%Reference: +% +% Xinlei Chen, Deng Cai, "Large Scale Spectral Clustering with +% Landmark-Based Representation," AAAI 2011. +% +% version 2.0 --Dec./2011 +% version 1.0 --Oct./2010 +% +% Written by Xinlei Chen (endernewton AT gmail.com) +% Deng Cai (dengcai AT gmail.com) + + + +% Set and parse parameters +if (~exist('opts','var')) + opts = []; +end + + +p = 1000; +if isfield(opts,'p') + p = opts.p; +end + +r = 5; +if isfield(opts,'r') + r = opts.r; +end + +maxIter = 100; +if isfield(opts,'maxIter') + maxIter = opts.maxIter; +end + +numRep = 10; +if isfield(opts,'numRep') + numRep = opts.numRep; +end + +mode = 'kmeans'; +if isfield(opts,'mode') + mode = opts.mode; +end + +nSmp=size(data,1); + +% Landmark selection +if strcmp(mode,'kmeans') + kmMaxIter = 5; + if isfield(opts,'kmMaxIter') + kmMaxIter = opts.kmMaxIter; + end + kmNumRep = 1; + if isfield(opts,'kmNumRep') + kmNumRep = opts.kmNumRep; + end + [dump,marks]=litekmeans(data,p,'MaxIter',kmMaxIter,'Replicates',kmNumRep); + clear kmMaxIter kmNumRep +elseif strcmp(mode,'random') + indSmp = randperm(nSmp); + marks = data(indSmp(1:p),:); + clear indSmp +else + error('mode does not support!'); +end + +% Z construction +D = EuDist2(data,marks,0); + +if isfield(opts,'sigma') + sigma = opts.sigma; +else + sigma = mean(mean(D)); +end + +dump = zeros(nSmp,r); +idx = dump; +for i = 1:r + [dump(:,i),idx(:,i)] = min(D,[],2); + temp = (idx(:,i)-1)*nSmp+[1:nSmp]'; + D(temp) = 1e100; +end + +dump = exp(-dump/(2*sigma^2)); +sumD = sum(dump,2); +Gsdx = bsxfun(@rdivide,dump,sumD); +Gidx = repmat([1:nSmp]',1,r); +Gjdx = idx; +Z=sparse(Gidx(:),Gjdx(:),Gsdx(:),nSmp,p); + +% Graph decomposition +feaSum = full(sqrt(sum(Z,1))); +feaSum = max(feaSum, 1e-12); +Z = Z./feaSum(ones(size(Z,1),1),:); +U = mySVD(Z,k+1); +U(:,1) = []; + +U=U./repmat(sqrt(sum(U.^2,2)),1,k); + +% Final kmeans +label=litekmeans(U,k,'MaxIter',maxIter,'Replicates',numRep); + + + + diff --git a/LSDA.m b/LSDA.m new file mode 100644 index 0000000..0027e3d --- /dev/null +++ b/LSDA.m @@ -0,0 +1,188 @@ +function [eigvector, eigvalue] = LSDA(gnd, options, data) +% LSDA: Locality Sensitive Discriminant Analysis +% +% [eigvector, eigvalue] = LSDA(gnd, options, data) +% +% Input: +% data - Data matrix. Each row vector of fea is a data point. +% +% gnd - Label vector. +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% k = 0 +% Wb: +% Put an edge between two nodes if and +% only if they belong to different classes. +% Ww: +% Put an edge between two nodes if and +% only if they belong to same class. +% > 0 +% Wb: +% Put an edge between two nodes if +% they belong to different classes +% and they are among the k nearst +% neighbors of each other. +% Ww: +% Put an edge between two nodes if +% they belong to same class and they +% are among the k nearst neighbors of +% each other. +% beta [0,1] Paramter to tune the weight between +% within-class graph and between-class +% graph. Default 0.1. +% beta*L_b+(1-beta)*W_w +% +% Please see LGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The eigvalue of LPP eigen-problem. sorted from +% smallest to largest. +% +% Examples: +% +% +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.k = 5; +% [eigvector, eigvalue] = LSDA(gnd, options, fea); +% Y = fea*eigvector; +% +% +% +% See also LPP, LGE +% +%Reference: +% +% Deng Cai, Xiaofei He, Kun Zhou, Jiawei Han and Hujun Bao, "Locality +% Sensitive Discriminant Analysis", IJCAI'2007 +% +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) + + + +if (~exist('options','var')) + options = []; +end + + +[nSmp,nFea] = size(data); +if length(gnd) ~= nSmp + error('gnd and data mismatch!'); +end + +k = 0; +if isfield(options,'k') && (options.k < nSmp-1) + k = options.k; +end + + +beta = 0.1; +if isfield(options,'beta') && (options.beta > 0) && (options.beta < 1) + beta = options.beta; +end + + +Label = unique(gnd); +nLabel = length(Label); + +Ww = zeros(nSmp,nSmp); +Wb = ones(nSmp,nSmp); +for idx=1:nLabel + classIdx = find(gnd==Label(idx)); + Ww(classIdx,classIdx) = 1; + Wb(classIdx,classIdx) = 0; +end + +if k > 0 + D = EuDist2(data,[],0); + [dump idx] = sort(D,2); % sort each row + clear D dump + idx = idx(:,1:options.k+1); + + G = sparse(repmat([1:nSmp]',[options.k+1,1]),idx(:),ones(prod(size(idx)),1),nSmp,nSmp); + G = max(G,G'); + Ww = Ww.*G; + Wb = Wb.*G; + clear G +end + + +Db = full(sum(Wb,2)); +Wb = -Wb; +for i=1:size(Wb,1) + Wb(i,i) = Wb(i,i) + Db(i); +end + +D = full(sum(Ww,2)); + +if isfield(options,'Regu') && options.Regu + options.ReguAlpha = options.ReguAlpha*sum(D)/length(D); +end + + +% W = sparse(beta*Wb+(1-beta)*Ww); +W = sparse((beta/(1-beta))*Wb+Ww); + +clear Wb Ww + + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + + +if ~isfield(options,'Regu') || ~options.Regu + DToPowerHalf = D.^.5; + D_mhalf = DToPowerHalf.^-1; + + if nSmp < 5000 + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + W = (tmpD_mhalf.*W).*tmpD_mhalf'; + clear tmpD_mhalf; + else + [i_idx,j_idx,v_idx] = find(W); + v1_idx = zeros(size(v_idx)); + for i=1:length(v_idx) + v1_idx(i) = v_idx(i)*D_mhalf(i_idx(i))*D_mhalf(j_idx(i)); + end + W = sparse(i_idx,j_idx,v1_idx); + clear i_idx j_idx v_idx v1_idx + end + W = max(W,W'); + + data = repmat(DToPowerHalf,1,nFea).*data; + [eigvector, eigvalue] = LGE(W, [], options, data); +else + + D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + + [eigvector, eigvalue] = LGE(W, D, options, data); +end + + +eigIdx = find(eigvalue < 1e-10); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + diff --git a/LTM.m b/LTM.m new file mode 100644 index 0000000..75b7ca6 --- /dev/null +++ b/LTM.m @@ -0,0 +1,225 @@ +function [Pz_d_final, Pw_z_final, LogL_final, nIter_final] = LTM(X, K, W, options, Pz_d, Pw_z) +% Locally-consistent Topic Modeling (LTM) using EM +% +% Notation: +% X ... (mFea x nSmp) term-document matrix (observed data) +% X(i,j) stores number of occurrences of word i in document j +% +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% K ... number of topics +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% options.alpha: local consistency regularization prameter (default 1000) +% if alpha=0, LTM boils down to the ordinary +% PLSA. Please see [1] for details. +% +% You only need to provide the above four inputs. +% +% Pz_d ... P(z|d) +% Pw_z ... P(w|z) corresponds to beta parameter in LDA +% +% +% References: +% [1] Deng Cai, Xuanhui Wang, Xiaofei He, "Probabilistic Dyadic Data +% Analysis with Local and Global Consistency", ICML 2009. +% +% +% This software is based on the implementation of pLSA from +% +% Peter Gehler +% Max Planck Institute for biological Cybernetics +% pgehler@tuebingen.mpg.de +% Feb 2006 +% http://www.kyb.mpg.de/bs/people/pgehler/code/index.html +% +% +% version 2.0 --Jan/2012 +% version 1.0 --Nov/2008 +% +% +% Written by Deng Cai (dengcai AT gmail.com) +% +ZERO_OFFSET = 1e-200; + +differror = 1e-7; +if isfield(options,'error') + differror = options.error; +end + +maxIter = []; +if isfield(options, 'maxIter') + maxIter = options.maxIter; +end + +nRepeat = 10; +if isfield(options,'nRepeat') + nRepeat = options.nRepeat; +end + +minIterOrig = 30; +if isfield(options,'minIter') + minIterOrig = options.minIter; +end +minIter = minIterOrig-1; + +meanFitRatio = 0.1; +if isfield(options,'meanFitRatio') + meanFitRatio = options.meanFitRatio; +end + +alpha = 1000; +if isfield(options,'alpha') + alpha = options.alpha; +end + +lambdaB = 0; +if isfield(options,'lambdaB') + lambdaB = options.lambdaB; +end + +Verbosity = 0; +if isfield(options,'Verbosity') + Verbosity = options.Verbosity; +end + +if min(min(X)) < 0 + error('Input should be nonnegative!'); +end + +[mFea,nSmp]=size(X); +if ~exist('Pz_d','var') + [Pz_d,Pw_z] = pLSA_init(X,K); +else + nRepeat = 1; +end + +Pd = sum(X)./sum(X(:)); +Pd = full(Pd); +Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + +if alpha > 0 + DCol = full(sum(W,2)); + D = spdiags(DCol,0,speye(size(W,1))); + L = D - W; + if isfield(options,'NormW') && options.NormW + D_mhalf = DCol.^-.5; + + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + L = (tmpD_mhalf.*L).*tmpD_mhalf'; + clear D_mhalf tmpD_mhalf; + + L = max(L, L'); + end + L = alpha*L; + dLen = full(sum(X,1)); + OmegaL = (1-lambdaB)*spdiags(dLen',0,speye(size(W,1)))+L; +end + +tryNo = 0; +selectInit = 1; +nIter = 0; +LogL = []; +while tryNo < nRepeat + tryNo = tryNo+1; + maxErr = 1; + while(maxErr > differror) + [Pw_z,Pz_d] = mex_EMstep(X,Pw_d,Pw_z,Pz_d); + if alpha >0 + Pz_d = Pz_d.*repmat(dLen,K,1); + Pz_d = (OmegaL\Pz_d')'; + end + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + + nIter = nIter + 1; + if nIter > minIter + if selectInit + newLogL = mex_logL(X,Pw_d,Pd); + if alpha > 0 + newLogL = newLogL - sum(sum((log(Pz_d + ZERO_OFFSET)*L).*Pz_d)); + end + LogL = [LogL newLogL]; %#ok + maxErr = 0; + else + newLogL = mex_logL(X,Pw_d,Pd); + if alpha > 0 + newLogL = newLogL - sum(sum((log(Pz_d + ZERO_OFFSET)*L).*Pz_d)); + end + LogL = [LogL newLogL]; %#ok + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*newLogL; + maxErr = (meanFit-newLogL)/meanFit; + + if ~isempty(maxIter) + if nIter >= maxIter + maxErr = 0; + end + end + end + else + newLogL = mex_logL(X,Pw_d,Pd); + if alpha > 0 + newLogL = newLogL - sum(sum((log(Pz_d + ZERO_OFFSET)*L).*Pz_d)); + end + LogL = [LogL newLogL]; %#ok + end + if Verbosity + if length(LogL) > 1 + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(LogL(end)),' deltaLogL: ',num2str(LogL(end)-LogL(end-1)),' maxErr:',num2str(maxErr)]); + else + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(LogL(end)),' maxErr:',num2str(maxErr)]); + end + end + end + + + if tryNo == 1 + Pz_d_final = Pz_d; + Pw_z_final = Pw_z; + nIter_final = nIter; + LogL_final = LogL; + Pw_d_final = Pw_d; + else + if LogL(end) > LogL_final(end) + Pz_d_final = Pz_d; + Pw_z_final = Pw_z; + nIter_final = nIter; + LogL_final = LogL; + Pw_d_final = Pw_d; + end + end + + if selectInit + if tryNo < nRepeat + %re-start + [Pz_d,Pw_z] = pLSA_init(X,K); + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + LogL = []; + nIter = 0; + else + tryNo = tryNo - 1; + minIter = 0; + selectInit = 0; + Pz_d = Pz_d_final; + Pw_z= Pw_z_final; + LogL = LogL_final; + nIter = nIter_final; + meanFit = LogL(end)*10; + Pw_d = Pw_d_final; + end + end +end + + +% initialize the probability distributions +function [Pz_d,Pw_z] = pLSA_init(X,K) + +[mFea,nSmp] = size(X); +[label, center] = litekmeans(X',K,'maxIter',10); +E = sparse(1:nSmp,label,1,nSmp,K,nSmp); +Pz_d = full(E'); +center = max(0,center); +Pw_z = center'; +Pw_z = Pw_z./repmat(sum(Pw_z,1),mFea,1); + + diff --git a/LapPLSI.m b/LapPLSI.m new file mode 100644 index 0000000..637cfbc --- /dev/null +++ b/LapPLSI.m @@ -0,0 +1,320 @@ +function [Pz_d_final, Pw_z_final, Obj_final, nIter_final] = LapPLSI(X, K, W, options, Pz_d, Pw_z) +% Laplacian Probabilistic Latent Semantic Indexing/Alnalysis (LapPLSI) using generalized EM +% +% where +% X +% Notation: +% X ... (mFea x nSmp) term-document matrix (observed data) +% X(i,j) stores number of occurrences of word i in document j +% +% mFea ... number of words (vocabulary size) +% nSmp ... number of documents +% K ... number of topics +% W ... weight matrix of the affinity graph +% +% options ... Structure holding all settings +% options.lambda: manifold regularization prameter (default 1000) +% +% You only need to provide the above four inputs. +% +% Pz_d ... P(z|d) +% Pw_z ... P(w|z) corresponds to beta parameter in LDA +% +% +% References: +% [1] Deng Cai, Qiaozhu Mei, Jiawei Han, ChengXiang Zhai, "Modeling Hidden +% Topics on Document Manifold", Proc. 2008 ACM Conf. on Information and +% Knowledge Management (CIKM'08), Napa Valley, CA, Oct. 2008. +% [2] Qiaozhu Mei, Deng Cai, Duo Zhang, ChengXiang Zhai, "Topic Modeling +% with Network Regularization", Proceedings of the World Wide Web +% Conference ( WWW'08), 2008 +% +% +% This software is based on the implementation of pLSA from +% +% Peter Gehler +% Max Planck Institute for biological Cybernetics +% pgehler@tuebingen.mpg.de +% Feb 2006 +% http://www.kyb.mpg.de/bs/people/pgehler/code/index.html +% +% version 2.0 --June/2009 +% version 1.1 --June/2008 +% version 1.0 --Nov/2007 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +maxLoop = 100; + +if min(min(X)) < 0 + error('Input should be nonnegative!'); +end + + +differror = 1e-7; +if isfield(options,'error') + differror = options.error; +end + +maxIter = []; +if isfield(options, 'maxIter') + maxIter = options.maxIter; +end + +nRepeat = 10; +if isfield(options,'nRepeat') + nRepeat = options.nRepeat; +end + +minIter = 30; +if isfield(options,'minIter') + minIter = options.minIter; +end + +meanFitRatio = 0.1; +if isfield(options,'meanFitRatio') + meanFitRatio = options.meanFitRatio; +end + +lambda = 1000; +if isfield(options,'lambda') + lambda = options.lambda; +end + +gamma = 0.1; +if isfield(options,'gamma') + gamma = options.gamma; +end + +nStep = 200; +if isfield(options,'nStep') + nStep = options.nStep; +end + + +Verbosity = 0; +if isfield(options,'Verbosity') + Verbosity = options.Verbosity; +end + + +DCol = full(sum(W,2)); +D = spdiags(DCol,0,speye(size(W,1))); +L = D - W; +if isfield(options,'NormW') && options.NormW + D_mhalf = DCol.^-.5; + + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + L = (tmpD_mhalf.*L).*tmpD_mhalf'; + clear D_mhalf tmpD_mhalf; + + L = max(L, L'); +end + +if ~exist('Pz_d','var') + [Pz_d,Pw_z] = pLSA_init(X,K); +else + nRepeat = 1; +end + +Pd = sum(X)./sum(X(:)); +Pd = full(Pd); +Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + + +selectInit = 1; +if nRepeat == 1 + selectInit = 0; + minIter = 0; +end + +tryNo = 0; + + +[Pw_z,Pz_d] = mex_EMstep(X,Pw_d,Pw_z,Pz_d); +Pw_d = mex_Pw_d(X,Pw_z,Pz_d); +LogL = mex_logL(X,Pw_d,Pd); +ObjLap = sum(sum(Pz_d*L.*Pz_d))*2; +Obj = LogL-ObjLap*lambda; +nIter = 1; + +meanFit = Obj*10; + +for i=1:nStep + Pz_d = (1-gamma)*Pz_d + gamma*Pz_d*W./repmat(DCol',K,1); +end +Pw_d = mex_Pw_d(X,Pw_z,Pz_d); +LogLNew = mex_logL(X,Pw_d,Pd); +ObjLapNew = sum(sum(Pz_d*L.*Pz_d))*2; +ObjNew = LogLNew-ObjLapNew*lambda; +if ObjNew > Obj + nIter = nIter + 1; + LogL(end+1) = LogLNew; + ObjLap(end+1) = ObjLapNew; + Obj(end+1) = ObjNew; +end + + + +while tryNo < nRepeat + tryNo = tryNo+1; + maxErr = 1; + while(maxErr > differror) + Pw_z_old = Pw_z; + Pz_d_old = Pz_d; + [Pw_z,Pz_d] = mex_EMstep(X,Pw_d,Pw_z,Pz_d); + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + LogLNew = mex_logL(X,Pw_d,Pd); + deltaLogL = LogLNew-LogL(end); + + ObjLapNew = sum(sum(Pz_d*L.*Pz_d))*2; + deltaObjLap = ObjLapNew - ObjLap(end); + deltaObj = deltaLogL-deltaObjLap*lambda; + + loopNo = 0; + loopNo2 = 0; + while deltaObj < 0 + loopNo = 0; + while deltaObj < 0 + for i=1:nStep + Pz_d = (1-gamma)*Pz_d + gamma*Pz_d*W./repmat(DCol',K,1); + end + ObjLapNew = sum(sum(Pz_d*L.*Pz_d))*2; + deltaObjLap = ObjLapNew - ObjLap(end); + deltaObj = deltaLogL-deltaObjLap*lambda; + loopNo = loopNo + 1; + if loopNo > maxLoop + break; + end + end + if loopNo > maxLoop + break; + end + loopNo2 = loopNo2 + 1; + if loopNo2 > maxLoop + break; + end + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + LogLNew = mex_logL(X,Pw_d,Pd); + deltaLogL = LogLNew-LogL(end); + deltaObj = deltaLogL-deltaObjLap*lambda; + end + + IterValid = 1; + StopIter = 0; + if loopNo > maxLoop || loopNo2 > maxLoop + Pw_z = Pw_z_old; + Pz_d = Pz_d_old; + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + if nStep > 1 + nStep = ceil(nStep/2); + IterValid = 0; + elseif gamma > 0.01 + gamma = gamma/2; + IterValid = 0; + else + StopIter = 1; + end + else + LogL(end+1) = LogLNew; + ObjLap(end+1) = ObjLapNew; + Obj(end+1) = LogL(end)-ObjLap(end)*lambda; + end + + if StopIter + maxErr = 0; + elseif IterValid + nIter = nIter + 1; + if nIter > minIter + if selectInit + maxErr = 0; + else + meanFit = meanFitRatio*meanFit + (1-meanFitRatio)*Obj(end); + maxErr = (meanFit-Obj(end))/meanFit; + + if ~isempty(maxIter) + if nIter >= maxIter + maxErr = 0; + end + end + end + end + if Verbosity + if length(LogL) > 1 + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(Obj(end)),' deltaLogL: ',num2str(Obj(end)-Obj(end-1)),' maxErr:',num2str(maxErr)]); + else + disp(['tryNo: ',num2str(tryNo),' Iteration: ',num2str(nIter),' LogL: ',num2str(Obj(end)),' maxErr:',num2str(maxErr)]); + end + end + end + end + + + if tryNo == 1 + Pz_d_final = Pz_d; + Pw_z_final = Pw_z; + nIter_final = nIter; + LogL_final = LogL; + ObjLap_final = ObjLap; + Obj_final = Obj; + Pw_d_final = Pw_d; + else + if Obj(end) > Obj_final(end) + Pz_d_final = Pz_d; + Pw_z_final = Pw_z; + nIter_final = nIter; + LogL_final = LogL; + ObjLap_final = ObjLap; + Obj_final = Obj; + Pw_d_final = Pw_d; + end + end + + if selectInit + if tryNo < nRepeat + %re-start + [Pz_d,Pw_z] = pLSA_init(X,K); + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + + [Pw_z,Pz_d] = mex_EMstep(X,Pw_d,Pw_z,Pz_d); + Pw_d = mex_Pw_d(X,Pw_z,Pz_d); + LogL = mex_logL(X,Pw_d,Pd); + ObjLap = sum(sum(Pz_d*L.*Pz_d))*2; + Obj = LogL-ObjLap*lambda; + nIter = 1; + else + tryNo = tryNo - 1; + minIter = 0; + selectInit = 0; + Pz_d = Pz_d_final; + Pw_z= Pw_z_final; + LogL = LogL_final; + ObjLap = ObjLap_final; + Obj = Obj_final; + nIter = nIter_final; + meanFit = Obj(end)*10; + Pw_d = Pw_d_final; + end + end +end + + + +function [Pz_d,Pw_z,Pz_q] = pLSA_init(X,K,Xtest) + +[mFea,nSmp] = size(X); + +Pz_d = rand(K,nSmp); +Pz_d = Pz_d ./ repmat(sum(Pz_d,1),K,1); + +% random assignment +Pw_z = rand(mFea,K); +Pw_z = Pw_z./repmat(sum(Pw_z,1),mFea,1); + +if nargin > 2 + nTest = size(Xtest,2); + Pz_q = rand(K,nTest); + Pz_q = Pz_q ./ repmat(sum(Pz_q),K,1); +end + + diff --git a/LaplacianScore.m b/LaplacianScore.m new file mode 100644 index 0000000..d9d9cc3 --- /dev/null +++ b/LaplacianScore.m @@ -0,0 +1,113 @@ +function [Y] = LaplacianScore(X, W) +% Usage: +% [Y] = LaplacianScore(X, W) +% +% X: Rows of vectors of data points +% W: The affinity matrix. +% Y: Vector of (1-LaplacianScore) for each feature. +% The features with larger y are more important. +% +% Examples: +% +% fea = rand(50,70); +% options = []; +% options.Metric = 'Cosine'; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'Cosine'; +% W = constructW(fea,options); +% +% LaplacianScore = LaplacianScore(fea,W); +% [junk, index] = sort(-LaplacianScore); +% +% newfea = fea(:,index); +% %the features in newfea will be sorted based on their importance. +% +% Type "LaplacianScore" for a self-demo. +% +% See also constructW +% +%Reference: +% +% Xiaofei He, Deng Cai and Partha Niyogi, "Laplacian Score for Feature Selection". +% Advances in Neural Information Processing Systems 18 (NIPS 2005), +% Vancouver, Canada, 2005. +% +% Deng Cai, 2004/08 + + +if nargin == 0, selfdemo; return; end + +[nSmp,nFea] = size(X); + +if size(W,1) ~= nSmp + error('W is error'); +end + +D = full(sum(W,2)); +L = W; + +allone = ones(nSmp,1); + + +tmp1 = D'*X; + +D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + +DPrime = sum((X'*D)'.*X)-tmp1.*tmp1/sum(diag(D)); +LPrime = sum((X'*L)'.*X)-tmp1.*tmp1/sum(diag(D)); + +DPrime(find(DPrime < 1e-12)) = 10000; + +Y = LPrime./DPrime; +Y = Y'; +Y = full(Y); + + + + +%--------------------------------------------------- +function selfdemo +% ====== Self demo using IRIS dataset +% ====== 1. Plot IRIS data after LDA for dimension reduction to 2D +load iris.dat + +feaNorm = mynorm(iris(:,1:4),2); +fea = iris(:,1:4) ./ repmat(max(1e-10,feaNorm),1,4); + +options = []; +options.Metric = 'Cosine'; +options.NeighborMode = 'KNN'; +options.WeightMode = 'Cosine'; +options.k = 3; + +W = constructW(fea,options); + +[LaplacianScore] = feval(mfilename,iris(:,1:4),W); +[junk, index] = sort(-LaplacianScore); + +index1 = find(iris(:,5)==1); +index2 = find(iris(:,5)==2); +index3 = find(iris(:,5)==3); +figure; +plot(iris(index1, index(1)), iris(index1, index(2)), '*', ... + iris(index2, index(1)), iris(index2, index(2)), 'o', ... + iris(index3, index(1)), iris(index3, index(2)), 'x'); +legend('Class 1', 'Class 2', 'Class 3'); +title('IRIS data onto the first and second feature (Laplacian Score)'); +axis equal; axis tight; + +figure; +plot(iris(index1, index(3)), iris(index1, index(4)), '*', ... + iris(index2, index(3)), iris(index2, index(4)), 'o', ... + iris(index3, index(3)), iris(index3, index(4)), 'x'); +legend('Class 1', 'Class 2', 'Class 3'); +title('IRIS data onto the third and fourth feature (Laplacian Score)'); +axis equal; axis tight; + +disp('Laplacian Score:'); +for i = 1:length(LaplacianScore) + disp(num2str(LaplacianScore(i))); +end + + diff --git a/LeastR.m b/LeastR.m new file mode 100644 index 0000000..158650a --- /dev/null +++ b/LeastR.m @@ -0,0 +1,669 @@ +function [x, funVal, ValueL]=LeastR(A, y, z, opts) +% +%% +% Function LeastR +% Least Squares Loss with the L1-norm Regularization +% +%% Problem +% +% min 1/2 || A x - y||^2 + 1/2 rsL2 * ||x||_2^2 + z * ||x||_1 +% +% By default, rsL2=0. +% When rsL2 is nonzero, this corresponds to the well-know elastic net. +% +%% Input parameters: +% +% A- Matrix of size m x n +% A can be a dense matrix +% a sparse matrix +% or a DCT matrix +% y - Response vector (of size mx1) +% z - L_1 norm regularization parameter (z >=0) +% opts- Optional inputs (default value: opts=[]) +% +%% Output parameters: +% x- Solution +% funVal- Function value during iterations +% +%% Copyright (C) 2009-2010 Jun Liu, and Jieping Ye +% +% You are suggested to first read the Manual. +% +% For any problem, please contact with Jun Liu via j.liu@asu.edu +% +% Last modified on February 18, 2010. +% +%% Related papers +% +% [1] Jun Liu and Jieping Ye, Efficient Euclidean Projections +% in Linear Time, ICML 2009. +% +% [2] Jun Liu and Jieping Ye, Sparse Learning with Efficient Euclidean +% Projections onto the L1 Ball, Technical Report ASU, 2008. +% +%% Related functions +% +% sll_opts, initFactor, pathSolutionLeast +% LeastC, nnLeastR, nnLeastC, +% +%% + +%% Verify and initialize the parameters +%% + +% Verify the number of input parameters +if (nargin <3) + error('\n Inputs: A, y and z should be specified!\n'); +elseif (nargin==3) + opts=[]; +end + +% Get the size of the matrix A +[m,n]=size(A); + +% Verify the length of y +if (length(y) ~=m) + error('\n Check the length of y!\n'); +end + +% Verify the value of z +if (z<0) + error('\n z should be nonnegative!\n'); +end + +% run sll_opts to set default values (flags) +opts=sll_opts(opts); + +%% Detailed initialization +%% Normalization + +% Please refer to sll_opts for the definitions of mu, nu and nFlag +% +% If .nFlag =1, the input matrix A is normalized to +% A= ( A- repmat(mu, m,1) ) * diag(nu)^{-1} +% +% If .nFlag =2, the input matrix A is normalized to +% A= diag(nu)^{-1} * ( A- repmat(mu, m,1) ) +% +% Such normalization is done implicitly +% This implicit normalization is suggested for the sparse matrix +% but not for the dense matrix +% + +if (opts.nFlag~=0) + if (isfield(opts,'mu')) + mu=opts.mu; + if(size(mu,2)~=n) + error('\n Check the input .mu'); + end + else + mu=mean(A,1); + end + + if (opts.nFlag==1) + if (isfield(opts,'nu')) + nu=opts.nu; + if(size(nu,1)~=n) + error('\n Check the input .nu!'); + end + else + nu=(sum(A.^2,1)/m).^(0.5); nu=nu'; + end + else % .nFlag=2 + if (isfield(opts,'nu')) + nu=opts.nu; + if(size(nu,1)~=m) + error('\n Check the input .nu!'); + end + else + nu=(sum(A.^2,2)/n).^(0.5); + end + end + + ind_zero=find(abs(nu)<= 1e-10); nu(ind_zero)=1; + % If some values in nu is typically small, it might be that, + % the entries in a given row or column in A are all close to zero. + % For numerical stability, we set the corresponding value to 1. +end + +if (~issparse(A)) && (opts.nFlag~=0) + fprintf('\n -----------------------------------------------------'); + fprintf('\n The data is not sparse or not stored in sparse format'); + fprintf('\n The code still works.'); + fprintf('\n But we suggest you to normalize the data directly,'); + fprintf('\n for achieving better efficiency.'); + fprintf('\n -----------------------------------------------------'); +end + +%% Starting point initialization + +% compute AT y +if (opts.nFlag==0) + ATy=A'*y; +elseif (opts.nFlag==1) + ATy=A'*y - sum(y) * mu'; ATy=ATy./nu; +else + invNu=y./nu; ATy=A'*invNu-sum(invNu)*mu'; +end + +% process the regularization parameter + +% L2 norm regularization +if isfield(opts,'rsL2') + rsL2=opts.rsL2; + if (rsL2<0) + error('\n opts.rsL2 should be nonnegative!'); + end +else + rsL2=0; +end + +% L1 norm regularization +if (opts.rFlag==0) + lambda=z; +else % z here is the scaling factor lying in [0,1] + if (z<0 || z>1) + error('\n opts.rFlag=1, and z should be in [0,1]'); + end + + lambda_max=max(abs(ATy)); + lambda=z*lambda_max; + + rsL2=rsL2*lambda_max; % the input rsL2 is a ratio of lambda_max +end + +% initialize a starting point +if opts.init==2 + x=zeros(n,1); +else + if isfield(opts,'x0') + x=opts.x0; + if (length(x)~=n) + error('\n Check the input .x0'); + end + else + x=ATy; % if .x0 is not specified, we use ratio*ATy, + % where ratio is a positive value + end +end + +% compute A x +if (opts.nFlag==0) + Ax=A* x; +elseif (opts.nFlag==1) + invNu=x./nu; mu_invNu=mu * invNu; + Ax=A*invNu -repmat(mu_invNu, m, 1); +else + Ax=A*x-repmat(mu*x, m, 1); Ax=Ax./nu; +end + +if (opts.init==0) % If .init=0, we set x=ratio*x by "initFactor" + % Please refer to the function initFactor for detail + + x_norm=sum(abs(x)); x_2norm=x'*x; + if x_norm>=1e-6 + ratio=initFactor(x_norm, Ax, y, lambda,'LeastR', rsL2, x_2norm); + x=ratio*x; Ax=ratio*Ax; + end +end + +%% The main program + +%% The Armijo Goldstein line search scheme + accelearted gradient descent +if (opts.mFlag==0 && opts.lFlag==0) + + bFlag=0; % this flag tests whether the gradient step only changes a little + + L=1 + rsL2; + % We assume that the maximum eigenvalue of A'A is over 1 + + % assign xp with x, and Axp with Ax + xp=x; Axp=Ax; xxp=zeros(n,1); + + % alphap and alpha are used for computing the weight in forming search point + alphap=0; alpha=1; + + for iterStep=1:opts.maxIter + % --------------------------- step 1 --------------------------- + % compute search point s based on xp and x (with beta) + beta=(alphap-1)/alpha; s=x + beta* xxp; + + % --------------------------- step 2 --------------------------- + % line search for L and compute the new approximate solution x + + % compute the gradient (g) at s + As=Ax + beta* (Ax-Axp); + + % compute AT As + if (opts.nFlag==0) + ATAs=A'*As; + elseif (opts.nFlag==1) + ATAs=A'*As - sum(As) * mu'; ATAs=ATAs./nu; + else + invNu=As./nu; ATAs=A'*invNu-sum(invNu)*mu'; + end + + % obtain the gradient g + g=ATAs-ATy + rsL2 * s; + + % copy x and Ax to xp and Axp + xp=x; Axp=Ax; + + while (1) + % let s walk in a step in the antigradient of s to get v + % and then do the l1-norm regularized projection + v=s-g/L; + + % L1-norm regularized projection + x=sign(v).*max(abs(v)-lambda / L,0); + + v=x-s; % the difference between the new approximate solution x + % and the search point s + + % compute A x + if (opts.nFlag==0) + Ax=A* x; + elseif (opts.nFlag==1) + invNu=x./nu; mu_invNu=mu * invNu; + Ax=A*invNu -repmat(mu_invNu, m, 1); + else + Ax=A*x-repmat(mu*x, m, 1); Ax=Ax./nu; + end + + Av=Ax -As; + r_sum=v'*v; l_sum=Av'*Av; + + if (r_sum <=1e-20) + bFlag=1; % this shows that, the gradient step makes little improvement + break; + end + + % the condition is ||Av||_2^2 <= (L - rsL2) * ||v||_2^2 + if(l_sum <= r_sum * (L-rsL2)) + break; + else + L=max(2*L, l_sum/r_sum + rsL2); + % fprintf('\n L=%5.6f',L); + end + end + + ValueL(iterStep)=L; + + % --------------------------- step 3 --------------------------- + % update alpha and alphap, and check whether converge + alphap=alpha; alpha= (1+ sqrt(4*alpha*alpha +1))/2; + + xxp=x-xp; Axy=Ax-y; + funVal(iterStep)=Axy'* Axy/2 + rsL2/2 * x'*x + sum(abs(x)) * lambda; + + if (bFlag) + % fprintf('\n The program terminates as the gradient step changes the solution very small.'); + break; + end + + switch(opts.tFlag) + case 0 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <= opts.tol) + break; + end + end + case 1 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <=... + opts.tol* funVal(iterStep-1)) + break; + end + end + case 2 + if ( funVal(iterStep)<= opts.tol) + break; + end + case 3 + norm_xxp=sqrt(xxp'*xxp); + if ( norm_xxp <=opts.tol) + break; + end + case 4 + norm_xp=sqrt(xp'*xp); norm_xxp=sqrt(xxp'*xxp); + if ( norm_xxp <=opts.tol * max(norm_xp,1)) + break; + end + case 5 + if iterStep>=opts.maxIter + break; + end + end + end +end + +%% Reformulated problem + Nemirovski's scheme + +% .mFlag=1, and .lFlag=0 +% refomulate the problem as the constrained convex optimization +% problem, and then apply Armijo Goldstein line search scheme + +% Problem: +% min 1/2 || A x - y||^2 + 1/2 rsL2 * ||x||_2^2 + z * t' * 1 +% s.t. |x| <= t + +if(opts.mFlag==1 && opts.lFlag==0) + + bFlag=0; % this flag tests whether the gradient step only changes a little + + L=1 + rsL2; + % We assume that the maximum eigenvalue of A'A is over 1 + + % assign xp with x, and Axp with Ax + xp=x; Axp=Ax; xxp=zeros(n,1); + t=abs(x); tp=t; + % t is the upper bound of absolute value of x + + % alphap and alpha are used for computing the weight in forming search point + alphap=0; alpha=1; + + for iterStep=1:opts.maxIter + % --------------------------- step 1 --------------------------- + % compute search point s based on xp and x (with beta) + beta=(alphap-1)/alpha; s=x + beta* xxp; s_t= t + beta * (t -tp); + + % --------------------------- step 2 --------------------------- + % line search for L and compute the new approximate solution x + + % compute the gradient (g) at s + As=Ax + beta* (Ax-Axp); + + % compute AT As + if (opts.nFlag==0) + ATAs=A'*As; + elseif (opts.nFlag==1) + ATAs=A'*As - sum(As) * mu'; ATAs=ATAs./nu; + else + invNu=As./nu; ATAs=A'*invNu-sum(invNu)*mu'; + end + + % obtain the gradient g + g=ATAs-ATy + rsL2 * s; + + % copy x and Ax to xp and Axp + xp=x; Axp=Ax; + tp=t; + + while (1) + % let s walk in a step in the antigradient of s to get v + % and then do the l1-norm regularized projection + + u=s-g/L; + v= s_t - lambda / L; + + % projection + [x, t]=ep1R(u, v, n); + + v=x-s; % the difference between the new approximate solution x + % and the search point s + + v_t=t-s_t; + + % compute A x + if (opts.nFlag==0) + Ax=A* x; + elseif (opts.nFlag==1) + invNu=x./nu; mu_invNu=mu * invNu; + Ax=A*invNu -repmat(mu_invNu, m, 1); + else + Ax=A*x-repmat(mu*x, m, 1); Ax=Ax./nu; + end + + Av=Ax -As; + r_sum=v'*v + v_t'*v_t; l_sum=Av'*Av + v'*v * rsL2; + + if (r_sum <=1e-20) + bFlag=1; % this shows that, the gradient step makes little improvement + break; + end + + % the condition is ||Av||_2^2 + rsL2 * ||v||_2^2 + % <= L * (||v||_2^2 + ||v_t|| _2^2 ) + if(l_sum <= r_sum * L) + break; + else + L=max(2*L, l_sum/r_sum); + % fprintf('\n L=%5.6f',L); + end + end + + ValueL(iterStep)=L; + + % --------------------------- step 3 --------------------------- + % update alpha and alphap, and check whether converge + alphap=alpha; alpha= (1+ sqrt(4*alpha*alpha +1))/2; + + xxp=x-xp; Axy=Ax-y; + funVal(iterStep)=Axy'* Axy/2 + rsL2/2 * x'*x + sum(t) * lambda; + + if (bFlag) + % fprintf('\n The program terminates as the gradient step changes the solution very small.'); + break; + end + + switch(opts.tFlag) + case 0 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <= opts.tol) + break; + end + end + case 1 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <=... + opts.tol* funVal(iterStep-1)) + break; + end + end + case 2 + if ( funVal(iterStep)<= opts.tol) + break; + end + case 3 + norm_xxp=sqrt(xxp'*xxp + norm(t-tp)^2); + if ( norm_xxp <=opts.tol) + break; + end + case 4 + norm_xp=sqrt(xp'*xp + tp'*tp); norm_xxp=sqrt(xxp'*xxp+ norm(t-tp)^2); + if ( norm_xxp <=opts.tol * max(norm_xp,1)) + break; + end + case 5 + if iterStep>=opts.maxIter + break; + end + end + end + +end + + +%% adaptive line search + +% .mFlag=1, and .lFlag=1 +% refomulate the problem as the constrained convex optimization +% problem, and then apply adaptive line search scheme + +% Problem: +% min 1/2 || A x - y||^2 + 1/2 rsL2 * ||x||_2^2 + z * t' * 1 +% s.t. |x| <= t + +if(opts.mFlag==1 && opts.lFlag==1) + + bFlag=0; % this flag tests whether the gradient step only changes a little + + L=1 + rsL2; + % We assume that the maximum eigenvalue of A'A is over 1 + + gamma=1; + % we shall set the value of gamma = L, + % where L is appropriate for the starting point + + xp=x; Axp=Ax; + % store x and Ax + xxp=zeros(n,1); + % the difference of x and xp + t=abs(x); tp=t; + % t is the upper bound of absolute value of x + + % compute AT Ax + if (opts.nFlag==0) + ATAx=A'*Ax; + elseif (opts.nFlag==1) + ATAx=A'*Ax - sum(Ax) * mu'; ATAx=ATAx./nu; + else + invNu=Ax./nu; ATAx=A'*invNu-sum(invNu)*mu'; + end + + % We begin the adaptive line search in the following + % + % Note that, in the line search, L and beta are changing + + for iterStep=1:opts.maxIter + + ATAxp=ATAx; + % store ATAx to ATAxp + + if (iterStep~=1) + % compute AT Ax + if (opts.nFlag==0) + ATAx=A'*Ax; + elseif (opts.nFlag==1) + ATAx=A'*Ax - sum(Ax) * mu'; ATAx=ATAx./nu; + else + invNu=Ax./nu; ATAx=A'*invNu-sum(invNu)*mu'; + end + end + + %--------- Line Search for L begins + while (1) + if (iterStep~=1) + alpha= (-gamma+ sqrt(gamma*gamma + 4* L * gamma)) / (2*L); + beta= (gamma - gamma* alphap) / (alphap * gamma + alphap* L * alpha); + % beta is the coefficient for generating search point s + + s=x + beta* xxp; s_t= t + beta * (t -tp); + As=Ax + beta* (Ax-Axp); + ATAs=ATAx + beta * (ATAx- ATAxp); + % compute the search point s, A * s, and A' * A * s + else + alpha= (-1+ sqrt(5)) / 2; + beta=0; s=x; s_t=t; As=Ax; ATAs=ATAx; + end + + g=ATAs-ATy + rsL2 * s; + % compute the gradient g + + % let s walk in a step in the antigradient of s + u=s-g/L; + v= s_t - lambda / L; + + % projection + [xnew, tnew]=ep1R(u,v,n); + + v=xnew-s; % the difference between the new approximate solution x + % and the search point s + v_t=tnew-s_t; + + % compute A xnew + if (opts.nFlag==0) + Axnew=A* xnew; + elseif (opts.nFlag==1) + invNu=xnew./nu; mu_invNu=mu * invNu; + Axnew=A*invNu -repmat(mu_invNu, m, 1); + else + Axnew=A*xnew-repmat(mu*xnew, m, 1); Axnew=Axnew./nu; + end + + Av=Axnew -As; + r_sum=v'*v + v_t'*v_t; l_sum=Av'*Av + v'*v * rsL2; + + if (r_sum <=1e-20) + bFlag=1; % this shows that, the gradient step makes little improvement + break; + end + + % the condition is ||Av||_2^2 + rsL2 * ||v||_2^2 + % <= L * (||v||_2^2 + ||v_t|| _2^2 ) + if(l_sum <= r_sum * L) + break; + else + L=max(2*L, l_sum/r_sum); + % fprintf('\n L=%5.6f',L); + end + end + %--------- Line Search for L ends + + gamma=L* alpha* alpha; alphap=alpha; + % update gamma, and alphap + + ValueL(iterStep)=L; + + tao=L * r_sum / l_sum; + if (tao >=5) + L=L*0.8; + end + % decrease the value of L + + xp=x; x=xnew; xxp=x-xp; + Axp=Ax; Ax=Axnew; + % update x and Ax with xnew and Axnew + tp=t; t=tnew; + % update tp and t + + Axy=Ax-y; + funVal(iterStep)=Axy' * Axy/2 + rsL2/2 * x'*x + lambda * sum(t); + % compute function value + + if (bFlag) + % fprintf('\n The program terminates as the gradient step changes the solution very small.'); + break; + end + + switch(opts.tFlag) + case 0 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <= opts.tol) + break; + end + end + case 1 + if iterStep>=2 + if (abs( funVal(iterStep) - funVal(iterStep-1) ) <=... + opts.tol* funVal(iterStep-1)) + break; + end + end + case 2 + if ( funVal(iterStep)<= opts.tol) + break; + end + case 3 + norm_xxp=sqrt(xxp'*xxp+ norm(t-tp)^2); + if ( norm_xxp <=opts.tol) + break; + end + case 4 + norm_xp=sqrt(xp'*xp + tp'*tp); norm_xxp=sqrt(xxp'*xxp+ norm(t-tp)^2); + if ( norm_xxp <=opts.tol * max(norm_xp,1)) + break; + end + case 5 + if iterStep>=opts.maxIter + break; + end + end + end +end + + +%% +if(opts.mFlag==0 && opts.lFlag==1) + error('\n The function does not support opts.mFlag=0 & opts.lFlag=1!'); +end \ No newline at end of file diff --git a/MAED.m b/MAED.m new file mode 100644 index 0000000..6d2d8b8 --- /dev/null +++ b/MAED.m @@ -0,0 +1,107 @@ +function sampleList = MAED(fea,selectNum,options) +% MAED: Manifold Adaptive Experimental Design +% +% sampleList = MAED(fea,selectNum,options) +% +% Input: +% +% fea - Data matrix. Each row of fea is a sample. +% selectNum - The number of samples to select. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% splitLabel - logical array with the size as the number +% of samples. If some of the inputs already +% have the label and there is no need +% select these samples, you should set the +% corresponding entry in 'splitLabel' as +% true; (Default: all false) +% +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If 'W' is not provided and 'ReguBeta'>0 , +% MAED will build a k-NN graph with Heat kernel +% weight, where 'k' is a prameter. +% +% k - The parameter for k-NN graph (Default is 5) +% If 'W' is provided, this parameter will be +% ignored. +% +% ReguBeta - regularization paramter for manifold +% adaptive kernel. +% +% ReguAlpha - ridge regularization paramter. Default 0.01 +% +% +% Output: +% +% sampleList - The index of the sample which should be labeled. +% +% Examples: +% +% See: http://www.zjucadcg.cn/dengcai/Data/ReproduceExp.html#MAED +% +%Reference: +% +% [1] Deng Cai and Xiaofei He, "Manifold Adaptive Experimental Design for +% Text Categorization", IEEE Transactions on Knowledge and Data +% Engineering, vol. 24, no. 4, pp. 707-719, 2012. +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +nSmp = size(fea,1); + +splitLabel = false(nSmp,1); +if isfield(options,'splitLabel') + splitLabel = options.splitLabel; +end + +K = constructKernel(fea,[],options); +if isfield(options,'ReguBeta') && options.ReguBeta > 0 + if isfield(options,'W') + W = options.W; + else + if isfield(options,'k') + Woptions.k = options.k; + else + Woptions.k = 5; + end + if nSmp > 3000 + tmpD = EuDist2(fea(randsample(nSmp,3000),:)); + else + tmpD = EuDist2(fea); + end + Woptions.t = mean(mean(tmpD)); + W = constructW(fea,Woptions); + end + D = full(sum(W,2)); + L = spdiags(D,0,nSmp,nSmp)-W; + + K=(speye(size(K,1))+options.ReguBeta*K*L)\K; + K = max(K,K'); +end + +if ~isfield(options,'Method') + options.Method = 'Seq'; +end + +ReguAlpha = 0.01; +if isfield(options,'ReguAlpha') + ReguAlpha = options.ReguAlpha; +end + + +switch lower(options.Method) + case {lower('Seq')} + sampleList = MAEDseq(K,selectNum,splitLabel,ReguAlpha); + otherwise + error('Optimization method does not exist!'); +end + + diff --git a/MAEDseq.m b/MAEDseq.m new file mode 100644 index 0000000..84a5803 --- /dev/null +++ b/MAEDseq.m @@ -0,0 +1,45 @@ +function smpRank = MAEDseq(K,selectNum,splitLabel,ReguAlpha) +% MAED: Manifold Adaptive Experimental Design with Sequential Optimization +% +% smpRank = MAEDseq(K,selectNum,splitLabel,ReguAlpha) +% +% This function will be called by MAED.m +% +%Reference: +% +% [1] Deng Cai and Xiaofei He, "Manifold Adaptive Experimental Design for +% Text Categorization", IEEE Transactions on Knowledge and Data +% Engineering, vol. 24, no. 4, pp. 707-719, 2012. +% +% [2] K. Yu, J. Bi, and V. Tresp, "Active Learning via Transductive +% Experimental Design," ICML 2006. +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2008 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if sum(splitLabel) + selectNum > size(K,1) + error('You are requiring too many points!'); +end + +if sum(splitLabel) + Klabel = K(splitLabel,splitLabel); + K = K - (K(:,splitLabel)/(Klabel+ReguAlpha*speye(size(Klabel))))*K(splitLabel,:); +end + +splitCandi = true(size(K,2),1); +if sum(splitLabel) + splitCandi = splitCandi & ~splitLabel; +end + +smpRank = zeros(selectNum,1); +for sel = 1:selectNum + DValue = sum(K(:,splitCandi).^2,1)./(diag(K(splitCandi,splitCandi))'+ReguAlpha); + [value,idx] = max(DValue); + CandiIdx = find(splitCandi); + smpRank(sel) = CandiIdx(idx); + splitCandi(CandiIdx(idx)) = false; + K = K - (K(:,CandiIdx(idx))*K(CandiIdx(idx),:))/(K(CandiIdx(idx),CandiIdx(idx))+ReguAlpha); +end diff --git a/MCFS_p.m b/MCFS_p.m new file mode 100644 index 0000000..07ebd56 --- /dev/null +++ b/MCFS_p.m @@ -0,0 +1,236 @@ +function [FeaIndex,FeaNumCandi] = MCFS_p(fea,FeaNumCandi,options) +% MCFS: Feature Section for Multi Class/Cluster data +% +% FeaIndex = MCFS_p(data,FeaNumCandi,options) +% +% Input: +% fea - data matrix. Each row vector of data is a +% sample vector. +% FeaNumCandi - The number of featuers to be selected +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% gnd - The label of the data. You can provide +% gnd if it is a supervised feature +% selection problem. +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If W is not provided, MCFS_p will +% build a k-NN graph with Heat kernel +% weight, where k is a prameter. (If gnd is +% provided, this parameter will be ignored) +% k - The parameter for k-NN graph (Default is 5) +% If gnd or W is provided, this parameter will be +% ignored. +% nUseEigenfunction - Indicate how many eigen functions will be +% used. If gnd is provided, this parameter +% will be ignored. (Default is 5) +% +% Method - Method used to select features. Choices +% are: +% {'LASSO_LARs'} - (the default) +% 'LASSO_SLEP' +% 'GROUPLASSO_SLEP' +% +% Other fields are: +% * ratio: [default 1] when trying to select M features, +% keep ratio*M non-zero entries in each eigenvector +% (dimension). +% * NotEnoughNonZero: strategy when non-zero entries are +% not enough to select the required number of +% features. This parameter is only used when `ratio' +% is less than 1. It can be the following values: +% * 0: fire an error and exit +% * 1: ignore +% * 2: [default] try to find more non-zero entries, fire error +% when fail +% * 3: try to find more non-zero entries, ignore when +% fail +% +% +% Output: +% FeaIndex - cell variable. Each element in FeaIndex is the +% index of the selected features (the number of +% feature is specified in FeaNumCandi). +% length(FeaIndex) == length(FeaNumCandi) +% +% +% +%=================================================================== +% Examples: +% +%------------------------------------------------------------------- +% (Supervised feature selection) +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% +% options = []; +% options.gnd = gnd; +% FeaNumCandi = [10:5:60]; +% +% [FeaIndex,FeaNumCandi] = MCFS_p(fea, FeaNumCandi,options); +% +% for i = 1:length(FeaNumCandi) +% SelectFeaIdx = FeaIndex{i}; +% feaNew = fea(:,SelectFeaIdx); +% end +% +%------------------------------------------------------------------- +% (Unsupervised feature selection) +% +% fea = rand(50,70); +% +% options = []; +% options.k = 5; %For unsupervised feature selection, you should tune +% %this parameter k, the default k is 5. +% options.nUseEigenfunction = 4; %You should tune this parameter. +% +% FeaNumCandi = [10:5:60]; +% +% [FeaIndex,FeaNumCandi] = MCFS_p(fea,FeaNumCandi,options); +% +% for i = 1:length(FeaNumCandi) +% SelectFeaIdx = FeaIndex{i}; +% feaNew = fea(:,SelectFeaIdx); +% end +% +%=================================================================== +% +%Reference: +% +% Deng Cai, Chiyuan Zhang, Xiaofei He, "Unsupervised Feature Selection +% for Multi-cluster Data",16th ACM SIGKDD Conference on Knowledge +% Discovery and Data Mining (KDD'10), July 2010. +% +% version 1.1 --Dec/2011 +% version 1.0 --Dec/2009 +% +% Written by Deng Cai (dengcai AT gmail.com) +% Chiyuan Zhang (pluskid AT gmail.com) +% + +[nSmp,mFea] = size(fea); +FeaNumCandi = unique(FeaNumCandi); +FeaNumCandi(FeaNumCandi > mFea) = []; + +nUseEigenfunction = 5; +if isfield(options,'nUseEigenfunction') + nUseEigenfunction = options.nUseEigenfunction; +end + +k = 5; +if isfield(options,'k') + k = options.k; +end + +if isfield(options,'ratio') + ratio = options.ratio; +else + ratio = 1; +end + +if isfield(options, 'NotEnoughNonZero') + NotEnoughNonZero = options.NotEnoughNonZero; +else + NotEnoughNonZero = 3; +end + +if isfield(options,'gnd') + if length(options.gnd) ~= nSmp + error('gnd does not match!'); + else + gnd = options.gnd; + end + ClassLabel = unique(gnd); + nClass = length(ClassLabel); + + rand('state',0); + Y = rand(nClass,nClass); + Z = zeros(nSmp,nClass); + for i=1:nClass + idx = find(gnd==ClassLabel(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); + end + Z(:,1) = ones(nSmp,1); + [Y,R] = qr(Z,0); + Y(:,1) = []; +else + if isfield(options,'W') + W = options.W; + else + Woptions.k = k; + if nSmp > 3000 + tmpD = EuDist2(fea(randsample(nSmp,3000),:)); + else + tmpD = EuDist2(fea); + end + Woptions.t = mean(mean(tmpD)); + W = constructW(fea,Woptions); + end + + Y = Eigenmap(W,nUseEigenfunction); +end + +options.ReguType = 'RidgeLasso'; +if ~isfield(options,'Method') + options.Method = 'LASSO_LARs'; +end + +switch lower(options.Method) + case {lower('LASSO_LARs')} + options.LASSOway = 'LARs'; + options.LassoCardi = ceil(FeaNumCandi*ratio); + eigvectorAll = SR(options, Y, fea); + + FeaIndex = cell(1,length(FeaNumCandi)); + for i = 1:length(FeaNumCandi) + eigvector = eigvectorAll{i}; + eigvector = max(abs(eigvector),[],2); + + [dump,idx] = sort(eigvector,'descend'); + if dump(FeaNumCandi(i)) == 0 + if NotEnoughNonZero == 0 % fire error + error('Not enough fea!'); + elseif NotEnoughNonZero == 1 % ignore + warning('Not enough fea!'); + else + for j = i+1:length(FeaNumCandi) + eigvec = eigvectorAll{j}; + eigvec = max(abs(eigvec),[],2); + [dump2,idx2] = sort(eigvec,'descend'); + if (dump2(FeaNumCandi(i)) > 0) + break; + end + end + if (dump2(FeaNumCandi(i)) > 0) + idx = idx2; + else + if (NotEnoughNonZero == 2) + error('Not enough fea, tried to find more but failed!'); + else + warning('Not enough fea, tried to find more but failed!'); + idx = idx2; + end + end + end + end + FeaIndex{i} = idx(1:FeaNumCandi(i)); + end + case {lower('LASSO_SLEP')} + error('Comming soon!'); + case {lower('GROUPLASSO_SLEP')} + error('Comming soon!'); + otherwise + error('method does not exist!'); +end + + + + + + + + diff --git a/MMP.m b/MMP.m new file mode 100644 index 0000000..2db2b18 --- /dev/null +++ b/MMP.m @@ -0,0 +1,166 @@ +function [eigvector, eigvalue] = MMP(gnd,feaLabel,feaUnlabel,options) +% MMP: Maximum Margin Projection +% Semi-supervised version of LSDA (Locality Sensitive Discriminant Analysis) +% +% [eigvector, eigvalue] = MMP(gnd,feaLabel,feaUnlabel,options) +% +% Input: +% gnd - Label vector. +% +% feaLabel - Labeled data matrix. Each row vector of fea is a +% data point. +% feaUnlabel - Unlabeled data matrix. Each row vector of fea is +% a data point. +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% WOptions Please see ConstructW.m for detailed options. +% +% beta [0,1] Paramter to tune the weight between +% within-class graph and between-class +% graph. Default 0.1. +% beta*L_b+(1-beta)*W_w +% +% Please see LGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The eigvalue of LPP eigen-problem. sorted from +% smallest to largest. +% +% +% +% +% See also LPP, LGE, LSDA +% +%Reference: +% +% Xiaofei He, Deng Cai and Jiawei Han, "Learning a Maximum Margin +% Subspace for Image Retrieval", IEEE Transactions on Knowledge and Data +% Engineering, vol. 20, no. 2, pp. 189-201, February, 2008. +% +% version 2.1 --July/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) + + + +[nSmpLabel, nFea] = size(feaLabel); +[nSmpUnlabel, nFea2] = size(feaUnlabel); + +nSmp = nSmpLabel+nSmpUnlabel; + +if nFea ~= nFea2 + error('Fea error!'); +end + +data = [feaLabel;feaUnlabel]; +Ww = constructW(data,options.WOptions); + + +Wb = Ww(1:nSmpLabel,1:nSmpLabel); +[i_idx,j_idx,v_idx] = find(Wb); +for i=1:length(i_idx) + if (gnd(i_idx(i)) == gnd(j_idx(i))) + v_idx(i) = 0; + end +end +Wb = sparse(i_idx,j_idx,v_idx,nSmp,nSmp); + + +if isfield(options.WOptions,'bSemiSupervised') && options.WOptions.bSemiSupervised + if ~isfield(options.WOptions,'SameCategoryWeight') + options.WOptions.SameCategoryWeight = 1; + end + + G2 = zeros(nSmpLabel,nSmpLabel); + Label = unique(gnd); + nLabel = length(Label); + for idx=1:nLabel + classIdx = find(gnd==Label(idx)); + G2(classIdx,classIdx) = options.WOptions.SameCategoryWeight; + end + Ww(1:nSmpLabel,1:nSmpLabel) = G2; +end + + +Db = full(sum(Wb,2)); +Wb = -Wb; +for i=1:size(Wb,1) + Wb(i,i) = Wb(i,i) + Db(i); +end + +D = full(sum(Ww,2)); +if isfield(options,'Regu') && options.Regu + options.ReguAlpha = options.ReguAlpha*sum(D)/length(D); +end + + +beta = 0.1; +if isfield(options,'beta') && (options.beta > 0) && (options.beta < 1) + beta = options.beta; +end + +W = sparse((beta/(1-beta))*Wb+Ww); +clear Wb Ww + + + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean + +elseif isfield(options,'weightMean') && options.keepMean + if issparse(data) + data = full(data); + end + sampleMean = sum(repmat(D,1,nFea).*data,1)./sum(D); + data = (data - repmat(sampleMean,nSmp,1)); +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data,1); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + +if (~isfield(options,'Regu') || ~options.Regu) + DToPowerHalf = D.^.5; + D_mhalf = DToPowerHalf.^-1; + + if nSmp < 5000 + tmpD_mhalf = repmat(D_mhalf,1,nSmp); + W = (tmpD_mhalf.*W).*tmpD_mhalf'; + clear tmpD_mhalf; + else + [i_idx,j_idx,v_idx] = find(W); + v1_idx = zeros(size(v_idx)); + for i=1:length(v_idx) + v1_idx(i) = v_idx(i)*D_mhalf(i_idx(i))*D_mhalf(j_idx(i)); + end + W = sparse(i_idx,j_idx,v1_idx); + clear i_idx j_idx v_idx v1_idx + end + W = max(W,W'); + + data = repmat(DToPowerHalf,1,nFea).*data; + [eigvector, eigvalue] = LGE(W, [], options, data); +else + D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + [eigvector, eigvalue] = LGE(W, D, options, data); +end + + +eigIdx = find(eigvalue < 1e-10); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + diff --git a/MutualInfo.m b/MutualInfo.m new file mode 100644 index 0000000..7076614 --- /dev/null +++ b/MutualInfo.m @@ -0,0 +1,57 @@ +function MIhat = MutualInfo(L1,L2) +% mutual information +% +% version 2.0 --May/2007 +% version 1.0 --November/2003 +% +% Written by Deng Cai (dengcai AT gmail.com) +%=========== +L1 = L1(:); +L2 = L2(:); +if size(L1) ~= size(L2) + error('size(L1) must == size(L2)'); +end + +Label = unique(L1); +nClass = length(Label); + +Label2 = unique(L2); +nClass2 = length(Label2); +if nClass2 < nClass + % smooth + L1 = [L1; Label]; + L2 = [L2; Label]; +elseif nClass2 > nClass + % smooth + L1 = [L1; Label2]; + L2 = [L2; Label2]; +end + + +G = zeros(nClass); +for i=1:nClass + for j=1:nClass + G(i,j) = sum(L1 == Label(i) & L2 == Label(j)); + end +end +sumG = sum(G(:)); + +P1 = sum(G,2); P1 = P1/sumG; +P2 = sum(G,1); P2 = P2/sumG; +if sum(P1==0) > 0 || sum(P2==0) > 0 + % smooth + error('Smooth fail!'); +else + H1 = sum(-P1.*log2(P1)); + H2 = sum(-P2.*log2(P2)); + P12 = G/sumG; + PPP = P12./repmat(P2,nClass,1)./repmat(P1,1,nClass); + PPP(abs(PPP) < 1e-12) = 1; + MI = sum(P12(:) .* log2(PPP(:))); + MIhat = MI / max(H1,H2); + %%%%%%%%%%%%% why complex ? %%%%%%%% + MIhat = real(MIhat); +end + + + diff --git a/NPE.m b/NPE.m new file mode 100644 index 0000000..e50137f --- /dev/null +++ b/NPE.m @@ -0,0 +1,219 @@ +function [eigvector, eigvalue] = NPE(options, data) +% NPE: Neighborhood Preserving Embedding +% +% [eigvector, eigvalue] = NPE(options, data) +% +% Input: +% data - Data matrix. Each row vector of data is a data point. +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% NeighborMode - Indicates how to construct the graph. Choices +% are: +% 'KNN' - Put an edge between two nodes if and +% only if they are among the k nearst +% neighbors of each other. Default +% option. +% 'Supervised' - Two variations: +% 1. k=0, Put an edge between two nodes +% if and only if they belong to +% same class. +% 2. k>0, The distance between two nodes +% in the same class will be smaller than +% two nodes have diff. labels +% The label information 'gnd' should be +% provided. +% +% k - The number of neighbors. +% Default k = 5; +% gnd - The parameter needed under 'Supervised' +% NeighborMode. Colunm vector of the label +% information for each data point. +% +% Please see LGE.m for other options. +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The eigvalue of LPP eigen-problem. sorted from +% smallest to largest. +% +% Examples: +% +% +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.k = 5; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% [eigvector, eigvalue] = NPE(options, fea); +% Y = fea*eigvector; +% +% +% +% See also LPP, LGE +% +%Reference: +% +% Xiaofei He, Deng Cai, Shuicheng Yan, and Hong-Jiang +% Zhang, "Neighborhood Preserving Embedding", Tenth IEEE International +% Conference on Computer Vision (ICCV'2005), 2005 +% +% Sam Roweis & Lawrence Saul. "Nonlinear dimensionality reduction by +% locally linear embedding", Science, v.290 no.5500 , Dec.22, 2000. +% pp.2323--2326. +% +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.1 --May/2006 +% version 1.0 --Feb/2005 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) + + + +if (~exist('options','var')) + options = []; +end + +if ~isfield(options,'NeighborMode') + options.NeighborMode = 'KNN'; +end + +if ~isfield(options,'k') + options.k = 5; +end + + + +[nSmp,nFea] = size(data); + +if options.k >= nSmp + error('k is too large!'); +end + +if(options.k > nFea) + tol=1e-3; % regularlizer in case constrained fits are ill conditioned +else + tol=1e-12; +end + + + +if options.k <= 0 % Always supervised! + if ~isfield(options,'gnd') + error('gnd should be provided!'); + end + if length(options.gnd) ~= nSmp + error('gnd and data mismatch!'); + end + if ~isfield(options,'bEigs') + options.bEigs = 0; + end + + W = zeros(nSmp,nSmp); + for ii=1:nSmp + idx = find(options.gnd==options.gnd(ii)); + idx(find(idx==ii)) = []; + z = data(idx,:)-repmat(data(ii,:),length(idx),1); % shift ith pt to origin + C = z*z'; % local covariance + C = C + eye(size(C))*tol*trace(C); % regularlization + tW = C\ones(length(idx),1); % solve Cw=1 + tW = tW/sum(tW); % enforce sum(w)=1 + W(idx,ii) = tW; + end + M = (eye(size(W)) - W); + M = M*M'; + M = max(M,M'); + M = sparse(M); +else + switch lower(options.NeighborMode) + case {lower('KNN')} + + Distance = EuDist2(data,[],0); + [sorted,index] = sort(Distance,2); + neighborhood = index(:,2:(1+options.k)); + + case {lower('Supervised')} + if ~isfield(options,'gnd') + error('gnd should be provided!'); + end + if length(options.gnd) ~= nSmp + error('gnd and data mismatch!'); + end + if ~isfield(options,'bEigs') + options.bEigs = 0; + end + + Label = unique(options.gnd); + nLabel = length(Label); + neighborhood = zeros(nSmp,options.k); + for idx=1:nLabel + classIdx = find(options.gnd==Label(idx)); + if options.k >= length(classIdx) + error('k is too large!'); + end + Distance = EuDist2(data(classIdx,:),[],0); + [sorted,index] = sort(Distance,2); + neighborhood(classIdx,:) = classIdx(index(:,2:(1+options.k))); + end + otherwise + error('NeighborMode does not exist!'); + end + + W = zeros(options.k,nSmp); + for ii=1:nSmp + z = data(neighborhood(ii,:),:)-repmat(data(ii,:),options.k,1); % shift ith pt to origin + C = z*z'; % local covariance + C = C + eye(size(C))*tol*trace(C); % regularlization + W(:,ii) = C\ones(options.k,1); % solve Cw=1 + W(:,ii) = W(:,ii)/sum(W(:,ii)); % enforce sum(w)=1 + end + + M = sparse(1:nSmp,1:nSmp,ones(1,nSmp),nSmp,nSmp,4*options.k*nSmp); + for ii=1:nSmp + w = W(:,ii); + jj = neighborhood(ii,:)'; + M(ii,jj) = M(ii,jj) - w'; + M(jj,ii) = M(jj,ii) - w; + M(jj,jj) = M(jj,jj) + w*w'; + end + M = max(M,M'); + M = sparse(M); +end + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + +M = -M; +for i=1:size(M,1) + M(i,i) = M(i,i) + 1; +end + + + +[eigvector, eigvalue] = LGE(M, [], options, data); + + +eigIdx = find(eigvalue < 1e-10); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + diff --git a/NormalizeFea.m b/NormalizeFea.m new file mode 100644 index 0000000..d344e1b --- /dev/null +++ b/NormalizeFea.m @@ -0,0 +1,60 @@ +function fea = NormalizeFea(fea,row) +% if row == 1, normalize each row of fea to have unit norm; +% if row == 0, normalize each column of fea to have unit norm; +% +% version 3.0 --Jan/2012 +% version 2.0 --Jan/2012 +% version 1.0 --Oct/2003 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~exist('row','var') + row = 1; +end + +if row + nSmp = size(fea,1); + feaNorm = max(1e-14,full(sum(fea.^2,2))); + fea = spdiags(feaNorm.^-.5,0,nSmp,nSmp)*fea; +else + nSmp = size(fea,2); + feaNorm = max(1e-14,full(sum(fea.^2,1))'); + fea = fea*spdiags(feaNorm.^-.5,0,nSmp,nSmp); +end + +return; + + + + + + + +if row + [nSmp, mFea] = size(fea); + if issparse(fea) + fea2 = fea'; + feaNorm = mynorm(fea2,1); + for i = 1:nSmp + fea2(:,i) = fea2(:,i) ./ max(1e-10,feaNorm(i)); + end + fea = fea2'; + else + feaNorm = sum(fea.^2,2).^.5; + fea = fea./feaNorm(:,ones(1,mFea)); + end +else + [mFea, nSmp] = size(fea); + if issparse(fea) + feaNorm = mynorm(fea,1); + for i = 1:nSmp + fea(:,i) = fea(:,i) ./ max(1e-10,feaNorm(i)); + end + else + feaNorm = sum(fea.^2,1).^.5; + fea = fea./feaNorm(ones(1,mFea),:); + end +end + + diff --git a/OLGE.m b/OLGE.m new file mode 100644 index 0000000..2c4509c --- /dev/null +++ b/OLGE.m @@ -0,0 +1,289 @@ +function [eigvector, eigvalue, bSuccess] = OLGE(W, D, options, data) +% OLGE: Orthogonal Linear Graph Embedding +% +% [eigvector, eigvalue, bSuccess] = OLGE(W, D, options, data) +% +% Input: +% data - data matrix. Each row vector of data is a +% sample vector. +% W - Affinity graph matrix. +% D - Constraint graph matrix. +% LGE solves the optimization problem of +% a* = argmax (a'data'WXa)/(a'data'DXa) +% with the constraint a_i'*a_j=0 (i ~= j) +% Default: D = I +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReducedDim - The dimensionality of the reduced +% subspace. If 0, all the dimensions +% will be kept. Default is 30. +% +% Regu - 1: regularized solution, +% a* = argmax (a'data'WXa)/(a'data'DXa+ReguAlpha*I) +% 0: solve the sinularity problem by SVD (PCA) +% Default: 1 +% +% ReguAlpha - The regularization parameter. Valid +% when Regu==1. Default value is 0.1. +% +% ReguType - 'Ridge': Tikhonov regularization +% 'Custom': User provided +% regularization matrix +% Default: 'Ridge' +% regularizerR - (nFea x nFea) regularization +% matrix which should be provided +% if ReguType is 'Custom'. nFea is +% the feature number of data +% matrix +% +% PCARatio - The percentage of principal +% component kept in the PCA +% step. The percentage is +% calculated based on the +% eigenvalue. Default is 1 +% (100%, all the non-zero +% eigenvalues will be kept. +% If PCARatio > 1, the PCA step +% will keep exactly PCARatio principle +% components (does not exceed the +% exact number of non-zero components). +% +% bDisp - 0 or 1. diagnostic information +% display +% +% Output: +% eigvector - Each column is an embedding function, for a new +% sample vector (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of the eigen-problem. +% +% bSuccess - 0 or 1. Indicates whether the OLPP calcuation +% is successful. (OLPP needs matrix inverse, +% which will lead to eigen-decompose a +% non-symmetrical matrix. The caculation precsion +% of malab sometimes will cause imaginary numbers +% in eigenvectors. It seems that the caculation +% precsion of matlab is a little bit random, you +% can try again if not successful. More robust +% and efficient algorithms are welcome!) +% +% +% Examples: +% +% See also LGE, OLPP, LPP. +% +% +%Reference: +% +% 1. Deng Cai and Xiaofei He, "Orthogonal Locality Preserving Indexing" +% The 28th Annual International ACM SIGIR Conference (SIGIR'2005), +% Salvador, Brazil, Aug. 2005. +% +% 2. Deng Cai, Xiaofei He, Jiawei Han and Hong-Jiang Zhang, "Orthogonal +% Laplacianfaces for Face Recognition". IEEE Transactions on Image +% Processing, vol. 15, no. 11, pp. 3608-3614, November, 2006. +% +% 3. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for Efficient +% Regularized Subspace Learning", IEEE International Conference on +% Computer Vision (ICCV), Rio de Janeiro, Brazil, Oct. 2007. +% +% 4. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Dec/2011 +% version 2.0 --May/2007 +% version 1.0 --Sep/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) + +if (~exist('options','var')) + options = []; +end + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + +if ~isfield(options,'Regu') || ~options.Regu + bPCA = 1; + if ~isfield(options,'PCARatio') + options.PCARatio = 1; + end +else + bPCA = 0; + if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; + end + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end +end + +if ~isfield(options,'bDisp') + options.bDisp = 1; +end + +bD = 1; +if ~exist('D','var') || isempty(D) + bD = 0; +end + + +[nSmp,nFea] = size(data); +if size(W,1) ~= nSmp + error('W and data mismatch!'); +end +if bD && (size(D,1) ~= nSmp) + error('D and data mismatch!'); +end + + +%====================================== +% SVD +%====================================== +if bPCA + [U, S, V] = mySVD(data); + [U, S, V]=CutonRatio(U,S,V,options); + data = U*S; + eigvector_PCA = V; + if bD + DPrime = data'*D*data; + else + DPrime = data'*data; + end + DPrime = max(DPrime,DPrime'); +else + if bD + DPrime = data'*D*data; + else + DPrime = data'*data; + end + + switch lower(options.ReguType) + case {lower('Ridge')} + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + case {lower('Tensor')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + case {lower('Custom')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + otherwise + error('ReguType does not exist!'); + end + + DPrime = max(DPrime,DPrime'); +end + +WPrime = data'*W*data; +WPrime = max(WPrime,WPrime'); + + +%====================================== +% Generalized Eigen with Orthogonal Constraint +%====================================== + + +dimM = size(WPrime,2); + +if ReducedDim > dimM + ReducedDim = dimM; +end + + +rDPrime = chol(DPrime); +lDPrime = rDPrime'; + +Q0 = rDPrime\(lDPrime\WPrime); % Q0 = inv(DPrime)*WPrime; + +eigvector = []; +eigvalue = []; +tmpD = []; +Q = Q0; + +bSuccess = 1; +for i = 1:ReducedDim, + try + option = struct('disp',0); + [eigVec, eigv] = eigs(Q,1,'lr',option); + catch + disp('eigs Error!'); + bSuccess = 0; + return; + end + + if ~isreal(eigVec) + disp('Virtual part!'); + bSuccess = 0; + break; + end + + if eigv < 1e-3 + break; + end + + eigvector = [eigvector, eigVec]; % Each col of D is a eigenvector + eigvalue = [eigvalue;eigv]; + + + tmpD = [tmpD, rDPrime\(lDPrime\eigVec)]; % tmpD = inv(DPrime)*D; + DTran = eigvector'; + tmptmpD = DTran*tmpD; + tmptmpD = max(tmptmpD,tmptmpD'); + rtmptmpD = chol(tmptmpD); + tmptmpD = rtmptmpD\(rtmptmpD'\DTran); % tmptmpD = inv(D'*inv(DPrime)*D)*D' + + Q = -tmpD*tmptmpD; + for j=1:dimM + Q(j,j) = Q(j,j) + 1; + end + Q = Q*Q0; + + if (mod(i,10) == 0) && options.bDisp + disp([num2str(i),' eigenvector calculated!']); + end +end + + +if bPCA + if bSuccess + eigvector = eigvector_PCA*eigvector; + elseif size(eigvector,1) == size(eigvector_PCA,2) + eigvector = eigvector_PCA*eigvector; + end +end + + + + +function [U, S, V]=CutonRatio(U,S,V,options) + if ~isfield(options, 'PCARatio') + options.PCARatio = 1; + end + + eigvalue_PCA = full(diag(S)); + if options.PCARatio > 1 + idx = options.PCARatio; + if idx < length(eigvalue_PCA) + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end + elseif options.PCARatio < 1 + sumEig = sum(eigvalue_PCA); + sumEig = sumEig*options.PCARatio; + sumNow = 0; + for idx = 1:length(eigvalue_PCA) + sumNow = sumNow + eigvalue_PCA(idx); + if sumNow >= sumEig + break; + end + end + U = U(:,1:idx); + V = V(:,1:idx); + S = S(1:idx,1:idx); + end \ No newline at end of file diff --git a/OLPP.m b/OLPP.m new file mode 100644 index 0000000..65336e3 --- /dev/null +++ b/OLPP.m @@ -0,0 +1,131 @@ +function [eigvector, eigvalue, bSuccess] = OLPP(W, options, data) +% OLPP: Orthogonal Locality Preserving Projections +% +% [eigvector, eigvalue, bSuccess] = OLPP(W, options, data) +% +% Input: +% data - Data matrix. Each row vector of fea is a data point. +% W - Affinity matrix. You can either call "constructW" +% to construct the W, or construct it by yourself. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Please see OLGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of OLPP eigen-problem. +% +% bSuccess - 0 or 1. Indicates whether the OLPP calcuation +% is successful. (OLPP needs matrix inverse, +% which will lead to eigen-decompose a +% non-symmetrical matrix. The caculation precsion +% of malab sometimes will cause imaginary numbers +% in eigenvectors. It seems that the caculation +% precsion of matlab is a little bit random, you +% can try again if not successful. More robust +% and efficient algorithms are welcome!) +% +% Please see OLGE.m for other options. +% +% +% +% +% Examples: +% +% fea = rand(50,70); +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 1; +% W = constructW(fea,options); +% options.PCARatio = 0.99 +% options.ReducedDim = 5; +% bSuccess = 0 +% while ~bSuccess +% [eigvector, eigvalue, bSuccess] = OLPP(W, options, fea); +% end +% Y = fea*eigvector; +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.Metric = 'Euclidean'; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% options.bLDA = 1; +% W = constructW(fea,options); +% options.PCARatio = 1; +% options.ReducedDim = 5; +% while ~bSuccess +% [eigvector, eigvalue, bSuccess] = OLPP(W, options, fea); +% end +% Y = fea*eigvector; +% +% +% +% See also constructW, LPP, LGE, OLGE. +% +%Reference: +% +% Deng Cai and Xiaofei He, "Orthogonal Locality Preserving Indexing" +% The 28th Annual International ACM SIGIR Conference (SIGIR'2005), +% Salvador, Brazil, Aug. 2005. +% +% Deng Cai, Xiaofei He, Jiawei Han and Hong-Jiang Zhang, "Orthogonal +% Laplacianfaces for Face Recognition". IEEE Transactions on Image +% Processing, vol. 15, no. 11, pp. 3608-3614, November, 2006. +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu), August/2004, Feb/2006, +% Mar/2007, May/2007 + + + +if (~exist('options','var')) + options = []; +end + + +[nSmp] = size(data,1); +if size(W,1) ~= nSmp + error('W and data mismatch!'); +end + + +D = full(sum(W,2)); + +if isfield(options,'Regu') && options.Regu + options.ReguAlpha = options.ReguAlpha*sum(D)/length(D); +end + +D = sparse(1:nSmp,1:nSmp,D,nSmp,nSmp); + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean +else + if issparse(data) + data = full(data); + end + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); +end +%========================== + + +[eigvector, eigvalue, bSuccess] = OLGE(W, D, options, data); + + +eigIdx = find(eigvalue < 1e-3); +eigvalue (eigIdx) = []; +eigvector(:,eigIdx) = []; + + + + diff --git a/PCA.m b/PCA.m new file mode 100644 index 0000000..87f4756 --- /dev/null +++ b/PCA.m @@ -0,0 +1,76 @@ +function [eigvector, eigvalue] = PCA(data, options) +%PCA Principal Component Analysis +% +% Usage: +% [eigvector, eigvalue] = PCA(data, options) +% [eigvector, eigvalue] = PCA(data) +% +% Input: +% data - Data matrix. Each row vector of fea is a data point. +% +% options.ReducedDim - The dimensionality of the reduced subspace. If 0, +% all the dimensions will be kept. +% Default is 0. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The sorted eigvalue of PCA eigen-problem. +% +% Examples: +% fea = rand(7,10); +% options=[]; +% options.ReducedDim=4; +% [eigvector,eigvalue] = PCA(fea,4); +% Y = fea*eigvector; +% +% version 3.0 --Dec/2011 +% version 2.2 --Feb/2009 +% version 2.1 --June/2007 +% version 2.0 --May/2007 +% version 1.1 --Feb/2006 +% version 1.0 --April/2004 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if (~exist('options','var')) + options = []; +end + +ReducedDim = 0; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + + +[nSmp,nFea] = size(data); +if (ReducedDim > nFea) || (ReducedDim <=0) + ReducedDim = nFea; +end + + +if issparse(data) + data = full(data); +end +sampleMean = mean(data,1); +data = (data - repmat(sampleMean,nSmp,1)); + +[eigvector, eigvalue] = mySVD(data',ReducedDim); +eigvalue = full(diag(eigvalue)).^2; + +if isfield(options,'PCARatio') + sumEig = sum(eigvalue); + sumEig = sumEig*options.PCARatio; + sumNow = 0; + for idx = 1:length(eigvalue) + sumNow = sumNow + eigvalue(idx); + if sumNow >= sumEig + break; + end + end + eigvector = eigvector(:,1:idx); +end + + diff --git a/SCC.m b/SCC.m new file mode 100644 index 0000000..89fc157 --- /dev/null +++ b/SCC.m @@ -0,0 +1,136 @@ +function [B, V, cardiCandi] = SCC(X, nBasis, options) +% SCC: Sparse Concept Coding +% +% [B, V, cardiCandi] = SCC(X, nBasis, options) +% +% minimize_B,V ||X - B*V||^2, each column of V is a sparse vector +% +% Input: +% +% X - Data matrix. Each column of X is a sample. +% nBasis - The num of basis vectors. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% Parameters for basis learning: +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If 'W' is not provided, SCC will +% build a k-NN graph with Heat kernel +% weight, where 'k' is a prameter. +% +% k - The parameter for k-NN graph (Default is 5) +% If 'W' is provided, this parameter will be +% ignored. +% ReguAlpha - regularization paramter for basis +% learning using SR. Default 0.1 +% +% Parameters for sparse representation learning: +% ReguParaType - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in Cardi. +% +% 'SLEP': use SLEP to solve the LASSO +% problem. You need to specify the 'ReguGamma' +% Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% Cardi - An array to specify the cadinality requirement. +% Default [10:10:50] +% +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.05 +% +% +% Output: +% B - The basis matrix. Each column of B is a basis +% vector. +% +% V - If length(cardiCandi) == 1, V is the sparse +% representation of X with respect to basis B. +% If length(cardiCandi) > 1, V is a cell, and the +% i-th element in the cell will be the sparse +% representation with cardinality of cardiCandi(i). +% +% cardiCandi - If length(cardiCandi) == 1, this value is +% meaningless; +% If length(cardiCandi) > 1, each element of V +% tells the cardinality of the corresponding +% element in V. +% +% Examples: +% +% See: http://www.zjucadcg.cn/dengcai/Data/ReproduceExp.html#SCC +% +%Reference: +% +% [1] Deng Cai, Hujun Bao, Xiaofei He, "Sparse Concept Coding for Visual +% Analysis," The 24th IEEE Conference on Computer Vision and Pattern +% Recognition (CVPR 2011), pp. 2905-2910, Colorado Springs, CO, USA, +% 20-25 June 2011. +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2009 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +if ~exist('options','var') + options = []; +end + +% Basis learning +nSmp = size(X,2); +if isfield(options,'W') + W = options.W; +else + Woptions.k = 5; + if isfield(options,'k') + Woptions.k = k; + end + if nSmp > 3000 + tmpD = EuDist2(X(:,randsample(nSmp,3000))'); + else + tmpD = EuDist2(X'); + end + Woptions.t = mean(mean(tmpD)); + W = constructW(X',Woptions); +end + +SRoptions.W = W; +SRoptions.ReducedDim = nBasis; +SRoptions.ReguAlpha = 0.1; +if isfield(options,'ReguAlpha') + SRoptions.ReguAlpha = options.ReguAlpha; +end +B = SR_caller(SRoptions, X'); + + +% Sparse representation learning +if ~isfield(options,'ReguParaType') + options.ReguParaType = 'SLEP'; +end + +if strcmpi(options.ReguParaType,'LARs') + if ~isfield(options,'Cardi') + options.Cardi = 10:10:50; + end +end + +if ~isfield(options,'ReguGamma') + options.ReguAlpha = 0.05; +else + options.ReguAlpha = options.ReguGamma; +end + +[V, cardiCandi] = SparseCodingwithBasis(B, X, options); + +if length(cardiCandi) == 1 + V = V{1}'; +else + for i = 1:length(cardiCandi) + V{i} = V{i}'; + end +end diff --git a/SCCtest.m b/SCCtest.m new file mode 100644 index 0000000..b0fc0d9 --- /dev/null +++ b/SCCtest.m @@ -0,0 +1,92 @@ +function [V, cardiCandi] = SCCtest(X, B, options) +% SCC: Sparse Concept Coding +% +% [V, cardiCandi] = SCCtest(X, B, options) +% +% minimize_B,V ||X - B*V||^2, each column of V is a sparse vector +% +% Input: +% +% X - Data matrix. Each column of X is a sample. +% B - The basis matrix. Each column of B is a basis vector. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReguParaType - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in Cardi. +% +% 'SLEP': use SLEP to solve the LASSO +% problem. You need to specify the 'ReguGamma' +% Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% Cardi - An array to specify the cadinality requirement. +% Default [10:10:50] +% +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.05 +% +% +% Output: +% +% V - If length(cardiCandi) == 1, V is the sparse +% representation of X with respect to basis B. +% If length(cardiCandi) > 1, V is a cell, and the +% i-th element in the cell will be the sparse +% representation with cardinality of cardiCandi(i). +% +% cardiCandi - If length(cardiCandi) == 1, this value is +% meaningless; +% If length(cardiCandi) > 1, each element of V +% tells the cardinality of the corresponding +% element in V. +% +% Examples: +% +% See: http://www.zjucadcg.cn/dengcai/Data/Examples.html#SCC +% +%Reference: +% +% [1] Deng Cai, Hujun Bao, Xiaofei He, "Sparse Concept Coding for Visual +% Analysis," The 24th IEEE Conference on Computer Vision and Pattern +% Recognition (CVPR 2011), pp. 2905-2910, Colorado Springs, CO, USA, +% 20-25 June 2011. +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2009 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +if ~exist('options','var') + options = []; +end + +% Sparse representation learning +if ~isfield(options,'ReguParaType') + options.ReguParaType = 'SLEP'; +end + +if strcmpi(options.ReguParaType,'LARs') + if ~isfield(options,'Cardi') + options.Cardi = 10:10:50; + end +end + +if ~isfield(options,'ReguGamma') + options.ReguAlpha = 0.05; +else + options.ReguAlpha = options.ReguGamma; +end + +[V, cardiCandi] = SparseCodingwithBasis(B, X, options); + +if length(cardiCandi) == 1 + V = V{1}'; +else + for i = 1:length(cardiCandi) + V{i} = V{i}'; + end +end diff --git a/SDA.m b/SDA.m new file mode 100644 index 0000000..bd33f0d --- /dev/null +++ b/SDA.m @@ -0,0 +1,209 @@ +function [eigvector, eigvalue] = SDA(gnd,fea,semiSplit,options) +% SDA: Semi-supervised Discriminant Analysis +% +% [eigvector, eigvalue] = SDA(gnd,feaLabel,feaUnlabel,options) +% +% Input: +% gnd - Label vector. +% fea - data matrix. Each row vector of fea is a data point. +% +% semiSplit - fea(semiSplit,:) is the labeled data matrix. +% - fea(~semiSplit,:) is the unlabeled data matrix. +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% WOptions Please see ConstructW.m for detailed options. +% or +% W You can construct the W outside. +% +% ReguBeta Paramter to tune the weight between +% supervised info and local info +% Default 0.1. +% beta*L+\tilde{I} +% ReguAlpha Paramter of Tinkhonov regularizer +% Default 0.1. +% +% Please see LGE.m for other options. +% +% Output: +% eigvector - Each column is an embedding function, for a new +% data point (row vector) x, y = x*eigvector +% will be the embedding result of x. +% eigvalue - The eigvalue of SDA eigen-problem. sorted from +% smallest to largest. +% +% Examples: +% +% +% +% +% See also LPP, LGE +% +%Reference: +% +% Deng Cai, Xiaofei He and Jiawei Han, "Semi-Supervised Discriminant +% Analysis ", IEEE International Conference on Computer Vision (ICCV), +% Rio de Janeiro, Brazil, Oct. 2007. +% +% version 2.0 --July/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) + + + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end +if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; +end + + +[nSmp,nFea] = size(fea); +nSmpLabel = sum(semiSplit); +nSmpUnlabel = sum(~semiSplit); + + +if nSmpLabel+nSmpUnlabel ~= nSmp + error('input error!'); +end + +if ~isfield(options,'W') + options.WOptions.gnd = gnd; + options.WOptions.semiSplit = semiSplit; + W = constructW(fea,options.WOptions); +else + W = options.W; +end + +gnd = gnd(semiSplit); + +classLabel = unique(gnd); +nClass = length(classLabel); +Dim = nClass; + + +D = full(sum(W,2)); +W = -W; +for i=1:size(W,1) + W(i,i) = W(i,i) + D(i); +end + +ReguBeta = 0.1; +if isfield(options,'ReguBeta') && (options.ReguBeta > 0) + ReguBeta = options.ReguBeta; +end + +LabelIdx = find(semiSplit); +D = W*ReguBeta; +for i=1:nSmpLabel + D(LabelIdx(i),LabelIdx(i)) = D(LabelIdx(i),LabelIdx(i)) + 1; +end + + + + +%========================== +% If data is too large, the following centering codes can be commented +%========================== +if isfield(options,'keepMean') && options.keepMean +else + if issparse(fea) + fea = full(fea); + end + sampleMean = mean(fea,1); + fea = (fea - repmat(sampleMean,nSmp,1)); +end +%========================== + + +DPrime = fea'*D*fea; + +switch lower(options.ReguType) + case {lower('Ridge')} + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + case {lower('RidgeLPP')} + for i=1:size(DPrime,1) + DPrime(i,i) = DPrime(i,i) + options.ReguAlpha; + end + case {lower('Tensor')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + case {lower('Custom')} + DPrime = DPrime + options.ReguAlpha*options.regularizerR; + otherwise + error('ReguType does not exist!'); +end + +DPrime = max(DPrime,DPrime'); + + + +feaLabel = fea(LabelIdx,:); + +Hb = zeros(nClass,nFea); +for i = 1:nClass, + index = find(gnd==classLabel(i)); + classMean = mean(feaLabel(index,:),1); + Hb (i,:) = sqrt(length(index))*classMean; +end +WPrime = Hb'*Hb; +WPrime = max(WPrime,WPrime'); + + + + +dimMatrix = size(WPrime,2); + +if Dim > dimMatrix + Dim = dimMatrix; +end + + +if isfield(options,'bEigs') + if options.bEigs + bEigs = 1; + else + bEigs = 0; + end +else + if (dimMatrix > 1000 && Dim < dimMatrix/20) + bEigs = 1; + else + bEigs = 0; + end +end + + +if bEigs + %disp('use eigs to speed up!'); + option = struct('disp',0); + [eigvector, eigvalue] = eigs(WPrime,DPrime,Dim,'la',option); + eigvalue = diag(eigvalue); +else + [eigvector, eigvalue] = eig(WPrime,DPrime); + eigvalue = diag(eigvalue); + + [junk, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + eigvector = eigvector(:,index); + + if Dim < size(eigvector,2) + eigvector = eigvector(:, 1:Dim); + eigvalue = eigvalue(1:Dim); + end +end + +for i = 1:size(eigvector,2) + eigvector(:,i) = eigvector(:,i)./norm(eigvector(:,i)); +end + + + + + + + diff --git a/SR.m b/SR.m new file mode 100644 index 0000000..b13eab1 --- /dev/null +++ b/SR.m @@ -0,0 +1,325 @@ +function [eigvector, LassoCardi] = SR(options, Responses, data) +% SR: Spectral Regression +% +% [eigvector, LassoCardi] = SR(options, Responses, data) +% +% Input: +% data - data matrix. Each row vector of data is a +% sample vector. +% Responses - response vectors. Each column is a response vector +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReguType - 'Ridge': Tikhonov regularization +% L2-norm regularizer +% 'Lasso': L1-norm regularizer +% 'RidgeLasso': Combine Ridge and Lasso +% 'Custom': User provided +% regularization matrix +% Default: 'Ridge' +% +% 'Lasso' and 'RidgeLasso' will produce +% sparse solution [See Ref 8] +% +% ReguAlpha - The regularization parameter. +% Default value is 0.1 for 'Ridge' +% and 0.05 for 'Lasso' and 'RidgeLasso'. +% +% RidgeAlpha - Only useful if ReguType is 'RidgeLasso' +% 'ReguAlpha' will be the +% regularization parameter for L1-penalty +% 'RidgeAlpha' will be the +% regularization parameter for L2-penalty +% Default value is 0.001. +% +% regularizerR - (mFea x mFea) regularization +% matrix which should be provided +% if ReguType is 'Custom'. mFea is +% the feature number of data +% matrix +% +% LASSOway - 'LARs': use LARs to solve the +% LASSO problem. You need to +% specify the cardinality +% requirement in LassoCardi. +% +% 'SLEP': use SLEP to solve the +% LASSO problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% Output: +% eigvector - Each column is an embedding function, for a new +% sample vector (row vector) x, y = x*eigvector +% will be the embedding result of x. +% +% If 'Lasso' or 'RidgeLasso' regularization is +% used and 'LARs' is choosed to solve the +% problem, the output eigvector will be a cell, +% each element in the cell will be an eigenvector. +% +% LassoCardi - Only useful when ReguType is 'Lasso' and 'RidgeLasso' +% and LASSOway is 'LARs' +% +% Examples: +% +% See SR_caller.m +% +%Reference: +% +% 1. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% 2. Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% 3. Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% 4. Deng Cai, Xiaofei He, and Jiawei Han. "Isometric Projection", Proc. +% 22nd Conference on Artifical Intelligence (AAAI'07), Vancouver, Canada, +% July 2007. +% +% 6. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Subspace Learning Framework for Content-Based Image Retrieval", ACM +% Multimedia 2007, Augsburg, Germany, Sep. 2007. +% +% 7. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for Efficient +% Regularized Subspace Learning", IEEE International Conference on +% Computer Vision (ICCV), Rio de Janeiro, Brazil, Oct. 2007. +% +% 8. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Approach for Sparse Subspace Learning", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% 9. Deng Cai, Xiaofei He, Jiawei Han, "Efficient Kernel Discriminant +% Analysis via Spectral Regression", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% 10. Deng Cai, Xiaofei He, Wei Vivian Zhang, Jiawei Han, "Regularized +% Locality Preserving Indexing via Spectral Regression", Proc. 2007 ACM +% Int. Conf. on Information and Knowledge Management (CIKM'07), Lisboa, +% Portugal, Nov. 2007. +% +% +% version 3.0 --Jan/2012 +% version 2.0 --Aug/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 10000; % You can change this number according your machine computational power +if isfield(options,'MAX_MATRIX_SIZE') + MAX_MATRIX_SIZE = options.MAX_MATRIX_SIZE; +end + + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +[nSmp,mFea] = size(data); + +LassoCardi = 1; +switch lower(options.ReguType) + case {lower('Ridge')} + + KernelWay = 0; + if mFea > nSmp + KernelWay = 1; + end + nScale = min(nSmp,mFea); + if nScale < MAX_MATRIX_SIZE + if isfield(options,'LSQR') && options.LSQR + options.LSQR = 0; + end + end + nRepeat = 20; + if isfield(options,'nRepeat') + nRepeat = options.nRepeat; + end + + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end + + case {lower('Lasso')} + if mFea >= nSmp + warning(['You will only have ',num2str(nSmp),' non-zero coefficients']); + end + options.RidgeAlpha = 0; + options.ReguType = 'RidgeLasso'; + + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.05; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + + if strcmpi(options.LASSOway,'LARs') + if isfield(options,'LassoCardi') + LassoCardi = options.LassoCardi; + else + LassoCardi = 10:10:50; + end + LassoCardi(LassoCardi>mFea) = []; + else + if options.ReguAlpha >= 1 + error('ReguAlpha should be a ratio in (0, 1)!'); + end + end + case {lower('RidgeLasso')} + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.05; + end + if ~isfield(options,'RidgeAlpha') + options.RidgeAlpha = 0.001; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + if strcmpi(options.LASSOway,'LARs') + if isfield(options,'LassoCardi') + LassoCardi = options.LassoCardi; + else + LassoCardi = 10:10:50; + end + LassoCardi(LassoCardi>mFea) = []; + else + if options.ReguAlpha >= 1 + error('ReguAlpha should be a ratio in (0, 1)!'); + end + end + case {lower('Custom')} + otherwise + error('ReguType does not exist!'); +end + + + +switch lower(options.ReguType) + case {lower('Ridge')} + if isfield(options,'LSQR') && options.LSQR + [eigvector, istop] = lsqr2(data, Responses, options.ReguAlpha, nRepeat); + else + if KernelWay + ddata = full(data*data'); + if options.ReguAlpha > 0 + for i=1:size(ddata,1) + ddata(i,i) = ddata(i,i) + options.ReguAlpha; + end + end + + ddata = max(ddata,ddata'); + R = chol(ddata); + eigvector = R\(R'\Responses); + + eigvector = data'*eigvector; + else + ddata = full(data'*data); + + if options.ReguAlpha > 0 + for i=1:size(ddata,1) + ddata(i,i) = ddata(i,i) + options.ReguAlpha; + end + end + + ddata = max(ddata,ddata'); + B = data'*Responses; + + R = chol(ddata); + eigvector = R\(R'\B); + end + end + eigvector = eigvector./repmat(max(1e-10,sum(eigvector.^2,1).^.5),size(eigvector,1),1); + case {lower('RidgeLasso')} + nVector = size(Responses,2); + switch lower(options.LASSOway) + case {lower('LARs')} + eigvector = cell(nVector,1); + if mFea < MAX_MATRIX_SIZE + Gram = data'*data; + Gram = max(Gram,Gram'); + + if options.RidgeAlpha > 0 + for i=1:size(Gram,1) + Gram(i,i) = Gram(i,i) + options.RidgeAlpha; + end + end + + for i = 1:nVector + eigvector_T = lars(data, Responses(:,i),'lasso', -(max(LassoCardi)+5),1,Gram,LassoCardi); + eigvector{i} = eigvector_T; + end + else + if options.RidgeAlpha > 0 + data = [data;sqrt(options.RidgeAlpha)*speye(mFea)]; + Responses = [Responses;zeros(mFea,nVector)]; + end + + for i = 1:nVector + eigvector_T = lars(data, Responses(:,i),'lasso', -(max(LassoCardi)+5),0,[],LassoCardi); + eigvector{i} = eigvector_T; + end + end + case {lower('SLEP')} + eigvector = zeros(size(data,2),nVector); + opts=[]; + opts.rFlag=1; % the input parameter 'ReguAlpha' is a ratio in (0, 1) + opts.init = 2; + if options.RidgeAlpha > 0 + opts.rsL2=options.RidgeAlpha; + end + for i = 1:nVector + eigvector(:,i) = LeastR(data, Responses(:,i), options.ReguAlpha, opts); + end + eigvector = eigvector./repmat(max(1e-10,sum(eigvector.^2,1).^.5),size(eigvector,1),1); + otherwise + error('Method does not exist!'); + end + case {lower('Custom')} + ddata = full(data'*data); + ddata = ddata + options.RegularizerOptions.ReguAlpha*options.RegularizerOptions.regularizerR; + ddata = max(ddata,ddata'); + B = data'*Responses; + + R = chol(ddata); + eigvector = R\(R'\B); + + eigvector = eigvector./repmat(max(1e-10,sum(eigvector.^2,1).^.5),size(eigvector,1),1); + otherwise + error('ReguType does not exist!'); +end + + +if strcmpi(options.ReguType,'RidgeLasso') && strcmpi(options.LASSOway,'LARs') + eigvectorAll = eigvector; + eigvector = cell(length(LassoCardi),1); + + for i = 1:length(eigvectorAll) + eigvector_T = full(eigvectorAll{i}); + [tm,tn] = size(eigvector_T); + tCar = zeros(tn,1); + for k = 1:tn + tCar(k) = length(find(eigvector_T(:,k))); + end + + for cardidx = 1:length(LassoCardi) + ratio = LassoCardi(cardidx); + iMin = find(tCar == ratio); + if isempty(iMin) + error('Card dose not exist!'); + end + tmpEigvec = eigvector_T(:,iMin(end))/norm(eigvector_T(:,iMin(end))); + eigvector{cardidx} = [eigvector{cardidx} tmpEigvec]; + end + end +end + + + + diff --git a/SRDApredict.m b/SRDApredict.m new file mode 100644 index 0000000..1ab76f2 --- /dev/null +++ b/SRDApredict.m @@ -0,0 +1,79 @@ +function [accuracy,predictlabel] = SRDApredict(fea, gnd, model) +% SRDApredict: Spectral Regression Discriminant Analysis Prediction +% SRDApredict uses SRDA as a classifier. It used the nearest +% center rule in the SRDA subspace for classification. +% +% [predictlabel,accuracy,elapse] = SRDApredict(fea, gnd, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% gnd - Label vector of fea. +% model - model trained by SRKDAtrain.m +% +% Output: +% +% accuracy - classification accuracy +% predictlabel - predict label for fea +% +% Examples: +% +% +% See also SRDAtrain, SR, SR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% [3] Deng Cai, Xiaofei He and Jiawei Han, "Semi-Supervised Discriminant +% Analysis ", IEEE International Conference on Computer Vision (ICCV), +% Rio de Janeiro, Brazil, Oct. 2007. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~strcmp(model.TYPE,'SRDA') + error('model does not match!'); +end + + + +nTest = size(fea,1); + + +if model.LARs + accuracy = zeros(length(model.LassoCardi),1); + predictlabel = zeros(nTest,length(model.LassoCardi)); + for i=1:length(model.LassoCardi) + Embed_Test = fea*model.projection{i}; + + D = EuDist2(Embed_Test,model.ClassCenter{i},0); + [dump, idx] = min(D,[],2); + predictlabel(:,i) = model.ClassLabel(idx); + + accuracy(i) = 1 - length(find(predictlabel(:,i)-gnd))/nTest; + end +else + Embed_Test = fea*model.projection; + + D = EuDist2(Embed_Test,model.ClassCenter,0); + [dump, idx] = min(D,[],2); + predictlabel = model.ClassLabel(idx); + + accuracy = 1 - length(find(predictlabel-gnd))/nTest; +end + + + + + diff --git a/SRDAtest.m b/SRDAtest.m new file mode 100644 index 0000000..7ae091c --- /dev/null +++ b/SRDAtest.m @@ -0,0 +1,57 @@ +function [feaNew] = SRDAtest(fea, model) +% SRDAtest: Spectral Regression Discriminant Analysis Testing +% SRDAtest uses SRDA as a dimensionality reduction tool. +% +% [feaNew,elapse] = SRDAtest(fea, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% model - model trained by SRKDAtrain.m +% +% Output: +% +% feaNew - The data in the c-1 SRDA subspace, where c is the +% number of classes. +% +% Examples: +% +% +% See also SRDAtrain, SRDApredict, SR, SR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~strcmp(model.TYPE,'SRDA') + error('model does not match!'); +end + + + +if model.LARs + feaNew = cell(length(model.LassoCardi),1); + for i = 1:length(model.LassoCardi) + feaNew{i} = fea*model.projection{i}; + end +else + feaNew = fea*model.projection; +end + + + + diff --git a/SRDAtrain.m b/SRDAtrain.m new file mode 100644 index 0000000..e8d1940 --- /dev/null +++ b/SRDAtrain.m @@ -0,0 +1,230 @@ +function [model] = SRDAtrain(feaLabel, gnd, options, feaTrain) +% SRDAtrain: Training Spectral Regression Discriminant Analysis +% +% [model] = SRDAtrain(feaLabel, gnd) +% [model] = SRDAtrain(feaLabel, gnd, options) +% [model] = SRDAtrain(feaLabel, gnd, options, feaTrain) +% +% Input: +% +% feaLabel - data matrix. Each row is a data point. +% gnd - Label vector of feaLabel. +% feaTrain - data matrix. This input is optional. If provided, +% SRKDA will be performed in a semi-supervised way. +% feaTrain will be the training data without label. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReguType - 'Ridge': L2-norm regularizer (default) +% 'Lasso': L1-norm regularizer +% ReguAlpha - regularization paramter for L2-norm regularizer +% Default 0.1 +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.05 +% LASSOway - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in LassoCardi. +% 'SLEP': use SLEP to solve the LASSO +% problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% bCenter = 0 | 1 whether to center the data. (In some +% cases, e.g., text categorization, The +% data is very spase, centering the data +% will destroy the sparsity and consume +% too much memory. In this case, bCenter +% should be set to 0) +% Default: 1 +% +% The following fields are only useful when feaTrain is provided. +% +% ReguBeta - Paramter for manifold regularizer +% Default 1 +% Fields for W - Please see ConstructW.m for detailed options. +% +% LaplacianNorm = 0 | 1 (0 for un-normalized and 1 for +% normalized graph laplacian) +% Default: 0 +% LaplacianDegree - power of the graph Laplacian to use as +% the graph regularizer +% Default: 1 +% +% +% Output: +% model - used for SRDApredict.m +% +% +% Examples: +% +% +% +% See also SR, SR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% [3] Deng Cai, Xiaofei He and Jiawei Han, "Semi-Supervised Discriminant +% Analysis ", IEEE International Conference on Computer Vision (ICCV), +% Rio de Janeiro, Brazil, Oct. 2007. +% +% [4] V. Sindhwani, P. Niyogi, M. Belkin, "Beyond the Point Cloud: from +% Transductive to Semi-supervised Learning", ICML 2005. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +if ~exist('options','var') + options = []; +end + +if ~isfield(options,'bCenter') + options.bCenter = 1; +end + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +LARs = false; +switch lower(options.ReguType) + case {lower('Ridge')} + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end + case {lower('Lasso')} + if isfield(options,'ReguAlpha') && options.ReguAlpha > 0 + options.RidgeAlpha = options.ReguAlpha; + options.ReguType = 'RidgeLasso'; + end + if isfield(options,'ReguGamma') + options.ReguAlpha = options.ReguGamma; + else + options.ReguAlpha = 0.05; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + + if strcmpi(options.LASSOway,'LARs') + LARs = true; + if ~isfield(options,'LassoCardi') + options.LassoCardi = 10:10:50; + end + end + otherwise + error('ReguType does not exist!'); +end + + + + +nSmp = size(feaLabel,1); + +ClassLabel = unique(gnd); +model.ClassLabel = ClassLabel; +nClass = length(ClassLabel); + +% Response Generation +rand('state',0); +Y = rand(nClass,nClass); +Z = zeros(nSmp,nClass); +for i=1:nClass + idx = find(gnd==ClassLabel(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); +end +Z(:,1) = ones(nSmp,1); +[Y,R] = qr(Z,0); +Y(:,1) = []; + +feaLabelOrig = feaLabel; +if options.bCenter + sampleMean = mean(feaLabel); + feaLabel = (feaLabel - repmat(sampleMean,nSmp,1)); + if exist('feaTrain','var') + feaTrain = (feaTrain - repmat(sampleMean,size(feaTrain,1),1)); + end +end + +if exist('feaTrain','var') && ~(isfield(options,'ReguBeta') && options.ReguBeta <= 0) + if ~isfield(options,'ReguBeta') + options.ReguBeta = 1; + end + + feaAll = [feaLabel;feaTrain]; + W = constructW(feaAll,options); + + D = full(sum(W,2)); + sizeW = length(D); + if isfield(options,'LaplacianNorm') && options.LaplacianNorm + D=sqrt(1./D); + D=spdiags(D,0,sizeW,sizeW); + W=D*W*D; + L=speye(sizeW)-W; + else + L = spdiags(D,0,sizeW,sizeW)-W; + end + + if isfield(options,'LaplacianDegree') + L = L^options.LaplacianDegree; + end + + K = feaAll*feaAll'; + + I=speye(size(K,1)); + Ktilde=(I+options.ReguBeta*K*L)\K; + Ktilde = max(Ktilde,Ktilde'); + + [eigvector, LassoCardi] = KSR(options, Y, Ktilde(1:nSmp,1:nSmp)); + KtestHat = I-options.ReguBeta*L*Ktilde; + + if LARs + model.projection = cell(length(LassoCardi),1); + for i = 1:length(LassoCardi) + model.projection{i} = feaAll'*KtestHat(:,1:nSmp)*eigvector{i}; + end + else + model.projection = feaAll'*KtestHat(:,1:nSmp)*eigvector; + end +else + [model.projection, LassoCardi] = SR(options, Y, feaLabel); +end +model.LARs = LARs; +model.LassoCardi = LassoCardi; + +if LARs + model.ClassCenter = cell(length(LassoCardi),1); + for i = 1:length(LassoCardi) + Embed_Train = feaLabelOrig*model.projection{i}; + ClassCenter = zeros(nClass,size(Embed_Train,2)); + for j = 1:nClass + feaTmp = Embed_Train(gnd == ClassLabel(j),:); + ClassCenter(j,:) = mean(feaTmp,1); + end + model.ClassCenter{i} = ClassCenter; + end +else + Embed_Train = feaLabelOrig*model.projection; + ClassCenter = zeros(nClass,size(Embed_Train,2)); + for i = 1:nClass + feaTmp = Embed_Train(gnd == ClassLabel(i),:); + ClassCenter(i,:) = mean(feaTmp,1); + end + model.ClassCenter = ClassCenter; +end + +model.TYPE = 'SRDA'; +model.options = options; + + + + diff --git a/SRKDApredict.m b/SRKDApredict.m new file mode 100644 index 0000000..8d3c553 --- /dev/null +++ b/SRKDApredict.m @@ -0,0 +1,114 @@ +function [accuracy,predictlabel] = SRKDApredict(fea, gnd, model) +% SRKDApredict: Spectral Regression Kernel Discriminant Analysis Prediction +% SRKDApredict use SRKDA as a classifier. It used the nearest +% center rule in the SRKDA subspace for classification. +% +% [predictlabel,accuracy,elapse] = SRKDApredict(fea, gnd, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% gnd - Label vector of fea. +% model - model trained by SRKDAtrain.m +% +% Output: +% +% accuracy - classification accuracy +% predictlabel - predict label for fea +% +% Examples: +% +% +% See also SRKDAtrain, KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [2] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [3] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 8000; % You can change this number based on your memory. + +if ~strcmp(model.TYPE,'SRKDA') + error('model does not match!'); +end + + + +nTrain = size(model.fea,1); +nTest = size(fea,1); +nBlock = ceil(MAX_MATRIX_SIZE*MAX_MATRIX_SIZE/nTrain); +if model.LARs + accuracy = zeros(length(model.LassoCardi),1); + predictlabel = zeros(nTest,length(model.LassoCardi)); + Embed_Test = cell(length(model.LassoCardi),1); + for i=1:length(model.LassoCardi) + Embed_Test{i} = zeros(nTest,size(model.projection{i},2)); + end + for j = 1:ceil(nTest/nBlock) + if j == ceil(nTest/nBlock) + smpIdx = (j-1)*nBlock+1:nTest; + else + smpIdx = (j-1)*nBlock+1:j*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + if model.bSemi + KTest = KTest*model.KtestHat; + end + for i=1:length(model.LassoCardi) + if model.bSemi + Embed_Test{i}(smpIdx,:) = KTest(:,1:model.nLabel)*model.projection{i}; + else + Embed_Test{i}(smpIdx,:) = KTest*model.projection{i}; + end + end + clear KTest; + end + + for i=1:length(model.LassoCardi) + D = EuDist2(Embed_Test{i},model.ClassCenter{i},0); + [dump, idx] = min(D,[],2); + predictlabel(:,i) = model.ClassLabel(idx); + accuracy(i) = 1 - length(find(predictlabel(:,i)-gnd))/nTest; + end +else + Embed_Test = zeros(nTest,size(model.projection,2)); + for i = 1:ceil(nTest/nBlock) + if i == ceil(nTest/nBlock) + smpIdx = (i-1)*nBlock+1:nTest; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + if model.bSemi + KTest = KTest*model.KtestHat; + Embed_Test(smpIdx,:) = KTest(:,1:model.nLabel)*model.projection; + else + Embed_Test(smpIdx,:) = KTest*model.projection; + end + clear KTest; + end + D = EuDist2(Embed_Test,model.ClassCenter,0); + [dump, idx] = min(D,[],2); + predictlabel = model.ClassLabel(idx); + accuracy = 1 - length(find(predictlabel-gnd))/nTest; +end + + + + + diff --git a/SRKDAtest.m b/SRKDAtest.m new file mode 100644 index 0000000..23536da --- /dev/null +++ b/SRKDAtest.m @@ -0,0 +1,99 @@ +function [feaNew] = SRKDAtest(fea, model) +% SRKDAtest: Spectral Regression Kernel Discriminant Analysis Testing +% SRKDAtest uses SRKDA as a dimensionality reduction tool. +% +% [feaNew,elapse] = SRKDAtest(fea, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% model - model trained by SRKDAtrain.m +% +% Output: +% +% feaNew - The data in the c-1 SRKDA subspace, where c is the +% number of classes. +% +% Examples: +% +% +% See also SRKDAtrain, SRKDApredict, KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [2] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [3] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 8000; % You can change this number based on your memory. + +if ~strcmp(model.TYPE,'SRKDA') + error('model does not match!'); +end + + + +nTrain = size(model.fea,1); +nTest = size(fea,1); +nBlock = ceil(MAX_MATRIX_SIZE*MAX_MATRIX_SIZE/nTrain); +if model.LARs + feaNew = cell(length(model.LassoCardi),1); + for i=1:length(model.LassoCardi) + feaNew{i} = zeros(nTest,size(model.projection{i},2)); + end + for j = 1:ceil(nTest/nBlock) + if j == ceil(nTest/nBlock) + smpIdx = (j-1)*nBlock+1:nTest; + else + smpIdx = (j-1)*nBlock+1:j*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + if model.bSemi + KTest = KTest*model.KtestHat; + end + for i=1:length(model.LassoCardi) + if model.bSemi + feaNew{i}(smpIdx,:) = KTest(:,1:model.nLabel)*model.projection{i}; + else + feaNew{i}(smpIdx,:) = KTest*model.projection{i}; + end + end + clear KTest; + end +else + feaNew = zeros(nTest,size(model.projection,2)); + for i = 1:ceil(nTest/nBlock) + if i == ceil(nTest/nBlock) + smpIdx = (i-1)*nBlock+1:nTest; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + if model.bSemi + KTest = KTest*model.KtestHat; + feaNew(smpIdx,:) = KTest(:,1:model.nLabel)*model.projection; + else + feaNew(smpIdx,:) = KTest*model.projection; + end + clear KTest; + end +end + + + + + diff --git a/SRKDAtrain.m b/SRKDAtrain.m new file mode 100644 index 0000000..0f47570 --- /dev/null +++ b/SRKDAtrain.m @@ -0,0 +1,266 @@ +function [model] = SRKDAtrain(feaLabel, gnd, options, feaTrain) +% SRKDAtrain: Training Spectral Regression Kernel Discriminant Analysis +% +% [model] = SRKDAtrain(feaLabel, gnd) +% [model] = SRKDAtrain(feaLabel, gnd, options) +% [model] = SRKDAtrain(feaLabel, gnd, options, feaTrain) +% +% Input: +% +% feaLabel - data matrix. Each row is a data point. +% gnd - Label vector of feaLabel. +% feaTrain - data matrix. This input is optional. If provided, +% SRKDA will be performed in a semi-supervised way. +% feaTrain will be the training data without label. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% KernelType - Choices are: +% 'Gaussian' - e^{-(|x-y|^2)/2t^2} (Default) +% 'Polynomial' - (x'*y)^d +% 'PolyPlus' - (x'*y+1)^d +% 'Linear' - x'*y +% +% t - parameter for Gaussian (Default t will be +% estimated from the data) +% d - parameter for Poly +% +% ReguType - 'Ridge': L2-norm regularizer (default) +% 'Lasso': L1-norm regularizer +% ReguAlpha - regularization paramter for L2-norm regularizer +% Default 0.001 +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.1 +% LASSOway - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in LassoCardi. +% 'SLEP': use SLEP to solve the LASSO +% problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% The following fields are only useful when feaTrain is provided. +% +% ReguBeta - Paramter for manifold regularizer +% Default 1 +% Fields for W - Please see ConstructW.m for detailed options. +% +% LaplacianNorm = 0 | 1 (0 for un-normalized and 1 for +% normalized graph laplacian) +% Default: 0 +% LaplacianDegree - power of the graph Laplacian to use as +% the graph regularizer +% Default: 1 +% +% +% +% +% Output: +% model - used for SRKDApredict.m and SRKDAtest.m +% +% +% Examples: +% +% +% +% See also KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% [2] Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% [3] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% [4] V. Sindhwani, P. Niyogi, M. Belkin, "Beyond the Point Cloud: from +% Transductive to Semi-supervised Learning", ICML 2005. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% +MAX_SAMPLE_SIZE = 10000; % This number will only be used when options.approximate = 1; + % You can change this number based on your memory. + + +if ~exist('options','var') + options = []; +end + +if ~isfield(options,'KernelType') + options.KernelType = 'Gaussian'; +end + +if ~isfield(options,'t') + nSmp = size(feaLabel,1); + if nSmp > 3000 + D = EuDist2(feaLabel(randsample(nSmp,3000),:)); + else + D = EuDist2(feaLabel); + end + options.t = mean(mean(D)); +end + +approximate = 0; +if isfield(options,'approximate') + approximate = options.approximate; +end + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +LARs = false; +switch lower(options.ReguType) + case {lower('Ridge')} + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.001; + end + case {lower('Lasso')} + if isfield(options,'ReguAlpha') && options.ReguAlpha > 0 + options.RidgeAlpha = options.ReguAlpha; + options.ReguType = 'RidgeLasso'; + end + if isfield(options,'ReguGamma') + options.ReguAlpha = options.ReguGamma; + else + options.ReguAlpha = 0.1; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + if strcmpi(options.LASSOway,'LARs') + LARs = true; + if ~isfield(options,'LassoCardi') + options.LassoCardi = 10:10:50; + end + end + otherwise + error('ReguType does not exist!'); +end + + + + +nSmp = size(feaLabel,1); + +if nSmp <= MAX_SAMPLE_SIZE + approximate = 0; +end + + +ClassLabel = unique(gnd); +model.ClassLabel = ClassLabel; +nClass = length(ClassLabel); + +% Response Generation +rand('state',0); +Y = rand(nClass,nClass); +Z = zeros(nSmp,nClass); +for i=1:nClass + idx = find(gnd==ClassLabel(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); +end +Z(:,1) = ones(nSmp,1); +[Y,R] = qr(Z,0); +Y(:,1) = []; + +if exist('feaTrain','var') && ~(isfield(options,'ReguBeta') && options.ReguBeta <= 0) + if ~isfield(options,'ReguBeta') + options.ReguBeta = 1; + end + model.bSemi = 1; + model.nLabel = nSmp; + feaAll = [feaLabel;feaTrain]; + model.fea = feaAll; + W = constructW(feaAll,options); + + D = full(sum(W,2)); + sizeW = length(D); + if isfield(options,'LaplacianNorm') && options.LaplacianNorm + D=sqrt(1./D); + D=spdiags(D,0,sizeW,sizeW); + W=D*W*D; + L=speye(size(W,1))-W; + else + L = spdiags(D,0,sizeW,sizeW)-W; + end + + if isfield(options,'LaplacianDegree') + L = L^options.LaplacianDegree; + end + + K = constructKernel(feaAll,[],options); + + I=speye(size(K,1)); + Ktilde=(I+options.ReguBeta*K*L)\K; + Ktilde = max(Ktilde,Ktilde'); + model.KtestHat = I-options.ReguBeta*L*Ktilde; + + [model.projection , LassoCardi] = KSR(options, Y, Ktilde(1:nSmp,1:nSmp)); + if LARs + Embed_Train = cell(length(LassoCardi),1); + for i = 1:length(LassoCardi) + Embed_Train{i} = Ktilde(1:nSmp,1:nSmp)*model.projection{i}; + end + else + Embed_Train = Ktilde(1:nSmp,1:nSmp)*model.projection; + end +else + model.bSemi = 0; + if approximate + idx = randperm(nSmp); + selectIdx = idx(1:MAX_SAMPLE_SIZE); + model.fea = feaLabel(selectIdx,:); + K = constructKernel(feaLabel,model.fea,options); + [model.projection , LassoCardi] = SR(options, Y, K); + else + model.fea = feaLabel; + K = constructKernel(feaLabel,[],options); + [model.projection , LassoCardi] = KSR(options, Y, K); + end + + if LARs + Embed_Train = cell(length(LassoCardi),1); + for i = 1:length(LassoCardi) + Embed_Train{i} = K*model.projection{i}; + end + else + Embed_Train = K*model.projection; + end +end +model.LARs = LARs; +model.LassoCardi = LassoCardi; + +if LARs + model.ClassCenter = cell(length(LassoCardi),1); + for i = 1:length(LassoCardi) + ClassCenter = zeros(nClass,size(Embed_Train{i},2)); + for j = 1:nClass + feaTmp = Embed_Train{i}(gnd == ClassLabel(j),:); + ClassCenter(j,:) = mean(feaTmp,1); + end + model.ClassCenter{i} = ClassCenter; + end +else + ClassCenter = zeros(nClass,size(Embed_Train,2)); + for i = 1:nClass + feaTmp = Embed_Train(gnd == ClassLabel(i),:); + ClassCenter(i,:) = mean(feaTmp,1); + end + model.ClassCenter = ClassCenter; +end + +model.TYPE = 'SRKDA'; +model.options = options; + + + + diff --git a/SR_caller.m b/SR_caller.m new file mode 100644 index 0000000..81dac20 --- /dev/null +++ b/SR_caller.m @@ -0,0 +1,270 @@ +function [eigvector, LassoCardi] = SR_caller(options, data) +% SR_caller: Spectral Regression +% +% [eigvector, LassoCardi] = SR_caller(options, data) +% +% Input: +% data - data matrix. Each row vector of data is a +% sample vector. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% gnd - Colunm vector of the label information +% for each data point. +% If gnd is provided, SR will give the +% SRDA solution [See Ref 2] +% +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If gnd is not provided, W is required and +% SR will give the RLPI (RLPP) solution +% [See Ref 10] +% +% ReducedDim - The number of dimensions. If gnd is +% provided, ReducedDim=c-1 where c is the number +% of classes. Default ReducedDim = 30. +% +% Please see SR.m for other options. +% +% +% Output: +% eigvector - Each column is an embedding function, for a new +% sample vector (row vector) x, y = x*eigvector +% will be the embedding result of x. +% +% If 'Lasso' or 'RidgeLasso' regularization is +% used, the output eigvector will be a cell, +% each element in the cell will be an eigenvector. +% +% LassoCardi - Only useful when ReguType is 'Lasso' and 'RidgeLasso' +% and LASSOway is 'LARs' +% +%=================================================================== +% Examples: +% +% (Supervised case with L2-norm (ridge) regularizer, SR-LDA) +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.gnd = gnd; +% options.ReguAlpha = 0.01; +% options.ReguType = 'Ridge'; +% [eigvector] = SR_caller(options, fea); +% +% if size(eigvector,1) == size(fea,2) + 1 +% Y = [fea ones(size(fea,1),1)]*eigvector; % Y is samples in the SR subspace +% else +% Y = fea*eigvector; % Y is samples in the SR subspace +% end +%------------------------------------------------------------------- +% (Unsupervised case with L2-norm (ridge) regularizer, SR-LPP) +% +% fea = rand(50,70); +% options = []; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% +% options = []; +% options.W = W; +% options.ReguAlpha = 0.01; +% options.ReguType = 'Ridge'; +% options.ReducedDim = 10; +% [eigvector] = SR_caller(options, fea); +% +% if size(eigvector,1) == size(fea,2) + 1 +% Y = [fea ones(size(fea,1),1)]*eigvector; % Y is samples in the SR subspace +% else +% Y = fea*eigvector; % Y is samples in the SR subspace +% end +%------------------------------------------------------------------- +% (Supervised case with L1-norm (lasso) regularizer, SR-SpaseLDA) +% +% fea = rand(50,70); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.gnd = gnd; +% options.ReguType = 'RidgeLasso'; +% +% ---- use SLEP ---- Please see http://www.public.asu.edu/~jye02/Software/SLEP/ for details on SLEP. +% options.LASSOway = 'SLEP'; +% options.ReguAlpha = 0.1; +% eigvector = SR_caller(options, fea); +% +% if size(eigvector,1) == size(fea,2) + 1 +% Y = [fea ones(size(fea,1),1)]*eigvector; % Y is samples in the sparse SR subspace +% else +% Y = fea*eigvector; % Y is samples in the sparse SR subspace +% end +% +% ---- use LARs ---- +% options.LASSOway = 'LARs'; +% options.LassoCardi = [10:5:60]; +% options.RidgeAlpha = 0.001; +% [eigvectorAll,LassoCardi] = SR_caller(options, fea); +% +% for i = 1:length(LassoCardi) +% eigvector = eigvectorAll{i}; % projective functions with cardinality LassoCardi(i) +% +% if size(eigvector,1) == size(fea,2) + 1 +% Y = [fea ones(size(fea,1),1)]*eigvector; % Y is samples in the sparse SR subspace +% else +% Y = fea*eigvector; % Y is samples in the sparse SR subspace +% end +% end +% +%------------------------------------------------------------------- +% (Unsupervised case with L1-norm (lasso) regularizer, SR-SpaseLPP) +% +% fea = rand(50,70); +% options = []; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 5; +% W = constructW(fea,options); +% +% options = []; +% options.W = W; +% options.ReducedDim = 10; +% options.ReguType = 'RidgeLasso'; +% +% ---- use SLEP ---- Please see http://www.public.asu.edu/~jye02/Software/SLEP/ for details on SLEP. +% options.LASSOway = 'SLEP'; +% options.ReguAlpha = 0.1; +% eigvector = SR_caller(options, fea); +% +% Y = fea*eigvector; % Y is samples in the sparse SR subspace +% +% ---- use LARs ---- +% options.LASSOway = 'LARs'; +% options.LassoCardi = [10:5:60]; +% options.RidgeAlpha = 0.001; +% [eigvectorAll,LassoCardi] = SR_caller(options, fea); +% +% for i = 1:length(LassoCardi) +% eigvector = eigvectorAll{i}; % projective functions with cardinality LassoCardi(i) +% Y = fea*eigvector; % Y is samples in the sparse SR subspace +% end +% +%=================================================================== +% +%Reference: +% +% 1. Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% 2. Deng Cai, Xiaofei He and Jiawei Han, "SRDA: An Efficient Algorithm for +% Large Scale Discriminant Analysis" IEEE Transactions on Knowledge and +% Data Engineering, vol. 20, no. 1, pp. 1-12, January, 2008. +% +% 3. Deng Cai, Xiaofei He, and Jiawei Han. "Speed Up Kernel Discriminant +% Analysis", The VLDB Journal, vol. 20, no. 1, pp. 21-33, January, 2011. +% +% 4. Deng Cai, Xiaofei He, and Jiawei Han. "Isometric Projection", Proc. +% 22nd Conference on Artifical Intelligence (AAAI'07), Vancouver, Canada, +% July 2007. +% +% 6. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Subspace Learning Framework for Content-Based Image Retrieval", ACM +% Multimedia 2007, Augsburg, Germany, Sep. 2007. +% +% 7. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression for Efficient +% Regularized Subspace Learning", IEEE International Conference on +% Computer Vision (ICCV), Rio de Janeiro, Brazil, Oct. 2007. +% +% 8. Deng Cai, Xiaofei He, Jiawei Han, "Spectral Regression: A Unified +% Approach for Sparse Subspace Learning", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% 9. Deng Cai, Xiaofei He, Jiawei Han, "Efficient Kernel Discriminant +% Analysis via Spectral Regression", Proc. 2007 Int. Conf. on Data +% Mining (ICDM'07), Omaha, NE, Oct. 2007. +% +% 10. Deng Cai, Xiaofei He, Wei Vivian Zhang, Jiawei Han, "Regularized +% Locality Preserving Indexing via Spectral Regression", Proc. 2007 ACM +% Int. Conf. on Information and Knowledge Management (CIKM'07), Lisboa, +% Portugal, Nov. 2007. +% +% +% version 3.0 --Jan/2012 +% version 2.0 --Aug/2007 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 10000; % You can change this number according your machine computational power +if isfield(options,'MAX_MATRIX_SIZE') + MAX_MATRIX_SIZE = options.MAX_MATRIX_SIZE; +end + + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + + +[nSmp] = size(data,1); + +bSup = 0; +if isfield(options,'gnd') + gnd = options.gnd; + bSup = 1; + if length(gnd) ~= nSmp + error('Label vector wrong!'); + end +else + if ~isfield(options,'W') || (size(options.W,1) ~= nSmp) + error('Graph Error!'); + end + W = options.W; +end + +if ReducedDim > nSmp + ReducedDim = nSmp; +end + + + +if bSup + Label = unique(gnd); + nClass = length(Label); + + rand('state',0); + Y = rand(nClass,nClass); + Z = zeros(nSmp,nClass); + for i=1:nClass + idx = find(gnd==Label(i)); + Z(idx,:) = repmat(Y(i,:),length(idx),1); + end + Z(:,1) = ones(nSmp,1); + [Y,R] = qr(Z,0); + Y(:,1) = []; +else + Y = Eigenmap(W,ReducedDim); +end + +[nSmp,mFea] = size(data); +nScale = min(nSmp,mFea); +if nScale > MAX_MATRIX_SIZE && issparse(data) + options.LSQR = 1; + if isfield(options,'AppendConstant') && options.AppendConstant + data = [data ones(nSmp,1)]; + end +else + sampleMean = mean(data); + data = (data - repmat(sampleMean,nSmp,1)); + options.LSQR = 0; +end + +[eigvector,LassoCardi] = SR(options, Y, data); + + + diff --git a/SparseCodingwithBasis.m b/SparseCodingwithBasis.m new file mode 100644 index 0000000..f378893 --- /dev/null +++ b/SparseCodingwithBasis.m @@ -0,0 +1,113 @@ +function [Y, cardiCandi] = SparseCodingwithBasis(U, fea, options) +% SparseCodingwithBasis: Sparse Coding with the Provided Basis +% +% [Y, cardiCandi] = SparseCodingwithBasis(U, fea, options) +% +% Input: +% +% U - basis matrix. Each column of U is a basis vector. +% fea - data matrix. Each column of fea is a sample. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% ReguParaType - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in Cardi. +% +% 'SLEP': use SLEP to solve the LASSO +% problem. You need to specify the 'ReguAlpha' +% Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% Cardi - An array to specify the cadinality requirement. +% Default [10:10:50] +% +% ReguAlpha - regularization paramter for L1-norm regularizer +% Default 0.05 +% +% +% Output: +% +% Y - A cell. matrix. each element in the cell will be +% a sparse representation of the input fea. +% cardiCandi - An array. The length of 'cardiCandi' is the same +% as the length of 'Y'. +% elapse - Time on SparseCodingwithBasis +% +% +% Examples: +% +% See SCC +% +%Reference: +% +% [1] Deng Cai, Hujun Bao, Xiaofei He, "Sparse Concept Coding for Visual +% Analysis," The 24th IEEE Conference on Computer Vision and Pattern +% Recognition (CVPR 2011), pp. 2905-2910, Colorado Springs, CO, USA, +% 20-25 June 2011. +% +% version 2.0 --Jan/2012 +% version 1.0 --Aug/2009 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + +BasisNum = size(U,2); + +if strcmpi(options.ReguParaType,'LARs') + cardiCandi = options.Cardi; + cardiCandi(cardiCandi > BasisNum) = []; + cardiCandi = unique(cardiCandi); +else + cardiCandi = 1; +end + + +nSmp = size(fea,2); + +Y = cell(length(cardiCandi),1); +for i = 1:length(Y) + Y{i} = zeros(nSmp,BasisNum); +end + +if strcmpi(options.ReguParaType,'LARs') + Gram = U'*U; + Gram = max(Gram,Gram'); + + for i = 1:nSmp + eigvector_T = lars(U, fea(:,i),'lasso', -BasisNum,1,Gram,cardiCandi,0); + + [tm,tn] = size(eigvector_T); + tCar = zeros(tn,1); + for k = 1:tn + tCar(k) = length(find(eigvector_T(:,k))); + end + + for cardidx = 1:length(cardiCandi) + ratio = cardiCandi(cardidx); + iMin = find(tCar == ratio); + if isempty(iMin) + error('Card dose not exist!'); + end + Y{cardidx}(i,:) = eigvector_T(:,iMin(end)); + end + end + + for i = 1:length(Y) + if cardiCandi(i)/BasisNum < 0.6 + Y{i} = sparse(Y{i}); + end + end +elseif strcmpi(options.ReguParaType,'SLEP') + opts=[]; + opts.rFlag=1; % the input parameter 'ReguAlpha' is a ratio in (0, 1) + for i = 1:nSmp + x = LeastR(U, fea(:,i), options.ReguAlpha, opts); + Y{1}(i,:) = x'; + end +else + error('Method does not exist!'); +end + + diff --git a/TensorLGE.m b/TensorLGE.m new file mode 100644 index 0000000..1aba121 --- /dev/null +++ b/TensorLGE.m @@ -0,0 +1,209 @@ +function [U, V, eigvalue_U, eigvalue_V, posIdx, Y] = TensorLGE(X, W, D, options) +% TensorLGE: Tensor-based Linear Graph Embedding +% +% [U, V, eigvalue_U, eigvalue_V, posIdx] = TensorLGE(X, W) +% [U, V, eigvalue_U, eigvalue_V, posIdx] = TensorLGE(X, W, D) +% [U, V, eigvalue_U, eigvalue_V, posIdx] = TensorLGE(X, W, D, options) +% [U, V, eigvalue_U, eigvalue_V, posIdx, Y] = TensorLGE(X, W, D, options) +% +% Input: +% X - 3-d data matrix. X(:,:,i) is the i-th data +% sample. +% W - Affinity graph matrix. +% D - Constraint graph matrix. +% Default: D = I +% +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% nRepeat - The repeat times of the +% iterative procedure. Default +% 10 +% +% Output: +% U, V - Embedding functions, for a new data point +% (matrix) x, y = U'*x*V will be the embedding +% result of x. You might need to resort each +% element in y based on the posIdx. +% eigvalue_U +% eigvalue_V - corresponding eigenvalue. +% +% Y - The embedding results, Each row vector is a +% data point. The features in Y has been sorted +% that Y(:,i) will be important to Y(:,j) with +% respect to the objective function if i size(model.projection{1},2) + error(['d is too large for this model. The largest d is ',num2str(size(model.projection,2))]); + end + feaNew = cell(length(model.LassoCardi),1); + for i=1:length(model.LassoCardi) + feaNew{i} = zeros(nTest,d); + end + for j = 1:ceil(nTest/nBlock) + if j == ceil(nTest/nBlock) + smpIdx = (j-1)*nBlock+1:nTest; + else + smpIdx = (j-1)*nBlock+1:j*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + for i=1:length(model.LassoCardi) + feaNew{i}(smpIdx,:) = KTest*model.projection{i}(:,1:d); + end + clear KTest; + end +else + if d > size(model.projection,2) + error(['d is too large for this model. The largest d is ',num2str(size(model.projection,2))]); + end + feaNew = zeros(nTest,d); + for i = 1:ceil(nTest/nBlock) + if i == ceil(nTest/nBlock) + smpIdx = (i-1)*nBlock+1:nTest; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + KTest= constructKernel(fea(smpIdx,:),model.fea,model.options); + feaNew(smpIdx,:) = KTest*model.projection(:,1:d); + clear KTest; + end +end + + + + + + diff --git a/UKSRtrain.m b/UKSRtrain.m new file mode 100644 index 0000000..4891722 --- /dev/null +++ b/UKSRtrain.m @@ -0,0 +1,174 @@ +function [model] = UKSRtrain(fea, options) +% UKSRtrain: Training Unsupervised Kernel Spectral Regression Model +% +% [model] = UKSRtrain(fea, options) +% +% Input: +% +% fea - data matrix. Each row is a data point. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If W is not provided, USRtrain will +% build a k-NN graph with Heat kernel +% weight, where k is a prameter. +% +% k - The parameter for k-NN graph (Default is 5) +% If W is provided, this k will be ignored. +% +% ReducedDim - The number of reduced dimensions. +% Default ReducedDim = 30. +% +% KernelType - Choices are: +% 'Gaussian' - e^{-(|x-y|^2)/2t^2} (Default) +% 'Polynomial' - (x'*y)^d +% 'PolyPlus' - (x'*y+1)^d +% 'Linear' - x'*y +% +% t - parameter for Gaussian (Default t will be +% estimated from the data) +% d - parameter for Poly +% +% ReguType - 'Ridge': L2-norm regularizer (default) +% 'Lasso': L1-norm regularizer +% ReguAlpha - regularization paramter for L2-norm regularizer +% Default 0.001 +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.1 +% LASSOway - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in LassoCardi. +% 'SLEP': use SLEP to solve the LASSO +% problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% +% Output: +% model - used for USRtest.m +% +% +% Examples: +% +% +% +% See also KSR, KSR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, Wei Vivian Zhang, Jiawei Han, "Regularized +% Locality Preserving Indexing via Spectral Regression", Proc. 2007 ACM +% Int. Conf. on Information and Knowledge Management (CIKM'07), Lisboa, +% Portugal, Nov. 2007. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~exist('options','var') + options = []; +end + +k = 5; +if isfield(options,'k') + k = options.k; +end + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +LARs = false; +switch lower(options.ReguType) + case {lower('Ridge')} + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.001; + end + case {lower('Lasso')} + if isfield(options,'ReguAlpha') && options.ReguAlpha > 0 + options.RidgeAlpha = options.ReguAlpha; + options.ReguType = 'RidgeLasso'; + end + if isfield(options,'ReguGamma') + options.ReguAlpha = options.ReguGamma; + else + options.ReguAlpha = 0.1; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + if strcmpi(options.LASSOway,'LARs') + LARs = true; + if ~isfield(options,'LassoCardi') + options.LassoCardi = 10:10:50; + end + end + otherwise + error('ReguType does not exist!'); +end + + +nSmp=size(fea,1); +% Graph construction +if isfield(options,'W') + W = options.W; +else + Woptions.k = k; + if nSmp > 3000 + tmpIdx=randperm(nSmp); + tmpD = EuDist2(fea(tmpIdx(1:3000),:)); + else + tmpD = EuDist2(fea); + end + Woptions.t = mean(mean(tmpD)); + options.t = Woptions.t; + W = constructW(fea,Woptions); +end + + +% Respnse generation +Y = Eigenmap(W,ReducedDim); + + +% Projection learning +if ~isfield(options,'KernelType') + options.KernelType = 'Gaussian'; +end + +if ~isfield(options,'t') + nSmp = size(fea,1); + if nSmp > 3000 + D = EuDist2(fea(randsample(nSmp,3000),:)); + else + D = EuDist2(fea); + end + options.t = mean(mean(D)); +end + +model.fea = fea; +K = constructKernel(fea,[],options); + +[model.projection, LassoCardi] = KSR(options, Y, K); +model.LARs = LARs; +model.LassoCardi = LassoCardi; + +model.TYPE = 'UKSR'; +model.options = options; + + + + diff --git a/USRtest.m b/USRtest.m new file mode 100644 index 0000000..043ae0d --- /dev/null +++ b/USRtest.m @@ -0,0 +1,67 @@ +function feaNew = USRtest(fea, d, model) +% USRtest: Unsupervised Spectral Regression Testing +% USRtest uses SR as a dimensionality reduction tool. +% +% [feaNew,elapse] = USRtest(fea, d, model); +% +% Input: +% +% fea - data matrix. Each row is a data point. +% d - the reduced dimensions. d should not be larger +% than options.ReducedDim calling USRtrain.m +% model - model trained by USRtrain.m +% +% Output: +% +% feaNew - The data in the d-dimensional SR subspace. +% +% Examples: +% +% +% See also USRtrain, SR, SR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, Wei Vivian Zhang, Jiawei Han, "Regularized +% Locality Preserving Indexing via Spectral Regression", Proc. 2007 ACM +% Int. Conf. on Information and Knowledge Management (CIKM'07), Lisboa, +% Portugal, Nov. 2007. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + + + +if ~strcmp(model.TYPE,'USR') + error('model does not match!'); +end + + +if model.LARs + if d > size(model.projection{1},2) + error(['d is too large for this model. The largest d is ',num2str(size(model.projection,2))]); + end + feaNew = cell(length(model.LassoCardi),1); + for i = 1:length(model.LassoCardi) + feaNew{i} = fea*model.projection{i}(:,1:d); + end +else + if d > size(model.projection,2) + error(['d is too large for this model. The largest d is ',num2str(size(model.projection,2))]); + end + feaNew = fea*model.projection(:,1:d); +end + + + + + diff --git a/USRtrain.m b/USRtrain.m new file mode 100644 index 0000000..0c9d5b2 --- /dev/null +++ b/USRtrain.m @@ -0,0 +1,165 @@ +function [model] = USRtrain(fea, options) +% USRtrain: Training Unsupervised Spectral Regression Model +% +% [model] = USRtrain(fea, options) +% +% Input: +% +% fea - data matrix. Each row is a data point. +% options - Struct value in Matlab. The fields in options +% that can be set: +% +% W - Affinity matrix. You can either call +% "constructW" to construct the W, or +% construct it by yourself. +% If W is not provided, USRtrain will +% build a k-NN graph with Heat kernel +% weight, where k is a prameter. +% +% k - The parameter for k-NN graph (Default is 5) +% If W is provided, this k will be ignored. +% +% ReducedDim - The number of reduced dimensions. +% Default ReducedDim = 30. +% +% ReguType - 'Ridge': L2-norm regularizer (default) +% 'Lasso': L1-norm regularizer +% ReguAlpha - regularization paramter for L2-norm regularizer +% Default 0.1 +% ReguGamma - regularization paramter for L1-norm regularizer +% Default 0.05 +% LASSOway - 'LARs': use LARs to solve the LASSO +% problem. You need to specify the +% cardinality requirement in LassoCardi. +% 'SLEP': use SLEP to solve the LASSO +% problem. Please see http://www.public.asu.edu/~jye02/Software/SLEP/ +% for details on SLEP. (The Default) +% +% bCenter = 0 | 1 whether to center the data. (In some +% cases, e.g., text categorization, The +% data is very spase, centering the data +% will destroy the sparsity and consume +% too much memory. In this case, bCenter +% should be set to 0) +% Default: 1 +% +% +% Output: +% model - used for USRtest.m +% +% +% Examples: +% +% +% +% See also SR, SR_caller +% +%Reference: +% +% [1] Deng Cai, Xiaofei He, Wei Vivian Zhang, Jiawei Han, "Regularized +% Locality Preserving Indexing via Spectral Regression", Proc. 2007 ACM +% Int. Conf. on Information and Knowledge Management (CIKM'07), Lisboa, +% Portugal, Nov. 2007. +% +% [2] Deng Cai, "Spectral Regression: A Regression Framework for +% Efficient Regularized Subspace Learning", PhD Thesis, Department of +% Computer Science, UIUC, 2009. +% +% version 3.0 --Jan/2012 +% version 2.0 --December/2011 +% version 1.0 --May/2006 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~exist('options','var') + options = []; +end + +bCenter = 1; +if isfield(options,'bCenter') + bCenter = options.bCenter; +end + +k = 5; +if isfield(options,'k') + k = options.k; +end + +ReducedDim = 30; +if isfield(options,'ReducedDim') + ReducedDim = options.ReducedDim; +end + + +if ~isfield(options,'ReguType') + options.ReguType = 'Ridge'; +end + +LARs = false; +switch lower(options.ReguType) + case {lower('Ridge')} + if ~isfield(options,'ReguAlpha') + options.ReguAlpha = 0.1; + end + case {lower('Lasso')} + if isfield(options,'ReguAlpha') && options.ReguAlpha > 0 + options.RidgeAlpha = options.ReguAlpha; + options.ReguType = 'RidgeLasso'; + end + if isfield(options,'ReguGamma') + options.ReguAlpha = options.ReguGamma; + else + options.ReguAlpha = 0.05; + end + if ~isfield(options,'LASSOway') + options.LASSOway = 'SLEP'; + end + + if strcmpi(options.LASSOway,'LARs') + LARs = true; + if ~isfield(options,'LassoCardi') + options.LassoCardi = 10:10:50; + end + end + otherwise + error('ReguType does not exist!'); +end + + +nSmp=size(fea,1); +% Graph construction +if isfield(options,'W') + W = options.W; +else + Woptions.k = k; + if nSmp > 3000 + tmpD = EuDist2(fea(randsample(nSmp,3000),:)); + else + tmpD = EuDist2(fea); + end + Woptions.t = mean(mean(tmpD)); + W = constructW(fea,Woptions); +end + +% Respnse generation +Y = Eigenmap(W,ReducedDim); + +% Projection learning +if bCenter + sampleMean = mean(fea); + fea = (fea - repmat(sampleMean,nSmp,1)); +end + +[model.projection, LassoCardi] = SR(options, Y, fea); + +model.LARs = LARs; +model.LassoCardi = LassoCardi; + + +model.TYPE = 'USR'; +model.options = options; + + + + diff --git a/bestMap.m b/bestMap.m new file mode 100644 index 0000000..f5762aa --- /dev/null +++ b/bestMap.m @@ -0,0 +1,37 @@ +function [newL2] = bestMap(L1,L2) +%bestmap: permute labels of L2 to match L1 as good as possible +% [newL2] = bestMap(L1,L2); +% +% version 2.0 --May/2007 +% version 1.0 --November/2003 +% +% Written by Deng Cai (dengcai AT gmail.com) + + +%=========== + +L1 = L1(:); +L2 = L2(:); +if size(L1) ~= size(L2) + error('size(L1) must == size(L2)'); +end + +Label1 = unique(L1); +nClass1 = length(Label1); +Label2 = unique(L2); +nClass2 = length(Label2); + +nClass = max(nClass1,nClass2); +G = zeros(nClass); +for i=1:nClass1 + for j=1:nClass2 + G(i,j) = length(find(L1 == Label1(i) & L2 == Label2(j))); + end +end + +[c,t] = hungarian(-G); +newL2 = zeros(size(L2)); +for i=1:nClass2 + newL2(L2 == Label2(i)) = Label1(c(i)); +end + diff --git a/constructKernel.m b/constructKernel.m new file mode 100644 index 0000000..801d0a6 --- /dev/null +++ b/constructKernel.m @@ -0,0 +1,98 @@ +function K = constructKernel(fea_a,fea_b,options) +% function K = constructKernel(fea_a,fea_b,options) +% Usage: +% K = constructKernel(fea_a,[],options) +% +% K = constructKernel(fea_a,fea_b,options) +% +% fea_a, fea_b : Rows of vectors of data points. +% +% options : Struct value in Matlab. The fields in options that can +% be set: +% KernelType - Choices are: +% 'Gaussian' - e^{-(|x-y|^2)/2t^2} +% 'Polynomial' - (x'*y)^d +% 'PolyPlus' - (x'*y+1)^d +% 'Linear' - x'*y +% +% t - parameter for Gaussian +% d - parameter for Poly +% +% version 1.0 --Sep/2006 +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu) +% + +if (~exist('options','var')) + options = []; +else + if ~isstruct(options) + error('parameter error!'); + end +end + + + +%================================================= +if ~isfield(options,'KernelType') + options.KernelType = 'Gaussian'; +end + +switch lower(options.KernelType) + case {lower('Gaussian')} % e^{-(|x-y|^2)/2t^2} + if ~isfield(options,'t') + options.t = 1; + end + case {lower('Polynomial')} % (x'*y)^d + if ~isfield(options,'d') + options.d = 2; + end + case {lower('PolyPlus')} % (x'*y+1)^d + if ~isfield(options,'d') + options.d = 2; + end + case {lower('Linear')} % x'*y + otherwise + error('KernelType does not exist!'); +end + + +%================================================= + +switch lower(options.KernelType) + case {lower('Gaussian')} + if isempty(fea_b) + D = EuDist2(fea_a,[],0); + else + D = EuDist2(fea_a,fea_b,0); + end + K = exp(-D/(2*options.t^2)); + case {lower('Polynomial')} + if isempty(fea_b) + D = full(fea_a * fea_a'); + else + D = full(fea_a * fea_b'); + end + K = D.^options.d; + case {lower('PolyPlus')} + if isempty(fea_b) + D = full(fea_a * fea_a'); + else + D = full(fea_a * fea_b'); + end + K = (D+1).^options.d; + case {lower('Linear')} + if isempty(fea_b) + K = full(fea_a * fea_a'); + else + K = full(fea_a * fea_b'); + end + otherwise + error('KernelType does not exist!'); +end + +if isempty(fea_b) + K = max(K,K'); +end + + diff --git a/constructW.m b/constructW.m new file mode 100644 index 0000000..9519a8d --- /dev/null +++ b/constructW.m @@ -0,0 +1,526 @@ +function W = constructW(fea,options) +% Usage: +% W = constructW(fea,options) +% +% fea: Rows of vectors of data points. Each row is x_i +% options: Struct value in Matlab. The fields in options that can be set: +% +% NeighborMode - Indicates how to construct the graph. Choices +% are: [Default 'KNN'] +% 'KNN' - k = 0 +% Complete graph +% k > 0 +% Put an edge between two nodes if and +% only if they are among the k nearst +% neighbors of each other. You are +% required to provide the parameter k in +% the options. Default k=5. +% 'Supervised' - k = 0 +% Put an edge between two nodes if and +% only if they belong to same class. +% k > 0 +% Put an edge between two nodes if +% they belong to same class and they +% are among the k nearst neighbors of +% each other. +% Default: k=0 +% You are required to provide the label +% information gnd in the options. +% +% WeightMode - Indicates how to assign weights for each edge +% in the graph. Choices are: +% 'Binary' - 0-1 weighting. Every edge receiveds weight +% of 1. +% 'HeatKernel' - If nodes i and j are connected, put weight +% W_ij = exp(-norm(x_i - x_j)/2t^2). You are +% required to provide the parameter t. [Default One] +% 'Cosine' - If nodes i and j are connected, put weight +% cosine(x_i,x_j). +% +% k - The parameter needed under 'KNN' NeighborMode. +% Default will be 5. +% gnd - The parameter needed under 'Supervised' +% NeighborMode. Colunm vector of the label +% information for each data point. +% bLDA - 0 or 1. Only effective under 'Supervised' +% NeighborMode. If 1, the graph will be constructed +% to make LPP exactly same as LDA. Default will be +% 0. +% t - The parameter needed under 'HeatKernel' +% WeightMode. Default will be 1 +% bNormalized - 0 or 1. Only effective under 'Cosine' WeightMode. +% Indicates whether the fea are already be +% normalized to 1. Default will be 0 +% bSelfConnected - 0 or 1. Indicates whether W(i,i) == 1. Default 0 +% if 'Supervised' NeighborMode & bLDA == 1, +% bSelfConnected will always be 1. Default 0. +% bTrueKNN - 0 or 1. If 1, will construct a truly kNN graph +% (Not symmetric!). Default will be 0. Only valid +% for 'KNN' NeighborMode +% +% +% Examples: +% +% fea = rand(50,15); +% options = []; +% options.NeighborMode = 'KNN'; +% options.k = 5; +% options.WeightMode = 'HeatKernel'; +% options.t = 1; +% W = constructW(fea,options); +% +% +% fea = rand(50,15); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% options.WeightMode = 'HeatKernel'; +% options.t = 1; +% W = constructW(fea,options); +% +% +% fea = rand(50,15); +% gnd = [ones(10,1);ones(15,1)*2;ones(10,1)*3;ones(15,1)*4]; +% options = []; +% options.NeighborMode = 'Supervised'; +% options.gnd = gnd; +% options.bLDA = 1; +% W = constructW(fea,options); +% +% +% For more details about the different ways to construct the W, please +% refer: +% Deng Cai, Xiaofei He and Jiawei Han, "Document Clustering Using +% Locality Preserving Indexing" IEEE TKDE, Dec. 2005. +% +% +% Written by Deng Cai (dengcai2 AT cs.uiuc.edu), April/2004, Feb/2006, +% May/2007 +% + +bSpeed = 1; + +if (~exist('options','var')) + options = []; +end + +if isfield(options,'Metric') + warning('This function has been changed and the Metric is no longer be supported'); +end + + +if ~isfield(options,'bNormalized') + options.bNormalized = 0; +end + +%================================================= +if ~isfield(options,'NeighborMode') + options.NeighborMode = 'KNN'; +end + +switch lower(options.NeighborMode) + case {lower('KNN')} %For simplicity, we include the data point itself in the kNN + if ~isfield(options,'k') + options.k = 5; + end + case {lower('Supervised')} + if ~isfield(options,'bLDA') + options.bLDA = 0; + end + if options.bLDA + options.bSelfConnected = 1; + end + if ~isfield(options,'k') + options.k = 0; + end + if ~isfield(options,'gnd') + error('Label(gnd) should be provided under ''Supervised'' NeighborMode!'); + end + if ~isempty(fea) && length(options.gnd) ~= size(fea,1) + error('gnd doesn''t match with fea!'); + end + otherwise + error('NeighborMode does not exist!'); +end + +%================================================= + +if ~isfield(options,'WeightMode') + options.WeightMode = 'HeatKernel'; +end + +bBinary = 0; +bCosine = 0; +switch lower(options.WeightMode) + case {lower('Binary')} + bBinary = 1; + case {lower('HeatKernel')} + if ~isfield(options,'t') + nSmp = size(fea,1); + if nSmp > 3000 + D = EuDist2(fea(randsample(nSmp,3000),:)); + else + D = EuDist2(fea); + end + options.t = mean(mean(D)); + end + case {lower('Cosine')} + bCosine = 1; + otherwise + error('WeightMode does not exist!'); +end + +%================================================= + +if ~isfield(options,'bSelfConnected') + options.bSelfConnected = 0; +end + +%================================================= + +if isfield(options,'gnd') + nSmp = length(options.gnd); +else + nSmp = size(fea,1); +end +maxM = 62500000; %500M +BlockSize = floor(maxM/(nSmp*3)); + + +if strcmpi(options.NeighborMode,'Supervised') + Label = unique(options.gnd); + nLabel = length(Label); + if options.bLDA + G = zeros(nSmp,nSmp); + for idx=1:nLabel + classIdx = options.gnd==Label(idx); + G(classIdx,classIdx) = 1/sum(classIdx); + end + W = sparse(G); + return; + end + + switch lower(options.WeightMode) + case {lower('Binary')} + if options.k > 0 + G = zeros(nSmp*(options.k+1),3); + idNow = 0; + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D = EuDist2(fea(classIdx,:),[],0); + [dump idx] = sort(D,2); % sort each row + clear D dump; + idx = idx(:,1:options.k+1); + + nSmpClass = length(classIdx)*(options.k+1); + G(idNow+1:nSmpClass+idNow,1) = repmat(classIdx,[options.k+1,1]); + G(idNow+1:nSmpClass+idNow,2) = classIdx(idx(:)); + G(idNow+1:nSmpClass+idNow,3) = 1; + idNow = idNow+nSmpClass; + clear idx + end + G = sparse(G(:,1),G(:,2),G(:,3),nSmp,nSmp); + G = max(G,G'); + else + G = zeros(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + G(classIdx,classIdx) = 1; + end + end + + if ~options.bSelfConnected + for i=1:size(G,1) + G(i,i) = 0; + end + end + + W = sparse(G); + case {lower('HeatKernel')} + if options.k > 0 + G = zeros(nSmp*(options.k+1),3); + idNow = 0; + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D = EuDist2(fea(classIdx,:),[],0); + [dump idx] = sort(D,2); % sort each row + clear D; + idx = idx(:,1:options.k+1); + dump = dump(:,1:options.k+1); + dump = exp(-dump/(2*options.t^2)); + + nSmpClass = length(classIdx)*(options.k+1); + G(idNow+1:nSmpClass+idNow,1) = repmat(classIdx,[options.k+1,1]); + G(idNow+1:nSmpClass+idNow,2) = classIdx(idx(:)); + G(idNow+1:nSmpClass+idNow,3) = dump(:); + idNow = idNow+nSmpClass; + clear dump idx + end + G = sparse(G(:,1),G(:,2),G(:,3),nSmp,nSmp); + else + G = zeros(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D = EuDist2(fea(classIdx,:),[],0); + D = exp(-D/(2*options.t^2)); + G(classIdx,classIdx) = D; + end + end + + if ~options.bSelfConnected + for i=1:size(G,1) + G(i,i) = 0; + end + end + + W = sparse(max(G,G')); + case {lower('Cosine')} + if ~options.bNormalized + fea = NormalizeFea(fea); + end + + if options.k > 0 + G = zeros(nSmp*(options.k+1),3); + idNow = 0; + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + D = fea(classIdx,:)*fea(classIdx,:)'; + [dump idx] = sort(-D,2); % sort each row + clear D; + idx = idx(:,1:options.k+1); + dump = -dump(:,1:options.k+1); + + nSmpClass = length(classIdx)*(options.k+1); + G(idNow+1:nSmpClass+idNow,1) = repmat(classIdx,[options.k+1,1]); + G(idNow+1:nSmpClass+idNow,2) = classIdx(idx(:)); + G(idNow+1:nSmpClass+idNow,3) = dump(:); + idNow = idNow+nSmpClass; + clear dump idx + end + G = sparse(G(:,1),G(:,2),G(:,3),nSmp,nSmp); + else + G = zeros(nSmp,nSmp); + for i=1:nLabel + classIdx = find(options.gnd==Label(i)); + G(classIdx,classIdx) = fea(classIdx,:)*fea(classIdx,:)'; + end + end + + if ~options.bSelfConnected + for i=1:size(G,1) + G(i,i) = 0; + end + end + + W = sparse(max(G,G')); + otherwise + error('WeightMode does not exist!'); + end + return; +end + + +if bCosine && ~options.bNormalized + Normfea = NormalizeFea(fea); +end + +if strcmpi(options.NeighborMode,'KNN') && (options.k > 0) + if ~(bCosine && options.bNormalized) + G = zeros(nSmp*(options.k+1),3); + for i = 1:ceil(nSmp/BlockSize) + if i == ceil(nSmp/BlockSize) + smpIdx = (i-1)*BlockSize+1:nSmp; + dist = EuDist2(fea(smpIdx,:),fea,0); + + if bSpeed + nSmpNow = length(smpIdx); + dump = zeros(nSmpNow,options.k+1); + idx = dump; + for j = 1:options.k+1 + [dump(:,j),idx(:,j)] = min(dist,[],2); + temp = (idx(:,j)-1)*nSmpNow+[1:nSmpNow]'; + dist(temp) = 1e100; + end + else + [dump idx] = sort(dist,2); % sort each row + idx = idx(:,1:options.k+1); + dump = dump(:,1:options.k+1); + end + + if ~bBinary + if bCosine + dist = Normfea(smpIdx,:)*Normfea'; + dist = full(dist); + linidx = [1:size(idx,1)]'; + dump = dist(sub2ind(size(dist),linidx(:,ones(1,size(idx,2))),idx)); + else + dump = exp(-dump/(2*options.t^2)); + end + end + + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),1) = repmat(smpIdx',[options.k+1,1]); + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),2) = idx(:); + if ~bBinary + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),3) = dump(:); + else + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),3) = 1; + end + else + smpIdx = (i-1)*BlockSize+1:i*BlockSize; + + dist = EuDist2(fea(smpIdx,:),fea,0); + + if bSpeed + nSmpNow = length(smpIdx); + dump = zeros(nSmpNow,options.k+1); + idx = dump; + for j = 1:options.k+1 + [dump(:,j),idx(:,j)] = min(dist,[],2); + temp = (idx(:,j)-1)*nSmpNow+[1:nSmpNow]'; + dist(temp) = 1e100; + end + else + [dump idx] = sort(dist,2); % sort each row + idx = idx(:,1:options.k+1); + dump = dump(:,1:options.k+1); + end + + if ~bBinary + if bCosine + dist = Normfea(smpIdx,:)*Normfea'; + dist = full(dist); + linidx = [1:size(idx,1)]'; + dump = dist(sub2ind(size(dist),linidx(:,ones(1,size(idx,2))),idx)); + else + dump = exp(-dump/(2*options.t^2)); + end + end + + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),1) = repmat(smpIdx',[options.k+1,1]); + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),2) = idx(:); + if ~bBinary + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),3) = dump(:); + else + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),3) = 1; + end + end + end + + W = sparse(G(:,1),G(:,2),G(:,3),nSmp,nSmp); + else + G = zeros(nSmp*(options.k+1),3); + for i = 1:ceil(nSmp/BlockSize) + if i == ceil(nSmp/BlockSize) + smpIdx = (i-1)*BlockSize+1:nSmp; + dist = fea(smpIdx,:)*fea'; + dist = full(dist); + + if bSpeed + nSmpNow = length(smpIdx); + dump = zeros(nSmpNow,options.k+1); + idx = dump; + for j = 1:options.k+1 + [dump(:,j),idx(:,j)] = max(dist,[],2); + temp = (idx(:,j)-1)*nSmpNow+[1:nSmpNow]'; + dist(temp) = 0; + end + else + [dump idx] = sort(-dist,2); % sort each row + idx = idx(:,1:options.k+1); + dump = -dump(:,1:options.k+1); + end + + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),1) = repmat(smpIdx',[options.k+1,1]); + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),2) = idx(:); + G((i-1)*BlockSize*(options.k+1)+1:nSmp*(options.k+1),3) = dump(:); + else + smpIdx = (i-1)*BlockSize+1:i*BlockSize; + dist = fea(smpIdx,:)*fea'; + dist = full(dist); + + if bSpeed + nSmpNow = length(smpIdx); + dump = zeros(nSmpNow,options.k+1); + idx = dump; + for j = 1:options.k+1 + [dump(:,j),idx(:,j)] = max(dist,[],2); + temp = (idx(:,j)-1)*nSmpNow+[1:nSmpNow]'; + dist(temp) = 0; + end + else + [dump idx] = sort(-dist,2); % sort each row + idx = idx(:,1:options.k+1); + dump = -dump(:,1:options.k+1); + end + + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),1) = repmat(smpIdx',[options.k+1,1]); + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),2) = idx(:); + G((i-1)*BlockSize*(options.k+1)+1:i*BlockSize*(options.k+1),3) = dump(:); + end + end + + W = sparse(G(:,1),G(:,2),G(:,3),nSmp,nSmp); + end + + if bBinary + W(logical(W)) = 1; + end + + if isfield(options,'bSemiSupervised') && options.bSemiSupervised + tmpgnd = options.gnd(options.semiSplit); + + Label = unique(tmpgnd); + nLabel = length(Label); + G = zeros(sum(options.semiSplit),sum(options.semiSplit)); + for idx=1:nLabel + classIdx = tmpgnd==Label(idx); + G(classIdx,classIdx) = 1; + end + Wsup = sparse(G); + if ~isfield(options,'SameCategoryWeight') + options.SameCategoryWeight = 1; + end + W(options.semiSplit,options.semiSplit) = (Wsup>0)*options.SameCategoryWeight; + end + + if ~options.bSelfConnected + W = W - diag(diag(W)); + end + + if isfield(options,'bTrueKNN') && options.bTrueKNN + + else + W = max(W,W'); + end + + return; +end + + +% strcmpi(options.NeighborMode,'KNN') & (options.k == 0) +% Complete Graph + +switch lower(options.WeightMode) + case {lower('Binary')} + error('Binary weight can not be used for complete graph!'); + case {lower('HeatKernel')} + W = EuDist2(fea,[],0); + W = exp(-W/(2*options.t^2)); + case {lower('Cosine')} + W = full(Normfea*Normfea'); + otherwise + error('WeightMode does not exist!'); +end + +if ~options.bSelfConnected + for i=1:size(W,1) + W(i,i) = 0; + end +end + +W = max(W,W'); + + + + diff --git a/dijkstra.mexglx b/dijkstra.mexglx new file mode 100644 index 0000000..68deb21 Binary files /dev/null and b/dijkstra.mexglx differ diff --git a/dijkstra.mexw32 b/dijkstra.mexw32 new file mode 100644 index 0000000..c6884a7 Binary files /dev/null and b/dijkstra.mexw32 differ diff --git a/dijkstra.mexw64 b/dijkstra.mexw64 new file mode 100644 index 0000000..269ea6e Binary files /dev/null and b/dijkstra.mexw64 differ diff --git a/ep1R.mexw32 b/ep1R.mexw32 new file mode 100644 index 0000000..d7b69db Binary files /dev/null and b/ep1R.mexw32 differ diff --git a/ep1R.mexw64 b/ep1R.mexw64 new file mode 100644 index 0000000..a7e02a1 Binary files /dev/null and b/ep1R.mexw64 differ diff --git a/getargs.m b/getargs.m new file mode 100644 index 0000000..65a0323 --- /dev/null +++ b/getargs.m @@ -0,0 +1,85 @@ +function [eid,emsg,varargout]=getargs(pnames,dflts,varargin) +%GETARGS Process parameter name/value pairs +% [EID,EMSG,A,B,...]=GETARGS(PNAMES,DFLTS,'NAME1',VAL1,'NAME2',VAL2,...) +% accepts a cell array PNAMES of valid parameter names, a cell array +% DFLTS of default values for the parameters named in PNAMES, and +% additional parameter name/value pairs. Returns parameter values A,B,... +% in the same order as the names in PNAMES. Outputs corresponding to +% entries in PNAMES that are not specified in the name/value pairs are +% set to the corresponding value from DFLTS. If nargout is equal to +% length(PNAMES)+1, then unrecognized name/value pairs are an error. If +% nargout is equal to length(PNAMES)+2, then all unrecognized name/value +% pairs are returned in a single cell array following any other outputs. +% +% EID and EMSG are empty if the arguments are valid. If an error occurs, +% EMSG is the text of an error message and EID is the final component +% of an error message id. GETARGS does not actually throw any errors, +% but rather returns EID and EMSG so that the caller may throw the error. +% Outputs will be partially processed after an error occurs. +% +% This utility can be used for processing name/value pair arguments. +% +% Example: +% pnames = {'color' 'linestyle', 'linewidth'} +% dflts = { 'r' '_' '1'} +% varargin = {{'linew' 2 'nonesuch' [1 2 3] 'linestyle' ':'} +% [eid,emsg,c,ls,lw] = statgetargs(pnames,dflts,varargin{:}) % error +% [eid,emsg,c,ls,lw,ur] = statgetargs(pnames,dflts,varargin{:}) % ok + +% We always create (nparams+2) outputs: +% one each for emsg and eid +% nparams varargs for values corresponding to names in pnames +% If they ask for one more (nargout == nparams+3), it's for unrecognized +% names/values + +% Original Copyright 1993-2008 The MathWorks, Inc. +% Modified by Deng Cai (dengcai@gmail.com) 2011.11.27 + + + + +% Initialize some variables +emsg = ''; +eid = ''; +nparams = length(pnames); +varargout = dflts; +unrecog = {}; +nargs = length(varargin); + +% Must have name/value pairs +if mod(nargs,2)~=0 + eid = 'WrongNumberArgs'; + emsg = 'Wrong number of arguments.'; +else + % Process name/value pairs + for j=1:2:nargs + pname = varargin{j}; + if ~ischar(pname) + eid = 'BadParamName'; + emsg = 'Parameter name must be text.'; + break; + end + i = strcmpi(pname,pnames); + i = find(i); + if isempty(i) + % if they've asked to get back unrecognized names/values, add this + % one to the list + if nargout > nparams+2 + unrecog((end+1):(end+2)) = {varargin{j} varargin{j+1}}; + % otherwise, it's an error + else + eid = 'BadParamName'; + emsg = sprintf('Invalid parameter name: %s.',pname); + break; + end + elseif length(i)>1 + eid = 'BadParamName'; + emsg = sprintf('Ambiguous parameter name: %s.',pname); + break; + else + varargout{i} = varargin{j+1}; + end + end +end + +varargout{nparams+1} = unrecog; diff --git a/hungarian.m b/hungarian.m new file mode 100644 index 0000000..160b1d2 --- /dev/null +++ b/hungarian.m @@ -0,0 +1,465 @@ +function [C,T]=hungarian(A) +%HUNGARIAN Solve the Assignment problem using the Hungarian method. +% +%[C,T]=hungarian(A) +%A - a square cost matrix. +%C - the optimal assignment. +%T - the cost of the optimal assignment. +%s.t. T = trace(A(C,:)) is minimized over all possible assignments. + +% Adapted from the FORTRAN IV code in Carpaneto and Toth, "Algorithm 548: +% Solution of the assignment problem [H]", ACM Transactions on +% Mathematical Software, 6(1):104-111, 1980. + +% v1.0 96-06-14. Niclas Borlin, niclas@cs.umu.se. +% Department of Computing Science, Umeĺ University, +% Sweden. +% All standard disclaimers apply. + +% A substantial effort was put into this code. If you use it for a +% publication or otherwise, please include an acknowledgement or at least +% notify me by email. /Niclas + +[m,n]=size(A); + +if (m~=n) + error('HUNGARIAN: Cost matrix must be square!'); +end + +% Save original cost matrix. +orig=A; + +% Reduce matrix. +A=hminired(A); + +% Do an initial assignment. +[A,C,U]=hminiass(A); + +% Repeat while we have unassigned rows. +while (U(n+1)) + % Start with no path, no unchecked zeros, and no unexplored rows. + LR=zeros(1,n); + LC=zeros(1,n); + CH=zeros(1,n); + RH=[zeros(1,n) -1]; + + % No labelled columns. + SLC=[]; + + % Start path in first unassigned row. + r=U(n+1); + % Mark row with end-of-path label. + LR(r)=-1; + % Insert row first in labelled row set. + SLR=r; + + % Repeat until we manage to find an assignable zero. + while (1) + % If there are free zeros in row r + if (A(r,n+1)~=0) + % ...get column of first free zero. + l=-A(r,n+1); + + % If there are more free zeros in row r and row r in not + % yet marked as unexplored.. + if (A(r,l)~=0 & RH(r)==0) + % Insert row r first in unexplored list. + RH(r)=RH(n+1); + RH(n+1)=r; + + % Mark in which column the next unexplored zero in this row + % is. + CH(r)=-A(r,l); + end + else + % If all rows are explored.. + if (RH(n+1)<=0) + % Reduce matrix. + [A,CH,RH]=hmreduce(A,CH,RH,LC,LR,SLC,SLR); + end + + % Re-start with first unexplored row. + r=RH(n+1); + % Get column of next free zero in row r. + l=CH(r); + % Advance "column of next free zero". + CH(r)=-A(r,l); + % If this zero is last in the list.. + if (A(r,l)==0) + % ...remove row r from unexplored list. + RH(n+1)=RH(r); + RH(r)=0; + end + end + + % While the column l is labelled, i.e. in path. + while (LC(l)~=0) + % If row r is explored.. + if (RH(r)==0) + % If all rows are explored.. + if (RH(n+1)<=0) + % Reduce cost matrix. + [A,CH,RH]=hmreduce(A,CH,RH,LC,LR,SLC,SLR); + end + + % Re-start with first unexplored row. + r=RH(n+1); + end + + % Get column of next free zero in row r. + l=CH(r); + + % Advance "column of next free zero". + CH(r)=-A(r,l); + + % If this zero is last in list.. + if(A(r,l)==0) + % ...remove row r from unexplored list. + RH(n+1)=RH(r); + RH(r)=0; + end + end + + % If the column found is unassigned.. + if (C(l)==0) + % Flip all zeros along the path in LR,LC. + [A,C,U]=hmflip(A,C,LC,LR,U,l,r); + % ...and exit to continue with next unassigned row. + break; + else + % ...else add zero to path. + + % Label column l with row r. + LC(l)=r; + + % Add l to the set of labelled columns. + SLC=[SLC l]; + + % Continue with the row assigned to column l. + r=C(l); + + % Label row r with column l. + LR(r)=l; + + % Add r to the set of labelled rows. + SLR=[SLR r]; + end + end +end + +% Calculate the total cost. +T=sum(orig(logical(sparse(C,1:size(orig,2),1)))); + + +function A=hminired(A) +%HMINIRED Initial reduction of cost matrix for the Hungarian method. +% +%B=assredin(A) +%A - the unreduced cost matris. +%B - the reduced cost matrix with linked zeros in each row. + +% v1.0 96-06-13. Niclas Borlin, niclas@cs.umu.se. + +[m,n]=size(A); + +% Subtract column-minimum values from each column. +colMin=min(A); +A=A-colMin(ones(n,1),:); + +% Subtract row-minimum values from each row. +rowMin=min(A')'; +A=A-rowMin(:,ones(1,n)); + +% Get positions of all zeros. +[i,j]=find(A==0); + +% Extend A to give room for row zero list header column. +A(1,n+1)=0; +for k=1:n + % Get all column in this row. + cols=j(k==i)'; + % Insert pointers in matrix. + A(k,[n+1 cols])=[-cols 0]; +end + + +function [A,C,U]=hminiass(A) +%HMINIASS Initial assignment of the Hungarian method. +% +%[B,C,U]=hminiass(A) +%A - the reduced cost matrix. +%B - the reduced cost matrix, with assigned zeros removed from lists. +%C - a vector. C(J)=I means row I is assigned to column J, +% i.e. there is an assigned zero in position I,J. +%U - a vector with a linked list of unassigned rows. + +% v1.0 96-06-14. Niclas Borlin, niclas@cs.umu.se. + +[n,np1]=size(A); + +% Initalize return vectors. +C=zeros(1,n); +U=zeros(1,n+1); + +% Initialize last/next zero "pointers". +LZ=zeros(1,n); +NZ=zeros(1,n); + +for i=1:n + % Set j to first unassigned zero in row i. + lj=n+1; + j=-A(i,lj); + + % Repeat until we have no more zeros (j==0) or we find a zero + % in an unassigned column (c(j)==0). + + while (C(j)~=0) + % Advance lj and j in zero list. + lj=j; + j=-A(i,lj); + + % Stop if we hit end of list. + if (j==0) + break; + end + end + + if (j~=0) + % We found a zero in an unassigned column. + + % Assign row i to column j. + C(j)=i; + + % Remove A(i,j) from unassigned zero list. + A(i,lj)=A(i,j); + + % Update next/last unassigned zero pointers. + NZ(i)=-A(i,j); + LZ(i)=lj; + + % Indicate A(i,j) is an assigned zero. + A(i,j)=0; + else + % We found no zero in an unassigned column. + + % Check all zeros in this row. + + lj=n+1; + j=-A(i,lj); + + % Check all zeros in this row for a suitable zero in another row. + while (j~=0) + % Check the in the row assigned to this column. + r=C(j); + + % Pick up last/next pointers. + lm=LZ(r); + m=NZ(r); + + % Check all unchecked zeros in free list of this row. + while (m~=0) + % Stop if we find an unassigned column. + if (C(m)==0) + break; + end + + % Advance one step in list. + lm=m; + m=-A(r,lm); + end + + if (m==0) + % We failed on row r. Continue with next zero on row i. + lj=j; + j=-A(i,lj); + else + % We found a zero in an unassigned column. + + % Replace zero at (r,m) in unassigned list with zero at (r,j) + A(r,lm)=-j; + A(r,j)=A(r,m); + + % Update last/next pointers in row r. + NZ(r)=-A(r,m); + LZ(r)=j; + + % Mark A(r,m) as an assigned zero in the matrix . . . + A(r,m)=0; + + % ...and in the assignment vector. + C(m)=r; + + % Remove A(i,j) from unassigned list. + A(i,lj)=A(i,j); + + % Update last/next pointers in row r. + NZ(i)=-A(i,j); + LZ(i)=lj; + + % Mark A(r,m) as an assigned zero in the matrix . . . + A(i,j)=0; + + % ...and in the assignment vector. + C(j)=i; + + % Stop search. + break; + end + end + end +end + +% Create vector with list of unassigned rows. + +% Mark all rows have assignment. +r=zeros(1,n); +rows=C(C~=0); +r(rows)=rows; +empty=find(r==0); + +% Create vector with linked list of unassigned rows. +U=zeros(1,n+1); +U([n+1 empty])=[empty 0]; + + +function [A,C,U]=hmflip(A,C,LC,LR,U,l,r) +%HMFLIP Flip assignment state of all zeros along a path. +% +%[A,C,U]=hmflip(A,C,LC,LR,U,l,r) +%Input: +%A - the cost matrix. +%C - the assignment vector. +%LC - the column label vector. +%LR - the row label vector. +%U - the +%r,l - position of last zero in path. +%Output: +%A - updated cost matrix. +%C - updated assignment vector. +%U - updated unassigned row list vector. + +% v1.0 96-06-14. Niclas Borlin, niclas@cs.umu.se. + +n=size(A,1); + +while (1) + % Move assignment in column l to row r. + C(l)=r; + + % Find zero to be removed from zero list.. + + % Find zero before this. + m=find(A(r,:)==-l); + + % Link past this zero. + A(r,m)=A(r,l); + + A(r,l)=0; + + % If this was the first zero of the path.. + if (LR(r)<0) + ...remove row from unassigned row list and return. + U(n+1)=U(r); + U(r)=0; + return; + else + + % Move back in this row along the path and get column of next zero. + l=LR(r); + + % Insert zero at (r,l) first in zero list. + A(r,l)=A(r,n+1); + A(r,n+1)=-l; + + % Continue back along the column to get row of next zero in path. + r=LC(l); + end +end + + +function [A,CH,RH]=hmreduce(A,CH,RH,LC,LR,SLC,SLR) +%HMREDUCE Reduce parts of cost matrix in the Hungerian method. +% +%[A,CH,RH]=hmreduce(A,CH,RH,LC,LR,SLC,SLR) +%Input: +%A - Cost matrix. +%CH - vector of column of 'next zeros' in each row. +%RH - vector with list of unexplored rows. +%LC - column labels. +%RC - row labels. +%SLC - set of column labels. +%SLR - set of row labels. +% +%Output: +%A - Reduced cost matrix. +%CH - Updated vector of 'next zeros' in each row. +%RH - Updated vector of unexplored rows. + +% v1.0 96-06-14. Niclas Borlin, niclas@cs.umu.se. + +n=size(A,1); + +% Find which rows are covered, i.e. unlabelled. +coveredRows=LR==0; + +% Find which columns are covered, i.e. labelled. +coveredCols=LC~=0; + +r=find(~coveredRows); +c=find(~coveredCols); + +% Get minimum of uncovered elements. +m=min(min(A(r,c))); + +% Subtract minimum from all uncovered elements. +A(r,c)=A(r,c)-m; + +% Check all uncovered columns.. +for j=c + % ...and uncovered rows in path order.. + for i=SLR + % If this is a (new) zero.. + if (A(i,j)==0) + % If the row is not in unexplored list.. + if (RH(i)==0) + % ...insert it first in unexplored list. + RH(i)=RH(n+1); + RH(n+1)=i; + % Mark this zero as "next free" in this row. + CH(i)=j; + end + % Find last unassigned zero on row I. + row=A(i,:); + colsInList=-row(row<0); + if (length(colsInList)==0) + % No zeros in the list. + l=n+1; + else + l=colsInList(row(colsInList)==0); + end + % Append this zero to end of list. + A(i,l)=-j; + end + end +end + +% Add minimum to all doubly covered elements. +r=find(coveredRows); +c=find(coveredCols); + +% Take care of the zeros we will remove. +[i,j]=find(A(r,c)<=0); + +i=r(i); +j=c(j); + +for k=1:length(i) + % Find zero before this in this row. + lj=find(A(i(k),:)==-j(k)); + % Link past it. + A(i(k),lj)=A(i(k),j(k)); + % Mark it as assigned. + A(i(k),j(k))=0; +end + +A(r,c)=A(r,c)+m; diff --git a/initFactor.m b/initFactor.m new file mode 100644 index 0000000..f8b555b --- /dev/null +++ b/initFactor.m @@ -0,0 +1,84 @@ +function ratio=initFactor(x_norm, Ax , y, z, funName, rsL2, x_2norm) +% +%% function initFactor +% compute the an optimal constant factor for the initialization +% +% +% Input parameters: +% x_norm- the norm of the starting point +% Ax- A*x, with x being the initialization point +% y- the response matrix +% z- the regularization parameter or the ball +% funName- the name of the function +% +% Output parameter: +% ratio- the computed optimal initialization point is ratio*x +% +%% Copyright (C) 2009-2010 Jun Liu, and Jieping Ye +% +% For any problem, please contact with Jun Liu via j.liu@asu.edu +% +% Last revised on August 2, 2009. + +switch(funName) + case 'LeastC' + ratio_max = z / x_norm; + ratio_optimal = Ax'*y / (Ax'*Ax + rsL2 * x_2norm); + + if abs(ratio_optimal)<=ratio_max + ratio = ratio_optimal; + elseif ratio_optimal<0 + ratio = -ratio_max; + else + ratio = ratio_max; + end + % fprintf('\n ratio=%e,%e,%e',ratio,ratio_optimal,ratio_max); + + case 'LeastR' + ratio= (Ax'*y - z * x_norm) / (Ax'*Ax + rsL2 * x_2norm); + %fprintf('\n ratio=%e',ratio); + + case 'glLeastR' + ratio= (Ax'*y - z * x_norm) / (Ax'*Ax); + %fprintf('\n ratio=%e',ratio); + + case 'mcLeastR' + ratio= (Ax(:)'*y(:) - z * x_norm) / norm(Ax,'fro')^2; + %fprintf('\n ratio=%e',ratio); + + case 'mtLeastR' + ratio= (Ax'*y - z * x_norm) / (Ax'*Ax); + %fprintf('\n ratio=%e',ratio); + + case 'nnLeastR' + ratio= (Ax'*y - z * x_norm) / (Ax'*Ax + rsL2 * x_2norm); + ratio=max(0,ratio); + + case 'nnLeastC' + ratio_max = z / x_norm; + ratio_optimal = Ax'*y / (Ax'*Ax + rsL2 * x_2norm); + + if ratio_optimal<0 + ratio=0; + elseif ratio_optimal<=ratio_max + ratio = ratio_optimal; + else + ratio = ratio_max; + end + % fprintf('\n ratio=%e,%e,%e',ratio,ratio_optimal,ratio_max); + + case 'mcLeastC' + ratio_max = z / x_norm; + ratio_optimal = Ax(:)'*y(:) / (norm(Ax'*Ax,'fro')^2); + + if abs(ratio_optimal)<=ratio_max + ratio = ratio_optimal; + elseif ratio_optimal<0 + ratio = -ratio_max; + else + ratio = ratio_max; + end + + otherwise + fprintf('\n The specified funName is not supprted'); +end \ No newline at end of file diff --git a/lars.m b/lars.m new file mode 100644 index 0000000..6fb52dc --- /dev/null +++ b/lars.m @@ -0,0 +1,329 @@ +function beta = lars(X, y, method, stop, useGram, Gram, Cardi, bSparse, trace) +% This function is provided at +% http://www2.imm.dtu.dk/pubdb/views/publication_details.php?id=3897 +% I have made some small modifications -- Deng Cai, Feb/2008 + +% LARS The LARS algorithm for performing LAR or LASSO. +% BETA = LARS(X, Y) performs least angle regression on the variables in +% X to approximate the response Y. Variables X are assumed to be +% normalized (zero mean, unit length), the response Y is assumed to be +% centered. +% BETA = LARS(X, Y, METHOD), where METHOD is either 'LARS' or 'LASSO' +% determines whether least angle regression or lasso regression should +% be performed. +% BETA = LARS(X, Y, METHOD, STOP) with nonzero STOP will perform least +% angle or lasso regression with early stopping. If STOP is negative, +% STOP is an integer that determines the desired number of variables. If +% STOP is positive, it corresponds to an upper bound on the L1-norm of +% the BETA coefficients. +% BETA = LARS(X, Y, METHOD, STOP, USEGRAM) specifies whether the Gram +% matrix X'X should be calculated (USEGRAM = 1) or not (USEGRAM = 0). +% Calculation of the Gram matrix is suitable for low-dimensional +% problems. By default, the Gram matrix is calculated. +% BETA = LARS(X, Y, METHOD, STOP, USEGRAM, GRAM) makes it possible to +% supply a pre-computed Gram matrix. Set USEGRAM to 1 to enable. If no +% Gram matrix is available, exclude argument or set GRAM = []. +% BETA = LARS(X, Y, METHOD, STOP, USEGRAM, GRAM, TRACE) with nonzero +% TRACE will print the adding and subtracting of variables as all +% LARS/lasso solutions are found. +% Returns BETA where each row contains the predictor coefficients of +% one iteration. A suitable row is chosen using e.g. cross-validation, +% possibly including interpolation to achieve sub-iteration accuracy. +% +% Author: Karl Skoglund, IMM, DTU, kas@imm.dtu.dk +% Reference: 'Least Angle Regression' by Bradley Efron et al, 2003. + +%% Input checking +% Set default values. +if nargin < 9 + trace = 0; +end +if nargin < 8 + bSparse = 1; +end +if nargin < 7 + Cardi = []; +end +if nargin < 6 + Gram = []; +end +if nargin < 5 + useGram = 0; +end +if nargin < 4 + stop = 0; +end +if nargin < 3 + method = 'lasso'; +end +if strcmpi(method, 'lasso') + lasso = 1; +else + lasso = 0; +end + +if isempty(X) + error('The code has been updated. Please input the X'); +end + + +%% LARS variable setup +[n p] = size(X); +% nvars = min(n-1,p); % +nvars = p; % + +maxk = 512*nvars; % Maximum number of iterations + +if isempty(Cardi) + if stop == 0 + if bSparse + beta = sparse(p,2*nvars); + else + beta = zeros(p,2*nvars); + end + elseif stop < 0 + if bSparse + beta = sparse(p,2*round(-stop)); + else + beta = zeros(p,2*round(-stop)); + end + else + if bSparse + beta = sparse(p,100); + else + beta = zeros(p,100); + end + end +else + Cardi = unique(Cardi); + Cardi(Cardi>nvars) = []; + stop = -max(Cardi); + if bSparse + beta = sparse(p,length(Cardi)); + else + beta = zeros(p,length(Cardi)); + end + betak = zeros(p,1); +end + +mu = zeros(n, 1); % current "position" as LARS travels towards lsq solution +I = 1:p; % inactive set +A = []; % active set + +% Calculate Gram matrix if necessary +if isempty(Gram) && useGram + error('The code has been updated. Please input the Gram'); +% clear Gram; +% global Gram; + % Gram = X'*X; % Precomputation of the Gram matrix. Fast but memory consuming. +end + +if ~useGram + R = []; % Cholesky factorization R'R = X'X where R is upper triangular +end + + +lassocond = 0; % LASSO condition boolean +stopcond = 0; % Early stopping condition boolean +k = 0; % Iteration count +vars = 0; % Current number of variables + +if trace + disp(sprintf('Step\tAdded\tDropped\t\tActive set size')); +end + +% TimeLoop = zeros(2*nvars,1); +tmpT = cputime; + +%% LARS main loop +while vars < nvars && ~stopcond && k < maxk + k = k + 1; + c = X'*(y - mu); + [C j] = max(abs(c(I))); + j = I(j); + + if ~lassocond % if a variable has been dropped, do one iteration with this configuration (don't add new one right away) + if ~useGram + diag_k = X(:,j)'*X(:,j); % diagonal element k in X'X matrix + if isempty(R) + R = sqrt(diag_k); + else + col_k = X(:,j)'*X(:,A); % elements of column k in X'X matrix + R_k = R'\col_k'; % R'R_k = (X'X)_k, solve for R_k + R_kk = sqrt(diag_k - R_k'*R_k); % norm(x'x) = norm(R'*R), find last element by exclusion + R = [R R_k; [zeros(1,size(R,2)) R_kk]]; % update R + end + end + A = [A j]; + I(I == j) = []; + vars = vars + 1; + if trace + disp(sprintf('%d\t\t%d\t\t\t\t\t%d', k, j, vars)); + end + end + + s = sign(c(A)); % get the signs of the correlations + + if useGram + if vars <= 200 + R = chol(Gram(A,A)); + elseif lassocond + if (rJ <= 200) & vars <= 1000 + R = chol(Gram(A,A)); + else + R(:,rJ) = []; % remove column j + tmpn = size(R,2); + for tmpk = rJ:tmpn + tmpp = tmpk:tmpk+1; + [G,R(tmpp,tmpk)] = planerot(R(tmpp,tmpk)); % remove extra element in column + if tmpk < tmpn + R(tmpp,tmpk+1:tmpn) = G*R(tmpp,tmpk+1:tmpn); % adjust rest of row + end + end + R(end,:) = []; % remove zero'ed out row + end + else + R_k = R'\Gram(A(1:end-1),j); + R_kk = sqrt(Gram(j,j)-R_k'*R_k); + R = [R R_k; [zeros(1,size(R,2)) R_kk]]; % update R + end + GA1 = R\(R'\s); + AA = 1/sqrt(sum(GA1.*s)); + w = AA*GA1; + else + GA1 = R\(R'\s); + AA = 1/sqrt(sum(GA1.*s)); + w = AA*GA1; + end + u = X(:,A)*w; % equiangular direction (unit vector) + + if vars == nvars % if all variables active, go all the way to the lsq solution + gamma = C/AA; + else + a = X'*u; % correlation between each variable and eqiangular vector + temp = [(C - c(I))./(AA - a(I)); (C + c(I))./(AA + a(I))]; + gamma = min([temp(temp > 0); C/AA]); + end + + % LASSO modification + if lasso + lassocond = 0; + if isempty(Cardi) + temp = -beta(A,k)./w; + else + temp = -betak(A)./w; + end + [gamma_tilde] = min([temp(temp > 0); gamma]); + j = find(temp == gamma_tilde); + if gamma_tilde < gamma, + gamma = gamma_tilde; + lassocond = 1; + end + end + + mu = mu + gamma*u; + if isempty(Cardi) + if size(beta,2) < k+1 + if bSparse + beta = [beta sparse(p,size(beta,1))]; + else + beta = [beta zeros(p,size(beta,1))]; + end + end + beta(A,k+1) = beta(A,k) + gamma*w; + else + tmpbetak = betak(A) + gamma*w; + betak = zeros(p,1); + betak(A) = tmpbetak; + idx = find(Cardi==vars); + if ~isempty(idx) + beta(:,idx) = betak; + end + end + + % Early stopping at specified bound on L1 norm of beta + if isempty(Cardi) + if stop > 0 + t2 = sum(abs(beta(:,k+1))); + if t2 >= stop + t1 = sum(abs(beta(:,k))); + s = (stop - t1)/(t2 - t1); % interpolation factor 0 < s < 1 + beta(:,k+1) = beta(:,k) + s*(beta(:,k+1) - beta(:,k)); + stopcond = 1; + end + end + end + + % If LASSO condition satisfied, drop variable from active set + if lassocond == 1 + if ~useGram + R(:,j) = []; % remove column j + tmpn = size(R,2); + for tmpk = j:tmpn + tmpp = tmpk:tmpk+1; + [G,R(tmpp,tmpk)] = planerot(R(tmpp,tmpk)); % remove extra element in column + if tmpk < tmpn + R(tmpp,tmpk+1:tmpn) = G*R(tmpp,tmpk+1:tmpn); % adjust rest of row + end + end + R(end,:) = []; % remove zero'ed out row + end + rJ = j; + I = [I A(j)]; + A(j) = []; + vars = vars - 1; + if trace + disp(sprintf('%d\t\t\t\t%d\t\t\t%d', k, j, vars)); + end + end + + % Early stopping at specified number of variables + if stop < 0 + stopcond = vars >= -stop; + end + +% TimeLoop(k) = cputime - tmpT; +% tmpT = cputime; + +% if vars < 1000 +% if mod(vars,500) == 0 +% tmpT = cputime - tmpT; +% disp(['LARS: ',num2str(vars),' features selected. Time: ',num2str(tmpT)]); +% tmpT = cputime; +% end +% elseif vars < 2000 +% if mod(vars,200) == 0 +% tmpT = cputime - tmpT; +% disp(['LARS: ',num2str(vars),' features selected. Time: ',num2str(tmpT)]); +% tmpT = cputime; +% end +% elseif vars < 3000 +% if mod(vars,100) == 0 +% tmpT = cputime - tmpT; +% disp(['LARS: ',num2str(vars),' features selected. Time: ',num2str(tmpT)]); +% tmpT = cputime; +% end +% else +% if mod(vars,50) == 0 +% tmpT = cputime - tmpT; +% disp(['LARS: ',num2str(vars),' features selected. Time: ',num2str(tmpT)]); +% tmpT = cputime; +% end +% end +end + +if isempty(Cardi) + % trim beta + if size(beta,2) > k+1 + beta(:,k+2:end) = []; + end +end + +if k == maxk + disp('LARS warning: Forced exit. Maximum number of iteration reached.'); +end + +%% To do +% +% There is a modification that turns least angle regression into stagewise +% (epsilon) regression. This has not been implemented. diff --git a/learn_basis.m b/learn_basis.m new file mode 100644 index 0000000..c188837 --- /dev/null +++ b/learn_basis.m @@ -0,0 +1,90 @@ +function B = learn_basis(X, S, l2norm, Binit) +% Learning basis using Lagrange dual (with basis normalization) +% +% This code solves the following problem: +% +% minimize_B 0.5*||X - B*S||^2 +% subject to ||B(:,j)||_2 <= l2norm, forall j=1...size(S,1) +% +% The detail of the algorithm is described in the following paper: +% 'Efficient Sparse Codig Algorithms', Honglak Lee, Alexis Battle, Rajat Raina, Andrew Y. Ng, +% Advances in Neural Information Processing Systems (NIPS) 19, 2007 +% +% Written by Honglak Lee +% Copyright 2007 by Honglak Lee, Alexis Battle, Rajat Raina, and Andrew Y. Ng + +L = size(X,1); +N = size(X,2); +M = size(S, 1); + +% tic +SSt = S*S'; +XSt = X*S'; + +if exist('Binit', 'var') + dual_lambda = diag(Binit\XSt - SSt); +else + dual_lambda = 10*abs(rand(M,1)); % any arbitrary initialization should be ok. +end + +c = l2norm^2; +trXXt = sum(sum(X.^2)); + +lb=zeros(size(dual_lambda)); +options = optimset('GradObj','on', 'Hessian','on','Display','off'); +% options = optimset('GradObj','on', 'Hessian','on', 'TolFun', 1e-7); + +[x, fval, exitflag, output] = fmincon(@(x) fobj_basis_dual(x, SSt, XSt, X, c, trXXt), dual_lambda, [], [], [], [], lb, [], [], options); +% output.iterations +fval_opt = -0.5*N*fval; +dual_lambda= x; + +Bt = (SSt+diag(dual_lambda)) \ XSt'; +B_dual= Bt'; +fobjective_dual = fval_opt; + + +B= B_dual; +% fobjective = fobjective_dual; +% toc + +return; + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +function [f,g,H] = fobj_basis_dual(dual_lambda, SSt, XSt, X, c, trXXt) +% Compute the objective function value at x +L= size(XSt,1); +M= length(dual_lambda); + +SSt_inv = inv(SSt + diag(dual_lambda)); + +% trXXt = sum(sum(X.^2)); +if L>M + % (M*M)*((M*L)*(L*M)) => MLM + MMM = O(M^2(M+L)) + f = -trace(SSt_inv*(XSt'*XSt))+trXXt-c*sum(dual_lambda); + +else + % (L*M)*(M*M)*(M*L) => LMM + LML = O(LM(M+L)) + f = -trace(XSt*SSt_inv*XSt')+trXXt-c*sum(dual_lambda); +end +f= -f; + +if nargout > 1 % fun called with two output arguments + % Gradient of the function evaluated at x + g = zeros(M,1); + temp = XSt*SSt_inv; + g = sum(temp.^2) - c; + g= -g; + + + if nargout > 2 + % Hessian evaluated at x + % H = -2.*((SSt_inv*XSt'*XSt*SSt_inv).*SSt_inv); + H = -2.*((temp'*temp).*SSt_inv); + H = -H; + end +end + +return \ No newline at end of file diff --git a/learn_coefficients.m b/learn_coefficients.m new file mode 100644 index 0000000..8cd3feb --- /dev/null +++ b/learn_coefficients.m @@ -0,0 +1,269 @@ +function Sout = learn_coefficients(B, X, alpha, gamma, L, Sinit) +% The feature-sign search algorithm +% L1-regularized least squares problem solver +% +% This code solves the following problem: +% +% minimize_s 0.5*(||xi-B*si||^2 + alpha*si'si + 2*alpha*si(sigma(Lijsj)) + +% gamma*||si||_1 +% X : data matrix +% B : basis matrix +% Sinit : initial coefficient matrix +% +% References: +% [1] Miao Zheng, Jiajun Bu, Chun Chen, Can Wang, Lijun Zhang, Guang Qiu, Deng Cai. +% "Graph Regularized Sparse Coding for Image Representation", +% IEEE Transactions on Image Processing, Vol. 20, No. 5, pp. 1327-1336, 2011. +% +% Version1.0 -- Nov/2009 +% Version2.0 -- Jan/2012 +% Written by Miao Zheng +% +warning('off', 'MATLAB:divideByZero'); + +use_Sinit= false; +if exist('Sinit', 'var') + use_Sinit= true; +end + +Sout= zeros(size(B,2), size(X,2)); +BtB = B'*B; +BtX = B'*X; + +rankB = rank(BtB); +% rankB = min(size(B,1)-10, size(B,2)-10); + +for i=1:size(X,2) +% if mod(i, 100)==0, fprintf('.'); end %fprintf(1, 'l1ls_featuresign: %d/%d\r', i, size(X,2)); end + if use_Sinit + idx1 = find(Sinit(:,i)~=0); + maxn = min(length(idx1), rankB); + sinit = zeros(size(Sinit(:,i))); + sinit(idx1(1:maxn)) = Sinit(idx1(1:maxn), i); + a = sum(sum(Sinit,2)==0); + S = [Sout(:,1:i),Sinit(:,(i+1):size(X,2))]; + [Sout(:,i), fobj]= ls_featuresign_sub (B, S, X(:,i), BtB, BtX(:,i), L, i, alpha, gamma, sinit); + else + [Sout(:,i), fobj]= ls_featuresign_sub (B, Sout, X(:,i), BtB, BtX(:,i), L, i, alpha, gamma); + end +end + +% fprintf(1, '\n'); + +warning('on', 'MATLAB:divideByZero'); + +return; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +function [s, fobj] = ls_featuresign_sub (B, S, x, BtB, Btx, L, i, alpha, gamma, sinit) + +[N,M] = size(B); + +rankB = rank(BtB); +% rankB = min(size(B,1)-10, size(B,2)-10); + +% Step 1: Initialize +usesinit = false; +if ~exist('sinit', 'var') || isempty(sinit) + sinit= []; + s= sparse(zeros(M,1)); + theta= sparse(zeros(M,1)); + act= sparse(zeros(M,1)); + allowZero = false; +else + % sinit = []; + s= sparse(sinit); + theta= sparse(sign(s)); + act= sparse(abs(theta)); + usesinit = true; + allowZero = true; +end + + +L_new = L(:,i); +L_new(i) = 0; +P = S*L_new; + +fobj = fobj_featuresign(s, B, x, BtB, Btx, P, L, i, alpha, gamma); + +ITERMAX=1000; +optimality1=false; +for iter=1:ITERMAX + % check optimality0 + act_indx0 = find(act == 0); + + grad = BtB*sparse(s)+alpha*L(i,i)*sparse(s)+alpha*P - Btx; + theta = sign(s); + + optimality0= false; + % Step 2 + [mx,indx] = max (abs(grad(act_indx0))); + + if ~isempty(mx) && (mx >= gamma) && (iter>1 || ~usesinit) + act(act_indx0(indx)) = 1; + theta(act_indx0(indx)) = -sign(grad(act_indx0(indx))); + usesinit= false; + else + optimality0= true; + if optimality1 + break; + end + end + act_indx1 = find(act == 1); + + if length(act_indx1)>rankB + warning('sparsity penalty is too small: too many coefficients are activated'); + return; + end + + if isempty(act_indx1) %length(act_indx1)==0 + % if ~assert(max(abs(s))==0), save(fname_debug, 'B', 'x', 'gamma', 'sinit'); error('error'); end + if allowZero, allowZero= false; continue, end + return; + end + + % if ~assert(length(act_indx1) == length(find(act==1))), save(fname_debug, 'B', 'x', 'gamma', 'sinit'); error('error'); end + k=0; + while 1 + k=k+1; + + if k>ITERMAX + warning('Maximum number of iteration reached. The solution may not be optimal'); + % save(fname_debug, 'B', 'x', 'gamma', 'sinit'); + return; + end + + if isempty(act_indx1) % length(act_indx1)==0 + % if ~assert(max(abs(s))==0), save(fname_debug, 'B', 'x', 'gamma', 'sinit'); error('error'); end + if allowZero, allowZero= false; break, end + return; + end + + % Step 3: feature-sign step + [s, theta, act, act_indx1, optimality1, lsearch, fobj] = compute_FS_step (s, B, x, BtB, Btx, P, L, i, theta, act, act_indx1, alpha, gamma); + + % Step 4: check optimality condition 1 + if optimality1 break; end; + if lsearch >0 continue; end; + + end +end + +if iter >= ITERMAX + warning('Maximum number of iteration reached. The solution may not be optimal'); + % save(fname_debug, 'B', 'x', 'gamma', 'sinit'); +end + +if 0 % check if optimality + act_indx1 = find(act==1); + grad = BtB*sparse(s)+alpha*L(i,i)*sparse(s)+alpha*P - Btx; + norm(grad(act_indx1) + gamma.*sign(s(act_indx1)),'inf') + find(abs(grad(setdiff(1:M, act_indx1)))>gamma) +end + +fobj = fobj_featuresign(s, B, x, BtB, Btx, P, L, i, alpha, gamma); + +return; + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +function [s, theta, act, act_indx1, optimality1, lsearch, fobj] = compute_FS_step (s, B, x, BtB, Btx, P, L, idx, theta, act, act_indx1, alpha, gamma) + +s2 = s(act_indx1); +% B2 = B(:, act_indx1); +BtB2 = BtB(act_indx1, act_indx1); +theta2 = theta(act_indx1); + + +a = L(idx,idx)*alpha*ones(size(act_indx1)); +%a = L(idx,idx)*alpha*eye(size(act_indx1,1)); +% call matlab optimization solver.. +s_new = (BtB2+diag(a)) \ ( Btx(act_indx1) - gamma.*theta2 -alpha*P(act_indx1)); % RR + +% opts.POSDEF=true; opts.SYM=true; % RR +% s_new = linsolve(BtB2, ( Btx(act_indx1) - gamma.*theta2 ), opts); % RR +optimality1= false; +if (sign(s_new) == sign(s2)) + optimality1= true; + s(act_indx1) = s_new; + fobj = 0; + %fobj = fobj_featuresign(s, B, x, BtB, Btx, P, L, idx, alpha, gamma); + lsearch = 1; + return; +end + +% do line search: s -> s_new +progress = (0 - s2)./(s_new - s2); +lsearch=0; + +a= 0.5*sum((B(:,act_indx1)*(s_new-s2)).^2)+0.5*alpha*L(idx,idx)*(s_new-s2)'*(s_new-s2); +b= (s_new-s2)'*(alpha*P(act_indx1)-Btx(act_indx1)) + (s2'*BtB2+ alpha*L(idx,idx)*s2')*(s_new-s2); + +fobj_lsearch = gamma*sum(abs(s2)); +[sort_lsearch, ix_lsearch] = sort([progress',1]); +remove_idx=[]; +for i = 1:length(sort_lsearch) + t = sort_lsearch(i); if t<=0 || t>1 continue; end + s_temp= s2+ (s_new- s2).*t; + fobj_temp = a*t^2 + b*t + gamma*sum(abs(s_temp)); + if fobj_temp < fobj_lsearch + fobj_lsearch = fobj_temp; + lsearch = t; + if t<1 remove_idx = [remove_idx ix_lsearch(i)]; end % remove_idx can be more than two.. + elseif fobj_temp > fobj_lsearch + break; + else + if (sum(s2==0)) == 0 + lsearch = t; + fobj_lsearch = fobj_temp; + if t<1 remove_idx = [remove_idx ix_lsearch(i)]; end % remove_idx can be more than two.. + end + end +end + +% if ~assert(lsearch >=0 && lsearch <=1), save(fname_debug, 'B', 'x', 'gamma', 'sinit'); error('error'); end + +if lsearch >0 + % update s + s_new = s2 + (s_new - s2).*lsearch; + s(act_indx1) = s_new; + theta(act_indx1) = sign(s_new); % this is not clear... +end + +% if s encounters zero along the line search, then remove it from +% active set +if lsearch<1 && lsearch>0 + %remove_idx = find(s(act_indx1)==0); + remove_idx = find(abs(s(act_indx1)) < eps); + s(act_indx1(remove_idx))=0; + + theta(act_indx1(remove_idx))=0; + act(act_indx1(remove_idx))=0; + act_indx1(remove_idx)=[]; +end +%fobj_new = 0; +fobj_new = fobj_featuresign(s, B, x, BtB, Btx, P, L, idx, alpha, gamma); + +fobj = fobj_new; + +return; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +function [f, g] = fobj_featuresign(s, B, x, BtB, Btx, P, L, i, alpha, gamma) + +f= 0.5*(norm(x-B*s)^2+alpha*L(i,i).*(s'*(s+2*P))); +f= f+ gamma*norm(s,1); + +if nargout >1 + g = BtB*s+alpha*L(i,i)*s+alpha*P - Btx; + g= g+ gamma*sign(s); +end + +return; + +%%%%%%%%%%%%%%%%%%%%% \ No newline at end of file diff --git a/litekmeans.m b/litekmeans.m new file mode 100644 index 0000000..fa25784 --- /dev/null +++ b/litekmeans.m @@ -0,0 +1,457 @@ +function [label, center, bCon, sumD, D] = litekmeans(X, k, varargin) +%LITEKMEANS K-means clustering, accelerated by matlab matrix operations. +% +% label = LITEKMEANS(X, K) partitions the points in the N-by-P data matrix +% X into K clusters. This partition minimizes the sum, over all +% clusters, of the within-cluster sums of point-to-cluster-centroid +% distances. Rows of X correspond to points, columns correspond to +% variables. KMEANS returns an N-by-1 vector label containing the +% cluster indices of each point. +% +% [label, center] = LITEKMEANS(X, K) returns the K cluster centroid +% locations in the K-by-P matrix center. +% +% [label, center, bCon] = LITEKMEANS(X, K) returns the bool value bCon to +% indicate whether the iteration is converged. +% +% [label, center, bCon, SUMD] = LITEKMEANS(X, K) returns the +% within-cluster sums of point-to-centroid distances in the 1-by-K vector +% sumD. +% +% [label, center, bCon, SUMD, D] = LITEKMEANS(X, K) returns +% distances from each point to every centroid in the N-by-K matrix D. +% +% [ ... ] = LITEKMEANS(..., 'PARAM1',val1, 'PARAM2',val2, ...) specifies +% optional parameter name/value pairs to control the iterative algorithm +% used by KMEANS. Parameters are: +% +% 'Distance' - Distance measure, in P-dimensional space, that KMEANS +% should minimize with respect to. Choices are: +% {'sqEuclidean'} - Squared Euclidean distance (the default) +% 'cosine' - One minus the cosine of the included angle +% between points (treated as vectors). Each +% row of X SHOULD be normalized to unit. If +% the intial center matrix is provided, it +% SHOULD also be normalized. +% +% 'Start' - Method used to choose initial cluster centroid positions, +% sometimes known as "seeds". Choices are: +% {'sample'} - Select K observations from X at random (the default) +% 'cluster' - Perform preliminary clustering phase on random 10% +% subsample of X. This preliminary phase is itself +% initialized using 'sample'. An additional parameter +% clusterMaxIter can be used to control the maximum +% number of iterations in each preliminary clustering +% problem. +% matrix - A K-by-P matrix of starting locations; or a K-by-1 +% indicate vector indicating which K points in X +% should be used as the initial center. In this case, +% you can pass in [] for K, and KMEANS infers K from +% the first dimension of the matrix. +% +% 'MaxIter' - Maximum number of iterations allowed. Default is 100. +% +% 'Replicates' - Number of times to repeat the clustering, each with a +% new set of initial centroids. Default is 1. If the +% initial centroids are provided, the replicate will be +% automatically set to be 1. +% +% 'clusterMaxIter' - Only useful when 'Start' is 'cluster'. Maximum number +% of iterations of the preliminary clustering phase. +% Default is 10. +% +% +% Examples: +% +% fea = rand(500,10); +% [label, center] = litekmeans(fea, 5, 'MaxIter', 50); +% +% fea = rand(500,10); +% [label, center] = litekmeans(fea, 5, 'MaxIter', 50, 'Replicates', 10); +% +% fea = rand(500,10); +% [label, center, bCon, sumD, D] = litekmeans(fea, 5, 'MaxIter', 50); +% TSD = sum(sumD); +% +% fea = rand(500,10); +% initcenter = rand(5,10); +% [label, center] = litekmeans(fea, 5, 'MaxIter', 50, 'Start', initcenter); +% +% fea = rand(500,10); +% idx=randperm(500); +% [label, center] = litekmeans(fea, 5, 'MaxIter', 50, 'Start', idx(1:5)); +% +% +% See also KMEANS +% +% [Cite] Deng Cai, "Litekmeans: the fastest matlab implementation of +% kmeans," Available at: +% http://www.zjucadcg.cn/dengcai/Data/Clustering.html, 2011. +% +% version 2.0 --December/2011 +% version 1.0 --November/2011 +% +% Written by Deng Cai (dengcai AT gmail.com) + + +if nargin < 2 + error('litekmeans:TooFewInputs','At least two input arguments required.'); +end + +[n, p] = size(X); + + +pnames = { 'distance' 'start' 'maxiter' 'replicates' 'onlinephase' 'clustermaxiter'}; +dflts = {'sqeuclidean' 'sample' [] [] 'off' [] }; +[eid,errmsg,distance,start,maxit,reps,online,clustermaxit] = getargs(pnames, dflts, varargin{:}); +if ~isempty(eid) + error(sprintf('litekmeans:%s',eid),errmsg); +end + +if ischar(distance) + distNames = {'sqeuclidean','cosine'}; + j = strcmpi(distance, distNames); + j = find(j); + if length(j) > 1 + error('litekmeans:AmbiguousDistance', ... + 'Ambiguous ''Distance'' parameter value: %s.', distance); + elseif isempty(j) + error('litekmeans:UnknownDistance', ... + 'Unknown ''Distance'' parameter value: %s.', distance); + end + distance = distNames{j}; +else + error('litekmeans:InvalidDistance', ... + 'The ''Distance'' parameter value must be a string.'); +end + + +center = []; +if ischar(start) + startNames = {'sample','cluster'}; + j = find(strncmpi(start,startNames,length(start))); + if length(j) > 1 + error(message('litekmeans:AmbiguousStart', start)); + elseif isempty(j) + error(message('litekmeans:UnknownStart', start)); + elseif isempty(k) + error('litekmeans:MissingK', ... + 'You must specify the number of clusters, K.'); + end + if j == 2 + if floor(.1*n) < 5*k + j = 1; + end + end + start = startNames{j}; +elseif isnumeric(start) + if size(start,2) == p + center = start; + elseif (size(start,2) == 1 || size(start,1) == 1) + center = X(start,:); + else + error('litekmeans:MisshapedStart', ... + 'The ''Start'' matrix must have the same number of columns as X.'); + end + if isempty(k) + k = size(center,1); + elseif (k ~= size(center,1)) + error('litekmeans:MisshapedStart', ... + 'The ''Start'' matrix must have K rows.'); + end + start = 'numeric'; +else + error('litekmeans:InvalidStart', ... + 'The ''Start'' parameter value must be a string or a numeric matrix or array.'); +end + +% The maximum iteration number is default 100 +if isempty(maxit) + maxit = 100; +end + +% The maximum iteration number for preliminary clustering phase on random +% 10% subsamples is default 10 +if isempty(clustermaxit) + clustermaxit = 10; +end + + +% Assume one replicate +if isempty(reps) || ~isempty(center) + reps = 1; +end + +if ~(isscalar(k) && isnumeric(k) && isreal(k) && k > 0 && (round(k)==k)) + error('litekmeans:InvalidK', ... + 'X must be a positive integer value.'); +elseif n < k + error('litekmeans:TooManyClusters', ... + 'X must have more rows than the number of clusters.'); +end + + +bestlabel = []; +sumD = zeros(1,k); +bCon = false; + +for t=1:reps + switch start + case 'sample' + center = X(randsample(n,k),:); + case 'cluster' + Xsubset = X(randsample(n,floor(.1*n)),:); + [dump, center] = litekmeans(Xsubset, k, varargin{:}, 'start','sample', 'replicates',1 ,'MaxIter',clustermaxit); + case 'numeric' + end + + last = 0;label=1; + it=0; + + switch distance + case 'sqeuclidean' + while any(label ~= last) && it1 + if it>=maxit + aa = full(sum(X.*X,2)); + bb = full(sum(center.*center,2)); + ab = full(X*center'); + D = bsxfun(@plus,aa,bb') - 2*ab; + D(D<0) = 0; + else + aa = full(sum(X.*X,2)); + D = aa(:,ones(1,k)) + D; + D(D<0) = 0; + end + D = sqrt(D); + for j = 1:k + sumD(j) = sum(D(label==j,j)); + end + bestsumD = sumD; + bestD = D; + end + else + if it>=maxit + aa = full(sum(X.*X,2)); + bb = full(sum(center.*center,2)); + ab = full(X*center'); + D = bsxfun(@plus,aa,bb') - 2*ab; + D(D<0) = 0; + else + aa = full(sum(X.*X,2)); + D = aa(:,ones(1,k)) + D; + D(D<0) = 0; + end + D = sqrt(D); + for j = 1:k + sumD(j) = sum(D(label==j,j)); + end + if sum(sumD) < sum(bestsumD) + bestlabel = label; + bestcenter = center; + bestsumD = sumD; + bestD = D; + end + end + case 'cosine' + while any(label ~= last) && it1 + if any(label ~= last) + W=full(X*center'); + end + D = 1-W; + for j = 1:k + sumD(j) = sum(D(label==j,j)); + end + bestsumD = sumD; + bestD = D; + end + else + if any(label ~= last) + W=full(X*center'); + end + D = 1-W; + for j = 1:k + sumD(j) = sum(D(label==j,j)); + end + if sum(sumD) < sum(bestsumD) + bestlabel = label; + bestcenter = center; + bestsumD = sumD; + bestD = D; + end + end + end +end + +label = bestlabel; +center = bestcenter; +if reps>1 + sumD = bestsumD; + D = bestD; +elseif nargout > 3 + switch distance + case 'sqeuclidean' + if it>=maxit + aa = full(sum(X.*X,2)); + bb = full(sum(center.*center,2)); + ab = full(X*center'); + D = bsxfun(@plus,aa,bb') - 2*ab; + D(D<0) = 0; + else + aa = full(sum(X.*X,2)); + D = aa(:,ones(1,k)) + D; + D(D<0) = 0; + end + D = sqrt(D); + case 'cosine' + if it>=maxit + W=full(X*center'); + end + D = 1-W; + end + for j = 1:k + sumD(j) = sum(D(label==j,j)); + end +end + + + + +function [eid,emsg,varargout]=getargs(pnames,dflts,varargin) +%GETARGS Process parameter name/value pairs +% [EID,EMSG,A,B,...]=GETARGS(PNAMES,DFLTS,'NAME1',VAL1,'NAME2',VAL2,...) +% accepts a cell array PNAMES of valid parameter names, a cell array +% DFLTS of default values for the parameters named in PNAMES, and +% additional parameter name/value pairs. Returns parameter values A,B,... +% in the same order as the names in PNAMES. Outputs corresponding to +% entries in PNAMES that are not specified in the name/value pairs are +% set to the corresponding value from DFLTS. If nargout is equal to +% length(PNAMES)+1, then unrecognized name/value pairs are an error. If +% nargout is equal to length(PNAMES)+2, then all unrecognized name/value +% pairs are returned in a single cell array following any other outputs. +% +% EID and EMSG are empty if the arguments are valid. If an error occurs, +% EMSG is the text of an error message and EID is the final component +% of an error message id. GETARGS does not actually throw any errors, +% but rather returns EID and EMSG so that the caller may throw the error. +% Outputs will be partially processed after an error occurs. +% +% This utility can be used for processing name/value pair arguments. +% +% Example: +% pnames = {'color' 'linestyle', 'linewidth'} +% dflts = { 'r' '_' '1'} +% varargin = {{'linew' 2 'nonesuch' [1 2 3] 'linestyle' ':'} +% [eid,emsg,c,ls,lw] = statgetargs(pnames,dflts,varargin{:}) % error +% [eid,emsg,c,ls,lw,ur] = statgetargs(pnames,dflts,varargin{:}) % ok + +% We always create (nparams+2) outputs: +% one each for emsg and eid +% nparams varargs for values corresponding to names in pnames +% If they ask for one more (nargout == nparams+3), it's for unrecognized +% names/values + +% Original Copyright 1993-2008 The MathWorks, Inc. +% Modified by Deng Cai (dengcai@gmail.com) 2011.11.27 + + + + +% Initialize some variables +emsg = ''; +eid = ''; +nparams = length(pnames); +varargout = dflts; +unrecog = {}; +nargs = length(varargin); + +% Must have name/value pairs +if mod(nargs,2)~=0 + eid = 'WrongNumberArgs'; + emsg = 'Wrong number of arguments.'; +else + % Process name/value pairs + for j=1:2:nargs + pname = varargin{j}; + if ~ischar(pname) + eid = 'BadParamName'; + emsg = 'Parameter name must be text.'; + break; + end + i = strcmpi(pname,pnames); + i = find(i); + if isempty(i) + % if they've asked to get back unrecognized names/values, add this + % one to the list + if nargout > nparams+2 + unrecog((end+1):(end+2)) = {varargin{j} varargin{j+1}}; + % otherwise, it's an error + else + eid = 'BadParamName'; + emsg = sprintf('Invalid parameter name: %s.',pname); + break; + end + elseif length(i)>1 + eid = 'BadParamName'; + emsg = sprintf('Ambiguous parameter name: %s.',pname); + break; + else + varargout{i} = varargin{j+1}; + end + end +end + +varargout{nparams+1} = unrecog; diff --git a/lsqr2.m b/lsqr2.m new file mode 100644 index 0000000..0686640 --- /dev/null +++ b/lsqr2.m @@ -0,0 +1,348 @@ +function [X, istop, itn] = lsqr2( A, B, damp, itnlim, atol, btol, show) + +% This function is provided at +% http://www.stanford.edu/group/SOL/software/lsqr.html +% I have made small modifications -- Deng Cai, April/2006 +% +% +% function [X, istop, itn] = lsqr2( A, B, damp, itnlim, atol, btol, show) +% +% [ x, istop, itn, r1norm, r2norm, anorm, acond, arnorm, xnorm, var ]... +% = lsqr( m, n, 'aprod', iw, rw, b, damp, atol, btol, conlim, itnlim, show ); +% +% LSQR solves Ax = b or min ||b - Ax||_2 if damp = 0, +% or min || (b) - ( A )x || otherwise. +% || (0) (damp I) ||2 +% A is an m by n matrix defined by y = aprod( mode,m,n,x,iw,rw ), +% where the parameter 'aprodname' refers to a function 'aprod' that +% performs the matrix-vector operations. +% If mode = 1, aprod must return y = Ax without altering x. +% If mode = 2, aprod must return y = A'x without altering x. +% WARNING: The file containing the function 'aprod' +% must not be called aprodname.m !!!! + +%----------------------------------------------------------------------- +% LSQR uses an iterative (conjugate-gradient-like) method. +% For further information, see +% 1. C. C. Paige and M. A. Saunders (1982a). +% LSQR: An algorithm for sparse linear equations and sparse least squares, +% ACM TOMS 8(1), 43-71. +% 2. C. C. Paige and M. A. Saunders (1982b). +% Algorithm 583. LSQR: Sparse linear equations and least squares problems, +% ACM TOMS 8(2), 195-209. +% 3. M. A. Saunders (1995). Solution of sparse rectangular systems using +% LSQR and CRAIG, BIT 35, 588-604. +% +% Input parameters: +% iw, rw are not used by lsqr, but are passed to aprod. +% atol, btol are stopping tolerances. If both are 1.0e-9 (say), +% the final residual norm should be accurate to about 9 digits. +% (The final x will usually have fewer correct digits, +% depending on cond(A) and the size of damp.) +% conlim is also a stopping tolerance. lsqr terminates if an estimate +% of cond(A) exceeds conlim. For compatible systems Ax = b, +% conlim could be as large as 1.0e+12 (say). For least-squares +% problems, conlim should be less than 1.0e+8. +% Maximum precision can be obtained by setting +% atol = btol = conlim = zero, but the number of iterations +% may then be excessive. +% itnlim is an explicit limit on iterations (for safety). +% show = 1 gives an iteration log, +% show = 0 suppresses output. +% +% Output parameters: +% x is the final solution. +% istop gives the reason for termination. +% istop = 1 means x is an approximate solution to Ax = b. +% = 2 means x approximately solves the least-squares problem. +% r1norm = norm(r), where r = b - Ax. +% r2norm = sqrt( norm(r)^2 + damp^2 * norm(x)^2 ) +% = r1norm if damp = 0. +% anorm = estimate of Frobenius norm of Abar = [ A ]. +% [damp*I] +% acond = estimate of cond(Abar). +% arnorm = estimate of norm(A'*r - damp^2*x). +% xnorm = norm(x). +% var (if present) estimates all diagonals of (A'A)^{-1} (if damp=0) +% or more generally (A'A + damp^2*I)^{-1}. +% This is well defined if A has full column rank or damp > 0. +% (Not sure what var means if rank(A) < n and damp = 0.) +% +% +% 1990: Derived from Fortran 77 version of LSQR. +% 22 May 1992: bbnorm was used incorrectly. Replaced by anorm. +% 26 Oct 1992: More input and output parameters added. +% 01 Sep 1994: Matrix-vector routine is now a parameter 'aprodname'. +% Print log reformatted. +% 14 Jun 1997: show added to allow printing or not. +% 30 Jun 1997: var added as an optional output parameter. +% 07 Aug 2002: Output parameter rnorm replaced by r1norm and r2norm. +% Michael Saunders, Systems Optimization Laboratory, +% Dept of MS&E, Stanford University. +%----------------------------------------------------------------------- + +% Initialize. + +if nargin < 2 + error('Not enough input arguments.'); +end + +[m,n] = size(A); +[tm,d] = size(B); +if ~isequal(tm,m) + error(['Right hand side must be a column vector of' ... + ' length %d to match the coefficient matrix.'],m); +end + + +% Assign default values to unspecified parameters +if nargin < 3 || isempty(damp) + damp = 0; +end + +if nargin < 4 || isempty(itnlim) + itnlim = min([m,n,200]); +end + + +if nargin < 5 || isempty(atol) + atol = 1e-6; +end + +if nargin < 6 || isempty(btol) + btol = 1e-6; +end + +if nargin < 7 || isempty(show) + show = 0; +end + + + +if show + msg=['The exact solution is x = 0 ' + 'Ax - b is small enough, given atol, btol ' + 'The least-squares solution is good enough, given atol ' + 'The estimate of cond(Abar) has exceeded conlim ' + 'Ax - b is small enough for this machine ' + 'The least-squares solution is good enough for this machine' + 'Cond(Abar) seems to be too large for this machine ' + 'The iteration limit has been reached ']; + + disp(' ') + disp('LSQR Least-squares solution of Ax = b') + str1 = sprintf('The matrix A has %8g rows and %8g cols', m, n); + str2 = sprintf('damp = %20.14e ', damp); + str3 = sprintf('atol = %8.2e ', atol); + str4 = sprintf('btol = %8.2e itnlim = %8g' , btol, itnlim); + disp(str1); disp(str2); disp(str3); disp(str4); +end + +dampsq = damp^2; + + +X = zeros(n,d); +istop = zeros(1,d); +itn = zeros(1,d); + + +for t=1:d + nstop = 0; + anorm = 0; acond = 0; + ddnorm = 0; res2 = 0; + xnorm = 0; xxnorm = 0; z = 0; + cs2 = -1; sn2 = 0; + + % Set up the first vectors u and v for the bidiagonalization. + % These satisfy beta*u = b, alfa*v = A'u. + + u = B(:,t); + alfa = 0; beta = norm( u ); + if beta > 0 + u = (1/beta) * u; v = A'*u; + alfa = norm( v ); + else + break; + end + if alfa > 0 + v = (1/alfa) * v; w = v; + end + + arnorm = alfa * beta; if arnorm == 0, disp(msg(1,:)); continue, end + + rhobar = alfa; phibar = beta; bnorm = beta; + rnorm = beta; + r1norm = rnorm; + r2norm = rnorm; + + if show + head1 = ' Itn x(1) r1norm r2norm '; + head2 = ' Compatible LS Norm A Cond A'; + + disp(' ') + disp([head1 head2]) + test1 = 1; test2 = alfa / beta; + str1 = sprintf( '%6g %12.5e', itn, X(1,t) ); + str2 = sprintf( ' %10.3e %10.3e', r1norm, r2norm ); + str3 = sprintf( ' %8.1e %8.1e', test1, test2 ); + disp([str1 str2 str3]) + end + + %------------------------------------------------------------------ + % Main iteration loop. + %------------------------------------------------------------------ + while itn(t) < itnlim + itn(t) = itn(t) + 1; + % Perform the next step of the bidiagonalization to obtain the + % next beta, u, alfa, v. These satisfy the relations + % beta*u = A*v - alfa*u, + % alfa*v = A'*u - beta*v. + + u = A*v - alfa*u; + beta = norm( u ); + if beta > 0 + u = (1/beta) * u; + anorm = norm([anorm alfa beta damp]); + v = A'*u - beta*v; + alfa = norm( v ); + if alfa > 0, v = (1/alfa) * v; end + end + + % Use a plane rotation to eliminate the damping parameter. + % This alters the diagonal (rhobar) of the lower-bidiagonal matrix. + + rhobar1 = norm([rhobar damp]); + cs1 = rhobar / rhobar1; + sn1 = damp / rhobar1; + psi = sn1 * phibar; + phibar = cs1 * phibar; + + % Use a plane rotation to eliminate the subdiagonal element (beta) + % of the lower-bidiagonal matrix, giving an upper-bidiagonal matrix. + + rho = norm([rhobar1 beta]); + cs = rhobar1/ rho; + sn = beta / rho; + theta = sn * alfa; + rhobar = - cs * alfa; + phi = cs * phibar; + phibar = sn * phibar; + tau = sn * phi; + + % Update x and w. + + t1 = phi /rho; + t2 = - theta/rho; + dk = (1/rho)*w; + + X(:,t) = X(:,t) + t1*w; + w = v + t2*w; + ddnorm = ddnorm + norm(dk)^2; + + % Use a plane rotation on the right to eliminate the + % super-diagonal element (theta) of the upper-bidiagonal matrix. + % Then use the result to estimate norm(x). + + delta = sn2 * rho; + gambar = - cs2 * rho; + rhs = phi - delta * z; + zbar = rhs / gambar; + xnorm = sqrt(xxnorm + zbar^2); + gamma = norm([gambar theta]); + cs2 = gambar / gamma; + sn2 = theta / gamma; + z = rhs / gamma; + xxnorm = xxnorm + z^2; + + % Test for convergence. + % First, estimate the condition of the matrix Abar, + % and the norms of rbar and Abar'rbar. + + acond = anorm * sqrt( ddnorm ); + res1 = phibar^2; + res2 = res2 + psi^2; + rnorm = sqrt( res1 + res2 ); + arnorm = alfa * abs( tau ); + + % 07 Aug 2002: + % Distinguish between + % r1norm = ||b - Ax|| and + % r2norm = rnorm in current code + % = sqrt(r1norm^2 + damp^2*||x||^2). + % Estimate r1norm from + % r1norm = sqrt(r2norm^2 - damp^2*||x||^2). + % Although there is cancellation, it might be accurate enough. + + r1sq = rnorm^2 - dampsq * xxnorm; + r1norm = sqrt( abs(r1sq) ); if r1sq < 0, r1norm = - r1norm; end + r2norm = rnorm; + + % Now use these norms to estimate certain other quantities, + % some of which will be small near a solution. + + test1 = rnorm / bnorm; + test2 = arnorm/( anorm * rnorm ); + test3 = 1 / acond; + t1 = test1 / (1 + anorm * xnorm / bnorm); + rtol = btol + atol * anorm * xnorm / bnorm; + + % The following tests guard against extremely small values of + % atol, btol or ctol. (The user may have set any or all of + % the parameters atol, btol, conlim to 0.) + % The effect is equivalent to the normal tests using + % atol = eps, btol = eps, conlim = 1/eps. + + if itn(t) >= itnlim, istop(t) = 7; end % 'The iteration limit has been reached ' + if 1 + test3 <= 1, istop(t) = 6; end % 'Cond(Abar) seems to be too large for this machine ' + if 1 + test2 <= 1, istop(t) = 5; end % 'The least-squares solution is good enough for this machine' + if 1 + t1 <= 1, istop(t) = 4; end % 'Ax - b is small enough for this machine ' + + % Allow for tolerances set by the user. + + if test2 <= atol, istop(t) = 2; end % 'The least-squares solution is good enough, given atol ' + if test1 <= rtol, istop(t) = 1; end % 'Ax - b is small enough, given atol, btol ' + + % See if it is time to print something. + + if show + prnt = 0; + if n <= 40 , prnt = 1; end + if itn(t) <= 10 , prnt = 1; end + if itn(t) >= itnlim-10, prnt = 1; end + if rem(itn(t),10) == 0 , prnt = 1; end + if test2 <= 10*atol , prnt = 1; end + if test1 <= 10*rtol , prnt = 1; end + if istop ~= 0 , prnt = 1; end + + if prnt == 1 + str1 = sprintf( '%6g %12.5e', itn(t), X(1,t) ); + str2 = sprintf( ' %10.3e %10.3e', r1norm, r2norm ); + str3 = sprintf( ' %8.1e %8.1e', test1, test2 ); + str4 = sprintf( ' %8.1e %8.1e', anorm, acond ); + disp([str1 str2 str3 str4]) + end + end + if istop > 0, break, end + end + + % End of iteration loop. + % Print the stopping condition. + + if show + disp(' ') + disp('LSQR finished') + disp(msg(istop(t)+1,:)) + disp(' ') + str1 = sprintf( 'istop =%8g r1norm =%8.1e', istop(t), r1norm ); + str2 = sprintf( 'anorm =%8.1e arnorm =%8.1e', anorm, arnorm ); + str3 = sprintf( 'itn =%8g r2norm =%8.1e', itn(t), r2norm ); + str4 = sprintf( 'acond =%8.1e xnorm =%8.1e', acond, xnorm ); + disp([str1 ' ' str2]) + disp([str3 ' ' str4]) + disp(' ') + end + +end +%----------------------------------------------------------------------- +% End of lsqr.m +%----------------------------------------------------------------------- diff --git a/mex_EMstep.mexw32 b/mex_EMstep.mexw32 new file mode 100644 index 0000000..c4dcf03 Binary files /dev/null and b/mex_EMstep.mexw32 differ diff --git a/mex_EMstep.mexw64 b/mex_EMstep.mexw64 new file mode 100644 index 0000000..968494d Binary files /dev/null and b/mex_EMstep.mexw64 differ diff --git a/mex_Pw_d.mexw32 b/mex_Pw_d.mexw32 new file mode 100644 index 0000000..2eecd62 Binary files /dev/null and b/mex_Pw_d.mexw32 differ diff --git a/mex_Pw_d.mexw64 b/mex_Pw_d.mexw64 new file mode 100644 index 0000000..f01876e Binary files /dev/null and b/mex_Pw_d.mexw64 differ diff --git a/mex_logL.mexw32 b/mex_logL.mexw32 new file mode 100644 index 0000000..25896a8 Binary files /dev/null and b/mex_logL.mexw32 differ diff --git a/mex_logL.mexw64 b/mex_logL.mexw64 new file mode 100644 index 0000000..dcbbe70 Binary files /dev/null and b/mex_logL.mexw64 differ diff --git a/mySVD.m b/mySVD.m new file mode 100644 index 0000000..903731d --- /dev/null +++ b/mySVD.m @@ -0,0 +1,118 @@ +function [U, S, V] = mySVD(X,ReducedDim) +%mySVD Accelerated singular value decomposition. +% [U,S,V] = mySVD(X) produces a diagonal matrix S, of the +% dimension as the rank of X and with nonnegative diagonal elements in +% decreasing order, and unitary matrices U and V so that +% X = U*S*V'. +% +% [U,S,V] = mySVD(X,ReducedDim) produces a diagonal matrix S, of the +% dimension as ReducedDim and with nonnegative diagonal elements in +% decreasing order, and unitary matrices U and V so that +% Xhat = U*S*V' is the best approximation (with respect to F norm) of X +% among all the matrices with rank no larger than ReducedDim. +% +% Based on the size of X, mySVD computes the eigvectors of X*X^T or X^T*X +% first, and then convert them to the eigenvectors of the other. +% +% See also SVD. +% +% version 2.0 --Feb/2009 +% version 1.0 --April/2004 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +MAX_MATRIX_SIZE = 1600; % You can change this number according your machine computational power +EIGVECTOR_RATIO = 0.1; % You can change this number according your machine computational power + + +if ~exist('ReducedDim','var') + ReducedDim = 0; +end + +[nSmp, mFea] = size(X); +if mFea/nSmp > 1.0713 + ddata = X*X'; + ddata = max(ddata,ddata'); + + dimMatrix = size(ddata,1); + if (ReducedDim > 0) && (dimMatrix > MAX_MATRIX_SIZE) && (ReducedDim < dimMatrix*EIGVECTOR_RATIO) + option = struct('disp',0); + [U, eigvalue] = eigs(ddata,ReducedDim,'la',option); + eigvalue = diag(eigvalue); + else + if issparse(ddata) + ddata = full(ddata); + end + + [U, eigvalue] = eig(ddata); + eigvalue = diag(eigvalue); + [dump, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + U = U(:, index); + end + clear ddata; + + maxEigValue = max(abs(eigvalue)); + eigIdx = find(abs(eigvalue)/maxEigValue < 1e-10); + eigvalue(eigIdx) = []; + U(:,eigIdx) = []; + + if (ReducedDim > 0) && (ReducedDim < length(eigvalue)) + eigvalue = eigvalue(1:ReducedDim); + U = U(:,1:ReducedDim); + end + + eigvalue_Half = eigvalue.^.5; + S = spdiags(eigvalue_Half,0,length(eigvalue_Half),length(eigvalue_Half)); + + if nargout >= 3 + eigvalue_MinusHalf = eigvalue_Half.^-1; + V = X'*(U.*repmat(eigvalue_MinusHalf',size(U,1),1)); + end +else + ddata = X'*X; + ddata = max(ddata,ddata'); + + dimMatrix = size(ddata,1); + if (ReducedDim > 0) && (dimMatrix > MAX_MATRIX_SIZE) && (ReducedDim < dimMatrix*EIGVECTOR_RATIO) + option = struct('disp',0); + [V, eigvalue] = eigs(ddata,ReducedDim,'la',option); + eigvalue = diag(eigvalue); + else + if issparse(ddata) + ddata = full(ddata); + end + + [V, eigvalue] = eig(ddata); + eigvalue = diag(eigvalue); + + [dump, index] = sort(-eigvalue); + eigvalue = eigvalue(index); + V = V(:, index); + end + clear ddata; + + maxEigValue = max(abs(eigvalue)); + eigIdx = find(abs(eigvalue)/maxEigValue < 1e-10); + eigvalue(eigIdx) = []; + V(:,eigIdx) = []; + + if (ReducedDim > 0) && (ReducedDim < length(eigvalue)) + eigvalue = eigvalue(1:ReducedDim); + V = V(:,1:ReducedDim); + end + + eigvalue_Half = eigvalue.^.5; + S = spdiags(eigvalue_Half,0,length(eigvalue_Half),length(eigvalue_Half)); + + eigvalue_MinusHalf = eigvalue_Half.^-1; + U = X*(V.*repmat(eigvalue_MinusHalf',size(V,1),1)); +end + + + + + + + diff --git a/sll_opts.m b/sll_opts.m new file mode 100644 index 0000000..433d71b --- /dev/null +++ b/sll_opts.m @@ -0,0 +1,187 @@ +function opts = sll_opts(opts) + +% Options for Sparse Learning Library +% +% Notice: +% If one or several (even all) fields are empty, sll_opts shall assign the +% default settings. +% +% If some fields of opts have been defined, sll_opts shall check the fields +% for possible errors. +% +% +% Table of Options. * * indicates default value. +% +%% FIELD DESCRIPTION +%% Starting point +% +% .x0 Starting point of x. +% Initialized according to .init. +% +% .c0 Starting point for the intercept c (for Logistic Loss) +% Initialized according to .init. +% +% .init .init specifies how to initialize x. +% * 0 => .x0 is set by the function initFactor * +% 1 => .x0 and .c0 are defined +% 2 => .x0= zeros(n,1), .c0=0 +% +%% Termination +% +% .maxIter Maximum number of iterations. +% *1e4* +% +% .tol Tolerance parameter. +% *1e-4* +% +% .tFlag Flag for termination. +% * 0 => abs( funVal(i)- funVal(i-1) ) <= .tol * +% 1 => abs( funVal(i)- funVal(i-1) ) +% <= .tol max( funVal(i-1), 1) +% 2 => funVal(i) <= .tol +% 3 => norm( x_i - x_{i-1}, 2) <= .tol +% 4 => norm( x_i - x_{i-1}, 2) <= +% <= .tol max( norm( x_{i-1}, 2), 1 ) +% 5 => Run the code for .maxIter iterations +% +%% Normalization +% +% .nFlag Flag for implicit normalization of A. +% * 0 => Do not normalize A * +% 1 => A=(A-repmat(mu, m, 1))*diag(nu)^{-1} +% 2 => A=diag(nu)^{-1}*(A-repmat(mu,m,1) +% +% .mu Row vector to be substracted from each sample. +% (.mu is used when .nFlag=1 or 2) +% If .mu is not specified, then +% * .mu=mean(A,1) * +% +% .nu Weight (column) vector for normalization +% (.mu is used when .nFlag=1 or 2) +% If .nu is not specified, then +% * .nFlag=1 => .nu=(sum(A.^2, 1)'/m.^{0.5} * +% * .nFlag=2 => .nu=(sum(A.^2, 2)/n.^{0.5} * +% +%% Regularization +% +% .rFlag Flag for regularization +% (.rFlag is used for the functions with "R") +% * 0 => lambda is the regularization parameter * +% 1 => lambda = lambda * lambda_{max} +% where lambda_{max} is the maximum lambda +% that yields the zero solution +% .rsL2 Regularization parameter value of the squared L2 norm +% (.rsL2 is used only for l1 regularization) +% *.rsL2=0* +% If .rFlag=0, .rsL2 is used without scaling +% .rFlag=1, .rsL2=.rsL2 * lambda_{max} +% +%% Method & Line Search +% .lFlag +% +%% Grooup & Others +% +% .ind Indices for k groups (a k+1 row vector) +% For group lasso only +% Indices for the i-th group are (ind(i)+1):ind(i+1) +% +% .q Value of q in L1/Lq regularization +% *.q=2* +% +% .sWeight The sample (positive and negative) weight +% For the Logistic Loss only +% Positive sample: .sWeight(1) +% Negative sample: sWeight(2) +% *1/m for both positive and negative samples* +% +% .gWeight The weight for different groups +% *.gWeight=1* +% +% .fName The name of the function +% +%% Copyright (C) 2009-2010 Jun Liu, and Jieping Ye +% +% You are suggested to first read the Manual. +% +% For any problem, please contact with Jun Liu via j.liu@asu.edu +% +% Last modified 7 August 2009. + +%% Starting point + +if isfield(opts,'init') + if (opts.init~=0) && (opts.init~=1) && (opts.init~=2) + opts.init=0; % if .init is not 0, 1, or 2, then use the default 0 + end + + if ~isfield(opts,'x0') && (opts.init==1) + opts.init=0; % if .x0 is not defined and .init=1, set .init=0 + end +else + opts.init = 0; + % if .init is not specified, use "0" +end + +%% Termination + +if isfield(opts,'maxIter') + if (opts.maxIter<1) + opts.maxIter=10000; + end +else + opts.maxIter=10000; +end + +if ~isfield(opts,'tol') + opts.tol=1e-3; +end + +if isfield(opts,'tFlag') + if opts.tFlag<0 + opts.tFlag=0; + elseif opts.tFlag>5 + opts.tFlag=5; + else + opts.tFlag=floor(opts.tFlag); + end +else + opts.tFlag=0; +end + +%% Normalization + +if isfield(opts,'nFlag') + if (opts.nFlag~=1) && (opts.nFlag~=2) + opts.nFlag=0; + end +else + opts.nFlag=0; +end + +%% Regularization + +if isfield(opts,'rFlag') + if (opts.rFlag~=1) + opts.rFlag=0; + end +else + opts.rFlag=0; +end +%% Method (Line Search) + +if isfield(opts,'lFlag') + if (opts.lFlag~=1) + opts.lFlag=0; + end +else + opts.lFlag=0; +end + +if isfield(opts,'mFlag') + if (opts.mFlag~=1) + opts.mFlag=0; + end +else + opts.mFlag=0; +end + diff --git a/tfidf.m b/tfidf.m new file mode 100644 index 0000000..e92c7ab --- /dev/null +++ b/tfidf.m @@ -0,0 +1,49 @@ +function fea = tfidf(fea,bNorm) +% fea is a document-term frequency matrix, this function return the tfidf ([1+log(tf)]*log[N/df]) +% weighted document-term matrix. +% +% If bNorm == 1, each document verctor will be further normalized to +% have unit norm. (default) +% +% version 2.0 --Jan/2012 +% version 1.0 --Oct/2003 +% +% Written by Deng Cai (dengcai AT gmail.com) +% + +if ~exist('bNorm','var') + bNorm = 1; +end + + +[nSmp,mFea] = size(fea); +[idx,jdx,vv] = find(fea); +df = full(sum(sparse(idx,jdx,1),1)); + +df(df==0) = 1; +idf = log(nSmp./df); + +tffea = sparse(idx,jdx,log(vv)+1); + +fea2 = tffea'; +idf = idf'; + +MAX_MATRIX_SIZE = 5000; % You can change this number based on your memory. +nBlock = ceil(MAX_MATRIX_SIZE*MAX_MATRIX_SIZE/mFea); +for i = 1:ceil(nSmp/nBlock) + if i == ceil(nSmp/nBlock) + smpIdx = (i-1)*nBlock+1:nSmp; + else + smpIdx = (i-1)*nBlock+1:i*nBlock; + end + fea2(:,smpIdx) = fea2(:,smpIdx) .* idf(:,ones(1,length(smpIdx))); +end + +%Now each column of fea2 is the tf-idf vector. +%One can further normalize each vector to unit by using following codes: + +if bNorm + fea = NormalizeFea(fea2,0)'; +end + +% fea is the final document-term matrix.