add data

reference: Jianwei Yang Implementation for Face alignment at 3000 FPS
via Regressing Local Binary Features

To fit the data, need to revise the input of the functions

Author: ShownX

ESR_Train.m

@@ -1,14 +1,39 @@
 function ESR_Train()
-    % load data
-    % load('../Data/toyData.mat', 'images', 'bbx', 'pts');
-    load('../../Data/toyData', 'images_aug', 'bbx_aug', 'pts_aug', 'current_shapes','ground_truth');
-    
+    %% load parameters
     params = Train_params;
-    params.N_img = size(images_aug, 1);
     % create paralllel local jobs note
     if isempty(gcp('nocreate'))
         parpool(2);
     end
+    %% load data
+    if exist('Data/train_init.mat', 'file')
+        load('Data/train_init.mat', 'data');
+    else
+        data = loadsamples('D:\Dataset\lfpw\annotations\trainset', 'png');
+        %mkdir Data;
+        save('Data/train_init.mat', 'data');
+    end
+    
+    load('Data/InitialShape_68');
+    params.meanshape = S0(params.ind_usedpts, :);
+    params.N_fp = size(params.meanshape, 1);
+    %% flip data
+    if params.flip
+        data_flip = fliplrdata(data);
+    else
+        data_flip = [];
+    end
+    Data = [data; data_flip];
+    %% choose corresponding points for training
+    parfor i = 1:length(Data)
+        Data{i}.shape_gt = Data{i}.shape_gt(params.ind_usedpts, :);
+        Data{i}.bbox_gt = getbbox(Data{i}.shape_gt);
+    end
+    %% augment the data
+    Data = augmtdata(Data, params);
+    %% Explicit Shape Regression
+    params.N_img = size(images_aug, 1);
+    
 
     Y = cell(params.N_img, 1);
     Error = zeros(1, params.T+1);

Train_params.m

@@ -6,12 +6,15 @@
     % the local scale of search, it set 0.3 times of the distance between two pupils on the mean shape
     params.F = 5; % the number of features in fern
 
-    load('../../Data/mean_shape.mat', 'S0');
-    params.mean_shape = S0;
+    %params.mean_shape = S0;
 
-    params.N_fp = size(params.mean_shape, 1);
+    params.N_aug = 20;
+    
+    params.N_fp = 0; %size(params.mean_shape, 1);
     params.N_img = 0;
 
-    params.k = params.k*pdist([mean([params.mean_shape(20, :); params.mean_shape(23, :)]);...
-        mean([params.mean_shape(26, :); params.mean_shape(29, :)])])/2;
+    params.flip = 1; % mirror the data
+    params.ind_usedpts = 18:68;
+    %params.k = params.k*pdist([mean([params.mean_shape(20, :); params.mean_shape(23, :)]);...
+    %    mean([params.mean_shape(26, :); params.mean_shape(29, :)])])/2;
 end

augmtdata.m

@@ -0,0 +1,53 @@
+function data_aug = augmtdata(data, params)
+    data_len = length(data);
+    data_aug = cell(params.N_aug*data_len, 1);
+    
+    
+    for i = 1: data_len
+        % random select initial shape without replacement
+        rand_index_ = randperm(data_len, params.N_aug);
+        while ismember(i, rand_index_) % if rand_index contain i, rand select again
+            rand_index_ = randperm(data_len, params.N_aug);
+        end
+        % expand the data
+        for j = 1: params.N_aug
+            r_index = rand_index_(j);
+            % copy the original stuff
+            data_index = (j-1)*data_len + i;
+            data_aug{data_index}.img_gray = data{i}.img_gray;
+            data_aug{data_index}.width_orig  = data{i}.width_orig;
+            data_aug{data_index}.height_orig = data{i}.height_orig;    
+            data_aug{data_index}.width       = data{i}.width;
+            data_aug{data_index}.height      = data{i}.height;
+            data_aug{data_index}.shape_gt    = data{i}.shape_gt;
+            data_aug{data_index}.bbox_gt        = data{i}.bbox_gt;
+            % add the new stuff
+            data_aug{data_index}.intermediate_shapes = cell(1, params.T);
+            data_aug{data_index}.intermediate_bboxes = cell(1, params.T);
+            % scale and translate the sampled shape to ground-truth
+            % face rectangle region
+            select_shape = resetshape(data{i}.bbox_gt, [data{r_index}.shape_gt]);
+
+            data_aug{data_index}.intermediate_shapes{1} = select_shape;
+            data_aug{data_index}.intermediate_bboxes{1} = getbbox(select_shape);
+
+            meanshape_resize = resetshape(data_aug{data_index}.intermediate_bboxes{1}, params.meanshape);
+
+            data_aug{data_index}.tf2meanshape = fitgeotrans(bsxfun(@minus, ...
+                data_aug{data_index}.intermediate_shapes{1}, mean(data_aug{data_index}.intermediate_shapes{1})), ...
+                bsxfun(@minus, meanshape_resize, mean(meanshape_resize)),...
+                'nonreflectivesimilarity');
+            data_aug{data_index}.meanshape2tf = fitgeotrans(bsxfun(@minus, meanshape_resize, mean(meanshape_resize)), ...
+                bsxfun(@minus, data_aug{data_index}.intermediate_shapes{1}, mean(data_aug{data_index}.intermediate_shapes{1})),...
+                'nonreflectivesimilarity');                
+
+
+            shape_residual = bsxfun(@rdivide, data_aug{data_index}.shape_gt - data_aug{data_index}.intermediate_shapes{1},...
+                [data_aug{data_index}.intermediate_bboxes{1}(3) data_aug{data_index}.intermediate_bboxes{1}(4)]);  
+
+            [u, v] = transformPointsForward(data_aug{data_index}.tf2meanshape, shape_residual(:, 1), shape_residual(:, 2));
+            data_aug{data_index}.shapes_residual = [u, v];
+        end
+    end
+    
+end

fliplrdata.m

@@ -0,0 +1,18 @@
+function data_flip = fliplrdata(data)
+    data_len = size(data, 1);
+    data_flip = cell(data_len, 1);
+    parfor i = 1:data_len
+        data_flip{i}.img_gray    = fliplr(data{i}.img_gray);
+        data_flip{i}.width_orig  = data{i}.width_orig;
+        data_flip{i}.height_orig = data{i}.height_orig;    
+        data_flip{i}.width       = data{i}.width;
+        data_flip{i}.height      = data{i}.height; 
+        
+        data_flip{i}.shape_gt    = flipshape(data{i}.shape_gt);        
+        data_flip{i}.shape_gt(:, 1)  =  data{i}.width - data_flip{i}.shape_gt(:, 1);
+        
+        data_flip{i}.bbox_gt        = data{i}.bbox_gt;
+        data_flip{i}.bbox_gt(1)     = data_flip{i}.width - data_flip{i}.bbox_gt(1) - data_flip{i}.bbox_gt(3);       
+        
+    end
+end

flipshape.m

@@ -0,0 +1,32 @@
+function shape_flipped = flipshape(shape)
+%FLIPSHAPE Summary of this function goes here
+%   Function: flip input shape horizonically
+%   Detailed explanation goes here
+
+if size(shape, 1) == 68
+    shape_flipped = shape;
+    % flip check
+    shape_flipped(1:17, :) = shape(17:-1:1, :);
+    % flip eyebows
+    shape_flipped(18:27, :) = shape(27:-1:18, :);
+    % flip eyes
+    shape_flipped(32:36, :) = shape(36:-1:32, :);
+    % flip eyes
+    shape_flipped(37:40, :) = shape(46:-1:43, :);
+    shape_flipped(41:42, :) = shape(48:-1:47, :);
+    shape_flipped(43:46, :) = shape(40:-1:37, :);
+    shape_flipped(47:48, :) = shape(42:-1:41, :);
+    
+    % flip mouth
+    shape_flipped(49:55, :) = shape(55:-1:49, :);
+    shape_flipped(56:60, :) = shape(60:-1:56, :);   
+    
+    shape_flipped(61:65, :) = shape(65:-1:61, :);
+    shape_flipped(66:68, :) = shape(68:-1:66, :);   
+    
+else
+    disp('The Flip Funtion is Error!')
+end
+
+end
+

loadsamples.m

@@ -0,0 +1,68 @@
+function data = loadsamples(datapath, imgtype)
+    dd = dir(fullfile(datapath,['*.', imgtype]));
+    data_len = length(dd);
+    data = cell(data_len, 1);
+
+    parfor i = 1: data_len
+        
+        img = im2uint8(imread(fullfile(datapath, dd(i).name)));
+        data{i}.height_orig = size(img, 1);
+        data{i}.width_orig = size(img, 2);
+        shapepath = strrep(fullfile(datapath, dd(i).name), imgtype, 'pts');
+        data{i}.shape_gt = double(loadshape(shapepath));
+        data{i}.bbox_gt = getbbox(data{i}.shape_gt);
+        
+        region = enlargingbbox(data{i}.bbox_gt, 2.0);
+    
+        region(2) = double(max(region(2), 1));
+        region(1) = double(max(region(1), 1));
+
+        bottom_y = double(min(region(2) + region(4) - 1, data{i}.height_orig));
+        right_x = double(min(region(1) + region(3) - 1, data{i}.width_orig));
+
+        img_region = img(region(2):bottom_y, region(1):right_x, :);
+
+        % recalculate the location of groundtruth shape and bounding box
+        data{i}.shape_gt = bsxfun(@minus, data{i}.shape_gt, double([region(1) region(2)]));
+        data{i}.bbox_gt = getbbox(data{i}.shape_gt);
+
+        % only use inner points
+        data{i}.shape_gt = data{i}.shape_gt;
+
+        data{i}.isdet = 0;
+
+        if size(img_region, 3) == 1
+            data{i}.img_gray = img_region;
+        else
+            % hsv = rgb2hsv(img_region);
+            data{i}.img_gray = rgb2gray(img_region);
+        end    
+
+        data{i}.width    = size(img_region, 2);
+        data{i}.height   = size(img_region, 1);
+    end
+end
+
+function shape = loadshape(path)
+    % function: load shape from pts file
+    file = fopen(path);
+
+    if ~isempty(strfind(path, 'COFW'))
+        shape = textscan(file, '%d16 %d16 %d8', 'HeaderLines', 3, 'CollectOutput', 3);
+    else
+        shape = textscan(file, '%d16 %d16', 'HeaderLines', 3, 'CollectOutput', 2);
+    end
+    fclose(file);
+
+    shape = shape{1};
+end
+
+function region = enlargingbbox(bbox, scale)
+
+    region(1) = floor(bbox(1) - (scale - 1)/2*bbox(3));
+    region(2) = floor(bbox(2) - (scale - 1)/2*bbox(4));
+
+    region(3) = floor(scale*bbox(3));
+    region(4) = floor(scale*bbox(4));
+
+end

resetshape.m

@@ -0,0 +1,22 @@
+function [shape_initial] = resetshape(bbox, shape_union)
+%RESETSHAPE Summary of this function goes here
+%   Function: reset the initial shape according to the groundtruth shape and union shape for all faces
+%   Detailed explanation goes here
+%   Input: 
+%       bbox: bbounding box of groundtruth shape
+%       shape_union: uniionshape
+%   Output:
+%       shape_initial: reset initial shape
+%       bbox: bounding box of face image
+
+% get the bounding box according to the ground truth shape
+width_union = (max(shape_union(:, 1)) - min(shape_union(:, 1)));
+height_union = (max(shape_union(:, 2)) - min(shape_union(:, 2)));
+
+shape_union = bsxfun(@minus, (shape_union), (min(shape_union)));
+
+shape_initial = bsxfun(@times, shape_union, [(bbox(3)/width_union) (bbox(4)/height_union)]);
+shape_initial = bsxfun(@plus, shape_initial, double([bbox(1) bbox(2)]));
+
+end
+