-
Notifications
You must be signed in to change notification settings - Fork 5
/
modd2_evaluate_all_sequences_rectified.m
150 lines (135 loc) · 7.01 KB
/
modd2_evaluate_all_sequences_rectified.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
%% MODD2 EVALUATION SCRIPT
%
% This function evaluates segmentation outputs of the given method on raw
% sequences from the Multi-modal Marine Obstacle Detection Dataset 2 (MODD2)
%
% Input parameters:
% dataset_path - a path to the MODD2 dataset root folder
% output_path - a path to the output root folder
% method_name - name of the given method
% rectified (bool) - teels us if the segmentation method was performed
% on the rectified images (1) or raw images (0)
% segmentation_colors (optional) - a 3x3 matrix of colors used to
% represent labels in the output mask.
% In first row should be the RGB
% representation of the sky component,
% second row corresponds to the RGB
% representation of the
% obstacles/environment component, while
% the last row corresponds to the RGB
% representation of the water component.
%
function modd2_evaluate_all_sequences_rectified(dataset_path, output_path, method_name, is_rectified, segmentation_colors)
if(nargin < 3)
error('Not enough input parameters');
end
%% Evaluation parameters
% On rectified dataset?
eval_params.rectified = 1; % 1 = true, 0 = false
% Minimum overlap between two obstacles
eval_params.minoverlap = 0.15; % (default: 0.02)
% Freezone area below the water-edge where obstacles are removed
eval_params.freezone = 0; % (default: 0.01)
% Minimum surface area of obstacles to be considered as a threat
eval_params.area_threshold = 5*5; % (default: 15x15)
% Original image size (height x width)
eval_params.img_size = [958, 1278];
% RGB encoding of the labels
if(nargin == 3)
eval_params.labels = [ 0, 255, 0; ... % Sky represented with green
0, 0, 0; ... % obstacles with black
255, 0, 0]; % and water with red color.
else
eval_params.labels = segmentation_colors;
end
%% Extreme conditions information
% Sequence IDs where sudden movement occurres
extreme.sequences_sudden_movement = [1, 11];
% Sequence IDs where sun glitter occurres
extreme.sequences_sun_glitter = [11, 12, 13, 14, 20, 21, 22, 23, 24, 25, 26, 28];
% Sequence IDs where environmental reflections occurr
extreme.sequences_env_reflections = [1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, 16, 18, 23, 27, 28];
%% Makedir for storing interim results and posprocessed segmentation masks
if(~exist(fullfile('results', method_name, 'postprocessing'), 'dir'))
mkdir(fullfile('results', method_name, 'postprocessing'));
end
if(~exist(fullfile('results', method_name, 'eval_results'), 'dir'))
mkdir(fullfile('results', method_name, 'eval_results'), 'dir');
end
%% Initializations
% Evaluation results cell. Each cell represents each own sequence
eval_results = cell(28,2);
% For storing special sequence tag
seq.special = [];
% Evaluation measures
total_rmse = [];
total_tp = 0;
total_fp = 0;
total_fn = 0;
seq.is_rectified = is_rectified;
%% Loop through all sequences...
for seq_num_id = 1 : 28
seq.id = seq_num_id;
%% Fill in sequence information/details
% Add special sequences tag if necessary
if(ismember(seq.id, extreme.sequences_sudden_movement))
seq.special = [seq.special, 1];
end
if(ismember(seq.id, extreme.sequences_sun_glitter))
seq.special = [seq.special, 2];
end
if(ismember(seq.id, extreme.sequences_env_reflections))
seq.special = [seq.special, 3];
end
% Get sequence details (name, start frame, end frame)
[seq.name, seq.start_frame, seq.end_frame] = get_seq_details(seq.id);
%% Set paths accordingly...
paths.dataset_path = fullfile(dataset_path);
% Path to frames of the current sequence
paths.frames = fullfile(dataset_path, 'video_data', seq.name, 'frames');
% Path to ground-truth of the current sequence
paths.ground_truth = fullfile(dataset_path, 'annotationsV2_rectified', seq.name, 'ground_truth');
% Path to segmentation results of the current sequence
paths.output = fullfile(output_path, sprintf('seq%02d', seq.id), method_name);
% Path to USV masks
paths.USV_parts_masks = fullfile(dataset_path, 'USV_parts_masks');
%% Perform evaluation on sequence seq_num_id
% Results for each frame are written in format:
% 1 2 3 4 5
% RMSE TP FP FN Special sequence tag
fprintf('Evaluation on sequence %02d started...\n', seq.id);
[output_results, output_detections] = perform_evaluation_on_sequence(paths, seq, eval_params);
%% Save interim results
% Save detections information for the processed sequence...
save(fullfile('results', method_name, 'postprocessing', sprintf('seq%02d_%s_rectified.mat', seq.id, method_name)), 'output_detections');
% Save output results of the processed sequence...
eval_results{seq.id, 1} = output_results;
eval_results{seq.id, 2} = seq.special;
save(fullfile('results', method_name, 'eval_results', sprintf('seq%02d_%s_rectified.mat', seq.id, method_name)), 'eval_results');
%% Print interim results
fprintf('Seq %02d done\n', seq.id);
tmp_rmse = mean(cell2mat(output_results(:,1)));
tmp_tp = sum(cell2mat(output_results(:,2)));
tmp_fp = sum(cell2mat(output_results(:,3)));
tmp_fn = sum(cell2mat(output_results(:,4)));
fprintf('RMSE: %f\nTotal TP: %d\nTotal FP: %d\nTotal FN: %d\n', tmp_rmse, tmp_tp, tmp_fp, tmp_fn);
%% Update total results
total_rmse = [total_rmse; cell2mat(output_results(:,1))];
total_tp = total_tp + tmp_tp;
total_fp = total_fp + tmp_fp;
total_fn = total_fn + tmp_fn;
end
fprintf('--- all done ---\n');
fprintf('**********************************\n');
fprintf('* Evaluation on RECTIFIED images *\n');
fprintf('* Method: %*.*s *\n', 22, 22, method_name);
fprintf('**********************************\n');
fprintf('* Total RMSE: %04.01f px *\n', mean(total_rmse(:)) * 958);
fprintf('* Total STD: %04.01f px *\n', std(total_rmse(:)) * 958);
fprintf('* Total STE: %04.01f px *\n', std(total_rmse(:)) / sqrt(length(total_rmse(:))) * 958);
fprintf('* Total TP: %04d *\n', total_tp);
fprintf('* Total FP: %05d *\n', total_fp);
fprintf('* Total FN: %04d *\n', total_fn);
fprintf('* F-measure: %04.01f %% *\n', ((2 * total_tp) / (2 * total_tp + total_fp + total_fn) * 100));
fprintf('**********************************\n');
end