Skip to content

Commit af29c4f

Browse files
pdollarpdollar
committed
MatlabAPI/CocoEval.m: adding keypoint evaluation code!!! (needs more testing)
1 parent c95a22b commit af29c4f

File tree

4 files changed

+122
-19
lines changed

4 files changed

+122
-19
lines changed

‎MatlabAPI/CocoEval.m

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
% recThrs - [0:.01:1] R=101 recall thresholds for evaluation
2121
% areaRng - [...] A=4 object area ranges for evaluation
2222
% maxDets - [1 10 100] M=3 thresholds on max detections per image
23-
% iouType - ['segm'] set iouType to 'segm' or 'bbox'
23+
% iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints'
2424
% useCats - [1] if true use category labels for evaluation
2525
% Note: iouType replaced the now DEPRECATED useSegm parameter.
2626
% Note: if useCats=0 category labels are ignored as in proposal scoring.
@@ -83,17 +83,25 @@
8383
end
8484

8585
methods
86-
function ev = CocoEval( cocoGt, cocoDt )
86+
function ev = CocoEval( cocoGt, cocoDt, iouType )
8787
% Initialize CocoEval using coco APIs for gt and dt.
8888
if(nargin>0), ev.cocoGt = cocoGt; end
8989
if(nargin>1), ev.cocoDt = cocoDt; end
9090
if(nargin>0), ev.params.imgIds = sort(ev.cocoGt.getImgIds()); end
9191
if(nargin>0), ev.params.catIds = sort(ev.cocoGt.getCatIds()); end
92+
if(nargin<3), iouType='segm'; end
9293
ev.params.iouThrs = .5:.05:.95;
9394
ev.params.recThrs = 0:.01:1;
94-
ev.params.areaRng = [0 1e5; 0 32; 32 96; 96 1e5].^2;
95-
ev.params.maxDets = [1 10 100];
96-
ev.params.iouType = 'segm';
95+
if( any(strcmp(iouType,{'bbox','segm'})) )
96+
ev.params.areaRng = [0 1e5; 0 32; 32 96; 96 1e5].^2;
97+
ev.params.maxDets = [1 10 100];
98+
elseif( strcmp(iouType,'keypoints') )
99+
ev.params.areaRng = [0 1e5; 32 96; 96 1e5].^2;
100+
ev.params.maxDets = 20;
101+
else
102+
error('unknown iouType: %s',iouType);
103+
end
104+
ev.params.iouType = iouType;
97105
ev.params.useCats = 1;
98106
end
99107

@@ -122,6 +130,9 @@ function evaluate( ev )
122130
f='bbox'; if(isempty(dt)), [dt(:).(f)]=deal(); end
123131
if(~isfield(dt,f)), s=MaskApi.toBbox([dt.segmentation]);
124132
for d=1:nDt(i), dt(d).(f)=s(d,:); end; end
133+
elseif( strcmp(p.iouType,'keypoints') )
134+
gtIg=[gt.ignore]|[gt.num_keypoints]==0;
135+
for g=1:nGt(i), gt(g).ignore=gtIg(g); end
125136
else
126137
error('unknown iouType: %s',p.iouType);
127138
end
@@ -186,10 +197,17 @@ function accumulate( ev )
186197
function summarize( ev )
187198
% Compute and display summary metrics for evaluation results.
188199
if(isempty(ev.eval)), error('Please run accumulate() first'); end
189-
k=100; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
190-
{1,':','small',k}, {1,':','medium',k}, {1,':','large',k},...
191-
{0,':','all',1}, {0,':','all',10}, {0,':','all',k},...
192-
{0,':','small',k}, {0,':','medium',k}, {0,':','large',k}};
200+
if( any(strcmp(ev.params.iouType,{'bbox','segm'})) )
201+
k=100; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
202+
{1,':','small',k}, {1,':','medium',k}, {1,':','large',k},...
203+
{0,':','all',1}, {0,':','all',10}, {0,':','all',k},...
204+
{0,':','small',k}, {0,':','medium',k}, {0,':','large',k}};
205+
elseif( strcmp(ev.params.iouType,'keypoints') )
206+
k=20; M={{1,':','all',k},{1,.50,'all',k}, {1,.75,'all',k},...
207+
{1,':','medium',k}, {1,':','large',k},...
208+
{0,':','all',k},{0,.50,'all',k}, {0,.75,'all',k},...
209+
{0,':','medium',k}, {0,':','large',k}};
210+
end
193211
k=length(M); ev.stats=zeros(1,k);
194212
for s=1:k, ev.stats(s)=summarize1(M{s}{:}); end
195213

@@ -349,10 +367,11 @@ function makeplot( rs, ps, outDir, nm )
349367
if(D>p.maxDets), D=p.maxDets; dt=dt(1:D); end
350368
% compute iou between each dt and gt region
351369
iscrowd = uint8([gt.iscrowd]);
352-
t=find(strcmp(p.iouType,{'segm','bbox'}));
370+
t=find(strcmp(p.iouType,{'segm','bbox','keypoints'}));
353371
if(t==1), g=[gt.segmentation]; elseif(t==2), g=cat(1,gt.bbox); end
354372
if(t==1), d=[dt.segmentation]; elseif(t==2), d=cat(1,dt.bbox); end
355-
ious=MaskApi.iou(d,g,iscrowd);
373+
if(t<=2), ious=MaskApi.iou(d,g,iscrowd); else
374+
ious=CocoEval.oks(gt,dt); end
356375
% attempt to match each (sorted) dt to each (sorted) gt
357376
gtm=zeros(T,G); gtIds=[gt.id]; gtIg=[gt.ignore];
358377
dtm=zeros(T,D); dtIds=[dt.id]; dtIg=zeros(T,D);
@@ -380,5 +399,40 @@ function makeplot( rs, ps, outDir, nm )
380399
dtImgIds=ones(1,D)*p.imgIds; gtImgIds=ones(1,G)*p.imgIds;
381400
e = {dtIds,gtIds,dtImgIds,gtImgIds,dtm,gtm,[dt.score],dtIg,gtIg};
382401
end
402+
403+
function o = oks( gt, dt )
404+
% Compute Object Keypoint Similarity (OKS) between objects.
405+
G=length(gt); D=length(dt); o=zeros(D,G); if(~D||~G), return; end
406+
% sigmas hard-coded for person class, will need params eventually
407+
sigmas=[.26 .25 .25 .35 .35 .79 .79 .72 .72 .62 ...
408+
.62 1.07 1.07 .87 .87 .89 .89]/10;
409+
vars=(sigmas*2).^2; k=length(sigmas); m=k*3; bb=cat(1,gt.bbox);
410+
% create bounds for ignore regions (double the gt bbox)
411+
x0=bb(:,1)-bb(:,3); x1=bb(:,1)+bb(:,3)*2;
412+
y0=bb(:,2)-bb(:,4); y1=bb(:,2)+bb(:,4)*2;
413+
% extract keypoint locations and visibility flags
414+
gKp=cat(1,gt.keypoints); assert(size(gKp,2)==m);
415+
dKp=cat(1,dt.keypoints); assert(size(dKp,2)==m);
416+
xg=gKp(:,1:3:m); yg=gKp(:,2:3:m); vg=gKp(:,3:3:m);
417+
xd=dKp(:,1:3:m); yd=dKp(:,2:3:m);
418+
% compute oks between each detection and ground truth object
419+
for d=1:D
420+
for g=1:G
421+
v=vg(g,:); x=xd(d,:); y=yd(d,:); k1=nnz(v);
422+
if( k1>0 )
423+
% measure the per-keypoint distance if keypoints visible
424+
dx=x-xg(g,:); dy=y-yg(g,:);
425+
else
426+
% measure minimum distance to keypoints in (x0,y0) & (x1,y1)
427+
dx=max(0,x0(g,:)-x)+max(0,x-x1(g,:));
428+
dy=max(0,y0(g,:)-y)+max(0,y-y1(g,:));
429+
end
430+
% use the distances to compute the oks
431+
e=(dx.^2+dy.^2)./vars/gt(g).area/2;
432+
if(k1>0), e=e(v>0); else k1=k; end
433+
o(d,g)=sum(exp(-e))/k1;
434+
end
435+
end
436+
end
383437
end
384438
end

‎MatlabAPI/evalDemo.m

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
%% Demo demonstrating the algorithm result formats for COCO
22

33
%% select results type for demo (either bbox or segm)
4-
type = {'segm','bbox'}; type = type{1}; % specify type here
4+
type = {'segm','bbox','keypoints'}; type = type{1}; % specify type here
55
fprintf('Running demo for *%s* results.\n\n',type);
66

77
%% initialize COCO ground truth api
8-
dataDir='../'; dataType='val2014';
9-
annFile=sprintf('%s/annotations/instances_%s.json',dataDir,dataType);
10-
if(~exist('cocoGt','var')), cocoGt=CocoApi(annFile); end
8+
dataDir='../'; prefix='instances'; dataType='val2014';
9+
if(strcmp(type,'keypoints')), prefix='person_keypoints'; end
10+
annFile=sprintf('%s/annotations/%s_%s.json',dataDir,prefix,dataType);
11+
cocoGt=CocoApi(annFile);
1112

1213
%% initialize COCO detections api
13-
resFile='%s/results/instances_%s_fake%s100_results.json';
14-
resFile=sprintf(resFile,dataDir,dataType,type);
14+
resFile='%s/results/%s_%s_fake%s100_results.json';
15+
resFile=sprintf(resFile,dataDir,prefix,dataType,type);
1516
cocoDt=cocoGt.loadRes(resFile);
1617

1718
%% visialuze gt and dt side by side
@@ -33,9 +34,8 @@
3334
if(0), f=fopen(resFile,'w'); fwrite(f,gason(res)); fclose(f); end
3435

3536
%% run COCO evaluation code (see CocoEval.m)
36-
cocoEval=CocoEval(cocoGt,cocoDt);
37+
cocoEval=CocoEval(cocoGt,cocoDt,type);
3738
cocoEval.params.imgIds=imgIds;
38-
cocoEval.params.iouType=type;
3939
cocoEval.evaluate();
4040
cocoEval.accumulate();
4141
cocoEval.summarize();

‎results/person_keypoints_val2014_fakekeypoints100_results.json

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

‎results/val2014_fake_eval_res.txt

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
------------------------------------------------------------------------------
2+
type=segm
3+
Running per image evaluation... DONE (t=0.45s).
4+
Accumulating evaluation results... DONE (t=0.08s).
5+
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.320
6+
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.562
7+
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.299
8+
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.387
9+
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.310
10+
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.327
11+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.268
12+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.415
13+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.417
14+
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.469
15+
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.377
16+
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.381
17+
18+
------------------------------------------------------------------------------
19+
type=bbox
20+
Running per image evaluation... DONE (t=0.34s).
21+
Accumulating evaluation results... DONE (t=0.08s).
22+
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.505
23+
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.697
24+
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.573
25+
Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.586
26+
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.519
27+
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.501
28+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.387
29+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.594
30+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.595
31+
Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.640
32+
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.566
33+
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.564
34+
35+
------------------------------------------------------------------------------
36+
type=keypoints
37+
Running per image evaluation... DONE (t=0.06s).
38+
Accumulating evaluation results... DONE (t=0.00s).
39+
Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.372
40+
Average Precision (AP) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.636
41+
Average Precision (AP) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.348
42+
Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.384
43+
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.386
44+
Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 20 ] = 0.514
45+
Average Recall (AR) @[ IoU=0.50 | area= all | maxDets= 20 ] = 0.734
46+
Average Recall (AR) @[ IoU=0.75 | area= all | maxDets= 20 ] = 0.504
47+
Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets= 20 ] = 0.508
48+
Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets= 20 ] = 0.522

0 commit comments

Comments
 (0)