From 3cf75cb54dc95f34721e2896abf7e447515ade7c Mon Sep 17 00:00:00 2001 From: Karan Desai Date: Wed, 25 Jul 2018 16:17:28 -0400 Subject: [PATCH 1/2] Fix bug in evaluation for tie in scores. --- eval_utils/rank_answerer.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/eval_utils/rank_answerer.py b/eval_utils/rank_answerer.py index 9b4b036..d666290 100644 --- a/eval_utils/rank_answerer.py +++ b/eval_utils/rank_answerer.py @@ -20,14 +20,29 @@ def rankOptions(options, gtOptions, scores): '''Rank a batch of examples against a list of options.''' + # sort in descending order - largest score gets highest rank + sortedRanks, rankedIdx = scores.sort(1, descending=True) + + # convert from rankedIdx to ranks + ranks = rankedIdx.clone().fill_(0) + for i in range(rankedIdx.size(0)): + for j in range(100): + ranks[i][rankedIdx[i][j]] = j + ranks += 1 + + gtOptions = gtOptions.view(-1) numOptions = options.size(1) - # Compute score of GT options in 'scores' - gtScores = scores.gather(1, gtOptions.unsqueeze(1)) - # Sort all predicted scores - sortedScore, _ = torch.sort(scores, 1) - # In sorted scores, count how many are greater than the GT score - ranks = torch.sum(sortedScore.gt(gtScores).float(), 1) - return ranks + 1 + ranks = ranks.view(-1, numOptions) + gtRanks = torch.LongTensor(gtOptions.size(0)) + + for i in range(gtOptions.size(0)): + gtBinary = torch.zeros(numOptions) + gtBinary[gtOptions[i]] = 1 + sortedGt = gtBinary.index_select(0, ranks[i].sort()[1].cpu()) + gtRank = (sortedGt == 1).nonzero() + 1 + gtRanks[i] = int(gtRank) # gtRank is 1x1 LongTensor + + return gtRanks def rankABot(aBot, dataset, split, scoringFunction, exampleLimit=None): From 16480a08402313145b201208198c8239114ce417 Mon Sep 17 00:00:00 2001 From: Karan Desai Date: Thu, 26 Jul 2018 10:53:46 -0400 Subject: [PATCH 2/2] Simplify rank computation. --- eval_utils/rank_answerer.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/eval_utils/rank_answerer.py b/eval_utils/rank_answerer.py index d666290..b2f94cb 100644 --- a/eval_utils/rank_answerer.py +++ b/eval_utils/rank_answerer.py @@ -31,16 +31,9 @@ def rankOptions(options, gtOptions, scores): ranks += 1 gtOptions = gtOptions.view(-1) - numOptions = options.size(1) - ranks = ranks.view(-1, numOptions) gtRanks = torch.LongTensor(gtOptions.size(0)) - for i in range(gtOptions.size(0)): - gtBinary = torch.zeros(numOptions) - gtBinary[gtOptions[i]] = 1 - sortedGt = gtBinary.index_select(0, ranks[i].sort()[1].cpu()) - gtRank = (sortedGt == 1).nonzero() + 1 - gtRanks[i] = int(gtRank) # gtRank is 1x1 LongTensor + gtRanks[i] = int(ranks[i, gtOptions[i]]) return gtRanks