m_prefs_pair_ret.c

/* 
   Copyright (c) 2008 - Chris Buckley. 

   Permission is granted for use and modification of this file for
   research, non-commercial purposes. 
*/

#include "common.h"
#include "sysfunc.h"
#include "trec_eval.h"
#include "functions.h"
#include "trec_format.h"

static int 
te_calc_prefs_pair_ret (const EPI *epi, const REL_INFO *rel_info,
			const RESULTS *results, const TREC_MEAS *tm,
			TREC_EVAL *eval);

/* See trec_eval.h for definition of TREC_MEAS */
TREC_MEAS te_meas_prefs_pair_ret =
   {"prefs_pair_ret",
     "   Average over doc pairs of preference ratio for that pair.\n\
    If a doc pair satisfies 3 preferences but fails 2 preferences (preferences\n\
    from 5 different users),  then the score for doc pair is 3/5.\n\
    Same as prefs_simp if there are no doc_pairs in multiple judgment groups.\n\
    For doc pref A>B, A and B must both be retrieved to be counted as either\n\
    fulfilled or possible.\n\
    For doc pref A>B, this includes implied preferences (only one of A or B\n\
    retrieved), and counts as failure if neither A nor B retrieved.\n\
    pref_*_ret measures should be used for dynamic collections but are\n\
    inferior in most other applications.\n\
    Assumes '-R prefs' or '-R qrels_prefs'\n",
    te_init_meas_s_double,
    te_calc_prefs_pair_ret,
    te_acc_meas_s,
    te_calc_avg_meas_s,
    te_print_single_meas_s_double,
    te_print_final_meas_s_double,
    NULL, -1
};

static int 
te_calc_prefs_pair_ret (const EPI *epi, const REL_INFO *rel_info,
			const RESULTS *results, const TREC_MEAS *tm,
			TREC_EVAL *eval)
{
    RESULTS_PREFS rp;
    long i, j;
    double sum = 0;
    long num_pairs = 0;

    if (UNDEF == form_prefs_counts (epi, rel_info, results, &rp))
	return (UNDEF);

    for (i = 0; i < rp.num_judged_ret; i++) {
	for (j = i+1; j < rp.num_judged_ret; j++) {
	    if (rp.pref_counts.array[i][j] ||
		rp.pref_counts.array[j][i]) {
		num_pairs++;
		sum += (double) rp.pref_counts.array[i][j] /
		    (double) (rp.pref_counts.array[i][j] +
			     rp.pref_counts.array[j][i]);
	    }
	}
    }

    if (num_pairs) {
        eval->values[tm->eval_index].value =
	    sum / (double) num_pairs;
    }
    return (1);
}