m_utility.c

/* 
   Copyright (c) 2008 - Chris Buckley. 

   Permission is granted for use and modification of this file for
   research, non-commercial purposes. 
*/
#include "common.h"
#include "sysfunc.h"
#include "trec_eval.h"
#include "functions.h"
#include "trec_format.h"

static int
te_calc_utility(const EPI * epi, const REL_INFO * rel_info,
                const RESULTS * results, const TREC_MEAS * tm,
                TREC_EVAL * eval);
static double utility_param_array[] = { 1.0, -1.0, 0.0, 0.0 };

static PARAMS default_utility_params = {
    NULL, sizeof(utility_param_array) / sizeof(utility_param_array[0]),
    &utility_param_array[0]
};

/* See trec_eval.h for definition of TREC_MEAS */
TREC_MEAS te_meas_utility = { "utility",
    "    Set utility measure\n\
    Set evaluation based on contingency table:\n\
                        relevant  nonrelevant\n\
       retrieved            a          b\n\
       nonretrieved         c          d\n\
    where  utility = p1 * a + p2 * b + p3 * c + p4 * d\n\
    and p1-4 are parameters (given on command line in that order).\n\
    Conceptually, each retrieved relevant doc is worth something positive to\n\
    a user, each retrieved nonrelevant doc has a negative worth, each \n\
    relevant doc not retrieved may have a negative worth, and each\n\
    nonrelevant doc not retrieved may have a (small) positive worth.\n\
    The overall measure is simply a weighted sum of these values.\n\
    If p4 is non-zero, then '-N num_docs_in_coll' may also be needed - the\n\
    standard results and rel_info files do not contain that information.\n\
    Default usage: -m utility.1.0,-1.0,0.0,0.0 ...\n\
    Warning: Current version summary evaluation averages over all topics;\n\
    it could be argued that simply summing is more useful (but not backward\n\
    compatible)\n",
    te_init_meas_s_double_p_double,
    te_calc_utility,
    te_acc_meas_s,
    te_calc_avg_meas_s,
    te_print_single_meas_s_double,
    te_print_final_meas_s_double_p,
    (void *) &default_utility_params, -1
};

static int
te_calc_utility(const EPI * epi, const REL_INFO * rel_info,
                const RESULTS * results, const TREC_MEAS * tm, TREC_EVAL * eval)
{
    double *params = (double *) tm->meas_params->param_values;
    RES_RELS rr;

    if (UNDEF == te_form_res_rels(epi, rel_info, results, &rr))
        return (UNDEF);

    if (tm->meas_params->num_params != 4) {
        fprintf(stderr,
                "trec_eval.calc_utility: improper number of coefficients\n");
        return (UNDEF);
    }

    eval->values[tm->eval_index].value =
        params[0] * rr.num_rel_ret +
        params[1] * (rr.num_ret - rr.num_rel_ret) +
        params[2] * (rr.num_rel - rr.num_rel_ret) +
        params[3] * (epi->num_docs_in_coll + rr.num_rel_ret - rr.num_ret -
                     rr.num_rel);
    return (1);
}