Ereport.pl 1.01 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
#!/usr/bin/perl

die "Use: $0 eval_output qrels_file\n" unless @ARGV==2;

open(EOUT,$eout=shift) || die "Cannot open $eout: $!";
open(RELJ,$relj=shift) || die "Cannot open $relj: $!";

$_=<EOUT>;
die "$eout must start with a number!\n "unless /^[1-9][0-9]*\n/;
$ndocs=$_+0;

$qid=0;
$relj_str=<RELJ>;
$eout_str=<EOUT>;

while(!eof(RELJ) || !eof(EOUT)) {
  ++$qid;
  %dq=();
  $A=$B=$AB=0;
  $Ravg=$Pavg=0;

  while($relj_str =~ /^0*$qid\s+(\d+)/) {
    ++$A;
    $dq{$1+0}=1;
    last unless $relj_str=<RELJ>;
  }
  # Favg measure = 1/(a/Pavg+(1-a)/Ravg)
sub Favg { my $a=shift; $Pavg*$Ravg ? 1/($a/$Pavg+(1-$a)/$Ravg) : 0; }
  # F0    : a=0                 -- ignore precision
  # F5    : a=0.5
  # F1    : a=1                 -- ignore recall
  while($eout_str =~ /^$qid\s+(\d+)\s+(\d+\.\d+)/) {
    $B++;
    $AB++ if $dq{$1+0};
    $Ravg+=$AB;
    $Pavg+=$AB/$B;
    last unless $eout_str=<EOUT>;
  }
  next unless $A;

  $Ravg/=$B*$A if $B;
  $Pavg/=$B    if $B;

  write;
format=
@##### @#.#######
$qid, Favg(0.5)
.
}

exit 0;