1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
#!/usr/bin/env perl
# $Id: report-experiment-scores.perl 407 2008-11-10 14:43:31Z philipp $
use warnings;
use strict;
my $email;
my %TYPE;
$TYPE{"nist-bleu"} = "BLEU";
$TYPE{"multi-bleu"} = "BLEU";
$TYPE{"nist-bleu-c"} = "BLEU-c";
$TYPE{"multi-bleu-c"}= "BLEU-c";
$TYPE{"ibm-bleu"} = "IBM";
$TYPE{"ibm-bleu-c"} = "IBM-c";
$TYPE{"meteor"} = "METEOR";
$TYPE{"bolt-bleu"} = "BLEU";
$TYPE{"bolt-bleu-c"} = "BLEU-c";
$TYPE{"bolt-ter"} = "TER";
$TYPE{"bolt-ter-c"} = "TER-c";
my %SCORE;
my %AVERAGE;
foreach (@ARGV) {
if (/^email='(\S+)'/) {
$email = $1;
}
}
foreach (@ARGV) {
if (/^set=(\S+),type=(\S+),file=(\S+)$/) {
&process($1,$2,$3);
}
}
foreach my $set (keys %SCORE) {
my $score = $SCORE{$set};
chop($score);
print "$set: $score\n";
}
if ((scalar keys %SCORE) > 1) {
print "avg:";
my $first = 1;
foreach my $type (sort keys %AVERAGE) {
print " ;" unless $first; $first = 0;
printf " %.02f $TYPE{$type}",$AVERAGE{$type}/(scalar keys %SCORE);
}
print "\n";
}
sub process {
my ($set,$type,$file) = @_;
$SCORE{$set} .= "; " if defined($SCORE{$set});
if (! -e $file) {
print STDERR "ERROR (score $type for set $set): file '$file' does not exist!\n";
}
elsif ($type eq 'nist-bleu' || $type eq 'nist-bleu-c') {
$SCORE{$set} .= &extract_nist_bleu($file,$type)." ";
}
elsif ($type eq 'ibm-bleu' || $type eq 'ibm-bleu-c') {
$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
}
elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c') {
$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
}
elsif ($type eq 'meteor') {
$SCORE{$set} .= &extract_meteor($file,$type)." ";
}
elsif ($type =~ /^bolt-(.+)$/) {
$SCORE{$set} .= &extract_bolt($file,$1)." ";
}
}
sub extract_nist_bleu {
my ($file,$type) = @_;
my ($bleu,$ratio);
foreach (`cat $file`) {
$bleu = $1*100 if /BLEU score = (\S+)/;
$ratio = int(1000*$1)/1000 if /length ratio: (\S+)/;
}
if (!defined($bleu)) {
print STDERR "ERROR (extract_nist_bleu): could not find BLEU score in file '$file'\n";
return "";
}
my $output = sprintf("%.02f ",$bleu);
$output .= sprintf("(%.03f) ",$ratio) if $ratio;
$AVERAGE{$type} += $bleu;
return $output.$TYPE{$type};
}
sub extract_ibm_bleu {
my ($file,$type) = @_;
my ($bleu,$ratio);
foreach (`cat $file`) {
$bleu = $1*100 if /BLEUr\dn4c?,(\S+)/;
$ratio = int(1000*(1/$1))/1000 if /Ref2SysLen,(\S+)/;
}
if (!$bleu) {
print STDERR "ERROR (extract_ibm_bleu): could not find BLEU score in file '$file'\n";
return "";
}
my $output = sprintf("%.02f ",$bleu);
$output .= sprintf("(%.03f) ",$ratio) if $ratio;
$AVERAGE{$type} += $bleu;
return $output.$TYPE{$type};
}
sub extract_multi_bleu {
my ($file,$type) = @_;
my ($bleu,$ratio);
foreach (`cat $file`) {
$bleu = $1 if /BLEU = (\S+), /;
$ratio = $1 if / ration?=(\S+),/;
}
my $output = sprintf("%.02f ",$bleu);
$output .= sprintf("(%.03f) ",$ratio) if $ratio;
$AVERAGE{"multi-bleu"} += $bleu;
return $output.$TYPE{$type};
}
sub extract_bolt {
my ($file,$type) = @_;
my $score;
foreach (`cat $file`) {
$score = $1 if $type eq 'bleu' && /Lowercase BLEU\s+([\d\.]+)/;
$score = $1 if $type eq 'bleu-c' && /Cased BLEU\s+([\d\.]+)/;
$score = $1 if $type eq 'ter' && /Lowercase TER\s+([\d\.]+)/;
$score = $1 if $type eq 'ter-c' && /Cased TER\s+([\d\.]+)/;
}
my $output = sprintf("%.02f ",$score*100);
$AVERAGE{"bolt-".$type} += $score*100;
return $output.$TYPE{"bolt-".$type};
}
sub extract_meteor {
my ($file,$type) = @_;
my ($meteor, $precision);
foreach (`cat $file`) {
$meteor = $1*100 if /Final score:\s*(\S+)/;
$precision = $1 if /Precision:\s*(\S+)/;
}
my $output = sprintf("%.02f ",$meteor);
$output .= sprintf("(%.03f) ",$precision) if $precision;
$AVERAGE{"meteor"} += $meteor;
return $output.$TYPE{$type};
}
|
