sacre bleu

2 files changed, 49 insertions, 7 deletions

diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta
index c61165897..1b2df0f33 100644
--- a/scripts/ems/experiment.meta
+++ b/scripts/ems/experiment.meta
@@ -1514,6 +1514,23 @@ multi-bleu-c-detok
         template: $multi-bleu-c-detok IN1 < IN > OUT
         final-model: yes
 
+sacre-bleu
+        in: detokenized-output raw-reference
+        out: sacre-bleu-score
+        default-name: evaluation/sacre-bleu
+        ignore-unless: sacre-bleu
+        rerun-on-change: sacre-bleu
+        template: $sacre-bleu IN1 < IN > OUT
+        final-model: yes
+sacre-bleu-c
+        in: detokenized-output raw-reference
+        out: sacre-bleu-c-score
+        default-name: evaluation/sacre-bleu-c
+        ignore-unless: sacre-bleu-c
+        rerun-on-change: sacre-bleu-c
+        template: $sacre-bleu-c IN1 < IN > OUT
+        final-model: yes
+
 ter
 	in: wrapped-output reference-sgm
 	out: ter-score
@@ -1705,6 +1722,6 @@ train
 
 [REPORTING] single
 report
-	in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:multi-bleu-detok-score EVALUATION:multi-bleu-c-detok-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
+	in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:multi-bleu-detok-score EVALUATION:multi-bleu-c-detok-score EVALUATION:sacre-bleu-score EVALUATION:sacre-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
 	out: report
 	default-name: evaluation/report
diff --git a/scripts/ems/support/report-experiment-scores.perl b/scripts/ems/support/report-experiment-scores.perl
index c859508cb..de15fb82a 100755
--- a/scripts/ems/support/report-experiment-scores.perl
+++ b/scripts/ems/support/report-experiment-scores.perl
@@ -22,10 +22,12 @@ $TYPE{"bolt-bleu"}     = "BLEU";
 $TYPE{"bolt-bleu-c"}   = "BLEU-c";
 $TYPE{"bolt-ter"}      = "TER";
 $TYPE{"bolt-ter-c"}    = "TER-c";
-
 $TYPE{"multi-bleu-detok"}  = "BLEU";
 $TYPE{"multi-bleu-c-detok"}= "BLEU-c";
 
+$TYPE{"sacre-bleu"}  = "BLEU";
+$TYPE{"sacre-bleu-c"}= "BLEU-c";
+
 my %SCORE;
 my %AVERAGE;
 foreach (@ARGV) {
@@ -57,26 +59,49 @@ sub process {
     my ($set,$type,$file) = @_;
     $SCORE{$set} .= "; " if defined($SCORE{$set});
     if (! -e $file) {
-	print STDERR "ERROR (score $type for set $set): file '$file' does not exist!\n";
+	    print STDERR "ERROR (score $type for set $set): file '$file' does not exist!\n";
     }
     elsif ($type eq 'nist-bleu' || $type eq 'nist-bleu-c') {
-	$SCORE{$set} .= &extract_nist_bleu($file,$type)." ";
+	    $SCORE{$set} .= &extract_nist_bleu($file,$type)." ";
     }
     elsif ($type eq 'ibm-bleu' || $type eq 'ibm-bleu-c') {
-	$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
+	    $SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
     }
     elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c'
 	|| $type eq 'multi-bleu-detok' || $type eq 'multi-bleu-c-detok') {
-	$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
+        $SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
+    }
+    elsif ($type eq 'sacre-bleu' || $type eq 'sacre-bleu-c') {
+        $SCORE{$set} .= &extract_sacre_bleu($file,$type)." ";
     }
     elsif ($type eq 'meteor') {
-	$SCORE{$set} .= &extract_meteor($file,$type)." ";
+	    $SCORE{$set} .= &extract_meteor($file,$type)." ";
     }
     elsif ($type =~ /^bolt-(.+)$/) {
       $SCORE{$set} .= &extract_bolt($file,$1)." ";
     }
 }
 
+sub extract_sacre_bleu {
+    my ($file,$type) = @_;
+    my ($bleu,$ratio);
+    #print STDERR "type=$type, file=$file \n";
+
+    foreach (my $line = `cat $file`) {
+        chomp($line);
+        #print STDERR "line=$line \n";
+        my @toks = split(" ", $line);
+        $bleu = $toks[2];
+        $ratio = $toks[9];
+    }
+    my $output = sprintf("%.02f ",$bleu);
+    $output .= sprintf("(%.03f) ",$ratio) if $ratio;
+
+    $AVERAGE{"sacre-bleu"} += $bleu;
+
+    return $output.$TYPE{$type};
+}
+
 sub extract_nist_bleu {
     my ($file,$type) = @_;
     my ($bleu,$ratio);