diff options
Diffstat (limited to 'scripts/analysis/sentence-by-sentence.pl')
| -rwxr-xr-x | scripts/analysis/sentence-by-sentence.pl | 247 |
1 files changed, 164 insertions, 83 deletions
diff --git a/scripts/analysis/sentence-by-sentence.pl b/scripts/analysis/sentence-by-sentence.pl index d5ca9879c..47e4ef63d 100755 --- a/scripts/analysis/sentence-by-sentence.pl +++ b/scripts/analysis/sentence-by-sentence.pl @@ -7,32 +7,39 @@ use strict; use Getopt::Long; my $sourcefile = undef; +my @truthfiles; GetOptions( - "source=s" => \$sourcefile, + "source|s=s" => \$sourcefile, + "reference|r=s" => \@truthfiles ) or exit(1); -my ($sysoutfile, @truthfiles) = @ARGV; - -if (!defined $sysoutfile || scalar(@truthfiles) == 0) { - print STDERR " -usage: $0 system_output reference(s...) > sentence-by-sentence.html +my @sysoutfiles = @ARGV; +if (scalar(@sysoutfiles) == 0 || scalar(@truthfiles) == 0) +{ + print STDERR "usage: $0 system_output(s) > sentence-by-sentence.html Options: - --source STRING ... foreign original + --source,-s STRING foreign input (can be used multiple times) + --reference,-r STRING English truth (can be used multiple times) N-grams are colored by the number of supporting references: - red for fewest, green for most, mediate shades otherwise. -"; + red for fewest, green for most, mediate shades otherwise.\n"; exit(1); } +#################################################################################################################### + my @TRUTHS = () x scalar(@truthfiles); for(my $i = 0; $i < scalar(@truthfiles); $i++) { open($TRUTHS[$i], "<$truthfiles[$i]") or die "couldn't open '$truthfiles[$i]' for read: $!\n"; binmode($TRUTHS[$i], ":utf8"); } -open(SYSOUT, "<$sysoutfile") or die "couldn't open '$sysoutfile' for read: $!\n"; -binmode(SYSOUT, ":utf8"); +my @SYSOUTS = () x scalar(@sysoutfiles); +for(my $i = 0; $i < scalar(@sysoutfiles); $i++) +{ + open($SYSOUTS[$i], "<$sysoutfiles[$i]") or die "couldn't open '$sysoutfiles[$i]' for read: $!\n"; + binmode($SYSOUTS[$i], ":utf8"); +} binmode(STDOUT, ":utf8"); if (defined $sourcefile) { @@ -40,63 +47,64 @@ if (defined $sourcefile) binmode(SOURCE, ":utf8"); } my @bleuScores; +for(my $i = 0; $i < scalar(@sysoutfiles); $i++) {push @bleuScores, [];} my @htmlSentences; +my @javascripts; my @htmlColors = ('#99ff99', '#aaaaff', '#ffff99', '#ff9933', '#ff9999'); #color sentences by rank (split in n tiers) my $ngramSingleRefColor = '#aaffaa'; my @ngramMultirefColors = ('#ff9999', '#ff9933', '#ffff99', '#a0a0ff', '#99ff99'); #arbitrary-length list; first entry is used for worst n-grams -my $i = 0; -while(my $sLine = <SYSOUT>) +my $numSentences = 0; +my (@sLines, @eLines); +while(readLines(\@SYSOUTS, \@sLines) && readLines(\@TRUTHS, \@eLines)) { - escapeMetachars($sLine); #remove inconsistencies in encoding - my @sFactors = @{extractFactorArrays($sLine)}; - my @eLines = () x scalar(@truthfiles); - my @eFactors; - for(my $j = 0; $j < scalar(@truthfiles); $j++) - { - my $fh = $TRUTHS[$j]; - $eLines[$j] = <$fh>; - escapeMetachars($eLines[$j]); #remove inconsistencies in encoding - push @eFactors, extractFactorArrays($eLines[$j], "$truthfiles[$j] shorter than $sysoutfile"); - } - my $sourceFactors; + #create array of lines of HTML + my @html = ("<div class=\"sentence_%%%%\" id=\"sentence$numSentences\">"); #%%%% is a flag to be replaced + + my (@sFactors, @eFactors, $sourceFactors); + #process source if (defined $sourcefile) { my $sourceLine = <SOURCE>; escapeMetachars($sourceLine); #remove inconsistencies in encoding - $sourceFactors = extractFactorArrays($sourceLine, "$sourcefile shorter than $sysoutfile"); + $sourceFactors = extractFactorArrays($sourceLine); + push @html, "<tr><td class=\"sent_title\">Source</td><td class=\"source_sentence\" id=\"source$numSentences\">" + . getFactoredSentenceHTML($sourceFactors) . "</td></tr>\n"; } - - my $bleuData = getBLEUSentenceDetails(\@sFactors, \@eFactors, 0); - push @bleuScores, [$i, $bleuData->[0], 0]; #the last number will be the rank - my $pwerData = getPWERSentenceDetails(\@sFactors, \@eFactors, 0); - my $html = "<div class=\"sentence\" style=\"background-color: %%%%\" id=\"sentence$i\">"; #the %%%% and other tokens like it are flags to be replaced - $html .= "<div class=\"bleu_report\"><b>Sentence $i) BLEU:</b> " . sprintf("%.4lg", $bleuData->[0]->[0]) . " (" . join('/', map {sprintf("%.4lg", $_)} @{$bleuData->[0]}[1 .. 4]) . ")</div><table>\n"; - if(defined $sourcefile) + #process truth + for(my $j = 0; $j < scalar(@truthfiles); $j++) { - $html .= "<tr><td class=\"sent_title\">Source</td><td class=\"source_sentence\" id=\"source$i\">" . getFactoredSentenceHTML($sourceFactors) . "</td></tr>\n"; + escapeMetachars($eLines[$j]); #remove inconsistencies in encoding + push @eFactors, extractFactorArrays($eLines[$j]); + push @html, "<tr><td class=\"sent_title\">Ref $j</td><td class=\"truth_sentence\" id=\"truth${numSentences}_$j\">" + . getFactoredSentenceHTML($eFactors[$j]) . "</td></tr>\n"; } - for(my $j = 0; $j < scalar(@truthfiles); $j++) + #process sysouts + my @bleuData; + for(my $j = 0; $j < scalar(@sysoutfiles); $j++) { - $html .= "<tr><td class=\"sent_title\">Ref $j</td><td class=\"truth_sentence\" id=\"truth${i}_$j\">" . getFactoredSentenceHTML($eFactors[$j]) . "</td></tr>\n"; + escapeMetachars($sLines[$j]); #remove inconsistencies in encoding + push @sFactors, extractFactorArrays($sLines[$j]); + push @bleuData, getBLEUSentenceDetails($sFactors[$j], \@eFactors, 0); + push @{$bleuScores[$j]}, [$numSentences, $bleuData[$j]->[0], 0]; #the last number will be the rank + my $pwerData = getPWERSentenceDetails($sFactors[$j], \@eFactors, 0); + push @html, "<tr><td class=\"sent_title\">Output $j</td><td class=\"sysout_sentence\" id=\"sysout$numSentences\">" + . getFactoredSentenceHTML($sFactors[$j], $pwerData) . "</td></tr>\n"; + push @html, "<tr><td class=\"sent_title\">N-grams</td><td class=\"sysout_ngrams\" id=\"ngrams$numSentences\">" + . getAllNgramsHTML($sFactors[$j], $bleuData[$j]->[1], scalar(@truthfiles)) . "</td></tr>\n"; } - my $j = 0; - $html .= "<tr><td class=\"sent_title\">Output</td><td class=\"sysout_sentence\" id=\"sysout$i\">" . getFactoredSentenceHTML(\@sFactors, $pwerData) . "</td></tr>\n"; - $j = 0; - $html .= "<tr><td class=\"sent_title\">N-grams</td><td class=\"sysout_ngrams\" id=\"ngrams$i\">" . getAllNgramsHTML(\@sFactors, $bleuData->[1], scalar(@truthfiles)) . "</td></tr>\n"; - $html .= "</table></div>\n"; - push @htmlSentences, $html; - $i++; + splice(@html, 1, 0, "<div class=\"bleu_report\"><b>Sentence $numSentences) BLEU:</b> " + . join("; ", map {sprintf("%.4lg", $_->[0]->[0]) . " (" . join('/', map {sprintf("%.4lg", $_)} @{$_->[0]}[1 .. 4]) . ") "} @bleuData) . "</div><table>\n"); + push @html, "</table></div>\n"; + push @htmlSentences, join('', @html); + $numSentences++; + @sLines = (); @eLines = (); #clear writable arrays to be refilled } -close(SYSOUT); +foreach my $sysoutfh (@SYSOUTS) {close($sysoutfh);} foreach my $truthfh (@TRUTHS) {close($truthfh);} -rankSentencesByBLEU(\@bleuScores); -my $stylesheet = <<EOHTML; -<style type="text/css"> +my $stylesheet = "<style type=\"text/css\"> .legend {background: #fff; border: 1px solid #000; padding: 2px; margin-bottom: 10px; margin-right: 15px} .legend_title {font-weight: bold; font-size: medium; text-decoration: underline} -div.sentence {background: #ffffee; border: 1px solid #000088; padding: 0px 8px 0px 8px} //entire composition for a given sentence -div.sentence td {margin: 8px 0px 8px 0px} div.bleu_report {margin-bottom: 5px} td.sent_title {font-weight: bold; font-size: medium; margin-bottom: 12px} .source_sentence {background: #ffcccc; border: 1px solid #bbb} @@ -105,25 +113,67 @@ td.sent_title {font-weight: bold; font-size: medium; margin-bottom: 12px} table.sentence_table {border: none} .sysout_ngrams {background: #fff; border: 1px solid #bbb} table.ngram_table {} -td.ngram_cell {padding: 1px} -</style> -EOHTML -print "<html><head>\n"; -print "<meta http-equiv=\"Content-type: text/html; charset=utf-8\">\n"; -print "<title>$sysoutfile vs. [" . join(' ', @truthfiles) . "]: Sentence-by-sentence Comparison</title>$stylesheet</head><body>\n"; +td.ngram_cell {padding: 1px}\n"; +for(my $i = 0; $i < scalar(@htmlColors); $i++) +{ + $stylesheet .= ".sentence_tier$i {background: $htmlColors[$i]; border: 1px solid #000088; padding: 0px 8px 0px 8px} //entire composition for a given sentence\n"; + $stylesheet .= "div.sentence_tier$i td {margin: 8px 0px 8px 0px}\n"; +} +$stylesheet .= "</style>\n"; + +print "<html><head><meta http-equiv=\"Content-type: text/html; charset=utf-8\">\n"; +print "<title>[" . join(', ', @sysoutfiles) . "] vs. [" . join(', ', @truthfiles) . "]: Sentence-by-Sentence Comparison</title>$stylesheet</head><body>\n"; -#javascript to sort by BLEU, by order in corpus, ... -my %rank2index = map {$bleuScores[$_]->[2] => $_} (0 .. scalar(@htmlSentences) - 1); +foreach my $systemScores (@bleuScores) {rankSentencesByBLEU($systemScores);} +#javascript to sort by BLEU for any system, by order in corpus ... print "<script type=\"text/javascript\"> +var selectedSysout = 0; //index of system currently being used to rank/sort + +function selectSysout(index) +{ + //update the BLEU-range text shown in the legend + var legend = document.getElementById('legendBLEU'); + var rows = legend.getElementsByTagName('tr'); + for(var i = 0; i < rows.length; i++) + { + var cell = rows[i].childNodes[1]; + var spans = cell.getElementsByTagName('span'); + cell.childNodes[0].nodeValue = spans[index].firstChild.nodeValue; //something like '0.1 - 0.3' + } + + //update the background colors of the sentence divs + var allSentences = document.getElementById('all_sentences'); + var sentences = allSentences.childNodes; + for(var i = 0; i < sentences.length; i++) + { + if(typeof sentences[i].tagName != 'undefined' && sentences[i].tagName.toLowerCase() == 'div') //text nodes have undefined tagName + { + var tierSpans = sentences[i].firstChild.childNodes; + sentences[i].childNodes[2].className = tierSpans[index].firstChild.nodeValue; //something like 'tier3' + } + } + selectedSysout = index; //selectedSysout is a flag to the sort functions +} + function sortByBLEU() { - var body = document.getElementById('all_sentences'); var row;\n"; -foreach my $rank (sort {$a <=> $b} keys %rank2index) + var body = document.getElementById('all_sentences'); var row; + switch(selectedSysout) + {\n"; +for(my $i = 0; $i < scalar(@sysoutfiles); $i++) { - print "\trow = document.getElementById('everything" . $rank2index{$rank} . "');\n"; - print "\tbody.removeChild(row); body.appendChild(row);\n"; + print "case $i: + {"; + my %rank2index = map {$bleuScores[$i]->[$_]->[2] => $_} (0 .. scalar(@htmlSentences) - 1); + foreach my $rank (sort {$a <=> $b} keys %rank2index) + { + print "\trow = document.getElementById('everything" . $rank2index{$rank} . "');\n"; + print "\tbody.removeChild(row); body.appendChild(row);\n"; + } + print "break;}\n"; } print "} +} function sortByCorpusOrder() { var body = document.getElementById('all_sentences'); var row;\n"; @@ -135,42 +185,57 @@ for(my $j = 0; $j < scalar(@htmlSentences); $j++) print "} </script>\n"; -#legend for background colors -my @minBLEU = (1e9) x scalar(@htmlColors); -my @maxBLEU = (-1e9) x scalar(@htmlColors); -for(my $k = 0; $k < scalar(@htmlSentences); $k++) +#legends for background colors of sentences and n-grams +my (@minBLEU, @maxBLEU); +my @bleuTiers = () x scalar(@htmlSentences); #for each sentence, arrayref of tier indices for each system +for(my $i = 0; $i < scalar(@sysoutfiles); $i++) { - my $tier = int($bleuScores[$k]->[2] / (scalar(@htmlSentences) / scalar(@htmlColors))); - if($bleuScores[$k]->[1]->[0] < $minBLEU[$tier]) {$minBLEU[$tier] = $bleuScores[$k]->[1]->[0];} - elsif($bleuScores[$k]->[1]->[0] > $maxBLEU[$tier]) {$maxBLEU[$tier] = $bleuScores[$k]->[1]->[0];} + my @a = (1e9) x scalar(@htmlColors); + my @b = (-1e9) x scalar(@htmlColors); + for(my $k = 0; $k < scalar(@htmlSentences); $k++) + { + my $tier = int($bleuScores[$i]->[$k]->[2] / (scalar(@htmlSentences) / scalar(@htmlColors))); + push @{$bleuTiers[$k]}, $tier; + if($bleuScores[$i]->[$k]->[1]->[0] < $a[$tier]) {$a[$tier] = $bleuScores[$i]->[$k]->[1]->[0];} + if($bleuScores[$i]->[$k]->[1]->[0] > $b[$tier]) {$b[$tier] = $bleuScores[$i]->[$k]->[1]->[0];} + } + push @minBLEU, \@a; + push @maxBLEU, \@b; } -print "<table border=0><tr><td><div class=\"legend\"><span class=\"legend_title\">Sentence Background Colors => BLEU Ranges</span><table border=0>"; +print "<table border=0><tr><td><div id=\"legendBLEU\" class=\"legend\"><span class=\"legend_title\">Sentence Background Colors => BLEU Ranges</span><table border=0>"; for(my $k = 0; $k < scalar(@htmlColors); $k++) { print "<tr><td style=\"width: 15px; height: 15px; background: " . $htmlColors[$k] . "\"></td><td align=left style=\"padding-left: 12px\">" - . sprintf("%.4lg", $minBLEU[$k]) . " - " . sprintf("%.4lg", $maxBLEU[$k]) . "</td>"; + . sprintf("%.4lg", $minBLEU[0]->[$k]) . " - " . sprintf("%.4lg", $maxBLEU[0]->[$k]); + for(my $j = 0; $j < scalar(@sysoutfiles); $j++) + { + print "<span style=\"display: none\">" . sprintf("%.4lg", $minBLEU[$j]->[$k]) . " - " . sprintf("%.4lg", $maxBLEU[$j]->[$k]) . "</span>"; + } + print "</td></tr>"; } print "</table></div></td>\n"; print "<td><div class=\"legend\"><span class=\"legend_title\">N-gram Colors => Number of Matching Reference Translations</span><table border=0>"; for(my $k = 1; $k <= scalar(@truthfiles); $k++) { - print "<tr><td style=\"width: 15px; height: 15px; background: " . getNgramColorHTML($k, scalar(@truthfiles)) . "\"></td><td align=left style=\"padding-left: 12px\">$k</td>"; + print "<tr><td style=\"width: 15px; height: 15px; background: " . getNgramColorHTML($k, scalar(@truthfiles)) . "\"></td><td align=left style=\"padding-left: 12px\">$k</td></tr>"; } print "</table></div></td></tr></table><div style=\"font-weight: bold; margin-bottom: 15px\"> PWER errors are marked in red on output sentence displays.</div> +<div style=\"margin-bottom: 8px\">Color by system # " + . join(' | ', map {"<a href=\"javascript:selectSysout($_);\">$_</a>" . (($_ == '0') ? " (default)" : "")} (0 .. scalar(@sysoutfiles) - 1)) . "</div> <div style=\"margin-bottom: 8px\">Sort by <a href=\"javascript:sortByBLEU();\">BLEU score</a> | <a href=\"javascript:sortByCorpusOrder();\">corpus order</a> (default)</div>\n"; #sentence boxes print "<div id=\"all_sentences\">"; -my $j = 0; -foreach my $sentenceHTML (@htmlSentences) +for(my $j = 0; $j < scalar(@htmlSentences); $j++) { print "<div id=\"everything$j\" style=\"margin: 0px; padding: 0px\">"; + print "<div class=\"ranks_container\" style=\"display: none\">" . join('', map {"<span>sentence_tier$_</span>"} @{$bleuTiers[$j]}) . "</div>"; print "<hr width=98%>"; - my $bgcolor = getSentenceBGColorHTML($bleuScores[$j], $i); #i is now # of sentences - $sentenceHTML =~ s/%%%%/$bgcolor/; - print "$sentenceHTML</div>\n"; - $j++; +# my $bgcolor = getSentenceBGColorHTML($bleuScores[0]->[$j], $i); #i is now # of sentences + my $tierNum = $bleuTiers[$j]->[0]; + $htmlSentences[$j] =~ s/%%%%/tier$tierNum/; + print "$htmlSentences[$j]</div>\n"; } print "</div></body></html>"; @@ -211,7 +276,7 @@ sub round } #escape HTML metacharacters for display purposes and to allow for consistent string comparison -#arguments: string to be formatted +#arguments: string to be formatted in place #return: none sub escapeMetachars { @@ -223,13 +288,27 @@ sub escapeMetachars ############################################################################################################################################################### -#arguments: line read from corpus file, (optionally) string to die with if line isn't defined (default die-msg is empty) +#read one line from each of any number of filehandles +#arguments: arrayref of filehandles, (empty) arrayref to be filled with read lines +#return: 1 on success, 0 on failure (on failure the lines arrayref's value isn't defined) +sub readLines +{ + my ($refFilehandles, $refLines) = @_; + foreach my $fh (@$refFilehandles) + { + my $line = <$fh>; + return 0 unless defined($line); + push @$refLines, $line; + } + return 1; +} + +#arguments: line read from corpus file #return: sentence (arrayref of arrayrefs of factor strings) taken from line sub extractFactorArrays { - my ($line, $msg) = (shift, ''); - $msg = shift if scalar(@_); - die $msg if !defined $line; + my $line = shift; + die "" if !defined $line; chomp $line; $line =~ s/^\s*|\s*$//g; #added by Ondrej to handle moses-mert-parallel output my @words = split(/\s+/, $line); @@ -284,15 +363,17 @@ sub getBLEUSentenceDetails { $correct[$k]++; my @indices = (); + my $notOvercounting = 0; #make sure we don't 'match' against truth n-grams whose instances have all been used already for(my $m = 0; $m < scalar(@{$REF_GRAM{$gram}}); $m++) { if($REF_GRAM{$gram}->[$m] > 0) { push @indices, $m; $REF_GRAM{$gram}->[$m]--; + $notOvercounting = 1; } } - push @$ngramMatches, [$i - $k, $k + 1, \@indices]; + if($notOvercounting == 1) {push @$ngramMatches, [$i - $k, $k + 1, \@indices];} } } } |