list frequent mismatched tokenizations first

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@2852 1f5c12ca-751b-0410-a591-d2e778427230

1 files changed, 5 insertions, 1 deletions

diff --git a/scripts/analysis/suspicious_tokenization.pl b/scripts/analysis/suspicious_tokenization.pl
index f7ca3c60d..29e32d271 100755
--- a/scripts/analysis/suspicious_tokenization.pl
+++ b/scripts/analysis/suspicious_tokenization.pl
@@ -49,10 +49,14 @@ foreach my $ngr (keys %$ngrams) {
   $report->{$ngr}->{"tok"} = $tokcnt;
   $report->{$ngr}->{"untok"} = $untokcnt;
   $report->{$ngr}->{"diff"} = abs($untokcnt-$tokcnt);
+  $report->{$ngr}->{"sum"} = $untokcnt+$tokcnt;
 }
 
 # Report
-foreach my $ngr (sort {$report->{$a}->{"diff"} <=> $report->{$b}->{"diff"}}
+foreach my $ngr (sort {
+                         $report->{$a}->{"diff"} <=> $report->{$b}->{"diff"}
+                      || $report->{$b}->{"sum"} <=> $report->{$a}->{"sum"}
+                      }
                   keys %$report) {
   print "$ngr\t$report->{$ngr}->{untok}\t$report->{$ngr}->{tok}\t$report->{$ngr}->{diff}\n";
 }