kenlm regression tests

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4284 1f5c12ca-751b-0410-a591-d2e778427230

22 files changed, 416 insertions, 1 deletions

diff --git a/regression-testing/MosesRegressionTesting.pm b/regression-testing/MosesRegressionTesting.pm
index 77d6e730c..5a5b977ba 100644
--- a/regression-testing/MosesRegressionTesting.pm
+++ b/regression-testing/MosesRegressionTesting.pm
@@ -5,7 +5,7 @@ use strict;
 # if your tests need a new version of the test data, increment this
 # and make sure that a moses-regression-tests-vX.Y is available for
 # download from statmt.org (redpony AT umd dot edu for more info)
-use constant TESTING_DATA_VERSION => '7';
+use constant TESTING_DATA_VERSION => '8';
 
 # find the data directory in a few likely locations and make sure
 # that it is the correct version
diff --git a/regression-testing/run-test-suite.perl b/regression-testing/run-test-suite.perl
index 2eabce02c..8a05e15bb 100755
--- a/regression-testing/run-test-suite.perl
+++ b/regression-testing/run-test-suite.perl
@@ -23,10 +23,15 @@ my @tests = qw (
   chart.target-syntax.ondisk
   chart.hierarchical
   chart.hierarchical-withsrilm
+  chart.hierarchical-withkenlm
   chart.hierarchical.ondisk
   phrase.basic-surface-only
   phrase.basic-surface-only-withirstlm
   phrase.basic-surface-only-withirstlm-binlm
+  phrase.basic-surface-only-withkenlm
+  phrase.basic-surface-only-withkenlm.bin
+  phrase.basic-lm-oov
+  phrase.basic-lm-oov-withkenlm
   phrase.ptable-filtering
   phrase.multi-factor
   phrase.multi-factor-drop
diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl
new file mode 100755
index 000000000..2f6e17607
--- /dev/null
+++ b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl
new file mode 100755
index 000000000..476ddf6e9
--- /dev/null
+++ b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini b/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini
new file mode 100644
index 000000000..d8c7694fb
--- /dev/null
+++ b/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini
@@ -0,0 +1,69 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+
+# mapping steps
+[mapping]
+0 T 0
+1 T 1
+
+# translation tables: source-factors, target-factors, number of scores, file 
+[ttable-file]
+6 0 0 5 ${MODEL_PATH}/hierarchical/phrase-table.0-0.1
+6 0 0 1 ${MODEL_PATH}/hierarchical/phrase-table.0-0.2
+
+# no generation models, no generation-file section
+
+# language models: type(srilm/irstlm), factors, order, file
+[lmodel-file]
+8 0 3 ${LM_PATH}/news.de.kenlm
+
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+
+
+# language model weights
+[weight-l]
+0.273687
+
+
+# translation model weights
+[weight-t]
+0.077197
+0.014533
+0.059406
+0.001891
+-0.098314
+0.131945
+
+# no generation models, no weight-generation section
+
+# word penalty
+[weight-w]
+-0.343026
+
+[cube-pruning-pop-limit]
+1000
+
+[non-terminals]
+X
+
+[search-algorithm]
+3
+
+[inputtype]
+3
+
+[max-chart-span]
+20
+1000
+
+[weight-d]
+1
diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt b/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt
new file mode 100644
index 000000000..142ac43e0
--- /dev/null
+++ b/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt
@@ -0,0 +1,5 @@
+Prague Stock Market falls to minus by the end of the trading day 
+after a sharp drop in the morning , the Prague Stock Market corrected its losses . 
+transactions with stocks from the Czech Energy Enterprise ( ÄŒEZ ) reached nearly half of the regular daily trading . 
+the Prague Stock Market immediately continued its fall from Monday at the beginning of Tuesday 's trading , when it dropped by nearly six percent . 
+this time the fall in stocks on Wall Street is responsible for the drop . 
diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt b/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt
new file mode 100644
index 000000000..c77f070ce
--- /dev/null
+++ b/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt
@@ -0,0 +1,15 @@
+TRANSLATION_1=Prag Stock Markt fällt , durch das Ende der Handel Tag
+TRANSLATION_2=nach einem steilen Rückgang der Morgen , dem Prager Stock Markt korrigiert ihre Verluste .
+TRANSLATION_3=Transaktionen mit Aktien von der tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der regelmäßigen täglich Handel .
+TRANSLATION_4=der Prager Stock Markt sofort ihren Sturz von Montag zu Beginn der Handel Tuesday , wenn es um beinahe sechs Prozent .
+TRANSLATION_5=dieses Mal den Rückgang der Aktien an der Wall Street verantwortlich ist für die fallen .
+LMLOAD_TIME ~ 1.000
+PTLOAD_TIME ~ 1.000
+PTLOAD_TIME ~ 83.000
+PTLOAD_TIME ~ 83.000
+SCORE_1 = -12.100
+SCORE_2 = -12.344
+SCORE_3 = -116.629
+SCORE_4 = -121.386
+SCORE_5 = -8.128
+TOTAL_WALLTIME ~ 105
diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl
new file mode 100755
index 000000000..2f6e17607
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl
new file mode 100755
index 000000000..476ddf6e9
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini
new file mode 100644
index 000000000..3c2558505
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini
@@ -0,0 +1,53 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+0.1
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+[lmodel-oov-feature]
+1
+
+
+[verbose]
+2
+
diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt
new file mode 100644
index 000000000..a505702d1
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt
new file mode 100644
index 000000000..974429d61
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+LMLOAD_TIME ~ 8.00
+PTLOAD_TIME ~ 9.00
+SCORE_1 = -14.843
+SCORE_2 = -152.932
+SCORE_3 = -277.957
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 28
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl
new file mode 100755
index 000000000..2f6e17607
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl
new file mode 100755
index 000000000..476ddf6e9
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini
new file mode 100644
index 000000000..25bd27173
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini
@@ -0,0 +1,49 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt
new file mode 100644
index 000000000..a505702d1
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt
new file mode 100644
index 000000000..44a0502c8
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+LMLOAD_TIME ~ 8.00
+PTLOAD_TIME ~ 9.00
+SCORE_1 = -14.843
+SCORE_2 = -153.032
+SCORE_3 = -278.157
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 28
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl
new file mode 100755
index 000000000..2f6e17607
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl
new file mode 100755
index 000000000..476ddf6e9
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini b/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini
new file mode 100644
index 000000000..25bd27173
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini
@@ -0,0 +1,49 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt
new file mode 100644
index 000000000..a505702d1
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt
new file mode 100644
index 000000000..44a0502c8
--- /dev/null
+++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+LMLOAD_TIME ~ 8.00
+PTLOAD_TIME ~ 9.00
+SCORE_1 = -14.843
+SCORE_2 = -153.032
+SCORE_3 = -278.157
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 28