Merge /home/hieu/workspace/github/moses-regression-tests

71 files changed, 1311 insertions, 0 deletions

diff --git a/tests/phrase.basic-lm-oov.oldformat/filter-stderr.pl b/tests/phrase.basic-lm-oov.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.basic-lm-oov.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.basic-lm-oov.oldformat/filter-stdout.pl b/tests/phrase.basic-lm-oov.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.basic-lm-oov.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.basic-lm-oov.oldformat/moses.ini b/tests/phrase.basic-lm-oov.oldformat/moses.ini
new file mode 100644
index 0000000..3c25585
--- /dev/null
+++ b/tests/phrase.basic-lm-oov.oldformat/moses.ini
@@ -0,0 +1,53 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+0.1
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+[lmodel-oov-feature]
+1
+
+
+[verbose]
+2
+
diff --git a/tests/phrase.basic-lm-oov.oldformat/to-translate.txt b/tests/phrase.basic-lm-oov.oldformat/to-translate.txt
new file mode 100644
index 0000000..a505702
--- /dev/null
+++ b/tests/phrase.basic-lm-oov.oldformat/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.basic-lm-oov.oldformat/truth/results.txt b/tests/phrase.basic-lm-oov.oldformat/truth/results.txt
new file mode 100644
index 0000000..b3b2e17
--- /dev/null
+++ b/tests/phrase.basic-lm-oov.oldformat/truth/results.txt
@@ -0,0 +1,11 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+SCORE_1 = -14.843
+SCORE_2 = -152.932
+SCORE_3 = -277.957
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 8
diff --git a/tests/phrase.basic-surface-binptable.oldformat/filter-stderr.pl b/tests/phrase.basic-surface-binptable.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.basic-surface-binptable.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.basic-surface-binptable.oldformat/filter-stdout.pl b/tests/phrase.basic-surface-binptable.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.basic-surface-binptable.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.basic-surface-binptable.oldformat/moses.ini b/tests/phrase.basic-surface-binptable.oldformat/moses.ini
new file mode 100644
index 0000000..cedc536
--- /dev/null
+++ b/tests/phrase.basic-surface-binptable.oldformat/moses.ini
@@ -0,0 +1,56 @@
+# Moses configuration file
+# automatic exodus from pharaoh.ini Wed Jul 12 18:24:14 EDT 2006
+
+###########################
+### PHARAOH CONFIG FILE ###
+###########################
+
+# phrase table f, n, p(n|f)
+[ttable-file]
+1 0 0 5 ${MODEL_PATH}/basic-surface-binptable/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/tests/phrase.basic-surface-binptable.oldformat/to-translate.txt b/tests/phrase.basic-surface-binptable.oldformat/to-translate.txt
new file mode 100644
index 0000000..a505702
--- /dev/null
+++ b/tests/phrase.basic-surface-binptable.oldformat/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.basic-surface-binptable.oldformat/truth/results.txt b/tests/phrase.basic-surface-binptable.oldformat/truth/results.txt
new file mode 100644
index 0000000..3c04124
--- /dev/null
+++ b/tests/phrase.basic-surface-binptable.oldformat/truth/results.txt
@@ -0,0 +1,11 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+SCORE_1 = -14.843
+SCORE_2 = -153.032
+SCORE_3 = -278.157
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 23
diff --git a/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stderr.pl b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stdout.pl b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/moses.ini b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/moses.ini
new file mode 100644
index 0000000..271b8b3
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/moses.ini
@@ -0,0 +1,49 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+1 0 3 ${LM_PATH}/europarl.en.binlm
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/to-translate.txt b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/to-translate.txt
new file mode 100644
index 0000000..a505702
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/truth/results.txt b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/truth/results.txt
new file mode 100644
index 0000000..9d9f192
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm-binlm.oldformat/truth/results.txt
@@ -0,0 +1,11 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+SCORE_1 = -14.843
+SCORE_2 = -122.483
+SCORE_3 = -217.059
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 28
diff --git a/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stderr.pl b/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stdout.pl b/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.basic-surface-only-withirstlm.oldformat/moses.ini b/tests/phrase.basic-surface-only-withirstlm.oldformat/moses.ini
new file mode 100644
index 0000000..f4c6fee
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm.oldformat/moses.ini
@@ -0,0 +1,49 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+1 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/tests/phrase.basic-surface-only-withirstlm.oldformat/to-translate.txt b/tests/phrase.basic-surface-only-withirstlm.oldformat/to-translate.txt
new file mode 100644
index 0000000..a505702
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm.oldformat/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.basic-surface-only-withirstlm.oldformat/truth/results.txt b/tests/phrase.basic-surface-only-withirstlm.oldformat/truth/results.txt
new file mode 100644
index 0000000..2aae7cb
--- /dev/null
+++ b/tests/phrase.basic-surface-only-withirstlm.oldformat/truth/results.txt
@@ -0,0 +1,11 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways . 
+TRANSLATION_5=only then will the european institutions to its mandate . 
+SCORE_1 = -14.843
+SCORE_2 = -122.483
+SCORE_3 = -217.059
+SCORE_4 = -5.995
+SCORE_5 = -7.015
+TOTAL_WALLTIME ~ 30
diff --git a/tests/phrase.basic-surface-only.oldformat/filter-stderr.pl b/tests/phrase.basic-surface-only.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.basic-surface-only.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.basic-surface-only.oldformat/filter-stdout.pl b/tests/phrase.basic-surface-only.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.basic-surface-only.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.basic-surface-only.oldformat/moses.ini b/tests/phrase.basic-surface-only.oldformat/moses.ini
new file mode 100644
index 0000000..25bd271
--- /dev/null
+++ b/tests/phrase.basic-surface-only.oldformat/moses.ini
@@ -0,0 +1,49 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
diff --git a/tests/phrase.basic-surface-only.oldformat/to-translate.txt b/tests/phrase.basic-surface-only.oldformat/to-translate.txt
new file mode 100644
index 0000000..079abe9
--- /dev/null
+++ b/tests/phrase.basic-surface-only.oldformat/to-translate.txt
@@ -0,0 +1,6 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.basic-surface-only.oldformat/truth/results.txt b/tests/phrase.basic-surface-only.oldformat/truth/results.txt
new file mode 100644
index 0000000..88fa606
--- /dev/null
+++ b/tests/phrase.basic-surface-only.oldformat/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? 
+TRANSLATION_2=
+TRANSLATION_3=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . 
+TRANSLATION_4=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_5=parliament wants the in two ways . 
+TRANSLATION_6=only then will the european institutions to its mandate . 
+SCORE_1 = -14.843
+SCORE_2 = 0.000
+SCORE_3 = -153.032
+SCORE_4 = -278.157
+SCORE_5 = -5.995
+SCORE_6 = -7.015
+TOTAL_WALLTIME ~ 31
diff --git a/tests/phrase.compresspt.oldformat/filter-nbest.pl b/tests/phrase.compresspt.oldformat/filter-nbest.pl
new file mode 100755
index 0000000..4d34fe0
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/filter-nbest.pl
@@ -0,0 +1,15 @@
+#!/usr/bin/perl
+
+use strict;
+
+my $x=0;
+my $oldcode = "";
+while (<>) {
+  chomp;
+  my ($code, $trans, $featscores, $globscores, $align1, $align2 ) = split(/[\s]*\|\|\|[\s]*/,$_);
+  $x = 0 if $oldcode ne $code;
+  $x++;
+  chomp($code);
+  print "TRANSLATION_${code}_NBEST_${x}=$trans ||| $featscores ||| $globscores ||| $align1 ||| $align2\n";
+  $oldcode = $code;
+}
diff --git a/tests/phrase.compresspt.oldformat/filter-stderr.pl b/tests/phrase.compresspt.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..a359091
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  print "SCORE_$x = $pscore\n";
+  $x++;
+}
diff --git a/tests/phrase.compresspt.oldformat/filter-stdout.pl b/tests/phrase.compresspt.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..3f2ec42
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  print "TRANSLATION_$x=$_\n";
+  $x++;
+}
diff --git a/tests/phrase.compresspt.oldformat/moses.ini b/tests/phrase.compresspt.oldformat/moses.ini
new file mode 100644
index 0000000..899e2c1
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/moses.ini
@@ -0,0 +1,72 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+1
+2
+3
+
+# mapping steps
+[mapping]
+0 T 0
+
+# translation tables: table type (hierarchical(0), textual (0), binary (1)), source-factors, target-factors, number of scores, file 
+# OLD FORMAT is still handled for back-compatibility
+# OLD FORMAT translation tables: source-factors, target-factors, number of scores, file 
+# OLD FORMAT a binary table type (1) is assumed 
+[ttable-file]
+12 0 0 5 ${TEST_PATH}/phrase-table.minphr
+
+# no generation models, no generation-file section
+
+# language models: type(srilm/irstlm), factors, order, file
+[lmodel-file]
+1 0 3 ${MODEL_PATH}/ptable-with-alignment/mini.irstlm.en.3g.lm
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.6
+
+# language model weights
+[weight-l]
+0.5000
+
+
+# translation model weights
+[weight-t]
+0.20
+0.20
+0.20
+0.20
+0.20
+
+# no generation models, no weight-generation section
+
+# word penalty
+[weight-w]
+-1
+
+[distortion-limit]
+6
+
+[n-best-list]
+nbest
+5
+
+
+[print-alignment-info-in-n-best]
+true
+
+[include-segmentation-in-n-best]
+true
+
+
+
diff --git a/tests/phrase.compresspt.oldformat/phrase-table.minphr b/tests/phrase.compresspt.oldformat/phrase-table.minphr
new file mode 100644
index 0000000..b6ad6ac
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/phrase-table.minphr
diff --git a/tests/phrase.compresspt.oldformat/reordering-table.minlexr b/tests/phrase.compresspt.oldformat/reordering-table.minlexr
new file mode 100644
index 0000000..6f22336
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/reordering-table.minlexr
diff --git a/tests/phrase.compresspt.oldformat/to-translate.txt b/tests/phrase.compresspt.oldformat/to-translate.txt
new file mode 100644
index 0000000..7c24b74
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/to-translate.txt
@@ -0,0 +1 @@
+beide|PIAT|2|NK Versäumnisse|NN|3|SB haben|VAFIN|0|ROOT terroristische|ADJA|5|NK Gruppen|NN|8|MO in|APPR|8|MO Pakistan|NE|6|NK gestärkt|VVPP|3|OC .|$.|3|PUNC
diff --git a/tests/phrase.compresspt.oldformat/truth/results.txt b/tests/phrase.compresspt.oldformat/truth/results.txt
new file mode 100644
index 0000000..c35cf21
--- /dev/null
+++ b/tests/phrase.compresspt.oldformat/truth/results.txt
@@ -0,0 +1,8 @@
+TRANSLATION_0=beide Versäumnisse , terroristische Gruppen in Pakistan gestärkt . 
+SCORE_0 = -549.862
+TRANSLATION_0_NBEST_1=beide Versäumnisse , terroristische Gruppen in Pakistan gestärkt . ||| IRSTLM0= -112.888 Distortion0= 0 WordPenalty0= -9 PhraseDictionaryCompact0= -5.01064 -2.17433 -1.73026 -6.17638 2.99969 ||| -549.862 ||| 0=0 1=1 2=2 3=3 4=4 5-6=5-6 7=7 8=8 ||| 0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 
+TRANSLATION_0_NBEST_2=beide Versäumnisse , terroristische Gruppen in Pakistan gestärkt . ||| IRSTLM0= -112.888 Distortion0= 0 WordPenalty0= -9 PhraseDictionaryCompact0= -6.00156 -2.17433 -1.82688 -6.17638 3.99959 ||| -549.88 ||| 0=0 1=1 2=2 3=3 4=4 5=5 6=6 7=7 8=8 ||| 0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 
+TRANSLATION_0_NBEST_3=beide Versäumnisse made terroristische Gruppen in Pakistan gestärkt . ||| IRSTLM0= -116.003 Distortion0= 0 WordPenalty0= -9 PhraseDictionaryCompact0= -1.37305 -2.17433 -1.73026 -2.3922 2.99969 ||| -549.936 ||| 0=0 1=1 2=2 3=3 4=4 5-6=5-6 7=7 8=8 ||| 0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 
+TRANSLATION_0_NBEST_4=beide Versäumnisse made terroristische Gruppen in Pakistan gestärkt . ||| IRSTLM0= -116.003 Distortion0= 0 WordPenalty0= -9 PhraseDictionaryCompact0= -2.36397 -2.17433 -1.82688 -2.3922 3.99959 ||| -549.953 ||| 0=0 1=1 2=2 3=3 4=4 5=5 6=6 7=7 8=8 ||| 0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 
+TRANSLATION_0_NBEST_5=beide Versäumnisse , terroristische Gruppen in Pakistan . gestärkt ||| IRSTLM0= -110.706 Distortion0= -3 WordPenalty0= -9 PhraseDictionaryCompact0= -5.01064 -2.17433 -1.73026 -6.17638 2.99969 ||| -550.572 ||| 0=0 1=1 2=2 3=3 4=4 5-6=5-6 8=7 7=8 ||| 0-0 1-1 2-2 3-3 4-4 5-5 6-6 8-7 7-8 
+TOTAL_WALLTIME ~ 0
diff --git a/tests/phrase.consensus-decoding-surface.oldformat/filter-stderr.pl b/tests/phrase.consensus-decoding-surface.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.consensus-decoding-surface.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.consensus-decoding-surface.oldformat/filter-stdout.pl b/tests/phrase.consensus-decoding-surface.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.consensus-decoding-surface.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.consensus-decoding-surface.oldformat/moses.ini b/tests/phrase.consensus-decoding-surface.oldformat/moses.ini
new file mode 100644
index 0000000..f78e5b9
--- /dev/null
+++ b/tests/phrase.consensus-decoding-surface.oldformat/moses.ini
@@ -0,0 +1,60 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
+[consensus-decoding]
+1
+
+[mbr-size]
+1000
+
+[lmbr-pruning-factor]
+100
+
+[mbr-scale]
+1
diff --git a/tests/phrase.consensus-decoding-surface.oldformat/to-translate.txt b/tests/phrase.consensus-decoding-surface.oldformat/to-translate.txt
new file mode 100644
index 0000000..a505702
--- /dev/null
+++ b/tests/phrase.consensus-decoding-surface.oldformat/to-translate.txt
@@ -0,0 +1,5 @@
+ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ?
+schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind .
+vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen .
+das parlament will das auf zweierlei weise tun .
+nur dann werden die europäischen institutionen auch ihrem auftrag gerecht .
diff --git a/tests/phrase.consensus-decoding-surface.oldformat/truth/results.txt b/tests/phrase.consensus-decoding-surface.oldformat/truth/results.txt
new file mode 100644
index 0000000..326917b
--- /dev/null
+++ b/tests/phrase.consensus-decoding-surface.oldformat/truth/results.txt
@@ -0,0 +1,6 @@
+TRANSLATION_1=i ask you , therefore , mr president , are the different labour costs not therefore also a reduction of the free competition in the european union ? 
+TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of all commitments by payments are met . 
+TRANSLATION_3=three years ago , we have with our employment initiative started by small and medium-sized enterprises , halfen chancenkapital to obtain . 
+TRANSLATION_4=parliament wants the in two ways to do . 
+TRANSLATION_5=only then will the european institutions also its mandate fair . 
+TOTAL_WALLTIME ~ 262
diff --git a/tests/phrase.continue-partial-translation.oldformat/filter-stderr.pl b/tests/phrase.continue-partial-translation.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.continue-partial-translation.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.continue-partial-translation.oldformat/filter-stdout.pl b/tests/phrase.continue-partial-translation.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.continue-partial-translation.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.continue-partial-translation.oldformat/moses.ini b/tests/phrase.continue-partial-translation.oldformat/moses.ini
new file mode 100644
index 0000000..86e99bb
--- /dev/null
+++ b/tests/phrase.continue-partial-translation.oldformat/moses.ini
@@ -0,0 +1,51 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+1.42658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[continue-partial-translation]
+1
+
+[verbose]
+2
diff --git a/tests/phrase.continue-partial-translation.oldformat/to-translate.txt b/tests/phrase.continue-partial-translation.oldformat/to-translate.txt
new file mode 100644
index 0000000..9283512
--- /dev/null
+++ b/tests/phrase.continue-partial-translation.oldformat/to-translate.txt
@@ -0,0 +1,6 @@
+das
+keep ||| 00000 ||| das
+keep ||| 10000 ||| behalten das
+test ||| 0000 ||| raz dva tri ctyri
+two ||| 0100 ||| raz dva tri ctyri
+three ||| 0010 ||| raz dva tri ctyri
diff --git a/tests/phrase.continue-partial-translation.oldformat/truth/results.txt b/tests/phrase.continue-partial-translation.oldformat/truth/results.txt
new file mode 100644
index 0000000..d9d3626
--- /dev/null
+++ b/tests/phrase.continue-partial-translation.oldformat/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=' 
+TRANSLATION_2=keep the 
+TRANSLATION_3=keep the 
+TRANSLATION_4=test raz dva tri ctyri 
+TRANSLATION_5=two raz tri ctyri 
+TRANSLATION_6=three raz dva ctyri 
+SCORE_1 = -17.279
+SCORE_2 = -19.343
+SCORE_3 = -19.343
+SCORE_4 = -1731.443
+SCORE_5 = -1303.374
+SCORE_6 = -1303.374
+TOTAL_WALLTIME ~ 8
diff --git a/tests/phrase.multi-factor.oldformat/filter-stderr.pl b/tests/phrase.multi-factor.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.multi-factor.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.multi-factor.oldformat/filter-stdout.pl b/tests/phrase.multi-factor.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.multi-factor.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.multi-factor.oldformat/moses.ini b/tests/phrase.multi-factor.oldformat/moses.ini
new file mode 100644
index 0000000..6e12521
--- /dev/null
+++ b/tests/phrase.multi-factor.oldformat/moses.ini
@@ -0,0 +1,71 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+1
+
+# indicates that factors are separated by '++' in the input
+[factor-delimiter]
+++
+
+# mapping steps
+[mapping]
+T 0
+G 0
+T 1
+
+# translation tables: source-factors, target-factors, number of scores, file 
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/multi-factor/phrase-table.0-0.gz
+1 1 1 5 ${MODEL_PATH}/multi-factor/phrase-table.1-1.gz
+
+# generation models: source-factors, target-factors
+[generation-file]
+0 1 2 ${MODEL_PATH}/multi-factor/generation.0-1.gz
+
+# language models: 0, factors, type, file
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+0
+
+# distortion (reordering) weight
+[weight-d]
+0.6
+
+# language model weights
+[weight-l]
+0.5
+
+# translation model weights
+[weight-t]
+0.2
+0.2
+0.2
+0.2
+0.2
+0.2
+0.2
+0.2
+0.2
+0.2
+
+# generation model weights
+[weight-generation]
+0.3
+0
+
+# word penalty
+[weight-w]
+-1
+
+[verbose]
+2
+
diff --git a/tests/phrase.multi-factor.oldformat/to-translate.txt b/tests/phrase.multi-factor.oldformat/to-translate.txt
new file mode 100644
index 0000000..5868686
--- /dev/null
+++ b/tests/phrase.multi-factor.oldformat/to-translate.txt
@@ -0,0 +1 @@
+es++PPER gibt++VVFIN verschiedene++ADJ andere++ADJ meinungen++NN .++PE
diff --git a/tests/phrase.multi-factor.oldformat/truth/results.txt b/tests/phrase.multi-factor.oldformat/truth/results.txt
new file mode 100644
index 0000000..f3de782
--- /dev/null
+++ b/tests/phrase.multi-factor.oldformat/truth/results.txt
@@ -0,0 +1,3 @@
+TRANSLATION_1=there are various different opinions . 
+SCORE_1 = -113.734
+TOTAL_WALLTIME ~ 10
diff --git a/tests/phrase.show-weights.lex-reorder.oldformat/filter-stderr.pl b/tests/phrase.show-weights.lex-reorder.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..9580c19
--- /dev/null
+++ b/tests/phrase.show-weights.lex-reorder.oldformat/filter-stderr.pl
@@ -0,0 +1,9 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+}
diff --git a/tests/phrase.show-weights.lex-reorder.oldformat/filter-stdout.pl b/tests/phrase.show-weights.lex-reorder.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..bf00754
--- /dev/null
+++ b/tests/phrase.show-weights.lex-reorder.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "WEIGHT_$x=$_\n";
+}
diff --git a/tests/phrase.show-weights.lex-reorder.oldformat/moses.ini b/tests/phrase.show-weights.lex-reorder.oldformat/moses.ini
new file mode 100644
index 0000000..bbe14fa
--- /dev/null
+++ b/tests/phrase.show-weights.lex-reorder.oldformat/moses.ini
@@ -0,0 +1,66 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+
+# mapping steps
+[mapping]
+T 0
+
+# translation tables: source-factors, target-factors, number of scores, file 
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/lexicalized-reordering/phrase-table.0-0.gz
+
+# no generation models, no generation-file section
+
+# language models: type(srilm/irstlm), factors, order, file
+[lmodel-file]
+8 0 5 ${MODEL_PATH}/lexicalized-reordering/europarl.lm
+
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+0
+# distortion (reordering) files
+[distortion-file]
+0-0 msd-bidirectional-fe 6 ${MODEL_PATH}/lexicalized-reordering/reordering-table.msd-bidirectional-fe.0.5.0-0.gz
+
+# distortion (reordering) weight
+[weight-d]
+0.5
+0.3
+0.3
+0.3
+0.3
+0.3
+0.3
+
+# language model weights
+[weight-l]
+0.5000
+
+
+# translation model weights
+[weight-t]
+0.2
+0.2
+0.2
+0.2
+0.2
+
+# no generation models, no weight-generation section
+
+# word penalty
+[weight-w]
+-1
+
+[distortion-limit]
+6
+
+[show-weights]
+1
diff --git a/tests/phrase.show-weights.lex-reorder.oldformat/to-translate.txt b/tests/phrase.show-weights.lex-reorder.oldformat/to-translate.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/phrase.show-weights.lex-reorder.oldformat/to-translate.txt
diff --git a/tests/phrase.show-weights.lex-reorder.oldformat/truth/results.txt b/tests/phrase.show-weights.lex-reorder.oldformat/truth/results.txt
new file mode 100644
index 0000000..a242d07
--- /dev/null
+++ b/tests/phrase.show-weights.lex-reorder.oldformat/truth/results.txt
@@ -0,0 +1,6 @@
+WEIGHT_1=KENLM0= 0.500000
+WEIGHT_2=Distortion0= 0.500000
+WEIGHT_3=LexicalReordering0= 0.300000 0.300000 0.300000 0.300000 0.300000 0.300000
+WEIGHT_4=WordPenalty0= -1.000000
+WEIGHT_5=PhraseDictionaryMemory0= 0.200000 0.200000 0.200000 0.200000 0.200000
+TOTAL_WALLTIME ~ 1
diff --git a/tests/phrase.sparse-phrase-table.oldformat/filter-stderr.pl b/tests/phrase.sparse-phrase-table.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..2f6e176
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/tests/phrase.sparse-phrase-table.oldformat/filter-stdout.pl b/tests/phrase.sparse-phrase-table.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..476ddf6
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/tests/phrase.sparse-phrase-table.oldformat/moses.ini b/tests/phrase.sparse-phrase-table.oldformat/moses.ini
new file mode 100644
index 0000000..5673ecc
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/moses.ini
@@ -0,0 +1,51 @@
+# moses.ini for regression test
+
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/sparse-phrase-table/phrase-table.gz sparse
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
+[weight-file]
+${TEST_PATH}/weights
diff --git a/tests/phrase.sparse-phrase-table.oldformat/to-translate.txt b/tests/phrase.sparse-phrase-table.oldformat/to-translate.txt
new file mode 100644
index 0000000..92d74ce
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/to-translate.txt
@@ -0,0 +1 @@
+a b c c
diff --git a/tests/phrase.sparse-phrase-table.oldformat/truth/results.txt b/tests/phrase.sparse-phrase-table.oldformat/truth/results.txt
new file mode 100644
index 0000000..af8d526
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/truth/results.txt
@@ -0,0 +1,3 @@
+TRANSLATION_1=it me she she 
+SCORE_1 = -7.448
+TOTAL_WALLTIME ~ 7
diff --git a/tests/phrase.sparse-phrase-table.oldformat/weights b/tests/phrase.sparse-phrase-table.oldformat/weights
new file mode 100644
index 0000000..7986538
--- /dev/null
+++ b/tests/phrase.sparse-phrase-table.oldformat/weights
@@ -0,0 +1,3 @@
+PhraseDictionaryMemory0_po 0.5
+PhraseDictionaryMemory0_io -0.01
+PhraseDictionaryMemory0_o 0.1
diff --git a/tests/phrase.xml-markup.oldformat/filter-nbest.pl b/tests/phrase.xml-markup.oldformat/filter-nbest.pl
new file mode 100755
index 0000000..705fcee
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/filter-nbest.pl
@@ -0,0 +1,12 @@
+#!/usr/bin/perl
+$x=0;
+$oldcode = "";
+while (<>) {
+  chomp;
+  ($code,$trans,$featscores,$globscores) = split(/[\s]*\|\|\|[\s]*/,$_);
+  $x = 0 if $oldcode ne $code;
+  $x++;
+  chomp($code);
+  print "TRANSLATION_${code}_NBEST_${x}=$trans ||| $featscores\n";
+  $oldcode = $code;
+}
diff --git a/tests/phrase.xml-markup.oldformat/filter-stderr.pl b/tests/phrase.xml-markup.oldformat/filter-stderr.pl
new file mode 100755
index 0000000..a359091
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  print "SCORE_$x = $pscore\n";
+  $x++;
+}
diff --git a/tests/phrase.xml-markup.oldformat/filter-stdout.pl b/tests/phrase.xml-markup.oldformat/filter-stdout.pl
new file mode 100755
index 0000000..3f2ec42
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  print "TRANSLATION_$x=$_\n";
+  $x++;
+}
diff --git a/tests/phrase.xml-markup.oldformat/moses.ini b/tests/phrase.xml-markup.oldformat/moses.ini
new file mode 100644
index 0000000..8c8542f
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/moses.ini
@@ -0,0 +1,57 @@
+# Moses configuration file
+# automatic exodus from pharaoh.ini Wed Jul 12 18:24:14 EDT 2006
+
+###########################
+### PHARAOH CONFIG FILE ###
+###########################
+
+# phrase table f, n, p(n|f)
+[ttable-file]
+1 0 0 1 ${TEST_PATH}/phrase-table
+
+# language model
+[lmodel-file]
+8 0 3 ${LM_PATH}/europarl.en.srilm.gz
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+0
+
+# distortion (reordering) weight
+[weight-d]
+1.0
+
+# language model weight
+[weight-l]
+1.0
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+1.0
+
+# word penalty
+[weight-w]
+-5.0
+
+[distortion-limit]
+25
+
+[beam-threshold]
+0.000
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+[verbose]
+2
+
+[n-best-list]
+nbest
+10
+
+[xml-input]
+inclusive
+
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table b/tests/phrase.xml-markup.oldformat/phrase-table
new file mode 100644
index 0000000..fc67473
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table
@@ -0,0 +1,22 @@
+der ||| the ||| 0.3
+das ||| the ||| 0.4
+das ||| it ||| 0.1
+das ||| this ||| 0.1
+die ||| the ||| 0.3
+ist ||| is ||| 1.0
+ist ||| 's ||| 1.0
+das ist ||| it is ||| 0.2
+das ist ||| this is ||| 0.8
+es ist ||| it is ||| 0.8
+es ist ||| this is ||| 0.2
+ein ||| a ||| 1.0
+ein ||| an ||| 1.0
+klein ||| small ||| 0.8
+klein ||| little ||| 0.8
+kleines ||| small ||| 0.2
+kleines ||| little ||| 0.2
+haus ||| house ||| 1.0
+alt ||| old ||| 0.8
+altes ||| old ||| 0.2
+gibt ||| gives ||| 1.0
+es gibt ||| there is ||| 1.0
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table.binphr.idx b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.idx
new file mode 100644
index 0000000..a18ceaf
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.idx
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srctree b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srctree
new file mode 100644
index 0000000..9335395
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srctree
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srcvoc b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srcvoc
new file mode 100644
index 0000000..cc94a13
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.srcvoc
@@ -0,0 +1,12 @@
+11 gibt
+10 altes
+9 alt
+8 haus
+7 kleines
+6 klein
+5 ein
+4 es
+3 ist
+2 die
+1 das
+0 der
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtdata b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtdata
new file mode 100644
index 0000000..1af05d7
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtdata
diff --git a/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtvoc b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtvoc
new file mode 100644
index 0000000..a00b16f
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/phrase-table.binphr.tgtvoc
@@ -0,0 +1,13 @@
+12 there
+11 gives
+10 old
+9 house
+8 little
+7 small
+6 an
+5 a
+4 's
+3 is
+2 this
+1 it
+0 the
diff --git a/tests/phrase.xml-markup.oldformat/to-translate.txt b/tests/phrase.xml-markup.oldformat/to-translate.txt
new file mode 100644
index 0000000..3e0d71f
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/to-translate.txt
@@ -0,0 +1,6 @@
+das ist ein kleines <n english="dwelling" prob="0.8">haus</n> .
+das ist ein kleines <n english="dwelling||building" prob="0.6||0.4">haus</n> .
+das ist ein kleines <n english="dwelling||building" prob="0.6||100.0">haus</n> .
+<n english="dwelling||building" prob="10.6||0.4" span="4-4" />das ist ein kleines haus .
+<n english="dwelling" prob="10.6" span="4-4" />das ist ein kleines haus .
+<n english="nice place" prob="4.0" span="3-4" /><n english="dwelling" prob="10.6" span="4-4" />das ist ein kleines haus .
diff --git a/tests/phrase.xml-markup.oldformat/truth/results.txt b/tests/phrase.xml-markup.oldformat/truth/results.txt
new file mode 100644
index 0000000..734002c
--- /dev/null
+++ b/tests/phrase.xml-markup.oldformat/truth/results.txt
@@ -0,0 +1,73 @@
+TRANSLATION_0=this is a small house . 
+TRANSLATION_1=this is a small house . 
+TRANSLATION_2=this is a little building . 
+TRANSLATION_3=this is a small house . 
+TRANSLATION_4=this is a small house . 
+TRANSLATION_5=this is a nice place . 
+SCORE_0 = -94.029
+SCORE_1 = -94.029
+SCORE_2 = -92.554
+SCORE_3 = -94.029
+SCORE_4 = -94.029
+SCORE_5 = -93.875
+TRANSLATION_0_NBEST_1=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_0_NBEST_2=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_0_NBEST_3=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_0_NBEST_4=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_0_NBEST_5=this is a little dwelling . ||| KENLM0= -31.093 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.05573
+TRANSLATION_0_NBEST_6=this is an small house . ||| KENLM0= -31.6579 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_0_NBEST_7=this is a small dwelling . ||| KENLM0= -31.5834 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.05573
+TRANSLATION_0_NBEST_8=it is a little dwelling . ||| KENLM0= -31.1384 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.44202
+TRANSLATION_0_NBEST_9=this is a house small . ||| KENLM0= -29.1723 Distortion0= -4 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_0_NBEST_10=it is a small dwelling . ||| KENLM0= -31.6288 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.44202
+TRANSLATION_1_NBEST_1=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_1_NBEST_2=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_1_NBEST_3=this is a little building . ||| KENLM0= -25.3263 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.74887
+TRANSLATION_1_NBEST_4=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_1_NBEST_5=this is a small building . ||| KENLM0= -25.8167 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.74887
+TRANSLATION_1_NBEST_6=it is a little building . ||| KENLM0= -25.3718 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -4.13517
+TRANSLATION_1_NBEST_7=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_1_NBEST_8=it is a small building . ||| KENLM0= -25.8622 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -4.13517
+TRANSLATION_1_NBEST_9=this is a little dwelling . ||| KENLM0= -31.093 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.34341
+TRANSLATION_1_NBEST_10=this is an small house . ||| KENLM0= -31.6579 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_2_NBEST_1=this is a little building . ||| KENLM0= -25.3263 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 2.77259
+TRANSLATION_2_NBEST_2=this is a small building . ||| KENLM0= -25.8167 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 2.77259
+TRANSLATION_2_NBEST_3=it is a little building . ||| KENLM0= -25.3718 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 1.38629
+TRANSLATION_2_NBEST_4=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_2_NBEST_5=it is a small building . ||| KENLM0= -25.8622 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 1.38629
+TRANSLATION_2_NBEST_6=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_2_NBEST_7=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_2_NBEST_8=this is an small building . ||| KENLM0= -32.235 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 2.77259
+TRANSLATION_2_NBEST_9=this is an little building . ||| KENLM0= -32.3167 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 2.77259
+TRANSLATION_2_NBEST_10=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_3_NBEST_1=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_3_NBEST_2=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_3_NBEST_3=this is a little building . ||| KENLM0= -25.3263 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.74887
+TRANSLATION_3_NBEST_4=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_3_NBEST_5=this is a small building . ||| KENLM0= -25.8167 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -2.74887
+TRANSLATION_3_NBEST_6=it is a little building . ||| KENLM0= -25.3718 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -4.13517
+TRANSLATION_3_NBEST_7=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_3_NBEST_8=it is a small building . ||| KENLM0= -25.8622 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -4.13517
+TRANSLATION_3_NBEST_9=this is a little dwelling . ||| KENLM0= -31.093 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_3_NBEST_10=this is a small dwelling . ||| KENLM0= -31.5834 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_4_NBEST_1=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_4_NBEST_2=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_4_NBEST_3=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_4_NBEST_4=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_4_NBEST_5=this is a little dwelling . ||| KENLM0= -31.093 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_4_NBEST_6=this is a small dwelling . ||| KENLM0= -31.5834 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_4_NBEST_7=it is a little dwelling . ||| KENLM0= -31.1384 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -0.858022
+TRANSLATION_4_NBEST_8=it is a small dwelling . ||| KENLM0= -31.6288 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -0.858022
+TRANSLATION_4_NBEST_9=this is an small house . ||| KENLM0= -31.6579 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_4_NBEST_10=this is a house small . ||| KENLM0= -29.1723 Distortion0= -4 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_5_NBEST_1=this is a nice place . ||| KENLM0= -25.0386 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 1.16315
+TRANSLATION_5_NBEST_2=this is a small house . ||| KENLM0= -22.1968 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_5_NBEST_3=it is a nice place . ||| KENLM0= -25.084 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -0.223144
+TRANSLATION_5_NBEST_4=it is a small house . ||| KENLM0= -22.2422 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_5_NBEST_5=this is a little house . ||| KENLM0= -26.517 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -1.83258
+TRANSLATION_5_NBEST_6=it is a little house . ||| KENLM0= -26.5625 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -3.21888
+TRANSLATION_5_NBEST_7=this is an nice place . ||| KENLM0= -31.5425 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 1.16315
+TRANSLATION_5_NBEST_8=this is a little dwelling . ||| KENLM0= -31.093 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_5_NBEST_9=this is a small dwelling . ||| KENLM0= -31.5834 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= 0.528273
+TRANSLATION_5_NBEST_10=it is a little dwelling . ||| KENLM0= -31.1384 Distortion0= 0 WordPenalty0= -6 PhraseDictionaryBinary0= -0.858022
+TOTAL_WALLTIME ~ 3