Merge branch 'master' into miramerge.

Also fix mert-moses.pl to use correct flag for specifying
weights of non-core features.

Conflicts:
	.gitignore
	configure.in
	ltmain.sh
	moses/src/LM/Factory.cpp
	moses/src/LMList.cpp
	moses/src/LMList.h
	moses/src/LanguageModel.cpp
	moses/src/LanguageModel.h
	moses/src/LanguageModelKen.h
	moses/src/Makefile.am
	moses/src/Manager.cpp
	moses/src/PhraseDictionaryMemory.cpp
	moses/src/PhraseDictionaryTree.cpp
	moses/src/StaticData.cpp
	moses/src/TargetPhrase.h

13 files changed, 146 insertions, 32 deletions

diff --git a/regression-testing/MosesRegressionTesting.pm b/regression-testing/MosesRegressionTesting.pm
index 77d6e730c..9eb94a838 100644
--- a/regression-testing/MosesRegressionTesting.pm
+++ b/regression-testing/MosesRegressionTesting.pm
@@ -5,7 +5,7 @@ use strict;
 # if your tests need a new version of the test data, increment this
 # and make sure that a moses-regression-tests-vX.Y is available for
 # download from statmt.org (redpony AT umd dot edu for more info)
-use constant TESTING_DATA_VERSION => '7';
+use constant TESTING_DATA_VERSION => '9';
 
 # find the data directory in a few likely locations and make sure
 # that it is the correct version
diff --git a/regression-testing/run-test-suite.perl b/regression-testing/run-test-suite.perl
index 08e3cea1a..f59e75e1f 100755
--- a/regression-testing/run-test-suite.perl
+++ b/regression-testing/run-test-suite.perl
@@ -23,15 +23,11 @@ my @tests = qw (
   chart.target-syntax.ondisk
   chart.hierarchical
   chart.hierarchical-withsrilm
-  #chart.hierarchical-withkenlm
   chart.hierarchical.ondisk
   phrase.basic-surface-only
   phrase.basic-surface-only-withirstlm
   phrase.basic-surface-only-withirstlm-binlm
-  #phrase.basic-surface-only-withkenlm
-  #phrase.basic-surface-only-withkenlm.bin
   phrase.basic-lm-oov
-  #phrase.basic-lm-oov-withkenlm
   phrase.ptable-filtering
   phrase.multi-factor
   phrase.multi-factor-drop
@@ -43,6 +39,7 @@ my @tests = qw (
   phrase.lattice-surface
   phrase.lattice-distortion
   phrase.lexicalized-reordering
+  phrase.lexicalized-reordering-bin
   phrase.lexicalized-reordering-cn
   phrase.consensus-decoding-surface
   phrase.continue-partial-translation
@@ -50,6 +47,10 @@ my @tests = qw (
   phrase.show-weights
   phrase.xml-markup
 );
+  #phrase.basic-lm-oov-withkenlm
+  #phrase.basic-surface-only-withkenlm
+  #phrase.basic-surface-only-withkenlm.bin
+  #chart.hierarchical-withkenlm
 
 ############################################################
 use MosesRegressionTesting;
@@ -148,6 +149,7 @@ my $pass_percentage = int(100 * ($total-$fail) / $total);
 print "\n$pass_percentage% of the tests passed.\n";
 print "$fail_percentage% of the tests failed.\n";
 if ($fail_percentage>0) { print "\nPLEASE INVESTIGATE THESE FAILED TESTS: @failed\n"; }
+exit 2 if $fail > 0;
 
 sub do_test {
   my ($test) = @_;
diff --git a/regression-testing/tests/chart.hierarchical.ondisk/truth/results.txt b/regression-testing/tests/chart.hierarchical.ondisk/truth/results.txt
index 5ed37b255..c48776609 100644
--- a/regression-testing/tests/chart.hierarchical.ondisk/truth/results.txt
+++ b/regression-testing/tests/chart.hierarchical.ondisk/truth/results.txt
@@ -7,7 +7,7 @@ LMLOAD_TIME ~ 1.000
 PTLOAD_TIME ~ 1.000
 SCORE_1 = -12.918
 SCORE_2 = -12.745
-SCORE_3 = -117.149
+SCORE_3 = -117.584
 SCORE_4 = -122.970
 SCORE_5 = -9.131
 TOTAL_WALLTIME ~ 23
diff --git a/regression-testing/tests/chart.hierarchical/truth/results.txt b/regression-testing/tests/chart.hierarchical/truth/results.txt
index ff9efc6c8..a3491649b 100644
--- a/regression-testing/tests/chart.hierarchical/truth/results.txt
+++ b/regression-testing/tests/chart.hierarchical/truth/results.txt
@@ -9,7 +9,7 @@ PTLOAD_TIME ~ 83.000
 PTLOAD_TIME ~ 83.000
 SCORE_1 = -12.918
 SCORE_2 = -12.745
-SCORE_3 = -117.149
+SCORE_3 = -117.584
 SCORE_4 = -122.749
 SCORE_5 = -9.149
 TOTAL_WALLTIME ~ 105
diff --git a/regression-testing/tests/chart.target-syntax.ondisk/moses.ini b/regression-testing/tests/chart.target-syntax.ondisk/moses.ini
index c04c13872..0e40974f0 100644
--- a/regression-testing/tests/chart.target-syntax.ondisk/moses.ini
+++ b/regression-testing/tests/chart.target-syntax.ondisk/moses.ini
@@ -20,7 +20,7 @@
 
 # language models: type(srilm/irstlm), factors, order, file
 [lmodel-file]
-1 0 5 ${LM_PATH}/news.de.binlm
+8 0 5 ${LM_PATH}/news.de.arpalm.gz
 
 
 # limit on how many phrase translations e for each phrase f are loaded
diff --git a/regression-testing/tests/chart.target-syntax.ondisk/truth/results.txt b/regression-testing/tests/chart.target-syntax.ondisk/truth/results.txt
index b9af41b98..ac9629ca9 100644
--- a/regression-testing/tests/chart.target-syntax.ondisk/truth/results.txt
+++ b/regression-testing/tests/chart.target-syntax.ondisk/truth/results.txt
@@ -1,13 +1,15 @@
-TRANSLATION_1=Prag an Market fällt zu ohne am Ende des Handel Tag
-TRANSLATION_2=nach einem steilen Rückgang der Morgen , die Prag Stock Market korrigiert ihre Verluste .
-TRANSLATION_3=Geschäfte mit Aktien aus der tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der regelmäßigen Tageszeitung Handel .
-TRANSLATION_4=durch die Prag Stock Market sofort ihren fallen aus Montag Anfang Tuesday der Handel , als es um beinahe sechs Prozent .
-TRANSLATION_5=dieses Mal den Rückgang der Aktien an der Wall Street verantwortlich ist für den Einbruch .
-LMLOAD_TIME ~ 1.000
-PTLOAD_TIME ~ 1.000
-SCORE_1 = -9.173
-SCORE_2 = -9.857
-SCORE_3 = -113.066
-SCORE_4 = -117.916
-SCORE_5 = -7.325
-TOTAL_WALLTIME ~ 51
+TRANSLATION_1=Prag Stock Markt fällt , ohne Ende der Handel mit jedem Tag
+TRANSLATION_2=nach einem steilen Rückgang der Morgen , dem Prager Stock Markt ihre Verluste korrigiert .
+TRANSLATION_3=Geschäfte mit Aktien von der tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der regelmäßigen täglich Handel .
+TRANSLATION_4=der Prager Stock Markt sofort ihren Sturz von Montag Anfang Tuesday der Handel , wenn es um fast 6 % gefallen .
+TRANSLATION_5=dieses Mal den Rückgang der Aktien an der Wall Street sind verantwortlich für die fallen .
+LMLOAD_TIME ~ 0.000
+PTLOAD_TIME ~ 0.000
+PTLOAD_TIME ~ 7.000
+PTLOAD_TIME ~ 7.000
+SCORE_1 = -9.294
+SCORE_2 = -9.256
+SCORE_3 = -112.621
+SCORE_4 = -116.639
+SCORE_5 = -6.029
+TOTAL_WALLTIME ~ 23
diff --git a/regression-testing/tests/chart.target-syntax/moses.ini b/regression-testing/tests/chart.target-syntax/moses.ini
index 39cfd2c54..81f4e3961 100644
--- a/regression-testing/tests/chart.target-syntax/moses.ini
+++ b/regression-testing/tests/chart.target-syntax/moses.ini
@@ -20,7 +20,7 @@
 
 # language models: type(srilm/irstlm), factors, order, file
 [lmodel-file]
-1 0 5 ${LM_PATH}/news.de.binlm
+8 0 5 ${LM_PATH}/news.de.arpalm.gz
 
 
 # limit on how many phrase translations e for each phrase f are loaded
diff --git a/regression-testing/tests/chart.target-syntax/truth/results.txt b/regression-testing/tests/chart.target-syntax/truth/results.txt
index e58ab85cf..ac9629ca9 100644
--- a/regression-testing/tests/chart.target-syntax/truth/results.txt
+++ b/regression-testing/tests/chart.target-syntax/truth/results.txt
@@ -1,15 +1,15 @@
-TRANSLATION_1=Prag an Market fällt zu ohne am Ende des Handel Tag
-TRANSLATION_2=nach einem steilen Rückgang der Morgen , die Prag Stock Market korrigiert ihre Verluste .
-TRANSLATION_3=Geschäfte mit Aktien aus der tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der regelmäßigen Tageszeitung Handel .
-TRANSLATION_4=bei der Prag Stock Market sofort ihren fallen aus Montag Anfang Tuesday Handel , wenn es um beinahe sechs Prozent .
-TRANSLATION_5=an der Zeit in diesem Herbst in Aktien an der Wall Street dafür verantwortlich ist , die Einbruch .
+TRANSLATION_1=Prag Stock Markt fällt , ohne Ende der Handel mit jedem Tag
+TRANSLATION_2=nach einem steilen Rückgang der Morgen , dem Prager Stock Markt ihre Verluste korrigiert .
+TRANSLATION_3=Geschäfte mit Aktien von der tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der regelmäßigen täglich Handel .
+TRANSLATION_4=der Prager Stock Markt sofort ihren Sturz von Montag Anfang Tuesday der Handel , wenn es um fast 6 % gefallen .
+TRANSLATION_5=dieses Mal den Rückgang der Aktien an der Wall Street sind verantwortlich für die fallen .
 LMLOAD_TIME ~ 0.000
 PTLOAD_TIME ~ 0.000
 PTLOAD_TIME ~ 7.000
 PTLOAD_TIME ~ 7.000
-SCORE_1 = -9.173
-SCORE_2 = -9.857
-SCORE_3 = -113.066
-SCORE_4 = -117.959
-SCORE_5 = -7.223
+SCORE_1 = -9.294
+SCORE_2 = -9.256
+SCORE_3 = -112.621
+SCORE_4 = -116.639
+SCORE_5 = -6.029
 TOTAL_WALLTIME ~ 23
diff --git a/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stderr.pl b/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stderr.pl
new file mode 100755
index 000000000..2f6e17607
--- /dev/null
+++ b/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stderr.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+
+  if (/^Finished loading LanguageModels/) {
+    my $time = RegTestUtils::readTime($_);
+    print "LMLOAD_TIME ~ $time\n";
+  }
+  if (/^Finished loading phrase tables/) {
+    my $time = RegTestUtils::readTime($_);
+    print "PTLOAD_TIME ~ $time\n";
+  }
+  next unless /^BEST TRANSLATION:/;
+  my $pscore = RegTestUtils::readHypoScore($_);
+  $x++;
+  print "SCORE_$x = $pscore\n";
+}
diff --git a/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stdout.pl b/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stdout.pl
new file mode 100755
index 000000000..476ddf6e9
--- /dev/null
+++ b/regression-testing/tests/phrase.lexicalized-reordering-bin/filter-stdout.pl
@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "TRANSLATION_$x=$_\n";
+}
diff --git a/regression-testing/tests/phrase.lexicalized-reordering-bin/moses.ini b/regression-testing/tests/phrase.lexicalized-reordering-bin/moses.ini
new file mode 100644
index 000000000..fb536f9a9
--- /dev/null
+++ b/regression-testing/tests/phrase.lexicalized-reordering-bin/moses.ini
@@ -0,0 +1,63 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+
+# mapping steps
+[mapping]
+T 0
+
+# translation tables: source-factors, target-factors, number of scores, file 
+[ttable-file]
+1 0 0 5 ${MODEL_PATH}/lexicalized-reordering-bin/phrase-table
+
+# no generation models, no generation-file section
+
+# language models: type(srilm/irstlm), factors, order, file
+[lmodel-file]
+0 0 5 ${MODEL_PATH}/lexicalized-reordering/europarl.lm
+
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+0
+# distortion (reordering) files
+[distortion-file]
+0-0 msd-bidirectional-fe 6 ${MODEL_PATH}/lexicalized-reordering-bin/reordering-table.msd-bidirectional-fe
+
+# distortion (reordering) weight
+[weight-d]
+0.5
+0.3
+0.3
+0.3
+0.3
+0.3
+0.3
+
+# language model weights
+[weight-l]
+0.5000
+
+
+# translation model weights
+[weight-t]
+0.2
+0.2
+0.2
+0.2
+0.2
+
+# no generation models, no weight-generation section
+
+# word penalty
+[weight-w]
+-1
+
+[distortion-limit]
+6
diff --git a/regression-testing/tests/phrase.lexicalized-reordering-bin/to-translate.txt b/regression-testing/tests/phrase.lexicalized-reordering-bin/to-translate.txt
new file mode 100644
index 000000000..d336d82f3
--- /dev/null
+++ b/regression-testing/tests/phrase.lexicalized-reordering-bin/to-translate.txt
@@ -0,0 +1,5 @@
+monsieur le président , ce que nous devrons toutefois également faire à biarritz , c' est regarder un peu plus loin .
+les élus que nous sommes avons au moins autant le devoir de l' encourager à progresser , en dépit de l' adversité , que de relayer les messages que nous recevons de l' opinion publique dans chacun de nos pays .
+au regard des événements de ces derniers temps , la question du prix de l' essence me semble elle aussi particulièrement remarquable .
+à l' heure actuelle , le conseil est en train d' examiner l' inclusion de tels mécanismes dans l' article 7 .
+deuxièmement , dans la transparence pour les citoyens , qui connaissent à présent les droits dont ils disposent vis-à-vis de ceux qui appliquent et élaborent le droit européen , et pour ceux qui , justement , appliquent et élaborent ce droit européen .
diff --git a/regression-testing/tests/phrase.lexicalized-reordering-bin/truth/results.txt b/regression-testing/tests/phrase.lexicalized-reordering-bin/truth/results.txt
new file mode 100644
index 000000000..9c56d1e73
--- /dev/null
+++ b/regression-testing/tests/phrase.lexicalized-reordering-bin/truth/results.txt
@@ -0,0 +1,13 @@
+TRANSLATION_1=mr president , what we devrons but also to to biarritz , this is regarder a little more loin . 
+TRANSLATION_2=the élus that we are have the with difference the us of the encourager to progresser , in dépit of the adversité , that of relayer the messages that we recevons of the opinion all in each of our countries . 
+TRANSLATION_3=the prerequisites for the événements of these derniers time , the issue of the sufficiently of the essence to it also in remarquable . 
+TRANSLATION_4=in the , the council , it is train of examiner the inclusion of such mécanismes in article 7 . 
+TRANSLATION_5=secondly , in the transparency for the citizens , which are now the droits which they disposent vis-à-vis of those appliquent and élaborent the european law and , to those , precisely appliquent and élaborent this european law . 
+LMLOAD_TIME ~ 8.00
+PTLOAD_TIME ~ 9.00
+SCORE_1 = -621.292
+SCORE_2 = -1402.951
+SCORE_3 = -636.300
+SCORE_4 = -631.966
+SCORE_5 = -1110.523
+TOTAL_WALLTIME ~ 16