rescale LM score for chart decoding

17 files changed, 46 insertions, 213 deletions

diff --git a/tests/chart.nbest-with-alignment/truth/results.txt b/tests/chart.nbest-with-alignment/truth/results.txt
index 1766148..db45da0 100644
--- a/tests/chart.nbest-with-alignment/truth/results.txt
+++ b/tests/chart.nbest-with-alignment/truth/results.txt
@@ -10,25 +10,25 @@ SCORE_2 = -12.745
 SCORE_3 = -117.584
 SCORE_4 = -122.749
 SCORE_5 = -9.149
-TRANSLATION_0_NBEST_1=Prag überquillt Stock Markt , durch das Ende des Handel Tag ||| IRSTLM0= -33.5161 WordPenalty0= -13 PhraseDictionarySCFG0= -38.0654 -36.4637 -28.3614 -20.8156 8.99907 PhraseDictionarySCFG1= 2.99969 ||| -12.9179 |||  ||| 
-TRANSLATION_0_NBEST_2=Prag überquillt Stock Markt , durch das Ende des Devisen Tag ||| IRSTLM0= -33.5161 WordPenalty0= -13 PhraseDictionarySCFG0= -37.0299 -35.857 -29.8466 -22.9556 8.99907 PhraseDictionarySCFG1= 2.99969 ||| -12.9214 |||  ||| 
-TRANSLATION_0_NBEST_3=Prag überquillt Stock Markt fällt , durch das Ende des Handel Tag ||| IRSTLM0= -37.295 WordPenalty0= -14 PhraseDictionarySCFG0= -33.9846 -29.401 -25.5504 -22.3319 9.99896 PhraseDictionarySCFG1= 5.99938 ||| -12.9238 |||  ||| 
-TRANSLATION_0_NBEST_4=Prag überquillt Stock Markt fällt , durch das Ende des Handel Tag ||| IRSTLM0= -37.295 WordPenalty0= -14 PhraseDictionarySCFG0= -33.9846 -29.401 -25.5504 -22.3319 9.99896 PhraseDictionarySCFG1= 5.99938 ||| -12.9238 |||  ||| 
-TRANSLATION_1_NBEST_1= ||| IRSTLM0= -3.91806 WordPenalty0= -2 PhraseDictionarySCFG0= 0 0 0 0 0 PhraseDictionarySCFG1= 0 ||| -0.774374 |||  ||| 
-TRANSLATION_2_NBEST_1=nach einem steilen Rückgang der Morgen der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -42.7378 WordPenalty0= -16 PhraseDictionarySCFG0= -22.0149 -19.4038 -18.8721 -20.7816 10.9989 PhraseDictionarySCFG1= 5.99938 ||| -12.7448 |||  ||| 
-TRANSLATION_2_NBEST_2=nach einem steilen Rückgang der Morgen , der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -44.3854 WordPenalty0= -17 PhraseDictionarySCFG0= -20.8618 -18.2393 -18.5446 -21.4071 10.9989 PhraseDictionarySCFG1= 6.99927 ||| -12.7906 |||  ||| 
-TRANSLATION_2_NBEST_3=nach einem scharfen Rückgang der Morgen der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -42.7378 WordPenalty0= -16 PhraseDictionarySCFG0= -22.1582 -19.6551 -19.9314 -21.7261 10.9989 PhraseDictionarySCFG1= 5.99938 ||| -12.8243 |||  ||| 
-TRANSLATION_2_NBEST_4=nach einem steilen Rückgang der Morgen , der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -44.3854 WordPenalty0= -17 PhraseDictionarySCFG0= -20.402 -18.2393 -18.0671 -21.4071 11.9988 PhraseDictionarySCFG1= 6.99927 ||| -12.8251 |||  ||| 
-TRANSLATION_3_NBEST_1=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -63.3997 WordPenalty0= -22 PhraseDictionarySCFG0= -23.0803 -18.4766 -26.2273 -22.9922 15.9983 PhraseDictionarySCFG1= 12.9986 ||| -117.584 |||  ||| 
-TRANSLATION_3_NBEST_2=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreichte beinahe die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -63.3997 WordPenalty0= -22 PhraseDictionarySCFG0= -22.6125 -18.1042 -27.2882 -24.0585 15.9983 PhraseDictionarySCFG1= 12.9986 ||| -117.607 |||  ||| 
-TRANSLATION_3_NBEST_3=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der täglichen gehandelt . regelmäßigen ||| IRSTLM0= -63.1835 WordPenalty0= -22 PhraseDictionarySCFG0= -22.5634 -18.6849 -28.2097 -25.1323 15.9983 PhraseDictionarySCFG1= 12.9986 ||| -117.61 |||  ||| 
-TRANSLATION_3_NBEST_4=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht fast die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -63.3997 WordPenalty0= -22 PhraseDictionarySCFG0= -23.5327 -18.9771 -26.2457 -23.0196 15.9983 PhraseDictionarySCFG1= 12.9986 ||| -117.627 |||  ||| 
-TRANSLATION_4_NBEST_1=Über die Prag Stock Market sofort seinem Sturz von Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -63.8588 WordPenalty0= -24 PhraseDictionarySCFG0= -57.5379 -54.5468 -45.845 -40.7049 18.998 PhraseDictionarySCFG1= 7.99917 ||| -122.749 |||  ||| 
-TRANSLATION_4_NBEST_2=Über die Prag Stock Markt sofort ihren fallen aus Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -63.9225 WordPenalty0= -24 PhraseDictionarySCFG0= -56.6908 -57.8191 -41.4948 -38.5075 18.998 PhraseDictionarySCFG1= 5.99938 ||| -122.75 |||  ||| 
-TRANSLATION_4_NBEST_3=Über die Prag Stock Markt sofort ihren fallen aus Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -63.9225 WordPenalty0= -24 PhraseDictionarySCFG0= -56.6908 -57.8191 -41.4948 -38.5075 18.998 PhraseDictionarySCFG1= 5.99938 ||| -122.75 |||  ||| 
-TRANSLATION_4_NBEST_4=Über die Prag Stock Markt sofort fortgesetzte seinem Sturz von Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -67.0807 WordPenalty0= -25 PhraseDictionarySCFG0= -48.418 -50.1131 -42.5677 -42.2291 19.9979 PhraseDictionarySCFG1= 6.99927 ||| -122.752 |||  ||| 
-TRANSLATION_5_NBEST_1=diesmal in Aktien an der Wall Street dem Fall ist für die fallen . ||| IRSTLM0= -32.2937 WordPenalty0= -16 PhraseDictionarySCFG0= -17.3218 -26.5953 -14.5791 -23.5759 9.99896 PhraseDictionarySCFG1= 6.99927 ||| -9.14867 |||  ||| 
-TRANSLATION_5_NBEST_2=diesmal in Aktien an der Wall Street dem Fall ist dafür die fallen . ||| IRSTLM0= -32.2937 WordPenalty0= -16 PhraseDictionarySCFG0= -15.8795 -25.4668 -16.7206 -25.3747 9.99896 PhraseDictionarySCFG1= 6.99927 ||| -9.15155 |||  ||| 
-TRANSLATION_5_NBEST_3=diesmal die fallen in Aktien an der Wall Street ist verantwortlich für die fallen . ||| IRSTLM0= -37.6507 WordPenalty0= -17 PhraseDictionarySCFG0= -9.75694 -16.2587 -12.1807 -21.6218 10.9989 PhraseDictionarySCFG1= 10.9989 ||| -9.15603 |||  ||| 
-TRANSLATION_5_NBEST_4=diesmal die fallen in Aktien an der Wall Street ist verantwortungsvolle für die fallen . ||| IRSTLM0= -37.6507 WordPenalty0= -17 PhraseDictionarySCFG0= -8.97208 -15.2468 -13.6316 -23.5387 10.9989 PhraseDictionarySCFG1= 10.9989 ||| -9.17055 |||  ||| 
+TRANSLATION_0_NBEST_1=Prag überquillt Stock Markt , durch das Ende des Handel Tag ||| IRSTLM0= -77.1736 WordPenalty0= -13 PhraseDictionaryMemory0= -38.0654 -36.4637 -28.3614 -20.8156 8.99907 PhraseDictionaryMemory1= 2.99969 ||| -12.9179 |||  ||| 
+TRANSLATION_0_NBEST_2=Prag überquillt Stock Markt , durch das Ende des Devisen Tag ||| IRSTLM0= -77.1736 WordPenalty0= -13 PhraseDictionaryMemory0= -37.0299 -35.857 -29.8466 -22.9556 8.99907 PhraseDictionaryMemory1= 2.99969 ||| -12.9214 |||  ||| 
+TRANSLATION_0_NBEST_3=Prag überquillt Stock Markt fällt , durch das Ende des Handel Tag ||| IRSTLM0= -85.8749 WordPenalty0= -14 PhraseDictionaryMemory0= -33.9846 -29.401 -25.5504 -22.3319 9.99896 PhraseDictionaryMemory1= 5.99938 ||| -12.9239 |||  ||| 
+TRANSLATION_0_NBEST_4=Prag überquillt Stock Markt fällt , durch das Ende des Handel Tag ||| IRSTLM0= -85.8749 WordPenalty0= -14 PhraseDictionaryMemory0= -33.9846 -29.401 -25.5504 -22.3319 9.99896 PhraseDictionaryMemory1= 5.99938 ||| -12.9239 |||  ||| 
+TRANSLATION_1_NBEST_1= ||| IRSTLM0= -9.02167 WordPenalty0= -2 PhraseDictionaryMemory0= 0 0 0 0 0 PhraseDictionaryMemory1= 0 ||| -0.774376 |||  ||| 
+TRANSLATION_2_NBEST_1=nach einem steilen Rückgang der Morgen der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -98.4073 WordPenalty0= -16 PhraseDictionaryMemory0= -22.0149 -19.4038 -18.8721 -20.7816 10.9989 PhraseDictionaryMemory1= 5.99938 ||| -12.7449 |||  ||| 
+TRANSLATION_2_NBEST_2=nach einem steilen Rückgang der Morgen , der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -102.201 WordPenalty0= -17 PhraseDictionaryMemory0= -20.8618 -18.2393 -18.5446 -21.4071 10.9989 PhraseDictionaryMemory1= 6.99927 ||| -12.7907 |||  ||| 
+TRANSLATION_2_NBEST_3=nach einem scharfen Rückgang der Morgen der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -98.4073 WordPenalty0= -16 PhraseDictionaryMemory0= -22.1582 -19.6551 -19.9314 -21.7261 10.9989 PhraseDictionaryMemory1= 5.99938 ||| -12.8243 |||  ||| 
+TRANSLATION_2_NBEST_4=nach einem steilen Rückgang der Morgen , der Prager Stock Markt ihre Verluste . korrigiert ||| IRSTLM0= -102.201 WordPenalty0= -17 PhraseDictionaryMemory0= -20.402 -18.2393 -18.0671 -21.4071 11.9988 PhraseDictionaryMemory1= 6.99927 ||| -12.8251 |||  ||| 
+TRANSLATION_3_NBEST_1=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -145.983 WordPenalty0= -22 PhraseDictionaryMemory0= -23.0803 -18.4766 -26.2273 -22.9922 15.9983 PhraseDictionaryMemory1= 12.9986 ||| -117.584 |||  ||| 
+TRANSLATION_3_NBEST_2=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreichte beinahe die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -145.983 WordPenalty0= -22 PhraseDictionaryMemory0= -22.6125 -18.1042 -27.2882 -24.0585 15.9983 PhraseDictionaryMemory1= 12.9986 ||| -117.607 |||  ||| 
+TRANSLATION_3_NBEST_3=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht beinahe die Hälfte der täglichen gehandelt . regelmäßigen ||| IRSTLM0= -145.485 WordPenalty0= -22 PhraseDictionaryMemory0= -22.5634 -18.6849 -28.2097 -25.1323 15.9983 PhraseDictionaryMemory1= 12.9986 ||| -117.61 |||  ||| 
+TRANSLATION_3_NBEST_4=Geschäfte mit Aktien aus der Tschechischen Energy Enterprise ( ČEZ ) erreicht fast die Hälfte der täglichen Handel . regelmäßigen ||| IRSTLM0= -145.983 WordPenalty0= -22 PhraseDictionaryMemory0= -23.5327 -18.9771 -26.2457 -23.0196 15.9983 PhraseDictionaryMemory1= 12.9986 ||| -117.627 |||  ||| 
+TRANSLATION_4_NBEST_1=Über die Prag Stock Market sofort seinem Sturz von Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -147.04 WordPenalty0= -24 PhraseDictionaryMemory0= -57.5379 -54.5468 -45.845 -40.7049 18.998 PhraseDictionaryMemory1= 7.99917 ||| -122.749 |||  ||| 
+TRANSLATION_4_NBEST_2=Über die Prag Stock Markt sofort ihren fallen aus Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -147.187 WordPenalty0= -24 PhraseDictionaryMemory0= -56.6908 -57.8191 -41.4948 -38.5075 18.998 PhraseDictionaryMemory1= 5.99938 ||| -122.75 |||  ||| 
+TRANSLATION_4_NBEST_3=Über die Prag Stock Markt sofort ihren fallen aus Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -147.187 WordPenalty0= -24 PhraseDictionaryMemory0= -56.6908 -57.8191 -41.4948 -38.5075 18.998 PhraseDictionaryMemory1= 5.99938 ||| -122.75 |||  ||| 
+TRANSLATION_4_NBEST_4=Über die Prag Stock Markt sofort fortgesetzte seinem Sturz von Montag Anfang der Handel , wenn es um beinahe sechs Prozent . Tuesday ||| IRSTLM0= -154.459 WordPenalty0= -25 PhraseDictionaryMemory0= -48.418 -50.1131 -42.5677 -42.2291 19.9979 PhraseDictionaryMemory1= 6.99927 ||| -122.752 |||  ||| 
+TRANSLATION_5_NBEST_1=diesmal in Aktien an der Wall Street dem Fall ist für die fallen . ||| IRSTLM0= -74.359 WordPenalty0= -16 PhraseDictionaryMemory0= -17.3218 -26.5953 -14.5791 -23.5759 9.99896 PhraseDictionaryMemory1= 6.99927 ||| -9.14869 |||  ||| 
+TRANSLATION_5_NBEST_2=diesmal in Aktien an der Wall Street dem Fall ist dafür die fallen . ||| IRSTLM0= -74.359 WordPenalty0= -16 PhraseDictionaryMemory0= -15.8795 -25.4668 -16.7206 -25.3747 9.99896 PhraseDictionaryMemory1= 6.99927 ||| -9.15157 |||  ||| 
+TRANSLATION_5_NBEST_3=diesmal die fallen in Aktien an der Wall Street ist verantwortlich für die fallen . ||| IRSTLM0= -86.6939 WordPenalty0= -17 PhraseDictionaryMemory0= -9.75694 -16.2587 -12.1807 -21.6218 10.9989 PhraseDictionaryMemory1= 10.9989 ||| -9.15605 |||  ||| 
+TRANSLATION_5_NBEST_4=diesmal die fallen in Aktien an der Wall Street ist verantwortungsvolle für die fallen . ||| IRSTLM0= -86.6939 WordPenalty0= -17 PhraseDictionaryMemory0= -8.97208 -15.2468 -13.6316 -23.5387 10.9989 PhraseDictionaryMemory1= 10.9989 ||| -9.17057 |||  ||| 
 TOTAL_WALLTIME ~ 31
diff --git a/tests/chart.show-weights.target-syntax/truth/results.txt b/tests/chart.show-weights.target-syntax/truth/results.txt
index 2a184ce..6174a59 100644
--- a/tests/chart.show-weights.target-syntax/truth/results.txt
+++ b/tests/chart.show-weights.target-syntax/truth/results.txt
@@ -1,5 +1,5 @@
-WEIGHT_1=IRSTLM0= 0.210707
+WEIGHT_1=IRSTLM0= 0.091509
 WEIGHT_2=WordPenalty0= -0.233879
 WEIGHT_3=PhraseDictionaryOnDisk0= 0.067170 0.024389 0.013606 0.024433 -0.103362
 WEIGHT_4=PhraseDictionaryOnDisk1= 0.017807
-TOTAL_WALLTIME ~ 0
+TOTAL_WALLTIME ~ 1
diff --git a/tests/phrase.early-distortion-cost/truth/results.txt b/tests/phrase.early-distortion-cost/truth/results.txt
index 86cf69b..ac5b79e 100644
--- a/tests/phrase.early-distortion-cost/truth/results.txt
+++ b/tests/phrase.early-distortion-cost/truth/results.txt
@@ -37,7 +37,7 @@ TRANSLATION_0_NBEST_26=despite progress human rights of the end is unknown . |||
 TRANSLATION_0_NBEST_27=human rights despite progress is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -20 WordPenalty0= -10 PhraseDictionaryMemory0= -0.697919 -4.51473 -2.0892 -6.25646 6.99927 ||| -22.4174 ||| 2-3=0-1 0=2 1=3 7=4 8=5 9=6 4-6=7-9 ||| 
 TRANSLATION_0_NBEST_28=despite progress human rights is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -12 WordPenalty0= -10 PhraseDictionaryMemory0= -0.693147 -4.51473 -1.09861 -6.25646 3.99958 ||| -22.4183 ||| 0-1=0-1 2-3=2-3 7-9=4-6 4-6=7-9 ||| 
 TRANSLATION_0_NBEST_29=despite progress human rights is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -12 WordPenalty0= -10 PhraseDictionaryMemory0= -1.663 -4.51473 -3.14525 -6.25646 6.99927 ||| -22.4216 ||| 0-1=0-1 2-3=2-3 7=4 8=5 9=6 4-5=7-8 6=9 ||| 
-TRANSLATION_0_NBEST_30=despite progress human rights is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -12 WordPenalty0= -10 PhraseDictionaryMemory0= -1.65823 -4.51473 -2.15467 -6.25646 5.99938 ||| -22.4226 ||| 0=0 1=1 2-3=2-3 7-9=4-6 4-5=7-8 6=9 ||| 
+TRANSLATION_0_NBEST_30=despite progress human rights is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -12 WordPenalty0= -10 PhraseDictionaryMemory0= -1.65823 -4.51473 -2.15467 -6.25646 5.99938 ||| -22.4225 ||| 0=0 1=1 2-3=2-3 7-9=4-6 4-5=7-8 6=9 ||| 
 TRANSLATION_0_NBEST_31=despite progress human rights end of the is unknown . ||| IRSTLM0= -60.695 Distortion0= -6 WordPenalty0= -10 PhraseDictionaryMemory0= -3.30589 -4.51474 -1.87877 -6.25646 6.99927 ||| -22.4388 ||| 0=0 1=1 2-3=2-3 5-6=4-5 4=6 7=7 8-9=8-9 ||| 
 TRANSLATION_0_NBEST_32=despite progress human rights is unknown . the end of the ||| IRSTLM0= -61.4383 Distortion0= -12 WordPenalty0= -11 PhraseDictionaryMemory0= -1.14991 -7.38101 -2.82887 -6.25646 6.99927 ||| -22.4426 ||| 0=0 1=1 2=2 3=3 7-8=4-5 9=6 4-6=7-10 ||| 
 TRANSLATION_0_NBEST_33=despite progress human rights is unknown . the end of ||| IRSTLM0= -60.2112 Distortion0= -12 WordPenalty0= -10 PhraseDictionaryMemory0= -1.14991 -4.51473 -2.82887 -6.25646 5.99938 ||| -22.4557 ||| 0-1=0-1 2=2 3=3 7-8=4-5 9=6 4-6=7-9 ||| 
@@ -258,4 +258,4 @@ TRANSLATION_4_NBEST_47=especially the options . between China and ||| IRSTLM0= -
 TRANSLATION_4_NBEST_48=especially the between China and options . ||| IRSTLM0= -50.3579 Distortion0= 0 WordPenalty0= -7 PhraseDictionaryMemory0= -2.62417 -2.8165 -1.57435 -8.81916 3.99959 ||| -20.5459 ||| 0-2=0 3=1 4-6=2-4 7-8=5-6 ||| 
 TRANSLATION_4_NBEST_49=especially options . between China and the ||| IRSTLM0= -49.2721 Distortion0= -18 WordPenalty0= -7 PhraseDictionaryMemory0= -2.67546 -2.8165 -1.91831 -8.81916 5.99938 ||| -20.5821 ||| 0-2=0 7=1 8=2 4=3 5-6=4-5 3=6 ||| 
 TRANSLATION_4_NBEST_50=especially options . between China and the ||| IRSTLM0= -49.2721 Distortion0= -18 WordPenalty0= -7 PhraseDictionaryMemory0= -2.89168 -2.8165 -2.70598 -8.81916 6.99927 ||| -20.5829 ||| 0-2=0 7=1 8=2 4=3 5=4 6=5 3=6 ||| 
-TOTAL_WALLTIME ~ 0
+TOTAL_WALLTIME ~ 1
diff --git a/tests/phrase.ptable-filtering/truth/results.txt b/tests/phrase.ptable-filtering/truth/results.txt
index 68cbb53..4f7d8b4 100644
--- a/tests/phrase.ptable-filtering/truth/results.txt
+++ b/tests/phrase.ptable-filtering/truth/results.txt
@@ -1,22 +1,22 @@
-TRANSLATION_OPTION_1=form of :: c=-0.815 core=(0.000,0.000,0.000,0.000,-7.684,-10.313,-5.881,-7.564,1.000,0.000) 
-TRANSLATION_OPTION_2=from :: c=-0.783 core=(0.000,0.000,0.000,0.000,-11.882,-9.781,-5.881,-3.635,1.000,0.000) 
-TRANSLATION_OPTION_3=money :: c=-0.680 core=(0.000,0.000,0.000,0.000,-7.083,-6.272,-3.935,-2.920,1.000,0.000) 
-TRANSLATION_OPTION_4=money transfers :: c=-0.797 core=(0.000,0.000,0.000,0.000,-0.847,-5.598,-4.782,-6.755,1.000,0.000) 
-TRANSLATION_OPTION_5=of transfers :: c=-0.805 core=(0.000,0.000,0.000,0.000,-2.944,-5.199,-5.881,-4.673,1.000,0.000) 
-TRANSLATION_OPTION_6=payments :: c=-0.816 core=(0.000,0.000,0.000,0.000,-7.352,-8.544,-5.187,-6.033,1.000,0.000) 
-TRANSLATION_OPTION_7=providing :: c=-0.866 core=(0.000,0.000,0.000,0.000,-9.478,-8.443,-5.881,-5.117,1.000,0.000) 
-TRANSLATION_OPTION_8=remittance :: c=-0.627 core=(0.000,0.000,0.000,0.000,-0.426,-1.052,-1.296,-1.567,1.000,0.000) 
-TRANSLATION_OPTION_9=remittance of :: c=-0.726 core=(0.000,0.000,0.000,0.000,-0.693,-1.052,-5.881,-2.404,1.000,0.000) 
-TRANSLATION_OPTION_10=remittances , :: c=-0.793 core=(0.000,0.000,0.000,0.000,-2.773,-0.834,-5.881,-4.021,1.000,0.000) 
-TRANSLATION_OPTION_11=remittances , to :: c=-0.864 core=(-4.387,0.000,0.000,0.000,0.000,-0.834,-5.881,-5.558,1.000,0.000) 
-TRANSLATION_OPTION_12=remittances :: c=-0.611 core=(0.000,0.000,0.000,0.000,-0.357,-0.834,-0.975,-1.241,1.000,0.000) 
-TRANSLATION_OPTION_13=remittances from :: c=-0.753 core=(0.000,0.000,0.000,0.000,-0.827,-1.527,-3.935,-4.877,1.000,0.000) 
-TRANSLATION_OPTION_14=represents :: c=-0.852 core=(0.000,0.000,0.000,0.000,-8.504,-9.216,-5.881,-6.726,1.000,0.000) 
-TRANSLATION_OPTION_15=sending money :: c=-0.835 core=(0.000,0.000,0.000,0.000,-0.916,-6.398,-5.187,-8.260,1.000,0.000) 
-TRANSLATION_OPTION_16=sent :: c=-0.833 core=(0.000,0.000,0.000,0.000,-7.791,-7.930,-5.881,-5.117,1.000,0.000) 
-TRANSLATION_OPTION_17=transfer :: c=-0.717 core=(0.000,0.000,0.000,0.000,-6.775,-6.455,-3.396,-3.468,1.000,0.000) 
-TRANSLATION_OPTION_18=transfer of money :: c=-0.801 core=(-4.658,0.000,0.000,0.000,-1.792,-6.359,-5.881,-7.225,1.000,0.000) 
-TRANSLATION_OPTION_19=transfers :: c=-0.692 core=(0.000,0.000,0.000,0.000,-5.651,-5.199,-3.935,-3.836,1.000,0.000) 
-TRANSLATION_OPTION_20=transfers from :: c=-0.759 core=(0.000,0.000,0.000,0.000,-2.639,-5.882,-5.881,-7.471,1.000,0.000) 
+TRANSLATION_OPTION_1=form of :: c=-0.815 core=(0.000,0.000,-2.000,0.000,-7.684,-10.313,-5.881,-7.564,1.000,0.000) 
+TRANSLATION_OPTION_2=from :: c=-0.783 core=(0.000,0.000,-1.000,0.000,-11.882,-9.781,-5.881,-3.635,1.000,0.000) 
+TRANSLATION_OPTION_3=money :: c=-0.680 core=(0.000,0.000,-1.000,0.000,-7.083,-6.272,-3.935,-2.920,1.000,0.000) 
+TRANSLATION_OPTION_4=money transfers :: c=-0.797 core=(0.000,0.000,-2.000,0.000,-0.847,-5.598,-4.782,-6.755,1.000,0.000) 
+TRANSLATION_OPTION_5=of transfers :: c=-0.805 core=(0.000,0.000,-2.000,0.000,-2.944,-5.199,-5.881,-4.673,1.000,0.000) 
+TRANSLATION_OPTION_6=payments :: c=-0.816 core=(0.000,0.000,-1.000,0.000,-7.352,-8.544,-5.187,-6.033,1.000,0.000) 
+TRANSLATION_OPTION_7=providing :: c=-0.866 core=(0.000,0.000,-1.000,0.000,-9.478,-8.443,-5.881,-5.117,1.000,0.000) 
+TRANSLATION_OPTION_8=remittance :: c=-0.627 core=(0.000,0.000,-1.000,0.000,-0.426,-1.052,-1.296,-1.567,1.000,0.000) 
+TRANSLATION_OPTION_9=remittance of :: c=-0.726 core=(0.000,0.000,-2.000,0.000,-0.693,-1.052,-5.881,-2.404,1.000,0.000) 
+TRANSLATION_OPTION_10=remittances , :: c=-0.793 core=(0.000,0.000,-2.000,0.000,-2.773,-0.834,-5.881,-4.021,1.000,0.000) 
+TRANSLATION_OPTION_11=remittances , to :: c=-0.864 core=(-4.387,0.000,-3.000,0.000,0.000,-0.834,-5.881,-5.558,1.000,0.000) 
+TRANSLATION_OPTION_12=remittances :: c=-0.611 core=(0.000,0.000,-1.000,0.000,-0.357,-0.834,-0.975,-1.241,1.000,0.000) 
+TRANSLATION_OPTION_13=remittances from :: c=-0.753 core=(0.000,0.000,-2.000,0.000,-0.827,-1.527,-3.935,-4.877,1.000,0.000) 
+TRANSLATION_OPTION_14=represents :: c=-0.852 core=(0.000,0.000,-1.000,0.000,-8.504,-9.216,-5.881,-6.726,1.000,0.000) 
+TRANSLATION_OPTION_15=sending money :: c=-0.835 core=(0.000,0.000,-2.000,0.000,-0.916,-6.398,-5.187,-8.260,1.000,0.000) 
+TRANSLATION_OPTION_16=sent :: c=-0.833 core=(0.000,0.000,-1.000,0.000,-7.791,-7.930,-5.881,-5.117,1.000,0.000) 
+TRANSLATION_OPTION_17=transfer :: c=-0.717 core=(0.000,0.000,-1.000,0.000,-6.775,-6.455,-3.396,-3.468,1.000,0.000) 
+TRANSLATION_OPTION_18=transfer of money :: c=-0.801 core=(-4.658,0.000,-3.000,0.000,-1.792,-6.359,-5.881,-7.225,1.000,0.000) 
+TRANSLATION_OPTION_19=transfers :: c=-0.692 core=(0.000,0.000,-1.000,0.000,-5.651,-5.199,-3.935,-3.836,1.000,0.000) 
+TRANSLATION_OPTION_20=transfers from :: c=-0.759 core=(0.000,0.000,-2.000,0.000,-2.639,-5.882,-5.881,-7.471,1.000,0.000) 
 TRANSLATION_1 = from the west .
-TOTAL_WALLTIME ~ 8
+TOTAL_WALLTIME ~ 5
diff --git a/tests/phrase.source-word-deletion/filter-stderr.pl b/tests/phrase.source-word-deletion/filter-stderr.pl
deleted file mode 100755
index 2f6e176..0000000
--- a/tests/phrase.source-word-deletion/filter-stderr.pl
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/perl
-
-BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
-use RegTestUtils;
-
-$x=0;
-while (<>) {
-  chomp;
-
-  if (/^Finished loading LanguageModels/) {
-    my $time = RegTestUtils::readTime($_);
-    print "LMLOAD_TIME ~ $time\n";
-  }
-  if (/^Finished loading phrase tables/) {
-    my $time = RegTestUtils::readTime($_);
-    print "PTLOAD_TIME ~ $time\n";
-  }
-  next unless /^BEST TRANSLATION:/;
-  my $pscore = RegTestUtils::readHypoScore($_);
-  $x++;
-  print "SCORE_$x = $pscore\n";
-}
diff --git a/tests/phrase.source-word-deletion/filter-stdout.pl b/tests/phrase.source-word-deletion/filter-stdout.pl
deleted file mode 100755
index 476ddf6..0000000
--- a/tests/phrase.source-word-deletion/filter-stdout.pl
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/perl
-$x=0;
-while (<>) {
-  chomp;
-  $x++;
-  print "TRANSLATION_$x=$_\n";
-}
diff --git a/tests/phrase.source-word-deletion/moses.ini b/tests/phrase.source-word-deletion/moses.ini
deleted file mode 100644
index e9dbc5e..0000000
--- a/tests/phrase.source-word-deletion/moses.ini
+++ /dev/null
@@ -1,40 +0,0 @@
-# moses.ini for regression test
-
-# limit on how many phrase translations e for each phrase f are loaded
-[ttable-limit]
-#ttable element load limit 0 = all elements loaded
-20
-
-[distortion-limit]
-4
-
-[beam-threshold]
-0.03
-
-[input-factors]
-0
-
-[mapping]
-T 0
-
-
-[verbose]
-2
-
-[feature]
-WordPenalty
-Distortion
-KENLM lazyken=0 order=3 factor=0 path=${LM_PATH}/europarl.en.srilm.gz
-swd factor=0 path=${TEST_PATH}/words
-PhraseDictionaryMemory num-features=5 input-factor=0 output-factor=0 path=${MODEL_PATH}/ptable-with-alignment/phrase-table.gz
-
-[weight]
-WordPenalty0= -0.273416114951401
-KENLM0= 0.142658800199951
-Distortion0= 0.141806519223522
-PhraseDictionaryMemory0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851 -0.0426081987467227
-
-
-[weight-file]
-${TEST_PATH}/weights
-
diff --git a/tests/phrase.source-word-deletion/to-translate.txt b/tests/phrase.source-word-deletion/to-translate.txt
deleted file mode 100644
index 78d7ca1..0000000
--- a/tests/phrase.source-word-deletion/to-translate.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-vor allem aber die Anwalt
-Hand hat Mehrheit
diff --git a/tests/phrase.source-word-deletion/truth/results.txt b/tests/phrase.source-word-deletion/truth/results.txt
deleted file mode 100644
index 5862cdc..0000000
--- a/tests/phrase.source-word-deletion/truth/results.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-TRANSLATION_1=especially the lawyers 
-TRANSLATION_2=country majority 
-SCORE_1 = -2.819
-SCORE_2 = -0.390
-TOTAL_WALLTIME ~ 7
diff --git a/tests/phrase.source-word-deletion/weights b/tests/phrase.source-word-deletion/weights
deleted file mode 100644
index d921dd1..0000000
--- a/tests/phrase.source-word-deletion/weights
+++ /dev/null
@@ -1,3 +0,0 @@
-swd0_aber 2.0
-swd0_hat 3.0
-swd0_OTHER 4.0
diff --git a/tests/phrase.source-word-deletion/words b/tests/phrase.source-word-deletion/words
deleted file mode 100644
index d26b77d..0000000
--- a/tests/phrase.source-word-deletion/words
+++ /dev/null
@@ -1 +0,0 @@
-aber
diff --git a/tests/phrase.sparse-phrase-table/filter-stderr.pl b/tests/phrase.sparse-phrase-table/filter-stderr.pl
deleted file mode 100755
index 2f6e176..0000000
--- a/tests/phrase.sparse-phrase-table/filter-stderr.pl
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/perl
-
-BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
-use RegTestUtils;
-
-$x=0;
-while (<>) {
-  chomp;
-
-  if (/^Finished loading LanguageModels/) {
-    my $time = RegTestUtils::readTime($_);
-    print "LMLOAD_TIME ~ $time\n";
-  }
-  if (/^Finished loading phrase tables/) {
-    my $time = RegTestUtils::readTime($_);
-    print "PTLOAD_TIME ~ $time\n";
-  }
-  next unless /^BEST TRANSLATION:/;
-  my $pscore = RegTestUtils::readHypoScore($_);
-  $x++;
-  print "SCORE_$x = $pscore\n";
-}
diff --git a/tests/phrase.sparse-phrase-table/filter-stdout.pl b/tests/phrase.sparse-phrase-table/filter-stdout.pl
deleted file mode 100755
index 476ddf6..0000000
--- a/tests/phrase.sparse-phrase-table/filter-stdout.pl
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/perl
-$x=0;
-while (<>) {
-  chomp;
-  $x++;
-  print "TRANSLATION_$x=$_\n";
-}
diff --git a/tests/phrase.sparse-phrase-table/moses.ini b/tests/phrase.sparse-phrase-table/moses.ini
deleted file mode 100644
index 5673ecc..0000000
--- a/tests/phrase.sparse-phrase-table/moses.ini
+++ /dev/null
@@ -1,51 +0,0 @@
-# moses.ini for regression test
-
-[ttable-file]
-0 0 0 5 ${MODEL_PATH}/sparse-phrase-table/phrase-table.gz sparse
-
-# language model
-[lmodel-file]
-8 0 3 ${LM_PATH}/europarl.en.srilm.gz
-# limit on how many phrase translations e for each phrase f are loaded
-[ttable-limit]
-#ttable element load limit 0 = all elements loaded
-20
-
-# distortion (reordering) weight
-[weight-d]
-0.141806519223522
-
-# language model weight
-[weight-l]
-0.142658800199951
-
-# translation model weight (phrase translation, lexical weighting)
-[weight-t]
-0.00402447059454402
-0.0685647475075862
-0.294089113124688
-0.0328320356515851
--0.0426081987467227
-
-# word penalty
-[weight-w]
--0.273416114951401
-
-[distortion-limit]
-4
-
-[beam-threshold]
-0.03
-
-[input-factors]
-0
-
-[mapping]
-T 0
-
-
-[verbose]
-2
-
-[weight-file]
-${TEST_PATH}/weights
diff --git a/tests/phrase.sparse-phrase-table/to-translate.txt b/tests/phrase.sparse-phrase-table/to-translate.txt
deleted file mode 100644
index 92d74ce..0000000
--- a/tests/phrase.sparse-phrase-table/to-translate.txt
+++ /dev/null
@@ -1 +0,0 @@
-a b c c
diff --git a/tests/phrase.sparse-phrase-table/truth/results.txt b/tests/phrase.sparse-phrase-table/truth/results.txt
deleted file mode 100644
index af8d526..0000000
--- a/tests/phrase.sparse-phrase-table/truth/results.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-TRANSLATION_1=it me she she 
-SCORE_1 = -7.448
-TOTAL_WALLTIME ~ 7
diff --git a/tests/phrase.sparse-phrase-table/weights b/tests/phrase.sparse-phrase-table/weights
deleted file mode 100644
index a882b3c..0000000
--- a/tests/phrase.sparse-phrase-table/weights
+++ /dev/null
@@ -1,3 +0,0 @@
-SparsePhraseDictionaryFeature0_po 0.5
-SparsePhraseDictionaryFeature0_io -0.01
-SparsePhraseDictionaryFeature0_o 0.1