change multimodel tests to new phrase table format without phrase penalty.

differences in score are due to the fact that the new PhrasePenalty feature function also triggers for unknown words.

12 files changed, 33 insertions, 27 deletions

diff --git a/models/multimodel/model1/phrase-table.bin.binphr.srctree.wa b/models/multimodel/model1/phrase-table.bin.binphr.srctree.wa
index c7a785b..9026fcf 100644
--- a/models/multimodel/model1/phrase-table.bin.binphr.srctree.wa
+++ b/models/multimodel/model1/phrase-table.bin.binphr.srctree.wa
diff --git a/models/multimodel/model1/phrase-table.bin.binphr.tgtdata.wa b/models/multimodel/model1/phrase-table.bin.binphr.tgtdata.wa
index 8848033..921b945 100644
--- a/models/multimodel/model1/phrase-table.bin.binphr.tgtdata.wa
+++ b/models/multimodel/model1/phrase-table.bin.binphr.tgtdata.wa
diff --git a/models/multimodel/model1/phrase-table.gz b/models/multimodel/model1/phrase-table.gz
index 4273617..a7c45a8 100644
--- a/models/multimodel/model1/phrase-table.gz
+++ b/models/multimodel/model1/phrase-table.gz
diff --git a/models/multimodel/model2/phrase-table.bin.binphr.srctree.wa b/models/multimodel/model2/phrase-table.bin.binphr.srctree.wa
index 3582944..582a185 100644
--- a/models/multimodel/model2/phrase-table.bin.binphr.srctree.wa
+++ b/models/multimodel/model2/phrase-table.bin.binphr.srctree.wa
diff --git a/models/multimodel/model2/phrase-table.bin.binphr.tgtdata.wa b/models/multimodel/model2/phrase-table.bin.binphr.tgtdata.wa
index 3c2b0a7..45cd0b9 100644
--- a/models/multimodel/model2/phrase-table.bin.binphr.tgtdata.wa
+++ b/models/multimodel/model2/phrase-table.bin.binphr.tgtdata.wa
diff --git a/models/multimodel/model2/phrase-table.gz b/models/multimodel/model2/phrase-table.gz
index 59f3666..ba573b2 100644
--- a/models/multimodel/model2/phrase-table.gz
+++ b/models/multimodel/model2/phrase-table.gz
diff --git a/tests/phrase.multimodel-binptable/moses.ini b/tests/phrase.multimodel-binptable/moses.ini
index d6cbe7c..a6a77fb 100644
--- a/tests/phrase.multimodel-binptable/moses.ini
+++ b/tests/phrase.multimodel-binptable/moses.ini
@@ -19,13 +19,15 @@ T 2
 UnknownWordPenalty
 WordPenalty
 Distortion
-PhraseDictionaryBinary tuneable=false num-features=5 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/phrase-table.bin table-limit=20
-PhraseDictionaryBinary tuneable=false num-features=5 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/phrase-table.bin table-limit=20
-PhraseDictionaryMultiModel num-features=5 input-factor=0 output-factor=0 table-limit=20 mode=interpolate lambda=0.2,0.8 components=PhraseDictionaryBinary0,PhraseDictionaryBinary1
+PhrasePenalty
+PhraseDictionaryBinary tuneable=false num-features=4 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/phrase-table.bin table-limit=20
+PhraseDictionaryBinary tuneable=false num-features=4 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/phrase-table.bin table-limit=20
+PhraseDictionaryMultiModel num-features=4 input-factor=0 output-factor=0 table-limit=20 mode=interpolate lambda=0.2,0.8 components=PhraseDictionaryBinary0,PhraseDictionaryBinary1
 
 [weight]
 WordPenalty0= -0.273416114951401
 Distortion0= 0.141806519223522
-PhraseDictionaryMultiModel0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851 -0.0426081987467227
-PhraseDictionaryBinary0= 0 0 1 0 0
-PhraseDictionaryBinary1= 0 0 1 0 0
+PhrasePenalty0= -0.0426081987467227
+PhraseDictionaryMultiModel0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851
+PhraseDictionaryBinary0= 0 0 1 0
+PhraseDictionaryBinary1= 0 0 1 0
diff --git a/tests/phrase.multimodel-binptable/truth/results.txt b/tests/phrase.multimodel-binptable/truth/results.txt
index da0190b..440183e 100644
--- a/tests/phrase.multimodel-binptable/truth/results.txt
+++ b/tests/phrase.multimodel-binptable/truth/results.txt
@@ -4,10 +4,10 @@ TRANSLATION_3=the minutes for ' s sitting tomorrow thursday , the 3. february 's
 TRANSLATION_4=are there any comments ? 
 TRANSLATION_5=from their reaktion schließe i , that many members of the when this minutes 't obtain have , and it versteht of that kind to what is from even , that you a minutes , it you 't obtain have , 't zustimmen can . 
 TRANSLATION_6=i schlage so a , it minutes tomorrow to , there it the opportunity to propose to you however , now 't vorliegt , wofür i me entschuldige . 
-SCORE_1 = -301.671
+SCORE_1 = -301.799
 SCORE_2 = 0.036
-SCORE_3 = -102.520
+SCORE_3 = -102.562
 SCORE_4 = -0.102
-SCORE_5 = -410.750
-SCORE_6 = -402.951
-TOTAL_WALLTIME ~ 3
+SCORE_5 = -410.920
+SCORE_6 = -403.121
+TOTAL_WALLTIME ~ 3
\ No newline at end of file
diff --git a/tests/phrase.multimodel/moses.ini b/tests/phrase.multimodel/moses.ini
index f7d5801..d15c4d0 100644
--- a/tests/phrase.multimodel/moses.ini
+++ b/tests/phrase.multimodel/moses.ini
@@ -19,13 +19,15 @@ T 2
 UnknownWordPenalty
 WordPenalty
 Distortion
-PhraseDictionaryMemory tuneable=false num-features=5 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/phrase-table.gz table-limit=20
-PhraseDictionaryMemory tuneable=false num-features=5 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/phrase-table.gz table-limit=20
-PhraseDictionaryMultiModel num-features=5 input-factor=0 output-factor=0 table-limit=20 mode=interpolate lambda=0.2,0.8 components=PhraseDictionaryMemory0,PhraseDictionaryMemory1
+PhrasePenalty
+PhraseDictionaryMemory tuneable=false num-features=4 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/phrase-table.gz table-limit=20
+PhraseDictionaryMemory tuneable=false num-features=4 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/phrase-table.gz table-limit=20
+PhraseDictionaryMultiModel num-features=4 input-factor=0 output-factor=0 table-limit=20 mode=interpolate lambda=0.2,0.8 components=PhraseDictionaryMemory0,PhraseDictionaryMemory1
 
 [weight]
 WordPenalty0= -0.273416114951401
 Distortion0= 0.141806519223522
-PhraseDictionaryMultiModel0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851 -0.0426081987467227
-PhraseDictionaryMemory0= 0 0 1 0 0
-PhraseDictionaryMemory1= 0 0 1 0 0
+PhrasePenalty0= -0.0426081987467227
+PhraseDictionaryMultiModel0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851
+PhraseDictionaryMemory0= 0 0 1 0
+PhraseDictionaryMemory1= 0 0 1 0
diff --git a/tests/phrase.multimodel/truth/results.txt b/tests/phrase.multimodel/truth/results.txt
index da0190b..ac28a55 100644
--- a/tests/phrase.multimodel/truth/results.txt
+++ b/tests/phrase.multimodel/truth/results.txt
@@ -4,10 +4,10 @@ TRANSLATION_3=the minutes for ' s sitting tomorrow thursday , the 3. february 's
 TRANSLATION_4=are there any comments ? 
 TRANSLATION_5=from their reaktion schließe i , that many members of the when this minutes 't obtain have , and it versteht of that kind to what is from even , that you a minutes , it you 't obtain have , 't zustimmen can . 
 TRANSLATION_6=i schlage so a , it minutes tomorrow to , there it the opportunity to propose to you however , now 't vorliegt , wofür i me entschuldige . 
-SCORE_1 = -301.671
+SCORE_1 = -301.799
 SCORE_2 = 0.036
-SCORE_3 = -102.520
+SCORE_3 = -102.562
 SCORE_4 = -0.102
-SCORE_5 = -410.750
-SCORE_6 = -402.951
+SCORE_5 = -410.920
+SCORE_6 = -403.121
 TOTAL_WALLTIME ~ 3
diff --git a/tests/phrase.multimodelcounts/moses.ini b/tests/phrase.multimodelcounts/moses.ini
index b7271f7..b82ae00 100644
--- a/tests/phrase.multimodelcounts/moses.ini
+++ b/tests/phrase.multimodelcounts/moses.ini
@@ -20,18 +20,20 @@ T 4
 UnknownWordPenalty
 WordPenalty
 Distortion
+PhrasePenalty
 PhraseDictionaryMemory tuneable=false num-features=3 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/count-table.gz table-limit=20
 PhraseDictionaryMemory tuneable=false num-features=3 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/count-table.gz table-limit=20
 
 PhraseDictionaryMemory tuneable=false filterable=false num-features=1 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model1/count-table-target.gz
 PhraseDictionaryMemory tuneable=false filterable=false num-features=1 input-factor=0 output-factor=0 path=${MODEL_PATH}/multimodel/model2/count-table-target.gz
 
-PhraseDictionaryMultiModelCounts num-features=5 input-factor=0 output-factor=0 table-limit=20 mode=instance_weighting lambda=1.0,1.0 components=PhraseDictionaryMemory0,PhraseDictionaryMemory1 target-table=PhraseDictionaryMemory2,PhraseDictionaryMemory3 lex-e2f=${MODEL_PATH}/multimodel/model1/lex.counts.e2f,${MODEL_PATH}/multimodel/model2/lex.counts.e2f lex-f2e=${MODEL_PATH}/multimodel/model1/lex.counts.f2e,${MODEL_PATH}/multimodel/model2/lex.counts.f2e
+PhraseDictionaryMultiModelCounts num-features=4 input-factor=0 output-factor=0 table-limit=20 mode=instance_weighting lambda=1.0,1.0 components=PhraseDictionaryMemory0,PhraseDictionaryMemory1 target-table=PhraseDictionaryMemory2,PhraseDictionaryMemory3 lex-e2f=${MODEL_PATH}/multimodel/model1/lex.counts.e2f,${MODEL_PATH}/multimodel/model2/lex.counts.e2f lex-f2e=${MODEL_PATH}/multimodel/model1/lex.counts.f2e,${MODEL_PATH}/multimodel/model2/lex.counts.f2e
 
 [weight]
 WordPenalty0= -0.273416114951401
 Distortion0= 0.141806519223522
-PhraseDictionaryMultiModelCounts0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851 -0.0426081987467227
+PhraseDictionaryMultiModelCounts0= 0.00402447059454402 0.0685647475075862 0.294089113124688 0.0328320356515851
+PhrasePenalty0= -0.0426081987467227
 
 PhraseDictionaryMemory0= 1 0 0
 PhraseDictionaryMemory1= 1 0 0
diff --git a/tests/phrase.multimodelcounts/truth/results.txt b/tests/phrase.multimodelcounts/truth/results.txt
index d468725..c5aa7be 100644
--- a/tests/phrase.multimodelcounts/truth/results.txt
+++ b/tests/phrase.multimodelcounts/truth/results.txt
@@ -4,10 +4,10 @@ TRANSLATION_3=the minutes for ' s sitting tomorrow has been set , unfortunately
 TRANSLATION_4=are there any comments ? 
 TRANSLATION_5=are to succeed this state aid code their reaktion schließe , i that many members of the when this minutes not obtain have , and it versteht of that kind to what is of even , that you a minutes , none of this is to they do not obtain have , not zustimmen citizens . 
 TRANSLATION_6=i schlage therefore meps , the minutes tomorrow to , there it the opportunity to propose to you however , not vorliegt , wofür i me entschuldige . 
-SCORE_1 = -299.073
+SCORE_1 = -299.201
 SCORE_2 = 1.311
-SCORE_3 = -99.917
+SCORE_3 = -99.959
 SCORE_4 = 0.503
-SCORE_5 = -403.499
-SCORE_6 = -401.264
+SCORE_5 = -403.669
+SCORE_6 = -401.434
 TOTAL_WALLTIME ~ 2