make translation model combination more general: allow non-default number and position of phrase table features

14 files changed, 365 insertions, 202 deletions

diff --git a/contrib/tmcombine/test/model3/model/lex.counts.e2f b/contrib/tmcombine/test/model3/model/lex.counts.e2f
new file mode 100644
index 000000000..ed05c0b7d
--- /dev/null
+++ b/contrib/tmcombine/test/model3/model/lex.counts.e2f
@@ -0,0 +1,8 @@
+ad af 500 1000
+bd bf 5 10
+der le 20285 102586
+der NULL 12926 704917
+gipfel sommet 3485 7322
+pass col 419 2911
+pass passeport 7 28
+sitzung séance 14 59
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model3/model/lex.counts.f2e b/contrib/tmcombine/test/model3/model/lex.counts.f2e
new file mode 100644
index 000000000..ea31f690d
--- /dev/null
+++ b/contrib/tmcombine/test/model3/model/lex.counts.f2e
@@ -0,0 +1,8 @@
+af ad 500 1000
+bf bd 5 10
+col pass 419 615
+le der 20285 113635
+passeport pass 7 615
+retrouvé NULL 34 1016136
+séance sitzung 14 33
+sommet gipfel 3485 5700
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model3/model/lex.e2f b/contrib/tmcombine/test/model3/model/lex.e2f
new file mode 100644
index 000000000..f9263ffe5
--- /dev/null
+++ b/contrib/tmcombine/test/model3/model/lex.e2f
@@ -0,0 +1,8 @@
+ad af 0.5
+bd bf 0.5
+der le 0.1977365
+der NULL 0.0183369
+gipfel sommet 0.4759629
+pass col 0.1439368
+pass passeport 0.2500000
+sitzung séance 0.2372881
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model3/model/lex.f2e b/contrib/tmcombine/test/model3/model/lex.f2e
new file mode 100644
index 000000000..2bba51f01
--- /dev/null
+++ b/contrib/tmcombine/test/model3/model/lex.f2e
@@ -0,0 +1,8 @@
+af ad 0.5
+bf bd 0.5
+col pass 0.6813008
+le der 0.1785101
+passeport pass 0.0113821
+retrouvé NULL 0.0000335
+séance sitzung 0.4242424
+sommet gipfel 0.6114035
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model3/model/phrase-table b/contrib/tmcombine/test/model3/model/phrase-table
new file mode 100644
index 000000000..f5c8647de
--- /dev/null
+++ b/contrib/tmcombine/test/model3/model/phrase-table
@@ -0,0 +1,8 @@
+ad ||| af ||| 0.3 0.3 0.3 0.3 0.5 0.5 0.5 0.5 2.718 ||| 0-0 ||| 1000 1000
+bd ||| bf ||| 0.3 0.3 0.3 0.3 0.5 0.5 0.5 0.5 2.718 ||| 0-0 ||| 10 10
+der gipfel ||| sommet ||| 0.3 0.3 0.3 0.3 0.00327135 0.00872768 0.0366795 0.611403 2.718 ||| 1-0 ||| 5808 518
+der pass ||| le col ||| 0.3 0.3 0.3 0.3 0.0173565 0.0284616 0.288889 0.121619 2.718 ||| 0-0 1-1 ||| 749 45
+pass ||| col ||| 0.3 0.3 0.3 0.3 0.1952 0.143937 0.628866 0.681301 2.718 ||| 0-0 ||| 1875 582
+pass ||| passeport retrouvé ||| 0.3 0.3 0.3 0.3 0.5 0.25 0.00171821 3.813e-07 2.718 ||| 0-0 ||| 2 582
+pass ||| passeport ||| 0.3 0.3 0.3 0.3 0.266667 0.25 0.00687285 0.0113821 2.718 ||| 0-0 ||| 15 582
+sitzung ||| séance ||| 0.3 0.3 0.3 0.3 0.272727 0.237288 0.352941 0.424242 2.718 ||| 0-0 ||| 22 17
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model4/model/lex.counts.e2f b/contrib/tmcombine/test/model4/model/lex.counts.e2f
new file mode 100644
index 000000000..8475fcdf9
--- /dev/null
+++ b/contrib/tmcombine/test/model4/model/lex.counts.e2f
@@ -0,0 +1,8 @@
+ad af 100 1000
+bd bf 1 10
+der le 150181 944391
+der NULL 54483 3595140
+gipfel sommet 3421 9342
+pass col 2 70
+pass passeport 73 379
+sitzung séance 3441 5753
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model4/model/lex.counts.f2e b/contrib/tmcombine/test/model4/model/lex.counts.f2e
new file mode 100644
index 000000000..b0913088a
--- /dev/null
+++ b/contrib/tmcombine/test/model4/model/lex.counts.f2e
@@ -0,0 +1,8 @@
+af ad 100 1000
+bf bd 1 10
+col pass 2 108
+le der 150181 1356104
+passeport pass 73 108
+retrouvé NULL 43 6276240
+séance sitzung 3441 6142
+sommet gipfel 3421 4908
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model4/model/lex.e2f b/contrib/tmcombine/test/model4/model/lex.e2f
new file mode 100644
index 000000000..b1ce3a613
--- /dev/null
+++ b/contrib/tmcombine/test/model4/model/lex.e2f
@@ -0,0 +1,8 @@
+ad af 0.1
+bd bf 0.1
+der le 0.1590242
+der NULL 0.0151546
+gipfel sommet 0.366195
+pass col 0.0285714
+pass passeport 0.1926121
+sitzung séance 0.5981227
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model4/model/lex.f2e b/contrib/tmcombine/test/model4/model/lex.f2e
new file mode 100644
index 000000000..d931dcb72
--- /dev/null
+++ b/contrib/tmcombine/test/model4/model/lex.f2e
@@ -0,0 +1,8 @@
+af ad 0.1
+bf bd 0.1
+col pass 0.0185185
+le der 0.1107445
+passeport pass 0.6759259
+retrouvé NULL 0.0000069
+séance sitzung 0.5602410
+sommet gipfel 0.6970253
\ No newline at end of file
diff --git a/contrib/tmcombine/test/model4/model/phrase-table b/contrib/tmcombine/test/model4/model/phrase-table
new file mode 100644
index 000000000..494b6a37f
--- /dev/null
+++ b/contrib/tmcombine/test/model4/model/phrase-table
@@ -0,0 +1,5 @@
+ad ||| af ||| 0.6 0.6 0.6 0.6 0.1 0.1 0.1 0.1 2.718 ||| 0-0 ||| 1000 1000
+bd ||| bf ||| 0.6 0.6 0.6 0.6 0.1 0.1 0.1 0.1 2.718 ||| 0-0 ||| 10 10
+der pass ||| le passeport ||| 0.6 0.6 0.6 0.6 0.16 0.03063 0.4 0.0748551 2.718 ||| 0-0 1-1 ||| 25 10
+pass ||| passeport ||| 0.6 0.6 0.6 0.6 0.28022 0.192612 0.607143 0.675926 2.718 ||| 0-0 ||| 182 84
+sitzung ||| séance ||| 0.6 0.6 0.6 0.6 0.784521 0.598123 0.516654 0.560241 2.718 ||| 0-0 ||| 4251 6455
\ No newline at end of file
diff --git a/contrib/tmcombine/test/phrase-table_test10 b/contrib/tmcombine/test/phrase-table_test10
new file mode 100644
index 000000000..ee2aebeb1
--- /dev/null
+++ b/contrib/tmcombine/test/phrase-table_test10
@@ -0,0 +1,9 @@
+ad ||| af ||| 0.3 0.3 0.3 0.3 0.11579 0.35574 0.472359 0.469238 2.718 ||| 0-0 ||| 25332.4712297 1074.23173673
+bd ||| bf ||| 0.3 0.3 0.3 0.3 0.11579 0.35574 0.472359 0.469238 2.718 ||| 0-0 ||| 253.324712297 10.7423173673
+der gipfel ||| sommet ||| 0.3 0.3 0.3 0.3 0.00327135 0.00686984 0.0366795 0.617135 2.718 ||| 1-0 ||| 5808.0 518.0
+der pass ||| le col ||| 0.3 0.3 0.3 0.3 0.0173565 0.023534 0.284201 0.0972183 2.718 ||| 0-0 1-1 ||| 749.0 45.7423173673
+der pass ||| le passeport ||| 6e-10 6e-10 6e-10 6e-10 0.16 0.0329324 0.0064913 0.00303408 2.718 ||| 0-0 1-1 ||| 608.311780741 45.7423173673
+pass ||| col ||| 0.3 0.3 0.3 0.3 0.1952 0.142393 0.6222 0.671744 2.718 ||| 0-0 ||| 1875.0 588.235465885
+pass ||| passeport retrouvé ||| 0.3 0.3 0.3 0.3 0.5 0.199258 0.0017 5.11945e-07 2.718 ||| 0-0 ||| 2.0 588.235465885
+pass ||| passeport ||| 0.3 0.3 0.3 0.3 0.280174 0.199258 0.0132359 0.0209644 2.718 ||| 0-0 ||| 4443.5097638 588.235465885
+sitzung ||| séance ||| 0.3 0.3 0.3 0.3 0.784412 0.59168 0.511045 0.552002 2.718 ||| 0-0 ||| 103459.335197 496.165860589
diff --git a/contrib/tmcombine/test/phrase-table_test8 b/contrib/tmcombine/test/phrase-table_test8
index 74eb27c0c..f0776cd80 100644
--- a/contrib/tmcombine/test/phrase-table_test8
+++ b/contrib/tmcombine/test/phrase-table_test8
@@ -1,9 +1,9 @@
-ad ||| af ||| 0.5 0.398085 0.5 0.482814 2.718 ||| 0-0 ||| 1000.000001 1000.000001
-bd ||| bf ||| 0.5 0.111367 0.5 0.172867 2.718 ||| 0-0 ||| 10.00000001 10.00000001
+ad ||| af ||| 0.242966 0.398085 0.483231 0.482814 2.718 ||| 0-0 ||| 2797.86490081 1043.7557397
+bd ||| bf ||| 0.102213 0.111367 0.174411 0.172867 2.718 ||| 0-0 ||| 1807.86490081 53.7557396976
 der gipfel ||| sommet ||| 0.00327135 0.00863717 0.0366795 0.612073 2.718 ||| 1-0 ||| 5808.0 518.0
-der pass ||| le col ||| 0.0173565 0.0260469 0.288889 0.113553 2.718 ||| 0-0 1-1 ||| 749.0 45.00000001
-der pass ||| le passeport ||| 0.0064 0.0389201 8.88889e-12 0.0101009 2.718 ||| 0-0 1-1 ||| 2.5e-08 45.00000001
-pass ||| col ||| 0.1952 0.131811 0.628866 0.63621 2.718 ||| 0-0 ||| 1875.0 582.000000084
-pass ||| passeport retrouvé ||| 0.5 0.196956 0.00171821 1.89355e-06 2.718 ||| 0-0 ||| 2.0 582.000000084
-pass ||| passeport ||| 0.266667 0.196956 0.00687285 0.0565932 2.718 ||| 0-0 ||| 15.000000182 582.000000084
-sitzung ||| séance ||| 0.272727 0.545019 0.352941 0.502625 2.718 ||| 0-0 ||| 22.000004251 17.000006455
+der pass ||| le col ||| 0.0173565 0.0260469 0.146469 0.113553 2.718 ||| 0-0 1-1 ||| 749.0 88.7557396976
+der pass ||| le passeport ||| 0.16 0.0389201 0.197196 0.0101009 2.718 ||| 0-0 1-1 ||| 1797.86490081 88.7557396976
+pass ||| col ||| 0.1952 0.131811 0.584893 0.63621 2.718 ||| 0-0 ||| 1875.0 625.755739698
+pass ||| passeport retrouvé ||| 0.5 0.196956 0.00159806 1.89355e-06 2.718 ||| 0-0 ||| 2.0 625.755739698
+pass ||| passeport ||| 0.280108 0.196956 0.0488465 0.0565932 2.718 ||| 0-0 ||| 1812.86490081 625.755739698
+sitzung ||| séance ||| 0.778334 0.545019 0.470846 0.502625 2.718 ||| 0-0 ||| 1819.86490081 60.7557396976
diff --git a/contrib/tmcombine/test/phrase-table_test9 b/contrib/tmcombine/test/phrase-table_test9
new file mode 100644
index 000000000..017c97854
--- /dev/null
+++ b/contrib/tmcombine/test/phrase-table_test9
@@ -0,0 +1,9 @@
+ad ||| af ||| 0.45 0.45 0.45 0.45 0.14 0.136364 0.18 0.3 2.718 ||| 0-0 ||| 10000.0 5000.0
+bd ||| bf ||| 0.45 0.45 0.45 0.45 0.14 0.136364 0.18 0.3 2.718 ||| 0-0 ||| 100.0 50.0
+der gipfel ||| sommet ||| 0.15 0.15 0.15 0.15 0.00327135 0.00569336 0.0366795 0.651018 2.718 ||| 1-0 ||| 5808.0 518.0
+der pass ||| le col ||| 0.15 0.15 0.15 0.15 0.0173565 0.0193836 0.152941 0.0675369 2.718 ||| 0-0 1-1 ||| 749.0 85.0
+der pass ||| le passeport ||| 0.3 0.3 0.3 0.3 0.16 0.0307772 0.188235 0.0128336 2.718 ||| 0-0 1-1 ||| 225.0 85.0
+pass ||| col ||| 0.15 0.15 0.15 0.15 0.1952 0.121573 0.398693 0.582296 2.718 ||| 0-0 ||| 1875.0 918.0
+pass ||| passeport retrouvé ||| 0.15 0.15 0.15 0.15 0.5 0.193033 0.00108932 1.16835e-06 2.718 ||| 0-0 ||| 2.0 918.0
+pass ||| passeport ||| 0.45 0.45 0.45 0.45 0.280097 0.193033 0.22658 0.11065 2.718 ||| 0-0 ||| 1653.0 918.0
+sitzung ||| séance ||| 0.45 0.45 0.45 0.45 0.784227 0.597753 0.516546 0.559514 2.718 ||| 0-0 ||| 38281.0 25837.0
diff --git a/contrib/tmcombine/tmcombine.py b/contrib/tmcombine/tmcombine.py
index e166c33c8..f7696fbfe 100755
--- a/contrib/tmcombine/tmcombine.py
+++ b/contrib/tmcombine/tmcombine.py
@@ -36,7 +36,10 @@ from collections import defaultdict
 from operator import mul
 from tempfile import NamedTemporaryFile
 from subprocess import Popen
-from itertools import izip
+try:
+    from itertools import izip
+except:
+    izip = zip
 
 try:
     from lxml import etree as ET
@@ -85,76 +88,9 @@ class Moses():
         filename = os.path.join(model,'model',table)
         fileobj = handle_file(filename,'open',mode)
         return fileobj
-
-
-    def load_phrase_counts(self,line,priority,i,store='all',filter_by=None,filter_by_src=None,filter_by_target=None,inverted=False):
-        """take single phrase table line and store counts in internal data structure"""
         
-        src = line[0]
-        target = line[1]
         
-        if inverted:
-            src,target = target,src
-        
-        target_count = 0
-        src_count = 0
-            
-        if (store == 'all' or store == 'pairs') and not (filter_by and not (src in filter_by and target in filter_by[src])):
-            
-            if priority < 10 or (src in self.phrase_pairs and target in self.phrase_pairs[src]):
-
-                try:
-                    target_count,src_count = map(float,line[-1].split())
-                except:
-                    sys.stderr.write(str(line)+'\n')
-                    sys.stderr.write('Counts are missing. Maybe you have an older Moses version without counts?\n')
-                    return
-
-                scores = line[2].split()
-                pst = float(scores[0])
-                pts = float(scores[2])
-                
-                if priority == 2: #MAP
-                    self.phrase_pairs[src][target][0][0][i] = pst
-                    self.phrase_pairs[src][target][0][1][i] = pts
-                else:
-                    self.phrase_pairs[src][target][0][0][i] = pst*target_count
-                    self.phrase_pairs[src][target][0][1][i] = pts*src_count
-
-                self.store_info(src,target,line)
-        
-        if (store == 'all' or store == 'source') and not (filter_by_src and not filter_by_src.get(src)):
-
-            if not src_count:
-                try:
-                    if priority == 2: #MAP
-                        src_count = 1
-                    else:
-                        src_count = float(line[-1].split()[1])
-                except:
-                    sys.stderr.write(str(line)+'\n')
-                    sys.stderr.write('Counts are missing. Maybe you have an older Moses version without counts?\n')
-                    return
-                    
-            self.phrase_source[src][i] = src_count
-            
-        if (store == 'all' or store == 'target') and not (filter_by_target and not filter_by_target.get(target)):
-            
-            if not target_count:
-                try:
-                    if priority == 2: #MAP
-                        target_count = 1
-                    else:
-                        target_count = float(line[-1].split()[0])
-                except:
-                    sys.stderr.write(str(line)+'\n')
-                    sys.stderr.write('Counts are missing. Maybe you have an older Moses version without counts?\n')
-                    return
-                    
-            self.phrase_target[target][i] = target_count
-        
-        
-    def load_phrase_probabilities(self,line,priority,i,store='pairs',filter_by=None,filter_by_src=None,filter_by_target=None,inverted=False):
+    def load_phrase_features(self,line,priority,i,mode='interpolate',store='pairs',filter_by=None,filter_by_src=None,filter_by_target=None,inverted=False,flags=None):
         """take single phrase table line and store probablities in internal data structure"""
         
         src = line[0]
@@ -167,18 +103,54 @@ class Moses():
             
                 self.store_info(src,target,line)
                 
-                model_probabilities = map(float,line[2].split()[:-1])
+                scores = line[2].split()
+                if len(scores) <self.number_of_features:
+                    sys.stderr.write('Error: model only has {0} features. Expected {1}.\n'.format(len(scores),self.number_of_features))
+                    exit()
+                    
+                scores = scores[:self.number_of_features]
+                model_probabilities = map(float,scores)
                 phrase_probabilities = self.phrase_pairs[src][target][0]
                 
+                if mode == 'counts' and not priority == 2: #priority 2 is MAP
+                    try:
+                        target_count,src_count = map(float,line[-1].split())
+                    except:
+                        sys.stderr.write(str(line)+'\n')
+                        sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
+                        return
+                        
+                    i_e2f = flags['i_e2f']
+                    i_f2e = flags['i_f2e']
+                    model_probabilities[i_e2f] *= target_count
+                    model_probabilities[i_f2e] *= src_count
+                
                 for j,p in enumerate(model_probabilities):
                     phrase_probabilities[j][i] = p
                 
         # mark that the src/target phrase has been seen.
         # needed for re-normalization during linear interpolation
         if (store == 'all' or store == 'source') and not (filter_by_src and not src in filter_by_src):
-            self.phrase_source[src][i] = 1
+            if mode == 'counts' and not priority == 2: #priority 2 is MAP
+                try:
+                    self.phrase_source[src][i] = float(line[-1].split()[1])
+                except:
+                    sys.stderr.write(str(line)+'\n')
+                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
+                    return
+            else:
+                self.phrase_source[src][i] = 1
+                
         if (store == 'all' or store == 'target') and not (filter_by_target and not target in filter_by_target):
-            self.phrase_target[target][i] = 1
+            if mode == 'counts' and not priority == 2: #priority 2 is MAP
+                try:
+                    self.phrase_target[target][i] = float(line[-1].split()[0])
+                except:
+                    sys.stderr.write(str(line)+'\n')
+                    sys.stderr.write('Counts are missing. Maybe your phrase table is from an older Moses version that doesn\'t store counts?\n')
+                    return
+            else:
+                self.phrase_target[target][i] = 1
 
 
     def load_reordering_probabilities(self,line,priority,i,store='pairs'):
@@ -194,8 +166,7 @@ class Moses():
             reordering_probabilities[j][i] = p
 
 
-
-    def traverse_incrementally(self,table,models,load_lines,store_flag,inverted=False,lowmem=False):
+    def traverse_incrementally(self,table,models,load_lines,store_flag,mode='interpolate',inverted=False,lowmem=False,flags=None):
         """hack-ish way to find common phrase pairs in multiple models in one traversal without storing it all in memory
             relies on alphabetical sorting of phrase table.
         """
@@ -218,7 +189,7 @@ class Moses():
                     if increment != stack[i][0]:
                         continue
                     else:
-                        load_lines(stack[i],priority,i,store=store_flag,inverted=inverted)
+                        load_lines(stack[i],priority,i,mode=mode,store=store_flag,inverted=inverted,flags=flags)
                         stack[i] = ''
                     
                 for line in model:
@@ -229,7 +200,7 @@ class Moses():
                         stack[i] = line
                         break
                         
-                    load_lines(line,priority,i,store=store_flag,inverted=inverted)
+                    load_lines(line,priority,i,mode=mode,store=store_flag,inverted=inverted,flags=flags)
                 
             yield 1
 
@@ -255,7 +226,7 @@ class Moses():
             self.word_pairs_f2e[a][b][i] = float(prob)
     
 
-    def load_word_counts(self,line,side,i,priority,e2f_filter=None,f2e_filter=None):
+    def load_word_counts(self,line,side,i,priority,e2f_filter=None,f2e_filter=None,flags=None):
         """process single line of lexical table"""
         
         a, b, ab_count, b_count = line.split(b' ')
@@ -389,8 +360,10 @@ class Moses():
         # information specific to Moses model: alignment info and comment section with target and source counts
         alignment,comments = self.phrase_pairs[src][target][1]
         if mode == 'counts':
-            srccount =  dot_product(self.phrase_source[src],weights[2])
-            targetcount = dot_product(self.phrase_target[target],weights[0])
+            i_e2f = flags['i_e2f']
+            i_f2e = flags['i_f2e']
+            srccount =  dot_product(self.phrase_source[src],weights[i_f2e])
+            targetcount = dot_product(self.phrase_target[target],weights[i_e2f])
             comments = b"%s %s" %(targetcount,srccount)
             
         features = b' '.join([b'%.6g' %(f) for f in features])
@@ -471,7 +444,8 @@ class Moses():
 
 
     def merge(self,pt_normal, pt_inverse, pt_out, mode='interpolate'):
-        """merge two phrasetables (the latter having been inverted to calculate p(s|t) and lex(s|t) in sorted order)"""
+        """merge two phrasetables (the latter having been inverted to calculate p(s|t) and lex(s|t) in sorted order)
+           Assumes that p(s|t) and lex(s|t) are in first table half, p(t|s) and lex(t|s) in second"""
         
         for line,line2 in izip(pt_normal,pt_inverse):
             
@@ -479,9 +453,10 @@ class Moses():
             line2 = line2.split(b' ||| ')
             
             #scores
+            mid = int(self.number_of_features/2)
             scores1 = line[2].split()
             scores2 = line2[2].split()
-            line[2] = b' '.join(scores2[:2]+scores1[2:])
+            line[2] = b' '.join(scores2[:mid]+scores1[mid:])
             
             # marginal counts
             if mode == 'counts':
@@ -722,17 +697,14 @@ def cross_entropy(model_interface,reference_interface,weights,score,mode,flags):
        don't call this directly, but use one of the Combine_TMs methods.
     """
 
-    weights = normalize_weights(weights,mode)
+    weights = normalize_weights(weights,mode,flags)
     
     if 'compare_cross-entropies' in flags and flags['compare_cross-entropies']:
         num_results = len(model_interface.models)
     else:
         num_results = 1
     
-    cross_entropy_ts = [0]*num_results
-    cross_entropy_st = [0]*num_results
-    cross_entropy_lex_ts = [0]*num_results
-    cross_entropy_lex_st = [0]*num_results
+    cross_entropies = [[0]*num_results for i in range(model_interface.number_of_features)]
     oov = [0]*num_results
     oov2 = 0
     other_translations = [0]*num_results
@@ -780,11 +752,8 @@ def cross_entropy(model_interface,reference_interface,weights,score,mode,flags):
                         continue
                         
                     n[i] += c
-                    cross_entropy_ts[i] += -log(features[2],2)*c
-                    cross_entropy_st[i] += -log(features[0],2)*c
-                    
-                    cross_entropy_lex_ts[i] += -log(features[3],2)*c
-                    cross_entropy_lex_st[i] += -log(features[1],2)*c
+                    for j in range(model_interface.number_of_features):
+                        cross_entropies[j][i] -= log(features[j],2)*c
 
                 elif src in model_interface.phrase_source and not ('compare_cross-entropies' in flags and flags['compare_cross-entropies']):
                     other_translations[i] += c
@@ -799,22 +768,18 @@ def cross_entropy(model_interface,reference_interface,weights,score,mode,flags):
     
     for i in range(num_results):
         try:
-            cross_entropy_ts[i] /= n[i]
-            cross_entropy_st[i] /= n[i]
-            cross_entropy_lex_ts[i] /= n[i]
-            cross_entropy_lex_st[i] /= n[i]
+            for j in range(model_interface.number_of_features):
+                cross_entropies[j][i] /= n[i]
         except ZeroDivisionError:
             sys.stderr.write('Warning: no matching phrase pairs between reference set and model\n')
-            cross_entropy_ts[i] = 0
-            cross_entropy_st[i] = 0
-            cross_entropy_lex_ts[i] = 0
-            cross_entropy_lex_st[i] = 0
+            for j in range(model_interface.number_of_features):
+                cross_entropies[j][i] = 0
 
 
     if 'compare_cross-entropies' in flags and flags['compare_cross-entropies']:
-        return [(cross_entropy_st[i],cross_entropy_lex_st[i],cross_entropy_ts[i],cross_entropy_lex_ts[i],other_translations[i],oov[i],ignored[i],n[i],total_pairs) for i in range(num_results)], (n[0],total_pairs,oov2)
+        return [tuple([ce[i] for ce in cross_entropies]) + (other_translations[i],oov[i],ignored[i],n[i],total_pairs) for i in range(num_results)], (n[0],total_pairs,oov2)
     else:
-        return cross_entropy_st[0],cross_entropy_lex_st[0],cross_entropy_ts[0],cross_entropy_lex_ts[0],other_translations[0],oov2,total_pairs
+        return tuple([ce[0] for ce in cross_entropies]) + (other_translations[0],oov2,total_pairs)
 
 
 def cross_entropy_light(model_interface,reference_interface,weights,score,mode,flags,cache):
@@ -823,22 +788,16 @@ def cross_entropy_light(model_interface,reference_interface,weights,score,mode,f
        Same as cross_entropy, but optimized for speed: it doesn't generate all of the statistics,
        doesn't normalize, and uses caching.
     """
-    weights = normalize_weights(weights,mode)
-    cross_entropy_ts = 0
-    cross_entropy_st = 0
-    cross_entropy_lex_ts = 0
-    cross_entropy_lex_st = 0
+    weights = normalize_weights(weights,mode,flags)
+    cross_entropies = [0]*model_interface.number_of_features
 
     for (src,target,c) in cache:
         features = score(weights,src,target,model_interface,flags,cache=True)
 
-        cross_entropy_ts -= log(features[2],2)*c
-        cross_entropy_st -= log(features[0],2)*c
-        
-        cross_entropy_lex_ts -= log(features[3],2)*c
-        cross_entropy_lex_st -= log(features[1],2)*c
+        for i in range(model_interface.number_of_features):
+            cross_entropies[i] -= log(features[i],2)*c
                 
-    return cross_entropy_st,cross_entropy_lex_st,cross_entropy_ts,cross_entropy_lex_ts
+    return cross_entropies
 
 
 def _get_reference_cache(reference_interface,model_interface):
@@ -881,16 +840,16 @@ def _get_lexical_filter(reference_interface,model_interface):
     return e2f_filter,f2e_filter
 
 
-def _hillclimb_move(weights,stepsize,mode):
+def _hillclimb_move(weights,stepsize,mode,flags):
     """Move function for hillclimb algorithm. Updates each weight by stepsize."""
 
     for i,w in enumerate(weights):
-        yield normalize_weights(weights[:i]+[w+stepsize]+weights[i+1:],mode)
+        yield normalize_weights(weights[:i]+[w+stepsize]+weights[i+1:],mode,flags)
 
     for i,w in enumerate(weights):
         new = w-stepsize
         if new >= 1e-10:
-            yield normalize_weights(weights[:i]+[new]+weights[i+1:],mode)
+            yield normalize_weights(weights[:i]+[new]+weights[i+1:],mode,flags)
 
 def _hillclimb(scores,best_weights,objective,model_interface,reference_interface,score_function,mode,flags,precision,cache,n):
     """first (deprecated) implementation of iterative weight optimization."""
@@ -912,7 +871,7 @@ def _hillclimb(scores,best_weights,objective,model_interface,reference_interface
             
         move = 0
         
-        for w in _hillclimb_move(list(best_weights),stepsize,mode):
+        for w in _hillclimb_move(list(best_weights),stepsize,mode,flags):
             weights_tuple = tuple(w)
             
             if weights_tuple in scores:
@@ -948,22 +907,18 @@ def optimize_cross_entropy_hillclimb(model_interface,reference_interface,initial
     cache,n = _get_reference_cache(reference_interface,model_interface)
     
     # each objective is a triple: which score to minimize from cross_entropy(), which weights to update accordingly, and a comment that is printed
-    objectives = [(lambda x: scores[x][0]/n,[0],'minimize cross-entropy for p(s|t)'), #optimize cross-entropy for p(s|t)
-                    (lambda x: scores[x][1]/n,[1],'minimize cross-entropy for lex(s|t)'),
-                    (lambda x: scores[x][2]/n,[2],'minimize cross-entropy for p(t|s)'), #optimize for p(t|s)
-                    (lambda x: scores[x][3]/n,[3],'minimize cross-entropy for lex(t|s)')] 
-
+    objectives = [(lambda x: scores[x][i]/n,[i],'minimize cross-entropy for feature {0}'.format(i)) for i in range(model_interface.number_of_features)]
     
     scores[best_weights] = cross_entropy_light(model_interface,reference_interface,initial_weights,score_function,mode,flags,cache)
     final_weights = initial_weights[:]
     final_cross_entropy = [0]*model_interface.number_of_features
     
-    for objective, features, comment in objectives:
+    for i,(objective, features, comment) in enumerate(objectives):
         best_weights = min(scores,key=objective)
         sys.stderr.write('Optimizing objective "' + comment +'"\n')
-        best_weights = _hillclimb(scores,best_weights,objective,model_interface,reference_interface,score_function,mode,flags,precision,cache,n)
+        best_weights = _hillclimb(scores,best_weights,objective,model_interface,reference_interface,score_function,feature_specific_mode(mode,i,flags),flags,precision,cache,n)
         
-        sys.stderr.write('\nCross-entropy:' + str(objective(best_weights)) + '- weights: ' + str(best_weights)+'\n\n')
+        sys.stderr.write('\nCross-entropy:' + str(objective(best_weights)) + ' - weights: ' + str(best_weights)+'\n\n')
         
         for j in features:
             final_weights[j] = list(best_weights)
@@ -984,11 +939,8 @@ def optimize_cross_entropy(model_interface,reference_interface,initial_weights,s
     cache,n = _get_reference_cache(reference_interface,model_interface)
     
     # each objective is a triple: which score to minimize from cross_entropy(), which weights to update accordingly, and a comment that is printed
-    objectives = [(lambda w: cross_entropy_light(model_interface,reference_interface,[[1]+list(w) for m in range(model_interface.number_of_features)],score_function,mode,flags,cache)[0],[0],'minimize cross-entropy for p(s|t)'), #optimize cross-entropy for p(s|t)
-                    (lambda w: cross_entropy_light(model_interface,reference_interface,[[1]+list(w) for m in range(model_interface.number_of_features)],score_function,mode,flags,cache)[1],[1],'minimize cross-entropy for lex(s|t)'),
-                    (lambda w: cross_entropy_light(model_interface,reference_interface,[[1]+list(w) for m in range(model_interface.number_of_features)],score_function,mode,flags,cache)[2],[2],'minimize cross-entropy for p(t|s)'), #optimize for p(t|s)
-                    (lambda w: cross_entropy_light(model_interface,reference_interface,[[1]+list(w) for m in range(model_interface.number_of_features)],score_function,mode,flags,cache)[3],[3],'minimize cross-entropy for lex(t|s)')] 
-    
+    objectives = [(lambda w: cross_entropy_light(model_interface,reference_interface,[[1]+list(w) for m in range(model_interface.number_of_features)],score_function,feature_specific_mode(mode,i,flags),flags,cache)[i],[i],'minimize cross-entropy for feature {0}'.format(i)) for i in range(model_interface.number_of_features)] #optimize cross-entropy for p(s|t)
+
     final_weights = initial_weights[:]
     final_cross_entropy = [0]*model_interface.number_of_features
     
@@ -996,7 +948,7 @@ def optimize_cross_entropy(model_interface,reference_interface,initial_weights,s
         sys.stderr.write('Optimizing objective "' + comment +'"\n')
         initial_values = [1]*(len(model_interface.models)-1) # we leave value of first model at 1 and optimize all others (normalized of course)
         best_weights, best_point, data = fmin_l_bfgs_b(objective,initial_values,approx_grad=True,bounds=[(0.000000001,None)]*len(initial_values))
-        best_weights = normalize_weights([1]+list(best_weights),mode)
+        best_weights = normalize_weights([1]+list(best_weights),feature_specific_mode(mode,i,flags),flags)
         sys.stderr.write('Cross-entropy after L-BFGS optimization: ' + str(best_point/n) + ' - weights: ' + str(best_weights)+'\n')
     
         for j in features:
@@ -1006,41 +958,59 @@ def optimize_cross_entropy(model_interface,reference_interface,initial_weights,s
     return final_weights,final_cross_entropy
 
 
+def feature_specific_mode(mode,i,flags):
+    """in mode 'counts', only the default Moses features can be recomputed from raw frequencies;
+       all other features are interpolated by default. 
+       This fucntion mostly serves optical purposes (i.e. normalizing a single weight vector for logging),
+       since normalize_weights also handles a mix of interpolated and recomputed features.
+    """
+    
+    if mode == 'counts' and i not in [flags['i_e2f'],flags['i_e2f_lex'],flags['i_f2e'],flags['i_f2e_lex']]:
+        return 'interpolate'
+    else:
+        return mode
+
+
 def redistribute_probability_mass(weights,src,target,interface,flags,mode='interpolate'):
     """the conditional probability p(x|y) is undefined for cases where p(y) = 0
        this function redistributes the probability mass to only consider models for which p(y) > 0
     """
 
+    i_e2f = flags['i_e2f']
+    i_e2f_lex = flags['i_e2f_lex']
+    i_f2e = flags['i_f2e']
+    i_f2e_lex = flags['i_f2e_lex']
+
     new_weights = weights[:]
     
     if flags['normalize_s_given_t'] == 's':
     
         # set weight to 0 for all models where target phrase is unseen (p(s|t)   
-        new_weights[0] = map(mul,interface.phrase_source[src],weights[0])
+        new_weights[i_e2f] = map(mul,interface.phrase_source[src],weights[i_e2f])
         if flags['normalize-lexical_weights']:
-            new_weights[1] = map(mul,interface.phrase_source[src],weights[1])
+            new_weights[i_e2f_lex] = map(mul,interface.phrase_source[src],weights[i_e2f_lex])
         
     elif flags['normalize_s_given_t'] == 't':
         
         # set weight to 0 for all models where target phrase is unseen (p(s|t)   
-        new_weights[0] = map(mul,interface.phrase_target[target],weights[0])
+        new_weights[i_e2f] = map(mul,interface.phrase_target[target],weights[i_e2f])
         if flags['normalize-lexical_weights']:
-            new_weights[1] = map(mul,interface.phrase_target[target],weights[1])
+            new_weights[i_e2f_lex] = map(mul,interface.phrase_target[target],weights[i_e2f_lex])
 
     # set weight to 0 for all models where source phrase is unseen (p(t|s)
-    new_weights[2] = map(mul,interface.phrase_source[src],weights[2])
+    new_weights[i_f2e] = map(mul,interface.phrase_source[src],weights[i_f2e])
     if flags['normalize-lexical_weights']:
-        new_weights[3] = map(mul,interface.phrase_source[src],weights[3])
+        new_weights[i_f2e_lex] = map(mul,interface.phrase_source[src],weights[i_f2e_lex])
         
     
-    return normalize_weights(new_weights,mode)
+    return normalize_weights(new_weights,mode,flags)
 
 
 def score_interpolate(weights,src,target,interface,flags,cache=False):
     """linear interpolation of probabilites (and other feature values)
        if normalized is True, the probability mass for p(x|y) is redistributed to models with p(y) > 0
     """
-    
+
     model_values = interface.phrase_pairs[src][target][0]
     
     scores = [0]*len(model_values)
@@ -1059,12 +1029,12 @@ def score_interpolate(weights,src,target,interface,flags,cache=False):
             lts = 0
             
         else:
-            scores[1] = compute_lexicalweight(normalized_weights[1],e2f_alignment,interface.word_pairs_e2f,None,mode='interpolate')
-            scores[3] = compute_lexicalweight(normalized_weights[3],f2e_alignment,interface.word_pairs_f2e,None,mode='interpolate')
+            scores[flags['i_e2f_lex']] = compute_lexicalweight(normalized_weights[flags['i_e2f_lex']],e2f_alignment,interface.word_pairs_e2f,None,mode='interpolate')
+            scores[flags['i_f2e_lex']] = compute_lexicalweight(normalized_weights[flags['i_f2e_lex']],f2e_alignment,interface.word_pairs_f2e,None,mode='interpolate')
 
     
     for idx,prob in enumerate(model_values):
-        if not ('recompute_lexweights' in flags and flags['recompute_lexweights'] and (idx == 1 or idx == 3)):
+        if not ('recompute_lexweights' in flags and flags['recompute_lexweights'] and (idx == flags['i_e2f_lex'] or idx == flags['i_f2e_lex'])):
             scores[idx] = dot_product(prob,normalized_weights[idx])
     
     return scores
@@ -1093,32 +1063,43 @@ def score_counts(weights,src,target,interface,flags,cache=False):
        each count is multiplied by its weight; trivial case is weight 1 for each model, which corresponds to a concatentation
     """
     
+    i_e2f = flags['i_e2f']
+    i_e2f_lex = flags['i_e2f_lex']
+    i_f2e = flags['i_f2e']
+    i_f2e_lex = flags['i_f2e_lex']
+    
+    # if we have non-default number of weights, assume that we might have to do a mix of count-based and interpolated scores.
+    if len(weights) == 4:
+        scores = [0]*len(weights)
+    else:
+        scores = score_interpolate(weights,src,target,interface,flags,cache=cache)
+    
     try:
-        joined_count = dot_product(interface.phrase_pairs[src][target][0][0],weights[0])
-        target_count = dot_product(interface.phrase_target[target],weights[0])
-        pst = joined_count / target_count
+        joined_count = dot_product(interface.phrase_pairs[src][target][0][i_e2f],weights[i_e2f])
+        target_count = dot_product(interface.phrase_target[target],weights[i_e2f])
+        scores[i_e2f] = joined_count / target_count
     except ZeroDivisionError:
-        pst = 0
+        scores[i_e2f] = 0
 
     try:
-        joined_count = dot_product(interface.phrase_pairs[src][target][0][1],weights[2])
-        source_count = dot_product(interface.phrase_source[src],weights[2])
-        pts = joined_count / source_count
+        joined_count = dot_product(interface.phrase_pairs[src][target][0][i_f2e],weights[i_f2e])
+        source_count = dot_product(interface.phrase_source[src],weights[i_f2e])
+        scores[i_f2e] = joined_count / source_count
     except ZeroDivisionError:
-        pts = 0
+        scores[i_f2e] = 0
     
     e2f_alignment,f2e_alignment = interface.get_word_alignments(src,target,cache=cache)
     
     if not e2f_alignment or not f2e_alignment:
         sys.stderr.write('Error: no word alignments found, but necessary for lexical weight computation.\n')
-        lst = 0
-        lts = 0
+        scores[i_e2f_lex] = 0
+        scores[i_f2e_lex] = 0
     
     else:
-        lst = compute_lexicalweight(weights[1],e2f_alignment,interface.word_pairs_e2f,interface.word_target,mode='counts',cache=cache)
-        lts = compute_lexicalweight(weights[3],f2e_alignment,interface.word_pairs_f2e,interface.word_source,mode='counts',cache=cache)
+        scores[i_e2f_lex] = compute_lexicalweight(weights[i_e2f_lex],e2f_alignment,interface.word_pairs_e2f,interface.word_target,mode='counts',cache=cache)
+        scores[i_f2e_lex] = compute_lexicalweight(weights[i_f2e_lex],f2e_alignment,interface.word_pairs_f2e,interface.word_source,mode='counts',cache=cache)
     
-    return [pst,lst,pts,lts]
+    return scores
 
 
 def score_interpolate_reordering(weights,src,target,interface):
@@ -1174,7 +1155,7 @@ def compute_lexicalweight(weights,alignment,word_pairs,marginal,mode='counts',ca
     return lex
 
 
-def normalize_weights(weights,mode):
+def normalize_weights(weights,mode,flags=None):
     """make sure that probability mass in linear interpolation is 1
        for weighted counts, weight of first model is set to 1
     """
@@ -1203,7 +1184,7 @@ def normalize_weights(weights,mode):
             except ZeroDivisionError:
                 sys.stderr.write('Error: Zero division in weight normalization. Are some of your weights zero? This might lead to undefined behaviour if a phrase pair is only seen in model with weight 0\n')
 
-    elif mode == 'counts':
+    elif mode == 'counts_pure':
         
         if type(weights[0]) == list:
                 
@@ -1217,6 +1198,19 @@ def normalize_weights(weights,mode):
             ratio = 1/weights[0]
             new_weights = [weight * ratio for weight in weights]
 
+    # make sure that features other than the standard Moses features are always interpolated (since no count-based computation is defined)
+    elif mode == 'counts': 
+        
+        if type(weights[0]) == list:
+                norm_counts = normalize_weights(weights,'counts_pure')
+                new_weights = normalize_weights(weights,'interpolate')
+                for i in [flags['i_e2f'],flags['i_e2f_lex'],flags['i_f2e'],flags['i_f2e_lex']]:
+                    new_weights[i] = norm_counts[i]
+                return new_weights
+        
+        else:
+            return normalize_weights(weights,'counts_pure')
+
     return new_weights
 
 
@@ -1288,7 +1282,11 @@ class Combine_TMs():
             'normalize_s_given_t':None, 
             'normalize-lexical_weights':True, 
             'add_origin_features':False,
-            'lowmem': False
+            'lowmem': False,
+            'i_e2f':0,
+            'i_e2f_lex':1,
+            'i_f2e':2,
+            'i_f2e_lex':3
             }
 
     # each model needs a priority. See init docstring for more info
@@ -1296,7 +1294,17 @@ class Combine_TMs():
                     'map':2,
                     'supplementary':10}
     
-    def __init__(self,models,weights=None,output_file=None,mode='interpolate',number_of_features=4,model_interface=Moses,reference_interface=Moses_Alignment,reference_file=None,lang_src=None,lang_target=None,output_lexical=None,**flags):
+    def __init__(self,models,weights=None,
+                      output_file=None,
+                      mode='interpolate',
+                      number_of_features=4,
+                      model_interface=Moses,
+                      reference_interface=Moses_Alignment,
+                      reference_file=None,
+                      lang_src=None,
+                      lang_target=None,
+                      output_lexical=None,
+                      **flags):
         """The whole configuration of the task is done during intialization. Afterwards, you only need to call your intended method(s).
            You can change some of the class attributes afterwards (such as the weights, or the output file), but you should never change the models or mode after initialization.
            See unit_test function for example configurations
@@ -1317,12 +1325,17 @@ class Combine_TMs():
 
            mode: declares the basic mixture-model algorithm. there are currently three options:
                  'counts': weighted counts (requires some statistics that Moses doesn't produce. Apply train_model.patch to train_model.perl and repeat step 4 of Moses training to obtain them.)
+                           Only the standard Moses features are recomputed from weighted counts; additional features are linearly interpolated 
+                           (see number_of_features to allow more features, and i_e2f etc. if the standard features are in a non-standard position)
                  'interpolate': linear interpolation
                  'loglinear': loglinear interpolation (careful: this creates the intersection of phrase tables and is often of little use)
                  
            number_of_features: could be used to interpolate models with non-default Moses features. 4 features is currently still hardcoded in various places 
                                (e.g. cross_entropy calculations, mode 'counts')
            
+           i_e2f,i_e2f_lex,i_f2e,i_f2e_lex: Index of the (Moses) phrase table features p(s|t), lex(s|t), p(t|s) and lex(t|s). 
+                Relevant for mode 'counts', and if 'recompute_lexweights' is True in mode 'interpolate'. In mode 'counts', any additional features are combined through linear interpolation.
+           
            model_interface: class that handles reading phrase tables and lexical tables, and writing phrase tables. Currently only Moses is implemented.
                  default: Moses
            
@@ -1378,7 +1391,15 @@ class Combine_TMs():
         self.lang_target = lang_target
         self.loaded = defaultdict(int)
         self.output_lexical = output_lexical
+
+        self.flags = copy.copy(self.flags)
+        self.flags.update(flags)
         
+        self.flags['i_e2f'] = int(self.flags['i_e2f'])
+        self.flags['i_e2f_lex'] = int(self.flags['i_e2f_lex'])
+        self.flags['i_f2e'] = int(self.flags['i_f2e'])
+        self.flags['i_f2e_lex'] = int(self.flags['i_f2e_lex'])
+
         if reference_interface:
             self.reference_interface = reference_interface(reference_file)
 
@@ -1395,22 +1416,18 @@ class Combine_TMs():
 
         if mode == 'interpolate':
             self.score = score_interpolate
-            self.load_lines = self.model_interface.load_phrase_probabilities
         elif mode == 'loglinear':
             self.score = score_loglinear
-            self.load_lines = self.model_interface.load_phrase_probabilities
         elif mode == 'counts':
             self.score = score_counts
-            self.load_lines = self.model_interface.load_phrase_counts
-
-        self.flags = copy.copy(self.flags)
-        self.flags.update(flags)
             
 
     def _sanity_checks(self,models,number_of_features,weights):
         """check if input arguments make sense (correct number of weights, valid model priorities etc.)
            is only called on initialization. If you change weights afterwards, better know what you're doing.
         """
+        
+        number_of_features = int(number_of_features)
               
         for (model,priority) in models:
             assert(priority in self._priorities)
@@ -1437,12 +1454,12 @@ class Combine_TMs():
             sys.stderr.write('Warning: No weights defined: initializing with uniform weights\n')
 
 
-        new_weights = normalize_weights(weights,self.mode)
+        new_weights = normalize_weights(weights,self.mode,self.flags)
         if weights != new_weights:
             if self.mode == 'interpolate' or self.mode == 'loglinear':
                 sys.stderr.write('Warning: weights should sum to 1 - ')
             elif self.mode == 'counts':
-                sys.stderr.write('Warning: normalizing weights so that first model has weight 1 - ')
+                sys.stderr.write('Warning: normalizing weights so that first model has weight 1 (for features that are recomputed from counts) - ')
             sys.stderr.write('normalizing to: '+ str(new_weights) +'\n')
             weights = new_weights
             
@@ -1472,7 +1489,7 @@ class Combine_TMs():
         if 'pt-filtered' in data and not self.loaded['pt-filtered']:
             
             models_prioritized = [(self.model_interface.open_table(model,'phrase-table'),priority,i) for (model,priority,i) in priority_sort_models(self.models)]
-                
+            
             for model,priority,i in models_prioritized:
                 sys.stderr.write('Loading phrase table ' + str(i) + ' (only data relevant for reference set)')
                 j = 0
@@ -1481,7 +1498,7 @@ class Combine_TMs():
                         sys.stderr.write('...'+str(j))
                     j += 1
                     line = line.rstrip().split(b' ||| ')
-                    self.load_lines(line,priority,i,store='all',filter_by=self.reference_interface.word_pairs,filter_by_src=self.reference_interface.word_source,filter_by_target=self.reference_interface.word_target)
+                    self.model_interface.load_phrase_features(line,priority,i,store='all',mode=self.mode,filter_by=self.reference_interface.word_pairs,filter_by_src=self.reference_interface.word_source,filter_by_target=self.reference_interface.word_target,flags=self.flags)
                 sys.stderr.write(' done\n')
 
             self.loaded['pt-filtered'] = 1
@@ -1506,7 +1523,7 @@ class Combine_TMs():
                         sys.stderr.write('...'+str(j))
                     j += 1
                     line = line.rstrip().split(b' ||| ')
-                    self.load_lines(line,priority,i,store='target')
+                    self.model_interface.load_phrase_features(line,priority,i,mode=self.mode,store='target',flags=self.flags)
                 sys.stderr.write(' done\n')
 
             self.loaded['pt-target'] = 1
@@ -1554,8 +1571,8 @@ class Combine_TMs():
 
         i = 0
         sys.stderr.write('Incrementally loading and processing phrase tables...')
-        for block in self.model_interface.traverse_incrementally('phrase-table',models,self.load_lines,store_flag,inverted=inverted,lowmem=self.flags['lowmem']):
-            
+
+        for block in self.model_interface.traverse_incrementally('phrase-table',models,self.model_interface.load_phrase_features,store_flag,mode=self.mode,inverted=inverted,lowmem=self.flags['lowmem'],flags=self.flags):
             for src in sorted(self.model_interface.phrase_pairs, key = lambda x: x + b' |'):
                 for target in sorted(self.model_interface.phrase_pairs[src], key = lambda x: x + b' |'):
                     
@@ -1642,8 +1659,8 @@ class Combine_TMs():
         i = 0
         
         sys.stderr.write('Incrementally loading and processing phrase tables...')
-        for block in self.model_interface.traverse_incrementally('reordering-table',models,self.model_interface.load_reordering_probabilities,'pairs',lowmem=self.flags['lowmem']):
-            
+
+        for block in self.model_interface.traverse_incrementally('reordering-table',models,self.model_interface.load_reordering_probabilities,'pairs',mode=self.mode,lowmem=self.flags['lowmem'],flags=self.flags):
             for src in sorted(self.model_interface.reordering_pairs):
                 for target in sorted(self.model_interface.reordering_pairs[src]):
                     if not i % 1000000:
@@ -1676,18 +1693,21 @@ class Combine_TMs():
         results, (intersection,total_pairs,oov2) = cross_entropy(self.model_interface,self.reference_interface,self.weights,self.score,self.mode,self.flags)
         
         padding = 90
+        num_features = self.model_interface.number_of_features
         
         print('\nResults of model comparison\n')
         print('{0:<{padding}}: {1}'.format('phrase pairs in reference (tokens)',total_pairs, padding=padding))
         print('{0:<{padding}}: {1}'.format('phrase pairs in model intersection (tokens)',intersection, padding=padding))
         print('{0:<{padding}}: {1}\n'.format('phrase pairs in model union (tokens)',total_pairs-oov2, padding=padding))
         
-        for i,(cross_entropy_st,cross_entropy_lex_st,cross_entropy_ts,cross_entropy_lex_ts,other_translations,oov,ignored,n,total_pairs) in enumerate(results):
+        for i,data in enumerate(results):
+            
+            cross_entropies = data[:num_features]
+            (other_translations,oov,ignored,n,total_pairs) = data[num_features:]
+            
             print('model ' +str(i))
-            print('{0:<{padding}}: {1}'.format('cross-entropy p(s|t)', cross_entropy_st, padding=padding))
-            print('{0:<{padding}}: {1}'.format('cross-entropy lex(s|t)', cross_entropy_lex_st, padding=padding))
-            print('{0:<{padding}}: {1}'.format('cross-entropy p(t|s)', cross_entropy_ts, padding=padding))
-            print('{0:<{padding}}: {1}'.format('cross-entropy lex(t|s)', cross_entropy_lex_ts, padding=padding))
+            for j in range(num_features):
+                print('{0:<{padding}}: {1}'.format('cross-entropy for feature {0}'.format(j), cross_entropies[j], padding=padding))
             print('{0:<{padding}}: {1}'.format('phrase pairs in model (tokens)', n+ignored, padding=padding))
             print('{0:<{padding}}: {1}'.format('phrase pairs in model, but not in intersection (tokens)', ignored, padding=padding))
             print('{0:<{padding}}: {1}'.format('phrase pairs in union, but not in model (but source phrase is) (tokens)', other_translations, padding=padding))
@@ -1786,6 +1806,18 @@ def test():
     Combiner = Combine_TMs([[os.path.join('test','model1'),'primary'],[os.path.join('test','model2'),'map']],output_file=os.path.join('test','phrase-table_test8'),mode='counts',reference_file='test/extract')
     Combiner.combine_given_tuning_set()
 
+    # count-based combination of two non-default models, with fixed weights. Same as test 3, but with the standard features moved back
+    # command line: python tmcombine.py combine_given_weights test/model3 test/model4 -w "0.5,0.5;0.5,0.5;0.5,0.5;0.5,0.5;0.1,0.9;0.1,1;0.2,0.8;0.5,0.5" -o test/phrase-table_test9 -m counts --number_of_features 8 --i_e2f 4 --i_e2f_lex 5 --i_f2e 6 --i_f2e_lex 7 -r test/extract
+    sys.stderr.write('Regression test 9\n')
+    Combiner = Combine_TMs([[os.path.join('test','model3'),'primary'],[os.path.join('test','model4'),'primary']],[[0.5,0.5],[0.5,0.5],[0.5,0.5],[0.5,0.5],[0.1,0.9],[0.1,1],[0.2,0.8],[0.5,0.5]],os.path.join('test','phrase-table_test9'),mode='counts',number_of_features=8,i_e2f=4,i_e2f_lex=5,i_f2e=6,i_f2e_lex=7)
+    Combiner.combine_given_weights()
+
+    # count-based combination of two non-default models, with fixed weights. Same as test 5, but with the standard features moved back
+    # command line: python tmcombine.py combine_given_tuning_set test/model3 test/model4 -o test/phrase-table_test10 -m counts --number_of_features 8 --i_e2f 4 --i_e2f_lex 5 --i_f2e 6 --i_f2e_lex 7 -r test/extract
+    sys.stderr.write('Regression test 10\n')
+    Combiner = Combine_TMs([[os.path.join('test','model3'),'primary'],[os.path.join('test','model4'),'primary']],output_file=os.path.join('test','phrase-table_test10'),mode='counts',number_of_features=8,i_e2f=4,i_e2f_lex=5,i_f2e=6,i_f2e_lex=7,reference_file='test/extract')
+    Combiner.combine_given_tuning_set()
+    
 
 #convert weight vector passed as a command line argument
 class to_list(argparse.Action):
@@ -1800,46 +1832,68 @@ class to_list(argparse.Action):
 def parse_command_line():
     parser = argparse.ArgumentParser(description='Combine translation models. Check DOCSTRING of the class Combine_TMs() and its methods for a more in-depth documentation and additional configuration options not available through the command line. The function test() shows examples.')
     
-    parser.add_argument('action', metavar='ACTION', choices=["combine_given_weights","combine_given_tuning_set","combine_reordering_tables","compute_cross_entropy","return_best_cross_entropy","compare_cross_entropies"],
+    group1 = parser.add_argument_group('Main options')
+    group2 = parser.add_argument_group('More model combination options')
+    
+    group1.add_argument('action', metavar='ACTION', choices=["combine_given_weights","combine_given_tuning_set","combine_reordering_tables","compute_cross_entropy","return_best_cross_entropy","compare_cross_entropies"],
                     help='What you want to do with the models. One of %(choices)s.')
     
-    parser.add_argument('model', metavar='DIRECTORY', nargs='+',
+    group1.add_argument('model', metavar='DIRECTORY', nargs='+',
                     help='Model directory. Assumes default Moses structure (i.e. path to phrase table and lexical tables).')
                     
-    parser.add_argument('-w', '--weights', dest='weights', action=to_list,
+    group1.add_argument('-w', '--weights', dest='weights', action=to_list,
                     default=None,
                     help='weight vector. Format 1: single vector, one weight per model. Example: \"0.1,0.9\" ; format 2: one vector per feature, one weight per model: \"0.1,0.9;0.5,0.5;0.4,0.6;0.2,0.8\"')
 
-    parser.add_argument('-m', '--mode', type=str,
+    group1.add_argument('-m', '--mode', type=str,
                     default="interpolate",
                     choices=["counts","interpolate","loglinear"],
                     help='basic mixture-model algorithm. Default: %(default)s. Note: depending on mode and additional configuration, additional statistics are needed. Check docstring documentation of Combine_TMs() for more info.')
 
-    parser.add_argument('-r', '--reference', type=str,
+    group1.add_argument('-r', '--reference', type=str,
                     default=None,
                     help='File containing reference phrase pairs for cross-entropy calculation. Default interface expects \'path/model/extract.gz\' that is produced by training a model on the reference (i.e. development) corpus.')
 
-    parser.add_argument('-o', '--output', type=str,
+    group1.add_argument('-o', '--output', type=str,
                     default="-",
                     help='Output file (phrase table). If not specified, model is written to standard output.')
 
-    parser.add_argument('--output-lexical', type=str,
+    group1.add_argument('--output-lexical', type=str,
                     default=None,
                     help=('Not only create a combined phrase table, but also combined lexical tables. Writes to OUTPUT_LEXICAL.e2f and OUTPUT_LEXICAL.f2e, or OUTPUT_LEXICAL.counts.e2f in mode \'counts\'.'))
 
-    parser.add_argument('--lowmem', action="store_true",
+    group1.add_argument('--lowmem', action="store_true",
                     help=('Low memory mode: requires two passes (and sorting in between) to combine a phrase table, but loads less data into memory. Only relevant for mode "counts" and some configurations of mode "interpolate".'))
 
-    parser.add_argument('--normalized', action="store_true",
-                    help=('for each phrase pair x,y: ignore models with p(y)=0, and distribute probability mass among models with p(y)>0. (default: missing entries (x,y) are always interpreted as p(x|y)=0). Only relevant in mode "interpolate".'))
-
-    parser.add_argument('--recompute_lexweights', action="store_true",
-                    help=('don\'t directly interpolate lexical weights, but interpolate word translation probabilities instead and recompute the lexical weights. Only relevant in mode "interpolate".'))
-
-    parser.add_argument('--tempdir', type=str,
+    group1.add_argument('--tempdir', type=str,
                     default=None,
                     help=('Temporary directory in --lowmem mode.'))
 
+    group2.add_argument('--i_e2f', type=int,
+                    default=0, metavar='N',
+                    help=('Index of p(f|e) (relevant for mode counts if phrase table has custom feature order). (default: %(default)s)'))
+
+    group2.add_argument('--i_e2f_lex', type=int,
+                    default=1, metavar='N',
+                    help=('Index of lex(f|e) (relevant for mode counts or with option recompute_lexweights if phrase table has custom feature order). (default: %(default)s)'))
+
+    group2.add_argument('--i_f2e', type=int,
+                    default=2, metavar='N',
+                    help=('Index of p(e|f) (relevant for mode counts if phrase table has custom feature order). (default: %(default)s)'))
+
+    group2.add_argument('--i_f2e_lex', type=int,
+                    default=3, metavar='N',
+                    help=('Index of lex(e|f) (relevant for mode counts or with option recompute_lexweights if phrase table has custom feature order). (default: %(default)s)'))
+
+    group2.add_argument('--number_of_features', type=int,
+                    default=4, metavar='N',
+                    help=('Combine models with N + 1 features (last feature is constant phrase penalty). (default: %(default)s)'))
+
+    group2.add_argument('--normalized', action="store_true",
+                    help=('for each phrase pair x,y: ignore models with p(y)=0, and distribute probability mass among models with p(y)>0. (default: missing entries (x,y) are always interpreted as p(x|y)=0). Only relevant in mode "interpolate".'))
+
+    group2.add_argument('--recompute_lexweights', action="store_true",
+                    help=('don\'t directly interpolate lexical weights, but interpolate word translation probabilities instead and recompute the lexical weights. Only relevant in mode "interpolate".'))
 
     return parser.parse_args()
 
@@ -1854,7 +1908,21 @@ if __name__ == "__main__":
     else:
         args = parse_command_line()
         #initialize
-        combiner = Combine_TMs([(m,'primary') for m in args.model],weights=args.weights,mode=args.mode,output_file=args.output,reference_file=args.reference,output_lexical=args.output_lexical,lowmem=args.lowmem,normalized=args.normalized,recompute_lexweights=args.recompute_lexweights,tempdir=args.tempdir)
+        combiner = Combine_TMs([(m,'primary') for m in args.model],
+                               weights=args.weights,
+                               mode=args.mode,
+                               output_file=args.output,
+                               reference_file=args.reference,
+                               output_lexical=args.output_lexical,
+                               lowmem=args.lowmem,
+                               normalized=args.normalized,
+                               recompute_lexweights=args.recompute_lexweights,
+                               tempdir=args.tempdir,
+                               number_of_features=args.number_of_features,
+                               i_e2f=args.i_e2f,
+                               i_e2f_lex=args.i_e2f_lex,
+                               i_f2e=args.i_f2e,
+                               i_f2e_lex=args.i_f2e_lex)
         # execute right method
         f_string = "combiner."+args.action+'()'
         exec(f_string)