only filter rule and reordering tables, if the tables changes, not just when weights or other settings changed

3 files changed, 89 insertions, 16 deletions

diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta
index c68145b15..ea3b46823 100644
--- a/scripts/ems/experiment.meta
+++ b/scripts/ems/experiment.meta
@@ -520,12 +520,20 @@ split-reference
 	multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl
 	template: $output-splitter -model IN1.$output-extension < IN > OUT
 filter
-	in: TRAINING:config input
+	in: input TRAINING:phrase-translation-table TRAINING:reordering-table
+	out: filtered-dir
+	default-name: tuning/filtered
+	rerun-on-change: filter-settings
+	pass-if: TRAINING:binarize-all
+	ignore-if: use-hiero
+	error: already exists. Please delete
+apply-filter
+	in: TRAINING:config filtered-dir
 	out: filtered-config
 	default-name: tuning/moses.filtered.ini
-  rerun-on-change: filter-settings
-  pass-if: TRAINING:binarize-all
+	pass-if: TRAINING:binarize-all
 	ignore-if: use-hiero
+	template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT
 tune
 	in: filtered-config input reference
 	out: weight-config
@@ -616,12 +624,20 @@ split-input
 	pass-unless: input-splitter
 	template: $input-splitter -model IN1.$input-extension < IN > OUT
 filter
-	in: TUNING:config-with-reused-weights input
+	in: input TRAINING:phrase-translation-table TRAINING:reordering-table
+	out: filtered-dir
+	default-name: evaluation/filtered
+	rerun-on-change: filter-settings report-precision-by-coverage
+	pass-if: TRAINING:binarize-all
+	ignore-if: use-hiero
+	error: already exists. Please delete
+apply-filter
+	in: TUNING:config-with-reused-weights filtered-dir
 	out: filtered-config
 	default-name: evaluation/filtered.ini
 	pass-if: TRAINING:binarize-all
-	rerun-on-change: filter-settings report-precision-by-coverage
-	error: already exists. Please delete
+	ignore-if: use-hiero
+	template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT
 decode
 	in: filtered-config input
 	out: system-output
diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl
index c95db9623..023e4d613 100755
--- a/scripts/ems/experiment.perl
+++ b/scripts/ems/experiment.perl
@@ -2133,8 +2133,8 @@ sub define_tuningevaluation_filter {
     my $dir = &check_and_get("GENERAL:working-dir");
     my $tuning_flag = !defined($set);
 
-    my ($filter_config,
-	$config,$input) = &get_output_and_input($step_id);
+    my ($filter_dir,
+	$input,$phrase_translation_table,$reordering_table) = &get_output_and_input($step_id);
 
     my $binarizer = &get("GENERAL:ttable-binarizer");
     my $hierarchical = &get("TRAINING:hierarchical-rule-set");
@@ -2148,9 +2148,7 @@ sub define_tuningevaluation_filter {
     $input_filter = &get("TUNING:input-filter") if $tuning_flag;
     $input_filter = $input unless $input_filter;
 
-    my $filter_dir = "$dir/tuning/filtered.$VERSION";
-    $filter_dir = "$dir/evaluation/filtered.$set.$VERSION" unless $tuning_flag;
-
+    # additional settings
     my $settings = &backoff_and_get("EVALUATION:$set:filter-settings") unless $tuning_flag;
     $settings = &get("TUNING:filter-settings") if $tuning_flag;
     $settings = "" unless $settings;
@@ -2160,12 +2158,22 @@ sub define_tuningevaluation_filter {
     $settings .= " -Binarizer \"$binarizer\"" if $binarizer;
     $settings .= " --Hierarchical" if &get("TRAINING:hierarchical-rule-set");
 
-    my $cmd = "$scripts/training/filter-model-given-input.pl";
-    $cmd .= " $filter_dir $config $input_filter $settings";
+    # create pseudo-config file
+    my $config = "$dir/tuning/moses.table.ini.$VERSION";
+    my $cmd = &get_training_setting(9);
+    $cmd .= &get_table_name_settings("translation-factors","phrase-translation-table",$phrase_translation_table);
+    $cmd .= &get_table_name_settings("reordering-factors","reordering-table",$reordering_table)
+	if $reordering_table;
+    $cmd .= "-lm 0:3:$dir "; # dummy
+    $cmd .= "-config $config\n";
+    
+    # filter command
+    $cmd .= "$scripts/training/filter-model-given-input.pl";
+    $cmd .= " $filter_dir $config $input_filter $settings\n";
+
+    # clean-up
+    $cmd .= "rm $config";
 
-    # copy moses.ini into specified file location
-    $cmd .= "\ncp $filter_dir/moses.ini $filter_config\n";
- 
     &create_step($step_id,$cmd);
 }
 
diff --git a/scripts/ems/support/substitute-filtered-tables.perl b/scripts/ems/support/substitute-filtered-tables.perl
new file mode 100755
index 000000000..fda3195e9
--- /dev/null
+++ b/scripts/ems/support/substitute-filtered-tables.perl
@@ -0,0 +1,49 @@
+#!/usr/bin/perl -w
+
+# experiment.perl support script
+# get filtered rule and reordering tables and place them into a configuration file
+
+if (scalar @ARGV < 1 || ! -e $ARGV[0]) {
+  die("ERROR: could not find pseudo-config with filtered tables");
+}
+
+# read config sections about filtered tables
+my %CONFIG;
+my $status = 0;
+my $section;
+open(FILTERED,$ARGV[0]);
+while(<FILTERED>) {
+  if (/^\[(.+)\]\s*$/) {
+    if ($1 eq "ttable-file" || $1 eq "distortion-file") {
+      $section = $1;
+      $status = 1;
+      print STDERR "found $section\n";
+    }
+    else {
+      $status = 0;
+    }
+  }
+  elsif ($status) {
+    $CONFIG{$section} .= $_;
+  }
+}
+close(FILTERED);
+
+# pass through master config file and replace table sections
+($status,$section) = (0);
+while(<STDIN>) {
+  if (/^\[(.+)\]\s*$/) {
+    print $_;
+    if ($1 eq "ttable-file" || $1 eq "distortion-file") {
+      print STDERR "replacing $1\n";
+      print $CONFIG{$1};
+      $status = 1;
+    }
+    else {
+      $status = 0;
+    }
+  }
+  elsif (!$status) {
+    print $_;
+  }
+}