diff options
Diffstat (limited to 'scripts/ems')
54 files changed, 19 insertions, 72 deletions
diff --git a/scripts/ems/biconcor/Alignment.cpp b/scripts/ems/biconcor/Alignment.cpp index 979037ee6..979037ee6 100755..100644 --- a/scripts/ems/biconcor/Alignment.cpp +++ b/scripts/ems/biconcor/Alignment.cpp diff --git a/scripts/ems/biconcor/Alignment.h b/scripts/ems/biconcor/Alignment.h index 38e4dab17..38e4dab17 100755..100644 --- a/scripts/ems/biconcor/Alignment.h +++ b/scripts/ems/biconcor/Alignment.h diff --git a/scripts/ems/biconcor/Jamfile b/scripts/ems/biconcor/Jamfile index 003193067..003193067 100755..100644 --- a/scripts/ems/biconcor/Jamfile +++ b/scripts/ems/biconcor/Jamfile diff --git a/scripts/ems/biconcor/Mismatch.cpp b/scripts/ems/biconcor/Mismatch.cpp index 43e83e921..43e83e921 100755..100644 --- a/scripts/ems/biconcor/Mismatch.cpp +++ b/scripts/ems/biconcor/Mismatch.cpp diff --git a/scripts/ems/biconcor/Mismatch.h b/scripts/ems/biconcor/Mismatch.h index bfcbf4fd8..bfcbf4fd8 100755..100644 --- a/scripts/ems/biconcor/Mismatch.h +++ b/scripts/ems/biconcor/Mismatch.h diff --git a/scripts/ems/biconcor/PhrasePair.cpp b/scripts/ems/biconcor/PhrasePair.cpp index b5330a5fb..b5330a5fb 100755..100644 --- a/scripts/ems/biconcor/PhrasePair.cpp +++ b/scripts/ems/biconcor/PhrasePair.cpp diff --git a/scripts/ems/biconcor/PhrasePair.h b/scripts/ems/biconcor/PhrasePair.h index 079588277..079588277 100755..100644 --- a/scripts/ems/biconcor/PhrasePair.h +++ b/scripts/ems/biconcor/PhrasePair.h diff --git a/scripts/ems/biconcor/PhrasePairCollection.cpp b/scripts/ems/biconcor/PhrasePairCollection.cpp index 10d4e37c4..10d4e37c4 100755..100644 --- a/scripts/ems/biconcor/PhrasePairCollection.cpp +++ b/scripts/ems/biconcor/PhrasePairCollection.cpp diff --git a/scripts/ems/biconcor/PhrasePairCollection.h b/scripts/ems/biconcor/PhrasePairCollection.h index b2473515e..b2473515e 100755..100644 --- a/scripts/ems/biconcor/PhrasePairCollection.h +++ b/scripts/ems/biconcor/PhrasePairCollection.h diff --git a/scripts/ems/biconcor/SuffixArray.cpp b/scripts/ems/biconcor/SuffixArray.cpp index e7214bf43..e7214bf43 100755..100644 --- a/scripts/ems/biconcor/SuffixArray.cpp +++ b/scripts/ems/biconcor/SuffixArray.cpp diff --git a/scripts/ems/biconcor/SuffixArray.h b/scripts/ems/biconcor/SuffixArray.h index fc5ad7c1c..fc5ad7c1c 100755..100644 --- a/scripts/ems/biconcor/SuffixArray.h +++ b/scripts/ems/biconcor/SuffixArray.h diff --git a/scripts/ems/biconcor/TargetCorpus.cpp b/scripts/ems/biconcor/TargetCorpus.cpp index 78c67987f..78c67987f 100755..100644 --- a/scripts/ems/biconcor/TargetCorpus.cpp +++ b/scripts/ems/biconcor/TargetCorpus.cpp diff --git a/scripts/ems/biconcor/TargetCorpus.h b/scripts/ems/biconcor/TargetCorpus.h index 557b1fc4e..557b1fc4e 100755..100644 --- a/scripts/ems/biconcor/TargetCorpus.h +++ b/scripts/ems/biconcor/TargetCorpus.h diff --git a/scripts/ems/biconcor/Vocabulary.cpp b/scripts/ems/biconcor/Vocabulary.cpp index 6a9bea7e3..6a9bea7e3 100755..100644 --- a/scripts/ems/biconcor/Vocabulary.cpp +++ b/scripts/ems/biconcor/Vocabulary.cpp diff --git a/scripts/ems/biconcor/Vocabulary.h b/scripts/ems/biconcor/Vocabulary.h index dc0175387..dc0175387 100755..100644 --- a/scripts/ems/biconcor/Vocabulary.h +++ b/scripts/ems/biconcor/Vocabulary.h diff --git a/scripts/ems/biconcor/base64.cpp b/scripts/ems/biconcor/base64.cpp index 2a863d161..2a863d161 100755..100644 --- a/scripts/ems/biconcor/base64.cpp +++ b/scripts/ems/biconcor/base64.cpp diff --git a/scripts/ems/biconcor/base64.h b/scripts/ems/biconcor/base64.h index 65d5db8b2..65d5db8b2 100755..100644 --- a/scripts/ems/biconcor/base64.h +++ b/scripts/ems/biconcor/base64.h diff --git a/scripts/ems/biconcor/biconcor.cpp b/scripts/ems/biconcor/biconcor.cpp index 0b6218623..0b6218623 100755..100644 --- a/scripts/ems/biconcor/biconcor.cpp +++ b/scripts/ems/biconcor/biconcor.cpp diff --git a/scripts/ems/example/config.basic b/scripts/ems/example/config.basic index fb8d62c74..fb8d62c74 100755..100644 --- a/scripts/ems/example/config.basic +++ b/scripts/ems/example/config.basic diff --git a/scripts/ems/example/config.factored b/scripts/ems/example/config.factored index 0fb072c66..0fb072c66 100755..100644 --- a/scripts/ems/example/config.factored +++ b/scripts/ems/example/config.factored diff --git a/scripts/ems/example/config.hierarchical b/scripts/ems/example/config.hierarchical index 9284726c7..9284726c7 100755..100644 --- a/scripts/ems/example/config.hierarchical +++ b/scripts/ems/example/config.hierarchical diff --git a/scripts/ems/example/config.syntax b/scripts/ems/example/config.syntax index 085e210a5..085e210a5 100755..100644 --- a/scripts/ems/example/config.syntax +++ b/scripts/ems/example/config.syntax diff --git a/scripts/ems/example/config.toy b/scripts/ems/example/config.toy index 59753a50e..59753a50e 100755..100644 --- a/scripts/ems/example/config.toy +++ b/scripts/ems/example/config.toy diff --git a/scripts/ems/example/data/nc-5k.en b/scripts/ems/example/data/nc-5k.en index ea40d25fd..ea40d25fd 100755..100644 --- a/scripts/ems/example/data/nc-5k.en +++ b/scripts/ems/example/data/nc-5k.en diff --git a/scripts/ems/example/data/nc-5k.fr b/scripts/ems/example/data/nc-5k.fr index e8ac93b5d..e8ac93b5d 100755..100644 --- a/scripts/ems/example/data/nc-5k.fr +++ b/scripts/ems/example/data/nc-5k.fr diff --git a/scripts/ems/example/data/test-ref.en.sgm b/scripts/ems/example/data/test-ref.en.sgm index 8aa97ae5b..8aa97ae5b 100755..100644 --- a/scripts/ems/example/data/test-ref.en.sgm +++ b/scripts/ems/example/data/test-ref.en.sgm diff --git a/scripts/ems/example/data/test-src.fr.sgm b/scripts/ems/example/data/test-src.fr.sgm index ed5856a74..ed5856a74 100755..100644 --- a/scripts/ems/example/data/test-src.fr.sgm +++ b/scripts/ems/example/data/test-src.fr.sgm diff --git a/scripts/ems/example/data/weight.ini b/scripts/ems/example/data/weight.ini index 731de5535..731de5535 100755..100644 --- a/scripts/ems/example/data/weight.ini +++ b/scripts/ems/example/data/weight.ini diff --git a/scripts/ems/experiment.machines b/scripts/ems/experiment.machines index 9e0294d60..9e0294d60 100755..100644 --- a/scripts/ems/experiment.machines +++ b/scripts/ems/experiment.machines diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index 941d42fa7..941d42fa7 100755..100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index 60b6142b0..8530904c1 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -1552,7 +1552,7 @@ sub define_tuning_tune { # the last 3 variables are only used for mira tuning my ($tuned_config,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest) = &get_output_and_input($step_id); - + my $use_mira = &backoff_and_get("TUNING:use-mira"); my $cmd = ""; if ($use_mira && $use_mira eq "true") { @@ -1564,7 +1564,9 @@ sub define_tuning_tune { $mira_config .= "cfg"; write_mira_config($mira_config,$experiment_dir,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest); - $cmd = "$tuning_script -config $mira_config -exec >& $mira_config_log"; + #$cmd = "$tuning_script -config $mira_config -exec >& $mira_config_log"; + # we want error messages in top-level log file + $cmd = "$tuning_script -config $mira_config -exec "; # write script to select the best set of weights after training for the specified number of epochs --> # cp to tuning/tmp.?/moses.ini @@ -1611,70 +1613,21 @@ sub define_tuning_tune { } sub write_mira_config { - my ($config_filename,$expt_dir,$tune_filtered_ini,$input,$reference,$devtest_filtered_ini,$input_devtest,$reference_devtest) = @_; - + my ($config_filename,$expt_dir,$tune_filtered_ini,$input,$reference,$devtest_filtered_ini,$input_devtest,$reference_devtest) = @_; my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir"); my $tuning_decoder_settings = &check_and_get("TUNING:decoder-settings"); - my $core_weights = &backoff_and_get("TUNING:core-weight-config"); my $start_weights = &backoff_and_get("TUNING:start-weight-config"); my $tuning_settings = &check_and_get("TUNING:tuning-settings"); my @settings = split(/ /, $tuning_settings); my $mira_tuning_settings = &check_and_get("TUNING:mira-tuning-settings"); - # convert core weights into format expected by mira - my $core_file = "$expt_dir/core_weights"; - my $start_file = "$expt_dir/start_weights"; - if ($core_weights or $start_weights) { - open INI, $core_weights if $core_weights; - open INI, $start_weights if $start_weights; - #print STDERR "Reading core weights from file $core_weights \n"; - open CORE_OR_START, ">$core_file" if $core_weights; - open CORE_OR_START, ">$start_file" if $start_weights; - while(<INI>) { - if (/weight-l/) { - my @lm_weights; - while (<INI>) { - last if $_ eq "\n"; - push(@lm_weights, $_); - } - - print CORE_OR_START "LM ".$lm_weights[0]; - for my $i (1 .. $#lm_weights) { - print CORE_OR_START "LM:".($i+1)." ".$lm_weights[$i]; - } - - } elsif (/weight-t/) { - my @pm_weights; - while (<INI>) { - last if $_ eq "\n"; - push(@pm_weights, $_); - } - for my $i (0 .. $#pm_weights) { - print CORE_OR_START "PhraseModel_".($i+1)." ".$pm_weights[$i]; - } - } elsif (/weight-d/) { - my @d_weights; - while (<INI>) { - last if $_ eq "\n"; - push(@d_weights, $_); - } - for my $i (0 .. $#d_weights) { - if ($i == 0) { - print CORE_OR_START "Distortion ".$d_weights[0]; - } - else { - print CORE_OR_START "LexicalReordering_wbe-msd-bidirectional-fe-allff_".($i+1)." ".$d_weights[$i]; - } - } - - } elsif (/weight-w/) { - my $w = <INI>; - print CORE_OR_START "WordPenalty ".$w; - } - } - close INI; - close CORE_OR_START; - } + $tune_filtered_ini =~ /.*\/([A-Za-z0-9\.\-\_]*)$/; + my $tune_filtered_ini_start = $1; + $tune_filtered_ini_start = $expt_dir."/".$tune_filtered_ini_start.".start"; + if ($start_weights) { + # apply start weights to filtered ini file, and pass the new ini to mira + system("$Bin/support/reuse-weights.perl $start_weights < $tune_filtered_ini > $tune_filtered_ini_start"); + } # mira config file open(CFG, ">$config_filename"); @@ -1686,27 +1639,21 @@ sub write_mira_config { print CFG "working-dir=".$expt_dir."\n"; print CFG "wait-for-bleu=1 \n"; print CFG "decoder-settings=".$tuning_decoder_settings."\n\n"; - - if ($core_weights) { - print CFG "[core] \n"; - print CFG "weightfile=".$core_file."\n\n"; - } - if ($start_weights) { - print CFG "[start] \n"; - print CFG "weightfile=".$start_file."\n\n"; - } - print CFG "[train] \n"; print CFG "trainer=\${moses-home}/dist/bin/mira \n"; print CFG "input-file=".$input."\n"; print CFG "reference-files=".$reference."\n"; - print CFG "moses-ini-file=".$tune_filtered_ini."\n"; + if ($start_weights) { + print CFG "moses-ini-file=".$tune_filtered_ini_start."\n"; + } + else { + print CFG "moses-ini-file=".$tune_filtered_ini."\n"; + } print CFG "hours=48 \n"; foreach my $setting (@settings) { print CFG $setting."\n"; } - print CFG "extra-args=".$mira_tuning_settings."\n\n"; - + print CFG "extra-args=".$mira_tuning_settings."\n\n"; print CFG "[devtest] \n"; if (&get("TRAINING:hierarchical-rule-set")) { print CFG "moses=\${moses-home}/dist/bin/moses_chart \n"; diff --git a/scripts/ems/web/analysis.php b/scripts/ems/web/analysis.php index 489bdc50d..489bdc50d 100755..100644 --- a/scripts/ems/web/analysis.php +++ b/scripts/ems/web/analysis.php diff --git a/scripts/ems/web/analysis_diff.php b/scripts/ems/web/analysis_diff.php index 9cb853030..9cb853030 100755..100644 --- a/scripts/ems/web/analysis_diff.php +++ b/scripts/ems/web/analysis_diff.php diff --git a/scripts/ems/web/bilingual-concordance.css b/scripts/ems/web/bilingual-concordance.css index e232337d2..e232337d2 100755..100644 --- a/scripts/ems/web/bilingual-concordance.css +++ b/scripts/ems/web/bilingual-concordance.css diff --git a/scripts/ems/web/close.gif b/scripts/ems/web/close.gif Binary files differindex d45791ccd..d45791ccd 100755..100644 --- a/scripts/ems/web/close.gif +++ b/scripts/ems/web/close.gif diff --git a/scripts/ems/web/comment.php b/scripts/ems/web/comment.php index 04628ea4f..04628ea4f 100755..100644 --- a/scripts/ems/web/comment.php +++ b/scripts/ems/web/comment.php diff --git a/scripts/ems/web/diff.php b/scripts/ems/web/diff.php index 71e732af1..71e732af1 100755..100644 --- a/scripts/ems/web/diff.php +++ b/scripts/ems/web/diff.php diff --git a/scripts/ems/web/general.css b/scripts/ems/web/general.css index 5eb8e3eb8..5eb8e3eb8 100755..100644 --- a/scripts/ems/web/general.css +++ b/scripts/ems/web/general.css diff --git a/scripts/ems/web/hierarchical-segmentation.css b/scripts/ems/web/hierarchical-segmentation.css index 47f2c2693..47f2c2693 100755..100644 --- a/scripts/ems/web/hierarchical-segmentation.css +++ b/scripts/ems/web/hierarchical-segmentation.css diff --git a/scripts/ems/web/hierarchical-segmentation.js b/scripts/ems/web/hierarchical-segmentation.js index b4eb206ce..b4eb206ce 100755..100644 --- a/scripts/ems/web/hierarchical-segmentation.js +++ b/scripts/ems/web/hierarchical-segmentation.js diff --git a/scripts/ems/web/index.php b/scripts/ems/web/index.php index 466af9013..466af9013 100755..100644 --- a/scripts/ems/web/index.php +++ b/scripts/ems/web/index.php diff --git a/scripts/ems/web/javascripts/builder.js b/scripts/ems/web/javascripts/builder.js index f1f42b9ac..f1f42b9ac 100755..100644 --- a/scripts/ems/web/javascripts/builder.js +++ b/scripts/ems/web/javascripts/builder.js diff --git a/scripts/ems/web/javascripts/controls.js b/scripts/ems/web/javascripts/controls.js index 7392fb664..7392fb664 100755..100644 --- a/scripts/ems/web/javascripts/controls.js +++ b/scripts/ems/web/javascripts/controls.js diff --git a/scripts/ems/web/javascripts/dragdrop.js b/scripts/ems/web/javascripts/dragdrop.js index 15c6dbca6..15c6dbca6 100755..100644 --- a/scripts/ems/web/javascripts/dragdrop.js +++ b/scripts/ems/web/javascripts/dragdrop.js diff --git a/scripts/ems/web/javascripts/effects.js b/scripts/ems/web/javascripts/effects.js index 066ee5909..066ee5909 100755..100644 --- a/scripts/ems/web/javascripts/effects.js +++ b/scripts/ems/web/javascripts/effects.js diff --git a/scripts/ems/web/javascripts/prototype.js b/scripts/ems/web/javascripts/prototype.js index 845ab7fb4..845ab7fb4 100755..100644 --- a/scripts/ems/web/javascripts/prototype.js +++ b/scripts/ems/web/javascripts/prototype.js diff --git a/scripts/ems/web/javascripts/scriptaculous.js b/scripts/ems/web/javascripts/scriptaculous.js index 6bf437acc..6bf437acc 100755..100644 --- a/scripts/ems/web/javascripts/scriptaculous.js +++ b/scripts/ems/web/javascripts/scriptaculous.js diff --git a/scripts/ems/web/javascripts/slider.js b/scripts/ems/web/javascripts/slider.js index eb8305582..eb8305582 100755..100644 --- a/scripts/ems/web/javascripts/slider.js +++ b/scripts/ems/web/javascripts/slider.js diff --git a/scripts/ems/web/javascripts/sound.js b/scripts/ems/web/javascripts/sound.js index a286eb98e..a286eb98e 100755..100644 --- a/scripts/ems/web/javascripts/sound.js +++ b/scripts/ems/web/javascripts/sound.js diff --git a/scripts/ems/web/javascripts/unittest.js b/scripts/ems/web/javascripts/unittest.js index 33a0c7157..33a0c7157 100755..100644 --- a/scripts/ems/web/javascripts/unittest.js +++ b/scripts/ems/web/javascripts/unittest.js diff --git a/scripts/ems/web/lib.php b/scripts/ems/web/lib.php index 440940d9c..440940d9c 100755..100644 --- a/scripts/ems/web/lib.php +++ b/scripts/ems/web/lib.php diff --git a/scripts/ems/web/overview.php b/scripts/ems/web/overview.php index 7280bc35b..7280bc35b 100755..100644 --- a/scripts/ems/web/overview.php +++ b/scripts/ems/web/overview.php diff --git a/scripts/ems/web/setup b/scripts/ems/web/setup index ffab41501..ffab41501 100755..100644 --- a/scripts/ems/web/setup +++ b/scripts/ems/web/setup diff --git a/scripts/ems/web/spinner.gif b/scripts/ems/web/spinner.gif Binary files differindex c196a8289..c196a8289 100755..100644 --- a/scripts/ems/web/spinner.gif +++ b/scripts/ems/web/spinner.gif |
