diff options
author | Hieu Hoang <fishandfrolick@gmail.com> | 2012-06-01 22:28:22 +0400 |
---|---|---|
committer | Hieu Hoang <fishandfrolick@gmail.com> | 2012-06-01 22:28:22 +0400 |
commit | c6756f8301e4a735ecbf0299b4a1e7a642c34f03 (patch) | |
tree | a2928f67abe6103c754e46f669c3a6259a5c6f12 /scripts | |
parent | f0f4893a3dda8d76f482be001bbb7930ade74ce1 (diff) |
start integrating adam's suffix array
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/ems/experiment.meta | 13 | ||||
-rwxr-xr-x | scripts/ems/experiment.perl | 20 |
2 files changed, 31 insertions, 2 deletions
diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index 8fb50fb52..68c147819 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -332,6 +332,13 @@ build-biconcor default-name: model/biconcor ignore-unless: biconcor error: usage +build-suffix-array + in: word-alignment corpus + out: suffix-array-model + default-name: model/suffix-array + ignore-unless: suffix-array + error: usage + build-lex-trans in: word-alignment corpus out: lexical-translation-table @@ -362,6 +369,7 @@ extract-phrases out: extracted-phrases rerun-on-change: max-phrase-length translation-factors reordering-factors hierarchical-rule-set extract-settings training-options script use-ghkm default-name: model/extract + ignore-if: suffix-array build-reordering in: extracted-phrases out: reordering-table @@ -373,6 +381,7 @@ build-ttable out: phrase-translation-table rerun-on-change: translation-factors hierarchical-rule-set score-settings training-options script EVALUATION:report-precision-by-coverage include-word-alignment-in-rules default-name: model/phrase-table + ignore-if: suffix-array build-generation in: corpus out: generation-table @@ -387,7 +396,7 @@ build-generation-custom ignore-unless: AND generation-factors generation-corpus default-name: model/generation-table create-config - in: reordering-table phrase-translation-table generation-table INTERPOLATED-LM:binlm LM:binlm + in: reordering-table phrase-translation-table generation-table INTERPOLATED-LM:binlm LM:binlm suffix-array-model out: config ignore-if: use-hiero rerun-on-change: decoding-steps alignment-factors translation-factors reordering-factors generation-factors lexicalized-reordering training-options script decoding-graph-backoff score-settings additional-ini @@ -820,6 +829,6 @@ analysis-precision [REPORTING] single report - in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model + in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model out: report default-name: evaluation/report diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index b0e71c969..b9d025d65 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -916,6 +916,10 @@ sub define_step { elsif ($DO_STEP[$i] eq 'TRAINING:build-biconcor') { &define_training_build_biconcor($i); } + elsif ($DO_STEP[$i] eq 'TRAINING:build-suffix-array') { + &define_training_build_suffix_array($i); + } + elsif ($DO_STEP[$i] eq 'TRAINING:build-lex-trans') { &define_training_build_lex_trans($i); } @@ -1632,6 +1636,22 @@ sub define_training_symmetrize_giza { &create_step($step_id,$cmd); } +sub define_training_build_suffix_array { + my ($step_id) = @_; + + my $scripts = &check_and_get("GENERAL:moses-script-dir"); + + my ($model, $aligned,$corpus) = &get_output_and_input($step_id); + my $sa_exec_dir = &check_and_get("TRAINING:suffix-array"); + my $input_extension = &check_backoff_and_get("TRAINING:input-extension"); + my $output_extension = &check_backoff_and_get("TRAINING:output-extension"); + my $method = &check_and_get("TRAINING:alignment-symmetrization-method"); + + my $cmd = "$scripts/training/wrappers/adam-suffix-array/suffix-array-create.sh $sa_exec_dir $corpus.$input_extension $corpus.$output_extension $aligned.$method $model"; + + &create_step($step_id,$cmd); +} + sub define_training_build_biconcor { my ($step_id) = @_; |