diff options
author | Hieu Hoang <fishandfrolick@gmail.com> | 2012-05-25 15:42:11 +0400 |
---|---|---|
committer | Hieu Hoang <fishandfrolick@gmail.com> | 2012-05-25 15:42:11 +0400 |
commit | fcf817fda26a010d9f7563c7b14b28534816d7c3 (patch) | |
tree | 16b2c5de86c7b596b3dbff84a66767d7222052e7 /scripts/training | |
parent | e42f8d36aecdd484dfd5b6ff9393235af9036c38 (diff) |
add --parallel to sorting options
Diffstat (limited to 'scripts/training')
-rwxr-xr-x | scripts/training/train-model.perl.missing_bin_dir | 16 |
1 files changed, 10 insertions, 6 deletions
diff --git a/scripts/training/train-model.perl.missing_bin_dir b/scripts/training/train-model.perl.missing_bin_dir index f168f38cb..9d04cc1a1 100755 --- a/scripts/training/train-model.perl.missing_bin_dir +++ b/scripts/training/train-model.perl.missing_bin_dir @@ -19,7 +19,7 @@ if ($SCRIPTS_ROOTDIR eq '') { $SCRIPTS_ROOTDIR =~ s/\/training$//; $SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"}); -my($_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_DIR, $_TEMP_DIR, $_SORT_BUFFER_SIZE, $_SORT_BATCH_SIZE, $_SORT_COMPRESS, $_CORPUS, +my($_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_DIR, $_TEMP_DIR, $_SORT_BUFFER_SIZE, $_SORT_BATCH_SIZE, $_SORT_COMPRESS, $_SORT_PARALLEL, $_CORPUS, $_CORPUS_COMPRESSION, $_FIRST_STEP, $_LAST_STEP, $_F, $_E, $_MAX_PHRASE_LENGTH, $_LEXICAL_FILE, $_NO_LEXICAL_WEIGHTING, $_VERBOSE, $_ALIGNMENT, $_ALIGNMENT_FILE, $_ALIGNMENT_STEM, @_LM, $_EXTRACT_FILE, $_GIZA_OPTION, $_HELP, $_PARTS, @@ -58,8 +58,9 @@ $_HELP = 1 'model-dir=s' => \$_MODEL_DIR, 'temp-dir=s' => \$_TEMP_DIR, 'sort-buffer-size=s' => \$_SORT_BUFFER_SIZE, - 'sort-batch-size=s' => \$_SORT_BATCH_SIZE, + 'sort-batch-size=i' => \$_SORT_BATCH_SIZE, 'sort-compress=s' => \$_SORT_COMPRESS, + 'sort-parallel=i' => \$_SORT_PARALLEL, 'extract-file=s' => \$_EXTRACT_FILE, 'alignment=s' => \$_ALIGNMENT, 'alignment-file=s' => \$_ALIGNMENT_FILE, @@ -233,11 +234,14 @@ my $__SORT_BATCH_SIZE = ""; $__SORT_BATCH_SIZE = "--batch-size $_SORT_BATCH_SIZE" if $_SORT_BATCH_SIZE; my $__SORT_COMPRESS = ""; -$__SORT_COMPRESS = "--compress-program=$_SORT_COMPRESS" if $_SORT_COMPRESS; +$__SORT_COMPRESS = "--compress-program $_SORT_COMPRESS" if $_SORT_COMPRESS; + +my $__SORT_PARALLEL = ""; +$__SORT_PARALLEL = "--parallel $_SORT_PARALLEL" if $_SORT_PARALLEL; # supporting scripts/binaries from this package my $PHRASE_EXTRACT = "$SCRIPTS_ROOTDIR/training/phrase-extract/extract"; -$PHRASE_EXTRACT = "$SCRIPTS_ROOTDIR/generic/extract-parallel.perl $_CORES $SPLIT_EXEC \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS\" $PHRASE_EXTRACT"; +$PHRASE_EXTRACT = "$SCRIPTS_ROOTDIR/generic/extract-parallel.perl $_CORES $SPLIT_EXEC \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS $__SORT_PARALLEL\" $PHRASE_EXTRACT"; my $RULE_EXTRACT; if (defined($_GHKM)) { @@ -246,7 +250,7 @@ if (defined($_GHKM)) { else { $RULE_EXTRACT = "$SCRIPTS_ROOTDIR/training/phrase-extract/extract-rules"; } -$RULE_EXTRACT = "$SCRIPTS_ROOTDIR/generic/extract-parallel.perl $_CORES $SPLIT_EXEC \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS\" $RULE_EXTRACT"; +$RULE_EXTRACT = "$SCRIPTS_ROOTDIR/generic/extract-parallel.perl $_CORES $SPLIT_EXEC \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS $__SORT_PARALLEL\" $RULE_EXTRACT"; my $LEXICAL_REO_SCORER = "$SCRIPTS_ROOTDIR/training/lexical-reordering/score"; my $MEMSCORE = "$SCRIPTS_ROOTDIR/training/memscore/memscore"; @@ -255,7 +259,7 @@ my $SYMAL = "$SCRIPTS_ROOTDIR/training/symal/symal"; my $GIZA2BAL = "$SCRIPTS_ROOTDIR/training/symal/giza2bal.pl"; my $PHRASE_SCORE = "$SCRIPTS_ROOTDIR/training/phrase-extract/score"; -$PHRASE_SCORE = "$SCRIPTS_ROOTDIR/generic/score-parallel.perl $_CORES \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS\" $PHRASE_SCORE"; +$PHRASE_SCORE = "$SCRIPTS_ROOTDIR/generic/score-parallel.perl $_CORES \"$SORT_EXEC $__SORT_BUFFER_SIZE $__SORT_BATCH_SIZE $__SORT_COMPRESS $__SORT_PARALLEL\" $PHRASE_SCORE"; my $PHRASE_CONSOLIDATE = "$SCRIPTS_ROOTDIR/training/phrase-extract/consolidate"; |