diff options
author | Barry Haddow <barry.haddow@gmail.com> | 2016-01-06 14:55:12 +0300 |
---|---|---|
committer | Barry Haddow <barry.haddow@gmail.com> | 2016-01-06 14:55:12 +0300 |
commit | 7125096c294cdfa11086f48ee272d9321cf6d492 (patch) | |
tree | 9b7306f55460957bff17e92bf1f03862630d6b85 /scripts/training | |
parent | 5f4a3bd2d0544868fdeaaf757bb70bb5bdf21109 (diff) |
enable nplm training on separate host, fix ems for nplm
Diffstat (limited to 'scripts/training')
-rwxr-xr-x | scripts/training/bilingual-lm/train_nplm.py | 8 | ||||
-rwxr-xr-x | scripts/training/train-neurallm.py | 18 |
2 files changed, 22 insertions, 4 deletions
diff --git a/scripts/training/bilingual-lm/train_nplm.py b/scripts/training/bilingual-lm/train_nplm.py index 7fe8c9838..7cdb30cc4 100755 --- a/scripts/training/bilingual-lm/train_nplm.py +++ b/scripts/training/bilingual-lm/train_nplm.py @@ -43,6 +43,9 @@ parser.add_argument("--mmap", dest="mmap", action="store_true", help="Use memory-mapped file (for lower memory consumption).") parser.add_argument("--extra-settings", dest="extra_settings", help="Extra settings to be passed to NPLM") +parser.add_argument( + "--train-host", dest="train_host", + help="Execute nplm training on this host, via ssh") parser.set_defaults( working_dir="working", @@ -123,7 +126,10 @@ def main(options): model_prefix = os.path.join( options.output_dir, options.output_model + ".model.nplm") - train_args = [ + train_args = [] + if options.train_host: + train_args = ["ssh", options.train_host] + train_args += [ options.nplm_home + "/src/trainNeuralNetwork", "--train_file", in_file, "--num_epochs", str(options.epochs), diff --git a/scripts/training/train-neurallm.py b/scripts/training/train-neurallm.py index f8ef17ff9..848538354 100755 --- a/scripts/training/train-neurallm.py +++ b/scripts/training/train-neurallm.py @@ -101,6 +101,9 @@ parser.add_argument( parser.add_argument( "--extra-settings", dest="extra_settings", help="Extra settings for nplm") +parser.add_argument( + "--train-host", dest="train_host", + help="Execute nplm training on this host, via ssh") parser.set_defaults( working_dir="working", @@ -143,7 +146,10 @@ def main(options): if options.mmap: train_file += '.mmap' - extraction_cmd = [ + extraction_cmd = [] + if options.train_host: + extraction_cmd = ["ssh", options.train_host] + extraction_cmd += [ os.path.join(options.nplm_home, 'src', 'prepareNeuralLM'), '--train_text', options.corpus_stem, '--ngramize', '1', @@ -183,7 +189,10 @@ def main(options): os.remove(os.path.join(options.working_dir, train_file)) except OSError: pass - mmap_cmd = [ + mmap_cmd = [] + if options.train_host: + mmap_cmd = ["ssh", options.train_host] + mmap_cmd += [ os.path.join(options.nplm_home, 'src', 'createMmap'), '--input_file', os.path.join(options.working_dir, numberized_file), @@ -198,7 +207,10 @@ def main(options): if options.validation_corpus: - extraction_cmd = [ + extraction_cmd = [] + if options.train_host: + extraction_cmd = ["ssh", options.train_host] + extraction_cmd += [ os.path.join(options.nplm_home, 'src', 'prepareNeuralLM'), '--train_text', options.validation_corpus, '--ngramize', '1', |