enable nplm training on separate host, fix ems for nplm

2 files changed, 22 insertions, 4 deletions

diff --git a/scripts/training/bilingual-lm/train_nplm.py b/scripts/training/bilingual-lm/train_nplm.py
index 7fe8c9838..7cdb30cc4 100755
--- a/scripts/training/bilingual-lm/train_nplm.py
+++ b/scripts/training/bilingual-lm/train_nplm.py
@@ -43,6 +43,9 @@ parser.add_argument("--mmap", dest="mmap", action="store_true",
     help="Use memory-mapped file (for lower memory consumption).")
 parser.add_argument("--extra-settings", dest="extra_settings",
   help="Extra settings to be passed to NPLM")
+parser.add_argument(
+    "--train-host", dest="train_host",
+    help="Execute nplm training on this host, via ssh")
 
 parser.set_defaults(
     working_dir="working",
@@ -123,7 +126,10 @@ def main(options):
 
     model_prefix = os.path.join(
         options.output_dir, options.output_model + ".model.nplm")
-    train_args = [
+    train_args = []
+    if options.train_host:
+      train_args = ["ssh", options.train_host]
+    train_args += [
         options.nplm_home + "/src/trainNeuralNetwork",
         "--train_file", in_file,
         "--num_epochs", str(options.epochs),
diff --git a/scripts/training/train-neurallm.py b/scripts/training/train-neurallm.py
index f8ef17ff9..848538354 100755
--- a/scripts/training/train-neurallm.py
+++ b/scripts/training/train-neurallm.py
@@ -101,6 +101,9 @@ parser.add_argument(
 parser.add_argument(
     "--extra-settings", dest="extra_settings",
     help="Extra settings for nplm")
+parser.add_argument(
+    "--train-host", dest="train_host",
+    help="Execute nplm training on this host, via ssh")
 
 parser.set_defaults(
     working_dir="working",
@@ -143,7 +146,10 @@ def main(options):
     if options.mmap:
         train_file += '.mmap'
 
-    extraction_cmd = [
+    extraction_cmd = []
+    if options.train_host:
+      extraction_cmd = ["ssh", options.train_host]
+    extraction_cmd += [
         os.path.join(options.nplm_home, 'src', 'prepareNeuralLM'),
         '--train_text', options.corpus_stem,
         '--ngramize', '1',
@@ -183,7 +189,10 @@ def main(options):
             os.remove(os.path.join(options.working_dir, train_file))
         except OSError:
             pass
-        mmap_cmd = [
+        mmap_cmd = []
+        if options.train_host:
+          mmap_cmd = ["ssh", options.train_host]
+        mmap_cmd += [
             os.path.join(options.nplm_home, 'src', 'createMmap'),
             '--input_file',
             os.path.join(options.working_dir, numberized_file),
@@ -198,7 +207,10 @@ def main(options):
 
     if options.validation_corpus:
 
-        extraction_cmd = [
+        extraction_cmd = []
+        if options.train_host:
+          extraction_cmd = ["ssh", options.train_host]
+        extraction_cmd += [
             os.path.join(options.nplm_home, 'src', 'prepareNeuralLM'),
             '--train_text', options.validation_corpus,
             '--ngramize', '1',