Add test for training a LM from STDIN

3 files changed, 37 insertions, 0 deletions

diff --git a/tests/interface/input-tsv/.gitignore b/tests/interface/input-tsv/.gitignore
index e11922d..7f15397 100644
--- a/tests/interface/input-tsv/.gitignore
+++ b/tests/interface/input-tsv/.gitignore
@@ -11,6 +11,7 @@ train_vocabs
 train_vocabs_yml
 train_vocabs_nopaths
 train_empty_lines
+train_lm
 
 train.de
 train.en
diff --git a/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh b/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh
new file mode 100644
index 0000000..ca22c8f
--- /dev/null
+++ b/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Train a language model on data from a TSV file
+# TAGS: sentencepiece tsv train lm
+#####################################################################
+
+# Exit on error
+set -e
+
+# Remove old artifacts and create working directory
+rm -rf train_lm train_lm.{log,out,diff}
+mkdir -p train_lm
+
+# Run marian command
+cat train.en | $MRT_MARIAN/marian --type lm \
+    --no-shuffle --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \
+    -m train_lm/model.npz -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.spm \
+    --after-batches 10 --disp-freq 2 \
+    --log train_lm.log
+
+# Check if files exist
+test -e train_lm/model.npz
+test -e train_lm.log
+
+# Compare the current output with the expected output
+cat train_lm.log | $MRT_TOOLS/extract-costs.sh > train_lm.out
+$MRT_TOOLS/diff-nums.py train_lm.out train_lm.expected -p 0.01 -o train_lm.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/interface/input-tsv/train_lm.expected b/tests/interface/input-tsv/train_lm.expected
new file mode 100644
index 0000000..c768644
--- /dev/null
+++ b/tests/interface/input-tsv/train_lm.expected
@@ -0,0 +1,5 @@
+274.50836182
+274.87689209
+266.24481201
+259.36730957
+235.45114136