Add test for saving validation

4 files changed, 63 insertions, 0 deletions

diff --git a/tests/training/restart/.gitignore b/tests/training/restart/.gitignore
index 57fd85c..dc7409f 100644
--- a/tests/training/restart/.gitignore
+++ b/tests/training/restart/.gitignore
@@ -1,3 +1,5 @@
+valid
+valid_script.temp
 sgd_2e
 adam
 adam_async
diff --git a/tests/training/restart/test_restoring_validation.sh b/tests/training/restart/test_restoring_validation.sh
new file mode 100644
index 0000000..fda8c45
--- /dev/null
+++ b/tests/training/restart/test_restoring_validation.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf valid valid_?.log valid_script.temp
+mkdir -p valid
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none \
+    --dim-emb 128 --dim-rnn 256 --mini-batch 16 \
+    -m valid/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 10 --valid-freq 20 --after-batches 150 --early-stopping 5 \
+    --valid-metrics valid-script cross-entropy --valid-script-path ./valid_script.sh \
+    --valid-sets $MRT_DATA/europarl.de-en/toy.bpe.{en,de} \
+    --valid-log valid_1.log
+
+test -e valid/model.npz
+test -e valid/model.npz.yml
+test -e valid_1.log
+
+cp valid/model.npz.yml valid/model.npz.yml.bac
+
+cat valid_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "valid-script" > valid.out
+#$MRT_TOOLS/diff-floats.py valid_1.out valid_1.expected -p 0.2 > valid_1.diff
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none \
+    --dim-emb 128 --dim-rnn 256 --mini-batch 16 \
+    -m valid/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 10 --valid-freq 20 --after-batches 300 --early-stopping 5 \
+    --valid-metrics valid-script cross-entropy --valid-script-path ./valid_script.sh \
+    --valid-sets $MRT_DATA/europarl.de-en/toy.bpe.{en,de} \
+    --valid-log valid_2.log
+
+test -e valid/model.npz
+test -e valid_2.log
+
+cat valid_2.log | $MRT_TOOLS/strip-timestamps.sh | grep "valid-script" >> valid.out
+diff valid.out valid.expected > valid.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/restart/valid.expected b/tests/training/restart/valid.expected
new file mode 100644
index 0000000..322dbcc
--- /dev/null
+++ b/tests/training/restart/valid.expected
@@ -0,0 +1,10 @@
+[valid] 20 : valid-script : 222.4 : new best
+[valid] 40 : valid-script : 222.3 : stalled 1 times
+[valid] 60 : valid-script : 222.2 : stalled 2 times
+[valid] 80 : valid-script : 222.1 : stalled 3 times
+[valid] 100 : valid-script : 222.6 : new best
+[valid] 120 : valid-script : 222.5 : stalled 1 times
+[valid] 140 : valid-script : 222.4 : stalled 2 times
+[valid] 150 : valid-script : 222.3 : stalled 3 times
+[valid] 160 : valid-script : 222.2 : stalled 4 times
+[valid] 180 : valid-script : 222.1 : stalled 5 times
diff --git a/tests/training/restart/valid_script.sh b/tests/training/restart/valid_script.sh
new file mode 100755
index 0000000..ac1152b
--- /dev/null
+++ b/tests/training/restart/valid_script.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+num=$(cat valid_script.temp 2>/dev/null || echo 1)
+((num=(num+1)%6))
+echo $num > valid_script.temp
+((num=6-num))
+echo 222.$num