Merge branch 'romang/new-metrics-finetune' into romang/merge-prs

3 files changed, 73 insertions, 0 deletions

diff --git a/tests/training/restoring/validation/.gitignore b/tests/training/restoring/validation/.gitignore
index 88578bc..e6a5770 100644
--- a/tests/training/restoring/validation/.gitignore
+++ b/tests/training/restoring/validation/.gitignore
@@ -5,6 +5,7 @@ valid_reset_stalled
 valid_stalled
 valid_add
 valid_newbest
+valid_reset_add
 train.*
 valid.*
 dev.bpe.de
diff --git a/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh b/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh
new file mode 100644
index 0000000..95fe20b
--- /dev/null
+++ b/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh
@@ -0,0 +1,57 @@
+#!/bin/bash -x
+
+#####################################################################
+# TAGS: valid valid-restart
+#####################################################################
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf valid_reset_add valid_reset_add_?.log
+mkdir -p valid_reset_add
+
+extra_opts="--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd"
+extra_opts="$extra_opts --dim-emb 128 --dim-rnn 256 --mini-batch 16"
+extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false --clip-norm 0"
+
+#$MRT_MARIAN/marian $extra_opts \
+    #-m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
+    #--disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 \
+    #--valid-metrics cross-entropy perplexity \
+    #--valid-sets dev.bpe.{en,de} --valid-mini-batch 64 \
+    #--valid-log valid_reset_add.expected.log
+
+#cat valid_reset_add.expected.log | $MRT_TOOLS/strip-timestamps.sh > valid_reset_add.expected
+#exit 1
+
+
+$MRT_MARIAN/marian $extra_opts \
+    -m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
+    --disp-freq 10 --valid-freq 20 --after-batches 100 --early-stopping 5 \
+    --valid-metrics cross-entropy \
+    --valid-sets dev.bpe.{en,de} --valid-mini-batch 64 \
+    --valid-log valid_reset_add_1.log
+
+test -e valid_reset_add/model.npz
+test -e valid_reset_add/model.npz.yml
+test -e valid_reset_add_1.log
+
+cp valid_reset_add/model.npz.progress.yml valid_reset_add/model.npz.progress.yml.bac
+cat valid_reset_add_1.log | $MRT_TOOLS/strip-timestamps.sh > valid_reset_add.out
+
+$MRT_MARIAN/marian $extra_opts \
+    -m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
+    --disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 \
+    --valid-metrics cross-entropy ce-mean-words \
+    --valid-sets dev.bpe.{en,de} --valid-mini-batch 64 --valid-reset-stalled --no-restore-corpus \
+    --valid-log valid_reset_add_2.log
+
+test -e valid_reset_add/model.npz
+test -e valid_reset_add_2.log
+
+cat valid_reset_add_2.log | $MRT_TOOLS/strip-timestamps.sh >> valid_reset_add.out
+$MRT_TOOLS/diff-nums.py -p 0.003 valid_reset_add.out valid_reset_add.expected -o valid_reset_add.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/restoring/validation/valid_reset_add.expected b/tests/training/restoring/validation/valid_reset_add.expected
new file mode 100644
index 0000000..b3780a9
--- /dev/null
+++ b/tests/training/restoring/validation/valid_reset_add.expected
@@ -0,0 +1,15 @@
+[valid] Ep. 1 : Up. 20 : cross-entropy : 294.63 : new best
+[valid] Ep. 1 : Up. 40 : cross-entropy : 292.643 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 290.224 : new best
+[valid] Ep. 1 : Up. 80 : cross-entropy : 286.857 : new best
+[valid] Ep. 1 : Up. 100 : cross-entropy : 282.156 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 274.772 : new best
+[valid] Ep. 1 : Up. 120 : ce-mean-words : 9.42616 : new best
+[valid] Ep. 1 : Up. 140 : cross-entropy : 264.221 : new best
+[valid] Ep. 1 : Up. 140 : ce-mean-words : 9.06417 : new best
+[valid] Ep. 1 : Up. 160 : cross-entropy : 258.245 : new best
+[valid] Ep. 1 : Up. 160 : ce-mean-words : 8.85916 : new best
+[valid] Ep. 1 : Up. 180 : cross-entropy : 255.25 : new best
+[valid] Ep. 1 : Up. 180 : ce-mean-words : 8.75644 : new best
+[valid] Ep. 1 : Up. 200 : cross-entropy : 252.909 : new best
+[valid] Ep. 1 : Up. 200 : ce-mean-words : 8.67611 : new best