Add tests checking the validation of final models

11 files changed, 123 insertions, 1 deletions

diff --git a/tests/training/basic/test_translation_script.sh b/tests/training/basic/test_translation_script.sh
index 1249e25..124b1a9 100644
--- a/tests/training/basic/test_translation_script.sh
+++ b/tests/training/basic/test_translation_script.sh
@@ -29,7 +29,7 @@ test -e trans.log
 
 grep -q "/tmp/marian.*" trans_script.temp
 
-$MRT_TOOLS/strip-timestamps.sh < trans.log | grep -v "Total translation time" > trans.out
+$MRT_TOOLS/strip-timestamps.sh < trans.log | grep -v "Total translation time" | head -n 4 > trans.out
 $MRT_TOOLS/diff-floats.py trans.out trans.expected -p 0.2 > trans.diff
 
 # Exit with success code
diff --git a/tests/training/valid/.gitignore b/tests/training/valid/.gitignore
new file mode 100644
index 0000000..c8fb94f
--- /dev/null
+++ b/tests/training/valid/.gitignore
@@ -0,0 +1,4 @@
+final_batch
+final_match
+final_epoch
+train.bpe.*
diff --git a/tests/training/valid/final_batch.expected b/tests/training/valid/final_batch.expected
new file mode 100644
index 0000000..3849c28
--- /dev/null
+++ b/tests/training/valid/final_batch.expected
@@ -0,0 +1,3 @@
+[valid] 60 : cross-entropy : 161.56 : new best
+[valid] 120 : cross-entropy : 149.974 : new best
+[valid] 150 : cross-entropy : 148.977 : new best
diff --git a/tests/training/valid/final_epoch.expected b/tests/training/valid/final_epoch.expected
new file mode 100644
index 0000000..f666c8e
--- /dev/null
+++ b/tests/training/valid/final_epoch.expected
@@ -0,0 +1,3 @@
+[valid] 40 : cross-entropy : 179.973 : new best
+[valid] 80 : cross-entropy : 154.704 : new best
+[valid] 81 : cross-entropy : 154.563 : new best
diff --git a/tests/training/valid/final_match.expected b/tests/training/valid/final_match.expected
new file mode 100644
index 0000000..6c31288
--- /dev/null
+++ b/tests/training/valid/final_match.expected
@@ -0,0 +1,3 @@
+[valid] 60 : cross-entropy : 161.559 : new best
+[valid] 120 : cross-entropy : 149.974 : new best
+[valid] 180 : cross-entropy : 148.967 : new best
diff --git a/tests/training/valid/setup.sh b/tests/training/valid/setup.sh
new file mode 100644
index 0000000..8b8cd07
--- /dev/null
+++ b/tests/training/valid/setup.sh
@@ -0,0 +1,2 @@
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1
diff --git a/tests/training/valid/test_final_validation_after_batches.sh b/tests/training/valid/test_final_validation_after_batches.sh
new file mode 100644
index 0000000..78a3224
--- /dev/null
+++ b/tests/training/valid/test_final_validation_after_batches.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf final_batch final_batch.log vocab.*.yml
+mkdir -p final_batch
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 1111 \
+    -m final_batch/model.npz \
+    -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \
+    -v vocab.en.yml vocab.de.yml \
+    --dim-vocabs 50000 50000 \
+    --disp-freq 30 --valid-freq 60 --after-batches 150 \
+    --valid-metrics cross-entropy \
+    --valid-sets valid.bpe.en valid.bpe.de \
+    --valid-log final_batch.log
+
+test -e final_batch/model.npz
+test -e final_batch.log
+
+$MRT_TOOLS/strip-timestamps.sh < final_batch.log > final_batch.out
+$MRT_TOOLS/diff-floats.py final_batch.out final_batch.expected -p 0.1 > final_batch.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/valid/test_final_validation_after_batches_match.sh b/tests/training/valid/test_final_validation_after_batches_match.sh
new file mode 100644
index 0000000..6796795
--- /dev/null
+++ b/tests/training/valid/test_final_validation_after_batches_match.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf final_match final_match.log vocab.*.yml
+mkdir -p final_match
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 1111 \
+    -m final_match/model.npz \
+    -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \
+    -v vocab.en.yml vocab.de.yml \
+    --dim-vocabs 50000 50000 \
+    --disp-freq 30 --valid-freq 60 --after-batches 180 \
+    --valid-metrics cross-entropy \
+    --valid-sets valid.bpe.en valid.bpe.de \
+    --valid-log final_match.log
+
+test -e final_match/model.npz
+test -e final_match.log
+
+$MRT_TOOLS/strip-timestamps.sh < final_match.log > final_match.out
+$MRT_TOOLS/diff-floats.py final_match.out final_match.expected -p 0.1 > final_match.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/valid/test_final_validation_after_epochs.sh b/tests/training/valid/test_final_validation_after_epochs.sh
new file mode 100644
index 0000000..fee027d
--- /dev/null
+++ b/tests/training/valid/test_final_validation_after_epochs.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf final_epoch final_epoch.log vocab.*.yml
+mkdir -p final_epoch
+
+head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.en > train.bpe.en
+head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.de > train.bpe.de
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 1111 \
+    -m final_epoch/model.npz \
+    -t train.bpe.en train.bpe.de \
+    -v vocab.en.yml vocab.de.yml \
+    --dim-vocabs 50000 50000 \
+    --mini-batch 32 --disp-freq 20 --valid-freq 40 --after-epochs 1 \
+    --valid-metrics cross-entropy \
+    --valid-sets valid.bpe.en valid.bpe.de \
+    --valid-log final_epoch.log
+
+test -e final_epoch/model.npz
+test -e final_epoch.log
+
+$MRT_TOOLS/strip-timestamps.sh < final_epoch.log > final_epoch.out
+$MRT_TOOLS/diff-floats.py final_epoch.out final_epoch.expected -p 0.1 > final_epoch.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/valid/valid.bpe.de b/tests/training/valid/valid.bpe.de
new file mode 100644
index 0000000..d27fbcd
--- /dev/null
+++ b/tests/training/valid/valid.bpe.de
@@ -0,0 +1,10 @@
+damit , meine Damen und Herren , hat das Parlament die Tagesordnung abge@@ arbeitet .
+das Protokoll dieser Sitzung wird dem Plenum zu Beginn der nächsten Sitzungsperiode zur Genehmigung vorgelegt .
+das Wort hat Herr M@@ anders zur Geschäftsordnung .
+Herr Präsident ! ich möchte diese Gelegenheit nutzen und Ihnen , dem Präsidium und allen Kolleginnen und Kollegen alles Gute für das neue Jahr wünschen .
+meine Damen und Herren ! bevor Sie mich verlassen , möchte ich allen Abgeordneten , allen Diensten , Beamten , Assistenten und sonstigen Mitarbeitern im Namen des Präsidiums danken und , wenn Sie mir gestatten - obwohl alle Mitarbeiter für uns arbeiten - , vielleicht speziell die Sprachen@@ dienste nennen , die uns helfen , damit wir uns hier verständigen können .
+ich möchte mir sogar erlauben , die Kommission und den Rat zu nennen , auch wenn sie nicht anwesend sind .
+ich will die Aussprache über &quot; Millennium ja , Millennium nein &quot; nicht noch einmal eröffnen , aber ich möchte Ihnen allen und darüber hinaus den von uns vertretenen europäischen Bürgern ein glück@@ liches Jahr 2000 wünschen .
+Unterbrechung der Sitzungsperiode
+ich erkläre die Sitzungsperiode des Europäischen Parlaments für unterbrochen .
+( die Sitzung wird um 10@@ .@@ 50 Uhr geschlossen . )
diff --git a/tests/training/valid/valid.bpe.en b/tests/training/valid/valid.bpe.en
new file mode 100644
index 0000000..c573859
--- /dev/null
+++ b/tests/training/valid/valid.bpe.en
@@ -0,0 +1,10 @@
+having said that , Parliament has reached the end of the agenda .
+the Minutes of the present sitting will be subject to Parliament &apos; s approval at the beginning of the next part @-@ session .
+Mr M@@ anders has the floor for a procedural motion .
+Mr President , I would like to take this opportunity to wish you , the Bureau and all Members , a good transition into the new year .
+ladies and gentlemen , before you leave me alone , I would like on behalf of the Bureau , once again , to thank all the Members , all the services , officials , assistants and other co @-@ workers and , if you will allow me - although all the co @-@ workers work for us - perhaps a special mention should go to the language services which help us to understand each other here .
+I would also like , although they are absent , to mention the Commission and the Council .
+I am not going to re @-@ open the &apos; Millennium or not the Millennium &apos; debate , but I am going to wish all of you , and by extension , all the citizens of Europe which we represent , a happy year 2000 .
+ad@@ journ@@ ment of the session
+I declare the session of the European Parliament adjourned .
+( the sitting was closed at 10@@ .@@ 50 a.m. )