diff options
author | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2018-02-03 00:34:39 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2018-02-03 00:34:39 +0300 |
commit | 46b178b6056c20ea91ed7cef3ecd4adde8efd43d (patch) | |
tree | 3c40fa73e97b30958129681d342174800668f165 /tests | |
parent | d3dfbac5cfb2a84d6d3425f1de098b4d4fde5dea (diff) |
Add tests checking the validation of final models
Diffstat (limited to 'tests')
-rw-r--r-- | tests/training/basic/test_translation_script.sh | 2 | ||||
-rw-r--r-- | tests/training/valid/.gitignore | 4 | ||||
-rw-r--r-- | tests/training/valid/final_batch.expected | 3 | ||||
-rw-r--r-- | tests/training/valid/final_epoch.expected | 3 | ||||
-rw-r--r-- | tests/training/valid/final_match.expected | 3 | ||||
-rw-r--r-- | tests/training/valid/setup.sh | 2 | ||||
-rw-r--r-- | tests/training/valid/test_final_validation_after_batches.sh | 28 | ||||
-rw-r--r-- | tests/training/valid/test_final_validation_after_batches_match.sh | 28 | ||||
-rw-r--r-- | tests/training/valid/test_final_validation_after_epochs.sh | 31 | ||||
-rw-r--r-- | tests/training/valid/valid.bpe.de | 10 | ||||
-rw-r--r-- | tests/training/valid/valid.bpe.en | 10 |
11 files changed, 123 insertions, 1 deletions
diff --git a/tests/training/basic/test_translation_script.sh b/tests/training/basic/test_translation_script.sh index 1249e25..124b1a9 100644 --- a/tests/training/basic/test_translation_script.sh +++ b/tests/training/basic/test_translation_script.sh @@ -29,7 +29,7 @@ test -e trans.log grep -q "/tmp/marian.*" trans_script.temp -$MRT_TOOLS/strip-timestamps.sh < trans.log | grep -v "Total translation time" > trans.out +$MRT_TOOLS/strip-timestamps.sh < trans.log | grep -v "Total translation time" | head -n 4 > trans.out $MRT_TOOLS/diff-floats.py trans.out trans.expected -p 0.2 > trans.diff # Exit with success code diff --git a/tests/training/valid/.gitignore b/tests/training/valid/.gitignore new file mode 100644 index 0000000..c8fb94f --- /dev/null +++ b/tests/training/valid/.gitignore @@ -0,0 +1,4 @@ +final_batch +final_match +final_epoch +train.bpe.* diff --git a/tests/training/valid/final_batch.expected b/tests/training/valid/final_batch.expected new file mode 100644 index 0000000..3849c28 --- /dev/null +++ b/tests/training/valid/final_batch.expected @@ -0,0 +1,3 @@ +[valid] 60 : cross-entropy : 161.56 : new best +[valid] 120 : cross-entropy : 149.974 : new best +[valid] 150 : cross-entropy : 148.977 : new best diff --git a/tests/training/valid/final_epoch.expected b/tests/training/valid/final_epoch.expected new file mode 100644 index 0000000..f666c8e --- /dev/null +++ b/tests/training/valid/final_epoch.expected @@ -0,0 +1,3 @@ +[valid] 40 : cross-entropy : 179.973 : new best +[valid] 80 : cross-entropy : 154.704 : new best +[valid] 81 : cross-entropy : 154.563 : new best diff --git a/tests/training/valid/final_match.expected b/tests/training/valid/final_match.expected new file mode 100644 index 0000000..6c31288 --- /dev/null +++ b/tests/training/valid/final_match.expected @@ -0,0 +1,3 @@ +[valid] 60 : cross-entropy : 161.559 : new best +[valid] 120 : cross-entropy : 149.974 : new best +[valid] 180 : cross-entropy : 148.967 : new best diff --git a/tests/training/valid/setup.sh b/tests/training/valid/setup.sh new file mode 100644 index 0000000..8b8cd07 --- /dev/null +++ b/tests/training/valid/setup.sh @@ -0,0 +1,2 @@ +test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 +test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 diff --git a/tests/training/valid/test_final_validation_after_batches.sh b/tests/training/valid/test_final_validation_after_batches.sh new file mode 100644 index 0000000..78a3224 --- /dev/null +++ b/tests/training/valid/test_final_validation_after_batches.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x + +# Exit on error +set -e + +# Test code goes here +rm -rf final_batch final_batch.log vocab.*.yml +mkdir -p final_batch + +$MRT_MARIAN/build/marian \ + --no-shuffle --seed 1111 \ + -m final_batch/model.npz \ + -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \ + -v vocab.en.yml vocab.de.yml \ + --dim-vocabs 50000 50000 \ + --disp-freq 30 --valid-freq 60 --after-batches 150 \ + --valid-metrics cross-entropy \ + --valid-sets valid.bpe.en valid.bpe.de \ + --valid-log final_batch.log + +test -e final_batch/model.npz +test -e final_batch.log + +$MRT_TOOLS/strip-timestamps.sh < final_batch.log > final_batch.out +$MRT_TOOLS/diff-floats.py final_batch.out final_batch.expected -p 0.1 > final_batch.diff + +# Exit with success code +exit 0 diff --git a/tests/training/valid/test_final_validation_after_batches_match.sh b/tests/training/valid/test_final_validation_after_batches_match.sh new file mode 100644 index 0000000..6796795 --- /dev/null +++ b/tests/training/valid/test_final_validation_after_batches_match.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x + +# Exit on error +set -e + +# Test code goes here +rm -rf final_match final_match.log vocab.*.yml +mkdir -p final_match + +$MRT_MARIAN/build/marian \ + --no-shuffle --seed 1111 \ + -m final_match/model.npz \ + -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \ + -v vocab.en.yml vocab.de.yml \ + --dim-vocabs 50000 50000 \ + --disp-freq 30 --valid-freq 60 --after-batches 180 \ + --valid-metrics cross-entropy \ + --valid-sets valid.bpe.en valid.bpe.de \ + --valid-log final_match.log + +test -e final_match/model.npz +test -e final_match.log + +$MRT_TOOLS/strip-timestamps.sh < final_match.log > final_match.out +$MRT_TOOLS/diff-floats.py final_match.out final_match.expected -p 0.1 > final_match.diff + +# Exit with success code +exit 0 diff --git a/tests/training/valid/test_final_validation_after_epochs.sh b/tests/training/valid/test_final_validation_after_epochs.sh new file mode 100644 index 0000000..fee027d --- /dev/null +++ b/tests/training/valid/test_final_validation_after_epochs.sh @@ -0,0 +1,31 @@ +#!/bin/bash -x + +# Exit on error +set -e + +# Test code goes here +rm -rf final_epoch final_epoch.log vocab.*.yml +mkdir -p final_epoch + +head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.en > train.bpe.en +head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.de > train.bpe.de + +$MRT_MARIAN/build/marian \ + --no-shuffle --seed 1111 \ + -m final_epoch/model.npz \ + -t train.bpe.en train.bpe.de \ + -v vocab.en.yml vocab.de.yml \ + --dim-vocabs 50000 50000 \ + --mini-batch 32 --disp-freq 20 --valid-freq 40 --after-epochs 1 \ + --valid-metrics cross-entropy \ + --valid-sets valid.bpe.en valid.bpe.de \ + --valid-log final_epoch.log + +test -e final_epoch/model.npz +test -e final_epoch.log + +$MRT_TOOLS/strip-timestamps.sh < final_epoch.log > final_epoch.out +$MRT_TOOLS/diff-floats.py final_epoch.out final_epoch.expected -p 0.1 > final_epoch.diff + +# Exit with success code +exit 0 diff --git a/tests/training/valid/valid.bpe.de b/tests/training/valid/valid.bpe.de new file mode 100644 index 0000000..d27fbcd --- /dev/null +++ b/tests/training/valid/valid.bpe.de @@ -0,0 +1,10 @@ +damit , meine Damen und Herren , hat das Parlament die Tagesordnung abge@@ arbeitet . +das Protokoll dieser Sitzung wird dem Plenum zu Beginn der nächsten Sitzungsperiode zur Genehmigung vorgelegt . +das Wort hat Herr M@@ anders zur Geschäftsordnung . +Herr Präsident ! ich möchte diese Gelegenheit nutzen und Ihnen , dem Präsidium und allen Kolleginnen und Kollegen alles Gute für das neue Jahr wünschen . +meine Damen und Herren ! bevor Sie mich verlassen , möchte ich allen Abgeordneten , allen Diensten , Beamten , Assistenten und sonstigen Mitarbeitern im Namen des Präsidiums danken und , wenn Sie mir gestatten - obwohl alle Mitarbeiter für uns arbeiten - , vielleicht speziell die Sprachen@@ dienste nennen , die uns helfen , damit wir uns hier verständigen können . +ich möchte mir sogar erlauben , die Kommission und den Rat zu nennen , auch wenn sie nicht anwesend sind . +ich will die Aussprache über " Millennium ja , Millennium nein " nicht noch einmal eröffnen , aber ich möchte Ihnen allen und darüber hinaus den von uns vertretenen europäischen Bürgern ein glück@@ liches Jahr 2000 wünschen . +Unterbrechung der Sitzungsperiode +ich erkläre die Sitzungsperiode des Europäischen Parlaments für unterbrochen . +( die Sitzung wird um 10@@ .@@ 50 Uhr geschlossen . ) diff --git a/tests/training/valid/valid.bpe.en b/tests/training/valid/valid.bpe.en new file mode 100644 index 0000000..c573859 --- /dev/null +++ b/tests/training/valid/valid.bpe.en @@ -0,0 +1,10 @@ +having said that , Parliament has reached the end of the agenda . +the Minutes of the present sitting will be subject to Parliament ' s approval at the beginning of the next part @-@ session . +Mr M@@ anders has the floor for a procedural motion . +Mr President , I would like to take this opportunity to wish you , the Bureau and all Members , a good transition into the new year . +ladies and gentlemen , before you leave me alone , I would like on behalf of the Bureau , once again , to thank all the Members , all the services , officials , assistants and other co @-@ workers and , if you will allow me - although all the co @-@ workers work for us - perhaps a special mention should go to the language services which help us to understand each other here . +I would also like , although they are absent , to mention the Commission and the Council . +I am not going to re @-@ open the ' Millennium or not the Millennium ' debate , but I am going to wish all of you , and by extension , all the citizens of Europe which we represent , a happy year 2000 . +ad@@ journ@@ ment of the session +I declare the session of the European Parliament adjourned . +( the sitting was closed at 10@@ .@@ 50 a.m. ) |