diff options
55 files changed, 594 insertions, 51 deletions
@@ -45,3 +45,4 @@ data/*/*.bpe data/*/truecase* data/*/*.gz data/*.md5 +data/exdb_mnist diff --git a/data/download-data.sh b/data/download-data.sh index 37e71be..ce6ba2d 100755 --- a/data/download-data.sh +++ b/data/download-data.sh @@ -9,6 +9,7 @@ URL=https://romang.blob.core.windows.net/mariandev/regression-tests/data # name as the tarball DATA_TARBALLS=( europarl.de-en + exdb_mnist ) for name in ${DATA_TARBALLS[@]}; do diff --git a/tests/examples/mnist/setup.sh b/tests/examples/mnist/setup.sh index 8dead6a..1d0221f 100644 --- a/tests/examples/mnist/setup.sh +++ b/tests/examples/mnist/setup.sh @@ -6,14 +6,12 @@ set -e # Setup code goes here test -e $MRT_MARIAN/mnist_example -(( $(ls -1 *-ubyte 2>/dev/null | wc -l) == 4 )) && exit 0 +test -e $MRT_DATA/exdb_mnist/train-images-idx3-ubyte +test -e $MRT_DATA/exdb_mnist/train-labels-idx1-ubyte +test -e $MRT_DATA/exdb_mnist/t10k-images-idx3-ubyte +test -e $MRT_DATA/exdb_mnist/t10k-labels-idx1-ubyte -wget -nc http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz -wget -nc http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz -wget -nc http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz -wget -nc http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz - -gzip -d *-ubyte.gz +test -e *-ubyte || cp $MRT_DATA/exdb_mnist/*-ubyte . # Exit with success code exit 0 diff --git a/tests/interface/config/test_load_config.sh b/tests/interface/config/test_load_config.sh index 0e66fd4..17fe51a 100644 --- a/tests/interface/config/test_load_config.sh +++ b/tests/interface/config/test_load_config.sh @@ -7,7 +7,7 @@ rm -rf load_config load_config.log no_config.log mkdir -p load_config # Run with no config file -$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle --seed 2222 \ --type s2s --mini-batch 8 --dim-rnn 32 --dim-emb 16 --after-batches 2 --log load_config.log test -e load_config/model.npz @@ -19,7 +19,7 @@ rm -rf load_config mkdir -p load_config # Run with config file and the same options -$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle --seed 2222 \ --config load_config.yml --log load_config.log test -e load_config/model.npz diff --git a/tests/server/test_ende.sh b/tests/server/test_ende.sh index 75392cf..71bbb8d 100644 --- a/tests/server/test_ende.sh +++ b/tests/server/test_ende.sh @@ -9,7 +9,7 @@ set -e clean_up() { - kill $SERVER_PID + kill $SERVER_PID || : } trap clean_up EXIT diff --git a/tests/server/test_ende_align.sh b/tests/server/test_ende_align.sh index 49e4f40..6351889 100644 --- a/tests/server/test_ende_align.sh +++ b/tests/server/test_ende_align.sh @@ -9,7 +9,7 @@ set -e clean_up() { - kill $SERVER_PID + kill $SERVER_PID || : } trap clean_up EXIT diff --git a/tests/server/test_ende_batch32.sh b/tests/server/test_ende_batch32.sh index 2866e81..95e967b 100644 --- a/tests/server/test_ende_batch32.sh +++ b/tests/server/test_ende_batch32.sh @@ -9,7 +9,7 @@ set -e clean_up() { - kill $SERVER_PID + kill $SERVER_PID || : } trap clean_up EXIT diff --git a/tests/server/test_ende_cpu.sh b/tests/server/test_ende_cpu.sh index bc05803..2cef7c2 100644 --- a/tests/server/test_ende_cpu.sh +++ b/tests/server/test_ende_cpu.sh @@ -14,7 +14,7 @@ if [ ! $MRT_MARIAN_USE_MKL ]; then fi clean_up() { - kill $SERVER_PID + kill $SERVER_PID || : } trap clean_up EXIT diff --git a/tests/server/test_ende_with_empty_lines.sh b/tests/server/test_ende_with_empty_lines.sh index c089aaa..2103cf2 100644 --- a/tests/server/test_ende_with_empty_lines.sh +++ b/tests/server/test_ende_with_empty_lines.sh @@ -9,7 +9,7 @@ set -e clean_up() { - kill $SERVER_PID + kill $SERVER_PID || : } trap clean_up EXIT diff --git a/tests/training/features/lr-decay/.gitignore b/tests/training/features/lr-decay/.gitignore new file mode 100644 index 0000000..bff5533 --- /dev/null +++ b/tests/training/features/lr-decay/.gitignore @@ -0,0 +1,3 @@ +lrdecay_* +devset.* +*.temp diff --git a/tests/training/features/lr-decay/lrdecay_stalled.expected b/tests/training/features/lr-decay/lrdecay_stalled.expected new file mode 100644 index 0000000..0f0c7c2 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled.expected @@ -0,0 +1,25 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.68312 : stalled 6 times (last best: 2.56794) +Decaying learning rate to 4.78297e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_all.expected b/tests/training/features/lr-decay/lrdecay_stalled_all.expected new file mode 100644 index 0000000..0dda108 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_all.expected @@ -0,0 +1,23 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57813 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.5914 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +Decaying learning rate to 8.1e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.64345 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +Decaying learning rate to 7.29e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.67197 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.73563 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_any.expected b/tests/training/features/lr-decay/lrdecay_stalled_any.expected new file mode 100644 index 0000000..9b21998 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_any.expected @@ -0,0 +1,22 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 5 time(s) diff --git a/tests/training/features/lr-decay/setup.sh b/tests/training/features/lr-decay/setup.sh new file mode 100644 index 0000000..1dc82af --- /dev/null +++ b/tests/training/features/lr-decay/setup.sh @@ -0,0 +1,6 @@ +test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 +test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 + +test -f $MRT_MODELS/wngt19/model.small.npz || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.en || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.de || exit 1 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled.sh b/tests/training/features/lr-decay/test_lr_decay_stalled.sh new file mode 100644 index 0000000..6f9abd7 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled lrdecay_stalled.log valid_script.temp +mkdir -p lrdecay_stalled + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 + +test -e lrdecay_stalled/model.npz +test -e lrdecay_stalled/model.npz.yml +test -e lrdecay_stalled.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled.out lrdecay_stalled.expected -p 0.2 -o lrdecay_stalled.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh new file mode 100644 index 0000000..280ba9f --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_all lrdecay_stalled_all.log valid_script.temp +mkdir -p lrdecay_stalled_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_all.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 --early-stopping-on all + +test -e lrdecay_stalled_all/model.npz +test -e lrdecay_stalled_all/model.npz.yml +test -e lrdecay_stalled_all.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_all.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_all.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_all.out lrdecay_stalled_all.expected -p 0.2 -o lrdecay_stalled_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh new file mode 100644 index 0000000..55c8f81 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_any lrdecay_stalled_any.log valid_script.temp +mkdir -p lrdecay_stalled_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_any.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 5 --early-stopping-on any + +test -e lrdecay_stalled_any/model.npz +test -e lrdecay_stalled_any/model.npz.yml +test -e lrdecay_stalled_any.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_any.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_any.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_any.out lrdecay_stalled_any.expected -p 0.2 -o lrdecay_stalled_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/valid_script.sh b/tests/training/features/lr-decay/valid_script.sh new file mode 100755 index 0000000..faf1369 --- /dev/null +++ b/tests/training/features/lr-decay/valid_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=valid_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth.sh b/tests/training/restoring/exp-smoothing/test_expsmooth.sh index e7c7b6d..47040de 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth.sh @@ -60,7 +60,8 @@ $MRT_MARIAN/marian \ --log expsmooth_1.log test -e expsmooth/model.npz -test -e expsmooth/model.npz.orig.npz +# @TODO: check for averaged parameters in checkpoint file +# test -e expsmooth/model.npz.orig.npz test -e expsmooth_1.log cat expsmooth_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' > expsmooth.out @@ -74,7 +75,8 @@ $MRT_MARIAN/marian \ --log expsmooth_2.log test -e expsmooth/model.npz -test -e expsmooth/model.npz.orig.npz +# @TODO: check for averaged parameters in checkpoint file +#test -e expsmooth/model.npz.orig.npz test -e expsmooth_2.log cat expsmooth_2.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' >> expsmooth.out diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh b/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh index 831ebec..6062a24 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh @@ -60,7 +60,8 @@ $MRT_MARIAN/marian \ --log expsmooth_s2s_1.log test -e expsmooth_s2s/model.npz -test -e expsmooth_s2s/model.npz.orig.npz +# @TODO: test for smoothed parameters in checkpoint file +# test -e expsmooth_s2s/model.npz.orig.npz test -e expsmooth_s2s_1.log cat expsmooth_s2s_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' > expsmooth_s2s.out @@ -74,7 +75,8 @@ $MRT_MARIAN/marian \ --log expsmooth_s2s_2.log test -e expsmooth_s2s/model.npz -test -e expsmooth_s2s/model.npz.orig.npz +# @TODO: test for smoothed parameters in checkpoint file +# test -e expsmooth_s2s/model.npz.orig.npz test -e expsmooth_s2s_2.log cat expsmooth_s2s_2.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' >> expsmooth_s2s.out diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh b/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh index eafc1cc..6ab255c 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh @@ -65,7 +65,8 @@ $MRT_MARIAN/marian \ --log expsmooth_sync_1.log test -e expsmooth_sync/model.npz -test -e expsmooth_sync/model.npz.orig.npz +# @TODO: test for smoothed parameters in checkpoint file +# test -e expsmooth_sync/model.npz.orig.npz test -e expsmooth_sync_1.log cat expsmooth_sync_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' > expsmooth_sync.out @@ -79,7 +80,8 @@ $MRT_MARIAN/marian \ --log expsmooth_sync_2.log test -e expsmooth_sync/model.npz -test -e expsmooth_sync/model.npz.orig.npz +# @TODO: test for smoothed parameters in checkpoint file +# test -e expsmooth_sync/model.npz.orig.npz test -e expsmooth_sync_2.log cat expsmooth_sync_2.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'valid' | sed 's/ : Time.*//' >> expsmooth_sync.out diff --git a/tests/training/restoring/optimizer/adagrad.keys.expected b/tests/training/restoring/optimizer/adagrad.keys.expected index 2899089..f37412c 100644 --- a/tests/training/restoring/optimizer/adagrad.keys.expected +++ b/tests/training/restoring/optimizer/adagrad.keys.expected @@ -1 +1,2 @@ adagrad_gt +master_parameters diff --git a/tests/training/restoring/optimizer/adam.keys.expected b/tests/training/restoring/optimizer/adam.keys.expected index fe9ecb1..e616f76 100644 --- a/tests/training/restoring/optimizer/adam.keys.expected +++ b/tests/training/restoring/optimizer/adam.keys.expected @@ -1,3 +1,4 @@ adam_mt adam_vt adam_denoms +master_parameters diff --git a/tests/training/restoring/validation/.gitignore b/tests/training/restoring/validation/.gitignore index 88578bc..e6a5770 100644 --- a/tests/training/restoring/validation/.gitignore +++ b/tests/training/restoring/validation/.gitignore @@ -5,6 +5,7 @@ valid_reset_stalled valid_stalled valid_add valid_newbest +valid_reset_add train.* valid.* dev.bpe.de diff --git a/tests/training/restoring/validation/test_adding_validator_after_restart.sh b/tests/training/restoring/validation/test_adding_validator_after_restart.sh index 6a6f2f3..ec51b02 100644 --- a/tests/training/restoring/validation/test_adding_validator_after_restart.sh +++ b/tests/training/restoring/validation/test_adding_validator_after_restart.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh b/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh new file mode 100644 index 0000000..95fe20b --- /dev/null +++ b/tests/training/restoring/validation/test_adding_validator_for_finetuning.sh @@ -0,0 +1,57 @@ +#!/bin/bash -x + +##################################################################### +# TAGS: valid valid-restart +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf valid_reset_add valid_reset_add_?.log +mkdir -p valid_reset_add + +extra_opts="--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd" +extra_opts="$extra_opts --dim-emb 128 --dim-rnn 256 --mini-batch 16" +extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false --clip-norm 0" + +#$MRT_MARIAN/marian $extra_opts \ + #-m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ + #--disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 \ + #--valid-metrics cross-entropy perplexity \ + #--valid-sets dev.bpe.{en,de} --valid-mini-batch 64 \ + #--valid-log valid_reset_add.expected.log + +#cat valid_reset_add.expected.log | $MRT_TOOLS/strip-timestamps.sh > valid_reset_add.expected +#exit 1 + + +$MRT_MARIAN/marian $extra_opts \ + -m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ + --disp-freq 10 --valid-freq 20 --after-batches 100 --early-stopping 5 \ + --valid-metrics cross-entropy \ + --valid-sets dev.bpe.{en,de} --valid-mini-batch 64 \ + --valid-log valid_reset_add_1.log + +test -e valid_reset_add/model.npz +test -e valid_reset_add/model.npz.yml +test -e valid_reset_add_1.log + +cp valid_reset_add/model.npz.progress.yml valid_reset_add/model.npz.progress.yml.bac +cat valid_reset_add_1.log | $MRT_TOOLS/strip-timestamps.sh > valid_reset_add.out + +$MRT_MARIAN/marian $extra_opts \ + -m valid_reset_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ + --disp-freq 10 --valid-freq 20 --after-batches 200 --early-stopping 5 \ + --valid-metrics cross-entropy ce-mean-words \ + --valid-sets dev.bpe.{en,de} --valid-mini-batch 64 --valid-reset-stalled --no-restore-corpus \ + --valid-log valid_reset_add_2.log + +test -e valid_reset_add/model.npz +test -e valid_reset_add_2.log + +cat valid_reset_add_2.log | $MRT_TOOLS/strip-timestamps.sh >> valid_reset_add.out +$MRT_TOOLS/diff-nums.py -p 0.003 valid_reset_add.out valid_reset_add.expected -o valid_reset_add.diff + +# Exit with success code +exit 0 diff --git a/tests/training/restoring/validation/test_restoring_newbest_validators.sh b/tests/training/restoring/validation/test_restoring_newbest_validators.sh index fa8b37a..bfbff83 100644 --- a/tests/training/restoring/validation/test_restoring_newbest_validators.sh +++ b/tests/training/restoring/validation/test_restoring_newbest_validators.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_stalled_validators.sh b/tests/training/restoring/validation/test_restoring_stalled_validators.sh index 50a6d2e..a67af03 100644 --- a/tests/training/restoring/validation/test_restoring_stalled_validators.sh +++ b/tests/training/restoring/validation/test_restoring_stalled_validators.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_validation.sh b/tests/training/restoring/validation/test_restoring_validation.sh index 483e408..3f41960 100644 --- a/tests/training/restoring/validation/test_restoring_validation.sh +++ b/tests/training/restoring/validation/test_restoring_validation.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh index f700e3c..8802acf 100644 --- a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh +++ b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_valid_reset_stalled.sh b/tests/training/restoring/validation/test_valid_reset_stalled.sh index c2c7d4b..37da6e6 100644 --- a/tests/training/restoring/validation/test_valid_reset_stalled.sh +++ b/tests/training/restoring/validation/test_valid_reset_stalled.sh @@ -3,7 +3,7 @@ ##################################################################### # SUMMARY: Test the option for resetting stalled validations after the training is resumed # AUTHOR: snukky -# TAGS: restore valid validscript validreset +# TAGS: restore valid valid-script valid-reset ##################################################################### # Exit on error diff --git a/tests/training/restoring/validation/valid_reset_add.expected b/tests/training/restoring/validation/valid_reset_add.expected new file mode 100644 index 0000000..b3780a9 --- /dev/null +++ b/tests/training/restoring/validation/valid_reset_add.expected @@ -0,0 +1,15 @@ +[valid] Ep. 1 : Up. 20 : cross-entropy : 294.63 : new best +[valid] Ep. 1 : Up. 40 : cross-entropy : 292.643 : new best +[valid] Ep. 1 : Up. 60 : cross-entropy : 290.224 : new best +[valid] Ep. 1 : Up. 80 : cross-entropy : 286.857 : new best +[valid] Ep. 1 : Up. 100 : cross-entropy : 282.156 : new best +[valid] Ep. 1 : Up. 120 : cross-entropy : 274.772 : new best +[valid] Ep. 1 : Up. 120 : ce-mean-words : 9.42616 : new best +[valid] Ep. 1 : Up. 140 : cross-entropy : 264.221 : new best +[valid] Ep. 1 : Up. 140 : ce-mean-words : 9.06417 : new best +[valid] Ep. 1 : Up. 160 : cross-entropy : 258.245 : new best +[valid] Ep. 1 : Up. 160 : ce-mean-words : 8.85916 : new best +[valid] Ep. 1 : Up. 180 : cross-entropy : 255.25 : new best +[valid] Ep. 1 : Up. 180 : ce-mean-words : 8.75644 : new best +[valid] Ep. 1 : Up. 200 : cross-entropy : 252.909 : new best +[valid] Ep. 1 : Up. 200 : ce-mean-words : 8.67611 : new best diff --git a/tests/training/scheduler/log_epoch_e.expected b/tests/training/scheduler/log_epoch_e.expected index 87cf167..c262c26 100644 --- a/tests/training/scheduler/log_epoch_e.expected +++ b/tests/training/scheduler/log_epoch_e.expected @@ -1,20 +1,22 @@ Training started -Seen 1542 samples +Parameter type float32, optimization type float32, casting types false +Allocating memory for Adam-specific shards +Seen 1,542 samples Starting data epoch 2 in logical epoch 1.000 -Ep. 1.000 : Up. 10 : Sen. 768 : Cost 9.68880177 * 61,315 after 61,315 -Seen 1542 samples +Ep. 1.000 : Up. 10 : Sen. 768 : Cost 9.68879700 * 61,315 @ 6,851 after 61,315 +Seen 1,542 samples Starting data epoch 3 in logical epoch 1.500 -Ep. 1.500 : Up. 20 : Sen. 1,536 : Cost 9.67091751 * 61,279 after 122,594 -Seen 1542 samples +Ep. 1.500 : Up. 20 : Sen. 1,536 : Cost 9.67091274 * 61,279 @ 6,585 after 122,594 +Seen 1,542 samples Starting data epoch 4 in logical epoch 2.000 -Seen 1542 samples +Seen 1,542 samples Starting data epoch 5 in logical epoch 2.500 -Ep. 2.500 : Up. 30 : Sen. 512 : Cost 9.65089989 * 54,621 after 177,215 -Seen 1542 samples +Ep. 2.500 : Up. 30 : Sen. 512 : Cost 9.65089798 * 54,621 @ 7,219 after 177,215 +Seen 1,542 samples Starting data epoch 6 in logical epoch 3.000 -Ep. 3.000 : Up. 40 : Sen. 1,280 : Cost 9.63199997 * 61,545 after 238,760 -Seen 1542 samples +Ep. 3.000 : Up. 40 : Sen. 1,280 : Cost 9.63199615 * 61,545 @ 6,916 after 238,760 +Seen 1,542 samples Starting data epoch 7 in logical epoch 3.500 Training finished Saving model to log_epoch_e/model.npz -Saving Adam parameters to log_epoch_e/model.npz.optimizer.npz +Saving Adam parameters diff --git a/tests/training/scheduler/log_epoch_t.expected b/tests/training/scheduler/log_epoch_t.expected index 1f57c2e..0373755 100644 --- a/tests/training/scheduler/log_epoch_t.expected +++ b/tests/training/scheduler/log_epoch_t.expected @@ -1,12 +1,14 @@ Training started -Ep. 2.258 : Up. 4 : Sen. 512 : Cost 9.69286919 * 13,547 after 13,547 -Ep. 3.400 : Up. 6 : Sen. 768 : Cost 9.68953419 * 6,851 after 20,398 -Ep. 5.131 : Up. 9 : Sen. 1,152 : Cost 9.68455887 * 10,387 after 30,785 -Ep. 6.793 : Up. 12 : Sen. 1,536 : Cost 9.68291855 * 9,975 after 40,760 -Seen 1542 samples +Parameter type float32, optimization type float32, casting types false +Allocating memory for Adam-specific shards +Ep. 2.258 : Up. 4 : Sen. 512 : Cost 9.69286919 * 13,547 @ 3,630 after 13,547 +Ep. 3.400 : Up. 6 : Sen. 768 : Cost 9.68952084 * 6,851 @ 3,634 after 20,398 +Ep. 5.131 : Up. 9 : Sen. 1,152 : Cost 9.68455029 * 10,387 @ 3,526 after 30,785 +Ep. 6.793 : Up. 12 : Sen. 1,536 : Cost 9.68291855 * 9,975 @ 3,457 after 40,760 +Seen 1,542 samples Starting data epoch 2 in logical epoch 6.819 -Ep. 8.472 : Up. 16 : Sen. 384 : Cost 9.67040443 * 10,074 after 50,834 -Ep. 10.219 : Up. 19 : Sen. 768 : Cost 9.66528606 * 10,481 after 61,315 +Ep. 8.472 : Up. 16 : Sen. 384 : Cost 9.67040443 * 10,074 @ 3,589 after 50,834 +Ep. 10.219 : Up. 19 : Sen. 768 : Cost 9.66527557 * 10,481 @ 3,634 after 61,315 Training finished Saving model to log_epoch_t/model.npz -Saving Adam parameters to log_epoch_t/model.npz.optimizer.npz +Saving Adam parameters diff --git a/tests/training/scheduler/log_epoch_u.expected b/tests/training/scheduler/log_epoch_u.expected index a8855f2..14a3064 100644 --- a/tests/training/scheduler/log_epoch_u.expected +++ b/tests/training/scheduler/log_epoch_u.expected @@ -1,15 +1,17 @@ Training started -Seen 1542 samples +Parameter type float32, optimization type float32, casting types false +Allocating memory for Adam-specific shards +Seen 1,542 samples Starting data epoch 2 in logical epoch 0.700 -Ep. 1.000 : Up. 10 : Sen. 768 : Cost 9.68880177 * 61,315 after 61,315 -Seen 1542 samples +Ep. 1.000 : Up. 10 : Sen. 768 : Cost 9.68879700 * 61,315 @ 6,851 after 61,315 +Seen 1,542 samples Starting data epoch 3 in logical epoch 1.400 -Ep. 2.000 : Up. 20 : Sen. 1,536 : Cost 9.67091751 * 61,279 after 122,594 -Seen 1542 samples +Ep. 2.000 : Up. 20 : Sen. 1,536 : Cost 9.67091274 * 61,279 @ 6,585 after 122,594 +Seen 1,542 samples Starting data epoch 4 in logical epoch 2.100 -Seen 1542 samples +Seen 1,542 samples Starting data epoch 5 in logical epoch 2.800 -Ep. 3.000 : Up. 30 : Sen. 512 : Cost 9.65089989 * 54,621 after 177,215 +Ep. 3.000 : Up. 30 : Sen. 512 : Cost 9.65089798 * 54,621 @ 7,219 after 177,215 Training finished Saving model to log_epoch_u/model.npz -Saving Adam parameters to log_epoch_u/model.npz.optimizer.npz +Saving Adam parameters diff --git a/tests/training/scheduler/test_logical_epoch.sh b/tests/training/scheduler/test_logical_epoch.sh index fca9f52..dcfa95d 100644 --- a/tests/training/scheduler/test_logical_epoch.sh +++ b/tests/training/scheduler/test_logical_epoch.sh @@ -25,7 +25,7 @@ test -e log_epoch_e/model.npz test -e log_epoch_e.log # Compare actual and expected outputs -cat log_epoch_e.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | sed 's/ : Time.*//' > log_epoch_e.out +cat log_epoch_e.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | grep -v 'Synced' | sed 's/ : Time.*//' > log_epoch_e.out $MRT_TOOLS/diff-nums.py log_epoch_e.out log_epoch_e.expected -p 0.01 -o log_epoch_e.diff # Exit with success code diff --git a/tests/training/scheduler/test_logical_epoch_labels.sh b/tests/training/scheduler/test_logical_epoch_labels.sh index c37c0fa..acbeda6 100644 --- a/tests/training/scheduler/test_logical_epoch_labels.sh +++ b/tests/training/scheduler/test_logical_epoch_labels.sh @@ -25,7 +25,7 @@ test -e log_epoch_t/model.npz test -e log_epoch_t.log # Compare actual and expected outputs -cat log_epoch_t.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | sed 's/ : Time.*//' > log_epoch_t.out +cat log_epoch_t.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | grep -v 'Synced' | sed 's/ : Time.*//' > log_epoch_t.out $MRT_TOOLS/diff-nums.py log_epoch_t.out log_epoch_t.expected -p 0.01 -o log_epoch_t.diff # Exit with success code diff --git a/tests/training/scheduler/test_logical_epoch_updates.sh b/tests/training/scheduler/test_logical_epoch_updates.sh index 8582120..326d4a6 100644 --- a/tests/training/scheduler/test_logical_epoch_updates.sh +++ b/tests/training/scheduler/test_logical_epoch_updates.sh @@ -25,7 +25,7 @@ test -e log_epoch_u/model.npz test -e log_epoch_u.log # Compare actual and expected outputs -cat log_epoch_u.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | sed 's/ : Time.*//' > log_epoch_u.out +cat log_epoch_u.log | $MRT_TOOLS/strip-timestamps.sh | grep -v '^\[' | grep -v 'Synced' | sed 's/ : Time.*//' > log_epoch_u.out $MRT_TOOLS/diff-nums.py log_epoch_u.out log_epoch_u.expected -p 0.01 -o log_epoch_u.diff # Exit with success code diff --git a/tests/training/validation/.gitignore b/tests/training/validation/.gitignore index 36a9cb8..d87a09c 100644 --- a/tests/training/validation/.gitignore +++ b/tests/training/validation/.gitignore @@ -13,3 +13,7 @@ trans_maxlen compare-trans compare-trans.bleu *.temp +stop_on_any +stop_on_all +stop_on_1st +devset.?? diff --git a/tests/training/validation/stop_on_1st.expected b/tests/training/validation/stop_on_1st.expected new file mode 100644 index 0000000..394c115 --- /dev/null +++ b/tests/training/validation/stop_on_1st.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_all.expected b/tests/training/validation/stop_on_all.expected new file mode 100644 index 0000000..9aae90f --- /dev/null +++ b/tests/training/validation/stop_on_all.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) diff --git a/tests/training/validation/stop_on_any.expected b/tests/training/validation/stop_on_any.expected new file mode 100644 index 0000000..f860950 --- /dev/null +++ b/tests/training/validation/stop_on_any.expected @@ -0,0 +1,14 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_script.sh b/tests/training/validation/stop_on_script.sh new file mode 100755 index 0000000..c140ff9 --- /dev/null +++ b/tests/training/validation/stop_on_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=stop_on_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num diff --git a/tests/training/validation/test_early_stopping_on_1st.sh b/tests/training/validation/test_early_stopping_on_1st.sh new file mode 100644 index 0000000..0ebe779 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_1st.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on the first validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_1st stop_on_1st.log stop_on_script.temp +mkdir -p stop_on_1st + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_1st/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_1st/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_1st.log \ + --early-stopping 4 --early-stopping-on first + +test -e stop_on_1st/model.npz +test -e stop_on_1st/model.npz.yml +test -e stop_on_1st.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_1st.log | grep '\[valid\]' > stop_on_1st.out +$MRT_TOOLS/diff-nums.py stop_on_1st.out stop_on_1st.expected -p 0.2 -o stop_on_1st.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_all.sh b/tests/training/validation/test_early_stopping_on_all.sh new file mode 100644 index 0000000..e6f95d9 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_all.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on each validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_all stop_on_all.log stop_on_script.temp +mkdir -p stop_on_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_all.log \ + --early-stopping 4 --early-stopping-on all + +test -e stop_on_all/model.npz +test -e stop_on_all/model.npz.yml +test -e stop_on_all.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_all.log | grep '\[valid\]' > stop_on_all.out +$MRT_TOOLS/diff-nums.py stop_on_all.out stop_on_all.expected -p 0.2 -o stop_on_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_any.sh b/tests/training/validation/test_early_stopping_on_any.sh new file mode 100644 index 0000000..d1f83db --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_any.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on any validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_any stop_on_any.log stop_on_script.temp +mkdir -p stop_on_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_any.log \ + --early-stopping 4 --early-stopping-on any + +test -e stop_on_any/model.npz +test -e stop_on_any/model.npz.yml +test -e stop_on_any.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_any.log | grep '\[valid\]' > stop_on_any.out +$MRT_TOOLS/diff-nums.py stop_on_any.out stop_on_any.expected -p 0.2 -o stop_on_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_empty_valid_set_lm.sh b/tests/training/validation/test_empty_valid_set_lm.sh index 3c6afcb..cd403aa 100644 --- a/tests/training/validation/test_empty_valid_set_lm.sh +++ b/tests/training/validation/test_empty_valid_set_lm.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid lm +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_source.sh b/tests/training/validation/test_empty_valid_set_source.sh index 8082926..8707d24 100644 --- a/tests/training/validation/test_empty_valid_set_source.sh +++ b/tests/training/validation/test_empty_valid_set_source.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_target.sh b/tests/training/validation/test_empty_valid_set_target.sh index 640eb51..9079ee9 100644 --- a/tests/training/validation/test_empty_valid_set_target.sh +++ b/tests/training/validation/test_empty_valid_set_target.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_sets.sh b/tests/training/validation/test_empty_valid_sets.sh index 9e1e79f..52c295b 100644 --- a/tests/training/validation/test_empty_valid_sets.sh +++ b/tests/training/validation/test_empty_valid_sets.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches.sh b/tests/training/validation/test_final_validation_after_batches.sh index 84a3dda..5ac8c6e 100644 --- a/tests/training/validation/test_final_validation_after_batches.sh +++ b/tests/training/validation/test_final_validation_after_batches.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches_match.sh b/tests/training/validation/test_final_validation_after_batches_match.sh index dc64991..c68f9e1 100644 --- a/tests/training/validation/test_final_validation_after_batches_match.sh +++ b/tests/training/validation/test_final_validation_after_batches_match.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_epochs.sh b/tests/training/validation/test_final_validation_after_epochs.sh index e263259..b76d139 100644 --- a/tests/training/validation/test_final_validation_after_epochs.sh +++ b/tests/training/validation/test_final_validation_after_epochs.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_templated_valid_bleu_output.sh b/tests/training/validation/test_templated_valid_bleu_output.sh index 60d95ce..d098044 100644 --- a/tests/training/validation/test_templated_valid_bleu_output.sh +++ b/tests/training/validation/test_templated_valid_bleu_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs with BLEU # AUTHOR: snukky +# TAGS: valid ##################################################################### diff --git a/tests/training/validation/test_templated_valid_translation_output.sh b/tests/training/validation/test_templated_valid_translation_output.sh index 2f4204c..c763f26 100644 --- a/tests/training/validation/test_templated_valid_translation_output.sh +++ b/tests/training/validation/test_templated_valid_translation_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs # AUTHOR: snukky +# TAGS: valid ##################################################################### |