diff options
author | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2021-07-26 12:22:06 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2021-07-26 12:22:06 +0300 |
commit | 4ec211ce0d3bf6c94b1d814958cef0db6f6cc0aa (patch) | |
tree | 4175164cad48fc267803b65b466bad3f462998f0 /tests | |
parent | 2a6722b5059e5f1acce7f78b7a6c943b34e16178 (diff) | |
parent | 1afd4eb1014ac451c6a3d6f9b5d34c322902e624 (diff) |
Merge branch 'romang/early-stopping-on' into romang/merge-prs
Diffstat (limited to 'tests')
32 files changed, 457 insertions, 1 deletions
diff --git a/tests/training/features/lr-decay/.gitignore b/tests/training/features/lr-decay/.gitignore new file mode 100644 index 0000000..bff5533 --- /dev/null +++ b/tests/training/features/lr-decay/.gitignore @@ -0,0 +1,3 @@ +lrdecay_* +devset.* +*.temp diff --git a/tests/training/features/lr-decay/lrdecay_stalled.expected b/tests/training/features/lr-decay/lrdecay_stalled.expected new file mode 100644 index 0000000..0f0c7c2 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled.expected @@ -0,0 +1,25 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.68312 : stalled 6 times (last best: 2.56794) +Decaying learning rate to 4.78297e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_all.expected b/tests/training/features/lr-decay/lrdecay_stalled_all.expected new file mode 100644 index 0000000..0dda108 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_all.expected @@ -0,0 +1,23 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57813 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.5914 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +Decaying learning rate to 8.1e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.64345 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +Decaying learning rate to 7.29e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.67197 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.73563 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_any.expected b/tests/training/features/lr-decay/lrdecay_stalled_any.expected new file mode 100644 index 0000000..9b21998 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_any.expected @@ -0,0 +1,22 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 5 time(s) diff --git a/tests/training/features/lr-decay/setup.sh b/tests/training/features/lr-decay/setup.sh new file mode 100644 index 0000000..1dc82af --- /dev/null +++ b/tests/training/features/lr-decay/setup.sh @@ -0,0 +1,6 @@ +test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 +test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 + +test -f $MRT_MODELS/wngt19/model.small.npz || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.en || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.de || exit 1 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled.sh b/tests/training/features/lr-decay/test_lr_decay_stalled.sh new file mode 100644 index 0000000..6f9abd7 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled lrdecay_stalled.log valid_script.temp +mkdir -p lrdecay_stalled + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 + +test -e lrdecay_stalled/model.npz +test -e lrdecay_stalled/model.npz.yml +test -e lrdecay_stalled.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled.out lrdecay_stalled.expected -p 0.2 -o lrdecay_stalled.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh new file mode 100644 index 0000000..280ba9f --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_all lrdecay_stalled_all.log valid_script.temp +mkdir -p lrdecay_stalled_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_all.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 --early-stopping-on all + +test -e lrdecay_stalled_all/model.npz +test -e lrdecay_stalled_all/model.npz.yml +test -e lrdecay_stalled_all.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_all.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_all.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_all.out lrdecay_stalled_all.expected -p 0.2 -o lrdecay_stalled_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh new file mode 100644 index 0000000..55c8f81 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_any lrdecay_stalled_any.log valid_script.temp +mkdir -p lrdecay_stalled_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_any.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 5 --early-stopping-on any + +test -e lrdecay_stalled_any/model.npz +test -e lrdecay_stalled_any/model.npz.yml +test -e lrdecay_stalled_any.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_any.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_any.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_any.out lrdecay_stalled_any.expected -p 0.2 -o lrdecay_stalled_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/valid_script.sh b/tests/training/features/lr-decay/valid_script.sh new file mode 100755 index 0000000..faf1369 --- /dev/null +++ b/tests/training/features/lr-decay/valid_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=valid_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num diff --git a/tests/training/restoring/validation/test_adding_validator_after_restart.sh b/tests/training/restoring/validation/test_adding_validator_after_restart.sh index 6a6f2f3..ec51b02 100644 --- a/tests/training/restoring/validation/test_adding_validator_after_restart.sh +++ b/tests/training/restoring/validation/test_adding_validator_after_restart.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_newbest_validators.sh b/tests/training/restoring/validation/test_restoring_newbest_validators.sh index fa8b37a..bfbff83 100644 --- a/tests/training/restoring/validation/test_restoring_newbest_validators.sh +++ b/tests/training/restoring/validation/test_restoring_newbest_validators.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_stalled_validators.sh b/tests/training/restoring/validation/test_restoring_stalled_validators.sh index 50a6d2e..a67af03 100644 --- a/tests/training/restoring/validation/test_restoring_stalled_validators.sh +++ b/tests/training/restoring/validation/test_restoring_stalled_validators.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_validation.sh b/tests/training/restoring/validation/test_restoring_validation.sh index 483e408..3f41960 100644 --- a/tests/training/restoring/validation/test_restoring_validation.sh +++ b/tests/training/restoring/validation/test_restoring_validation.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh index f700e3c..8802acf 100644 --- a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh +++ b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid valid-restart +##################################################################### + # Exit on error set -e diff --git a/tests/training/restoring/validation/test_valid_reset_stalled.sh b/tests/training/restoring/validation/test_valid_reset_stalled.sh index c2c7d4b..37da6e6 100644 --- a/tests/training/restoring/validation/test_valid_reset_stalled.sh +++ b/tests/training/restoring/validation/test_valid_reset_stalled.sh @@ -3,7 +3,7 @@ ##################################################################### # SUMMARY: Test the option for resetting stalled validations after the training is resumed # AUTHOR: snukky -# TAGS: restore valid validscript validreset +# TAGS: restore valid valid-script valid-reset ##################################################################### # Exit on error diff --git a/tests/training/validation/.gitignore b/tests/training/validation/.gitignore index 36a9cb8..d87a09c 100644 --- a/tests/training/validation/.gitignore +++ b/tests/training/validation/.gitignore @@ -13,3 +13,7 @@ trans_maxlen compare-trans compare-trans.bleu *.temp +stop_on_any +stop_on_all +stop_on_1st +devset.?? diff --git a/tests/training/validation/stop_on_1st.expected b/tests/training/validation/stop_on_1st.expected new file mode 100644 index 0000000..394c115 --- /dev/null +++ b/tests/training/validation/stop_on_1st.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_all.expected b/tests/training/validation/stop_on_all.expected new file mode 100644 index 0000000..9aae90f --- /dev/null +++ b/tests/training/validation/stop_on_all.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) diff --git a/tests/training/validation/stop_on_any.expected b/tests/training/validation/stop_on_any.expected new file mode 100644 index 0000000..f860950 --- /dev/null +++ b/tests/training/validation/stop_on_any.expected @@ -0,0 +1,14 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_script.sh b/tests/training/validation/stop_on_script.sh new file mode 100755 index 0000000..c140ff9 --- /dev/null +++ b/tests/training/validation/stop_on_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=stop_on_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num diff --git a/tests/training/validation/test_early_stopping_on_1st.sh b/tests/training/validation/test_early_stopping_on_1st.sh new file mode 100644 index 0000000..0ebe779 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_1st.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on the first validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_1st stop_on_1st.log stop_on_script.temp +mkdir -p stop_on_1st + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_1st/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_1st/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_1st.log \ + --early-stopping 4 --early-stopping-on first + +test -e stop_on_1st/model.npz +test -e stop_on_1st/model.npz.yml +test -e stop_on_1st.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_1st.log | grep '\[valid\]' > stop_on_1st.out +$MRT_TOOLS/diff-nums.py stop_on_1st.out stop_on_1st.expected -p 0.2 -o stop_on_1st.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_all.sh b/tests/training/validation/test_early_stopping_on_all.sh new file mode 100644 index 0000000..e6f95d9 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_all.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on each validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_all stop_on_all.log stop_on_script.temp +mkdir -p stop_on_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_all.log \ + --early-stopping 4 --early-stopping-on all + +test -e stop_on_all/model.npz +test -e stop_on_all/model.npz.yml +test -e stop_on_all.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_all.log | grep '\[valid\]' > stop_on_all.out +$MRT_TOOLS/diff-nums.py stop_on_all.out stop_on_all.expected -p 0.2 -o stop_on_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_any.sh b/tests/training/validation/test_early_stopping_on_any.sh new file mode 100644 index 0000000..d1f83db --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_any.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on any validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_any stop_on_any.log stop_on_script.temp +mkdir -p stop_on_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_any.log \ + --early-stopping 4 --early-stopping-on any + +test -e stop_on_any/model.npz +test -e stop_on_any/model.npz.yml +test -e stop_on_any.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_any.log | grep '\[valid\]' > stop_on_any.out +$MRT_TOOLS/diff-nums.py stop_on_any.out stop_on_any.expected -p 0.2 -o stop_on_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_empty_valid_set_lm.sh b/tests/training/validation/test_empty_valid_set_lm.sh index 3c6afcb..cd403aa 100644 --- a/tests/training/validation/test_empty_valid_set_lm.sh +++ b/tests/training/validation/test_empty_valid_set_lm.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid lm +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_source.sh b/tests/training/validation/test_empty_valid_set_source.sh index 8082926..8707d24 100644 --- a/tests/training/validation/test_empty_valid_set_source.sh +++ b/tests/training/validation/test_empty_valid_set_source.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_target.sh b/tests/training/validation/test_empty_valid_set_target.sh index 640eb51..9079ee9 100644 --- a/tests/training/validation/test_empty_valid_set_target.sh +++ b/tests/training/validation/test_empty_valid_set_target.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_sets.sh b/tests/training/validation/test_empty_valid_sets.sh index 9e1e79f..52c295b 100644 --- a/tests/training/validation/test_empty_valid_sets.sh +++ b/tests/training/validation/test_empty_valid_sets.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches.sh b/tests/training/validation/test_final_validation_after_batches.sh index 84a3dda..5ac8c6e 100644 --- a/tests/training/validation/test_final_validation_after_batches.sh +++ b/tests/training/validation/test_final_validation_after_batches.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches_match.sh b/tests/training/validation/test_final_validation_after_batches_match.sh index dc64991..c68f9e1 100644 --- a/tests/training/validation/test_final_validation_after_batches_match.sh +++ b/tests/training/validation/test_final_validation_after_batches_match.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_epochs.sh b/tests/training/validation/test_final_validation_after_epochs.sh index e263259..b76d139 100644 --- a/tests/training/validation/test_final_validation_after_epochs.sh +++ b/tests/training/validation/test_final_validation_after_epochs.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_templated_valid_bleu_output.sh b/tests/training/validation/test_templated_valid_bleu_output.sh index 60d95ce..d098044 100644 --- a/tests/training/validation/test_templated_valid_bleu_output.sh +++ b/tests/training/validation/test_templated_valid_bleu_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs with BLEU # AUTHOR: snukky +# TAGS: valid ##################################################################### diff --git a/tests/training/validation/test_templated_valid_translation_output.sh b/tests/training/validation/test_templated_valid_translation_output.sh index 2f4204c..c763f26 100644 --- a/tests/training/validation/test_templated_valid_translation_output.sh +++ b/tests/training/validation/test_templated_valid_translation_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs # AUTHOR: snukky +# TAGS: valid ##################################################################### |