diff options
author | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2021-04-16 16:25:40 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2021-04-16 16:30:30 +0300 |
commit | 1afd4eb1014ac451c6a3d6f9b5d34c322902e624 (patch) | |
tree | e0382043091cd0d7f48b5a0895dc032a3e0cd52b | |
parent | c1dc9dd2bb826b7ba3202d2ed4ac0f95388de859 (diff) |
Add tests for --lr-decay stalled and --early-stopping-on first|all|any
9 files changed, 217 insertions, 0 deletions
diff --git a/tests/training/features/lr-decay/.gitignore b/tests/training/features/lr-decay/.gitignore new file mode 100644 index 0000000..bff5533 --- /dev/null +++ b/tests/training/features/lr-decay/.gitignore @@ -0,0 +1,3 @@ +lrdecay_* +devset.* +*.temp diff --git a/tests/training/features/lr-decay/lrdecay_stalled.expected b/tests/training/features/lr-decay/lrdecay_stalled.expected new file mode 100644 index 0000000..0f0c7c2 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled.expected @@ -0,0 +1,25 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.68312 : stalled 6 times (last best: 2.56794) +Decaying learning rate to 4.78297e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_all.expected b/tests/training/features/lr-decay/lrdecay_stalled_all.expected new file mode 100644 index 0000000..0dda108 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_all.expected @@ -0,0 +1,23 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57813 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.5914 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +Decaying learning rate to 8.1e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.64345 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +Decaying learning rate to 7.29e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.67197 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.73563 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) diff --git a/tests/training/features/lr-decay/lrdecay_stalled_any.expected b/tests/training/features/lr-decay/lrdecay_stalled_any.expected new file mode 100644 index 0000000..9b21998 --- /dev/null +++ b/tests/training/features/lr-decay/lrdecay_stalled_any.expected @@ -0,0 +1,22 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +Decaying learning rate to 9e-05 after having stalled 1 time(s) +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56794 : new best +Decaying learning rate to 8.1e-05 after having stalled 2 time(s) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.5826 : stalled 1 times (last best: 2.56794) +Decaying learning rate to 7.29e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.57143 : stalled 2 times (last best: 2.56794) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.57244 : stalled 3 times (last best: 2.56794) +Decaying learning rate to 6.561e-05 after having stalled 3 time(s) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.60265 : stalled 4 times (last best: 2.56794) +Decaying learning rate to 5.9049e-05 after having stalled 4 time(s) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.63735 : stalled 5 times (last best: 2.56794) +Decaying learning rate to 5.31441e-05 after having stalled 5 time(s) diff --git a/tests/training/features/lr-decay/setup.sh b/tests/training/features/lr-decay/setup.sh new file mode 100644 index 0000000..1dc82af --- /dev/null +++ b/tests/training/features/lr-decay/setup.sh @@ -0,0 +1,6 @@ +test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 +test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 + +test -f $MRT_MODELS/wngt19/model.small.npz || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.en || exit 1 +test -f $MRT_MODELS/wngt19/newstest2014.de || exit 1 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled.sh b/tests/training/features/lr-decay/test_lr_decay_stalled.sh new file mode 100644 index 0000000..6f9abd7 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled lrdecay_stalled.log valid_script.temp +mkdir -p lrdecay_stalled + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 + +test -e lrdecay_stalled/model.npz +test -e lrdecay_stalled/model.npz.yml +test -e lrdecay_stalled.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled.out lrdecay_stalled.expected -p 0.2 -o lrdecay_stalled.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh new file mode 100644 index 0000000..280ba9f --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_all.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_all lrdecay_stalled_all.log valid_script.temp +mkdir -p lrdecay_stalled_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_all.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 4 --early-stopping-on all + +test -e lrdecay_stalled_all/model.npz +test -e lrdecay_stalled_all/model.npz.yml +test -e lrdecay_stalled_all.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_all.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_all.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_all.out lrdecay_stalled_all.expected -p 0.2 -o lrdecay_stalled_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh new file mode 100644 index 0000000..55c8f81 --- /dev/null +++ b/tests/training/features/lr-decay/test_lr_decay_stalled_any.sh @@ -0,0 +1,43 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test learning rate decaying after stalled validation +# AUTHOR: snukky +# TAGS: valid valid-script lr-decay +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf lrdecay_stalled_any lrdecay_stalled_any.log valid_script.temp +mkdir -p lrdecay_stalled_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz lrdecay_stalled_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m lrdecay_stalled_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./valid_script.sh \ + --valid-sets devset.{en,de} \ + --log lrdecay_stalled_any.log \ + --lr-decay 0.9 --lr-decay-start 1 --lr-decay-strategy stalled \ + --early-stopping 5 --early-stopping-on any + +test -e lrdecay_stalled_any/model.npz +test -e lrdecay_stalled_any/model.npz.yml +test -e lrdecay_stalled_any.log + +$MRT_TOOLS/strip-timestamps.sh < lrdecay_stalled_any.log | grep -P '(\[valid\]|Decaying)' > lrdecay_stalled_any.out +$MRT_TOOLS/diff-nums.py lrdecay_stalled_any.out lrdecay_stalled_any.expected -p 0.2 -o lrdecay_stalled_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/features/lr-decay/valid_script.sh b/tests/training/features/lr-decay/valid_script.sh new file mode 100755 index 0000000..faf1369 --- /dev/null +++ b/tests/training/features/lr-decay/valid_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=valid_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num |