diff options
Diffstat (limited to 'tests/training/validation')
17 files changed, 219 insertions, 0 deletions
diff --git a/tests/training/validation/.gitignore b/tests/training/validation/.gitignore index 36a9cb8..d87a09c 100644 --- a/tests/training/validation/.gitignore +++ b/tests/training/validation/.gitignore @@ -13,3 +13,7 @@ trans_maxlen compare-trans compare-trans.bleu *.temp +stop_on_any +stop_on_all +stop_on_1st +devset.?? diff --git a/tests/training/validation/stop_on_1st.expected b/tests/training/validation/stop_on_1st.expected new file mode 100644 index 0000000..394c115 --- /dev/null +++ b/tests/training/validation/stop_on_1st.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_all.expected b/tests/training/validation/stop_on_all.expected new file mode 100644 index 0000000..9aae90f --- /dev/null +++ b/tests/training/validation/stop_on_all.expected @@ -0,0 +1,18 @@ +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 80 : ce-mean-words : 2.71896 : stalled 5 times (last best: 2.56738) +[valid] Ep. 1 : Up. 80 : valid-script : 111.3 : stalled 3 times (last best: 111.6) +[valid] Ep. 1 : Up. 90 : ce-mean-words : 2.8146 : stalled 6 times (last best: 2.56738) +[valid] Ep. 1 : Up. 90 : valid-script : 111.2 : stalled 4 times (last best: 111.6) diff --git a/tests/training/validation/stop_on_any.expected b/tests/training/validation/stop_on_any.expected new file mode 100644 index 0000000..f860950 --- /dev/null +++ b/tests/training/validation/stop_on_any.expected @@ -0,0 +1,14 @@ +[valid] Ep. 1 : Up. 10 : valid-script : 111.4 : new best +[valid] Ep. 1 : Up. 10 : ce-mean-words : 2.61662 : new best +[valid] Ep. 1 : Up. 20 : valid-script : 111.3 : stalled 1 times (last best: 111.4) +[valid] Ep. 1 : Up. 20 : ce-mean-words : 2.58011 : new best +[valid] Ep. 1 : Up. 30 : valid-script : 111.2 : stalled 2 times (last best: 111.4) +[valid] Ep. 1 : Up. 30 : ce-mean-words : 2.56738 : new best +[valid] Ep. 1 : Up. 40 : valid-script : 111.1 : stalled 3 times (last best: 111.4) +[valid] Ep. 1 : Up. 40 : ce-mean-words : 2.59199 : stalled 1 times (last best: 2.56738) +[valid] Ep. 1 : Up. 50 : valid-script : 111.6 : new best +[valid] Ep. 1 : Up. 50 : ce-mean-words : 2.58017 : stalled 2 times (last best: 2.56738) +[valid] Ep. 1 : Up. 60 : valid-script : 111.5 : stalled 1 times (last best: 111.6) +[valid] Ep. 1 : Up. 60 : ce-mean-words : 2.60083 : stalled 3 times (last best: 2.56738) +[valid] Ep. 1 : Up. 70 : valid-script : 111.4 : stalled 2 times (last best: 111.6) +[valid] Ep. 1 : Up. 70 : ce-mean-words : 2.67288 : stalled 4 times (last best: 2.56738) diff --git a/tests/training/validation/stop_on_script.sh b/tests/training/validation/stop_on_script.sh new file mode 100755 index 0000000..c140ff9 --- /dev/null +++ b/tests/training/validation/stop_on_script.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +prefix=stop_on_script + +num=$(cat $prefix.temp 2>/dev/null || echo 1) +((num=(num+1)%6)) +echo $num > $prefix.temp +((num=6-num)) +echo 111.$num diff --git a/tests/training/validation/test_early_stopping_on_1st.sh b/tests/training/validation/test_early_stopping_on_1st.sh new file mode 100644 index 0000000..0ebe779 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_1st.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on the first validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_1st stop_on_1st.log stop_on_script.temp +mkdir -p stop_on_1st + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_1st/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_1st/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_1st.log \ + --early-stopping 4 --early-stopping-on first + +test -e stop_on_1st/model.npz +test -e stop_on_1st/model.npz.yml +test -e stop_on_1st.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_1st.log | grep '\[valid\]' > stop_on_1st.out +$MRT_TOOLS/diff-nums.py stop_on_1st.out stop_on_1st.expected -p 0.2 -o stop_on_1st.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_all.sh b/tests/training/validation/test_early_stopping_on_all.sh new file mode 100644 index 0000000..e6f95d9 --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_all.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on each validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_all stop_on_all.log stop_on_script.temp +mkdir -p stop_on_all + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_all/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_all/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics ce-mean-words valid-script \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_all.log \ + --early-stopping 4 --early-stopping-on all + +test -e stop_on_all/model.npz +test -e stop_on_all/model.npz.yml +test -e stop_on_all.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_all.log | grep '\[valid\]' > stop_on_all.out +$MRT_TOOLS/diff-nums.py stop_on_all.out stop_on_all.expected -p 0.2 -o stop_on_all.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_early_stopping_on_any.sh b/tests/training/validation/test_early_stopping_on_any.sh new file mode 100644 index 0000000..d1f83db --- /dev/null +++ b/tests/training/validation/test_early_stopping_on_any.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x + +##################################################################### +# SUMMARY: Test early stopping after stalling on any validation metric +# AUTHOR: snukky +# TAGS: valid valid-script +##################################################################### + +# Exit on error +set -e + +# Test code goes here +rm -rf stop_on_any stop_on_any.log stop_on_script.temp +mkdir -p stop_on_any + +# Start training with a fully trained model so that validation metrics do not improve easily +cp $MRT_MODELS/wngt19/model.small.npz stop_on_any/model.npz + +test -s devset.en || head -n 50 $MRT_MODELS/wngt19/newstest2014.en | sed -r 's/@@ //g' > devset.en +test -s devset.de || head -n 50 $MRT_MODELS/wngt19/newstest2014.de | sed -r 's/@@ //g' > devset.de + +# Training sides are intentionaly reversed to test early stopping +$MRT_MARIAN/marian \ + --seed 2222 --no-shuffle --clip-norm 1 --maxi-batch 1 --mini-batch 32 -w 2500 \ + -m stop_on_any/model.npz -t $MRT_DATA/europarl.de-en/corpus.small.{de,en}.gz \ + -v $MRT_MODELS/wngt19/en-de.{spm,spm} \ + --disp-freq 5 --valid-freq 10 --after-batches 200 \ + --valid-metrics valid-script ce-mean-words \ + --valid-script-path ./stop_on_script.sh \ + --valid-sets devset.{en,de} \ + --valid-log stop_on_any.log \ + --early-stopping 4 --early-stopping-on any + +test -e stop_on_any/model.npz +test -e stop_on_any/model.npz.yml +test -e stop_on_any.log + +$MRT_TOOLS/strip-timestamps.sh < stop_on_any.log | grep '\[valid\]' > stop_on_any.out +$MRT_TOOLS/diff-nums.py stop_on_any.out stop_on_any.expected -p 0.2 -o stop_on_any.diff + +# Exit with success code +exit 0 diff --git a/tests/training/validation/test_empty_valid_set_lm.sh b/tests/training/validation/test_empty_valid_set_lm.sh index 3c6afcb..cd403aa 100644 --- a/tests/training/validation/test_empty_valid_set_lm.sh +++ b/tests/training/validation/test_empty_valid_set_lm.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid lm +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_source.sh b/tests/training/validation/test_empty_valid_set_source.sh index 8082926..8707d24 100644 --- a/tests/training/validation/test_empty_valid_set_source.sh +++ b/tests/training/validation/test_empty_valid_set_source.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_set_target.sh b/tests/training/validation/test_empty_valid_set_target.sh index 640eb51..9079ee9 100644 --- a/tests/training/validation/test_empty_valid_set_target.sh +++ b/tests/training/validation/test_empty_valid_set_target.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_empty_valid_sets.sh b/tests/training/validation/test_empty_valid_sets.sh index 9e1e79f..52c295b 100644 --- a/tests/training/validation/test_empty_valid_sets.sh +++ b/tests/training/validation/test_empty_valid_sets.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches.sh b/tests/training/validation/test_final_validation_after_batches.sh index 84a3dda..5ac8c6e 100644 --- a/tests/training/validation/test_final_validation_after_batches.sh +++ b/tests/training/validation/test_final_validation_after_batches.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_batches_match.sh b/tests/training/validation/test_final_validation_after_batches_match.sh index dc64991..c68f9e1 100644 --- a/tests/training/validation/test_final_validation_after_batches_match.sh +++ b/tests/training/validation/test_final_validation_after_batches_match.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_final_validation_after_epochs.sh b/tests/training/validation/test_final_validation_after_epochs.sh index e263259..b76d139 100644 --- a/tests/training/validation/test_final_validation_after_epochs.sh +++ b/tests/training/validation/test_final_validation_after_epochs.sh @@ -1,5 +1,9 @@ #!/bin/bash -x +##################################################################### +# TAGS: valid +##################################################################### + # Exit on error set -e diff --git a/tests/training/validation/test_templated_valid_bleu_output.sh b/tests/training/validation/test_templated_valid_bleu_output.sh index 60d95ce..d098044 100644 --- a/tests/training/validation/test_templated_valid_bleu_output.sh +++ b/tests/training/validation/test_templated_valid_bleu_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs with BLEU # AUTHOR: snukky +# TAGS: valid ##################################################################### diff --git a/tests/training/validation/test_templated_valid_translation_output.sh b/tests/training/validation/test_templated_valid_translation_output.sh index 2f4204c..c763f26 100644 --- a/tests/training/validation/test_templated_valid_translation_output.sh +++ b/tests/training/validation/test_templated_valid_translation_output.sh @@ -3,6 +3,7 @@ ##################################################################### # SUMMARY: Templated file names for translation outputs # AUTHOR: snukky +# TAGS: valid ##################################################################### |