diff options
author | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2020-11-16 12:44:19 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2020-11-16 12:44:19 +0300 |
commit | 189969816c8a3f7d23e9b418c35443e16859dc8e (patch) | |
tree | 03f46aeb4e91b0995fa69a10baa0ef95e7925fe2 | |
parent | 2d8d19731f1cd63fe99f2ed18417456940b82110 (diff) |
Remove --optimize from the WNGT18 test
-rw-r--r-- | tests/models/wnmt18/.gitignore | 5 | ||||
-rw-r--r-- | tests/models/wnmt18/optimize_aan.bleu.expected | 2 | ||||
-rw-r--r-- | tests/models/wnmt18/test_student_small_aan_intgemm16.sh | 41 | ||||
-rw-r--r-- | tests/models/wnmt18/test_student_small_aan_intgemm8.sh (renamed from tests/models/wnmt18/test_student_small_aan_optimize.sh) | 23 |
4 files changed, 60 insertions, 11 deletions
diff --git a/tests/models/wnmt18/.gitignore b/tests/models/wnmt18/.gitignore index 065225a..0bbd721 100644 --- a/tests/models/wnmt18/.gitignore +++ b/tests/models/wnmt18/.gitignore @@ -1,5 +1,6 @@ newstest2014.in newstest2014.ref -optimize.bleu -optimize_aan.bleu +*.bleu +*.bin +*.expected diff --git a/tests/models/wnmt18/optimize_aan.bleu.expected b/tests/models/wnmt18/optimize_aan.bleu.expected index ba7ca5e..8c9c6ee 100644 --- a/tests/models/wnmt18/optimize_aan.bleu.expected +++ b/tests/models/wnmt18/optimize_aan.bleu.expected @@ -1 +1 @@ -25.55 +25.78 diff --git a/tests/models/wnmt18/test_student_small_aan_intgemm16.sh b/tests/models/wnmt18/test_student_small_aan_intgemm16.sh new file mode 100644 index 0000000..e08e5de --- /dev/null +++ b/tests/models/wnmt18/test_student_small_aan_intgemm16.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +##################################################################### +# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN and intgemm 16bit +# TAGS: cpu wngt student shortlist intgemm +##################################################################### + +# Exit on error +set -e + +# Skip if no MKL found +if [ ! $MRT_MARIAN_USE_MKL ]; then + exit 100 +fi + +model=model.student.small.aan + +# Remove previous outputs +rm -f optimize_aan_16.out $model.intgemm16.bin + +# Pack the model +$MRT_MARIAN/marian-conv -f $MRT_MODELS/wnmt18/$model/model.npz -t $model.intgemm16.bin --gemm-type intgemm16 +test -s $model.intgemm16.bin + +# Run test +cat newstest2014.in | $MRT_MARIAN/marian-decoder \ + -m $model.intgemm16.bin \ + -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \ + --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 \ + --shortlist $MRT_MODELS/wnmt18/lex.s2t 100 75 --skip-cost --cpu-threads=1 --max-length-factor 1.2 \ + > optimize_aan_16.out + +cat optimize_aan_16.out | perl -pe 's/@@ //g' \ + | $MRT_TOOLS/moses-scripts/scripts/recaser/detruecase.perl \ + | $MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl newstest2014.ref \ + | $MRT_TOOLS/extract-bleu.sh > optimize_aan_16.bleu + +$MRT_TOOLS/diff-nums.py optimize_aan_16.bleu optimize_aan.bleu.expected -p 0.6 -o optimize_aan_16.bleu.diff + +# Exit with success code +exit 0 diff --git a/tests/models/wnmt18/test_student_small_aan_optimize.sh b/tests/models/wnmt18/test_student_small_aan_intgemm8.sh index 5010c8a..44bd527 100644 --- a/tests/models/wnmt18/test_student_small_aan_optimize.sh +++ b/tests/models/wnmt18/test_student_small_aan_intgemm8.sh @@ -1,8 +1,8 @@ #!/bin/bash ##################################################################### -# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN -# TAGS: cpu wngt student shortlist obsolete +# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN and intgemm 8bit +# TAGS: cpu wngt student shortlist intgemm ##################################################################### # Exit on error @@ -15,20 +15,27 @@ fi model=model.student.small.aan +# Remove previous outputs +rm -f optimize_aan_8.out $model.intgemm8.bin + +# Pack the model +$MRT_MARIAN/marian-conv -f $MRT_MODELS/wnmt18/$model/model.npz -t $model.intgemm8.bin --gemm-type intgemm8 +test -s $model.intgemm8.bin + # Run test cat newstest2014.in | $MRT_MARIAN/marian-decoder \ - -m $MRT_MODELS/wnmt18/$model/model.npz \ + -m $model.intgemm8.bin \ -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \ - --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 --optimize \ + --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 \ --shortlist $MRT_MODELS/wnmt18/lex.s2t 100 75 --skip-cost --cpu-threads=1 --max-length-factor 1.2 \ - > optimize_aan.out + > optimize_aan_8.out -cat optimize_aan.out | perl -pe 's/@@ //g' \ +cat optimize_aan_8.out | perl -pe 's/@@ //g' \ | $MRT_TOOLS/moses-scripts/scripts/recaser/detruecase.perl \ | $MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl newstest2014.ref \ - | $MRT_TOOLS/extract-bleu.sh > optimize_aan.bleu + | $MRT_TOOLS/extract-bleu.sh > optimize_aan_8.bleu -$MRT_TOOLS/diff-nums.py optimize_aan.bleu optimize_aan.bleu.expected -p 0.4 -o optimize_aan.bleu.diff +$MRT_TOOLS/diff-nums.py optimize_aan_8.bleu optimize_aan.bleu.expected -p 0.6 -o optimize_aan_8.bleu.diff # Exit with success code exit 0 |