Remove --optimize from the WNGT18 test

4 files changed, 60 insertions, 11 deletions

diff --git a/tests/models/wnmt18/.gitignore b/tests/models/wnmt18/.gitignore
index 065225a..0bbd721 100644
--- a/tests/models/wnmt18/.gitignore
+++ b/tests/models/wnmt18/.gitignore
@@ -1,5 +1,6 @@
 newstest2014.in
 newstest2014.ref
 
-optimize.bleu
-optimize_aan.bleu
+*.bleu
+*.bin
+*.expected
diff --git a/tests/models/wnmt18/optimize_aan.bleu.expected b/tests/models/wnmt18/optimize_aan.bleu.expected
index ba7ca5e..8c9c6ee 100644
--- a/tests/models/wnmt18/optimize_aan.bleu.expected
+++ b/tests/models/wnmt18/optimize_aan.bleu.expected
@@ -1 +1 @@
-25.55
+25.78
diff --git a/tests/models/wnmt18/test_student_small_aan_intgemm16.sh b/tests/models/wnmt18/test_student_small_aan_intgemm16.sh
new file mode 100644
index 0000000..e08e5de
--- /dev/null
+++ b/tests/models/wnmt18/test_student_small_aan_intgemm16.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#####################################################################
+# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN and intgemm 16bit
+# TAGS: cpu wngt student shortlist intgemm
+#####################################################################
+
+# Exit on error
+set -e
+
+# Skip if no MKL found
+if [ ! $MRT_MARIAN_USE_MKL ]; then
+    exit 100
+fi
+
+model=model.student.small.aan
+
+# Remove previous outputs
+rm -f optimize_aan_16.out $model.intgemm16.bin
+
+# Pack the model
+$MRT_MARIAN/marian-conv -f $MRT_MODELS/wnmt18/$model/model.npz -t $model.intgemm16.bin --gemm-type intgemm16
+test -s $model.intgemm16.bin
+
+# Run test
+cat newstest2014.in | $MRT_MARIAN/marian-decoder \
+    -m $model.intgemm16.bin \
+    -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \
+    --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 \
+    --shortlist $MRT_MODELS/wnmt18/lex.s2t 100 75 --skip-cost --cpu-threads=1 --max-length-factor 1.2 \
+    > optimize_aan_16.out
+
+cat optimize_aan_16.out | perl -pe 's/@@ //g' \
+    | $MRT_TOOLS/moses-scripts/scripts/recaser/detruecase.perl \
+    | $MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl newstest2014.ref \
+    | $MRT_TOOLS/extract-bleu.sh > optimize_aan_16.bleu
+
+$MRT_TOOLS/diff-nums.py optimize_aan_16.bleu optimize_aan.bleu.expected -p 0.6 -o optimize_aan_16.bleu.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/models/wnmt18/test_student_small_aan_optimize.sh b/tests/models/wnmt18/test_student_small_aan_intgemm8.sh
index 5010c8a..44bd527 100644
--- a/tests/models/wnmt18/test_student_small_aan_optimize.sh
+++ b/tests/models/wnmt18/test_student_small_aan_intgemm8.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
 
 #####################################################################
-# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN
-# TAGS: cpu wngt student shortlist obsolete
+# SUMMARY: CPU-based optimized decoding with the WNGT18 small student model with AAN and intgemm 8bit
+# TAGS: cpu wngt student shortlist intgemm
 #####################################################################
 
 # Exit on error
@@ -15,20 +15,27 @@ fi
 
 model=model.student.small.aan
 
+# Remove previous outputs
+rm -f optimize_aan_8.out $model.intgemm8.bin
+
+# Pack the model
+$MRT_MARIAN/marian-conv -f $MRT_MODELS/wnmt18/$model/model.npz -t $model.intgemm8.bin --gemm-type intgemm8
+test -s $model.intgemm8.bin
+
 # Run test
 cat newstest2014.in | $MRT_MARIAN/marian-decoder \
-    -m $MRT_MODELS/wnmt18/$model/model.npz \
+    -m $model.intgemm8.bin \
     -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \
-    --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 --optimize \
+    --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 \
     --shortlist $MRT_MODELS/wnmt18/lex.s2t 100 75 --skip-cost --cpu-threads=1 --max-length-factor 1.2 \
-    > optimize_aan.out
+    > optimize_aan_8.out
 
-cat optimize_aan.out | perl -pe 's/@@ //g' \
+cat optimize_aan_8.out | perl -pe 's/@@ //g' \
     | $MRT_TOOLS/moses-scripts/scripts/recaser/detruecase.perl \
     | $MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl newstest2014.ref \
-    | $MRT_TOOLS/extract-bleu.sh > optimize_aan.bleu
+    | $MRT_TOOLS/extract-bleu.sh > optimize_aan_8.bleu
 
-$MRT_TOOLS/diff-nums.py optimize_aan.bleu optimize_aan.bleu.expected -p 0.4 -o optimize_aan.bleu.diff
+$MRT_TOOLS/diff-nums.py optimize_aan_8.bleu optimize_aan.bleu.expected -p 0.6 -o optimize_aan_8.bleu.diff
 
 # Exit with success code
 exit 0