From 7c3a2be843fcff48bfb21c6ddbf4e2231093de8c Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Tue, 10 Nov 2020 05:22:37 +0000
Subject: add regression tests for model quantization

---
 .../quantized-model/quantized-no-opt.expected      | 10 +++++
 .../quantized-model/quantized-with-bias.expected   | 10 +++++
 .../features/quantized-model/quantized.expected    | 10 +++++
 .../features/quantized-model/test_quantmodel.sh    | 37 ++++++++++++++++++
 .../quantized-model/test_quantmodel_with_bias.sh   | 44 ++++++++++++++++++++++
 .../test_quantmodel_with_optimization.sh           | 37 ++++++++++++++++++
 6 files changed, 148 insertions(+)
 create mode 100644 tests/training/features/quantized-model/quantized-no-opt.expected
 create mode 100644 tests/training/features/quantized-model/quantized-with-bias.expected
 create mode 100644 tests/training/features/quantized-model/quantized.expected
 create mode 100644 tests/training/features/quantized-model/test_quantmodel.sh
 create mode 100644 tests/training/features/quantized-model/test_quantmodel_with_bias.sh
 create mode 100644 tests/training/features/quantized-model/test_quantmodel_with_optimization.sh

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/quantized-no-opt.expected b/tests/training/features/quantized-model/quantized-no-opt.expected
new file mode 100644
index 0000000..c4ac602
--- /dev/null
+++ b/tests/training/features/quantized-model/quantized-no-opt.expected
@@ -0,0 +1,10 @@
+225.10929871
+243.58345032
+229.45071411
+224.28813171
+212.65242004
+204.06596375
+197.81690979
+190.08908081
+193.72296143
+195.20832825
diff --git a/tests/training/features/quantized-model/quantized-with-bias.expected b/tests/training/features/quantized-model/quantized-with-bias.expected
new file mode 100644
index 0000000..4f56c41
--- /dev/null
+++ b/tests/training/features/quantized-model/quantized-with-bias.expected
@@ -0,0 +1,10 @@
+225.10006714
+243.58285522
+229.47399902
+224.31018066
+212.68742371
+204.09915161
+197.85253906
+190.12380981
+193.74653625
+195.23658752
diff --git a/tests/training/features/quantized-model/quantized.expected b/tests/training/features/quantized-model/quantized.expected
new file mode 100644
index 0000000..5f42a2c
--- /dev/null
+++ b/tests/training/features/quantized-model/quantized.expected
@@ -0,0 +1,10 @@
+225.25198364
+244.07009888
+230.06900024
+224.89492798
+213.12133789
+204.42272949
+198.22579956
+190.56004333
+194.42613220
+196.06826782
diff --git a/tests/training/features/quantized-model/test_quantmodel.sh b/tests/training/features/quantized-model/test_quantmodel.sh
new file mode 100644
index 0000000..75d3a1a
--- /dev/null
+++ b/tests/training/features/quantized-model/test_quantmodel.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Train a quantized marian model
+# AUTHOR: afaji
+#####################################################################
+
+# Exit on error
+set -e
+
+PREFIX=quantized
+
+# Remove old artifacts and create working directory
+rm -rf $PREFIX $PREFIX.{log,out,diff}
+mkdir -p $PREFIX
+
+
+# Train an 8-bits model 
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 \
+    --log $PREFIX.log
+
+# Check if files exist
+test -e $PREFIX/model.npz
+test -e $PREFIX.log
+
+# Compare the current output with the expected output
+cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff 
+
+# make sure that the resulting model has no more than 256 different values (i.e. quantized)
+$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
+
+# Exit with success code
+exit 0
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
new file mode 100644
index 0000000..1878ecb
--- /dev/null
+++ b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
@@ -0,0 +1,44 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Train a quantized marian model
+# AUTHOR: afaji
+#####################################################################
+
+# Exit on error
+set -e
+
+PREFIX=quantized-with-bias
+
+# Remove old artifacts and create working directory
+rm -rf $PREFIX $PREFIX.{log,out,diff}
+mkdir -p $PREFIX
+
+
+# training with quantized bias is tricky, so we start by training a normal model first before finetuning it to the quantized space.
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 20 --disp-freq 10 \
+    --log $PREFIX.log
+
+# Train an 8-bits model 
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 --quantize-biases \
+    --log $PREFIX.log
+
+# Check if files exist
+test -e $PREFIX/model.npz
+test -e $PREFIX.log
+
+# Compare the current output with the expected output
+cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
+
+# make sure that the resulting model has no more than 256 different values (i.e. quantized)
+$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8 --with_bias
+
+# Exit with success code
+exit 0
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
new file mode 100644
index 0000000..368d920
--- /dev/null
+++ b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Train a quantized marian model with a scale optimization
+# AUTHOR: afaji
+#####################################################################
+
+# Exit on error
+set -e
+
+PREFIX=quantized-opt
+
+# Remove old artifacts and create working directory
+rm -rf $PREFIX $PREFIX.{log,out,diff}
+mkdir -p $PREFIX
+
+
+# Train an 8-bits model 
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 --quantize-optimization-steps 3 \
+    --log $PREFIX.log
+
+# Check if files exist
+test -e $PREFIX/model.npz
+test -e $PREFIX.log
+
+# Compare the current output with the expected output
+cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff 
+
+# make sure that the resulting model has no more than 256 different values (i.e. quantized)
+$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
+
+# Exit with success code
+exit 0
-- 
cgit v1.2.3


From fae55cffea37d50f1130dabd5b0488df9fdd0628 Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Tue, 10 Nov 2020 05:41:19 +0000
Subject: add regression tests for model quantization

---
 .../features/quantized-model/quantized.expected      | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/quantized.expected b/tests/training/features/quantized-model/quantized.expected
index 5f42a2c..17620ec 100644
--- a/tests/training/features/quantized-model/quantized.expected
+++ b/tests/training/features/quantized-model/quantized.expected
@@ -1,10 +1,10 @@
-225.25198364
-244.07009888
-230.06900024
-224.89492798
-213.12133789
-204.42272949
-198.22579956
-190.56004333
-194.42613220
-196.06826782
+225.10929871
+243.58345032
+229.45071411
+224.28813171
+212.65242004
+204.06596375
+197.81690979
+190.08915710
+193.72299194
+195.20808411
-- 
cgit v1.2.3


From 01c568388c2441f722f04a803f3ab76c03ba9f76 Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Tue, 10 Nov 2020 06:33:49 +0000
Subject: correct the expected file

---
 .../features/quantized-model/quantized-no-opt.expected         | 10 ----------
 tests/training/features/quantized-model/quantized-opt.expected | 10 ++++++++++
 2 files changed, 10 insertions(+), 10 deletions(-)
 delete mode 100644 tests/training/features/quantized-model/quantized-no-opt.expected
 create mode 100644 tests/training/features/quantized-model/quantized-opt.expected

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/quantized-no-opt.expected b/tests/training/features/quantized-model/quantized-no-opt.expected
deleted file mode 100644
index c4ac602..0000000
--- a/tests/training/features/quantized-model/quantized-no-opt.expected
+++ /dev/null
@@ -1,10 +0,0 @@
-225.10929871
-243.58345032
-229.45071411
-224.28813171
-212.65242004
-204.06596375
-197.81690979
-190.08908081
-193.72296143
-195.20832825
diff --git a/tests/training/features/quantized-model/quantized-opt.expected b/tests/training/features/quantized-model/quantized-opt.expected
new file mode 100644
index 0000000..d413a9c
--- /dev/null
+++ b/tests/training/features/quantized-model/quantized-opt.expected
@@ -0,0 +1,10 @@
+225.25146484
+244.05734253
+230.04736328
+224.87918091
+213.11819458
+204.42819214
+198.22373962
+190.55807495
+194.42228699
+196.07421875
-- 
cgit v1.2.3


From 2e85eeb3f04dd1e346dac0172e3d72a4f4a510a3 Mon Sep 17 00:00:00 2001
From: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
Date: Tue, 10 Nov 2020 09:20:46 +0000
Subject: Create train/ directory for vocabs

---
 tests/training/features/quantized-model/test_quantmodel.sh             | 3 +--
 tests/training/features/quantized-model/test_quantmodel_with_bias.sh   | 3 +--
 .../features/quantized-model/test_quantmodel_with_optimization.sh      | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/test_quantmodel.sh b/tests/training/features/quantized-model/test_quantmodel.sh
index 75d3a1a..9bf6a34 100644
--- a/tests/training/features/quantized-model/test_quantmodel.sh
+++ b/tests/training/features/quantized-model/test_quantmodel.sh
@@ -12,8 +12,7 @@ PREFIX=quantized
 
 # Remove old artifacts and create working directory
 rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX
-
+mkdir -p $PREFIX train
 
 # Train an 8-bits model 
 $MRT_MARIAN/marian \
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
index 1878ecb..fa02213 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
@@ -12,8 +12,7 @@ PREFIX=quantized-with-bias
 
 # Remove old artifacts and create working directory
 rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX
-
+mkdir -p $PREFIX train
 
 # training with quantized bias is tricky, so we start by training a normal model first before finetuning it to the quantized space.
 $MRT_MARIAN/marian \
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
index 368d920..200a636 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
@@ -12,8 +12,7 @@ PREFIX=quantized-opt
 
 # Remove old artifacts and create working directory
 rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX
-
+mkdir -p $PREFIX train
 
 # Train an 8-bits model 
 $MRT_MARIAN/marian \
-- 
cgit v1.2.3


From ab06646770417f4481b2afe63cf723fe6a937f13 Mon Sep 17 00:00:00 2001
From: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
Date: Tue, 10 Nov 2020 09:32:09 +0000
Subject: Add setup.sh and .gitignore

---
 tests/training/features/quantized-model/.gitignore | 4 ++++
 tests/training/features/quantized-model/setup.sh   | 3 +++
 2 files changed, 7 insertions(+)
 create mode 100644 tests/training/features/quantized-model/.gitignore
 create mode 100644 tests/training/features/quantized-model/setup.sh

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/.gitignore b/tests/training/features/quantized-model/.gitignore
new file mode 100644
index 0000000..2fa5b37
--- /dev/null
+++ b/tests/training/features/quantized-model/.gitignore
@@ -0,0 +1,4 @@
+quantized
+quantized-with-bias
+quantized-opt
+train
diff --git a/tests/training/features/quantized-model/setup.sh b/tests/training/features/quantized-model/setup.sh
new file mode 100644
index 0000000..71af484
--- /dev/null
+++ b/tests/training/features/quantized-model/setup.sh
@@ -0,0 +1,3 @@
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1
+
-- 
cgit v1.2.3


From 844a4eb101002eba2d48c6e8ef60525a62eeeee1 Mon Sep 17 00:00:00 2001
From: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
Date: Tue, 10 Nov 2020 09:35:24 +0000
Subject: Remove trailing whitespaces

---
 tests/training/features/quantized-model/test_quantmodel.sh            | 4 ++--
 tests/training/features/quantized-model/test_quantmodel_with_bias.sh  | 2 +-
 .../features/quantized-model/test_quantmodel_with_optimization.sh     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/test_quantmodel.sh b/tests/training/features/quantized-model/test_quantmodel.sh
index 9bf6a34..b31a1d5 100644
--- a/tests/training/features/quantized-model/test_quantmodel.sh
+++ b/tests/training/features/quantized-model/test_quantmodel.sh
@@ -14,7 +14,7 @@ PREFIX=quantized
 rm -rf $PREFIX $PREFIX.{log,out,diff}
 mkdir -p $PREFIX train
 
-# Train an 8-bits model 
+# Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
     -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
@@ -27,7 +27,7 @@ test -e $PREFIX.log
 
 # Compare the current output with the expected output
 cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
-$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff 
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
 $MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
index fa02213..96380c3 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
@@ -21,7 +21,7 @@ $MRT_MARIAN/marian \
     --cost-type cross-entropy --sync-sgd --after-batches 20 --disp-freq 10 \
     --log $PREFIX.log
 
-# Train an 8-bits model 
+# Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
     -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
index 200a636..b4a8dbf 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
@@ -14,7 +14,7 @@ PREFIX=quantized-opt
 rm -rf $PREFIX $PREFIX.{log,out,diff}
 mkdir -p $PREFIX train
 
-# Train an 8-bits model 
+# Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
     -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
@@ -27,7 +27,7 @@ test -e $PREFIX.log
 
 # Compare the current output with the expected output
 cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
-$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff 
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
 $MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
-- 
cgit v1.2.3


From 96e40b77feda7a6742857c5d60ad8c62dc2e470b Mon Sep 17 00:00:00 2001
From: Roman Grundkiewicz <rgrundkiewicz@gmail.com>
Date: Tue, 10 Nov 2020 01:46:33 -0800
Subject: Update expected output and precision for optimized-opt

---
 .../features/quantized-model/quantized-opt.expected  | 20 ++++++++++----------
 .../test_quantmodel_with_optimization.sh             |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/quantized-opt.expected b/tests/training/features/quantized-model/quantized-opt.expected
index d413a9c..6ccc2e0 100644
--- a/tests/training/features/quantized-model/quantized-opt.expected
+++ b/tests/training/features/quantized-model/quantized-opt.expected
@@ -1,10 +1,10 @@
-225.25146484
-244.05734253
-230.04736328
-224.87918091
-213.11819458
-204.42819214
-198.22373962
-190.55807495
-194.42228699
-196.07421875
+225.53999329
+244.43618774
+230.57369995
+225.52883911
+213.84687805
+204.98857117
+198.73059082
+191.04969788
+194.96365356
+196.72579956
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
index b4a8dbf..4a88059 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
@@ -27,7 +27,7 @@ test -e $PREFIX.log
 
 # Compare the current output with the expected output
 cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
-$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff -p 0.01
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
 $MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
-- 
cgit v1.2.3


From 4dac388a39adc581ecba6ae88b5a889988d15729 Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Wed, 11 Nov 2020 04:27:38 +0000
Subject: simplify quantized-model testing by using a shared train folder

---
 tests/training/features/quantized-model/.gitignore   |  3 ---
 .../features/quantized-model/quantized-opt.expected  | 20 ++++++++++----------
 .../features/quantized-model/test_quantmodel.sh      | 10 +++++-----
 .../quantized-model/test_quantmodel_with_bias.sh     | 12 ++++++------
 .../test_quantmodel_with_optimization.sh             | 12 ++++++------
 5 files changed, 27 insertions(+), 30 deletions(-)

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/.gitignore b/tests/training/features/quantized-model/.gitignore
index 2fa5b37..08afa18 100644
--- a/tests/training/features/quantized-model/.gitignore
+++ b/tests/training/features/quantized-model/.gitignore
@@ -1,4 +1 @@
-quantized
-quantized-with-bias
-quantized-opt
 train
diff --git a/tests/training/features/quantized-model/quantized-opt.expected b/tests/training/features/quantized-model/quantized-opt.expected
index 6ccc2e0..12f21d3 100644
--- a/tests/training/features/quantized-model/quantized-opt.expected
+++ b/tests/training/features/quantized-model/quantized-opt.expected
@@ -1,10 +1,10 @@
-225.53999329
-244.43618774
-230.57369995
-225.52883911
-213.84687805
-204.98857117
-198.73059082
-191.04969788
-194.96365356
-196.72579956
+225.11299133
+243.58525085
+229.45321655
+224.28414917
+212.65376282
+204.06687927
+197.81901550
+190.08296204
+193.72265625
+195.21139526
diff --git a/tests/training/features/quantized-model/test_quantmodel.sh b/tests/training/features/quantized-model/test_quantmodel.sh
index b31a1d5..8b55697 100644
--- a/tests/training/features/quantized-model/test_quantmodel.sh
+++ b/tests/training/features/quantized-model/test_quantmodel.sh
@@ -11,18 +11,18 @@ set -e
 PREFIX=quantized
 
 # Remove old artifacts and create working directory
-rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX train
+rm -rf train $PREFIX.{log,out,diff}
+mkdir -p train
 
 # Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
-    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
     --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 \
     --log $PREFIX.log
 
 # Check if files exist
-test -e $PREFIX/model.npz
+test -e train/model.npz
 test -e $PREFIX.log
 
 # Compare the current output with the expected output
@@ -30,7 +30,7 @@ cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
 $MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
-$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
+$MRT_TOOLS/check-model-unique-vals.py train/model.npz -b 8
 
 # Exit with success code
 exit 0
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
index 96380c3..de14ffb 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_bias.sh
@@ -11,25 +11,25 @@ set -e
 PREFIX=quantized-with-bias
 
 # Remove old artifacts and create working directory
-rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX train
+rm -rf train $PREFIX.{log,out,diff}
+mkdir -p train
 
 # training with quantized bias is tricky, so we start by training a normal model first before finetuning it to the quantized space.
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
-    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
     --cost-type cross-entropy --sync-sgd --after-batches 20 --disp-freq 10 \
     --log $PREFIX.log
 
 # Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
-    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
     --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 --quantize-biases \
     --log $PREFIX.log
 
 # Check if files exist
-test -e $PREFIX/model.npz
+test -e train/model.npz
 test -e $PREFIX.log
 
 # Compare the current output with the expected output
@@ -37,7 +37,7 @@ cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
 $MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
-$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8 --with_bias
+$MRT_TOOLS/check-model-unique-vals.py train/model.npz -b 8 --with_bias
 
 # Exit with success code
 exit 0
diff --git a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
index 4a88059..fe7993e 100644
--- a/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
+++ b/tests/training/features/quantized-model/test_quantmodel_with_optimization.sh
@@ -11,26 +11,26 @@ set -e
 PREFIX=quantized-opt
 
 # Remove old artifacts and create working directory
-rm -rf $PREFIX $PREFIX.{log,out,diff}
-mkdir -p $PREFIX train
+rm -rf train $PREFIX.{log,out,diff}
+mkdir -p train
 
 # Train an 8-bits model
 $MRT_MARIAN/marian \
     --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
-    -m $PREFIX/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
     --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 8 --quantize-optimization-steps 3 \
     --log $PREFIX.log
 
 # Check if files exist
-test -e $PREFIX/model.npz
+test -e train/model.npz
 test -e $PREFIX.log
 
 # Compare the current output with the expected output
 cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
-$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff -p 0.01
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
 
 # make sure that the resulting model has no more than 256 different values (i.e. quantized)
-$MRT_TOOLS/check-model-unique-vals.py $PREFIX/model.npz -b 8
+$MRT_TOOLS/check-model-unique-vals.py train/model.npz -b 8
 
 # Exit with success code
 exit 0
-- 
cgit v1.2.3


From fdd7493cd2f922e65b6bb08b7ef84b711cf730da Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Wed, 11 Nov 2020 05:40:44 +0000
Subject: add log-based quantization test

---
 .../quantized-model/quantized-log4bit.expected     | 10 ++++++
 .../quantized-model/test_quantmodel_log.sh         | 36 ++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 tests/training/features/quantized-model/quantized-log4bit.expected
 create mode 100644 tests/training/features/quantized-model/test_quantmodel_log.sh

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/quantized-log4bit.expected b/tests/training/features/quantized-model/quantized-log4bit.expected
new file mode 100644
index 0000000..1de1b38
--- /dev/null
+++ b/tests/training/features/quantized-model/quantized-log4bit.expected
@@ -0,0 +1,10 @@
+225.47087097
+245.67460632
+232.80288696
+229.51301575
+217.59494019
+207.93991089
+201.96940613
+194.00720215
+197.69416809
+199.42311096
diff --git a/tests/training/features/quantized-model/test_quantmodel_log.sh b/tests/training/features/quantized-model/test_quantmodel_log.sh
new file mode 100644
index 0000000..f79809b
--- /dev/null
+++ b/tests/training/features/quantized-model/test_quantmodel_log.sh
@@ -0,0 +1,36 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Train a quantized marian model
+# AUTHOR: afaji
+#####################################################################
+
+# Exit on error
+set -e
+
+PREFIX=quantized-log4bit
+
+# Remove old artifacts and create working directory
+rm -rf train $PREFIX.{log,out,diff}
+mkdir -p train
+
+# Train an 8-bits model
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 100 --disp-freq 10 --quantize-bits 4 --quantize-log-based --quantize-optimization-steps 3 \
+    --log $PREFIX.log
+
+# Check if files exist
+test -e train/model.npz
+test -e $PREFIX.log
+
+# Compare the current output with the expected output
+cat $PREFIX.log | $MRT_TOOLS/extract-costs.sh > $PREFIX.out
+$MRT_TOOLS/diff-nums.py $PREFIX.out $PREFIX.expected -o $PREFIX.diff
+
+# make sure that the resulting model has no more than 256 different values (i.e. quantized)
+$MRT_TOOLS/check-model-unique-vals.py train/model.npz -b 4
+
+# Exit with success code
+exit 0
-- 
cgit v1.2.3


From 5fc2bab7c77e55410717c1f7229f97170353d20d Mon Sep 17 00:00:00 2001
From: Alham Fikri Aji <afaji321@gmail.com>
Date: Wed, 11 Nov 2020 06:14:34 +0000
Subject: test the resulting model, make sure it is quantized

---
 .../quantized-model/model_centers.expected         | 51 ++++++++++++++++++++++
 .../features/quantized-model/test_quant_centers.sh | 31 +++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 tests/training/features/quantized-model/model_centers.expected
 create mode 100644 tests/training/features/quantized-model/test_quant_centers.sh

(limited to 'tests')

diff --git a/tests/training/features/quantized-model/model_centers.expected b/tests/training/features/quantized-model/model_centers.expected
new file mode 100644
index 0000000..57380ae
--- /dev/null
+++ b/tests/training/features/quantized-model/model_centers.expected
@@ -0,0 +1,51 @@
+Tensor decoder_W_comb_att unique centers: [-0.17677179 -0.11784786 -0.05892393 -0.          0.05892393  0.11784786
+  0.17677179]
+Tensor decoder_Wc_att unique centers: [-0.15336949 -0.10224632 -0.05112316 -0.          0.05112316  0.10224632
+  0.15336949]
+Tensor Wemb_dec unique centers: [-0.32046145 -0.21364096 -0.10682048  0.          0.10682048  0.21364096
+  0.32046145]
+Tensor decoder_U unique centers: [-0.17687811 -0.11791874 -0.05895937 -0.          0.05895937  0.11791874
+  0.17687811]
+Tensor decoder_Ux unique centers: [-0.21770547 -0.14513698 -0.07256849  0.          0.07256849  0.14513698
+  0.21770547]
+Tensor decoder_W unique centers: [-0.19397542 -0.12931694 -0.06465847 -0.          0.06465847  0.12931694
+  0.19397542]
+Tensor decoder_Wx unique centers: [-0.25329626 -0.16886416 -0.08443208 -0.          0.08443208  0.16886416
+  0.25329626]
+Tensor decoder_U_nl unique centers: [-0.17696194 -0.11797463 -0.05898732  0.          0.05898732  0.11797463
+  0.17696194]
+Tensor decoder_Ux_nl unique centers: [-0.21896881 -0.14597921 -0.07298961  0.          0.07298961  0.14597921
+  0.21896881]
+Tensor decoder_Wc unique centers: [-0.15324192 -0.10216128 -0.05108064  0.          0.05108064  0.10216128
+  0.15324192]
+Tensor decoder_Wcx unique centers: [-0.18192002 -0.12128001 -0.06064001 -0.          0.06064001  0.12128001
+  0.18192002]
+Tensor ff_logit_prev_W unique centers: [-0.32183957 -0.2145597  -0.10727985 -0.          0.10727985  0.2145597
+  0.32183957]
+Tensor ff_logit_lstm_W unique centers: [-0.25455362 -0.16970241 -0.08485121  0.          0.08485121  0.16970241
+  0.25455362]
+Tensor ff_logit_ctx_W unique centers: [-0.19867198 -0.13244799 -0.06622399 -0.          0.06622399  0.13244799
+  0.19867198]
+Tensor decoder_ff_logit_l2_Wt unique centers: [-0.36124557 -0.24083039 -0.1204152   0.          0.1204152   0.24083039
+  0.36124557]
+Tensor ff_state_W unique centers: [-0.17704961 -0.11803307 -0.05901653  0.          0.05901653  0.11803307
+  0.17704961]
+Tensor Wemb unique centers: [-0.31208774 -0.20805849 -0.10402925  0.          0.10402925  0.20805849
+  0.31208774]
+Tensor encoder_U unique centers: [-0.17686225 -0.11790817 -0.05895409  0.          0.05895409  0.11790817
+  0.17686225]
+Tensor encoder_Ux unique centers: [-0.21824732 -0.14549822 -0.07274911  0.          0.07274911  0.14549822
+  0.21824732]
+Tensor encoder_W unique centers: [-0.19403435 -0.12935624 -0.06467812  0.          0.06467812  0.12935624
+  0.19403435]
+Tensor encoder_Wx unique centers: [-0.25213736 -0.16809157 -0.08404578 -0.          0.08404578  0.16809157
+  0.25213736]
+Tensor encoder_r_U unique centers: [-0.17699143 -0.11799429 -0.05899715  0.          0.05899715  0.11799429
+  0.17699143]
+Tensor encoder_r_Ux unique centers: [-0.21971346 -0.14647564 -0.07323782 -0.          0.07323782  0.14647564
+  0.21971346]
+Tensor encoder_r_W unique centers: [-0.19410282 -0.12940188 -0.06470094  0.          0.06470094  0.12940188
+  0.19410282]
+Tensor encoder_r_Wx unique centers: [-0.25225359 -0.16816907 -0.08408453 -0.          0.08408453  0.16816907
+  0.25225359]
+Tensor decoder_c_tt unique centers: []
diff --git a/tests/training/features/quantized-model/test_quant_centers.sh b/tests/training/features/quantized-model/test_quant_centers.sh
new file mode 100644
index 0000000..22dd863
--- /dev/null
+++ b/tests/training/features/quantized-model/test_quant_centers.sh
@@ -0,0 +1,31 @@
+#!/bin/bash -x
+
+#####################################################################
+# SUMMARY: Make sure that the resulting model is in quantized form
+# AUTHOR: afaji
+#####################################################################
+
+# Exit on error
+set -e
+
+PREFIX=test-center
+
+# Remove old artifacts and create working directory
+rm -rf train
+mkdir -p train
+
+# Train an 8-bits model
+$MRT_MARIAN/marian \
+    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --learn-rate 0.1 --optimizer sgd \
+    -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \
+    --cost-type cross-entropy --sync-sgd --after-batches 10 --disp-freq 2 --quantize-bits 3
+
+# Check if files exist
+test -e train/model.npz
+
+# make sure that the resulting model has no more than 256 different values (i.e. quantized)
+$MRT_TOOLS/check-model-unique-vals.py train/model.npz -b 3 --print_centers -o model_centers.out
+$MRT_TOOLS/diff-nums.py model_centers.out model_centers.expected -o model_centers.diff --numpy
+
+# Exit with success code
+exit 0
-- 
cgit v1.2.3