Add test for loading adam parameters

14 files changed, 148 insertions, 2 deletions

diff --git a/tests/training/restart/.gitignore b/tests/training/restart/.gitignore
index 7805a4d..1b9587e 100644
--- a/tests/training/restart/.gitignore
+++ b/tests/training/restart/.gitignore
@@ -1,3 +1,4 @@
-adam_npz
+adam_load
+adam
+adam_async
 sgd_2e
-adam_2e
diff --git a/tests/training/restart/adam.costs.expected b/tests/training/restart/adam.costs.expected
new file mode 100644
index 0000000..e82715d
--- /dev/null
+++ b/tests/training/restart/adam.costs.expected
@@ -0,0 +1,10 @@
+238.61
+240.18
+207.25
+186.16
+180.53
+186.15
+182.22
+168.12
+167.80
+167.27
diff --git a/tests/training/restart/adam.keys.expected b/tests/training/restart/adam.keys.expected
new file mode 100644
index 0000000..2eec3bc
--- /dev/null
+++ b/tests/training/restart/adam.keys.expected
@@ -0,0 +1,2 @@
+mt_
+vt_
diff --git a/tests/training/restart/adam.mt.expected b/tests/training/restart/adam.mt.expected
new file mode 100644
index 0000000..1c7682a
--- /dev/null
+++ b/tests/training/restart/adam.mt.expected
@@ -0,0 +1,2 @@
+[[  6.32751617e-05  -6.52741073e-05  -4.17807678e-05 ...,   0.00000000e+00
+    0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam.vt.expected b/tests/training/restart/adam.vt.expected
new file mode 100644
index 0000000..6772f97
--- /dev/null
+++ b/tests/training/restart/adam.vt.expected
@@ -0,0 +1,2 @@
+[[  9.30291080e-08   1.14721395e-07   1.63331570e-08 ...,   0.00000000e+00
+    0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam_async.costs.expected b/tests/training/restart/adam_async.costs.expected
new file mode 100644
index 0000000..c9efbad
--- /dev/null
+++ b/tests/training/restart/adam_async.costs.expected
@@ -0,0 +1,10 @@
+226.70
+249.24
+218.23
+201.51
+186.34
+177.74
+168.29
+159.49
+169.14
+168.87
diff --git a/tests/training/restart/adam_async.keys.expected b/tests/training/restart/adam_async.keys.expected
new file mode 100644
index 0000000..2eec3bc
--- /dev/null
+++ b/tests/training/restart/adam_async.keys.expected
@@ -0,0 +1,2 @@
+mt_
+vt_
diff --git a/tests/training/restart/adam_async.mt.expected b/tests/training/restart/adam_async.mt.expected
new file mode 100644
index 0000000..f812373
--- /dev/null
+++ b/tests/training/restart/adam_async.mt.expected
@@ -0,0 +1,2 @@
+[[ -6.81742749e-05   6.45068503e-05   1.26248915e-05 ...,   0.00000000e+00
+    0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam_async.vt.expected b/tests/training/restart/adam_async.vt.expected
new file mode 100644
index 0000000..3840517
--- /dev/null
+++ b/tests/training/restart/adam_async.vt.expected
@@ -0,0 +1,2 @@
+[[  3.90979540e-06   3.93529353e-06   4.17320479e-07 ...,   0.00000000e+00
+    0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam_load_1.expected b/tests/training/restart/adam_load_1.expected
new file mode 100644
index 0000000..ef5e82b
--- /dev/null
+++ b/tests/training/restart/adam_load_1.expected
@@ -0,0 +1,3 @@
+Ep. 1 : Up. 1 : Sen. 8 : Cost 279.36
+Ep. 1 : Up. 2 : Sen. 16 : Cost 244.33
+Ep. 1 : Up. 3 : Sen. 24 : Cost 190.35
diff --git a/tests/training/restart/adam_load_2.expected b/tests/training/restart/adam_load_2.expected
new file mode 100644
index 0000000..4d8b436
--- /dev/null
+++ b/tests/training/restart/adam_load_2.expected
@@ -0,0 +1,3 @@
+Ep. 1 : Up. 4 : Sen. 32 : Cost 183.98
+Ep. 1 : Up. 5 : Sen. 40 : Cost 207.76
+Ep. 1 : Up. 6 : Sen. 48 : Cost 164.58
diff --git a/tests/training/restart/test_loading_adam_params.sh b/tests/training/restart/test_loading_adam_params.sh
new file mode 100644
index 0000000..84fc7c6
--- /dev/null
+++ b/tests/training/restart/test_loading_adam_params.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf adam_load adam_load_?.log
+mkdir -p adam_load
+
+extra_opts="--no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 8 --dim-rnn 64 --dim-emb 32"
+
+$MRT_MARIAN/build/marian \
+    -m adam_load/model.npz -t train.max50.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 1 --after-batches 3 -l 0.1 $extra_opts \
+    --log adam_load_1.log
+
+test -e adam_load/model.npz
+test -e adam_load/model.npz.optimizer.npz
+test -e adam_load_1.log
+
+cat adam_load_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > adam_load_1.out
+
+$MRT_MARIAN/build/marian \
+    -m adam_load/model.npz -t train.max50.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 1 --after-batches 6 -l 0.1 $extra_opts \
+    --log adam_load_2.log
+
+test -e adam_load/model.npz
+test -e adam_load/model.npz.optimizer.npz
+test -e adam_load_2.log
+
+cat adam_load_2.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > adam_load_2.out
+
+$MRT_TOOLS/diff-floats.py -p 0.99 adam_load_1.out adam_load_1.expected > adam_load_1.diff
+$MRT_TOOLS/diff-floats.py -p 0.99 adam_load_2.out adam_load_2.expected > adam_load_2.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/restart/test_saving_adam_params.sh b/tests/training/restart/test_saving_adam_params.sh
new file mode 100644
index 0000000..6955390
--- /dev/null
+++ b/tests/training/restart/test_saving_adam_params.sh
@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf adam adam*.log
+mkdir -p adam
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none \
+    -m adam/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 10 --after-batches 100 --save-freq 60 \
+    --log adam.log
+
+test -e adam/model.npz
+test -e adam/model.npz.optimizer.npz
+test -e adam.log
+
+$MRT_TOOLS/extract-costs.sh < adam.log > adam.costs.out
+$MRT_TOOLS/diff-floats.py adam.costs.out adam.costs.expected -p 0.2 > adam.costs.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz > adam.keys.out
+diff adam.keys.out adam.keys.expected > adam.keys.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k mt_ > adam.mt.out
+$MRT_TOOLS/diff-floats.py -p 0.000005  adam.mt.out adam.mt.expected > adam.mt.diff
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k vt_ > adam.vt.out
+$MRT_TOOLS/diff-floats.py -p 0.000005 adam.vt.out adam.vt.expected > adam.vt.diff
+
+# Exit with success code
+exit 0
diff --git a/tests/training/restart/test_saving_adam_params_async.sh b/tests/training/restart/test_saving_adam_params_async.sh
new file mode 100644
index 0000000..66461e9
--- /dev/null
+++ b/tests/training/restart/test_saving_adam_params_async.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf adam_async adam_async*.log
+mkdir -p adam_async
+
+if (( $MRT_NUM_DEVICES < 2 )); then
+    echo "Too few devices available"
+    exit 100
+fi
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --devices 0 1 \
+    -m adam_async/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 10 --after-batches 100 --save-freq 60 \
+    --log adam_async.log
+
+test -e adam_async/model.npz
+test -e adam_async/model.npz.optimizer.npz
+test -e adam_async.log
+
+$MRT_TOOLS/extract-costs.sh < adam_async.log > adam_async.costs.out
+$MRT_TOOLS/diff-floats.py adam_async.costs.out adam_async.costs.expected -p 3.00 -n 2 > adam_async.costs.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz > adam_async.keys.out
+diff adam_async.keys.out adam_async.keys.expected > adam_async.keys.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k mt_ > adam_async.mt.out
+$MRT_TOOLS/diff-floats.py -p 0.000009  adam_async.mt.out adam_async.mt.expected > adam_async.mt.diff
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k vt_ > adam_async.vt.out
+$MRT_TOOLS/diff-floats.py -p 0.0000009 adam_async.vt.out adam_async.vt.expected > adam_async.vt.diff
+
+# Exit with success code
+exit 0