Update tests restoring adam parameters

14 files changed, 84 insertions, 33 deletions

diff --git a/tests/training/restart/.gitignore b/tests/training/restart/.gitignore
index 1b9587e..57fd85c 100644
--- a/tests/training/restart/.gitignore
+++ b/tests/training/restart/.gitignore
@@ -1,4 +1,5 @@
-adam_load
+sgd_2e
 adam
 adam_async
-sgd_2e
+adam_sync
+adam_load
diff --git a/tests/training/restart/adam.costs.expected b/tests/training/restart/adam.costs.expected
index e82715d..20cf3c5 100644
--- a/tests/training/restart/adam.costs.expected
+++ b/tests/training/restart/adam.costs.expected
@@ -1,10 +1,10 @@
-238.61
-240.18
-207.25
-186.16
-180.53
-186.15
-182.22
-168.12
-167.80
-167.27
+238.74
+245.65
+239.98
+231.62
+230.46
+233.95
+229.32
+212.29
+209.17
+199.81
diff --git a/tests/training/restart/adam.mt.expected b/tests/training/restart/adam.mt.expected
index 1c7682a..1594e99 100644
--- a/tests/training/restart/adam.mt.expected
+++ b/tests/training/restart/adam.mt.expected
@@ -1,2 +1 @@
-[[  6.32751617e-05  -6.52741073e-05  -4.17807678e-05 ...,   0.00000000e+00
-    0.00000000e+00   0.00000000e+00]]
+[[ 0.00052343 -0.00056336  0.00047079 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restart/adam.vt.expected b/tests/training/restart/adam.vt.expected
index 6772f97..d03a7e9 100644
--- a/tests/training/restart/adam.vt.expected
+++ b/tests/training/restart/adam.vt.expected
@@ -1,2 +1,2 @@
-[[  9.30291080e-08   1.14721395e-07   1.63331570e-08 ...,   0.00000000e+00
+[[  6.01100282e-07   6.87037470e-07   1.69330690e-07 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam_async.costs.expected b/tests/training/restart/adam_async.costs.expected
index c9efbad..4134e31 100644
--- a/tests/training/restart/adam_async.costs.expected
+++ b/tests/training/restart/adam_async.costs.expected
@@ -1,10 +1,10 @@
-226.70
-249.24
-218.23
-201.51
-186.34
-177.74
-168.29
-159.49
-169.14
-168.87
+226.77
+250.64
+243.71
+246.06
+236.14
+221.90
+211.39
+202.45
+204.81
+201.36
diff --git a/tests/training/restart/adam_async.mt.expected b/tests/training/restart/adam_async.mt.expected
index f812373..852700a 100644
--- a/tests/training/restart/adam_async.mt.expected
+++ b/tests/training/restart/adam_async.mt.expected
@@ -1,2 +1 @@
-[[ -6.81742749e-05   6.45068503e-05   1.26248915e-05 ...,   0.00000000e+00
-    0.00000000e+00   0.00000000e+00]]
+[[ 0.00351954 -0.00345516  0.00260725 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restart/adam_async.vt.expected b/tests/training/restart/adam_async.vt.expected
index 3840517..dfe7787 100644
--- a/tests/training/restart/adam_async.vt.expected
+++ b/tests/training/restart/adam_async.vt.expected
@@ -1,2 +1,2 @@
-[[  3.90979540e-06   3.93529353e-06   4.17320479e-07 ...,   0.00000000e+00
+[[  6.58408317e-06   6.16061698e-06   3.20382856e-06 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/adam_sync.costs.expected b/tests/training/restart/adam_sync.costs.expected
new file mode 100644
index 0000000..f01a299
--- /dev/null
+++ b/tests/training/restart/adam_sync.costs.expected
@@ -0,0 +1,10 @@
+226.76
+250.62
+243.61
+245.71
+233.83
+219.88
+210.65
+201.93
+204.41
+204.01
diff --git a/tests/training/restart/adam_sync.keys.expected b/tests/training/restart/adam_sync.keys.expected
new file mode 100644
index 0000000..2eec3bc
--- /dev/null
+++ b/tests/training/restart/adam_sync.keys.expected
@@ -0,0 +1,2 @@
+mt_
+vt_
diff --git a/tests/training/restart/adam_sync.mt.expected b/tests/training/restart/adam_sync.mt.expected
new file mode 100644
index 0000000..9d51448
--- /dev/null
+++ b/tests/training/restart/adam_sync.mt.expected
@@ -0,0 +1 @@
+[[ 0.00390074 -0.00370624  0.00307948 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restart/adam_sync.vt.expected b/tests/training/restart/adam_sync.vt.expected
new file mode 100644
index 0000000..1285390
--- /dev/null
+++ b/tests/training/restart/adam_sync.vt.expected
@@ -0,0 +1,2 @@
+[[  5.76593220e-06   5.19660261e-06   3.10109863e-06 ...,   0.00000000e+00
+    0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restart/test_saving_adam_params.sh b/tests/training/restart/test_saving_adam_params.sh
index 6955390..e6bdb20 100644
--- a/tests/training/restart/test_saving_adam_params.sh
+++ b/tests/training/restart/test_saving_adam_params.sh
@@ -8,7 +8,7 @@ rm -rf adam adam*.log
 mkdir -p adam
 
 $MRT_MARIAN/build/marian \
-    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none \
+    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --dim-emb 128 --dim-rnn 256 \
     -m adam/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
     --disp-freq 10 --after-batches 100 --save-freq 60 \
     --log adam.log
@@ -24,7 +24,7 @@ python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz
 diff adam.keys.out adam.keys.expected > adam.keys.diff
 
 python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k mt_ > adam.mt.out
-$MRT_TOOLS/diff-floats.py -p 0.000005  adam.mt.out adam.mt.expected > adam.mt.diff
+$MRT_TOOLS/diff-floats.py -p 0.0001  adam.mt.out adam.mt.expected > adam.mt.diff
 python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k vt_ > adam.vt.out
 $MRT_TOOLS/diff-floats.py -p 0.000005 adam.vt.out adam.vt.expected > adam.vt.diff
 
diff --git a/tests/training/restart/test_saving_adam_params_async.sh b/tests/training/restart/test_saving_adam_params_async.sh
index 66461e9..1cdad84 100644
--- a/tests/training/restart/test_saving_adam_params_async.sh
+++ b/tests/training/restart/test_saving_adam_params_async.sh
@@ -13,10 +13,10 @@ if (( $MRT_NUM_DEVICES < 2 )); then
 fi
 
 $MRT_MARIAN/build/marian \
-    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --devices 0 1 \
+    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --dim-emb 128 --dim-rnn 256 \
     -m adam_async/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
     --disp-freq 10 --after-batches 100 --save-freq 60 \
-    --log adam_async.log
+    --log adam_async.log --devices 0 1
 
 test -e adam_async/model.npz
 test -e adam_async/model.npz.optimizer.npz
@@ -29,7 +29,7 @@ python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimiz
 diff adam_async.keys.out adam_async.keys.expected > adam_async.keys.diff
 
 python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k mt_ > adam_async.mt.out
-$MRT_TOOLS/diff-floats.py -p 0.000009  adam_async.mt.out adam_async.mt.expected > adam_async.mt.diff
+$MRT_TOOLS/diff-floats.py -p 0.0001  adam_async.mt.out adam_async.mt.expected > adam_async.mt.diff
 python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k vt_ > adam_async.vt.out
 $MRT_TOOLS/diff-floats.py -p 0.0000009 adam_async.vt.out adam_async.vt.expected > adam_async.vt.diff
 
diff --git a/tests/training/restart/test_saving_adam_params_sync.sh b/tests/training/restart/test_saving_adam_params_sync.sh
new file mode 100644
index 0000000..762ea63
--- /dev/null
+++ b/tests/training/restart/test_saving_adam_params_sync.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+# Test code goes here
+rm -rf adam_sync adam_sync*.log
+mkdir -p adam_sync
+
+if (( $MRT_NUM_DEVICES < 2 )); then
+    echo "Too few devices available"
+    exit 100
+fi
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --dim-emb 128 --dim-rnn 256 \
+    -m adam_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.{en,de}.yml \
+    --disp-freq 10 --after-batches 100 --save-freq 60 \
+    --log adam_sync.log --devices 0 1 --sync-sgd
+
+test -e adam_sync/model.npz
+test -e adam_sync/model.npz.optimizer.npz
+test -e adam_sync.log
+
+$MRT_TOOLS/extract-costs.sh < adam_sync.log > adam_sync.costs.out
+$MRT_TOOLS/diff-floats.py adam_sync.costs.out adam_sync.costs.expected -p 3.00 -n 2 > adam_sync.costs.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz > adam_sync.keys.out
+diff adam_sync.keys.out adam_sync.keys.expected > adam_sync.keys.diff
+
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k mt_ > adam_sync.mt.out
+$MRT_TOOLS/diff-floats.py -p 0.0001  adam_sync.mt.out adam_sync.mt.expected > adam_sync.mt.diff
+python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k vt_ > adam_sync.vt.out
+$MRT_TOOLS/diff-floats.py -p 0.0000009 adam_sync.vt.out adam_sync.vt.expected > adam_sync.vt.diff
+
+# Exit with success code
+exit 0