Add test for multi-GPU training

4 files changed, 55 insertions, 0 deletions

diff --git a/tests/training/multi-gpu/.gitignore b/tests/training/multi-gpu/.gitignore
new file mode 100644
index 0000000..e42b6a6
--- /dev/null
+++ b/tests/training/multi-gpu/.gitignore
@@ -0,0 +1 @@
+async_sgd
diff --git a/tests/training/multi-gpu/async_sgd.expected b/tests/training/multi-gpu/async_sgd.expected
new file mode 100644
index 0000000..5bf739f
--- /dev/null
+++ b/tests/training/multi-gpu/async_sgd.expected
@@ -0,0 +1,20 @@
+Ep. 1 : Up. 20 : Sen. 1280 : Cost 408.93
+Ep. 1 : Up. 40 : Sen. 2560 : Cost 256.44
+Ep. 1 : Up. 60 : Sen. 3840 : Cost 173.64
+Ep. 1 : Up. 80 : Sen. 5120 : Cost 118.18
+Ep. 1 : Up. 100 : Sen. 6400 : Cost 64.24
+Ep. 1 : Up. 120 : Sen. 7680 : Cost 179.19
+Ep. 1 : Up. 140 : Sen. 8960 : Cost 190.98
+Ep. 1 : Up. 160 : Sen. 10240 : Cost 167.34
+Ep. 1 : Up. 180 : Sen. 11520 : Cost 167.91
+Ep. 1 : Up. 200 : Sen. 12800 : Cost 156.16
+Ep. 1 : Up. 220 : Sen. 14080 : Cost 189.85
+Ep. 1 : Up. 240 : Sen. 15360 : Cost 175.92
+Ep. 1 : Up. 260 : Sen. 16640 : Cost 167.99
+Ep. 1 : Up. 280 : Sen. 17920 : Cost 164.80
+Ep. 1 : Up. 300 : Sen. 19200 : Cost 186.69
+Ep. 1 : Up. 320 : Sen. 20480 : Cost 166.41
+Ep. 1 : Up. 340 : Sen. 21760 : Cost 168.86
+Ep. 1 : Up. 360 : Sen. 23040 : Cost 177.95
+Ep. 1 : Up. 380 : Sen. 24320 : Cost 203.43
+Ep. 1 : Up. 400 : Sen. 25600 : Cost 165.09
diff --git a/tests/training/multi-gpu/setup.sh b/tests/training/multi-gpu/setup.sh
new file mode 100644
index 0000000..8b8cd07
--- /dev/null
+++ b/tests/training/multi-gpu/setup.sh
@@ -0,0 +1,2 @@
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1
+test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1
diff --git a/tests/training/multi-gpu/test_async_sgd.sh b/tests/training/multi-gpu/test_async_sgd.sh
new file mode 100644
index 0000000..6e0ac6b
--- /dev/null
+++ b/tests/training/multi-gpu/test_async_sgd.sh
@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Exit on error
+set -e
+
+if (( $MRT_NUM_DEVICES < 2 )); then
+    echo "Too few devices available"
+    exit 100
+fi
+
+# Test code goes here
+rm -rf async_sgd async_sgd.log
+mkdir -p async_sgd
+
+$MRT_MARIAN/build/marian \
+    --no-shuffle --seed 1111 \
+    --devices 0 1 \
+    -m async_sgd/model.npz \
+    -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \
+    -v vocab.en.yml vocab.de.yml \
+    --disp-freq 20 --after-batches 400 \
+    --log async_sgd.log
+
+test -e vocab.en.yml
+test -e vocab.de.yml
+test -e async_sgd.log
+
+cat async_sgd.log | $MRT_TOOLS/strip-timestamps.sh | grep -oP "Ep\. 1 .* Cost [0-9.]*" > async_sgd.out
+$MRT_TOOLS/diff-floats.py async_sgd.out async_sgd.expected -p 0.8 --max-diff-nums 5 > async_sgd.diff
+
+# Exit with success code
+exit 0