Use clip-norm 0 in training/restarting

4 files changed, 31 insertions, 17 deletions

diff --git a/tests/training/restarting/sgd_2e.expected b/tests/training/restarting/sgd_2e.expected
index 2a801f2..15bbe18 100644
--- a/tests/training/restarting/sgd_2e.expected
+++ b/tests/training/restarting/sgd_2e.expected
@@ -1,16 +1,16 @@
 Ep. 1 : Up. 4 : Sen. 128 : Cost 257.99652100
 Ep. 1 : Up. 8 : Sen. 256 : Cost 267.93783569
-Ep. 1 : Up. 12 : Sen. 384 : Cost 243.39039612
-Ep. 1 : Up. 16 : Sen. 512 : Cost 235.87208557
+Ep. 1 : Up. 12 : Sen. 384 : Cost 243.39041138
+Ep. 1 : Up. 16 : Sen. 512 : Cost 235.87210083
 Ep. 1 : Up. 20 : Sen. 640 : Cost 204.79017639
-Ep. 1 : Up. 24 : Sen. 768 : Cost 240.11624146
-Ep. 1 : Up. 28 : Sen. 896 : Cost 208.47099304
-Ep. 1 : Up. 32 : Sen. 1,024 : Cost 199.36221313
-Ep. 2 : Up. 36 : Sen. 128 : Cost 213.58728027
-Ep. 2 : Up. 40 : Sen. 256 : Cost 220.51084900
-Ep. 2 : Up. 44 : Sen. 384 : Cost 199.10847473
-Ep. 2 : Up. 48 : Sen. 512 : Cost 194.56945801
-Ep. 2 : Up. 52 : Sen. 640 : Cost 171.77461243
-Ep. 2 : Up. 56 : Sen. 768 : Cost 208.75405884
-Ep. 2 : Up. 60 : Sen. 896 : Cost 186.17434692
-Ep. 2 : Up. 64 : Sen. 1,024 : Cost 179.63316345
+Ep. 1 : Up. 24 : Sen. 768 : Cost 240.11631775
+Ep. 1 : Up. 28 : Sen. 896 : Cost 208.47109985
+Ep. 1 : Up. 32 : Sen. 1,024 : Cost 199.36233521
+Ep. 2 : Up. 36 : Sen. 128 : Cost 213.58744812
+Ep. 2 : Up. 40 : Sen. 256 : Cost 220.51107788
+Ep. 2 : Up. 44 : Sen. 384 : Cost 199.10870361
+Ep. 2 : Up. 48 : Sen. 512 : Cost 194.56954956
+Ep. 2 : Up. 52 : Sen. 640 : Cost 171.77453613
+Ep. 2 : Up. 56 : Sen. 768 : Cost 208.75399780
+Ep. 2 : Up. 60 : Sen. 896 : Cost 186.17416382
+Ep. 2 : Up. 64 : Sen. 1,024 : Cost 179.63301086
diff --git a/tests/training/restarting/sgd_sync_2e.expected b/tests/training/restarting/sgd_sync_2e.expected
index 07061db..a83990d 100644
--- a/tests/training/restarting/sgd_sync_2e.expected
+++ b/tests/training/restarting/sgd_sync_2e.expected
@@ -7,10 +7,10 @@ Ep. 1 : Up. 24 : Sen. 768 : Cost 240.11631775
 Ep. 1 : Up. 28 : Sen. 896 : Cost 208.47109985
 Ep. 1 : Up. 32 : Sen. 1,024 : Cost 199.36233521
 Ep. 2 : Up. 36 : Sen. 128 : Cost 213.58744812
-Ep. 2 : Up. 40 : Sen. 256 : Cost 220.51107788
+Ep. 2 : Up. 40 : Sen. 256 : Cost 220.51104736
 Ep. 2 : Up. 44 : Sen. 384 : Cost 199.10870361
 Ep. 2 : Up. 48 : Sen. 512 : Cost 194.56954956
 Ep. 2 : Up. 52 : Sen. 640 : Cost 171.77453613
-Ep. 2 : Up. 56 : Sen. 768 : Cost 208.75396729
+Ep. 2 : Up. 56 : Sen. 768 : Cost 208.75399780
 Ep. 2 : Up. 60 : Sen. 896 : Cost 186.17416382
 Ep. 2 : Up. 64 : Sen. 1,024 : Cost 179.63301086
diff --git a/tests/training/restarting/test_sgd_for_two_epochs.sh b/tests/training/restarting/test_sgd_for_two_epochs.sh
index ad92b5f..11bf76e 100644
--- a/tests/training/restarting/test_sgd_for_two_epochs.sh
+++ b/tests/training/restarting/test_sgd_for_two_epochs.sh
@@ -1,5 +1,11 @@
 #!/bin/bash -x
 
+#####################################################################
+# SUMMARY: Restaring training after the 1st epoch (async)
+# AUTHOR: snukky
+# TAGS: optimizer clip-norm
+#####################################################################
+
 # Exit on error
 set -e
 
@@ -7,12 +13,13 @@ set -e
 rm -rf sgd_2e sgd_1st_epoch.log sgd_2nd_epoch.log
 mkdir -p sgd_2e
 
-extra_opts="--no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd"
+extra_opts="--no-shuffle --clip-norm 1 --seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd"
 # Added because default options has changes
 extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false"
 
 
 # Uncomment to prepare the expected output
+#rm -f sgd_two_epochs.log
 #$MRT_MARIAN/marian \
     #-m sgd_2e/model_2e.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \
     #--disp-freq 4 --save-freq 32 --after-epochs 2 -l 0.1 $extra_opts \
diff --git a/tests/training/restarting/test_sgd_for_two_epochs_sync.sh b/tests/training/restarting/test_sgd_for_two_epochs_sync.sh
index d3ee295..8615e80 100644
--- a/tests/training/restarting/test_sgd_for_two_epochs_sync.sh
+++ b/tests/training/restarting/test_sgd_for_two_epochs_sync.sh
@@ -1,5 +1,11 @@
 #!/bin/bash -x
 
+#####################################################################
+# SUMMARY: Restaring training after the 1st epoch (sync-sgd)
+# AUTHOR: snukky
+# TAGS: optimizer clip-norm
+#####################################################################
+
 # Exit on error
 set -e
 
@@ -7,12 +13,13 @@ set -e
 rm -rf sgd_sync_2e sgd_sync_*_epoch.log
 mkdir -p sgd_sync_2e
 
-extra_opts="--no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --sync-sgd"
+extra_opts="--no-shuffle --clip-norm 1 --seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --sync-sgd"
 # Added because default options has changes
 extra_opts="$extra_opts --cost-type ce-mean --disp-label-counts false"
 
 
 # Uncomment to prepare the expected output
+#rm -f sgd_sync_two_epochs.log
 #$MRT_MARIAN/marian \
     #-m sgd_sync_2e/model_2e.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \
     #--disp-freq 4 --save-freq 32 --after-epochs 2 -l 0.1 $extra_opts \