Use clip-norm 0 in training/features/

27 files changed, 238 insertions, 244 deletions

diff --git a/tests/training/features/data-weighting/maxibatch.expected b/tests/training/features/data-weighting/maxibatch.expected
index 7c0001c..ab99d76 100644
--- a/tests/training/features/data-weighting/maxibatch.expected
+++ b/tests/training/features/data-weighting/maxibatch.expected
@@ -1,10 +1,10 @@
-6924.51171875
-5225.91162109
-4136.33691406
-3092.30273438
-2086.31420898
-5388.93750000
-5653.18310547
-4460.88183594
-3398.95581055
-2398.78735352
+6904.40136719
+5177.73974609
+4077.06103516
+3035.06811523
+2036.97106934
+5234.54150391
+5421.84570312
+4205.61328125
+3152.19384766
+2185.01635742
diff --git a/tests/training/features/data-weighting/sqlite.expected b/tests/training/features/data-weighting/sqlite.expected
index d2f0f31..05298eb 100644
--- a/tests/training/features/data-weighting/sqlite.expected
+++ b/tests/training/features/data-weighting/sqlite.expected
@@ -1,100 +1,100 @@
-145.22386169
-408.22799683
-1194.63964844
-233.55360413
-1430.38696289
-126.14705658
-378.98550415
-96.71858215
-757.48663330
-330.56832886
-1604.02294922
-214.05000305
-438.19305420
-96.71667480
-466.88586426
-194.51495361
-1197.85363770
-146.07209778
-933.41638184
-389.17184448
-933.05151367
-622.52008057
-874.42907715
-825.77954102
-437.69128418
-155.24894714
-496.18453979
-232.95338440
-350.40795898
-321.22961426
-610.87152100
-670.83068848
-1778.17529297
-241.61517334
-1370.66113281
-136.27859497
-1510.65393066
-184.96159363
-815.58801270
-136.21365356
-729.38146973
-174.92094421
-408.49716187
-213.90168762
-233.55191040
-136.20471191
-290.47631836
-165.94480896
-584.11578369
-349.23815918
-958.81738281
-253.41488647
-437.03112793
-330.26031494
-435.45944214
-154.90957642
-521.61523438
-563.37170410
-496.03311157
-175.29434204
-438.16888428
-135.96989441
-496.29870605
-87.74061584
-436.65737915
-214.36607361
-58.15562820
-378.97769165
-348.81768799
-48.82063675
-993.13079834
-292.51773071
-524.27105713
-620.84045410
-1228.60827637
-535.02069092
-846.56628418
-291.05285645
-1658.58984375
-145.76992798
-1809.52172852
-680.63385010
-875.07159424
-78.35388947
-1049.29785156
-165.48069763
-611.14044189
-87.67588806
-438.09893799
-126.03882599
-262.80325317
-184.69609070
-262.46411133
-213.61804199
-1341.65759277
-106.89002228
-174.81327820
-116.23052979
-1165.48278809
-612.53643799
+145.22384644
+408.21359253
+1194.58532715
+233.50500488
+1429.99536133
+126.10730743
+378.85754395
+96.65737915
+757.20660400
+330.41281128
+1603.38867188
+213.94171143
+438.01345825
+96.60343933
+466.71673584
+194.41984558
+1197.37438965
+145.95291138
+932.55432129
+388.84637451
+932.01428223
+622.00317383
+873.45208740
+824.53771973
+437.26394653
+155.06787109
+495.64407349
+232.61149597
+349.86016846
+320.80847168
+610.00537109
+670.07312012
+1775.15356445
+241.20024109
+1368.22827148
+135.98722839
+1508.71228027
+184.60949707
+814.25140381
+135.82812500
+728.21057129
+174.76170349
+408.09228516
+213.54101562
+233.23889160
+136.04132080
+290.16552734
+165.65209961
+583.17614746
+348.41030884
+956.42248535
+252.87858582
+435.55230713
+329.51501465
+434.74572754
+154.38674927
+520.62304688
+562.26965332
+494.76245117
+174.66748047
+436.70803833
+135.57254028
+494.28051758
+87.53215790
+435.90051270
+213.83657837
+57.93138885
+377.68255615
+347.87652588
+48.66077423
+989.79504395
+291.43624878
+522.76562500
+619.18884277
+1225.72314453
+533.32568359
+844.25756836
+289.89556885
+1653.40588379
+144.93725586
+1802.95410156
+678.34832764
+870.25048828
+78.17111969
+1046.43676758
+164.60942078
+609.03454590
+87.29292297
+435.67877197
+125.71372223
+261.20431519
+184.11248779
+260.97955322
+212.86184692
+1336.96362305
+106.14862061
+173.34860229
+115.57688904
+1160.16357422
+609.54388428
diff --git a/tests/training/features/data-weighting/sqlite_word.expected b/tests/training/features/data-weighting/sqlite_word.expected
index fb557e0..2271c82 100644
--- a/tests/training/features/data-weighting/sqlite_word.expected
+++ b/tests/training/features/data-weighting/sqlite_word.expected
@@ -1,14 +1,14 @@
-853.40081787
-710.09143066
-610.75262451
-526.56585693
-439.88232422
-345.80377197
-234.99189758
-606.08709717
-734.31378174
-627.71948242
-544.36926270
-456.04959106
-364.86871338
-264.01800537
+846.28295898
+683.08270264
+546.49383545
+436.32812500
+349.99142456
+268.15167236
+178.71995544
+478.48889160
+563.18597412
+471.57501221
+402.95126343
+333.62921143
+264.42492676
+190.03326416
diff --git a/tests/training/features/data-weighting/test_maxi_batches_with_sentence_weights.sh b/tests/training/features/data-weighting/test_maxi_batches_with_sentence_weights.sh
index 58e9e82..bec6015 100644
--- a/tests/training/features/data-weighting/test_maxi_batches_with_sentence_weights.sh
+++ b/tests/training/features/data-weighting/test_maxi_batches_with_sentence_weights.sh
@@ -16,7 +16,7 @@ test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corp
 test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml
 
 $MRT_MARIAN/marian \
-    --seed 3333 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \
+    --seed 3333 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd \
     -m maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 --cost-type ce-sum --disp-label-counts false \
     --data-weighting train.1k.inc.txt --data-weighting-type sentence
diff --git a/tests/training/features/data-weighting/test_maxi_batches_with_word_weights.sh b/tests/training/features/data-weighting/test_maxi_batches_with_word_weights.sh
index 5cb5592..55bf916 100644
--- a/tests/training/features/data-weighting/test_maxi_batches_with_word_weights.sh
+++ b/tests/training/features/data-weighting/test_maxi_batches_with_word_weights.sh
@@ -16,7 +16,7 @@ test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corp
 test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml
 
 $MRT_MARIAN/marian \
-    --seed 6666 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \
+    --seed 6666 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd \
     -m word_maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log word_maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 --cost-type ce-mean \
     --data-weighting train.1k.wordinc.txt --data-weighting-type word
diff --git a/tests/training/features/data-weighting/test_word_weighting_sqlite.sh b/tests/training/features/data-weighting/test_word_weighting_sqlite.sh
index fe399c7..4518885 100644
--- a/tests/training/features/data-weighting/test_word_weighting_sqlite.sh
+++ b/tests/training/features/data-weighting/test_word_weighting_sqlite.sh
@@ -15,7 +15,7 @@ mkdir -p sqlite_word
 cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/2/g' > sqlite_word.weights.txt
 
 $MRT_MARIAN/marian \
-    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
+    --seed 1111 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
     -m sqlite_word/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log sqlite_word.log --disp-freq 5 -e 2 --mini-batch-fit -w 500 \
     --data-weighting sqlite_word.weights.txt --data-weighting-type word --sqlite sqlite_word/corpus.sqlite3
diff --git a/tests/training/features/data-weighting/test_word_weighting_with_eos.sh b/tests/training/features/data-weighting/test_word_weighting_with_eos.sh
index c3d7b93..2b7c3dd 100644
--- a/tests/training/features/data-weighting/test_word_weighting_with_eos.sh
+++ b/tests/training/features/data-weighting/test_word_weighting_with_eos.sh
@@ -17,7 +17,7 @@ cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r -e 's/[^ ]+/2/g' -e 's/$/ 2/' >
 
 # Train
 $MRT_MARIAN/marian \
-    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
+    --seed 1111 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
     -m word_eos/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log word_eos.log --disp-freq 5 -e 2 \
     --data-weighting word_eos.weights.txt --data-weighting-type word
diff --git a/tests/training/features/data-weighting/test_word_weighting_with_twos.sh b/tests/training/features/data-weighting/test_word_weighting_with_twos.sh
index 0660501..b514b28 100644
--- a/tests/training/features/data-weighting/test_word_weighting_with_twos.sh
+++ b/tests/training/features/data-weighting/test_word_weighting_with_twos.sh
@@ -18,7 +18,7 @@ cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/2/g' > word_twos.weigh
 
 # Train with word weighting
 $MRT_MARIAN/marian \
-    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
+    --seed 1111 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
     -m word_twos/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log word_twos.log --disp-freq 5 -e 2 \
     --data-weighting word_twos.weights.txt --data-weighting-type word
@@ -41,7 +41,7 @@ echo "data-weighting-type: word" >> word_twos.config.yml
 
 # Train with word weighting
 $MRT_MARIAN/marian \
-    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
+    --seed 1111 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
     -m word_twos_cfg/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log word_twos_cfg.log --disp-freq 5 -e 2 \
     -c word_twos.config.yml
diff --git a/tests/training/features/data-weighting/test_word_weighting_with_twos_sync.sh b/tests/training/features/data-weighting/test_word_weighting_with_twos_sync.sh
index 675ae8b..bf6d753 100644
--- a/tests/training/features/data-weighting/test_word_weighting_with_twos_sync.sh
+++ b/tests/training/features/data-weighting/test_word_weighting_with_twos_sync.sh
@@ -18,7 +18,7 @@ cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/2/g' > word_twos_sync.
 
 # Train with word weighting
 $MRT_MARIAN/marian \
-    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
+    --seed 1111 --no-shuffle --clip-norm 0 --dim-emb 128 --dim-rnn 256 --optimizer sgd --cost-type ce-mean \
     -m word_twos_sync/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml --sync-sgd \
     --log word_twos_sync.log --disp-freq 5 -e 2 \
     --data-weighting word_twos_sync.weights.txt --data-weighting-type word
diff --git a/tests/training/features/data-weighting/word_eos.expected b/tests/training/features/data-weighting/word_eos.expected
index a4ec027..da2dc29 100644
--- a/tests/training/features/data-weighting/word_eos.expected
+++ b/tests/training/features/data-weighting/word_eos.expected
@@ -1,17 +1,17 @@
-Ep. 1 : Up. 5 : Sen. 320 : Cost 856.40283203
-Ep. 1 : Up. 10 : Sen. 640 : Cost 705.00976562
-Ep. 1 : Up. 15 : Sen. 960 : Cost 604.22814941
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 518.43249512
-Ep. 1 : Up. 25 : Sen. 1,600 : Cost 442.49536133
-Ep. 1 : Up. 30 : Sen. 1,920 : Cost 367.35723877
-Ep. 1 : Up. 35 : Sen. 2,240 : Cost 301.55618286
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 230.20394897
-Ep. 2 : Up. 45 : Sen. 64 : Cost 340.81048584
-Ep. 2 : Up. 50 : Sen. 384 : Cost 818.84265137
-Ep. 2 : Up. 55 : Sen. 704 : Cost 681.78875732
-Ep. 2 : Up. 60 : Sen. 1,024 : Cost 587.13653564
-Ep. 2 : Up. 65 : Sen. 1,344 : Cost 501.12982178
-Ep. 2 : Up. 70 : Sen. 1,664 : Cost 427.36920166
-Ep. 2 : Up. 75 : Sen. 1,984 : Cost 354.46206665
-Ep. 2 : Up. 80 : Sen. 2,304 : Cost 287.68417358
-Ep. 2 : Up. 85 : Sen. 2,624 : Cost 212.97563171
+Ep. 1 : Up. 5 : Sen. 320 : Cost 848.23455811
+Ep. 1 : Up. 10 : Sen. 640 : Cost 672.56451416
+Ep. 1 : Up. 15 : Sen. 960 : Cost 525.11682129
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 418.14608765
+Ep. 1 : Up. 25 : Sen. 1,600 : Cost 345.36917114
+Ep. 1 : Up. 30 : Sen. 1,920 : Cost 280.52749634
+Ep. 1 : Up. 35 : Sen. 2,240 : Cost 227.94680786
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 167.04818726
+Ep. 2 : Up. 45 : Sen. 64 : Cost 262.95532227
+Ep. 2 : Up. 50 : Sen. 384 : Cost 660.01922607
+Ep. 2 : Up. 55 : Sen. 704 : Cost 521.61163330
+Ep. 2 : Up. 60 : Sen. 1,024 : Cost 435.67529297
+Ep. 2 : Up. 65 : Sen. 1,344 : Cost 364.16458130
+Ep. 2 : Up. 70 : Sen. 1,664 : Cost 305.08660889
+Ep. 2 : Up. 75 : Sen. 1,984 : Cost 250.36157227
+Ep. 2 : Up. 80 : Sen. 2,304 : Cost 204.07473755
+Ep. 2 : Up. 85 : Sen. 2,624 : Cost 142.30810547
diff --git a/tests/training/features/data-weighting/word_maxibatch.expected b/tests/training/features/data-weighting/word_maxibatch.expected
index 542de08..c71c457 100644
--- a/tests/training/features/data-weighting/word_maxibatch.expected
+++ b/tests/training/features/data-weighting/word_maxibatch.expected
@@ -1,10 +1,10 @@
-493.42471313
-355.55953979
-272.83404541
-199.24537659
-131.18077087
-386.23254395
-388.55151367
-296.93032837
-220.35517883
-152.07803345
+491.56161499
+351.35723877
+267.88531494
+194.60379028
+127.38488770
+371.88699341
+366.62310791
+272.43316650
+197.15826416
+132.87835693
diff --git a/tests/training/features/data-weighting/word_twos.expected b/tests/training/features/data-weighting/word_twos.expected
index cc235d0..95d13cc 100644
--- a/tests/training/features/data-weighting/word_twos.expected
+++ b/tests/training/features/data-weighting/word_twos.expected
@@ -1,17 +1,17 @@
-Ep. 1 : Up. 5 : Sen. 320 : Cost 846.69714355 :
-Ep. 1 : Up. 10 : Sen. 640 : Cost 695.30053711 :
-Ep. 1 : Up. 15 : Sen. 960 : Cost 594.51928711 :
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 508.72247314 :
-Ep. 1 : Up. 25 : Sen. 1,600 : Cost 432.78329468 :
-Ep. 1 : Up. 30 : Sen. 1,920 : Cost 357.64947510 :
-Ep. 1 : Up. 35 : Sen. 2,240 : Cost 291.84161377 :
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 220.49028015 :
-Ep. 2 : Up. 45 : Sen. 64 : Cost 331.08535767 :
-Ep. 2 : Up. 50 : Sen. 384 : Cost 809.13928223 :
-Ep. 2 : Up. 55 : Sen. 704 : Cost 672.08361816 :
-Ep. 2 : Up. 60 : Sen. 1,024 : Cost 577.43341064 :
-Ep. 2 : Up. 65 : Sen. 1,344 : Cost 491.42279053 :
-Ep. 2 : Up. 70 : Sen. 1,664 : Cost 417.66470337 :
-Ep. 2 : Up. 75 : Sen. 1,984 : Cost 344.76025391 :
-Ep. 2 : Up. 80 : Sen. 2,304 : Cost 277.97634888 :
-Ep. 2 : Up. 85 : Sen. 2,624 : Cost 203.26664734 :
+Ep. 1 : Up. 5 : Sen. 320 : Cost 838.97186279 :
+Ep. 1 : Up. 10 : Sen. 640 : Cost 665.44097900 :
+Ep. 1 : Up. 15 : Sen. 960 : Cost 523.22821045 :
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 417.61639404 :
+Ep. 1 : Up. 25 : Sen. 1,600 : Cost 343.39797974 :
+Ep. 1 : Up. 30 : Sen. 1,920 : Cost 278.35540771 :
+Ep. 1 : Up. 35 : Sen. 2,240 : Cost 225.92178345 :
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 165.37797546 :
+Ep. 2 : Up. 45 : Sen. 64 : Cost 257.18948364 :
+Ep. 2 : Up. 50 : Sen. 384 : Cost 635.78594971 :
+Ep. 2 : Up. 55 : Sen. 704 : Cost 507.77557373 :
+Ep. 2 : Up. 60 : Sen. 1,024 : Cost 431.42156982 :
+Ep. 2 : Up. 65 : Sen. 1,344 : Cost 361.39825439 :
+Ep. 2 : Up. 70 : Sen. 1,664 : Cost 302.86456299 :
+Ep. 2 : Up. 75 : Sen. 1,984 : Cost 248.74520874 :
+Ep. 2 : Up. 80 : Sen. 2,304 : Cost 203.10728455 :
+Ep. 2 : Up. 85 : Sen. 2,624 : Cost 141.87115479 :
diff --git a/tests/training/features/data-weighting/word_twos_sync.expected b/tests/training/features/data-weighting/word_twos_sync.expected
index df4c0d5..f199178 100644
--- a/tests/training/features/data-weighting/word_twos_sync.expected
+++ b/tests/training/features/data-weighting/word_twos_sync.expected
@@ -1,17 +1,17 @@
-Ep. 1 : Up. 5 : Sen. 320 : Cost 846.69714355 :
-Ep. 1 : Up. 10 : Sen. 640 : Cost 695.30053711 :
-Ep. 1 : Up. 15 : Sen. 960 : Cost 594.51928711 :
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 508.72241211 :
-Ep. 1 : Up. 25 : Sen. 1,600 : Cost 432.78320312 :
-Ep. 1 : Up. 30 : Sen. 1,920 : Cost 357.64950562 :
-Ep. 1 : Up. 35 : Sen. 2,240 : Cost 291.84161377 :
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 220.49028015 :
-Ep. 2 : Up. 45 : Sen. 64 : Cost 331.08532715 :
-Ep. 2 : Up. 50 : Sen. 384 : Cost 809.13928223 :
-Ep. 2 : Up. 55 : Sen. 704 : Cost 672.08367920 :
-Ep. 2 : Up. 60 : Sen. 1,024 : Cost 577.43341064 :
-Ep. 2 : Up. 65 : Sen. 1,344 : Cost 491.42279053 :
-Ep. 2 : Up. 70 : Sen. 1,664 : Cost 417.66470337 :
-Ep. 2 : Up. 75 : Sen. 1,984 : Cost 344.76025391 :
-Ep. 2 : Up. 80 : Sen. 2,304 : Cost 277.97634888 :
-Ep. 2 : Up. 85 : Sen. 2,624 : Cost 203.26664734 :
+Ep. 1 : Up. 5 : Sen. 320 : Cost 838.97186279 :
+Ep. 1 : Up. 10 : Sen. 640 : Cost 665.44097900 :
+Ep. 1 : Up. 15 : Sen. 960 : Cost 523.22821045 :
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 417.61639404 :
+Ep. 1 : Up. 25 : Sen. 1,600 : Cost 343.39797974 :
+Ep. 1 : Up. 30 : Sen. 1,920 : Cost 278.35540771 :
+Ep. 1 : Up. 35 : Sen. 2,240 : Cost 225.92178345 :
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 165.37797546 :
+Ep. 2 : Up. 45 : Sen. 64 : Cost 257.18945312 :
+Ep. 2 : Up. 50 : Sen. 384 : Cost 635.78594971 :
+Ep. 2 : Up. 55 : Sen. 704 : Cost 507.77557373 :
+Ep. 2 : Up. 60 : Sen. 1,024 : Cost 431.42156982 :
+Ep. 2 : Up. 65 : Sen. 1,344 : Cost 361.39825439 :
+Ep. 2 : Up. 70 : Sen. 1,664 : Cost 302.86456299 :
+Ep. 2 : Up. 75 : Sen. 1,984 : Cost 248.74520874 :
+Ep. 2 : Up. 80 : Sen. 2,304 : Cost 203.10728455 :
+Ep. 2 : Up. 85 : Sen. 2,624 : Cost 141.87115479 :
diff --git a/tests/training/features/exp-smoothing/test_expsmooth.sh b/tests/training/features/exp-smoothing/test_expsmooth.sh
index b27cee9..69d7072 100644
--- a/tests/training/features/exp-smoothing/test_expsmooth.sh
+++ b/tests/training/features/exp-smoothing/test_expsmooth.sh
@@ -1,5 +1,9 @@
 #!/bin/bash -x
 
+#####################################################################
+# TAGS: clip-norm
+#####################################################################
+
 # Exit on error
 set -e
 
@@ -8,7 +12,7 @@ rm -rf expsmooth expsmooth*.log
 mkdir -p expsmooth
 
 
-opts="--no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer sgd --learn-rate 0.5 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32 --cost-type ce-mean"
+opts="--no-shuffle --clip-norm 1 --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer sgd --learn-rate 0.5 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32 --cost-type ce-mean"
 
 # No exponential smoothing
 $MRT_MARIAN/marian \
diff --git a/tests/training/features/exp-smoothing/test_expsmooth_sync.sh b/tests/training/features/exp-smoothing/test_expsmooth_sync.sh
index 3bab8ee..29e2978 100644
--- a/tests/training/features/exp-smoothing/test_expsmooth_sync.sh
+++ b/tests/training/features/exp-smoothing/test_expsmooth_sync.sh
@@ -13,11 +13,11 @@ rm -rf expsmooth_sync expsmooth_sync*.log
 mkdir -p expsmooth_sync
 
 
-opts="--no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer adam --learn-rate 0.0001 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32 --devices 0 1 --sync-sgd"
+opts="--no-shuffle --clip-norm 0 --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer adam --learn-rate 0.0001 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32 --devices 0 1 --sync-sgd"
 
 # No exponential smoothing
 $MRT_MARIAN/marian \
-    -m expsmooth_sync/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml --clip-norm 0 --cost-type ce-mean-words \
+    -m expsmooth_sync/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml --cost-type ce-mean-words \
     --disp-freq 20 --valid-freq 20 --after-batches 200 $opts \
     --log expsmooth_sync_0.log
 
@@ -30,7 +30,7 @@ cat expsmooth_sync_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep
 
 # With exponential smoothing
 $MRT_MARIAN/marian \
-    -m expsmooth_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml --clip-norm 0 --cost-type ce-mean-words \
+    -m expsmooth_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml --cost-type ce-mean-words \
     --disp-freq 20 --valid-freq 20 --after-batches 200 --exponential-smoothing 0.0001 $opts \
     --log expsmooth_sync.log
 
diff --git a/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh b/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh
index a022e5c..925f36f 100644
--- a/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh
+++ b/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh
@@ -3,7 +3,7 @@
 #####################################################################
 # SUMMARY: Training S2S model with guided alignment
 # AUTHOR: snukky
-# TAGS: align rnn
+# TAGS: align rnn clip-norm
 #####################################################################
 
 # Exit on error
@@ -15,7 +15,7 @@ mkdir -p rnn
 
 # Run marian command
 $MRT_MARIAN/marian \
-    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
+    --no-shuffle --clip-norm 1 --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
     -m rnn/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
     --after-batches 100 --disp-freq 10 \
     --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \
diff --git a/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh b/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh
index f5f18b9..cd28f1c 100644
--- a/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh
+++ b/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh
@@ -15,7 +15,7 @@ mkdir -p transformer
 
 # Run marian command
 $MRT_MARIAN/marian --type transformer \
-    --no-shuffle --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
+    --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
     -m transformer/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
     --after-batches 100 --disp-freq 10 \
     --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \
diff --git a/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh b/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh
index 49675e8..963052d 100644
--- a/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh
+++ b/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh
@@ -15,7 +15,7 @@ mkdir -p transformer_sync
 
 # Run marian command
 $MRT_MARIAN/marian --type transformer \
-    --no-shuffle --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean --sync-sgd \
+    --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean --sync-sgd \
     -m transformer_sync/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
     --after-batches 100 --disp-freq 10 \
     --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \
diff --git a/tests/training/features/guided-alignment/transformer.expected b/tests/training/features/guided-alignment/transformer.expected
index 2513e28..e69de29 100644
--- a/tests/training/features/guided-alignment/transformer.expected
+++ b/tests/training/features/guided-alignment/transformer.expected
@@ -1,10 +0,0 @@
-244.42282104
-256.56842041
-238.89138794
-233.57333374
-223.44998169
-204.23277283
-232.98970032
-204.28886414
-215.29394531
-201.92327881
diff --git a/tests/training/features/mixed-ensembles/s2s_transf.expected b/tests/training/features/mixed-ensembles/s2s_transf.expected
index 1aba12f..3f2ff2d 100644
--- a/tests/training/features/mixed-ensembles/s2s_transf.expected
+++ b/tests/training/features/mixed-ensembles/s2s_transf.expected
@@ -1,5 +1,5 @@
-herrsch@@ Binnengrenzen gli@@ Borrell nische Millennium nun@@ Millennium nun@@ waren gessen@@ tentei@@ 41 typ@@ rig aufweisen ethn@@ Baum@@ nahe Unter@@ Unterzeichnung teure Wohl itäten ausgewogene Pläne persönliche agieren Meeres@@ persönliche agieren Meeres@@ persönliche Schlußfolgerungen Unterschied Fe@@ Unter@@ Somm@@ Pläne persönliche rain wunder@@ extended persönliche rain
-herrsch@@ Binnengrenzen Vorsitzes Pläne Kön@@ unterstützte tei@@ whol@@ Millennium wenngleich Kön@@ unterstützte tei@@ whol@@ VAT operator ethn@@ Baum@@ nahe Hague CI@@ COD rain will Kommissarin aush@@ wecken ASEM Konzep@@ Demokratisierungs@@ abzuwarten Voraussetzungen Kommissionspräsidenten unterbrochen COD Napole@@ Tür@@ log@@ Varela log@@ regulations wecken extreme Woh@@ log@@ Varela log@@ Varela
-waren gessen@@ nehme gli@@ cut@@ ethn@@ Baum@@ lich ca. EPL@@ ca. cycle tive ely Pazi@@ eben@@ agents ethn@@ agieren ethn@@ agieren Meeres@@ brachte Umweltverträglichkeitsprü@@ oring Genuss agieren ethn@@ agieren Meeres@@ ethn@@ agieren Meeres@@ nü@@ Instan@@ Geflügel@@ ahn finanziell bund@@ fortführen reform@@ Einklang need extreme agents
-herrsch@@ Binnengrenzen Binnengrenzen Binnengrenzen Dele@@ Tan@@ Texten Texten Texten Texten Texten Texten Texten wo nonsense thal@@ Sk@@ ethn@@ Baum@@ nahe nützlichen Konfrontation zielen Positionen mes@@ cor@@ Statistiken herrsch@@ Binnengrenzen nonsense reform just genügt erregend menschliche netz erregend menschliche tbewer@@ maj@@ coa log@@ Varela log@@ Varela log@@ Varela log@@ Varela log@@ Varela Budge@@ alitä@@ fit
-Statistiken rig ASEM Papier@@ ethn@@ itäts@@ zusätzlich itäts@@ zusätzlich itäts@@ zusätzlich itäts@@ zusätzlich will Capp@@ break ethn@@ agieren ethn@@ agieren ethn@@ agieren ethn@@ agieren
+
+
+
+
+
diff --git a/tests/training/features/mixed-ensembles/test_ensemble_of_different_s2s.sh b/tests/training/features/mixed-ensembles/test_ensemble_of_different_s2s.sh
index 42e231b..539956d 100644
--- a/tests/training/features/mixed-ensembles/test_ensemble_of_different_s2s.sh
+++ b/tests/training/features/mixed-ensembles/test_ensemble_of_different_s2s.sh
@@ -13,7 +13,7 @@ set -e
 rm -rf two_s2s two_s2s*.log
 mkdir -p two_s2s
 
-options="--no-shuffle --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 100"
+options="--no-shuffle --clip-norm 0 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 60"
 
 # Train model A
 $MRT_MARIAN/marian \
diff --git a/tests/training/features/mixed-ensembles/test_ensemble_of_s2s_and_transformer.sh b/tests/training/features/mixed-ensembles/test_ensemble_of_s2s_and_transformer.sh
index a80fe32..fe33de3 100644
--- a/tests/training/features/mixed-ensembles/test_ensemble_of_s2s_and_transformer.sh
+++ b/tests/training/features/mixed-ensembles/test_ensemble_of_s2s_and_transformer.sh
@@ -7,7 +7,7 @@ set -e
 rm -rf s2s_transf s2s_transf*.log
 mkdir -p s2s_transf
 
-options="--no-shuffle --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 100"
+options="--no-shuffle --clip-norm 0 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 60"
 
 # Train model A
 $MRT_MARIAN/marian \
diff --git a/tests/training/features/mixed-ensembles/two_s2s.expected b/tests/training/features/mixed-ensembles/two_s2s.expected
index 9cfe9b0..fe80305 100644
--- a/tests/training/features/mixed-ensembles/two_s2s.expected
+++ b/tests/training/features/mixed-ensembles/two_s2s.expected
@@ -1,5 +1,5 @@
-umgewandelt Davies Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt Standpunkt
-staff@@ thalten Blut@@ beschä@@ trade Entlastungsverfahren Strafgerichtshof versu@@ gla@@ Jose@@ trade Entlastungsverfahren Strafgerichtshof versu@@ gla@@ Jose@@ Einfuhren Jose@@ trade Entlastungsverfahren Koordination ω@@ Hilfest@@ versu@@ gla@@ Jose@@ Einfuhren Jose@@ trade Entlastungsverfahren Koordination ω@@ Hilfest@@ versu@@ gla@@ Jose@@ Einfuhren Jose@@ trade Entlastungsverfahren Koordination ω@@ Hilfest@@ versu@@ gla@@ Jose@@ Einfuhren ärz@@
-Hed@@ Warrant Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt schriftlichen Hed@@ GM@@ Lebensunterhalt
-tests Beihilfen General les gerich@@ verwendet Betrieben verei Prognosen les gerich@@ gerich@@ verwendet Betrieben anhaltenden freue day ismen Schle@@ gewünschte Nahrungsmittelhilfe sur Forscher les gerich@@ gerich@@ verwendet Betrieben verei Prognosen les gerich@@ gerich@@ verwendet Betrieben anhaltenden freue day ismen Schle@@ gewünschte Nahrungsmittelhilfe sur Forscher les gerich@@ gerich@@ verwendet Betrieben verei Prognosen les gerich@@ verwendet
-Sal@@ Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@ fil Υ@@
+.
+.
+.
+.
+.
diff --git a/tests/training/features/right-left/rnn.expected b/tests/training/features/right-left/rnn.expected
index c683efb..ff302f0 100644
--- a/tests/training/features/right-left/rnn.expected
+++ b/tests/training/features/right-left/rnn.expected
@@ -1,10 +1,10 @@
-227.26374817
-251.25552368
-244.43490601
-247.96240234
-242.51679993
-239.25460815
-236.51896667
-231.50540161
-238.35562134
-242.17578125
+226.89152527
+249.98703003
+242.43225098
+245.21345520
+239.08744812
+234.52084351
+230.54391479
+224.25790405
+228.97502136
+230.70504761
diff --git a/tests/training/features/right-left/test_right_left_rnn.sh b/tests/training/features/right-left/test_right_left_rnn.sh
index 245125e..ae3976b 100644
--- a/tests/training/features/right-left/test_right_left_rnn.sh
+++ b/tests/training/features/right-left/test_right_left_rnn.sh
@@ -14,7 +14,7 @@ mkdir -p rnn
 
 # Run marian command
 $MRT_MARIAN/marian \
-    --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
+    --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
     -m rnn/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
     --after-batches 100 --disp-freq 10 \
     --right-left --log rnn.log
diff --git a/tests/training/features/right-left/test_right_left_transformer.sh b/tests/training/features/right-left/test_right_left_transformer.sh
index 8a40bdd..2939c3c 100644
--- a/tests/training/features/right-left/test_right_left_transformer.sh
+++ b/tests/training/features/right-left/test_right_left_transformer.sh
@@ -14,7 +14,7 @@ mkdir -p transformer
 
 # Run marian command
 $MRT_MARIAN/marian --type transformer \
-    --no-shuffle --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
+    --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \
     -m transformer/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \
     --after-batches 100 --disp-freq 10 \
     --right-left --log transformer.log
diff --git a/tests/training/features/right-left/transformer.expected b/tests/training/features/right-left/transformer.expected
index a63420f..0fc2662 100644
--- a/tests/training/features/right-left/transformer.expected
+++ b/tests/training/features/right-left/transformer.expected
@@ -1,10 +1,10 @@
-237.99105835
-263.23455811
-255.45816040
-259.72146606
-254.48379517
-250.45918274
-248.04586792
-242.33943176
-249.78984070
-253.30130005
+233.16964722
+249.00422668
+235.84651184
+234.45532227
+225.37080383
+216.76550293
+210.73200989
+202.66915894
+207.25146484
+209.57803345