changing expected values due to changes in output layers

54 files changed, 545 insertions, 552 deletions

diff --git a/tests/training/basics/gzip.expected b/tests/training/basics/gzip.expected
index c6960ce..47c8765 100644
--- a/tests/training/basics/gzip.expected
+++ b/tests/training/basics/gzip.expected
@@ -1,5 +1,5 @@
-447.84643555
-374.76788330
-324.78485107
-284.22189331
-248.39517212
+448.35955811
+374.86718750
+324.67779541
+283.97842407
+248.12561035
diff --git a/tests/training/basics/toy.expected b/tests/training/basics/toy.expected
index f5094ef..4e03c75 100644
--- a/tests/training/basics/toy.expected
+++ b/tests/training/basics/toy.expected
@@ -1,44 +1,44 @@
-207.97480774
-179.29321289
-212.48278809
-236.76557922
-185.84127808
-233.23265076
-208.95817566
-217.90177917
-183.83541870
-201.72744751
-148.79904175
-269.31390381
-217.85786438
-130.89492798
-121.50901794
-134.44161987
-177.97956848
-143.88101196
-181.38346863
-101.61656189
-126.42018890
-141.65258789
-213.97846985
-245.28518677
-169.17837524
-102.50040436
-115.88985443
-176.02175903
-128.50570679
-138.32833862
-123.53054047
-213.36453247
-148.53483582
-198.26582336
-135.98811340
-173.58190918
-181.12872314
-187.30583191
-129.51103210
-126.53293610
-162.27357483
-95.55107117
-133.98471069
-206.47187805
+207.82556152
+178.71255493
+211.33903503
+234.55387878
+182.80172729
+227.75459290
+200.35102844
+207.02507019
+174.51194763
+191.75926208
+141.59402466
+256.85244751
+208.20707703
+125.15594482
+116.06490326
+129.00891113
+172.10009766
+138.76477051
+176.41941833
+98.28771210
+123.42759705
+138.35072327
+210.30610657
+241.78837585
+166.28385925
+99.98839569
+113.32307434
+173.32617188
+125.69969177
+135.87945557
+120.28639984
+209.76863098
+145.23466492
+195.52755737
+132.37802124
+169.65127563
+177.24984741
+182.49871826
+125.44963074
+122.40126038
+157.54493713
+91.15261841
+129.87872314
+200.61141968
diff --git a/tests/training/basics/trans.expected b/tests/training/basics/trans.expected
index 6b7678b..68987df 100644
--- a/tests/training/basics/trans.expected
+++ b/tests/training/basics/trans.expected
@@ -1,4 +1,4 @@
-[valid] Ep. 1 : Up. 60 : cross-entropy : 221.035 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 218.835 : new best
 [valid] Ep. 1 : Up. 60 : translation : 1 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 195.984 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 187.91 : new best
 [valid] Ep. 1 : Up. 120 : translation : 2 : new best
diff --git a/tests/training/basics/valid.expected b/tests/training/basics/valid.expected
index 63aac29..66764ec 100644
--- a/tests/training/basics/valid.expected
+++ b/tests/training/basics/valid.expected
@@ -1,10 +1,10 @@
-[valid] Ep. 1 : Up. 15 : cross-entropy : 308.376 : new best
+[valid] Ep. 1 : Up. 15 : cross-entropy : 307.647 : new best
 [valid] Ep. 1 : Up. 15 : valid-script : 1 : new best
-[valid] Ep. 1 : Up. 30 : cross-entropy : 307.337 : new best
+[valid] Ep. 1 : Up. 30 : cross-entropy : 305.551 : new best
 [valid] Ep. 1 : Up. 30 : valid-script : 2 : new best
-[valid] Ep. 1 : Up. 45 : cross-entropy : 301.132 : new best
+[valid] Ep. 1 : Up. 45 : cross-entropy : 299.442 : new best
 [valid] Ep. 1 : Up. 45 : valid-script : 3 : new best
-[valid] Ep. 1 : Up. 60 : cross-entropy : 285.37 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 281.549 : new best
 [valid] Ep. 1 : Up. 60 : valid-script : 4 : new best
-[valid] Ep. 1 : Up. 75 : cross-entropy : 275.612 : new best
+[valid] Ep. 1 : Up. 75 : cross-entropy : 268.403 : new best
 [valid] Ep. 1 : Up. 75 : valid-script : 5 : new best
diff --git a/tests/training/cost-functions/ce-mean-words.expected b/tests/training/cost-functions/ce-mean-words.expected
index f8e42ef..1962514 100644
--- a/tests/training/cost-functions/ce-mean-words.expected
+++ b/tests/training/cost-functions/ce-mean-words.expected
@@ -1,8 +1,8 @@
-8.48374081
-8.47330475
-8.45676517
-8.45006180
-8.42407513
-8.39919662
-8.34003067
-8.26019096
+8.48090363
+8.45937634
+8.42510605
+8.40673161
+8.35613060
+8.30123329
+8.18988895
+8.04098606
diff --git a/tests/training/cost-functions/ce-mean.expected b/tests/training/cost-functions/ce-mean.expected
index 9649b45..10a808e 100644
--- a/tests/training/cost-functions/ce-mean.expected
+++ b/tests/training/cost-functions/ce-mean.expected
@@ -1,8 +1,8 @@
-205.86325073
-203.75646973
-113.10920715
-291.26306152
-150.84355164
-260.24383545
-181.52583313
-247.80560303
+205.79443359
+203.42153931
+112.68572235
+289.76901245
+149.62661743
+257.20617676
+178.25546265
+241.22378540
diff --git a/tests/training/cost-functions/ce-sum.expected b/tests/training/cost-functions/ce-sum.expected
index b108bbe..a164bd1 100644
--- a/tests/training/cost-functions/ce-sum.expected
+++ b/tests/training/cost-functions/ce-sum.expected
@@ -1,8 +1,8 @@
-13175.24804688
-13040.41406250
-7238.98876953
-18640.82226562
-9653.98242188
-16655.56445312
-11617.62695312
-15859.46289062
+13170.84179688
+13018.96679688
+7211.87988281
+18545.13281250
+9576.06738281
+16460.98828125
+11408.13281250
+15437.80957031
diff --git a/tests/training/cost-functions/perplexity.expected b/tests/training/cost-functions/perplexity.expected
index c97e54c..04cbf84 100644
--- a/tests/training/cost-functions/perplexity.expected
+++ b/tests/training/cost-functions/perplexity.expected
@@ -1,8 +1,8 @@
-4835.50488281
-4785.30371094
-4706.80761719
-4675.36181641
-4555.41650391
-4443.47412109
-4188.16210938
-3866.76611328
+4821.80517578
+4719.09570312
+4560.10205078
+4477.03125000
+4256.11279297
+4028.61523438
+3604.00585938
+3105.27124023
diff --git a/tests/training/features/data-weighting/maxibatch.expected b/tests/training/features/data-weighting/maxibatch.expected
index 72fbb73..7c0001c 100644
--- a/tests/training/features/data-weighting/maxibatch.expected
+++ b/tests/training/features/data-weighting/maxibatch.expected
@@ -1,10 +1,10 @@
-6921.77050781
-5224.29541016
-4134.13769531
-3090.62255859
-2086.21093750
-5387.22900391
-5651.76074219
-4459.25537109
-3396.93627930
-2398.67187500
+6924.51171875
+5225.91162109
+4136.33691406
+3092.30273438
+2086.31420898
+5388.93750000
+5653.18310547
+4460.88183594
+3398.95581055
+2398.78735352
diff --git a/tests/training/features/data-weighting/sqlite.expected b/tests/training/features/data-weighting/sqlite.expected
index e5dc23a..d2f0f31 100644
--- a/tests/training/features/data-weighting/sqlite.expected
+++ b/tests/training/features/data-weighting/sqlite.expected
@@ -1,100 +1,100 @@
-145.76625061
-407.85418701
-1194.96923828
-233.20181274
-1427.79321289
-126.29446411
-378.78878784
-97.17931366
-757.44409180
-330.21224976
-1602.50830078
-213.69992065
-437.10656738
-97.17844391
-466.32504272
-194.22283936
-1194.76013184
-145.61857605
-932.43957520
-388.58657837
-932.55883789
-621.67999268
-873.99902344
-825.64160156
-437.16613770
-155.39439392
-495.47885132
-233.18109131
-349.39471436
-320.56796265
-612.07836914
-670.26416016
-1777.50268555
-242.84895325
-1369.95361328
-136.01496887
-1515.54931641
-184.50100708
-815.76287842
-135.94961548
-728.80718994
-174.92292786
-408.04016113
-213.76396179
-232.92919922
-135.99600220
-291.41558838
-165.11375427
-582.64404297
-349.64080811
-961.79394531
-252.50859070
-437.06246948
-330.19354248
-436.77810669
-155.42767334
-524.70916748
-563.38171387
-495.48129272
-174.89411926
-436.91708374
-135.98927307
-495.24850464
-87.39509583
-437.12753296
-213.65098572
-58.25393295
-378.78936768
-349.33700562
-48.53756332
-990.68878174
-291.55676270
-524.49810791
-621.72137451
-1223.91223145
-534.35400391
-845.23986816
-291.38967896
-1661.09838867
-145.75514221
-1806.95227051
-680.00848389
-874.08618164
-77.69504547
-1048.91845703
-165.15805054
-611.82751465
-87.41978455
-437.08914185
-126.29270172
-262.40588379
-184.46842957
-262.17800903
-213.70729065
-1340.37475586
-106.82720947
-174.82476807
-116.54352570
-1165.54418945
-611.97637939
+145.22386169
+408.22799683
+1194.63964844
+233.55360413
+1430.38696289
+126.14705658
+378.98550415
+96.71858215
+757.48663330
+330.56832886
+1604.02294922
+214.05000305
+438.19305420
+96.71667480
+466.88586426
+194.51495361
+1197.85363770
+146.07209778
+933.41638184
+389.17184448
+933.05151367
+622.52008057
+874.42907715
+825.77954102
+437.69128418
+155.24894714
+496.18453979
+232.95338440
+350.40795898
+321.22961426
+610.87152100
+670.83068848
+1778.17529297
+241.61517334
+1370.66113281
+136.27859497
+1510.65393066
+184.96159363
+815.58801270
+136.21365356
+729.38146973
+174.92094421
+408.49716187
+213.90168762
+233.55191040
+136.20471191
+290.47631836
+165.94480896
+584.11578369
+349.23815918
+958.81738281
+253.41488647
+437.03112793
+330.26031494
+435.45944214
+154.90957642
+521.61523438
+563.37170410
+496.03311157
+175.29434204
+438.16888428
+135.96989441
+496.29870605
+87.74061584
+436.65737915
+214.36607361
+58.15562820
+378.97769165
+348.81768799
+48.82063675
+993.13079834
+292.51773071
+524.27105713
+620.84045410
+1228.60827637
+535.02069092
+846.56628418
+291.05285645
+1658.58984375
+145.76992798
+1809.52172852
+680.63385010
+875.07159424
+78.35388947
+1049.29785156
+165.48069763
+611.14044189
+87.67588806
+438.09893799
+126.03882599
+262.80325317
+184.69609070
+262.46411133
+213.61804199
+1341.65759277
+106.89002228
+174.81327820
+116.23052979
+1165.48278809
+612.53643799
diff --git a/tests/training/features/data-weighting/sqlite_word.expected b/tests/training/features/data-weighting/sqlite_word.expected
index 3f43848..037b678 100644
--- a/tests/training/features/data-weighting/sqlite_word.expected
+++ b/tests/training/features/data-weighting/sqlite_word.expected
@@ -1,15 +1,15 @@
-861.55938721
-727.98687744
-630.16650391
-553.84191895
-469.53985596
-387.43038940
-299.73706055
-257.13076782
-828.24053955
-706.25305176
-615.00897217
-536.81494141
-454.13830566
-369.39624023
-274.19537354
+862.09051514
+728.33001709
+630.45318604
+554.12792969
+469.74276733
+387.57510376
+299.93112183
+257.35052490
+828.63208008
+706.60028076
+615.24633789
+537.03155518
+454.32131958
+369.54190063
+274.37387085
diff --git a/tests/training/features/data-weighting/train.expected b/tests/training/features/data-weighting/train.expected
index cd3f965..32e8451 100644
--- a/tests/training/features/data-weighting/train.expected
+++ b/tests/training/features/data-weighting/train.expected
@@ -1,10 +1,10 @@
-473.66577148
-495.55804443
-504.03683472
-498.77752686
-494.78051758
-475.22576904
-518.50439453
-495.43505859
-467.06631470
-523.22912598
+474.05703735
+495.60043335
+504.07754517
+498.34414673
+493.95330811
+474.32269287
+517.29327393
+493.80380249
+465.55847168
+521.18237305
diff --git a/tests/training/features/data-weighting/valid.expected b/tests/training/features/data-weighting/valid.expected
index 6afb154..c2937c9 100644
--- a/tests/training/features/data-weighting/valid.expected
+++ b/tests/training/features/data-weighting/valid.expected
@@ -1,8 +1,8 @@
-[valid] Ep. 2 : Up. 15 : cross-entropy : 277.031 : new best
+[valid] Ep. 2 : Up. 15 : cross-entropy : 277.417 : new best
 [valid] Ep. 2 : Up. 15 : valid-script : 1 : new best
-[valid] Ep. 3 : Up. 30 : cross-entropy : 276.978 : new best
+[valid] Ep. 3 : Up. 30 : cross-entropy : 276.641 : new best
 [valid] Ep. 3 : Up. 30 : valid-script : 2 : new best
-[valid] Ep. 4 : Up. 45 : cross-entropy : 276.894 : new best
+[valid] Ep. 4 : Up. 45 : cross-entropy : 275.773 : new best
 [valid] Ep. 4 : Up. 45 : valid-script : 3 : new best
-[valid] Ep. 4 : Up. 50 : cross-entropy : 276.855 : new best
+[valid] Ep. 4 : Up. 50 : cross-entropy : 275.441 : new best
 [valid] Ep. 4 : Up. 50 : valid-script : 4 : new best
diff --git a/tests/training/features/data-weighting/word_eos.expected b/tests/training/features/data-weighting/word_eos.expected
index 6965f3a..a4ec027 100644
--- a/tests/training/features/data-weighting/word_eos.expected
+++ b/tests/training/features/data-weighting/word_eos.expected
@@ -1,17 +1,17 @@
-Ep. 1 : Up. 5 : Sen. 320 : Cost 855.91491699
-Ep. 1 : Up. 10 : Sen. 640 : Cost 704.62048340
-Ep. 1 : Up. 15 : Sen. 960 : Cost 603.96533203
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 518.18493652
-Ep. 1 : Up. 25 : Sen. 1,600 : Cost 442.29742432
-Ep. 1 : Up. 30 : Sen. 1,920 : Cost 367.12915039
-Ep. 1 : Up. 35 : Sen. 2,240 : Cost 301.38732910
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 229.98681641
-Ep. 2 : Up. 45 : Sen. 64 : Cost 340.48742676
-Ep. 2 : Up. 50 : Sen. 384 : Cost 818.48626709
-Ep. 2 : Up. 55 : Sen. 704 : Cost 681.45458984
-Ep. 2 : Up. 60 : Sen. 1,024 : Cost 586.93054199
-Ep. 2 : Up. 65 : Sen. 1,344 : Cost 500.92144775
-Ep. 2 : Up. 70 : Sen. 1,664 : Cost 427.21176147
-Ep. 2 : Up. 75 : Sen. 1,984 : Cost 354.24258423
-Ep. 2 : Up. 80 : Sen. 2,304 : Cost 287.52239990
-Ep. 2 : Up. 85 : Sen. 2,624 : Cost 212.79341125
+Ep. 1 : Up. 5 : Sen. 320 : Cost 856.40283203
+Ep. 1 : Up. 10 : Sen. 640 : Cost 705.00976562
+Ep. 1 : Up. 15 : Sen. 960 : Cost 604.22814941
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 518.43249512
+Ep. 1 : Up. 25 : Sen. 1,600 : Cost 442.49536133
+Ep. 1 : Up. 30 : Sen. 1,920 : Cost 367.35723877
+Ep. 1 : Up. 35 : Sen. 2,240 : Cost 301.55618286
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 230.20394897
+Ep. 2 : Up. 45 : Sen. 64 : Cost 340.81048584
+Ep. 2 : Up. 50 : Sen. 384 : Cost 818.84265137
+Ep. 2 : Up. 55 : Sen. 704 : Cost 681.78875732
+Ep. 2 : Up. 60 : Sen. 1,024 : Cost 587.13653564
+Ep. 2 : Up. 65 : Sen. 1,344 : Cost 501.12982178
+Ep. 2 : Up. 70 : Sen. 1,664 : Cost 427.36920166
+Ep. 2 : Up. 75 : Sen. 1,984 : Cost 354.46206665
+Ep. 2 : Up. 80 : Sen. 2,304 : Cost 287.68417358
+Ep. 2 : Up. 85 : Sen. 2,624 : Cost 212.97563171
diff --git a/tests/training/features/data-weighting/word_maxibatch.expected b/tests/training/features/data-weighting/word_maxibatch.expected
index 72a1404..542de08 100644
--- a/tests/training/features/data-weighting/word_maxibatch.expected
+++ b/tests/training/features/data-weighting/word_maxibatch.expected
@@ -1,10 +1,10 @@
-493.10702515
-355.26889038
-272.68411255
-199.14933777
-131.11528015
-386.00247192
-388.27536011
-296.74649048
-220.32846069
-151.99040222
+493.42471313
+355.55953979
+272.83404541
+199.24537659
+131.18077087
+386.23254395
+388.55151367
+296.93032837
+220.35517883
+152.07803345
diff --git a/tests/training/features/data-weighting/word_twos.expected b/tests/training/features/data-weighting/word_twos.expected
index 24ed38d..cc235d0 100644
--- a/tests/training/features/data-weighting/word_twos.expected
+++ b/tests/training/features/data-weighting/word_twos.expected
@@ -1,17 +1,17 @@
-Ep. 1 : Up. 5 : Sen. 320 : Cost 846.21844482 :
-Ep. 1 : Up. 10 : Sen. 640 : Cost 694.92529297 :
-Ep. 1 : Up. 15 : Sen. 960 : Cost 594.27014160 :
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 508.49014282 :
-Ep. 1 : Up. 25 : Sen. 1,600 : Cost 432.60327148 :
-Ep. 1 : Up. 30 : Sen. 1,920 : Cost 357.43505859 :
-Ep. 1 : Up. 35 : Sen. 2,240 : Cost 291.69354248 :
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 220.29316711 :
-Ep. 2 : Up. 45 : Sen. 64 : Cost 330.79293823 :
-Ep. 2 : Up. 50 : Sen. 384 : Cost 808.79095459 :
-Ep. 2 : Up. 55 : Sen. 704 : Cost 671.76074219 :
-Ep. 2 : Up. 60 : Sen. 1,024 : Cost 577.23736572 :
-Ep. 2 : Up. 65 : Sen. 1,344 : Cost 491.22949219 :
-Ep. 2 : Up. 70 : Sen. 1,664 : Cost 417.52026367 :
-Ep. 2 : Up. 75 : Sen. 1,984 : Cost 344.55108643 :
-Ep. 2 : Up. 80 : Sen. 2,304 : Cost 277.83230591 :
-Ep. 2 : Up. 85 : Sen. 2,624 : Cost 203.10263062 :
+Ep. 1 : Up. 5 : Sen. 320 : Cost 846.69714355 :
+Ep. 1 : Up. 10 : Sen. 640 : Cost 695.30053711 :
+Ep. 1 : Up. 15 : Sen. 960 : Cost 594.51928711 :
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 508.72247314 :
+Ep. 1 : Up. 25 : Sen. 1,600 : Cost 432.78329468 :
+Ep. 1 : Up. 30 : Sen. 1,920 : Cost 357.64947510 :
+Ep. 1 : Up. 35 : Sen. 2,240 : Cost 291.84161377 :
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 220.49028015 :
+Ep. 2 : Up. 45 : Sen. 64 : Cost 331.08535767 :
+Ep. 2 : Up. 50 : Sen. 384 : Cost 809.13928223 :
+Ep. 2 : Up. 55 : Sen. 704 : Cost 672.08361816 :
+Ep. 2 : Up. 60 : Sen. 1,024 : Cost 577.43341064 :
+Ep. 2 : Up. 65 : Sen. 1,344 : Cost 491.42279053 :
+Ep. 2 : Up. 70 : Sen. 1,664 : Cost 417.66470337 :
+Ep. 2 : Up. 75 : Sen. 1,984 : Cost 344.76025391 :
+Ep. 2 : Up. 80 : Sen. 2,304 : Cost 277.97634888 :
+Ep. 2 : Up. 85 : Sen. 2,624 : Cost 203.26664734 :
diff --git a/tests/training/features/exp-smoothing/expsmooth.expected b/tests/training/features/exp-smoothing/expsmooth.expected
index 39b5a3f..0a332e7 100644
--- a/tests/training/features/exp-smoothing/expsmooth.expected
+++ b/tests/training/features/exp-smoothing/expsmooth.expected
@@ -1,10 +1,10 @@
-Ep. 1 : Up. 20 : Sen. 80 : Cost 188.28628540
-Ep. 1 : Up. 40 : Sen. 160 : Cost 158.74415588
-Ep. 1 : Up. 60 : Sen. 240 : Cost 186.73446655
-Ep. 1 : Up. 80 : Sen. 320 : Cost 187.43354797
-Ep. 1 : Up. 100 : Sen. 400 : Cost 191.98048401
-Ep. 1 : Up. 120 : Sen. 480 : Cost 197.26301575
-Ep. 1 : Up. 140 : Sen. 560 : Cost 188.48814392
-Ep. 1 : Up. 160 : Sen. 640 : Cost 138.19192505
-Ep. 1 : Up. 180 : Sen. 720 : Cost 178.13829041
-Ep. 1 : Up. 200 : Sen. 800 : Cost 172.75942993
+Ep. 1 : Up. 20 : Sen. 80 : Cost 189.12686157
+Ep. 1 : Up. 40 : Sen. 160 : Cost 164.48492432
+Ep. 1 : Up. 60 : Sen. 240 : Cost 191.08549500
+Ep. 1 : Up. 80 : Sen. 320 : Cost 193.08412170
+Ep. 1 : Up. 100 : Sen. 400 : Cost 197.98199463
+Ep. 1 : Up. 120 : Sen. 480 : Cost 203.05249023
+Ep. 1 : Up. 140 : Sen. 560 : Cost 194.73191833
+Ep. 1 : Up. 160 : Sen. 640 : Cost 141.65765381
+Ep. 1 : Up. 180 : Sen. 720 : Cost 182.43969727
+Ep. 1 : Up. 200 : Sen. 800 : Cost 175.64430237
diff --git a/tests/training/features/exp-smoothing/expsmooth.valid.expected b/tests/training/features/exp-smoothing/expsmooth.valid.expected
index c9cf296..5eb3194 100644
--- a/tests/training/features/exp-smoothing/expsmooth.valid.expected
+++ b/tests/training/features/exp-smoothing/expsmooth.valid.expected
@@ -1,10 +1,10 @@
-[valid] Ep. 1 : Up. 20 : cross-entropy : 222.122 : new best
-[valid] Ep. 1 : Up. 40 : cross-entropy : 204.974 : new best
-[valid] Ep. 1 : Up. 60 : cross-entropy : 198.165 : new best
-[valid] Ep. 1 : Up. 80 : cross-entropy : 193.006 : new best
-[valid] Ep. 1 : Up. 100 : cross-entropy : 189.999 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 188.414 : new best
-[valid] Ep. 1 : Up. 140 : cross-entropy : 187.68 : new best
-[valid] Ep. 1 : Up. 160 : cross-entropy : 185.726 : new best
-[valid] Ep. 1 : Up. 180 : cross-entropy : 185.776 : stalled 1 times (last best: 185.726)
-[valid] Ep. 1 : Up. 200 : cross-entropy : 185.164 : new best
+[valid] Ep. 1 : Up. 20 : cross-entropy : 227.393 : new best
+[valid] Ep. 1 : Up. 40 : cross-entropy : 210.417 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 203.053 : new best
+[valid] Ep. 1 : Up. 80 : cross-entropy : 198.014 : new best
+[valid] Ep. 1 : Up. 100 : cross-entropy : 194.883 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 192.912 : new best
+[valid] Ep. 1 : Up. 140 : cross-entropy : 191.532 : new best
+[valid] Ep. 1 : Up. 160 : cross-entropy : 189.221 : new best
+[valid] Ep. 1 : Up. 180 : cross-entropy : 188.993 : new best
+[valid] Ep. 1 : Up. 200 : cross-entropy : 188.035 : new best
diff --git a/tests/training/features/exp-smoothing/expsmooth_sync.expected b/tests/training/features/exp-smoothing/expsmooth_sync.expected
index 60c7b0f..48f75b5 100644
--- a/tests/training/features/exp-smoothing/expsmooth_sync.expected
+++ b/tests/training/features/exp-smoothing/expsmooth_sync.expected
@@ -1,10 +1,10 @@
-Ep. 1 : Up. 20 : Sen. 80 : Cost 182.65203857
-Ep. 1 : Up. 40 : Sen. 160 : Cost 155.88166809
-Ep. 1 : Up. 60 : Sen. 240 : Cost 184.18322754
-Ep. 1 : Up. 80 : Sen. 320 : Cost 186.25080872
-Ep. 1 : Up. 100 : Sen. 400 : Cost 191.62405396
-Ep. 1 : Up. 120 : Sen. 480 : Cost 197.04354858
-Ep. 1 : Up. 140 : Sen. 560 : Cost 188.32565308
-Ep. 1 : Up. 160 : Sen. 640 : Cost 138.54452515
-Ep. 1 : Up. 180 : Sen. 720 : Cost 178.15878296
-Ep. 1 : Up. 200 : Sen. 800 : Cost 172.65933228
+Ep. 1 : Up. 20 : Sen. 80 : Cost 187.71237183
+Ep. 1 : Up. 40 : Sen. 160 : Cost 163.85215759
+Ep. 1 : Up. 60 : Sen. 240 : Cost 190.63551331
+Ep. 1 : Up. 80 : Sen. 320 : Cost 192.62811279
+Ep. 1 : Up. 100 : Sen. 400 : Cost 197.13417053
+Ep. 1 : Up. 120 : Sen. 480 : Cost 202.37889099
+Ep. 1 : Up. 140 : Sen. 560 : Cost 193.67512512
+Ep. 1 : Up. 160 : Sen. 640 : Cost 140.65829468
+Ep. 1 : Up. 180 : Sen. 720 : Cost 180.93316650
+Ep. 1 : Up. 200 : Sen. 800 : Cost 174.50163269
diff --git a/tests/training/features/exp-smoothing/expsmooth_sync.valid.expected b/tests/training/features/exp-smoothing/expsmooth_sync.valid.expected
index 990b5bd..9185b54 100644
--- a/tests/training/features/exp-smoothing/expsmooth_sync.valid.expected
+++ b/tests/training/features/exp-smoothing/expsmooth_sync.valid.expected
@@ -1,10 +1,10 @@
-[valid] Ep. 1 : Up. 20 : cross-entropy : 217.366 : new best
-[valid] Ep. 1 : Up. 40 : cross-entropy : 201.761 : new best
-[valid] Ep. 1 : Up. 60 : cross-entropy : 195.803 : new best
-[valid] Ep. 1 : Up. 80 : cross-entropy : 191.88 : new best
-[valid] Ep. 1 : Up. 100 : cross-entropy : 189.405 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 188.01 : new best
-[valid] Ep. 1 : Up. 140 : cross-entropy : 187.335 : new best
-[valid] Ep. 1 : Up. 160 : cross-entropy : 185.499 : new best
-[valid] Ep. 1 : Up. 180 : cross-entropy : 185.617 : stalled 1 times (last best: 185.499)
-[valid] Ep. 1 : Up. 200 : cross-entropy : 184.961 : new best
+[valid] Ep. 1 : Up. 20 : cross-entropy : 226.312 : new best
+[valid] Ep. 1 : Up. 40 : cross-entropy : 209.256 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 202.403 : new best
+[valid] Ep. 1 : Up. 80 : cross-entropy : 197.33 : new best
+[valid] Ep. 1 : Up. 100 : cross-entropy : 194.074 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 191.846 : new best
+[valid] Ep. 1 : Up. 140 : cross-entropy : 190.357 : new best
+[valid] Ep. 1 : Up. 160 : cross-entropy : 187.689 : new best
+[valid] Ep. 1 : Up. 180 : cross-entropy : 187.172 : new best
+[valid] Ep. 1 : Up. 200 : cross-entropy : 186.016 : new best
diff --git a/tests/training/features/guided-alignment/rnn.expected b/tests/training/features/guided-alignment/rnn.expected
index 7c0ba57..0cefbcf 100644
--- a/tests/training/features/guided-alignment/rnn.expected
+++ b/tests/training/features/guided-alignment/rnn.expected
@@ -1,10 +1,10 @@
-237.17997742
-258.60153198
-244.89590454
-238.79472351
-224.54911804
-202.88813782
-231.55581665
-201.81591797
-211.45260620
-198.07586670
+236.62251282
+256.09661865
+241.58700562
+236.70104980
+224.99372864
+204.95254517
+235.38345337
+205.25239563
+215.87429810
+202.22142029
diff --git a/tests/training/features/guided-alignment/transformer.expected b/tests/training/features/guided-alignment/transformer.expected
index 54b713d..2513e28 100644
--- a/tests/training/features/guided-alignment/transformer.expected
+++ b/tests/training/features/guided-alignment/transformer.expected
@@ -1,10 +1,10 @@
-235.15219116
-248.11796570
-226.96292114
-220.31608582
-210.20214844
-190.91659546
-216.90063477
-189.43588257
-197.92109680
-185.81344604
+244.42282104
+256.56842041
+238.89138794
+233.57333374
+223.44998169
+204.23277283
+232.98970032
+204.28886414
+215.29394531
+201.92327881
diff --git a/tests/training/models/lm/lm-transformer.expected b/tests/training/models/lm/lm-transformer.expected
index 8db5d5c..a5e9556 100644
--- a/tests/training/models/lm/lm-transformer.expected
+++ b/tests/training/models/lm/lm-transformer.expected
@@ -1,5 +1,5 @@
-405.37481689
-294.14898682
-217.30960083
-152.40402222
-86.83746338
+405.95352173
+277.85601807
+198.49377441
+135.86233521
+74.85224152
diff --git a/tests/training/models/lm/lm-transformer.scores.expected b/tests/training/models/lm/lm-transformer.scores.expected
index a996d13..d40b653 100644
--- a/tests/training/models/lm/lm-transformer.scores.expected
+++ b/tests/training/models/lm/lm-transformer.scores.expected
@@ -1,10 +1,10 @@
--112.025002
--202.339981
--99.826500
--232.753708
--643.735596
--151.965912
--383.419220
--60.465172
--89.117729
--105.327499
+-90.117882
+-179.877197
+-81.371750
+-204.165802
+-636.969482
+-131.934113
+-359.475616
+-56.732944
+-68.373947
+-100.774132
diff --git a/tests/training/models/lm/lm.expected b/tests/training/models/lm/lm.expected
index dd06692..c6b5c74 100644
--- a/tests/training/models/lm/lm.expected
+++ b/tests/training/models/lm/lm.expected
@@ -1,5 +1,5 @@
-410.21221924
-307.32592773
-234.65556335
-165.87971497
-93.67483521
+410.02645874
+306.52648926
+233.58132935
+167.19117737
+91.86805725
diff --git a/tests/training/models/lm/lm.scores.expected b/tests/training/models/lm/lm.scores.expected
index 53bf2c6..9ea8ae4 100644
--- a/tests/training/models/lm/lm.scores.expected
+++ b/tests/training/models/lm/lm.scores.expected
@@ -1,10 +1,10 @@
--119.056870
--209.617035
--104.773262
--245.013489
--663.487915
--160.193527
--396.427094
--62.445087
--94.592773
--110.077751
+-114.927658
+-208.074463
+-102.252083
+-244.505508
+-677.256836
+-154.783279
+-411.580017
+-58.307816
+-89.968994
+-111.055710
diff --git a/tests/training/models/multi-source/multi-s2s.expected b/tests/training/models/multi-source/multi-s2s.expected
index 517f88e..61608c4 100644
--- a/tests/training/models/multi-source/multi-s2s.expected
+++ b/tests/training/models/multi-source/multi-s2s.expected
@@ -1,5 +1,5 @@
-389.64587402
-284.92941284
-207.10832214
-145.49560547
-82.26767731
+388.14068604
+279.26577759
+198.44155884
+137.92988586
+75.97171021
diff --git a/tests/training/models/multi-source/multi-transformer.expected b/tests/training/models/multi-source/multi-transformer.expected
index 09c3608..35c5e4b 100644
--- a/tests/training/models/multi-source/multi-transformer.expected
+++ b/tests/training/models/multi-source/multi-transformer.expected
@@ -1,5 +1,5 @@
-383.92016602
-276.54284668
-205.36067200
-145.78311157
-82.33700562
+382.23056030
+264.14666748
+193.33871460
+133.58370972
+71.43719482
diff --git a/tests/training/models/nematus/encdec_depth.expected b/tests/training/models/nematus/encdec_depth.expected
index 94c4672..3a4ebdf 100644
--- a/tests/training/models/nematus/encdec_depth.expected
+++ b/tests/training/models/nematus/encdec_depth.expected
@@ -1,5 +1,5 @@
-475.38677979
-452.63513184
-431.22705078
-414.68640137
-401.01882935
+488.74591064
+461.10632324
+438.10931396
+419.87353516
+404.20791626
diff --git a/tests/training/models/nematus/wmt17.expected b/tests/training/models/nematus/wmt17.expected
index 59e15b5..b35c165 100644
--- a/tests/training/models/nematus/wmt17.expected
+++ b/tests/training/models/nematus/wmt17.expected
@@ -1,5 +1,5 @@
-475.19924927
-452.67333984
-431.39782715
-415.06884766
-401.69699097
+490.18731689
+466.02075195
+442.22518921
+426.79705811
+411.55426025
diff --git a/tests/training/models/transformer/transformer.expected b/tests/training/models/transformer/transformer.expected
index c181f85..50d85ee 100644
--- a/tests/training/models/transformer/transformer.expected
+++ b/tests/training/models/transformer/transformer.expected
@@ -1,10 +1,10 @@
-226.87113953
-250.82170105
-244.17037964
-247.56513977
-242.15576172
-238.88996887
-236.09550476
-231.19555664
-237.84509277
-241.79611206
+236.04219055
+260.96929932
+254.12194824
+257.51263428
+253.00631714
+248.38674927
+245.93569946
+240.78047180
+248.01782227
+252.18527222
diff --git a/tests/training/multi-gpu/async_sgd.expected b/tests/training/multi-gpu/async_sgd.expected
index 5578a20..8b07ca5 100644
--- a/tests/training/multi-gpu/async_sgd.expected
+++ b/tests/training/multi-gpu/async_sgd.expected
@@ -1,20 +1,20 @@
-Ep. 1 : Up. 20 : Sen. 1,280 : Cost 242.11746216
-Ep. 1 : Up. 40 : Sen. 2,560 : Cost 235.72708130
-Ep. 1 : Up. 60 : Sen. 3,840 : Cost 236.84497070
-Ep. 1 : Up. 80 : Sen. 5,120 : Cost 226.40872192
-Ep. 1 : Up. 100 : Sen. 6,400 : Cost 210.93917847
-Ep. 1 : Up. 120 : Sen. 7,680 : Cost 216.88357544
-Ep. 1 : Up. 140 : Sen. 8,960 : Cost 206.62683105
-Ep. 1 : Up. 160 : Sen. 10,240 : Cost 196.11888123
-Ep. 1 : Up. 180 : Sen. 11,520 : Cost 193.01203918
-Ep. 1 : Up. 200 : Sen. 12,800 : Cost 194.11386108
-Ep. 1 : Up. 220 : Sen. 14,080 : Cost 199.61235046
-Ep. 1 : Up. 240 : Sen. 15,360 : Cost 191.05105591
-Ep. 1 : Up. 260 : Sen. 16,640 : Cost 190.15347290
-Ep. 1 : Up. 280 : Sen. 17,920 : Cost 182.99386597
-Ep. 1 : Up. 300 : Sen. 19,200 : Cost 172.30728149
-Ep. 1 : Up. 320 : Sen. 20,480 : Cost 181.16923523
-Ep. 1 : Up. 340 : Sen. 21,760 : Cost 179.05541992
-Ep. 1 : Up. 360 : Sen. 23,040 : Cost 180.18623352
-Ep. 1 : Up. 380 : Sen. 24,320 : Cost 178.12226868
-Ep. 1 : Up. 400 : Sen. 25,600 : Cost 177.11029053
+Ep. 1 : Up. 20 : Sen. 1,280 : Cost 241.64421082
+Ep. 1 : Up. 40 : Sen. 2,560 : Cost 234.09941101
+Ep. 1 : Up. 60 : Sen. 3,840 : Cost 234.62942505
+Ep. 1 : Up. 80 : Sen. 5,120 : Cost 222.46903992
+Ep. 1 : Up. 100 : Sen. 6,400 : Cost 205.35173035
+Ep. 1 : Up. 120 : Sen. 7,680 : Cost 209.94436646
+Ep. 1 : Up. 140 : Sen. 8,960 : Cost 198.47026062
+Ep. 1 : Up. 160 : Sen. 10,240 : Cost 187.63980103
+Ep. 1 : Up. 180 : Sen. 11,520 : Cost 183.61985779
+Ep. 1 : Up. 200 : Sen. 12,800 : Cost 185.31411743
+Ep. 1 : Up. 220 : Sen. 14,080 : Cost 191.44917297
+Ep. 1 : Up. 240 : Sen. 15,360 : Cost 183.22087097
+Ep. 1 : Up. 260 : Sen. 16,640 : Cost 183.28189087
+Ep. 1 : Up. 280 : Sen. 17,920 : Cost 176.65219116
+Ep. 1 : Up. 300 : Sen. 19,200 : Cost 167.07302856
+Ep. 1 : Up. 320 : Sen. 20,480 : Cost 176.93701172
+Ep. 1 : Up. 340 : Sen. 21,760 : Cost 175.71195984
+Ep. 1 : Up. 360 : Sen. 23,040 : Cost 177.32435608
+Ep. 1 : Up. 380 : Sen. 24,320 : Cost 175.38401794
+Ep. 1 : Up. 400 : Sen. 25,600 : Cost 174.90682983
diff --git a/tests/training/multi-gpu/sync_sgd.expected b/tests/training/multi-gpu/sync_sgd.expected
index 4167667..0871af8 100644
--- a/tests/training/multi-gpu/sync_sgd.expected
+++ b/tests/training/multi-gpu/sync_sgd.expected
@@ -1,10 +1,10 @@
-216.98922729
-189.01902771
-178.41192627
-192.66134644
-206.70254517
-212.32287598
-201.16146851
-222.01089478
-210.61257935
-220.22055054
+216.80216980
+189.78765869
+181.02754211
+198.78121948
+212.99096680
+220.08464050
+209.56250000
+230.61848450
+218.01846313
+229.08955383
diff --git a/tests/training/multi-gpu/sync_sgd_1gpu.expected b/tests/training/multi-gpu/sync_sgd_1gpu.expected
index 7940fff..73c9503 100644
--- a/tests/training/multi-gpu/sync_sgd_1gpu.expected
+++ b/tests/training/multi-gpu/sync_sgd_1gpu.expected
@@ -1,4 +1,4 @@
-236.82699585
-197.64729309
-199.82775879
-184.65493774
+236.64883423
+197.38874817
+198.74374390
+183.10134888
diff --git a/tests/training/multi-gpu/sync_sgd_1gpu_expsmooth.expected b/tests/training/multi-gpu/sync_sgd_1gpu_expsmooth.expected
index 7940fff..0600ba2 100644
--- a/tests/training/multi-gpu/sync_sgd_1gpu_expsmooth.expected
+++ b/tests/training/multi-gpu/sync_sgd_1gpu_expsmooth.expected
@@ -1,4 +1,4 @@
-236.82699585
-197.64729309
-199.82775879
-184.65493774
+236.64883423
+197.38874817
+198.74374390
+183.10137939
diff --git a/tests/training/restarting/sgd_2e.expected b/tests/training/restarting/sgd_2e.expected
index 9dacfd0..2a801f2 100644
--- a/tests/training/restarting/sgd_2e.expected
+++ b/tests/training/restarting/sgd_2e.expected
@@ -1,16 +1,16 @@
-Ep. 1 : Up. 4 : Sen. 128 : Cost 258.60427856
-Ep. 1 : Up. 8 : Sen. 256 : Cost 270.30130005
-Ep. 1 : Up. 12 : Sen. 384 : Cost 247.52519226
-Ep. 1 : Up. 16 : Sen. 512 : Cost 242.22946167
-Ep. 1 : Up. 20 : Sen. 640 : Cost 212.66212463
-Ep. 1 : Up. 24 : Sen. 768 : Cost 252.52357483
-Ep. 1 : Up. 28 : Sen. 896 : Cost 220.55186462
-Ep. 1 : Up. 32 : Sen. 1,024 : Cost 208.12597656
-Ep. 2 : Up. 36 : Sen. 128 : Cost 219.92158508
-Ep. 2 : Up. 40 : Sen. 256 : Cost 226.15286255
-Ep. 2 : Up. 44 : Sen. 384 : Cost 203.31124878
-Ep. 2 : Up. 48 : Sen. 512 : Cost 197.85574341
-Ep. 2 : Up. 52 : Sen. 640 : Cost 173.79647827
-Ep. 2 : Up. 56 : Sen. 768 : Cost 209.79650879
-Ep. 2 : Up. 60 : Sen. 896 : Cost 186.76977539
-Ep. 2 : Up. 64 : Sen. 1,024 : Cost 179.61027527
+Ep. 1 : Up. 4 : Sen. 128 : Cost 257.99652100
+Ep. 1 : Up. 8 : Sen. 256 : Cost 267.93783569
+Ep. 1 : Up. 12 : Sen. 384 : Cost 243.39039612
+Ep. 1 : Up. 16 : Sen. 512 : Cost 235.87208557
+Ep. 1 : Up. 20 : Sen. 640 : Cost 204.79017639
+Ep. 1 : Up. 24 : Sen. 768 : Cost 240.11624146
+Ep. 1 : Up. 28 : Sen. 896 : Cost 208.47099304
+Ep. 1 : Up. 32 : Sen. 1,024 : Cost 199.36221313
+Ep. 2 : Up. 36 : Sen. 128 : Cost 213.58728027
+Ep. 2 : Up. 40 : Sen. 256 : Cost 220.51084900
+Ep. 2 : Up. 44 : Sen. 384 : Cost 199.10847473
+Ep. 2 : Up. 48 : Sen. 512 : Cost 194.56945801
+Ep. 2 : Up. 52 : Sen. 640 : Cost 171.77461243
+Ep. 2 : Up. 56 : Sen. 768 : Cost 208.75405884
+Ep. 2 : Up. 60 : Sen. 896 : Cost 186.17434692
+Ep. 2 : Up. 64 : Sen. 1,024 : Cost 179.63316345
diff --git a/tests/training/restoring/optimizer/adagrad.costs.expected b/tests/training/restoring/optimizer/adagrad.costs.expected
index 31f312e..7b4f7e1 100644
--- a/tests/training/restoring/optimizer/adagrad.costs.expected
+++ b/tests/training/restoring/optimizer/adagrad.costs.expected
@@ -1,10 +1,10 @@
-238.72998047
-245.70014954
-240.35234070
-233.43530273
-239.59432983
-254.54069519
-256.56338501
-243.96914673
-245.85603333
-236.53524780
+238.52751160
+245.27938843
+239.83557129
+232.83401489
+238.87149048
+253.74154663
+255.69897461
+243.06086731
+244.85818481
+235.55209351
diff --git a/tests/training/restoring/optimizer/adagrad.gt.expected b/tests/training/restoring/optimizer/adagrad.gt.expected
index 227cd1a..faacfeb 100644
--- a/tests/training/restoring/optimizer/adagrad.gt.expected
+++ b/tests/training/restoring/optimizer/adagrad.gt.expected
@@ -1,2 +1,2 @@
-[[  2.13038606e-06   2.62594858e-05   9.69232133e-06 ...,   0.00000000e+00
+[[  3.54592659e-04   5.64868504e-04   8.30977151e-05 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restoring/optimizer/adam.costs.expected b/tests/training/restoring/optimizer/adam.costs.expected
index 7272a33..a6b5f9a 100644
--- a/tests/training/restoring/optimizer/adam.costs.expected
+++ b/tests/training/restoring/optimizer/adam.costs.expected
@@ -1,10 +1,10 @@
-238.70394897
-245.51347351
-239.71450806
-231.26594543
-231.99667358
-237.87966919
-234.03823853
-217.62667847
-215.01562500
-205.33581543
+238.40983582
+244.61091614
+238.22981262
+229.24475098
+230.14970398
+234.50399780
+228.12467957
+210.38107300
+206.17379761
+196.83959961
diff --git a/tests/training/restoring/optimizer/adam.mt.expected b/tests/training/restoring/optimizer/adam.mt.expected
index 64bcee9..bd411e5 100644
--- a/tests/training/restoring/optimizer/adam.mt.expected
+++ b/tests/training/restoring/optimizer/adam.mt.expected
@@ -1,2 +1 @@
-[[ -5.87761315e-05   1.77891372e-04  -1.15002964e-04 ...,   0.00000000e+00
-    0.00000000e+00   0.00000000e+00]]
+[[-0.00032975  0.00083305 -0.00051456 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restoring/optimizer/adam.vt.expected b/tests/training/restoring/optimizer/adam.vt.expected
index e08fe13..8205e2b 100644
--- a/tests/training/restoring/optimizer/adam.vt.expected
+++ b/tests/training/restoring/optimizer/adam.vt.expected
@@ -1,2 +1,2 @@
-[[  3.27316441e-09   9.30779294e-08   4.11611119e-08 ...,   0.00000000e+00
+[[  2.10904631e-07   7.14701400e-07   1.94320407e-07 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restoring/optimizer/adam_async.costs.expected b/tests/training/restoring/optimizer/adam_async.costs.expected
index 2c2bcce..5c13c3a 100644
--- a/tests/training/restoring/optimizer/adam_async.costs.expected
+++ b/tests/training/restoring/optimizer/adam_async.costs.expected
@@ -1,10 +1,10 @@
-7255.70458984
-8017.33105469
-7792.87353516
-7871.14453125
-7605.37109375
-7275.98925781
-6925.38037109
-6636.69628906
-6718.36572266
-6711.51171875
+7247.86181641
+7994.63183594
+7750.25000000
+7804.45703125
+7530.29150391
+7192.14550781
+6780.81640625
+6450.87402344
+6511.03564453
+6491.26269531
diff --git a/tests/training/restoring/optimizer/adam_async.mt.expected b/tests/training/restoring/optimizer/adam_async.mt.expected
index 36442fa..ff60b5d 100644
--- a/tests/training/restoring/optimizer/adam_async.mt.expected
+++ b/tests/training/restoring/optimizer/adam_async.mt.expected
@@ -1 +1 @@
-[[ 0.00348229  0.00959761 -0.00544088 ...,  0.          0.          0.        ]]
+[[-0.00195269  0.00663034 -0.00437212 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restoring/optimizer/adam_async.vt.expected b/tests/training/restoring/optimizer/adam_async.vt.expected
index 147b02c..49106ba 100644
--- a/tests/training/restoring/optimizer/adam_async.vt.expected
+++ b/tests/training/restoring/optimizer/adam_async.vt.expected
@@ -1,2 +1,2 @@
-[[  8.52484561e-07   4.59003832e-06   1.47568892e-06 ...,   0.00000000e+00
+[[  1.19943786e-06   6.02165346e-06   2.27189776e-06 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restoring/optimizer/adam_load.expected b/tests/training/restoring/optimizer/adam_load.expected
index ee730d5..bf5fef4 100644
--- a/tests/training/restoring/optimizer/adam_load.expected
+++ b/tests/training/restoring/optimizer/adam_load.expected
@@ -1,6 +1,6 @@
-Ep. 1 : Up. 1 : Sen. 2 : Cost 223.47967529
-Ep. 1 : Up. 2 : Sen. 4 : Cost 255.49520874
-Ep. 1 : Up. 3 : Sen. 6 : Cost 213.80761719
-Ep. 1 : Up. 4 : Sen. 8 : Cost 352.16754150
-Ep. 1 : Up. 5 : Sen. 10 : Cost 367.21719360
-Ep. 1 : Up. 6 : Sen. 12 : Cost 196.68669128
+Ep. 1 : Up. 1 : Sen. 2 : Cost 223.64685059
+Ep. 1 : Up. 2 : Sen. 4 : Cost 258.80792236
+Ep. 1 : Up. 3 : Sen. 6 : Cost 255.67260742
+Ep. 1 : Up. 4 : Sen. 8 : Cost 346.67749023
+Ep. 1 : Up. 5 : Sen. 10 : Cost 278.72695923
+Ep. 1 : Up. 6 : Sen. 12 : Cost 178.23016357
diff --git a/tests/training/restoring/optimizer/adam_sync.costs.expected b/tests/training/restoring/optimizer/adam_sync.costs.expected
index 2faf27e..eb87705 100644
--- a/tests/training/restoring/optimizer/adam_sync.costs.expected
+++ b/tests/training/restoring/optimizer/adam_sync.costs.expected
@@ -1,10 +1,10 @@
-7255.24365234
-8016.10302734
-7789.29394531
-7859.13769531
-7564.06005859
-7214.78515625
-6903.62011719
-6621.17675781
-6706.66259766
-6701.24316406
+7245.90527344
+7991.28515625
+7743.94384766
+7789.80712891
+7495.90478516
+7120.76269531
+6739.33447266
+6421.30761719
+6486.60400391
+6468.15869141
diff --git a/tests/training/restoring/optimizer/adam_sync.mt.expected b/tests/training/restoring/optimizer/adam_sync.mt.expected
index 5cdf31d..6cb2aaa 100644
--- a/tests/training/restoring/optimizer/adam_sync.mt.expected
+++ b/tests/training/restoring/optimizer/adam_sync.mt.expected
@@ -1 +1 @@
-[[ 0.00084951  0.00929706 -0.00577381 ...,  0.          0.          0.        ]]
+[[-0.00177634  0.00746338 -0.00486176 ...,  0.          0.          0.        ]]
diff --git a/tests/training/restoring/optimizer/adam_sync.vt.expected b/tests/training/restoring/optimizer/adam_sync.vt.expected
index 7de36bb..b86c122 100644
--- a/tests/training/restoring/optimizer/adam_sync.vt.expected
+++ b/tests/training/restoring/optimizer/adam_sync.vt.expected
@@ -1,2 +1,2 @@
-[[  2.50489421e-07   4.21463801e-06   1.65782251e-06 ...,   0.00000000e+00
+[[  1.04040987e-06   6.49143612e-06   2.48227298e-06 ...,   0.00000000e+00
     0.00000000e+00   0.00000000e+00]]
diff --git a/tests/training/restoring/validation/valid_add.expected b/tests/training/restoring/validation/valid_add.expected
index 397895f..fb2d8a5 100644
--- a/tests/training/restoring/validation/valid_add.expected
+++ b/tests/training/restoring/validation/valid_add.expected
@@ -1,15 +1,15 @@
-[valid] Ep. 1 : Up. 20 : cross-entropy : 296.108 : new best
-[valid] Ep. 1 : Up. 40 : cross-entropy : 296.103 : new best
-[valid] Ep. 1 : Up. 60 : cross-entropy : 296.097 : new best
-[valid] Ep. 1 : Up. 80 : cross-entropy : 296.092 : new best
-[valid] Ep. 1 : Up. 100 : cross-entropy : 296.088 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 296.082 : new best
-[valid] Ep. 1 : Up. 120 : ce-mean-words : 10.1572 : new best
-[valid] Ep. 1 : Up. 140 : cross-entropy : 296.078 : new best
-[valid] Ep. 1 : Up. 140 : ce-mean-words : 10.157 : new best
-[valid] Ep. 1 : Up. 160 : cross-entropy : 296.072 : new best
-[valid] Ep. 1 : Up. 160 : ce-mean-words : 10.1569 : new best
-[valid] Ep. 1 : Up. 180 : cross-entropy : 296.068 : new best
-[valid] Ep. 1 : Up. 180 : ce-mean-words : 10.1567 : new best
-[valid] Ep. 1 : Up. 200 : cross-entropy : 296.062 : new best
-[valid] Ep. 1 : Up. 200 : ce-mean-words : 10.1565 : new best
+[valid] Ep. 1 : Up. 20 : cross-entropy : 296.282 : new best
+[valid] Ep. 1 : Up. 40 : cross-entropy : 296.269 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 296.255 : new best
+[valid] Ep. 1 : Up. 80 : cross-entropy : 296.242 : new best
+[valid] Ep. 1 : Up. 100 : cross-entropy : 296.229 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 296.216 : new best
+[valid] Ep. 1 : Up. 120 : ce-mean-words : 10.1618 : new best
+[valid] Ep. 1 : Up. 140 : cross-entropy : 296.202 : new best
+[valid] Ep. 1 : Up. 140 : ce-mean-words : 10.1613 : new best
+[valid] Ep. 1 : Up. 160 : cross-entropy : 296.189 : new best
+[valid] Ep. 1 : Up. 160 : ce-mean-words : 10.1609 : new best
+[valid] Ep. 1 : Up. 180 : cross-entropy : 296.176 : new best
+[valid] Ep. 1 : Up. 180 : ce-mean-words : 10.1604 : new best
+[valid] Ep. 1 : Up. 200 : cross-entropy : 296.162 : new best
+[valid] Ep. 1 : Up. 200 : ce-mean-words : 10.1599 : new best
diff --git a/tests/training/restoring/validation/valid_lowisbet.expected b/tests/training/restoring/validation/valid_lowisbet.expected
index 7ddb2d1..5af7cbd 100644
--- a/tests/training/restoring/validation/valid_lowisbet.expected
+++ b/tests/training/restoring/validation/valid_lowisbet.expected
@@ -1,11 +1,5 @@
-[valid] Ep. 1 : Up. 30 : cross-entropy : 299.666 : new best
-[valid] Ep. 2 : Up. 60 : cross-entropy : 299.155 : new best
-[valid] Ep. 3 : Up. 90 : cross-entropy : 295.698 : new best
-[valid] Ep. 4 : Up. 120 : cross-entropy : 293.975 : new best
-[valid] Ep. 5 : Up. 150 : cross-entropy : 293.203 : new best
-[valid] Ep. 5 : Up. 160 : cross-entropy : 293.042 : new best
-[valid] Ep. 6 : Up. 180 : cross-entropy : 291.761 : new best
-[valid] Ep. 7 : Up. 210 : cross-entropy : 291.089 : new best
-[valid] Ep. 8 : Up. 240 : cross-entropy : 291.087 : new best
-[valid] Ep. 9 : Up. 270 : cross-entropy : 291.422 : stalled 1 times (last best: 291.087)
-[valid] Ep. 10 : Up. 300 : cross-entropy : 291.984 : stalled 2 times (last best: 291.087)
+[valid] Ep. 1 : Up. 30 : cross-entropy : 299.128 : new best
+[valid] Ep. 2 : Up. 60 : cross-entropy : 298.528 : new best
+[valid] Ep. 3 : Up. 90 : cross-entropy : 296.425 : new best
+[valid] Ep. 4 : Up. 120 : cross-entropy : 297.839 : stalled 1 times (last best: 296.425)
+[valid] Ep. 5 : Up. 150 : cross-entropy : 297.727 : stalled 2 times (last best: 296.425)
diff --git a/tests/training/restoring/validation/valid_newbest.expected b/tests/training/restoring/validation/valid_newbest.expected
index ec21cb2..d03d098 100644
--- a/tests/training/restoring/validation/valid_newbest.expected
+++ b/tests/training/restoring/validation/valid_newbest.expected
@@ -1,20 +1,20 @@
-[valid] Ep. 1 : Up. 10 : cross-entropy : 250.085 : new best
-[valid] Ep. 1 : Up. 10 : translation : 4 : new best
-[valid] Ep. 1 : Up. 20 : cross-entropy : 250.084 : new best
-[valid] Ep. 1 : Up. 20 : translation : 4 : stalled 1 times (last best: 4)
-[valid] Ep. 1 : Up. 30 : cross-entropy : 250.083 : new best
-[valid] Ep. 1 : Up. 30 : translation : 4 : stalled 2 times (last best: 4)
-[valid] Ep. 1 : Up. 40 : cross-entropy : 250.081 : new best
+[valid] Ep. 1 : Up. 10 : cross-entropy : 250.506 : new best
+[valid] Ep. 1 : Up. 10 : translation : 8 : new best
+[valid] Ep. 1 : Up. 20 : cross-entropy : 250.501 : new best
+[valid] Ep. 1 : Up. 20 : translation : 8 : stalled 1 times (last best: 8)
+[valid] Ep. 1 : Up. 30 : cross-entropy : 250.497 : new best
+[valid] Ep. 1 : Up. 30 : translation : 8 : stalled 2 times (last best: 8)
+[valid] Ep. 1 : Up. 40 : cross-entropy : 250.491 : new best
 [valid] Ep. 1 : Up. 40 : translation : 9 : new best
-[valid] Ep. 1 : Up. 50 : cross-entropy : 250.08 : new best
-[valid] Ep. 1 : Up. 50 : translation : 0 : stalled 1 times (last best: 9)
-[valid] Ep. 1 : Up. 60 : cross-entropy : 250.078 : new best
-[valid] Ep. 1 : Up. 60 : translation : 0 : stalled 2 times (last best: 9)
-[valid] Ep. 1 : Up. 70 : cross-entropy : 250.077 : new best
-[valid] Ep. 1 : Up. 70 : translation : 5 : stalled 3 times (last best: 9)
-[valid] Ep. 1 : Up. 80 : cross-entropy : 250.075 : new best
-[valid] Ep. 1 : Up. 80 : translation : 6 : stalled 4 times (last best: 9)
-[valid] Ep. 1 : Up. 90 : cross-entropy : 250.074 : new best
-[valid] Ep. 1 : Up. 90 : translation : 6 : stalled 5 times (last best: 9)
-[valid] Ep. 1 : Up. 100 : cross-entropy : 250.072 : new best
+[valid] Ep. 1 : Up. 50 : cross-entropy : 250.486 : new best
+[valid] Ep. 1 : Up. 50 : translation : 7 : stalled 1 times (last best: 9)
+[valid] Ep. 1 : Up. 60 : cross-entropy : 250.481 : new best
+[valid] Ep. 1 : Up. 60 : translation : 3 : stalled 2 times (last best: 9)
+[valid] Ep. 1 : Up. 70 : cross-entropy : 250.476 : new best
+[valid] Ep. 1 : Up. 70 : translation : 6 : stalled 3 times (last best: 9)
+[valid] Ep. 1 : Up. 80 : cross-entropy : 250.471 : new best
+[valid] Ep. 1 : Up. 80 : translation : 0 : stalled 4 times (last best: 9)
+[valid] Ep. 1 : Up. 90 : cross-entropy : 250.465 : new best
+[valid] Ep. 1 : Up. 90 : translation : 9 : stalled 5 times (last best: 9)
+[valid] Ep. 1 : Up. 100 : cross-entropy : 250.461 : new best
 [valid] Ep. 1 : Up. 100 : translation : 6 : stalled 6 times (last best: 9)
diff --git a/tests/training/validation/final_batch.expected b/tests/training/validation/final_batch.expected
index 893f451..eac5cdc 100644
--- a/tests/training/validation/final_batch.expected
+++ b/tests/training/validation/final_batch.expected
@@ -1,3 +1,3 @@
-[valid] Ep. 1 : Up. 60 : cross-entropy : 240.178 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 240.164 : new best
-[valid] Ep. 1 : Up. 150 : cross-entropy : 240.157 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 240.376 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 240.348 : new best
+[valid] Ep. 1 : Up. 150 : cross-entropy : 240.332 : new best
diff --git a/tests/training/validation/final_epoch.expected b/tests/training/validation/final_epoch.expected
index decf6d3..ebcb25c 100644
--- a/tests/training/validation/final_epoch.expected
+++ b/tests/training/validation/final_epoch.expected
@@ -1,3 +1,3 @@
-[valid] Ep. 1 : Up. 40 : cross-entropy : 240.203 : new best
-[valid] Ep. 1 : Up. 80 : cross-entropy : 240.194 : new best
-[valid] Ep. 2 : Up. 81 : cross-entropy : 240.194 : new best
+[valid] Ep. 1 : Up. 40 : cross-entropy : 240.475 : new best
+[valid] Ep. 1 : Up. 80 : cross-entropy : 240.459 : new best
+[valid] Ep. 2 : Up. 81 : cross-entropy : 240.459 : new best
diff --git a/tests/training/validation/final_match.expected b/tests/training/validation/final_match.expected
index 29ae2c8..87c9cc2 100644
--- a/tests/training/validation/final_match.expected
+++ b/tests/training/validation/final_match.expected
@@ -1,3 +1,3 @@
-[valid] Ep. 1 : Up. 60 : cross-entropy : 240.178 : new best
-[valid] Ep. 1 : Up. 120 : cross-entropy : 240.164 : new best
-[valid] Ep. 1 : Up. 180 : cross-entropy : 240.15 : new best
+[valid] Ep. 1 : Up. 60 : cross-entropy : 240.376 : new best
+[valid] Ep. 1 : Up. 120 : cross-entropy : 240.348 : new best
+[valid] Ep. 1 : Up. 180 : cross-entropy : 240.317 : new best