Update data weighting tests

11 files changed, 137 insertions, 137 deletions

diff --git a/tests/training/weights/maxibatch.expected b/tests/training/weights/maxibatch.expected
index d191734..216e307 100644
--- a/tests/training/weights/maxibatch.expected
+++ b/tests/training/weights/maxibatch.expected
@@ -1,10 +1,10 @@
-6916.29
-5106.83
-3438.09
-2349.86
-1473.54
-3595.42
-3671.98
-2831.25
-2118.49
-1484.09
+6921.24
+5224.01
+4133.74
+3090.42
+2085.98
+5415.40
+5651.44
+4458.85
+3396.66
+2398.46
diff --git a/tests/training/weights/sqlite.expected b/tests/training/weights/sqlite.expected
index 4986c74..f572cd2 100644
--- a/tests/training/weights/sqlite.expected
+++ b/tests/training/weights/sqlite.expected
@@ -1,100 +1,100 @@
+145.71
+407.97
+1194.81
+233.12
+1427.90
+126.27
+378.83
+97.13
+757.66
+330.25
+1602.69
+213.70
+437.12
+97.13
+466.23
+194.27
+1194.74
 145.70
-407.94
-1194.70
-233.07
-1427.39
-126.23
-378.66
-97.07
-757.37
-330.12
-1602.07
-213.48
-436.69
-96.88
-465.92
-194.09
-1193.40
-145.47
-930.50
-387.66
-927.88
-619.45
-869.34
-818.98
-434.26
-154.00
-489.57
-228.58
-344.03
-306.33
-575.79
-594.91
-1552.05
-212.38
-1133.21
-117.73
-1334.77
-156.53
-690.57
-107.32
-613.12
-156.52
-364.33
-179.15
-203.65
-114.84
-252.92
-135.13
-449.82
-272.53
-761.90
-188.65
-303.93
-267.39
-354.33
-110.67
-381.25
-435.42
-382.01
-116.72
-325.70
-94.44
-312.32
-58.83
-339.84
-157.14
-49.69
-274.61
-264.53
-38.22
-713.85
-183.54
-380.32
-469.33
-853.95
-359.63
-611.92
-187.32
-1294.60
-84.43
-1289.77
-484.42
-535.36
-60.39
-730.99
-111.69
-429.13
-56.42
-267.68
-92.46
-148.99
-123.08
-143.95
-155.08
-940.73
-64.80
-86.15
-71.10
-783.37
-408.54
+932.48
+388.53
+932.45
+621.65
+874.20
+825.62
+437.12
+155.41
+495.40
+233.12
+349.69
+320.55
+611.94
+670.22
+1777.47
+242.83
+1369.54
+135.99
+1515.31
+184.55
+815.88
+135.98
+728.46
+174.84
+407.95
+213.68
+233.13
+135.98
+291.39
+165.13
+582.84
+349.68
+961.59
+252.55
+437.11
+330.26
+437.10
+155.41
+524.51
+563.36
+495.39
+174.84
+437.06
+135.98
+495.34
+87.42
+437.08
+213.68
+58.28
+378.81
+349.66
+48.56
+990.65
+291.40
+524.50
+621.64
+1223.85
+534.22
+844.99
+291.38
+1660.95
+145.69
+1806.65
+679.91
+874.14
+77.71
+1049.04
+165.14
+611.91
+87.43
+437.06
+126.26
+262.22
+184.54
+262.25
+213.68
+1340.38
+106.84
+174.82
+116.55
+1165.53
+611.90
diff --git a/tests/training/weights/test_maxi_batches_with_sentence_weights.sh b/tests/training/weights/test_maxi_batches_with_sentence_weights.sh
index 49007fc..8e7d69b 100644
--- a/tests/training/weights/test_maxi_batches_with_sentence_weights.sh
+++ b/tests/training/weights/test_maxi_batches_with_sentence_weights.sh
@@ -8,7 +8,7 @@ mkdir -p maxibatch
 rm -rf maxibatch/* maxibatch.log
 
 $MRT_MARIAN/build/marian \
-    --seed 3333 --no-shuffle \
+    --seed 3333 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 --cost-type ce-sum \
     --data-weighting train.1k.inc.txt --data-weighting-type sentence
@@ -17,7 +17,7 @@ test -e maxibatch/model.npz
 test -e maxibatch.log
 
 $MRT_TOOLS/extract-costs.sh < maxibatch.log > maxibatch.out
-$MRT_TOOLS/diff-floats.py maxibatch.out maxibatch.expected -p 0.4 > maxibatch.diff
+$MRT_TOOLS/diff-floats.py maxibatch.out maxibatch.expected -p 0.1 > maxibatch.diff
 
 # Exit with success code
 exit 0
diff --git a/tests/training/weights/test_maxi_batches_with_word_weights.sh b/tests/training/weights/test_maxi_batches_with_word_weights.sh
index 82457a2..bfee15f 100644
--- a/tests/training/weights/test_maxi_batches_with_word_weights.sh
+++ b/tests/training/weights/test_maxi_batches_with_word_weights.sh
@@ -8,7 +8,7 @@ mkdir -p word_maxibatch
 rm -rf word_maxibatch/* word_maxibatch.log
 
 $MRT_MARIAN/build/marian \
-    --seed 6666 --no-shuffle \
+    --seed 6666 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m word_maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log word_maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 \
     --data-weighting train.1k.wordinc.txt --data-weighting-type word
@@ -17,7 +17,7 @@ test -e word_maxibatch/model.npz
 test -e word_maxibatch.log
 
 $MRT_TOOLS/extract-costs.sh < word_maxibatch.log > word_maxibatch.out
-$MRT_TOOLS/diff-floats.py word_maxibatch.out word_maxibatch.expected -p 0.3 > word_maxibatch.diff
+$MRT_TOOLS/diff-floats.py word_maxibatch.out word_maxibatch.expected -p 0.1 > word_maxibatch.diff
 
 # Exit with success code
 exit 0
diff --git a/tests/training/weights/test_sentence_weighting_sqlite.sh b/tests/training/weights/test_sentence_weighting_sqlite.sh
index b6b7a16..93f9bee 100644
--- a/tests/training/weights/test_sentence_weighting_sqlite.sh
+++ b/tests/training/weights/test_sentence_weighting_sqlite.sh
@@ -8,7 +8,7 @@ rm -rf sqlite sqlite.log
 mkdir -p sqlite
 
 $MRT_MARIAN/build/marian \
-    --seed 1111 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 \
+    --seed 1111 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 -o sgd \
     -m sqlite/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log sqlite.log --disp-freq 1 --after-batches 100 --mini-batch 1 \
     --data-weighting train.1k.weights.txt --data-weighting-type sentence --sqlite sqlite/corpus.sqlite3
@@ -19,7 +19,7 @@ test -e sqlite.log
 
 cat sqlite.log | $MRT_TOOLS/extract-costs.sh > sqlite.out
 
-$MRT_TOOLS/diff-floats.py sqlite.out sqlite.expected -p 0.3 > sqlite.diff
+$MRT_TOOLS/diff-floats.py sqlite.out sqlite.expected -p 0.1 > sqlite.diff
 
 # Exit with success code
 exit 0
diff --git a/tests/training/weights/test_sentence_weighting_with_ones.sh b/tests/training/weights/test_sentence_weighting_with_ones.sh
index 6f70894..cfe31b8 100644
--- a/tests/training/weights/test_sentence_weighting_with_ones.sh
+++ b/tests/training/weights/test_sentence_weighting_with_ones.sh
@@ -8,7 +8,7 @@ rm -rf noweights* ones*
 mkdir -p noweights ones
 
 $MRT_MARIAN/build/marian \
-    --seed 2222 --no-shuffle \
+    --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m noweights/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log noweights.log --disp-freq 5 -e 2
 
@@ -19,7 +19,7 @@ cat noweights.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed -r 's/ T
 cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/.*/1/g' > ones.weights.txt
 
 $MRT_MARIAN/build/marian \
-    --seed 2222 --no-shuffle \
+    --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m ones/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log ones.log --disp-freq 5 -e 2 \
     --data-weighting ones.weights.txt
diff --git a/tests/training/weights/test_sentence_weights_x3.sh b/tests/training/weights/test_sentence_weights_x3.sh
index a37d821..a211052 100644
--- a/tests/training/weights/test_sentence_weights_x3.sh
+++ b/tests/training/weights/test_sentence_weights_x3.sh
@@ -8,7 +8,7 @@ rm -rf x3copied* x3weights*
 mkdir -p x3copied x3weights
 
 $MRT_MARIAN/build/marian \
-    --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 \
+    --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 -o sgd \
     -m x3copied/model.npz -t train.x3.{de,en} -v vocab.{de,en}.yml \
     --log x3copied.log --disp-freq 1 --after-batches 100 1 --mini-batch 4 --cost-type ce-sum
 
@@ -17,7 +17,7 @@ test -e x3copied.log
 cat x3copied.log | grep 'Cost ' | sed -r 's/.*Cost (.*) : Time.*/\1/' > x3copied.out
 
 $MRT_MARIAN/build/marian \
-    --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 \
+    --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 -o sgd \
     -m x3weights/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \
     --log x3weights.log --disp-freq 1 --after-batches 100 --mini-batch 2 --cost-type ce-sum \
     --data-weighting train.1k.weights.txt --data-weighting-type sentence
@@ -27,7 +27,7 @@ test -e x3weights.log
 
 cat x3weights.log | grep 'Cost ' | sed -r 's/.*Cost (.*) : Time.*/\1/' > x3weights.out
 
-$MRT_TOOLS/diff-floats.py x3copied.out x3weights.out -p 0.99 > x3weights.diff
+$MRT_TOOLS/diff-floats.py x3copied.out x3weights.out -p 0.1 > x3weights.diff
 
 # Exit with success code
 exit 0
diff --git a/tests/training/weights/test_validation.sh b/tests/training/weights/test_validation.sh
index 54f900d..cf1e36c 100644
--- a/tests/training/weights/test_validation.sh
+++ b/tests/training/weights/test_validation.sh
@@ -21,10 +21,10 @@ test -e valid/valid.log
 test -e valid/train.log
 
 $MRT_TOOLS/strip-timestamps.sh < valid/valid.log > valid.out
-$MRT_TOOLS/diff-floats.py valid.out valid.expected -p 0.3 > valid.diff
+$MRT_TOOLS/diff-floats.py valid.out valid.expected -p 0.1 > valid.diff
 
 $MRT_TOOLS/extract-costs.sh < valid/train.log > train.out
-$MRT_TOOLS/diff-floats.py train.out train.expected -p 0.3 > train.diff
+$MRT_TOOLS/diff-floats.py train.out train.expected -p 0.1 > train.diff
 
 # Exit with success code
 exit 0
diff --git a/tests/training/weights/test_word_weighting_with_ones.sh b/tests/training/weights/test_word_weighting_with_ones.sh
index bf6b918..082032e 100644
--- a/tests/training/weights/test_word_weighting_with_ones.sh
+++ b/tests/training/weights/test_word_weighting_with_ones.sh
@@ -8,7 +8,7 @@ rm -rf word_noweights* word_ones*
 mkdir -p word_noweights word_ones
 
 $MRT_MARIAN/build/marian \
-    --seed 1111 --no-shuffle \
+    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m word_noweights/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log word_noweights.log --disp-freq 5 -e 2
 
@@ -19,7 +19,7 @@ cat word_noweights.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed -r
 cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/1/g' > word_ones.weights.txt
 
 $MRT_MARIAN/build/marian \
-    --seed 1111 --no-shuffle \
+    --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 -o sgd \
     -m word_ones/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \
     --log word_ones.log --disp-freq 5 -e 2 \
     --data-weighting word_ones.weights.txt --data-weighting-type word
diff --git a/tests/training/weights/valid.expected b/tests/training/weights/valid.expected
index ff40944..cd689ed 100644
--- a/tests/training/weights/valid.expected
+++ b/tests/training/weights/valid.expected
@@ -1,8 +1,8 @@
 [valid] 15 : cross-entropy : 276.231 : new best
 [valid] 15 : valid-script : 1 : new best
-[valid] 30 : cross-entropy : 271.226 : new best
+[valid] 30 : cross-entropy : 271.225 : new best
 [valid] 30 : valid-script : 2 : new best
-[valid] 45 : cross-entropy : 278.417 : stalled 1 times
+[valid] 45 : cross-entropy : 278.419 : stalled 1 times
 [valid] 45 : valid-script : 3 : new best
-[valid] 50 : cross-entropy : 282.218 : stalled 2 times
+[valid] 50 : cross-entropy : 282.22 : stalled 2 times
 [valid] 50 : valid-script : 4 : new best
diff --git a/tests/training/weights/word_maxibatch.expected b/tests/training/weights/word_maxibatch.expected
index c932724..5a0b271 100644
--- a/tests/training/weights/word_maxibatch.expected
+++ b/tests/training/weights/word_maxibatch.expected
@@ -1,10 +1,10 @@
-492.75
-347.77
-227.56
-151.63
-92.78
-257.22
-252.89
-188.91
-137.61
-94.50
+493.09
+355.25
+272.68
+199.14
+131.11
+385.98
+388.26
+296.74
+220.32
+151.99