diff options
author | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2020-11-16 20:38:41 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundkiewicz@gmail.com> | 2020-11-16 20:38:41 +0300 |
commit | 035e84815592250600df09df751378689d7808c3 (patch) | |
tree | 823ae88d6dfb632ffd36b0830a6120fa8064e692 /tests/interface | |
parent | 525fc8697e4436ab9f1b58abd3e5de409ba28e3b (diff) |
Update tests with TSV inputs
Diffstat (limited to 'tests/interface')
39 files changed, 190 insertions, 163 deletions
diff --git a/tests/interface/input-tsv/restore_stdin.expected b/tests/interface/input-tsv/restore_stdin.expected index b9cc6d4..d26f7ff 100644 --- a/tests/interface/input-tsv/restore_stdin.expected +++ b/tests/interface/input-tsv/restore_stdin.expected @@ -1,30 +1,30 @@ -Ep. 1 : Up. 2 : Sen. 32 : Cost 214.87408447 -Ep. 1 : Up. 4 : Sen. 64 : Cost 195.88232422 -Ep. 1 : Up. 6 : Sen. 96 : Cost 222.75996399 -Ep. 1 : Up. 8 : Sen. 128 : Cost 232.49481201 -Ep. 1 : Up. 10 : Sen. 160 : Cost 204.78642273 -Ep. 1 : Up. 12 : Sen. 192 : Cost 256.89501953 -Ep. 1 : Up. 14 : Sen. 224 : Cost 237.73818970 -Ep. 1 : Up. 16 : Sen. 256 : Cost 210.21063232 -Ep. 1 : Up. 18 : Sen. 288 : Cost 178.70904541 -Ep. 1 : Up. 20 : Sen. 320 : Cost 224.30038452 -Ep. 1 : Up. 22 : Sen. 352 : Cost 225.22837830 -Ep. 1 : Up. 24 : Sen. 384 : Cost 210.81533813 -Ep. 1 : Up. 26 : Sen. 416 : Cost 202.19320679 -Ep. 1 : Up. 28 : Sen. 448 : Cost 211.53353882 -Ep. 1 : Up. 30 : Sen. 480 : Cost 209.39002991 -Ep. 1 : Up. 32 : Sen. 512 : Cost 206.38954163 -Ep. 1 : Up. 34 : Sen. 544 : Cost 202.88201904 -Ep. 1 : Up. 36 : Sen. 576 : Cost 192.36555481 -Ep. 1 : Up. 38 : Sen. 608 : Cost 179.21670532 -Ep. 1 : Up. 40 : Sen. 640 : Cost 164.29644775 -Ep. 1 : Up. 42 : Sen. 672 : Cost 187.61584473 -Ep. 1 : Up. 44 : Sen. 704 : Cost 244.09938049 -Ep. 1 : Up. 46 : Sen. 736 : Cost 266.25546265 -Ep. 1 : Up. 48 : Sen. 768 : Cost 197.74813843 -Ep. 1 : Up. 50 : Sen. 800 : Cost 187.12585449 -Ep. 1 : Up. 52 : Sen. 832 : Cost 186.14714050 -Ep. 1 : Up. 54 : Sen. 864 : Cost 227.19046021 -Ep. 1 : Up. 56 : Sen. 896 : Cost 210.90580750 -Ep. 1 : Up. 58 : Sen. 928 : Cost 210.68801880 -Ep. 1 : Up. 60 : Sen. 960 : Cost 182.89875793 +Ep. 1 : Up. 2 : Sen. 32 : Cost 214.83363342 +Ep. 1 : Up. 4 : Sen. 64 : Cost 195.70648193 +Ep. 1 : Up. 6 : Sen. 96 : Cost 222.41781616 +Ep. 1 : Up. 8 : Sen. 128 : Cost 231.91462708 +Ep. 1 : Up. 10 : Sen. 160 : Cost 204.08346558 +Ep. 1 : Up. 12 : Sen. 192 : Cost 255.86239624 +Ep. 1 : Up. 14 : Sen. 224 : Cost 236.60090637 +Ep. 1 : Up. 16 : Sen. 256 : Cost 209.00881958 +Ep. 1 : Up. 18 : Sen. 288 : Cost 177.51702881 +Ep. 1 : Up. 20 : Sen. 320 : Cost 222.74383545 +Ep. 1 : Up. 22 : Sen. 352 : Cost 223.34017944 +Ep. 1 : Up. 24 : Sen. 384 : Cost 208.93505859 +Ep. 1 : Up. 26 : Sen. 416 : Cost 200.02706909 +Ep. 1 : Up. 28 : Sen. 448 : Cost 209.29515076 +Ep. 1 : Up. 30 : Sen. 480 : Cost 207.00128174 +Ep. 1 : Up. 32 : Sen. 512 : Cost 203.81817627 +Ep. 1 : Up. 34 : Sen. 544 : Cost 200.10937500 +Ep. 1 : Up. 36 : Sen. 576 : Cost 189.81176758 +Ep. 1 : Up. 38 : Sen. 608 : Cost 176.77787781 +Ep. 1 : Up. 40 : Sen. 640 : Cost 161.60902405 +Ep. 1 : Up. 42 : Sen. 672 : Cost 184.40527344 +Ep. 1 : Up. 44 : Sen. 704 : Cost 239.88012695 +Ep. 1 : Up. 46 : Sen. 736 : Cost 262.33227539 +Ep. 1 : Up. 48 : Sen. 768 : Cost 194.13323975 +Ep. 1 : Up. 50 : Sen. 800 : Cost 183.32736206 +Ep. 1 : Up. 52 : Sen. 832 : Cost 181.78253174 +Ep. 1 : Up. 54 : Sen. 864 : Cost 222.31034851 +Ep. 1 : Up. 56 : Sen. 896 : Cost 206.36886597 +Ep. 1 : Up. 58 : Sen. 928 : Cost 205.69429016 +Ep. 1 : Up. 60 : Sen. 960 : Cost 178.27331543 diff --git a/tests/interface/input-tsv/test_tsv_train.sh b/tests/interface/input-tsv/test_tsv_train.sh index 72b87d8..822b249 100644 --- a/tests/interface/input-tsv/test_tsv_train.sh +++ b/tests/interface/input-tsv/test_tsv_train.sh @@ -14,7 +14,7 @@ mkdir -p train # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train/model.npz --tsv -t train.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 10 --disp-freq 2 \ --log train.log diff --git a/tests/interface/input-tsv/test_tsv_train_assume_stdin.sh b/tests/interface/input-tsv/test_tsv_train_assume_stdin.sh index 87f8c7c..75a2537 100644 --- a/tests/interface/input-tsv/test_tsv_train_assume_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_assume_stdin.sh @@ -14,7 +14,7 @@ mkdir -p train_stdin2 # Run marian command cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_stdin2/model.npz -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 10 --disp-freq 2 \ --log train_stdin2.log diff --git a/tests/interface/input-tsv/test_tsv_train_create_vocab_joint.sh b/tests/interface/input-tsv/test_tsv_train_create_vocab_joint.sh index 88b5b62..7cad7d2 100644 --- a/tests/interface/input-tsv/test_tsv_train_create_vocab_joint.sh +++ b/tests/interface/input-tsv/test_tsv_train_create_vocab_joint.sh @@ -14,7 +14,7 @@ mkdir -p train_vocab # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_vocab/model.npz --tsv -t train.tsv -v train_vocab/vocab.spm train_vocab/vocab.spm --dim-vocabs 2000 2000 -T train_vocab \ --after-batches 20 --disp-freq 2 \ --log train_vocab.log diff --git a/tests/interface/input-tsv/test_tsv_train_create_vocabs.sh b/tests/interface/input-tsv/test_tsv_train_create_vocabs.sh index a23e7ac..0a3da3f 100644 --- a/tests/interface/input-tsv/test_tsv_train_create_vocabs.sh +++ b/tests/interface/input-tsv/test_tsv_train_create_vocabs.sh @@ -14,7 +14,7 @@ mkdir -p train_vocabs # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_vocabs/model.npz --tsv -t train.tsv -v train_vocabs/vocab.de.spm train_vocabs/vocab.en.spm --dim-vocabs 2000 2000 -T train_vocabs \ --after-batches 20 --disp-freq 2 \ --log train_vocabs.log diff --git a/tests/interface/input-tsv/test_tsv_train_create_vocabs_yml.sh b/tests/interface/input-tsv/test_tsv_train_create_vocabs_yml.sh index 26d20a6..b257ea6 100644 --- a/tests/interface/input-tsv/test_tsv_train_create_vocabs_yml.sh +++ b/tests/interface/input-tsv/test_tsv_train_create_vocabs_yml.sh @@ -14,7 +14,7 @@ mkdir -p train_vocabs_yml # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_vocabs_yml/model.npz --tsv -t train.bpe.tsv -v train_vocabs_yml/vocab.de.yml train_vocabs_yml/vocab.en.yml --dim-vocabs 2000 2000 -T train_vocabs_yml \ --after-batches 20 --disp-freq 2 \ --log train_vocabs_yml.log diff --git a/tests/interface/input-tsv/test_tsv_train_inputtypes_stdin.sh b/tests/interface/input-tsv/test_tsv_train_inputtypes_stdin.sh index 07f2eab..0f824ac 100644 --- a/tests/interface/input-tsv/test_tsv_train_inputtypes_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_inputtypes_stdin.sh @@ -14,7 +14,7 @@ mkdir -p train_intypes_stdin # Run marian command cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_intypes_stdin/model.npz --tsv -t stdin --input-types sequence sequence -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 10 --disp-freq 2 \ --log train_intypes_stdin.log diff --git a/tests/interface/input-tsv/test_tsv_train_mini_batch_fit.sh b/tests/interface/input-tsv/test_tsv_train_mini_batch_fit.sh index 42afeac..401c7f9 100644 --- a/tests/interface/input-tsv/test_tsv_train_mini_batch_fit.sh +++ b/tests/interface/input-tsv/test_tsv_train_mini_batch_fit.sh @@ -14,7 +14,7 @@ mkdir -p train_fit # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --mini-batch-fit -w 500 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd --sync-sgd \ + --cost-type ce-mean --mini-batch-fit -w 500 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd --sync-sgd --clip-norm 0 \ -m train_fit/model.npz --tsv -t train.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 20 --disp-freq 4 \ --log train_fit.log diff --git a/tests/interface/input-tsv/test_tsv_train_mini_batch_fit_stdin.sh b/tests/interface/input-tsv/test_tsv_train_mini_batch_fit_stdin.sh index 0b4ba42..81e9410 100644 --- a/tests/interface/input-tsv/test_tsv_train_mini_batch_fit_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_mini_batch_fit_stdin.sh @@ -14,7 +14,7 @@ mkdir -p train_fit_stdin # Run marian command cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --mini-batch-fit -w 500 --seed 2222 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --mini-batch-fit -w 500 --seed 2222 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd \ -m train_fit_stdin/model.npz --tsv -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --disp-freq 4 --log train_fit_stdin.log diff --git a/tests/interface/input-tsv/test_tsv_train_restore_from_stdin.sh b/tests/interface/input-tsv/test_tsv_train_restore_from_stdin.sh index f8953ef..36b7685 100644 --- a/tests/interface/input-tsv/test_tsv_train_restore_from_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_restore_from_stdin.sh @@ -11,7 +11,7 @@ test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < train.bpe.de > vocab.de.yml test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < train.bpe.en > vocab.en.yml # TODO: Weight decaying in Adam is disabled, because it gives unstable results on GPU -extra_opts="--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --mini-batch 16 --dim-emb 128 --dim-rnn 256 --disp-freq 2 --type s2s --sync-sgd --optimizer sgd --cost-type ce-mean" +extra_opts="--no-shuffle --clip-norm 0 --seed 2222 --maxi-batch 1 --maxi-batch-sort none --mini-batch 16 --dim-emb 128 --dim-rnn 256 --disp-freq 2 --type s2s --sync-sgd --optimizer sgd --cost-type ce-mean" # Step 1: Train a model in one go, up to the update no. 70, and save training logs #$MRT_MARIAN/marian \ diff --git a/tests/interface/input-tsv/test_tsv_train_shuffle.sh b/tests/interface/input-tsv/test_tsv_train_shuffle.sh index e5ca216..9dd4ac9 100644 --- a/tests/interface/input-tsv/test_tsv_train_shuffle.sh +++ b/tests/interface/input-tsv/test_tsv_train_shuffle.sh @@ -14,7 +14,7 @@ mkdir -p train_shuffle # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd --sync-sgd \ + --cost-type ce-mean --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd --sync-sgd --clip-norm 0 \ -m train_shuffle/model.npz --tsv --tsv-fields 2 -t train.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 20 --disp-freq 4 \ --log train_shuffle.log diff --git a/tests/interface/input-tsv/test_tsv_train_shuffle_in_ram.sh b/tests/interface/input-tsv/test_tsv_train_shuffle_in_ram.sh index 8ce67c3..c6e0421 100644 --- a/tests/interface/input-tsv/test_tsv_train_shuffle_in_ram.sh +++ b/tests/interface/input-tsv/test_tsv_train_shuffle_in_ram.sh @@ -14,7 +14,7 @@ mkdir -p train_shuffle_ram # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --shuffle-in-ram --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd \ + --cost-type ce-mean --shuffle-in-ram --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 10 --optimizer sgd \ -m train_shuffle_ram/model.npz --tsv --tsv-fields 2 -t train.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 20 --disp-freq 4 \ --log train_shuffle_ram.log diff --git a/tests/interface/input-tsv/test_tsv_train_stdin.sh b/tests/interface/input-tsv/test_tsv_train_stdin.sh index 2a77516..7050b36 100644 --- a/tests/interface/input-tsv/test_tsv_train_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_stdin.sh @@ -14,7 +14,7 @@ mkdir -p train_stdin # Run marian command cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_stdin/model.npz --tsv -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 10 --disp-freq 2 \ --log train_stdin.log diff --git a/tests/interface/input-tsv/test_tsv_train_stdin_2_epochs.sh b/tests/interface/input-tsv/test_tsv_train_stdin_2_epochs.sh index af35c5c..654f019 100644 --- a/tests/interface/input-tsv/test_tsv_train_stdin_2_epochs.sh +++ b/tests/interface/input-tsv/test_tsv_train_stdin_2_epochs.sh @@ -14,7 +14,7 @@ mkdir -p train_stdin_2e # Train for the 1st epoch cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_stdin_2e/model.npz --tsv -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --disp-freq 5 \ --log train_stdin_2e.log @@ -25,7 +25,7 @@ test -e train_stdin_2e.log # Train for the 2nd epoch cat train.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.002 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.002 \ -m train_stdin_2e/model.npz --tsv -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --disp-freq 5 \ --log train_stdin_2e.log diff --git a/tests/interface/input-tsv/test_tsv_train_stdin_empty_fields.sh b/tests/interface/input-tsv/test_tsv_train_stdin_empty_fields.sh index e9b0b33..d76ad80 100644 --- a/tests/interface/input-tsv/test_tsv_train_stdin_empty_fields.sh +++ b/tests/interface/input-tsv/test_tsv_train_stdin_empty_fields.sh @@ -20,7 +20,7 @@ paste train.{de,en} \ # Run marian command cat train_empty_lines.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --max-length 200 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --max-length 200 \ -m train_empty_lines/model.npz --tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-epochs 1 --disp-freq 2 \ --log train_empty_lines.log diff --git a/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh b/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh index e9a25c6..8ebd9ce 100644 --- a/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh +++ b/tests/interface/input-tsv/test_tsv_train_stdin_lm.sh @@ -14,7 +14,7 @@ mkdir -p train_lm # Run marian command cat train.en | $MRT_MARIAN/marian --type lm \ - --cost-type ce-mean --no-shuffle --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ -m train_lm/model.npz -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.spm \ --after-batches 10 --disp-freq 2 \ --log train_lm.log diff --git a/tests/interface/input-tsv/test_tsv_train_with_align.sh b/tests/interface/input-tsv/test_tsv_train_with_align.sh index c101d51..a95ad9d 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align.sh @@ -14,7 +14,7 @@ mkdir -p train_align # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ -m train_align/model.npz --tsv -t train2.de-en-aln.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --guided-alignment 2 --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh index b02488a..efa1793 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh @@ -14,7 +14,7 @@ mkdir -p train_align_weights # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ -m train_align_weights/model.npz --tsv -t train2.de-w-aln-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 60 --disp-freq 4 \ --guided-alignment 2 --guided-alignment-weight 1.0 --data-weighting 1 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh index 8653a67..c0e175e 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh @@ -14,7 +14,7 @@ mkdir -p train_align_weights_intypes # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ -m train_align_weights_intypes/model.npz --tsv -t train2.de-w-aln-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 60 --disp-freq 4 \ --input-types sequence weight alignment sequence --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh b/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh index 55f9995..34c829f 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh @@ -14,7 +14,7 @@ mkdir -p train_align0 # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ -m train_align0/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --guided-alignment 0 --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh index d02f8dc..7a0503c 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh @@ -14,7 +14,7 @@ mkdir -p train_align_shuffle # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 --sync-sgd \ + --cost-type ce-mean --clip-norm 1 --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 --sync-sgd \ -m train_align_shuffle/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --guided-alignment 0 --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh index 2c4dc1a..49e74de 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh @@ -14,7 +14,7 @@ mkdir -p train_align_shuffle_ram # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --shuffle-in-ram --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --shuffle-in-ram --clip-norm 1 --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ -m train_align_shuffle_ram/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --guided-alignment 0 --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh b/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh index b266f3d..6eb5386 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh @@ -14,7 +14,7 @@ mkdir -p train_align_stdin # Run marian command cat train2.aln-de-en.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ -m train_align_stdin/model.npz -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --disp-freq 4 \ --guided-alignment 0 --guided-alignment-weight 1.0 \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_weights.sh b/tests/interface/input-tsv/test_tsv_train_with_weights.sh index 6528713..5b08134 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_weights.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_weights.sh @@ -14,7 +14,7 @@ mkdir -p train_weights # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ -m train_weights/model.npz --tsv -t train2.de-en-w.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --data-weighting 2 --data-weighting-type sentence \ diff --git a/tests/interface/input-tsv/test_tsv_train_with_weights_pos0.sh b/tests/interface/input-tsv/test_tsv_train_with_weights_pos0.sh index ee4522f..5065dd3 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_weights_pos0.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_weights_pos0.sh @@ -14,7 +14,7 @@ mkdir -p train_weights0 # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ -m train_weights0/model.npz --tsv -t train2.w-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ --data-weighting 0 --data-weighting-type sentence \ diff --git a/tests/interface/input-tsv/test_tsv_valid.sh b/tests/interface/input-tsv/test_tsv_valid.sh index df72551..94c46f8 100644 --- a/tests/interface/input-tsv/test_tsv_valid.sh +++ b/tests/interface/input-tsv/test_tsv_valid.sh @@ -18,7 +18,7 @@ test -e valid/vocab.spm || cp $MRT_MODELS/rnn-spm/vocab.deen.spm valid/vocab.spm # Train $MRT_MARIAN/marian \ - --seed 2222 --no-shuffle --mini-batch 32 --maxi-batch 1 --optimizer sgd \ + --seed 2222 --no-shuffle --clip-norm 1 --mini-batch 32 --maxi-batch 1 --optimizer sgd \ -m valid/model.npz --tsv -t train.tsv -v valid/vocab.{spm,spm} \ --disp-freq 20 --valid-freq 30 --after-batches 30 \ --valid-metrics cross-entropy translation --valid-translation-output valid.out \ diff --git a/tests/interface/input-tsv/train.expected b/tests/interface/input-tsv/train.expected index 1cf2ecf..ed76575 100644 --- a/tests/interface/input-tsv/train.expected +++ b/tests/interface/input-tsv/train.expected @@ -1,5 +1,5 @@ -261.83258057 -257.78665161 -262.48895264 -253.13388062 -234.16917419 +261.58084106 +256.52697754 +260.31454468 +250.19743347 +230.69422913 diff --git a/tests/interface/input-tsv/train_align.expected b/tests/interface/input-tsv/train_align.expected index 2de51f1..6d1c8b3 100644 --- a/tests/interface/input-tsv/train_align.expected +++ b/tests/interface/input-tsv/train_align.expected @@ -1,25 +1,25 @@ -272.57867432 -267.45211792 -245.10440063 -243.12583923 -254.65167236 -251.95730591 -259.63885498 -243.55068970 -236.45735168 -215.81468201 -212.01930237 -222.31713867 -220.01065063 -230.49443054 -219.36715698 -214.80720520 -198.64233398 -195.14010620 -205.54002380 -204.59991455 -215.78044128 -205.05665588 -201.84078979 -187.56027222 -184.19506836 +267.55108643 +238.90954590 +203.89498901 +194.40493774 +201.26391602 +196.71656799 +205.61053467 +190.75955200 +190.06002808 +175.82437134 +171.81805420 +183.65437317 +183.88174438 +195.11131287 +181.43255615 +182.07211304 +168.76817322 +167.46075439 +175.70928955 +179.71203613 +187.29899597 +175.05770874 +175.59832764 +165.63943481 +163.86834717 diff --git a/tests/interface/input-tsv/train_align_stdin.expected b/tests/interface/input-tsv/train_align_stdin.expected index a468d22..e7a4610 100644 --- a/tests/interface/input-tsv/train_align_stdin.expected +++ b/tests/interface/input-tsv/train_align_stdin.expected @@ -1,7 +1,7 @@ -272.57867432 -267.45211792 -245.10440063 -243.12583923 -254.65167236 -251.95730591 -259.63885498 +267.55108643 +238.90954590 +203.89497375 +194.40493774 +201.26391602 +196.71656799 +205.61053467 diff --git a/tests/interface/input-tsv/train_empty_lines.expected b/tests/interface/input-tsv/train_empty_lines.expected index bf4b72e..acc5403 100644 --- a/tests/interface/input-tsv/train_empty_lines.expected +++ b/tests/interface/input-tsv/train_empty_lines.expected @@ -1,16 +1,16 @@ -270.58209229 -212.28765869 -285.14007568 -286.86123657 -168.05017090 -281.66876221 -267.08026123 -269.00738525 -262.89984131 -224.43609619 -284.56796265 -281.34075928 -231.55950928 -286.07806396 -249.47781372 -264.77264404 +270.44882202 +211.79843140 +284.02545166 +285.28485107 +166.57519531 +279.17941284 +264.18832397 +265.68725586 +259.24093628 +221.06784058 +279.52667236 +276.05969238 +226.25883484 +279.85809326 +242.84822083 +257.72427368 diff --git a/tests/interface/input-tsv/train_fit.expected b/tests/interface/input-tsv/train_fit.expected index 1c2d2eb..c577e77 100644 --- a/tests/interface/input-tsv/train_fit.expected +++ b/tests/interface/input-tsv/train_fit.expected @@ -1,5 +1,5 @@ -251.22476196 -252.45635986 -252.93251038 -258.67086792 -232.73229980 +250.19946289 +249.29103088 +247.88410950 +250.90098572 +222.61479187 diff --git a/tests/interface/input-tsv/train_fit_stdin.expected b/tests/interface/input-tsv/train_fit_stdin.expected index b9ce675..8d06b37 100644 --- a/tests/interface/input-tsv/train_fit_stdin.expected +++ b/tests/interface/input-tsv/train_fit_stdin.expected @@ -1,3 +1,3 @@ -344.08602905 -173.01716614 -248.64839172 +342.83029175 +170.86856079 +244.25839233 diff --git a/tests/interface/input-tsv/train_lm.expected b/tests/interface/input-tsv/train_lm.expected index c768644..fdbed00 100644 --- a/tests/interface/input-tsv/train_lm.expected +++ b/tests/interface/input-tsv/train_lm.expected @@ -1,5 +1,5 @@ -274.50836182 -274.87689209 -266.24481201 -259.36730957 -235.45114136 +274.36938477 +274.21545410 +265.08605957 +257.78823853 +233.71450806 diff --git a/tests/interface/input-tsv/train_shuffle.expected b/tests/interface/input-tsv/train_shuffle.expected index 912fc9a..f9d2a72 100644 --- a/tests/interface/input-tsv/train_shuffle.expected +++ b/tests/interface/input-tsv/train_shuffle.expected @@ -1,5 +1,5 @@ -216.91867065 -261.72125244 -267.90841675 -266.43109131 -300.19824219 +216.37680054 +259.30813599 +263.75015259 +260.62384033 +291.27304077 diff --git a/tests/interface/input-tsv/train_stdin_2e.expected b/tests/interface/input-tsv/train_stdin_2e.expected index 279c4ff..f80f79b 100644 --- a/tests/interface/input-tsv/train_stdin_2e.expected +++ b/tests/interface/input-tsv/train_stdin_2e.expected @@ -1,6 +1,6 @@ -262.65640259 -245.10810852 -248.28816223 -262.62274170 -245.00149536 -248.09956360 +261.66619873 +242.05940247 +243.13801575 +238.52865601 +317.93377686 +355.64865112 diff --git a/tests/interface/input-tsv/train_vocab.expected b/tests/interface/input-tsv/train_vocab.expected index 7e5f15e..2196992 100644 --- a/tests/interface/input-tsv/train_vocab.expected +++ b/tests/interface/input-tsv/train_vocab.expected @@ -1,10 +1,10 @@ -221.76995850 -213.01387024 -217.94252014 -216.67770386 -209.55206299 -233.91926575 -206.07537842 -231.02885437 -206.19801331 -222.06900024 +221.08058167 +211.53454590 +216.04510498 +213.98677063 +207.70535278 +230.34349060 +201.30010986 +224.49655151 +201.09184265 +214.30133057 diff --git a/tests/interface/input-tsv/train_vocabs.expected b/tests/interface/input-tsv/train_vocabs.expected index f003061..18492a4 100644 --- a/tests/interface/input-tsv/train_vocabs.expected +++ b/tests/interface/input-tsv/train_vocabs.expected @@ -1,10 +1,10 @@ -225.92840576 -209.46032715 -223.60330200 -207.56042480 -223.16098022 -198.65243530 -225.91799927 -209.45040894 -223.59326172 -207.55093384 +225.66580200 +208.27442932 +221.54891968 +204.83132935 +219.20014954 +194.64096069 +220.58721924 +203.36322021 +216.46800232 +199.75990295 diff --git a/tests/interface/input-tsv/train_vocabs_yml.expected b/tests/interface/input-tsv/train_vocabs_yml.expected index becf7bf..c6b5967 100644 --- a/tests/interface/input-tsv/train_vocabs_yml.expected +++ b/tests/interface/input-tsv/train_vocabs_yml.expected @@ -1,10 +1,10 @@ -200.10346985 -199.77453613 -196.38256836 -185.44483948 -168.59661865 -192.99969482 -181.77833557 -177.61343384 -200.09066772 -199.76245117 +199.88481140 +198.79373169 +194.57501221 +183.03530884 +165.68594360 +189.14419556 +177.06405640 +172.28703308 +193.72500610 +192.91064453 diff --git a/tests/interface/input-tsv/update_outputs.sh b/tests/interface/input-tsv/update_outputs.sh new file mode 100755 index 0000000..82436fb --- /dev/null +++ b/tests/interface/input-tsv/update_outputs.sh @@ -0,0 +1,27 @@ +#!/bin/sh -x +cp train.out train.expected +cp train_stdin2.out train.expected +cp train_vocab.out train_vocab.expected +cp train_vocabs.out train_vocabs.expected +cp train_vocabs.de.spm.out train_vocabs.de.spm.expected +cp train_vocabs.en.spm.out train_vocabs.en.spm.expected +cp train_vocabs_yml.out train_vocabs_yml.expected +cp train_intypes_stdin.out train.expected +cp train_fit.out train_fit.expected +cp train_fit_stdin.out train_fit_stdin.expected +cp restore_stdin.out restore_stdin.expected +cp train_shuffle.out train_shuffle.expected +cp train_shuffle_ram.out train_shuffle.expected +cp train_stdin.out train.expected +cp train_stdin_2e.out train_stdin_2e.expected +cp train_empty_lines.out train_empty_lines.expected +cp train_lm.out train_lm.expected +cp train_align.out train_align.expected +cp train_align_weights.out train_align_weights.expected +cp train_align_weights_intypes.out train_align_weights.expected +cp train_align0.out train_align.expected +cp train_align_shuffle.out train_align_shuffle.expected +cp train_align_shuffle_ram.out train_align_shuffle.expected +cp train_align_stdin.out train_align_stdin.expected +cp train_weights.out train_weights.expected +cp train_weights0.out train_weights.expected |