From da95717d416234859527af77960acca366d58d5c Mon Sep 17 00:00:00 2001 From: Marcin Junczys-Dowmunt Date: Fri, 11 Feb 2022 06:03:54 -0800 Subject: Fix guided alignment regressions tests (#84) * fix guided alignment regressions tests * update regression tests --- .../input-tsv/test_tsv_train_with_align.sh | 5 ++- .../test_tsv_train_with_align_and_weights.sh | 5 ++- ..._tsv_train_with_align_and_weights_inputtypes.sh | 5 ++- .../input-tsv/test_tsv_train_with_align_pos0.sh | 5 ++- .../input-tsv/test_tsv_train_with_align_shuffle.sh | 5 ++- .../test_tsv_train_with_align_shuffle_in_ram.sh | 5 ++- .../input-tsv/test_tsv_train_with_align_stdin.sh | 5 ++- tests/interface/input-tsv/train_align.expected | 50 +++++++++++----------- .../input-tsv/train_align_shuffle.expected | 50 +++++++++++----------- .../interface/input-tsv/train_align_stdin.expected | 14 +++--- .../input-tsv/train_align_weights.expected | 30 ++++++------- .../features/guided-alignment/rnn.expected | 20 ++++----- .../guided-alignment/test_guided_alignment_rnn.sh | 2 +- .../test_guided_alignment_transformer.sh | 4 +- .../test_guided_alignment_transformer_sync.sh | 4 +- .../features/guided-alignment/transformer.expected | 10 +++++ 16 files changed, 118 insertions(+), 101 deletions(-) diff --git a/tests/interface/input-tsv/test_tsv_train_with_align.sh b/tests/interface/input-tsv/test_tsv_train_with_align.sh index a95ad9d..c2d79a7 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align.sh @@ -14,10 +14,11 @@ mkdir -p train_align # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.001 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align/model.npz --tsv -t train2.de-en-aln.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ - --guided-alignment 2 --guided-alignment-weight 1.0 \ + --guided-alignment 2 --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align.log diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh index efa1793..ee0832d 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights.sh @@ -14,10 +14,11 @@ mkdir -p train_align_weights # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.1 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align_weights/model.npz --tsv -t train2.de-w-aln-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 60 --disp-freq 4 \ - --guided-alignment 2 --guided-alignment-weight 1.0 --data-weighting 1 \ + --guided-alignment 2 --guided-alignment-weight 1.0 --data-weighting 1 --guided-alignment-cost ce \ --log train_align_weights.log diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh index c0e175e..a049bc1 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_and_weights_inputtypes.sh @@ -14,10 +14,11 @@ mkdir -p train_align_weights_intypes # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --no-shuffle --clip-norm 1 --seed 7777 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.1 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align_weights_intypes/model.npz --tsv -t train2.de-w-aln-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 60 --disp-freq 4 \ - --input-types sequence weight alignment sequence --guided-alignment-weight 1.0 \ + --input-types sequence weight alignment sequence --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align_weights_intypes.log # Check if files exist diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh b/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh index 34c829f..c7e9929 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_pos0.sh @@ -14,10 +14,11 @@ mkdir -p train_align0 # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.001 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align0/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ - --guided-alignment 0 --guided-alignment-weight 1.0 \ + --guided-alignment 0 --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align0.log diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh index 7a0503c..925b3ec 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle.sh @@ -14,10 +14,11 @@ mkdir -p train_align_shuffle # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --clip-norm 1 --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 --sync-sgd \ + --cost-type ce-mean --clip-norm 1 --seed 4444 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.1 --sync-sgd \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align_shuffle/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ - --guided-alignment 0 --guided-alignment-weight 1.0 \ + --guided-alignment 0 --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align_shuffle.log # Check if files exist diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh index 49e74de..a5db769 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_shuffle_in_ram.sh @@ -14,10 +14,11 @@ mkdir -p train_align_shuffle_ram # Run marian command $MRT_MARIAN/marian \ - --cost-type ce-mean --shuffle-in-ram --clip-norm 1 --seed 4444 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.1 \ + --cost-type ce-mean --shuffle-in-ram --clip-norm 1 --seed 4444 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.1 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align_shuffle_ram/model.npz --tsv -t train2.aln-de-en.tsv -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --after-batches 100 --disp-freq 4 \ - --guided-alignment 0 --guided-alignment-weight 1.0 \ + --guided-alignment 0 --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align_shuffle_ram.log # Check if files exist diff --git a/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh b/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh index 6eb5386..4e0bd32 100644 --- a/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh +++ b/tests/interface/input-tsv/test_tsv_train_with_align_stdin.sh @@ -14,10 +14,11 @@ mkdir -p train_align_stdin # Run marian command cat train2.aln-de-en.tsv | $MRT_MARIAN/marian \ - --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --learn-rate 0.001 \ + --cost-type ce-mean --no-shuffle --clip-norm 0 --seed 5555 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --learn-rate 0.001 \ + --dim-emb 32 --transformer-dim-ffn 64 --type transformer --enc-depth 3 --dec-depth 3 \ -m train_align_stdin/model.npz -t stdin -v $MRT_MODELS/rnn-spm/vocab.deen.{spm,spm} \ --disp-freq 4 \ - --guided-alignment 0 --guided-alignment-weight 1.0 \ + --guided-alignment 0 --guided-alignment-weight 1.0 --guided-alignment-cost ce \ --log train_align_stdin.log diff --git a/tests/interface/input-tsv/train_align.expected b/tests/interface/input-tsv/train_align.expected index 6d1c8b3..8114a79 100644 --- a/tests/interface/input-tsv/train_align.expected +++ b/tests/interface/input-tsv/train_align.expected @@ -1,25 +1,25 @@ -267.55108643 -238.90954590 -203.89498901 -194.40493774 -201.26391602 -196.71656799 -205.61053467 -190.75955200 -190.06002808 -175.82437134 -171.81805420 -183.65437317 -183.88174438 -195.11131287 -181.43255615 -182.07211304 -168.76817322 -167.46075439 -175.70928955 -179.71203613 -187.29899597 -175.05770874 -175.59832764 -165.63943481 -163.86834717 +349.90066528 +330.57034302 +301.72637939 +294.93923950 +311.47195435 +307.98052979 +318.67773438 +297.93566895 +289.55316162 +265.69094849 +258.18420410 +271.55532837 +267.22616577 +277.04589844 +259.60607910 +252.39709473 +232.80690002 +225.52084351 +237.79919434 +234.49322510 +244.92973328 +230.92803955 +226.21130371 +210.91824341 +204.61116028 diff --git a/tests/interface/input-tsv/train_align_shuffle.expected b/tests/interface/input-tsv/train_align_shuffle.expected index 4c7909c..2db8857 100644 --- a/tests/interface/input-tsv/train_align_shuffle.expected +++ b/tests/interface/input-tsv/train_align_shuffle.expected @@ -1,25 +1,25 @@ -267.62625122 -260.25723267 -256.07604980 -251.13800049 -248.25172424 -242.42607117 -243.15475464 -226.86355591 -237.34564209 -228.19276428 -212.93722534 -214.65142822 -218.42727661 -213.79447937 -213.66851807 -201.51606750 -210.07385254 -198.46676636 -197.96029663 -205.45782471 -209.31356812 -201.94723511 -187.90913391 -197.95065308 -196.05712891 +305.95520020 +252.33764648 +241.97651672 +237.92004395 +241.01110840 +235.38877869 +244.05845642 +228.61547852 +244.48324585 +240.67021179 +225.93069458 +230.75164795 +239.92446899 +234.91127014 +236.48362732 +225.34948730 +236.05422974 +226.67541504 +227.21850586 +240.31086731 +247.07595825 +240.54412842 +221.21253967 +337.10803223 +236.50607300 diff --git a/tests/interface/input-tsv/train_align_stdin.expected b/tests/interface/input-tsv/train_align_stdin.expected index e7a4610..6d8095f 100644 --- a/tests/interface/input-tsv/train_align_stdin.expected +++ b/tests/interface/input-tsv/train_align_stdin.expected @@ -1,7 +1,7 @@ -267.55108643 -238.90954590 -203.89497375 -194.40493774 -201.26391602 -196.71656799 -205.61053467 +349.90066528 +330.57034302 +301.72637939 +294.93923950 +311.47195435 +307.98052979 +318.67773438 diff --git a/tests/interface/input-tsv/train_align_weights.expected b/tests/interface/input-tsv/train_align_weights.expected index 4092789..77d155e 100644 --- a/tests/interface/input-tsv/train_align_weights.expected +++ b/tests/interface/input-tsv/train_align_weights.expected @@ -1,15 +1,15 @@ -341.56268311 -328.85687256 -300.68945312 -295.04937744 -322.07330322 -306.52780151 -332.50936890 -305.94641113 -290.83953857 -265.52841187 -256.46743774 -278.98114014 -265.27020264 -292.67654419 -273.39343262 +366.47402954 +311.10235596 +278.69085693 +266.78692627 +289.20837402 +277.70391846 +308.79412842 +286.08660889 +281.75717163 +269.53497314 +256.59310913 +292.10559082 +316.60882568 +305.43435669 +286.01943970 diff --git a/tests/training/features/guided-alignment/rnn.expected b/tests/training/features/guided-alignment/rnn.expected index 0cefbcf..3b07346 100644 --- a/tests/training/features/guided-alignment/rnn.expected +++ b/tests/training/features/guided-alignment/rnn.expected @@ -1,10 +1,10 @@ -236.62251282 -256.09661865 -241.58700562 -236.70104980 -224.99372864 -204.95254517 -235.38345337 -205.25239563 -215.87429810 -202.22142029 +297.51348877 +323.35086060 +308.66668701 +304.00738525 +290.16757202 +264.55703735 +307.27072144 +267.86627197 +281.67095947 +264.18920898 diff --git a/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh b/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh index 925f36f..ca9f030 100644 --- a/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh +++ b/tests/training/features/guided-alignment/test_guided_alignment_rnn.sh @@ -18,7 +18,7 @@ $MRT_MARIAN/marian \ --no-shuffle --clip-norm 1 --seed 1111 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \ -m rnn/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 100 --disp-freq 10 \ - --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \ + --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --guided-alignment-cost ce --learn-rate 0.1 \ --log rnn.log # Check if files exist diff --git a/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh b/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh index cd28f1c..cdca3c4 100644 --- a/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh +++ b/tests/training/features/guided-alignment/test_guided_alignment_transformer.sh @@ -15,10 +15,10 @@ mkdir -p transformer # Run marian command $MRT_MARIAN/marian --type transformer \ - --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean \ + --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --cost-type ce-mean \ -m transformer/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 100 --disp-freq 10 \ - --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \ + --guided-alignment corpus.bpe.align --guided-alignment-weight 1 --guided-alignment-cost ce --learn-rate 0.01 \ --log transformer.log # Check if files exist diff --git a/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh b/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh index 963052d..eb1bb70 100644 --- a/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh +++ b/tests/training/features/guided-alignment/test_guided_alignment_transformer_sync.sh @@ -15,10 +15,10 @@ mkdir -p transformer_sync # Run marian command $MRT_MARIAN/marian --type transformer \ - --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --cost-type ce-mean --sync-sgd \ + --no-shuffle --clip-norm 0 --seed 2222 --dim-emb 32 --dim-rnn 64 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer adam --cost-type ce-mean --sync-sgd \ -m transformer_sync/model.npz -t corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 100 --disp-freq 10 \ - --guided-alignment corpus.bpe.align --guided-alignment-weight 1.0 --learn-rate 0.1 \ + --guided-alignment corpus.bpe.align --guided-alignment-weight 1 --guided-alignment-cost ce --learn-rate 0.01 \ --log transformer_sync.log # Check if files exist diff --git a/tests/training/features/guided-alignment/transformer.expected b/tests/training/features/guided-alignment/transformer.expected index e69de29..56a6ff2 100644 --- a/tests/training/features/guided-alignment/transformer.expected +++ b/tests/training/features/guided-alignment/transformer.expected @@ -0,0 +1,10 @@ +276.90930176 +258.53686523 +244.95483398 +240.12124634 +234.31253052 +208.04850769 +234.38320923 +205.47990417 +212.96972656 +199.58921814 -- cgit v1.2.3