diff options
| author | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2018-11-29 15:08:07 +0300 |
|---|---|---|
| committer | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2018-11-29 15:08:07 +0300 |
| commit | 583011cb994a2fe23f9187e55c6079fa41dc5878 (patch) | |
| tree | 7d63870523b9ffc3ec9d86ea36174319cbf86e44 | |
| parent | 9cc169495729ce9d25ffae7492f08fd6d8619915 (diff) | |
| parent | 47a8eac0de6be7e9e7d2128e71bdd43b7cbbfb85 (diff) | |
Merge last changes
172 files changed, 292 insertions, 299 deletions
@@ -1,24 +1,17 @@ THREADS=16 -GIT_MARIAN_DEV=http://github.com/marian-nmt/marian-dev.git -GIT_MARIAN=http://github.com/marian-nmt/marian.git GIT_MOSES_SCRIPTS=http://github.com/marian-nmt/moses-scripts.git GIT_SUBWORD_NMT=http://github.com/rsennrich/subword-nmt.git -BRANCH=master - -EXTRA_FLAGS= -CMAKE_FLAGS=-DUSE_CUDNN=off -DCOMPILE_EXAMPLES=on -DCMAKE_BUILD_TYPE=Release $(EXTRA_FLAGS) - PIP_PACKAGES=websocket-client pyyaml -.PHONY: marian install tools models data run tools/marian +.PHONY: install tools models data run .SECONDARY: ##################################################################### -run: install marian +run: install bash ./run_mrt.sh install: tools models data @@ -41,12 +34,6 @@ data: mkdir -p $@ cd $@ && bash ./download-data.sh -marian: tools/marian -tools/marian: - git -C $@ pull || git clone $(GIT_MARIAN_DEV) -b $(BRANCH) $@ - rm -rf $@/build - mkdir -p $@/build && cd $@/build && cmake .. $(CMAKE_FLAGS) && make -j$(THREADS) - clean: git clean -x -d -f tests @@ -26,18 +26,16 @@ Each test consists of: ## Usage -Downloading data and tools, compiling the most recent version of marian-dev, and running -single-GPU tests: +Downloading data and tools, and running single-GPU tests: make install - make marian make run # or simply make Testing a custom version of Marian: make install - MARIAN=/path/to/marian-dev ./run_mrt.sh + MARIAN=/path/to/marian-dev/build ./run_mrt.sh Enabling multi-GPU tests: @@ -8,7 +8,7 @@ # where previous.log contains a list of test files in separate lines. # Environment variables: -# - MARIAN - path to Marian root directory +# - MARIAN - path to Marian build directory # - CUDA_VISIBLE_DEVICES - CUDA's variable specifying GPU device IDs # - NUM_DEVICES - maximum number of GPU devices to be used @@ -26,19 +26,27 @@ function logn { export MRT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export MRT_TOOLS=$MRT_ROOT/tools -export MRT_MARIAN="$( realpath ${MARIAN:-$MRT_TOOLS/marian} )" +export MRT_MARIAN="$( realpath ${MARIAN:-../build} )" export MRT_MODELS=$MRT_ROOT/models export MRT_DATA=$MRT_ROOT/data log "Using Marian: $MRT_MARIAN" +# Check if required tools are present in marian directory +for cmd in marian marian-decoder marian-scorer marian-server marian-vocab; do + if [ ! -e $MRT_MARIAN/$cmd ]; then + echo "Error: '$cmd' is not installed in '$MRT_MARIAN', you need to compile the toolkit first" + exit 1 + fi +done + # Get CMake settings -cd $MRT_MARIAN/build +cd $MRT_MARIAN cmake -L 2> /dev/null > $MRT_ROOT/cmake.log cd $MRT_ROOT # Check Marian compilation settings -export MRT_MARIAN_VERSION=$($MRT_MARIAN/build/marian --version 2>&1) +export MRT_MARIAN_VERSION=$($MRT_MARIAN/marian --version 2>&1) export MRT_MARIAN_USE_MKL=$(cat $MRT_ROOT/cmake.log | grep -P "MKL_ROOT" | grep -vP "MKL_ROOT.*NOTFOUND|USE_CUDNN:BOOL=(OFF|off|0)") export MRT_MARIAN_USE_CUDNN=$(cat $MRT_ROOT/cmake.log | grep -P "USE_CUDNN:BOOL=(ON|on|1)") diff --git a/tests/_self-adaptive/setup.sh b/tests/_self-adaptive/setup.sh index 2370926..11a463a 100644 --- a/tests/_self-adaptive/setup.sh +++ b/tests/_self-adaptive/setup.sh @@ -1,3 +1,3 @@ -test -f $MRT_MARIAN/build/marian-adaptive || exit 1 +test -f $MRT_MARIAN/marian-adaptive || exit 1 test -f $MRT_MODELS/wmt16_systems/en-de/model.npz || exit 1 python -c "import websocket" diff --git a/tests/_self-adaptive/test_context_empty.sh b/tests/_self-adaptive/test_context_empty.sh index 01a2301..3fd4b22 100644 --- a/tests/_self-adaptive/test_context_empty.sh +++ b/tests/_self-adaptive/test_context_empty.sh @@ -7,7 +7,7 @@ set -e rm -f nocontext.log # Run Marian -$MRT_MARIAN/build/marian-adaptive \ +$MRT_MARIAN/marian-adaptive \ -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json -v $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ --dim-vocabs 85000 85000 --dim-emb 500 --after-epochs 1 \ diff --git a/tests/_self-adaptive/test_context_partial.sh b/tests/_self-adaptive/test_context_partial.sh index d956526..a062b51 100644 --- a/tests/_self-adaptive/test_context_partial.sh +++ b/tests/_self-adaptive/test_context_partial.sh @@ -7,7 +7,7 @@ set -e rm -f contextpart.log # Run Marian -$MRT_MARIAN/build/marian-adaptive \ +$MRT_MARIAN/marian-adaptive \ -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json -v $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ --dim-vocabs 85000 85000 --dim-emb 500 --after-epochs 1 \ diff --git a/tests/_self-adaptive/test_oracle_1_sent_2_epochs.sh b/tests/_self-adaptive/test_oracle_1_sent_2_epochs.sh index 568104e..2374ee1 100644 --- a/tests/_self-adaptive/test_oracle_1_sent_2_epochs.sh +++ b/tests/_self-adaptive/test_oracle_1_sent_2_epochs.sh @@ -6,14 +6,14 @@ set -e # Test code goes here rm -f oracle_1s2e.log -#$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < ubuntu.in > trans.out +#$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < ubuntu.in > trans.out #diff trans.out trans.expected > trans.diff #$MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl -lc ubuntu.ref < trans.out > trans.bleu #diff trans.bleu trans.bleu.expected > trans.bleu.diff # Run Marian -$MRT_MARIAN/build/marian-adaptive \ +$MRT_MARIAN/marian-adaptive \ -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json -v $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ --dim-vocabs 85000 85000 --dim-emb 500 \ diff --git a/tests/_self-adaptive/test_oracle_2_sent_1_epoch.sh b/tests/_self-adaptive/test_oracle_2_sent_1_epoch.sh index 6547a61..aa47c4a 100644 --- a/tests/_self-adaptive/test_oracle_2_sent_1_epoch.sh +++ b/tests/_self-adaptive/test_oracle_2_sent_1_epoch.sh @@ -6,14 +6,14 @@ set -e # Test code goes here rm -f oracle_2s1e.log -#$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < ubuntu.in > trans.out +#$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < ubuntu.in > trans.out #diff trans.out trans.expected > trans.diff #$MRT_TOOLS/moses-scripts/scripts/generic/multi-bleu.perl -lc ubuntu.ref < trans.out > trans.bleu #diff trans.bleu trans.bleu.expected > trans.bleu.diff # Run Marian -$MRT_MARIAN/build/marian-adaptive \ +$MRT_MARIAN/marian-adaptive \ -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json -v $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ --dim-vocabs 85000 85000 --dim-emb 500 --after-epochs 1 \ diff --git a/tests/_self-adaptive/test_server_mode.sh b/tests/_self-adaptive/test_server_mode.sh index 305f94d..32e438e 100644 --- a/tests/_self-adaptive/test_server_mode.sh +++ b/tests/_self-adaptive/test_server_mode.sh @@ -9,12 +9,12 @@ clean_up() { trap clean_up EXIT # Test code goes here -$MRT_MARIAN/build/marian-adaptive -m $MRT_MODELS/wmt16_systems/en-de/model.npz -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8766 > server.log 2>&1 & +$MRT_MARIAN/marian-adaptive -m $MRT_MODELS/wmt16_systems/en-de/model.npz -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8766 > server.log 2>&1 & SERVER_PID=$! sleep 20 -python3 $MRT_MARIAN/scripts/self-adaptive/client_example.py -p 8766 > text.out +python3 $MRT_MARIAN/../scripts/self-adaptive/client_example.py -p 8766 > text.out kill $SERVER_PID test -e server.log diff --git a/tests/_template/test_decoder.sh b/tests/_template/test_decoder.sh index bc0e200..ca39319 100644 --- a/tests/_template/test_decoder.sh +++ b/tests/_template/test_decoder.sh @@ -12,7 +12,7 @@ set -e rm -f decoder.{out,diff} # Run marian decoder -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in > decoder.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in > decoder.out # Compare the output with the expected output $MRT_TOOLS/diff.sh decoder.out text.expected > decoder.diff diff --git a/tests/_template/test_server.sh b/tests/_template/test_server.sh index 1714d85..ef71d19 100644 --- a/tests/_template/test_server.sh +++ b/tests/_template/test_server.sh @@ -18,7 +18,7 @@ trap clean_up EXIT rm -f server.{log,out,diff} # Start marian server -$MRT_MARIAN/build/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 > server.log 2>&1 & +$MRT_MARIAN/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 > server.log 2>&1 & SERVER_PID=$! # Wait for server initialization @@ -26,7 +26,7 @@ sleep 20 grep -q "listening on port" server.log # Run client -python3 $MRT_MARIAN/scripts/server/client_example.py -p 8765 < text.in > server.out +python3 $MRT_MARIAN/../scripts/server/client_example.py -p 8765 < text.in > server.out kill $SERVER_PID # Compare the current output with the expected output diff --git a/tests/_template/test_training.sh b/tests/_template/test_training.sh index 551db67..e8fe321 100644 --- a/tests/_template/test_training.sh +++ b/tests/_template/test_training.sh @@ -13,7 +13,7 @@ rm -rf train train.{log,out,diff} mkdir -p train # Run marian command -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --dim-emb 32 --dim-rnn 64 --maxi-batch 1 --maxi-batch-sort none \ -m train/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v train/vocab.en.yml train/vocab.de.yml \ --after-batches 10 --disp-freq 2 \ diff --git a/tests/decoder/align-ensemble/test_align_ensemble.sh b/tests/decoder/align-ensemble/test_align_ensemble.sh index 066702c..4d77748 100644 --- a/tests/decoder/align-ensemble/test_align_ensemble.sh +++ b/tests/decoder/align-ensemble/test_align_ensemble.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.ensemble.yml --mini-batch 32 -b 5 --alignment < text.in > align.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.ensemble.yml --mini-batch 32 -b 5 --alignment < text.in > align.out $MRT_TOOLS/diff-nums.py -p 0.0001 align.out align.expected -o align.diff # Exit with success code diff --git a/tests/decoder/align-ensemble/test_align_ensemble_beam_1.sh b/tests/decoder/align-ensemble/test_align_ensemble_beam_1.sh index b4809d6..791a09e 100644 --- a/tests/decoder/align-ensemble/test_align_ensemble_beam_1.sh +++ b/tests/decoder/align-ensemble/test_align_ensemble_beam_1.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.ensemble.yml --mini-batch 1 -b 1 --alignment < text.in > align.b1.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.ensemble.yml --mini-batch 1 -b 1 --alignment < text.in > align.b1.out $MRT_TOOLS/diff-nums.py -p 0.0001 align.b1.out align.b1.expected -o align.b1.diff # Exit with success code diff --git a/tests/decoder/align/test_align.sh b/tests/decoder/align/test_align.sh index 7c0028d..f33f6b0 100644 --- a/tests/decoder/align/test_align.sh +++ b/tests/decoder/align/test_align.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 5 --alignment < text.in > align.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 5 --alignment < text.in > align.out $MRT_TOOLS/diff-nums.py -p 0.0001 align.out align.expected -o align.diff # Exit with success code diff --git a/tests/decoder/align/test_align_beam_1.sh b/tests/decoder/align/test_align_beam_1.sh index f4869ac..041ef5a 100644 --- a/tests/decoder/align/test_align_beam_1.sh +++ b/tests/decoder/align/test_align_beam_1.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 1 -b 1 --alignment < text.in > align.b1.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 1 -b 1 --alignment < text.in > align.b1.out $MRT_TOOLS/diff-nums.py -p 0.0001 align.b1.out align.b1.expected -o align.b1.diff # Exit with success code diff --git a/tests/decoder/align/test_align_beam_1_batched.sh b/tests/decoder/align/test_align_beam_1_batched.sh index 68ca362..a952a16 100644 --- a/tests/decoder/align/test_align_beam_1_batched.sh +++ b/tests/decoder/align/test_align_beam_1_batched.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 32 -b 1 --alignment < text.in > align.batched.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 32 -b 1 --alignment < text.in > align.batched.out $MRT_TOOLS/diff-nums.py -p 0.0001 align.batched.out align.batched.expected -o align.batched.diff # Exit with success code diff --git a/tests/decoder/align/test_align_nbest.sh b/tests/decoder/align/test_align_nbest.sh index 8add32f..28a7841 100644 --- a/tests/decoder/align/test_align_nbest.sh +++ b/tests/decoder/align/test_align_nbest.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 3 --n-best --alignment < text.in > align_nbest.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 3 --n-best --alignment < text.in > align_nbest.out $MRT_TOOLS/diff-nums.py -p 0.0001 align_nbest.out align_nbest.expected -o align_nbest.diff # Exit with success code diff --git a/tests/decoder/align/test_align_threshold.sh b/tests/decoder/align/test_align_threshold.sh index 6c672b1..e000ec6 100644 --- a/tests/decoder/align/test_align_threshold.sh +++ b/tests/decoder/align/test_align_threshold.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 5 --alignment 0.35 < text.in > align_threshold.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 16 -b 5 --alignment 0.35 < text.in > align_threshold.out $MRT_TOOLS/diff-nums.py -p 0.0001 align_threshold.out align_threshold.expected -o align_threshold.diff # Exit with success code diff --git a/tests/decoder/align/test_soft_align.sh b/tests/decoder/align/test_soft_align.sh index c08a2e2..77fcfaf 100644 --- a/tests/decoder/align/test_soft_align.sh +++ b/tests/decoder/align/test_soft_align.sh @@ -5,7 +5,7 @@ set -e # Test code goes here rm -f soft.out soft.raw.out -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 5 -b 5 --alignment soft < text.in > soft.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 5 -b 5 --alignment soft < text.in > soft.out $MRT_TOOLS/diff-nums.py -s , -p 0.0001 soft.out soft.expected -o soft.diff # Exit with success code diff --git a/tests/decoder/align/test_soft_align_nbest.sh b/tests/decoder/align/test_soft_align_nbest.sh index 57418c3..813bd87 100644 --- a/tests/decoder/align/test_soft_align_nbest.sh +++ b/tests/decoder/align/test_soft_align_nbest.sh @@ -5,7 +5,7 @@ set -e # Test code goes here rm -f soft.nbest.out soft.nbest.raw.out -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 5 -b 3 --n-best --alignment soft < text.in > soft.nbest.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --mini-batch 5 -b 3 --n-best --alignment soft < text.in > soft.nbest.out $MRT_TOOLS/diff-nums.py -s , -p 0.0001 soft.nbest.out soft.nbest.expected -o soft.nbest.diff # Exit with success code diff --git a/tests/decoder/wmt16/test_ende.sh b/tests/decoder/wmt16/test_ende.sh index 04b5f5b..d1ba84c 100644 --- a/tests/decoder/wmt16/test_ende.sh +++ b/tests/decoder/wmt16/test_ende.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in > text.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in > text.out $MRT_TOOLS/diff.sh text.out text.expected > text.diff # Exit with success code diff --git a/tests/decoder/wmt16/test_ende_cpu.sh b/tests/decoder/wmt16/test_ende_cpu.sh index f183315..7937fc2 100644 --- a/tests/decoder/wmt16/test_ende_cpu.sh +++ b/tests/decoder/wmt16/test_ende_cpu.sh @@ -10,7 +10,7 @@ if [ ! $MRT_MARIAN_USE_MKL ]; then fi # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --cpu-threads 4 < text.in > text_cpu.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml --cpu-threads 4 < text.in > text_cpu.out $MRT_TOOLS/diff.sh text_cpu.out text.expected > text_cpu.diff # Exit with success code diff --git a/tests/decoder/wmt16/test_ende_logs.sh b/tests/decoder/wmt16/test_ende_logs.sh index 55b6542..179f846 100644 --- a/tests/decoder/wmt16/test_ende_logs.sh +++ b/tests/decoder/wmt16/test_ende_logs.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in 2> logs.raw +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml < text.in 2> logs.raw cat logs.raw | grep "] Best translation" | sed -r "s/.*Best translation [0-9]+ : (.*)/\1/" > logs.out $MRT_TOOLS/diff.sh logs.out text.expected > logs.diff diff --git a/tests/decoder/wmt16/test_nbest.sh b/tests/decoder/wmt16/test_nbest.sh index a002bdb..cf8d6be 100644 --- a/tests/decoder/wmt16/test_nbest.sh +++ b/tests/decoder/wmt16/test_nbest.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 5 --n-best < text.in > nbest.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 5 --n-best < text.in > nbest.out $MRT_TOOLS/diff-nums.py nbest.out nbest.expected -o nbest.diff # Exit with success code diff --git a/tests/decoder/wmt17/test_ende.sh b/tests/decoder/wmt17/test_ende.sh index 902b16e..baa2cad 100644 --- a/tests/decoder/wmt17/test_ende.sh +++ b/tests/decoder/wmt17/test_ende.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml < text.in | tail -n 99 > text.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml < text.in | tail -n 99 > text.out $MRT_TOOLS/diff.sh text.out text.expected > text.diff # Exit with success code diff --git a/tests/decoder/wmt17/test_nbest.sh b/tests/decoder/wmt17/test_nbest.sh index 05a4053..8d5a0e5 100644 --- a/tests/decoder/wmt17/test_nbest.sh +++ b/tests/decoder/wmt17/test_nbest.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml \ +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml \ -b 5 --n-best --normalize < text.in | tail -n +6 > nbest.out # Compare n-best lists diff --git a/tests/examples/iris/setup.sh b/tests/examples/iris/setup.sh index 98f9868..4b3b57e 100644 --- a/tests/examples/iris/setup.sh +++ b/tests/examples/iris/setup.sh @@ -1 +1 @@ -test -e $MRT_MARIAN/build/iris_example || exit 1 +test -e $MRT_MARIAN/iris_example || exit 1 diff --git a/tests/examples/iris/test_iris.sh b/tests/examples/iris/test_iris.sh index ec6bde3..1bf6e66 100644 --- a/tests/examples/iris/test_iris.sh +++ b/tests/examples/iris/test_iris.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/iris_example > iris.out +$MRT_MARIAN/iris_example > iris.out $MRT_TOOLS/diff-nums.py iris.out iris.expected -o iris.diff # Exit with success code diff --git a/tests/examples/mnist/setup.sh b/tests/examples/mnist/setup.sh index 820da25..8dead6a 100644 --- a/tests/examples/mnist/setup.sh +++ b/tests/examples/mnist/setup.sh @@ -4,7 +4,7 @@ set -e # Setup code goes here -test -e $MRT_MARIAN/build/mnist_example +test -e $MRT_MARIAN/mnist_example (( $(ls -1 *-ubyte 2>/dev/null | wc -l) == 4 )) && exit 0 diff --git a/tests/examples/mnist/test_mnist_ffnn.sh b/tests/examples/mnist/test_mnist_ffnn.sh index eeb3f30..bbd1115 100644 --- a/tests/examples/mnist/test_mnist_ffnn.sh +++ b/tests/examples/mnist/test_mnist_ffnn.sh @@ -10,7 +10,7 @@ clean_up() { trap clean_up EXIT # Test code goes here -$MRT_MARIAN/build/mnist_example \ +$MRT_MARIAN/mnist_example \ --after-epochs 20 --mini-batch 200 --valid-freq 600 --maxi-batch 1 \ --train-sets train-images-idx3-ubyte train-labels-idx1-ubyte \ --valid-sets t10k-images-idx3-ubyte t10k-labels-idx1-ubyte \ diff --git a/tests/interface/config/test_alias_best_deep.sh b/tests/interface/config/test_alias_best_deep.sh index 06aa0d4..24ba88e 100644 --- a/tests/interface/config/test_alias_best_deep.sh +++ b/tests/interface/config/test_alias_best_deep.sh @@ -8,7 +8,7 @@ mkdir -p bestdeep # Test -$MRT_MARIAN/build/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m bestdeep/model.npz -v vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m bestdeep/model.npz -v vocab.de.yml vocab.en.yml \ --type s2s --dim-emb 32 --dim-rnn 16 --mini-batch 1 --after-batches 1 --no-shuffle \ --best-deep --log bestdeep.log diff --git a/tests/interface/config/test_alias_can_be_overwritten_by_cmd_options.sh b/tests/interface/config/test_alias_can_be_overwritten_by_cmd_options.sh index a601bcd..3febf1c 100644 --- a/tests/interface/config/test_alias_can_be_overwritten_by_cmd_options.sh +++ b/tests/interface/config/test_alias_can_be_overwritten_by_cmd_options.sh @@ -8,7 +8,7 @@ mkdir -p bestdeep_overwrite # Test -$MRT_MARIAN/build/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m bestdeep_overwrite/model.npz -v vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m bestdeep_overwrite/model.npz -v vocab.de.yml vocab.en.yml \ --type s2s --dim-emb 32 --dim-rnn 16 --mini-batch 1 --after-batches 1 --no-shuffle \ --best-deep --enc-depth 6 --dec-depth 6 --log bestdeep_overwrite.log diff --git a/tests/interface/config/test_cmd_options_overwrite_config_file.sh b/tests/interface/config/test_cmd_options_overwrite_config_file.sh index de15137..2ca97fd 100644 --- a/tests/interface/config/test_cmd_options_overwrite_config_file.sh +++ b/tests/interface/config/test_cmd_options_overwrite_config_file.sh @@ -7,7 +7,7 @@ rm -rf overwrite overwrite.log mkdir -p overwrite # Test -$MRT_MARIAN/build/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m overwrite/model.npz -v vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m overwrite/model.npz -v vocab.de.yml vocab.en.yml \ --config overwrite.yml --type s2s --mini-batch 1 --after-batches 1 --no-shuffle \ --log overwrite.log diff --git a/tests/interface/config/test_dump_config.sh b/tests/interface/config/test_dump_config.sh index 12c9530..c98d544 100644 --- a/tests/interface/config/test_dump_config.sh +++ b/tests/interface/config/test_dump_config.sh @@ -6,7 +6,7 @@ set -e rm -rf dump_config.yml # Run with no config file -$MRT_MARIAN/build/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --vocabs vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --vocabs vocab.de.yml vocab.en.yml \ --type s2s --mini-batch 8 --dim-rnn 32 --dim-emb 16 --after-batches 2 --dump-config > dump_config.yml test -e dump_config.yml diff --git a/tests/interface/config/test_dump_config_minimal.sh b/tests/interface/config/test_dump_config_minimal.sh index 430b761..5f6f0eb 100644 --- a/tests/interface/config/test_dump_config_minimal.sh +++ b/tests/interface/config/test_dump_config_minimal.sh @@ -6,7 +6,7 @@ set -e rm -f dump_minimal.{yml,out} # Run with no config file -$MRT_MARIAN/build/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --vocabs vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --vocabs vocab.de.yml vocab.en.yml \ --type s2s --mini-batch 8 --dim-rnn 32 --dim-emb 16 --after-batches 2 --dump-config minimal > dump_minimal.yml # Remove first line and paths to train sets and vocabs diff --git a/tests/interface/config/test_dump_config_with_relative_paths.sh b/tests/interface/config/test_dump_config_with_relative_paths.sh index 63db969..c15edc6 100644 --- a/tests/interface/config/test_dump_config_with_relative_paths.sh +++ b/tests/interface/config/test_dump_config_with_relative_paths.sh @@ -6,7 +6,7 @@ set -e rm -rf dump_relpaths.yml # Run with no config file -$MRT_MARIAN/build/marian-decoder -c relpaths.yml --mini-batch 8 --dump-config > dump_relpaths.yml +$MRT_MARIAN/marian-decoder -c relpaths.yml --mini-batch 8 --dump-config > dump_relpaths.yml test -e dump_relpaths.yml diff --git a/tests/interface/config/test_load_config.sh b/tests/interface/config/test_load_config.sh index 8947347..0e66fd4 100644 --- a/tests/interface/config/test_load_config.sh +++ b/tests/interface/config/test_load_config.sh @@ -7,7 +7,7 @@ rm -rf load_config load_config.log no_config.log mkdir -p load_config # Run with no config file -$MRT_MARIAN/build/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ --type s2s --mini-batch 8 --dim-rnn 32 --dim-emb 16 --after-batches 2 --log load_config.log test -e load_config/model.npz @@ -19,7 +19,7 @@ rm -rf load_config mkdir -p load_config # Run with config file and the same options -$MRT_MARIAN/build/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ +$MRT_MARIAN/marian --train-sets $MRT_DATA/europarl.de-en/corpus.bpe.{de,en} --model load_config/model.npz --vocabs vocab.de.yml vocab.en.yml --no-shuffle \ --config load_config.yml --log load_config.log test -e load_config/model.npz diff --git a/tests/interface/config/test_relative_paths.sh b/tests/interface/config/test_relative_paths.sh index 2e5b199..c37e579 100644 --- a/tests/interface/config/test_relative_paths.sh +++ b/tests/interface/config/test_relative_paths.sh @@ -6,7 +6,7 @@ set -e rm -f relpaths.log # Test code goes here -echo "this is a test" | $MRT_MARIAN/build/marian-decoder -c relpaths.yml --log relpaths.log +echo "this is a test" | $MRT_MARIAN/marian-decoder -c relpaths.yml --log relpaths.log test -e relpaths.log if ! grep -q "\.\." relpaths.log; then exit 0; else exit 1; fi diff --git a/tests/interface/config/test_relative_paths_apply_only_to_config_files.sh b/tests/interface/config/test_relative_paths_apply_only_to_config_files.sh index c8ab7d4..8f2220a 100644 --- a/tests/interface/config/test_relative_paths_apply_only_to_config_files.sh +++ b/tests/interface/config/test_relative_paths_apply_only_to_config_files.sh @@ -6,7 +6,7 @@ set -e rm -f relpaths_to_cfgfile.log # Test code goes here -echo "this is a test" | $MRT_MARIAN/build/marian-decoder -c relpaths_to_cfgfile.yml -m ../../../models/wmt16_systems/en-de/model.npz --log relpaths_to_cfgfile.log +echo "this is a test" | $MRT_MARIAN/marian-decoder -c relpaths_to_cfgfile.yml -m ../../../models/wmt16_systems/en-de/model.npz --log relpaths_to_cfgfile.log test -e relpaths_to_cfgfile.log diff --git a/tests/interface/config/test_relative_paths_are_not_applied_to_cmd_options.sh b/tests/interface/config/test_relative_paths_are_not_applied_to_cmd_options.sh index 61de1fe..5ea7273 100644 --- a/tests/interface/config/test_relative_paths_are_not_applied_to_cmd_options.sh +++ b/tests/interface/config/test_relative_paths_are_not_applied_to_cmd_options.sh @@ -9,7 +9,7 @@ mkdir -p subdir # Test code goes here echo "this is a test" > relpaths_nocmd.in -$MRT_MARIAN/build/marian-decoder -c relpaths.yml -i ./subdir/../relpaths_nocmd.in -o ./subdir/../relpaths_nocmd.out --log relpaths_nocmd.log +$MRT_MARIAN/marian-decoder -c relpaths.yml -i ./subdir/../relpaths_nocmd.in -o ./subdir/../relpaths_nocmd.out --log relpaths_nocmd.log test -e relpaths_nocmd.log test -e relpaths_nocmd.out diff --git a/tests/interface/config/test_relative_paths_for_each_config_file.sh b/tests/interface/config/test_relative_paths_for_each_config_file.sh index aa88007..b1b24f5 100644 --- a/tests/interface/config/test_relative_paths_for_each_config_file.sh +++ b/tests/interface/config/test_relative_paths_for_each_config_file.sh @@ -6,7 +6,7 @@ set -e rm -f relpaths_subdir.log # Test code goes here -echo "this is a test" | $MRT_MARIAN/build/marian-decoder -c relpaths_subdir.yml subdir/relpaths_subdir.yml --log relpaths_subdir.log +echo "this is a test" | $MRT_MARIAN/marian-decoder -c relpaths_subdir.yml subdir/relpaths_subdir.yml --log relpaths_subdir.log test -e relpaths_subdir.log diff --git a/tests/interface/config/test_relative_paths_for_input_in_config_file.sh b/tests/interface/config/test_relative_paths_for_input_in_config_file.sh index b635a1a..f070f74 100644 --- a/tests/interface/config/test_relative_paths_for_input_in_config_file.sh +++ b/tests/interface/config/test_relative_paths_for_input_in_config_file.sh @@ -9,7 +9,7 @@ mkdir -p subdir # Test code goes here echo "this is a test" > relpaths_input.in -$MRT_MARIAN/build/marian-decoder -c ./subdir/relpaths_input.yml -o relpaths_input.out --log relpaths_input.log +$MRT_MARIAN/marian-decoder -c ./subdir/relpaths_input.yml -o relpaths_input.out --log relpaths_input.log test -e relpaths_input.log test -e relpaths_input.out diff --git a/tests/interface/config/test_turn_off_boolean_from_config.sh b/tests/interface/config/test_turn_off_boolean_from_config.sh index 16cf6b1..617bc90 100644 --- a/tests/interface/config/test_turn_off_boolean_from_config.sh +++ b/tests/interface/config/test_turn_off_boolean_from_config.sh @@ -7,7 +7,7 @@ rm -rf booloff booloff.log mkdir -p booloff # Test -$MRT_MARIAN/build/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m booloff/model.npz -v vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m booloff/model.npz -v vocab.de.yml vocab.en.yml \ --dim-emb 32 --dim-rnn 16 --mini-batch 1 --after-batches 1 --no-shuffle \ --config booloff.yml --layer-normalization false --log booloff.log diff --git a/tests/interface/config/test_turn_off_vector_from_config.sh b/tests/interface/config/test_turn_off_vector_from_config.sh index 83608c0..4eb29fa 100644 --- a/tests/interface/config/test_turn_off_vector_from_config.sh +++ b/tests/interface/config/test_turn_off_vector_from_config.sh @@ -7,7 +7,7 @@ rm -rf vectoroff vectoroff.log mkdir -p vectoroff # Test -$MRT_MARIAN/build/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m vectoroff/model.npz -v vocab.de.yml vocab.en.yml \ +$MRT_MARIAN/marian -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -m vectoroff/model.npz -v vocab.de.yml vocab.en.yml \ --dim-emb 32 --dim-rnn 16 --mini-batch 1 --after-batches 1 --no-shuffle \ --valid-metrics [] --transformer-tied-layers [] --config vectoroff.yml --log vectoroff.log diff --git a/tests/interface/envvars/test_interpolate_envvars.sh b/tests/interface/envvars/test_interpolate_envvars.sh index d331559..dfe629a 100644 --- a/tests/interface/envvars/test_interpolate_envvars.sh +++ b/tests/interface/envvars/test_interpolate_envvars.sh @@ -7,11 +7,11 @@ set -e rm -f envvars.out export MRTMODELDIR=wmt16_systems -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/'${MRTMODELDIR}'/marian.en-de.yml --interpolate-env-vars -i text.in > envvars.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/'${MRTMODELDIR}'/marian.en-de.yml --interpolate-env-vars -i text.in > envvars.out $MRT_TOOLS/diff.sh envvars.out text.expected > envvars.diff # Without --interpolate-env-vars this should fail -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/'${MRTMODELDIR}'/marian.en-de.yml -i text.in > envvars.log 2>&1 || true +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/'${MRTMODELDIR}'/marian.en-de.yml -i text.in > envvars.log 2>&1 || true grep -q "could not be opened" envvars.log # Exit with success code diff --git a/tests/interface/input/test_empty_file.sh b/tests/interface/input/test_empty_file.sh index dd6a45d..81e9cee 100644 --- a/tests/interface/input/test_empty_file.sh +++ b/tests/interface/input/test_empty_file.sh @@ -7,7 +7,7 @@ rm -f empty_file.log empty_file.in touch empty_file.in # Test code goes here -$MRT_MARIAN/build/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ +$MRT_MARIAN/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 -v $MRT_MODELS/wmt16_systems/en-de/vocab.{en,de}.json \ -i empty_file.in > empty_file.log 2>&1 || true diff --git a/tests/interface/model/_test_no_permission_for_saving.sh b/tests/interface/model/_test_no_permission_for_saving.sh index a4f84dc..0ee789c 100644 --- a/tests/interface/model/_test_no_permission_for_saving.sh +++ b/tests/interface/model/_test_no_permission_for_saving.sh @@ -12,7 +12,7 @@ trap clean_up EXIT mkdir -p nowrite chmod a-w nowrite -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m nowrite/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --no-shuffle --after-batches 1 \ diff --git a/tests/interface/model/test_decoding_with_no_model.sh b/tests/interface/model/test_decoding_with_no_model.sh index b027326..efc1980 100644 --- a/tests/interface/model/test_decoding_with_no_model.sh +++ b/tests/interface/model/test_decoding_with_no_model.sh @@ -7,7 +7,7 @@ rm -f nomodel_decoder.log echo "this is a test ." > file.in # Test code goes here -$MRT_MARIAN/build/marian-decoder --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ +$MRT_MARIAN/marian-decoder --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -i file.in > nomodel_decoder.log 2>&1 || true test -e nomodel_decoder.log diff --git a/tests/interface/model/test_decoding_with_nonexistent_1_of_2_models.sh b/tests/interface/model/test_decoding_with_nonexistent_1_of_2_models.sh index baecaff..cfd91ce 100644 --- a/tests/interface/model/test_decoding_with_nonexistent_1_of_2_models.sh +++ b/tests/interface/model/test_decoding_with_nonexistent_1_of_2_models.sh @@ -7,7 +7,7 @@ rm -f nomodel_1of2_decoder.log echo "this is a test ." > file.in # Test code goes here -$MRT_MARIAN/build/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz /non/existent/path/to/1/of/2/models.npz \ +$MRT_MARIAN/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz /non/existent/path/to/1/of/2/models.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ -i file.in > nomodel_1of2_decoder.log 2>&1 || true diff --git a/tests/interface/model/test_decoding_with_nonexistent_model.sh b/tests/interface/model/test_decoding_with_nonexistent_model.sh index 85b4693..49c1be3 100644 --- a/tests/interface/model/test_decoding_with_nonexistent_model.sh +++ b/tests/interface/model/test_decoding_with_nonexistent_model.sh @@ -7,7 +7,7 @@ rm -f nomodel_1_decoder.log echo "this is a test ." > file.in # Test code goes here -$MRT_MARIAN/build/marian-decoder -m /non/existent/path/to/model.npz \ +$MRT_MARIAN/marian-decoder -m /non/existent/path/to/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ -i file.in > nomodel_1_decoder.log 2>&1 || true diff --git a/tests/interface/model/test_model_without_npz_suffix.sh b/tests/interface/model/test_model_without_npz_suffix.sh index 7467269..47193e4 100644 --- a/tests/interface/model/test_model_without_npz_suffix.sh +++ b/tests/interface/model/test_model_without_npz_suffix.sh @@ -7,7 +7,7 @@ rm -rf npzsuffix mkdir -p npzsuffix # Test code goes here -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m ./npzsuffix/model \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v $MRT_MODELS/wmt16_systems/en-de/vocab.{en,de}.json \ --no-shuffle --dim-emb 32 --dim-rnn 32 \ diff --git a/tests/interface/model/test_nonexistent_model_path.sh b/tests/interface/model/test_nonexistent_model_path.sh index 433e089..12c84d1 100644 --- a/tests/interface/model/test_nonexistent_model_path.sh +++ b/tests/interface/model/test_nonexistent_model_path.sh @@ -4,7 +4,7 @@ set -e # Test code goes here -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m /non/existent/path/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v $MRT_MODELS/wmt16_systems/en-de/vocab.{en,de}.json \ --no-shuffle --after-batches 1 \ diff --git a/tests/interface/model/test_scoring_with_nonexistent_model.sh b/tests/interface/model/test_scoring_with_nonexistent_model.sh index 4efd309..f42cbe9 100644 --- a/tests/interface/model/test_scoring_with_nonexistent_model.sh +++ b/tests/interface/model/test_scoring_with_nonexistent_model.sh @@ -8,7 +8,7 @@ echo "this is a test ." > file.in echo "das ist ein test ." > file.out # Test code goes here -$MRT_MARIAN/build/marian-scorer -m /non/existent/path/to/model.npz \ +$MRT_MARIAN/marian-scorer -m /non/existent/path/to/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -v $MRT_MODELS/wmt16_systems/en-de/vocab.en.json $MRT_MODELS/wmt16_systems/en-de/vocab.de.json \ -t file.in file.out > nomodel_scorer.log 2>&1 || true diff --git a/tests/interface/version/test_model_has_version.sh b/tests/interface/version/test_model_has_version.sh index 62e0578..8e5837c 100644 --- a/tests/interface/version/test_model_has_version.sh +++ b/tests/interface/version/test_model_has_version.sh @@ -8,7 +8,7 @@ rm -rf version version.log mkdir -p version # Train a model -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m version/model.npz \ -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} \ -v version/vocab.en.yml version/vocab.de.yml \ @@ -21,10 +21,10 @@ rm -f version.log # Check if the model contains a version test -e version/model.npz -python3 $MRT_MARIAN/scripts/contrib/model_info.py -s -m version/model.npz | grep -qP "version: v[1-9]+\.[0-9]+\.[0-9]+.*" +python3 $MRT_MARIAN/../scripts/contrib/model_info.py -s -m version/model.npz | grep -qP "version: v[1-9]+\.[0-9]+\.[0-9]+.*" # Check if the version is printed during decoding -echo "test" | $MRT_MARIAN/build/marian-decoder \ +echo "test" | $MRT_MARIAN/marian-decoder \ -m version/model.npz -v version/vocab.en.yml version/vocab.de.yml \ --log version.log diff --git a/tests/interface/version/test_no_version_from_old_models.sh b/tests/interface/version/test_no_version_from_old_models.sh index e532053..9735f70 100644 --- a/tests/interface/version/test_no_version_from_old_models.sh +++ b/tests/interface/version/test_no_version_from_old_models.sh @@ -7,7 +7,7 @@ set -e rm -rf noversion noversion.log mkdir -p noversion -echo "test" | $MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml 2> noversion.log +echo "test" | $MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml 2> noversion.log test -e noversion.log if grep -q "created with Marian" noversion.log; then diff --git a/tests/interface/vocab/_test_training_with_no_permission_for_vocab.sh b/tests/interface/vocab/_test_training_with_no_permission_for_vocab.sh index 6816618..8a55cec 100644 --- a/tests/interface/vocab/_test_training_with_no_permission_for_vocab.sh +++ b/tests/interface/vocab/_test_training_with_no_permission_for_vocab.sh @@ -8,7 +8,7 @@ rm -rf nowrite nowrite.log mkdir -p nowrite chmod a-w nowrite -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -v nowrite/vocab.en.yml nowrite/vocab.de.yml \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \ --no-shuffle --after-batches 1 \ diff --git a/tests/interface/vocab/test_decoding_with_empty_vocab.sh b/tests/interface/vocab/test_decoding_with_empty_vocab.sh index cf66cd2..d8c8f51 100644 --- a/tests/interface/vocab/test_decoding_with_empty_vocab.sh +++ b/tests/interface/vocab/test_decoding_with_empty_vocab.sh @@ -8,7 +8,7 @@ echo "this is a test ." > file.in touch emptyvoc.yml # Test code goes here -$MRT_MARIAN/build/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ +$MRT_MARIAN/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 -v emptyvoc.yml emptyvoc.yml \ -i file.in > emptyvoc_decoder.log 2>&1 || true diff --git a/tests/interface/vocab/test_decoding_with_no_vocab.sh b/tests/interface/vocab/test_decoding_with_no_vocab.sh index 3f9283b..2dc0abd 100644 --- a/tests/interface/vocab/test_decoding_with_no_vocab.sh +++ b/tests/interface/vocab/test_decoding_with_no_vocab.sh @@ -7,7 +7,7 @@ rm -f novocab_decoder.log echo "this is a test ." > file.in # Test code goes here -$MRT_MARIAN/build/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ +$MRT_MARIAN/marian-decoder -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -i file.in > novocab_decoder.log 2>&1 || true diff --git a/tests/interface/vocab/test_scoring_with_no_vocab.sh b/tests/interface/vocab/test_scoring_with_no_vocab.sh index e4035df..aa7d2ef 100644 --- a/tests/interface/vocab/test_scoring_with_no_vocab.sh +++ b/tests/interface/vocab/test_scoring_with_no_vocab.sh @@ -9,7 +9,7 @@ echo "this is a test ." > file.in echo "das ist ein test ." > file.out # Test code goes here -$MRT_MARIAN/build/marian-scorer -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ +$MRT_MARIAN/marian-scorer -m $MRT_MODELS/wmt16_systems/en-de/model.npz \ --type amun --dim-emb 500 --dim-vocabs 85000 85000 \ -t file.in file.out > novocab_scorer.log 2>&1 || true diff --git a/tests/interface/vocab/test_training_with_no_vocab.sh b/tests/interface/vocab/test_training_with_no_vocab.sh index a94a1e6..a970088 100644 --- a/tests/interface/vocab/test_training_with_no_vocab.sh +++ b/tests/interface/vocab/test_training_with_no_vocab.sh @@ -18,7 +18,7 @@ rm -f $MRT_DATA/europarl.de-en/corpus.bpe.en.yml $MRT_DATA/europarl.de-en/corpus mkdir -p novocab # Run Marian with no vocabs provided, expect to create new vocabs -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m novocab/model1.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \ $opts \ @@ -31,7 +31,7 @@ test -e novocab_create.log grep -q "Creating vocabulary" novocab_create.log # Run Marian with no vocabs provided, expect to load existing vocabs -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m novocab/model2.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de \ $opts \ diff --git a/tests/models/_char-s2s/test_char_s2s.sh b/tests/models/_char-s2s/test_char_s2s.sh index 4d9cd62..318c675 100644 --- a/tests/models/_char-s2s/test_char_s2s.sh +++ b/tests/models/_char-s2s/test_char_s2s.sh @@ -9,7 +9,7 @@ if [ ! $MRT_MARIAN_USE_CUDNN ]; then fi # Test code goes here -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/char-s2s/translate.yml < text.in > text.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/char-s2s/translate.yml < text.in > text.out $MRT_TOOLS/diff.sh text.out text.expected > text.diff # Exit with success code diff --git a/tests/models/_char-s2s/test_compare_scores.sh b/tests/models/_char-s2s/test_compare_scores.sh index 51df88c..127f836 100755 --- a/tests/models/_char-s2s/test_compare_scores.sh +++ b/tests/models/_char-s2s/test_compare_scores.sh @@ -9,7 +9,7 @@ if [ ! $MRT_MARIAN_USE_CUDNN ]; then fi # Translate with s2s -$MRT_MARIAN/build/marian-decoder \ +$MRT_MARIAN/marian-decoder \ -c $MRT_MODELS/char-s2s/translate.yml \ -b 12 \ --n-best < text.in > nbest.out @@ -22,7 +22,7 @@ cat text.in | perl -ne 'for$i(1..12){print}' > compare.char.src cat nbest.out | sed 's/ ||| /\t/g' | cut -f2 > compare.char.trg # Run rescorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/char-s2s/translate.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/char-s2s/translate.yml \ -m $MRT_MODELS/char-s2s/model.npz \ --max-length 7000 \ --workspace 256 \ diff --git a/tests/models/transformer/test_hard_aligns.sh b/tests/models/transformer/test_hard_aligns.sh index 471d1e7..f3e58fd 100644 --- a/tests/models/transformer/test_hard_aligns.sh +++ b/tests/models/transformer/test_hard_aligns.sh @@ -6,7 +6,7 @@ set -e rm -f hardalign.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --alignment < text.in > hardalign.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --alignment < text.in > hardalign.out $MRT_TOOLS/diff.sh hardalign.out text.b6.hardalign.expected > hardalign.diff diff --git a/tests/models/transformer/test_nbest.sh b/tests/models/transformer/test_nbest.sh index 1d725db..007227a 100644 --- a/tests/models/transformer/test_nbest.sh +++ b/tests/models/transformer/test_nbest.sh @@ -6,7 +6,7 @@ set -e rm -f nbest.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --n-best < text.in > nbest.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --n-best < text.in > nbest.out $MRT_TOOLS/diff-nums.py -p 0.0003 nbest.out text.b6.nbest.expected -o nbest.diff diff --git a/tests/models/transformer/test_soft_aligns.sh b/tests/models/transformer/test_soft_aligns.sh index 651392c..4e00989 100644 --- a/tests/models/transformer/test_soft_aligns.sh +++ b/tests/models/transformer/test_soft_aligns.sh @@ -6,7 +6,7 @@ set -e rm -f softalign.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --alignment soft < text.in > softalign.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 --alignment soft < text.in > softalign.out $MRT_TOOLS/diff-nums.py -s , -p 0.0001 softalign.out text.b6.softalign.expected -o softalign.diff diff --git a/tests/models/transformer/test_translation.sh b/tests/models/transformer/test_translation.sh index 2ad73e3..551a757 100644 --- a/tests/models/transformer/test_translation.sh +++ b/tests/models/transformer/test_translation.sh @@ -6,7 +6,7 @@ set -e rm -f transformer.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 < text.in > transformer.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 < text.in > transformer.out $MRT_TOOLS/diff.sh transformer.out text.b6.expected > transformer.diff diff --git a/tests/models/transformer/test_translation_batched.sh b/tests/models/transformer/test_translation_batched.sh index 1da9fda..2e06827 100644 --- a/tests/models/transformer/test_translation_batched.sh +++ b/tests/models/transformer/test_translation_batched.sh @@ -6,7 +6,7 @@ set -e rm -f batched.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 < text.in > batched.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/transformer/decode.yml -b 6 --mini-batch 32 < text.in > batched.out $MRT_TOOLS/diff.sh batched.out text.b6.expected > batched.diff diff --git a/tests/models/wmt16-ende/test_translation_b6n.sh b/tests/models/wmt16-ende/test_translation_b6n.sh index e712a29..1ef676e 100644 --- a/tests/models/wmt16-ende/test_translation_b6n.sh +++ b/tests/models/wmt16-ende/test_translation_b6n.sh @@ -4,7 +4,7 @@ set -e # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 < text.b6n.in > marian.b6n.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 < text.b6n.in > marian.b6n.out # Compare with Marian and Nematus $MRT_TOOLS/diff.sh marian.b6n.out marian.b6n.expected > marian.b6n.diff diff --git a/tests/models/wmt16-ende/test_translation_b6n_batch32.sh b/tests/models/wmt16-ende/test_translation_b6n_batch32.sh index 831b845..671d3b2 100644 --- a/tests/models/wmt16-ende/test_translation_b6n_batch32.sh +++ b/tests/models/wmt16-ende/test_translation_b6n_batch32.sh @@ -4,7 +4,7 @@ set -e # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 \ +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 \ --mini-batch 32 --maxi-batch 10 --maxi-batch-sort src -w 2500 \ < text.b6n.in > marian.batch32.out diff --git a/tests/models/wmt16-ende/test_translation_b6n_batch64.sh b/tests/models/wmt16-ende/test_translation_b6n_batch64.sh index 76ebea0..5fcc928 100644 --- a/tests/models/wmt16-ende/test_translation_b6n_batch64.sh +++ b/tests/models/wmt16-ende/test_translation_b6n_batch64.sh @@ -4,7 +4,7 @@ set -e # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 \ +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -b 6 -n 1.0 \ --mini-batch 64 --maxi-batch 10 --maxi-batch-sort src -w 2500 \ < text.b6n.in > marian.batch64.out diff --git a/tests/models/wmt17-ende/test_translation_b6n.sh b/tests/models/wmt17-ende/test_translation_b6n.sh index 07ec8e6..7b52f8b 100644 --- a/tests/models/wmt17-ende/test_translation_b6n.sh +++ b/tests/models/wmt17-ende/test_translation_b6n.sh @@ -6,7 +6,7 @@ set -e rm -f marian.b6n.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml -b 6 -n 1.0 < text.b6n.in > marian.b6n.out +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml -b 6 -n 1.0 < text.b6n.in > marian.b6n.out # Compare with Marian and Nematus $MRT_TOOLS/diff.sh marian.b6n.out marian.b6n.expected > marian.b6n.diff diff --git a/tests/models/wmt17-ende/test_translation_b6n_batch32.sh b/tests/models/wmt17-ende/test_translation_b6n_batch32.sh index 49ae6af..9379609 100644 --- a/tests/models/wmt17-ende/test_translation_b6n_batch32.sh +++ b/tests/models/wmt17-ende/test_translation_b6n_batch32.sh @@ -7,7 +7,7 @@ set -e rm -f marian.batch32.out # Run Marian -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml -b 6 -n 1.0 \ +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt17_systems/marian.en-de.yml -b 6 -n 1.0 \ --mini-batch 32 --maxi-batch 10 --maxi-batch-sort src -w 2500 \ < text.b6n.in > marian.batch32.out diff --git a/tests/models/wnmt18/test_student_small.sh b/tests/models/wnmt18/test_student_small.sh index 74724f9..716a4f5 100644 --- a/tests/models/wnmt18/test_student_small.sh +++ b/tests/models/wnmt18/test_student_small.sh @@ -11,7 +11,7 @@ fi model=model.student.small # Run test -cat newstest2014.in | $MRT_MARIAN/build/marian-decoder \ +cat newstest2014.in | $MRT_MARIAN/marian-decoder \ -m $MRT_MODELS/wnmt18/$model/model.npz \ -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \ --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b 1 \ diff --git a/tests/models/wnmt18/test_student_small_aan.sh b/tests/models/wnmt18/test_student_small_aan.sh index 73f3fd7..48645a8 100644 --- a/tests/models/wnmt18/test_student_small_aan.sh +++ b/tests/models/wnmt18/test_student_small_aan.sh @@ -11,7 +11,7 @@ fi model=model.student.small.aan # Run test -cat newstest2014.in | $MRT_MARIAN/build/marian-decoder \ +cat newstest2014.in | $MRT_MARIAN/marian-decoder \ -m $MRT_MODELS/wnmt18/$model/model.npz \ -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \ --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b 1 \ diff --git a/tests/models/wnmt18/test_student_small_aan_optimize.sh b/tests/models/wnmt18/test_student_small_aan_optimize.sh index fcfbe6d..19ea5a7 100644 --- a/tests/models/wnmt18/test_student_small_aan_optimize.sh +++ b/tests/models/wnmt18/test_student_small_aan_optimize.sh @@ -11,7 +11,7 @@ fi model=model.student.small.aan # Run test -cat newstest2014.in | $MRT_MARIAN/build/marian-decoder \ +cat newstest2014.in | $MRT_MARIAN/marian-decoder \ -m $MRT_MODELS/wnmt18/$model/model.npz \ -v $MRT_MODELS/wnmt18/vocab.ende.{yml,yml} \ --mini-batch-words 384 --mini-batch 100 --maxi-batch 100 --maxi-batch-sort src -b1 --optimize \ diff --git a/tests/scorer/align/test_scorer_align.sh b/tests/scorer/align/test_scorer_align.sh index 68fe68e..95fec1d 100644 --- a/tests/scorer/align/test_scorer_align.sh +++ b/tests/scorer/align/test_scorer_align.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/text.src.in $(pwd)/text.trg.in --alignment --mini-batch 16 \ | sed 's/^.* ||| //' > align.out diff --git a/tests/scorer/align/test_scorer_align_batch_1.sh b/tests/scorer/align/test_scorer_align_batch_1.sh index d81374b..b82f290 100644 --- a/tests/scorer/align/test_scorer_align_batch_1.sh +++ b/tests/scorer/align/test_scorer_align_batch_1.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/text.src.in $(pwd)/text.trg.in --alignment --mini-batch 1 \ | sed 's/^.* ||| //' > align.b1.out diff --git a/tests/scorer/align/test_scorer_align_nbest.sh b/tests/scorer/align/test_scorer_align_nbest.sh index 3f58630..680d918 100644 --- a/tests/scorer/align/test_scorer_align_nbest.sh +++ b/tests/scorer/align/test_scorer_align_nbest.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/text.src.in $(pwd)/nbest.trg.in --alignment --mini-batch 16 --n-best > nbest.out # Compare n-best lists diff --git a/tests/scorer/align/test_scorer_soft_align.sh b/tests/scorer/align/test_scorer_soft_align.sh index 54601c9..d4a7bef 100644 --- a/tests/scorer/align/test_scorer_soft_align.sh +++ b/tests/scorer/align/test_scorer_soft_align.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/text.src.in $(pwd)/text.trg.in --alignment soft --mini-batch 16 \ | sed 's/^.* ||| //' > soft.out diff --git a/tests/scorer/nbest/test_compare_parallel_and_nbest.sh b/tests/scorer/nbest/test_compare_parallel_and_nbest.sh index 8304c04..5658fdd 100644 --- a/tests/scorer/nbest/test_compare_parallel_and_nbest.sh +++ b/tests/scorer/nbest/test_compare_parallel_and_nbest.sh @@ -7,11 +7,11 @@ test -e text.srcall.in || cat text.src.in | sed 'p;p;p;p' > text.srcall.in test -e text.trg.in || cat text.nbest.in | sed 's/ ||| /\t/g' | cut -f2 > text.trg.in # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t text.srcall.in text.trg.in \ > parallel.scores.out -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ --n-best -t text.src.in text.nbest.in \ > parallel.nbest.out diff --git a/tests/scorer/nbest/test_custom_feature_name.sh b/tests/scorer/nbest/test_custom_feature_name.sh index 957d701..7a8277f 100644 --- a/tests/scorer/nbest/test_custom_feature_name.sh +++ b/tests/scorer/nbest/test_custom_feature_name.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ --n-best --n-best-feature FeatureName -t text.src.in text.nbest.in \ > custom.out diff --git a/tests/scorer/nbest/test_score_nbest_list.sh b/tests/scorer/nbest/test_score_nbest_list.sh index 4cf3cb3..2199a61 100644 --- a/tests/scorer/nbest/test_score_nbest_list.sh +++ b/tests/scorer/nbest/test_score_nbest_list.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ --n-best -t text.src.in text.nbest.in \ > nbest.out diff --git a/tests/scorer/scores/test_compare_with_decoder_scores.sh b/tests/scorer/scores/test_compare_with_decoder_scores.sh index e4a7440..77f2a95 100644 --- a/tests/scorer/scores/test_compare_with_decoder_scores.sh +++ b/tests/scorer/scores/test_compare_with_decoder_scores.sh @@ -4,7 +4,7 @@ set -e # Translate with s2s -$MRT_MARIAN/build/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml \ +$MRT_MARIAN/marian-decoder -c $MRT_MODELS/wmt16_systems/marian.en-de.yml \ -b 12 --n-best < text.in > nbest.out # Compare translations @@ -16,7 +16,7 @@ cat text.in | perl -ne 'for$i(1..12){print}' > compare.src cat nbest.out | sed 's/ ||| /\t/g' | cut -f2 > compare.trg # Run rescorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/compare.src $(pwd)/compare.trg > compare.scorer.out # Compare scores diff --git a/tests/scorer/scores/test_scores.sh b/tests/scorer/scores/test_scores.sh index 540e102..6afbf71 100644 --- a/tests/scorer/scores/test_scores.sh +++ b/tests/scorer/scores/test_scores.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/scores.src.in $(pwd)/scores.trg.in > scores.out # Compare scores diff --git a/tests/scorer/scores/test_summary.sh b/tests/scorer/scores/test_summary.sh index d2fc27f..523551b 100644 --- a/tests/scorer/scores/test_summary.sh +++ b/tests/scorer/scores/test_summary.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/scores.src.in $(pwd)/scores.trg.in --summary > summary.out # Compare scores diff --git a/tests/scorer/scores/test_summary_perplexity.sh b/tests/scorer/scores/test_summary_perplexity.sh index 73e93d8..bb629c9 100644 --- a/tests/scorer/scores/test_summary_perplexity.sh +++ b/tests/scorer/scores/test_summary_perplexity.sh @@ -4,7 +4,7 @@ set -e # Run scorer -$MRT_MARIAN/build/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ +$MRT_MARIAN/marian-scorer -c $MRT_MODELS/wmt16_systems/marian.en-de.scorer.yml \ -t $(pwd)/scores.src.in $(pwd)/scores.trg.in --summary perplexity > summary_perplexity.out # Compare scores diff --git a/tests/server/setup.sh b/tests/server/setup.sh index 5f40f8d..79ab0cd 100644 --- a/tests/server/setup.sh +++ b/tests/server/setup.sh @@ -1,3 +1,3 @@ -test -f $MRT_MARIAN/build/marian-server || exit 1 +test -f $MRT_MARIAN/marian-server || exit 1 test -f $MRT_MODELS/wmt16_systems/en-de/model.npz || exit 1 python -c "import websocket" diff --git a/tests/server/test_ende.sh b/tests/server/test_ende.sh index a309d31..b68411d 100644 --- a/tests/server/test_ende.sh +++ b/tests/server/test_ende.sh @@ -9,12 +9,12 @@ clean_up() { trap clean_up EXIT # Test code goes here -$MRT_MARIAN/build/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 > server.log 2>&1 & +$MRT_MARIAN/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 > server.log 2>&1 & SERVER_PID=$! sleep 20 -python3 $MRT_MARIAN/scripts/server/client_example.py -p 8765 < text.in > text.out +python3 $MRT_MARIAN/../scripts/server/client_example.py -p 8765 < text.in > text.out kill $SERVER_PID $MRT_TOOLS/diff.sh text.out text.expected > text.diff diff --git a/tests/server/test_ende_align.sh b/tests/server/test_ende_align.sh index 5242d7b..c3480b6 100644 --- a/tests/server/test_ende_align.sh +++ b/tests/server/test_ende_align.sh @@ -9,13 +9,13 @@ clean_up() { trap clean_up EXIT # Test code goes here -$MRT_MARIAN/build/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 \ +$MRT_MARIAN/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8765 \ --alignment > server.align.log 2>&1 & SERVER_PID=$! sleep 20 -python3 $MRT_MARIAN/scripts/server/client_example.py -p 8765 < text.in > text.align.out +python3 $MRT_MARIAN/../scripts/server/client_example.py -p 8765 < text.in > text.align.out kill $SERVER_PID $MRT_TOOLS/diff.sh text.align.out text.align.expected > text.align.diff diff --git a/tests/server/test_ende_batch32.sh b/tests/server/test_ende_batch32.sh index c9955e6..b38ad0e 100644 --- a/tests/server/test_ende_batch32.sh +++ b/tests/server/test_ende_batch32.sh @@ -9,13 +9,13 @@ clean_up() { trap clean_up EXIT # Test code goes here -$MRT_MARIAN/build/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8766 \ +$MRT_MARIAN/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8766 \ --mini-batch 32 --maxi-batch 1 > server.b32.log 2>&1 & SERVER_PID=$! sleep 20 -python3 $MRT_MARIAN/scripts/server/client_example.py -p 8766 -b 32 < text.in > text.b32.out +python3 $MRT_MARIAN/../scripts/server/client_example.py -p 8766 -b 32 < text.in > text.b32.out kill $SERVER_PID $MRT_TOOLS/diff.sh text.b32.out text.expected > text.diff diff --git a/tests/server/test_ende_cpu.sh b/tests/server/test_ende_cpu.sh index 8925561..311490a 100644 --- a/tests/server/test_ende_cpu.sh +++ b/tests/server/test_ende_cpu.sh @@ -17,12 +17,12 @@ head -n 4 text.in > text4.in head -n 4 text.expected > text4.expected # Test code goes here -$MRT_MARIAN/build/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8768 --cpu-threads 4 > server_cpu.log 2>&1 & +$MRT_MARIAN/marian-server -c $MRT_MODELS/wmt16_systems/marian.en-de.yml -p 8768 --cpu-threads 4 > server_cpu.log 2>&1 & SERVER_PID=$! sleep 20 -python3 $MRT_MARIAN/scripts/server/client_example.py -p 8768 < text4.in > text4.cpu.out +python3 $MRT_MARIAN/../scripts/server/client_example.py -p 8768 < text4.in > text4.cpu.out kill $SERVER_PID $MRT_TOOLS/diff.sh text4.cpu.out text4.expected > text4.cpu.diff diff --git a/tests/training/basics/setup.sh b/tests/training/basics/setup.sh index 1bcb2f9..5bb2760 100644 --- a/tests/training/basics/setup.sh +++ b/tests/training/basics/setup.sh @@ -3,7 +3,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 test -f $MRT_DATA/europarl.de-en/toy.bpe.en || exit 1 test -f $MRT_DATA/europarl.de-en/toy.bpe.de || exit 1 -test -s vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -s vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -s vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -s vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -s vocab.de.yml test -s vocab.en.yml diff --git a/tests/training/basics/test_gzipped_train_sets.sh b/tests/training/basics/test_gzipped_train_sets.sh index b28c41c..528c0eb 100644 --- a/tests/training/basics/test_gzipped_train_sets.sh +++ b/tests/training/basics/test_gzipped_train_sets.sh @@ -10,7 +10,7 @@ mkdir -p gzip test -e $MRT_DATA/europarl.de-en/corpus.bpe.de.gz || cat $MRT_DATA/europarl.de-en/corpus.bpe.de | gzip > $MRT_DATA/europarl.de-en/corpus.bpe.de.gz test -e $MRT_DATA/europarl.de-en/corpus.bpe.en.gz || cat $MRT_DATA/europarl.de-en/corpus.bpe.en | gzip > $MRT_DATA/europarl.de-en/corpus.bpe.en.gz -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --dim-emb 64 --dim-rnn 64 \ -m gzip/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de}.gz -v vocab.en.yml vocab.de.yml \ --log gzip.log --disp-freq 10 --after-batches 50 diff --git a/tests/training/basics/test_mini_batch_fit.sh b/tests/training/basics/test_mini_batch_fit.sh index bef96b9..7d4f7c2 100644 --- a/tests/training/basics/test_mini_batch_fit.sh +++ b/tests/training/basics/test_mini_batch_fit.sh @@ -10,7 +10,7 @@ type bc || exit 100 rm -rf batch_fit batch_fit.log mkdir -p batch_fit -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --dim-emb 128 --dim-rnn 256 \ -m batch_fit/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 20 --after-batches 100 \ diff --git a/tests/training/basics/test_sqlite.sh b/tests/training/basics/test_sqlite.sh index 0dec245..dc2b319 100644 --- a/tests/training/basics/test_sqlite.sh +++ b/tests/training/basics/test_sqlite.sh @@ -7,7 +7,7 @@ set -e rm -rf sqlite *sqlite.log mkdir -p sqlite -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 64 --dim-rnn 128 --optimizer sgd \ -m sqlite/model.nosqlite.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -19,7 +19,7 @@ test -e nosqlite.log $MRT_TOOLS/extract-costs.sh < nosqlite.log > nosqlite.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 64 --dim-rnn 128 --optimizer sgd \ -m sqlite/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} --sqlite \ diff --git a/tests/training/basics/test_sqlite_random_seed.sh b/tests/training/basics/test_sqlite_random_seed.sh index a443f59..6c1ddd9 100644 --- a/tests/training/basics/test_sqlite_random_seed.sh +++ b/tests/training/basics/test_sqlite_random_seed.sh @@ -7,7 +7,7 @@ set -e rm -rf sqlite_seed sqlite_seed_?.log mkdir -p sqlite_seed -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 3333 --dim-emb 64 --dim-rnn 128 --optimizer sgd \ -m sqlite_seed/model1.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} --sqlite -v vocab.en.yml vocab.de.yml \ @@ -17,7 +17,7 @@ $MRT_MARIAN/build/marian \ test -e sqlite_seed/model1.npz test -e sqlite_seed_1.log -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 3333 --dim-emb 64 --dim-rnn 128 --optimizer sgd \ -m sqlite_seed/model2.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} --sqlite \ diff --git a/tests/training/basics/test_toy_vocab.sh b/tests/training/basics/test_toy_vocab.sh index 2c7c253..b24ef3e 100644 --- a/tests/training/basics/test_toy_vocab.sh +++ b/tests/training/basics/test_toy_vocab.sh @@ -7,7 +7,7 @@ set -e mkdir -p toy rm -f toy/* toy.log -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --dim-emb 256 --dim-rnn 512 \ -m toy/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v toy/vocab.de.yml toy/vocab.en.yml \ --log toy.log --disp-freq 5 -e 5 diff --git a/tests/training/basics/test_translation_script.sh b/tests/training/basics/test_translation_script.sh index 83b102c..8956ab1 100644 --- a/tests/training/basics/test_translation_script.sh +++ b/tests/training/basics/test_translation_script.sh @@ -7,7 +7,7 @@ set -e rm -rf trans trans.log trans_script.temp mkdir -p trans -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 --maxi-batch 1 --mini-batch 16 \ -m trans/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/basics/test_valid_script.sh b/tests/training/basics/test_valid_script.sh index e638854..c168e0f 100644 --- a/tests/training/basics/test_valid_script.sh +++ b/tests/training/basics/test_valid_script.sh @@ -7,7 +7,7 @@ set -e rm -rf valid valid.log valid_script.temp mkdir -p valid -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 --maxi-batch 1 --mini-batch 16 \ -m valid/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} \ -v vocab.en.yml vocab.de.yml --dim-vocabs 50000 50000 \ diff --git a/tests/training/cost-functions/test_ce-mean-words.sh b/tests/training/cost-functions/test_ce-mean-words.sh index faaf113..243eac9 100644 --- a/tests/training/cost-functions/test_ce-mean-words.sh +++ b/tests/training/cost-functions/test_ce-mean-words.sh @@ -7,7 +7,7 @@ set -e rm -rf ce-mean-words ce-mean-words.log mkdir -p ce-mean-words -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --cost-type ce-mean-words \ --seed 9999 \ -m ce-mean-words/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/cost-functions/test_ce-mean.sh b/tests/training/cost-functions/test_ce-mean.sh index c2ef4d1..223e09c 100644 --- a/tests/training/cost-functions/test_ce-mean.sh +++ b/tests/training/cost-functions/test_ce-mean.sh @@ -7,7 +7,7 @@ set -e rm -rf ce-mean ce-mean.log mkdir -p ce-mean -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 9999 \ -m ce-mean/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 2 --after-epochs 1 \ diff --git a/tests/training/cost-functions/test_ce-sum.sh b/tests/training/cost-functions/test_ce-sum.sh index 540e82d..8d8acaa 100644 --- a/tests/training/cost-functions/test_ce-sum.sh +++ b/tests/training/cost-functions/test_ce-sum.sh @@ -7,7 +7,7 @@ set -e rm -rf ce-sum ce-sum.log mkdir -p ce-sum -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --cost-type ce-sum \ --seed 9999 \ -m ce-sum/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/cost-functions/test_perplexity.sh b/tests/training/cost-functions/test_perplexity.sh index ae828a4..36d224a 100644 --- a/tests/training/cost-functions/test_perplexity.sh +++ b/tests/training/cost-functions/test_perplexity.sh @@ -7,7 +7,7 @@ set -e rm -rf perplexity perplexity.log mkdir -p perplexity -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --cost-type perplexity \ --seed 9999 \ -m perplexity/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/data-weighting/setup.sh b/tests/training/data-weighting/setup.sh index 6d1a190..d6fd51e 100644 --- a/tests/training/data-weighting/setup.sh +++ b/tests/training/data-weighting/setup.sh @@ -1,7 +1,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 -test -s vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -s vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -s vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -s vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -s vocab.de.yml test -s vocab.en.yml diff --git a/tests/training/data-weighting/test_compare_word_and_sentence_weighting.sh b/tests/training/data-weighting/test_compare_word_and_sentence_weighting.sh index 6529857..da8a76c 100644 --- a/tests/training/data-weighting/test_compare_word_and_sentence_weighting.sh +++ b/tests/training/data-weighting/test_compare_word_and_sentence_weighting.sh @@ -11,7 +11,7 @@ cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r -e 's/[^ ]+/3/g' -e 's/$/ 3/' > cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r -e 's/.*/3/' > compare.sents.weights.txt # Train on sentence-level, each sentence has weight 3 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 3333 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m compare/model.sents.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log compare.sents.log --disp-freq 5 -e 2 \ @@ -23,7 +23,7 @@ test -e compare.sents.log cat compare.sents.log | $MRT_TOOLS/extract-disp.sh > compare.sents.out # Train on word-level, each word has weight 3, including EOS -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 3333 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m compare/model.words.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log compare.words.log --disp-freq 5 -e 2 \ diff --git a/tests/training/data-weighting/test_length_mismatch_warnings.sh b/tests/training/data-weighting/test_length_mismatch_warnings.sh index 9a52fcc..9480677 100644 --- a/tests/training/data-weighting/test_length_mismatch_warnings.sh +++ b/tests/training/data-weighting/test_length_mismatch_warnings.sh @@ -12,7 +12,7 @@ cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/1/g' > warn.weights.tx sed -i '2s/1 1 /1 /g' warn.weights.txt sed -i '3s/1 /1 1 /g' warn.weights.txt -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --dim-emb 64 --dim-rnn 128 --optimizer sgd \ -m warn/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log warn.log -e 1 \ diff --git a/tests/training/data-weighting/test_maxi_batches_with_sentence_weights.sh b/tests/training/data-weighting/test_maxi_batches_with_sentence_weights.sh index 071fd94..ae2cb20 100644 --- a/tests/training/data-weighting/test_maxi_batches_with_sentence_weights.sh +++ b/tests/training/data-weighting/test_maxi_batches_with_sentence_weights.sh @@ -7,10 +7,10 @@ set -e mkdir -p maxibatch rm -rf maxibatch/* maxibatch.log -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 3333 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \ --log maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 --cost-type ce-sum \ diff --git a/tests/training/data-weighting/test_maxi_batches_with_word_weights.sh b/tests/training/data-weighting/test_maxi_batches_with_word_weights.sh index 7b87e3d..c0babe4 100644 --- a/tests/training/data-weighting/test_maxi_batches_with_word_weights.sh +++ b/tests/training/data-weighting/test_maxi_batches_with_word_weights.sh @@ -7,10 +7,10 @@ set -e mkdir -p word_maxibatch rm -rf word_maxibatch/* word_maxibatch.log -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 6666 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_maxibatch/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \ --log word_maxibatch.log --disp-freq 10 --after-batches 100 --mini-batch 16 \ diff --git a/tests/training/data-weighting/test_sentence_weighting_sqlite.sh b/tests/training/data-weighting/test_sentence_weighting_sqlite.sh index 3f393fc..b1f9fc3 100644 --- a/tests/training/data-weighting/test_sentence_weighting_sqlite.sh +++ b/tests/training/data-weighting/test_sentence_weighting_sqlite.sh @@ -7,7 +7,7 @@ set -e rm -rf sqlite sqlite.log mkdir -p sqlite -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m sqlite/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \ --log sqlite.log --disp-freq 1 --after-batches 100 --mini-batch 1 \ diff --git a/tests/training/data-weighting/test_sentence_weighting_with_ones.sh b/tests/training/data-weighting/test_sentence_weighting_with_ones.sh index 6ff43ef..27ed5c4 100644 --- a/tests/training/data-weighting/test_sentence_weighting_with_ones.sh +++ b/tests/training/data-weighting/test_sentence_weighting_with_ones.sh @@ -7,10 +7,10 @@ set -e rm -rf noweights* ones* mkdir -p noweights ones -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m noweights/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log noweights.log --disp-freq 5 -e 2 @@ -21,7 +21,7 @@ cat noweights.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed -r 's/ T cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/.*/1/g' > ones.weights.txt -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m ones/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log ones.log --disp-freq 5 -e 2 \ diff --git a/tests/training/data-weighting/test_sentence_weights_x3.sh b/tests/training/data-weighting/test_sentence_weights_x3.sh index 45b92d3..5fc4f2b 100644 --- a/tests/training/data-weighting/test_sentence_weights_x3.sh +++ b/tests/training/data-weighting/test_sentence_weights_x3.sh @@ -7,10 +7,10 @@ set -e rm -rf x3copied* x3weights* mkdir -p x3copied x3weights -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m x3copied/model.npz -t train.x3.{de,en} -v vocab.{de,en}.yml \ --log x3copied.log --disp-freq 1 --after-batches 100 --mini-batch 4 --cost-type ce-sum @@ -19,7 +19,7 @@ test -e x3copied/model.npz test -e x3copied.log cat x3copied.log | grep 'Cost ' | sed -r 's/.*Cost (.*) : Time.*/\1/' > x3copied.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 2222 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --max-length 100 --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m x3weights/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \ --log x3weights.log --disp-freq 1 --after-batches 100 --mini-batch 2 --cost-type ce-sum \ diff --git a/tests/training/data-weighting/test_validation.sh b/tests/training/data-weighting/test_validation.sh index 3bb3c63..8099280 100644 --- a/tests/training/data-weighting/test_validation.sh +++ b/tests/training/data-weighting/test_validation.sh @@ -7,7 +7,7 @@ set -e rm -rf valid valid_script.temp mkdir -p valid -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 4444 --no-shuffle --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 \ -m valid/model.npz -t train.1k.{de,en} -v vocab.{de,en}.yml \ --disp-freq 5 --valid-freq 15 --after-batches 50 \ diff --git a/tests/training/data-weighting/test_word_weighting_sqlite.sh b/tests/training/data-weighting/test_word_weighting_sqlite.sh index bb2452b..229d5a8 100644 --- a/tests/training/data-weighting/test_word_weighting_sqlite.sh +++ b/tests/training/data-weighting/test_word_weighting_sqlite.sh @@ -9,7 +9,7 @@ mkdir -p sqlite_word cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/2/g' > sqlite_word.weights.txt -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m sqlite_word/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log sqlite_word.log --disp-freq 5 -e 2 --mini-batch-fit -w 500 \ diff --git a/tests/training/data-weighting/test_word_weighting_with_eos.sh b/tests/training/data-weighting/test_word_weighting_with_eos.sh index 4044b77..1f8a75d 100644 --- a/tests/training/data-weighting/test_word_weighting_with_eos.sh +++ b/tests/training/data-weighting/test_word_weighting_with_eos.sh @@ -11,7 +11,7 @@ mkdir -p word_eos cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r -e 's/[^ ]+/2/g' -e 's/$/ 2/' > word_eos.weights.txt # Train -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_eos/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log word_eos.log --disp-freq 5 -e 2 \ diff --git a/tests/training/data-weighting/test_word_weighting_with_ones.sh b/tests/training/data-weighting/test_word_weighting_with_ones.sh index 9e47cdc..78eae39 100644 --- a/tests/training/data-weighting/test_word_weighting_with_ones.sh +++ b/tests/training/data-weighting/test_word_weighting_with_ones.sh @@ -7,7 +7,7 @@ set -e rm -rf word_noweights* word_ones* mkdir -p word_noweights word_ones -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_noweights/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log word_noweights.log --disp-freq 5 -e 2 @@ -18,7 +18,7 @@ cat word_noweights.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed -r cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/1/g' > word_ones.weights.txt -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_ones/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log word_ones.log --disp-freq 5 -e 2 \ diff --git a/tests/training/data-weighting/test_word_weighting_with_twos.sh b/tests/training/data-weighting/test_word_weighting_with_twos.sh index a39ec07..bbf5b55 100644 --- a/tests/training/data-weighting/test_word_weighting_with_twos.sh +++ b/tests/training/data-weighting/test_word_weighting_with_twos.sh @@ -9,7 +9,7 @@ mkdir -p word_twos cat $MRT_DATA/europarl.de-en/toy.bpe.en | sed -r 's/[^ ]+/2/g' > word_twos.weights.txt -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_twos/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log word_twos.log --disp-freq 5 -e 2 \ @@ -27,7 +27,7 @@ mkdir -p word_twos_cfg echo "data-weighting: word_twos.weights.txt" > word_twos.config.yml echo "data-weighting-type: word" >> word_twos.config.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle --dim-emb 128 --dim-rnn 256 --optimizer sgd \ -m word_twos_cfg/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{de,en} -v vocab.{de,en}.yml \ --log word_twos_cfg.log --disp-freq 5 -e 2 \ diff --git a/tests/training/embeddings/setup.sh b/tests/training/embeddings/setup.sh index 4739558..f285aad 100644 --- a/tests/training/embeddings/setup.sh +++ b/tests/training/embeddings/setup.sh @@ -1,7 +1,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/embeddings/test_custom_embeddings.sh b/tests/training/embeddings/test_custom_embeddings.sh index 3edd00a..c80ab2b 100644 --- a/tests/training/embeddings/test_custom_embeddings.sh +++ b/tests/training/embeddings/test_custom_embeddings.sh @@ -8,7 +8,7 @@ rm -rf custom_emb custom_emb.log mkdir -p custom_emb # Train with custom embeddings only for one update with the smallest possible mini-batch, so that they should barely change -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m custom_emb/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --embedding-vectors word2vec.en word2vec.de --dim-emb 64 --dim-rnn 64 \ --no-shuffle --mini-batch 1 --after-batches 1 --log custom_emb.log @@ -20,7 +20,7 @@ test -e custom_emb.log grep -q "Loading embedding vectors from" custom_emb.log # Check if embeddings in the saved model are very similar to the original vectors -$MRT_MARIAN/scripts/embeddings/export_embeddings.py -m custom_emb/model.npz -o custom_emb.all +$MRT_MARIAN/../scripts/embeddings/export_embeddings.py -m custom_emb/model.npz -o custom_emb.all # The custom embeddings have been trained only for the first 100 words from each vocabulary cat custom_emb.all.src | head -n 101 > custom_emb.src diff --git a/tests/training/exp-smoothing/test_expsmooth.sh b/tests/training/exp-smoothing/test_expsmooth.sh index 2428dfc..f55f64d 100644 --- a/tests/training/exp-smoothing/test_expsmooth.sh +++ b/tests/training/exp-smoothing/test_expsmooth.sh @@ -11,7 +11,7 @@ mkdir -p expsmooth opts="--no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer sgd --learn-rate 0.5 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32" # No exponential smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 20 --valid-freq 20 --after-batches 200 $opts \ --log expsmooth_0.log @@ -24,7 +24,7 @@ cat expsmooth_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep -v 'v # With exponential smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 20 --valid-freq 20 --after-batches 200 --exponential-smoothing 0.0001 $opts \ --log expsmooth.log diff --git a/tests/training/exp-smoothing/test_expsmooth_sync.sh b/tests/training/exp-smoothing/test_expsmooth_sync.sh index 14d5442..19ef4e1 100644 --- a/tests/training/exp-smoothing/test_expsmooth_sync.sh +++ b/tests/training/exp-smoothing/test_expsmooth_sync.sh @@ -16,7 +16,7 @@ mkdir -p expsmooth_sync opts="--no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer sgd --learn-rate 0.5 --valid-sets valid.bpe.en valid.bpe.de --valid-metrics cross-entropy --valid-mini-batch 32 --devices 0 1 --sync-sgd" # No exponential smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 20 --valid-freq 20 --after-batches 200 $opts \ --log expsmooth_sync_0.log @@ -29,7 +29,7 @@ cat expsmooth_sync_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep # With exponential smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 20 --valid-freq 20 --after-batches 200 --exponential-smoothing 0.0001 $opts \ --log expsmooth_sync.log diff --git a/tests/training/lm/test_lm-transformer.sh b/tests/training/lm/test_lm-transformer.sh index 47737e5..a8bb5d2 100644 --- a/tests/training/lm/test_lm-transformer.sh +++ b/tests/training/lm/test_lm-transformer.sh @@ -7,7 +7,7 @@ set -e rm -rf lm-transformer lm-transformer.log mkdir -p lm-transformer -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle \ --type lm-transformer --dim-emb 128 --dim-rnn 256 \ -m lm-transformer/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.en -v vocab.en.yml \ @@ -24,7 +24,7 @@ $MRT_TOOLS/diff-nums.py lm-transformer.out lm-transformer.expected -p 0.02 -o lm # Scoring with LM test -s temp.bpe.en || tail $MRT_DATA/europarl.de-en/corpus.bpe.en > test.bpe.en -$MRT_MARIAN/build/marian-scorer -m lm-transformer/model.npz -t test.bpe.en -v vocab.en.yml > lm-transformer.scores.out +$MRT_MARIAN/marian-scorer -m lm-transformer/model.npz -t test.bpe.en -v vocab.en.yml > lm-transformer.scores.out $MRT_TOOLS/diff-nums.py lm-transformer.scores.out lm-transformer.scores.expected -p 0.002 -o lm-transformer.scores.diff # Exit with success code diff --git a/tests/training/lm/test_lm.sh b/tests/training/lm/test_lm.sh index e3c32a0..127193b 100644 --- a/tests/training/lm/test_lm.sh +++ b/tests/training/lm/test_lm.sh @@ -7,7 +7,7 @@ set -e rm -rf lm lm.log mkdir -p lm -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle \ --type lm --dim-emb 128 --dim-rnn 256 \ -m lm/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.en -v vocab.en.yml \ @@ -24,7 +24,7 @@ $MRT_TOOLS/diff-nums.py lm.out lm.expected -p 0.02 -o lm.diff # Scoring with LM test -s temp.bpe.en || tail $MRT_DATA/europarl.de-en/corpus.bpe.en > test.bpe.en -$MRT_MARIAN/build/marian-scorer -m lm/model.npz -t test.bpe.en -v vocab.en.yml > lm.scores.out +$MRT_MARIAN/marian-scorer -m lm/model.npz -t test.bpe.en -v vocab.en.yml > lm.scores.out $MRT_TOOLS/diff-nums.py lm.scores.out lm.scores.expected -p 0.002 -o lm.scores.diff # Exit with success code diff --git a/tests/training/mixed-models/setup.sh b/tests/training/mixed-models/setup.sh index b060767..d59e7bd 100644 --- a/tests/training/mixed-models/setup.sh +++ b/tests/training/mixed-models/setup.sh @@ -1,8 +1,8 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.en || exit 1 test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/mixed-models/test_ensemble_of_different_s2s.sh b/tests/training/mixed-models/test_ensemble_of_different_s2s.sh index 4fb5a4d..3f125ee 100644 --- a/tests/training/mixed-models/test_ensemble_of_different_s2s.sh +++ b/tests/training/mixed-models/test_ensemble_of_different_s2s.sh @@ -10,7 +10,7 @@ mkdir -p two_s2s options="--no-shuffle --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 100" # Train model A -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s -m two_s2s/modelA.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $options --seed 1111 --dim-emb 128 --dim-rnn 256 \ --log two_s2s_A.log @@ -19,7 +19,7 @@ test -e two_s2s/modelA.npz test -e two_s2s_A.log # Train model B with different architecture -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s -m two_s2s/modelB.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $options --seed 2222 --dim-emb 256 --dim-rnn 128 --layer-normalization --enc-depth 2 \ --log two_s2s_B.log @@ -28,7 +28,7 @@ test -e two_s2s/modelB.npz test -e two_s2s_B.log # Check if the ensemble of two different s2s models works -$MRT_MARIAN/build/marian-decoder -m two_s2s/modelA.npz two_s2s/modelB.npz -v vocab.en.yml vocab.de.yml \ +$MRT_MARIAN/marian-decoder -m two_s2s/modelA.npz two_s2s/modelB.npz -v vocab.en.yml vocab.de.yml \ -i text.in -o two_s2s.out --log two_s2s.log $MRT_TOOLS/diff.sh two_s2s.out two_s2s.expected > two_s2s.diff diff --git a/tests/training/mixed-models/test_ensemble_of_s2s_and_transformer.sh b/tests/training/mixed-models/test_ensemble_of_s2s_and_transformer.sh index 2b0142e..a80fe32 100644 --- a/tests/training/mixed-models/test_ensemble_of_s2s_and_transformer.sh +++ b/tests/training/mixed-models/test_ensemble_of_s2s_and_transformer.sh @@ -10,7 +10,7 @@ mkdir -p s2s_transf options="--no-shuffle --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --disp-freq 20 --after-batches 100" # Train model A -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s -m s2s_transf/modelA.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $options --seed 1111 --dim-emb 128 --dim-rnn 256 \ --log s2s_transf_A.log @@ -19,7 +19,7 @@ test -e s2s_transf/modelA.npz test -e s2s_transf_A.log # Train model B with different architecture -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type transformer -m s2s_transf/modelB.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $options --seed 2222 --dim-emb 256 --dim-rnn 128 \ --log s2s_transf_B.log @@ -28,7 +28,7 @@ test -e s2s_transf/modelB.npz test -e s2s_transf_B.log # Check if the ensemble of two different s2s models works -$MRT_MARIAN/build/marian-decoder -m s2s_transf/modelA.npz s2s_transf/modelB.npz -v vocab.en.yml vocab.de.yml \ +$MRT_MARIAN/marian-decoder -m s2s_transf/modelA.npz s2s_transf/modelB.npz -v vocab.en.yml vocab.de.yml \ -i text.in -o s2s_transf.out --log s2s_transf.log $MRT_TOOLS/diff.sh s2s_transf.out s2s_transf.expected > s2s_transf.diff diff --git a/tests/training/model-types/test_amun_has_decoder_c_tt.sh b/tests/training/model-types/test_amun_has_decoder_c_tt.sh index fb44930..8aa4f3d 100644 --- a/tests/training/model-types/test_amun_has_decoder_c_tt.sh +++ b/tests/training/model-types/test_amun_has_decoder_c_tt.sh @@ -9,13 +9,13 @@ mkdir -p decoder_c_tt opts="--no-shuffle --seed 1111 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --dim-emb 64 --dim-rnn 128" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m decoder_c_tt/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $opts --after-batches 1 test -e decoder_c_tt/model.npz -python $MRT_MARIAN/scripts/contrib/model_info.py -m decoder_c_tt/model.npz > decoder_c_tt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m decoder_c_tt/model.npz > decoder_c_tt.out grep -q "decoder_c_tt" decoder_c_tt.out # Exit with success code diff --git a/tests/training/model-types/test_transformer.sh b/tests/training/model-types/test_transformer.sh index 35f8f09..e9754cf 100644 --- a/tests/training/model-types/test_transformer.sh +++ b/tests/training/model-types/test_transformer.sh @@ -9,7 +9,7 @@ mkdir -p transformer opts="--no-shuffle --seed 1111 --mini-batch 32 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd --dim-emb 64 --dim-rnn 128" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type transformer -m transformer/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ $opts --disp-freq 10 --after-batches 100 \ --log transformer.log diff --git a/tests/training/multi-gpu/test_async_sgd_runs.sh b/tests/training/multi-gpu/test_async_sgd_runs.sh index 15b0348..1094cd2 100644 --- a/tests/training/multi-gpu/test_async_sgd_runs.sh +++ b/tests/training/multi-gpu/test_async_sgd_runs.sh @@ -12,7 +12,7 @@ fi rm -rf async_sgd async_sgd.log mkdir -p async_sgd -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none --dim-emb 128 --dim-rnn 256 \ --devices 0 1 \ -m async_sgd/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/multi-gpu/test_sync_sgd.sh b/tests/training/multi-gpu/test_sync_sgd.sh index 0d511b3..829c918 100644 --- a/tests/training/multi-gpu/test_sync_sgd.sh +++ b/tests/training/multi-gpu/test_sync_sgd.sh @@ -12,7 +12,7 @@ fi rm -rf sync_sgd sync_sgd.log mkdir -p sync_sgd -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none \ --dim-rnn 64 --dim-emb 32 --learn-rate 0.1 \ --devices 0 1 --sync-sgd --optimizer sgd \ diff --git a/tests/training/multi-source/test_multi-s2s.sh b/tests/training/multi-source/test_multi-s2s.sh index e55a835..a1555c6 100644 --- a/tests/training/multi-source/test_multi-s2s.sh +++ b/tests/training/multi-source/test_multi-s2s.sh @@ -7,7 +7,7 @@ set -e rm -rf multi-s2s multi-s2s.log mkdir -p multi-s2s -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle \ --type multi-s2s --dim-emb 128 --dim-rnn 256 \ -m multi-s2s/model.npz -t train.bpe.{en,xx,de} -v vocab.en.yml vocab.xx.yml vocab.de.yml \ diff --git a/tests/training/multi-source/test_multi-transformer.sh b/tests/training/multi-source/test_multi-transformer.sh index cafd2ad..4349291 100644 --- a/tests/training/multi-source/test_multi-transformer.sh +++ b/tests/training/multi-source/test_multi-transformer.sh @@ -7,7 +7,7 @@ set -e rm -rf multi-transformer multi-transformer.log mkdir -p multi-transformer -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --no-shuffle \ --type multi-transformer --dim-emb 128 --dim-rnn 256 \ -m multi-transformer/model.npz -t train.bpe.{en,xx,de} -v vocab.en.yml vocab.xx.yml vocab.de.yml \ diff --git a/tests/training/nematus/test_dec_cell_high_depth.sh b/tests/training/nematus/test_dec_cell_high_depth.sh index 52af123..3f8a6aa 100644 --- a/tests/training/nematus/test_dec_cell_high_depth.sh +++ b/tests/training/nematus/test_dec_cell_high_depth.sh @@ -7,7 +7,7 @@ set -e mkdir -p dec_high rm -f dec_high/* dec_high.log -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type nematus --enc-cell gru-nematus --dec-cell gru-nematus \ --enc-depth 2 --enc-cell-depth 2 --enc-type bidirectional --dec-depth 4 --dec-cell-base-depth 4 --dec-cell-high-depth 4 \ --layer-normalization \ diff --git a/tests/training/nematus/test_encdec_depth.sh b/tests/training/nematus/test_encdec_depth.sh index 99c4c7c..c2ce41a 100644 --- a/tests/training/nematus/test_encdec_depth.sh +++ b/tests/training/nematus/test_encdec_depth.sh @@ -7,7 +7,7 @@ set -e mkdir -p encdec_depth rm -f encdec_depth/* encdec_depth.log -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type nematus --enc-cell gru-nematus --dec-cell gru-nematus \ --enc-depth 4 --enc-cell-depth 4 --enc-type bidirectional --dec-depth 4 --dec-cell-base-depth 4 --dec-cell-high-depth 1 \ --layer-normalization \ diff --git a/tests/training/nematus/test_wmt17_model.sh b/tests/training/nematus/test_wmt17_model.sh index 43cdebe..1d88bb8 100644 --- a/tests/training/nematus/test_wmt17_model.sh +++ b/tests/training/nematus/test_wmt17_model.sh @@ -7,7 +7,7 @@ set -e mkdir -p wmt17 rm -f wmt17/* wmt17.log -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type nematus --enc-cell gru-nematus --dec-cell gru-nematus \ --enc-depth 1 --enc-cell-depth 4 --enc-type bidirectional --dec-depth 1 --dec-cell-base-depth 8 --dec-cell-high-depth 1 \ --layer-normalization \ diff --git a/tests/training/pretraining/test_weights_from_pretrained_model.sh b/tests/training/pretraining/test_weights_from_pretrained_model.sh index d7087c2..fdb80d4 100644 --- a/tests/training/pretraining/test_weights_from_pretrained_model.sh +++ b/tests/training/pretraining/test_weights_from_pretrained_model.sh @@ -8,7 +8,7 @@ rm -rf model lm.log orig.log model.log key-*.txt mkdir -p model # Train LM -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --seed 1111 --type lm -m model/lm.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.de --no-shuffle \ -v model/vocab.de.yml \ @@ -18,7 +18,7 @@ test -e lm.log test -e model/lm.npz # Train model without pretrained weights -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s -m model/orig.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de --no-shuffle \ -v model/vocab.en.yml model/vocab.de.yml \ @@ -29,7 +29,7 @@ test -e orig.log test -e model/orig.npz # Train model with weights initialized from LM -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s -m model/model.npz --pretrained-model model/lm.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en $MRT_DATA/europarl.de-en/corpus.bpe.de --no-shuffle \ -v model/vocab.en.yml model/vocab.de.yml \ @@ -41,15 +41,15 @@ test -e model/model.npz # Test if selected weights are initialized randomly for key in encoder_Wemb encoder_bi_U encoder_bi_r_Wx; do - python3 $MRT_MARIAN/scripts/contrib/model_info.py -m model/orig.npz -k $key > key-orig-$key.txt - python3 $MRT_MARIAN/scripts/contrib/model_info.py -m model/model.npz -k $key > key-model-$key.txt + python3 $MRT_MARIAN/../scripts/contrib/model_info.py -m model/orig.npz -k $key > key-orig-$key.txt + python3 $MRT_MARIAN/../scripts/contrib/model_info.py -m model/model.npz -k $key > key-model-$key.txt $MRT_TOOLS/diff-nums.py --numpy -p 0.000001 key-orig-$key.txt key-model-$key.txt -o key-diff-$key.txt done # Test if selected weights are identical with LM for key in decoder_Wemb decoder_cell1_U decoder_cell2_bx decoder_ff_logit_l1_W0; do - python3 $MRT_MARIAN/scripts/contrib/model_info.py -m model/lm.npz -k $key > key-lm-$key.txt - python3 $MRT_MARIAN/scripts/contrib/model_info.py -m model/model.npz -k $key > key-model-$key.txt + python3 $MRT_MARIAN/../scripts/contrib/model_info.py -m model/lm.npz -k $key > key-lm-$key.txt + python3 $MRT_MARIAN/../scripts/contrib/model_info.py -m model/model.npz -k $key > key-model-$key.txt $MRT_TOOLS/diff-nums.py --numpy -p 0.000001 key-lm-$key.txt key-model-$key.txt -o key-diff-$key.txt done diff --git a/tests/training/restarting/setup.sh b/tests/training/restarting/setup.sh index 92aadd2..1a15d62 100644 --- a/tests/training/restarting/setup.sh +++ b/tests/training/restarting/setup.sh @@ -4,7 +4,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 test -e $MRT_DATA/train.max50.en test -e $MRT_DATA/train.max50.de -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/restarting/test_restarting_finished.sh b/tests/training/restarting/test_restarting_finished.sh index c355038..90d57f1 100644 --- a/tests/training/restarting/test_restarting_finished.sh +++ b/tests/training/restarting/test_restarting_finished.sh @@ -8,7 +8,7 @@ rm -rf finish finish_?.log mkdir -p finish -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m finish/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 5 --log finish_1.log @@ -18,7 +18,7 @@ test -e finish_1.log grep -q "Training finished" finish_1.log -$MRT_MARIAN/build/marian -c finish/model.npz.yml --log finish_2.log +$MRT_MARIAN/marian -c finish/model.npz.yml --log finish_2.log grep -q "Loading model" finish_2.log grep -q "Training finished" finish_2.log diff --git a/tests/training/restarting/test_sgd_for_two_epochs.sh b/tests/training/restarting/test_sgd_for_two_epochs.sh index 3cb09d2..5a41fc1 100644 --- a/tests/training/restarting/test_sgd_for_two_epochs.sh +++ b/tests/training/restarting/test_sgd_for_two_epochs.sh @@ -11,7 +11,7 @@ extra_opts="--no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none --min # Uncomment to prepare the expected output -#$MRT_MARIAN/build/marian \ +#$MRT_MARIAN/marian \ #-m sgd_2e/model_2e.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ #--disp-freq 4 --save-freq 32 --after-epochs 2 -l 0.1 $extra_opts \ #--log sgd_two_epochs.log @@ -22,7 +22,7 @@ extra_opts="--no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none --min #exit 1 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sgd_2e/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 4 --save-freq 32 --after-epochs 1 -l 0.1 $extra_opts \ --log sgd_1st_epoch.log @@ -33,7 +33,7 @@ test -e sgd_1st_epoch.log cat sgd_1st_epoch.log | $MRT_TOOLS/extract-disp.sh > sgd_1st_epoch.out cp sgd_2e/model.npz.yml sgd_2e/model.npz.1st_epoch.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sgd_2e/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 4 --save-freq 32 --after-epochs 2 -l 0.1 $extra_opts \ --log sgd_2nd_epoch.log diff --git a/tests/training/restoring/corpus/setup.sh b/tests/training/restoring/corpus/setup.sh index 92aadd2..1a15d62 100644 --- a/tests/training/restoring/corpus/setup.sh +++ b/tests/training/restoring/corpus/setup.sh @@ -4,7 +4,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 test -e $MRT_DATA/train.max50.en test -e $MRT_DATA/train.max50.de -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/restoring/corpus/test_corpus_restoration.sh b/tests/training/restoring/corpus/test_corpus_restoration.sh index 8b3f625..acb5b1e 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus.log @@ -22,7 +22,7 @@ test -e corpus.log cat corpus.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 40 $extra_opts \ --log corpus_1.log @@ -33,7 +33,7 @@ test -e corpus_1.log cat corpus_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_1.out cp corpus/model.npz.yml corpus/model.npz.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_2.log diff --git a/tests/training/restoring/corpus/test_corpus_restoration_maxi_batch.sh b/tests/training/restoring/corpus/test_corpus_restoration_maxi_batch.sh index fe5ff6e..c439cb4 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration_maxi_batch.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration_maxi_batch.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 1111 --maxi-batch 20 --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_maxi/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_maxi.log @@ -22,7 +22,7 @@ test -e corpus_maxi.log cat corpus_maxi.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_maxi.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_maxi/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 40 $extra_opts \ --log corpus_maxi_1.log @@ -35,7 +35,7 @@ cat corpus_maxi_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ cp corpus_maxi/model.npz.yml corpus_maxi/model.npz.yml.bac cp corpus_maxi/model.npz.progress.yml corpus_maxi/model.npz.progress.yml.bac -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_maxi/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_maxi_2.log diff --git a/tests/training/restoring/corpus/test_corpus_restoration_mini_batch_fit.sh b/tests/training/restoring/corpus/test_corpus_restoration_mini_batch_fit.sh index 2b2ade7..862fb5f 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration_mini_batch_fit.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration_mini_batch_fit.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 5555 --maxi-batch 8 --maxi-batch-sort src --mini-batch 32 --mini-batch-fit -w 100 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_fit/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 80 $extra_opts \ --log corpus_fit.log @@ -22,7 +22,7 @@ test -e corpus_fit.log cat corpus_fit.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_fit.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_fit/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 60 $extra_opts \ --log corpus_fit_1.log @@ -34,7 +34,7 @@ cat corpus_fit_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : cp corpus_fit/model.npz.yml corpus_fit/model.npz.1.yml cp corpus_fit/model.npz.progress.yml corpus_fit/model.npz.progress.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_fit/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 80 $extra_opts \ --log corpus_fit_2.log diff --git a/tests/training/restoring/corpus/test_corpus_restoration_no_shuffle.sh b/tests/training/restoring/corpus/test_corpus_restoration_no_shuffle.sh index 3a25ca1..63105e4 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration_no_shuffle.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration_no_shuffle.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 1234 --no-shuffle --maxi-batch 8 --maxi-batch-sort src --mini-batch 32 --mini-batch-fit -w 100 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_noshuf/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 80 $extra_opts \ --log corpus_noshuf.log @@ -22,7 +22,7 @@ test -e corpus_noshuf.log cat corpus_noshuf.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_noshuf.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_noshuf/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 60 $extra_opts \ --log corpus_noshuf_1.log @@ -34,7 +34,7 @@ cat corpus_noshuf_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's cp corpus_noshuf/model.npz.yml corpus_noshuf/model.npz.1.yml cp corpus_noshuf/model.npz.progress.yml corpus_noshuf/model.npz.progress.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_noshuf/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 80 $extra_opts \ --log corpus_noshuf_2.log diff --git a/tests/training/restoring/corpus/test_corpus_restoration_one_epoch.sh b/tests/training/restoring/corpus/test_corpus_restoration_one_epoch.sh index affc5d7..1001d74 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration_one_epoch.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration_one_epoch.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 9999 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_one/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_one.log @@ -22,7 +22,7 @@ test -e corpus_one.log cat corpus_one.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_one.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_one/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 25 $extra_opts \ --log corpus_one_1.log @@ -33,7 +33,7 @@ test -e corpus_one_1.log cat corpus_one_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_one_1.out cp corpus_one/model.npz.yml corpus_one/model.npz.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_one/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_one_2.log diff --git a/tests/training/restoring/corpus/test_corpus_restoration_s2s.sh b/tests/training/restoring/corpus/test_corpus_restoration_s2s.sh index 606acbb..3cac8dd 100644 --- a/tests/training/restoring/corpus/test_corpus_restoration_s2s.sh +++ b/tests/training/restoring/corpus/test_corpus_restoration_s2s.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 1111 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4 --type s2s" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_s2s/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_s2s.log @@ -22,7 +22,7 @@ test -e corpus_s2s.log cat corpus_s2s.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_s2s.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_s2s/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 40 $extra_opts \ --log corpus_s2s_1.log @@ -33,7 +33,7 @@ test -e corpus_s2s_1.log cat corpus_s2s_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > corpus_s2s_1.out cp corpus_s2s/model.npz.yml corpus_s2s/model.npz.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m corpus_s2s/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log corpus_s2s_2.log diff --git a/tests/training/restoring/corpus/test_sqlite_restoration.sh b/tests/training/restoring/corpus/test_sqlite_restoration.sh index 0dcbdf8..369cfad 100644 --- a/tests/training/restoring/corpus/test_sqlite_restoration.sh +++ b/tests/training/restoring/corpus/test_sqlite_restoration.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 3333 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 75 --sqlite sqlite/dbfull.sqlite3 $extra_opts \ --log sqlite.log @@ -22,7 +22,7 @@ test -e sqlite.log cat sqlite.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sqlite.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 50 --sqlite sqlite/db.sqlite3 $extra_opts \ --log sqlite_1.log @@ -33,7 +33,7 @@ test -e sqlite_1.log cat sqlite_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sqlite_1.out cp sqlite/model.npz.yml sqlite/model.npz.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 75 --sqlite sqlite/db.sqlite3 $extra_opts \ --log sqlite_2.log diff --git a/tests/training/restoring/corpus/test_sqlite_restoration_maxi_batch.sh b/tests/training/restoring/corpus/test_sqlite_restoration_maxi_batch.sh index 09770d1..77305c5 100644 --- a/tests/training/restoring/corpus/test_sqlite_restoration_maxi_batch.sh +++ b/tests/training/restoring/corpus/test_sqlite_restoration_maxi_batch.sh @@ -12,7 +12,7 @@ test -e vocab.en.yml extra_opts="--seed 4444 --maxi-batch 20 --mini-batch 32 --optimizer sgd --dim-emb 128 --dim-rnn 256 --disp-freq 4" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite_maxi/model_full.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log sqlite_maxi.log @@ -22,7 +22,7 @@ test -e sqlite_maxi.log cat sqlite_maxi.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sqlite_maxi.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite_maxi/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 40 $extra_opts \ --log sqlite_maxi_1.log @@ -33,7 +33,7 @@ test -e sqlite_maxi_1.log cat sqlite_maxi_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sqlite_maxi_1.out cp sqlite_maxi/model.npz.yml sqlite_maxi/model.npz.1.yml -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sqlite_maxi/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 70 $extra_opts \ --log sqlite_maxi_2.log diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth.sh b/tests/training/restoring/exp-smoothing/test_expsmooth.sh index 624329f..6fac713 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth.sh @@ -23,7 +23,7 @@ opt_exp=0.0001 # Full pass, no exponential-smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish $opts \ --log expsmooth_0.log @@ -36,7 +36,7 @@ cat expsmooth_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep 'vali # Full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.full.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_f.log @@ -49,7 +49,7 @@ cat expsmooth_f.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep 'vali # A first part of batches -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_save --exponential-smoothing $opt_exp $opts \ --log expsmooth_1.log @@ -63,7 +63,7 @@ cat expsmooth_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep 'vali # Continue training until full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_2.log diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh b/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh index 3a74557..a0ca89c 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth_s2s.sh @@ -23,7 +23,7 @@ opt_exp=0.0001 # Full pass, no exponential-smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_s2s/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish $opts \ --log expsmooth_s2s_0.log @@ -36,7 +36,7 @@ cat expsmooth_s2s_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep ' # Full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_s2s/model.full.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_s2s_f.log @@ -49,7 +49,7 @@ cat expsmooth_s2s_f.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep ' # A first part of batches -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_s2s/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_save --exponential-smoothing $opt_exp $opts \ --log expsmooth_s2s_1.log @@ -63,7 +63,7 @@ cat expsmooth_s2s_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep ' # Continue training until full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_s2s/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_s2s_2.log diff --git a/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh b/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh index 20aadcf..dd440c8 100644 --- a/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh +++ b/tests/training/restoring/exp-smoothing/test_expsmooth_sync.sh @@ -23,7 +23,7 @@ opt_exp=0.00001 # Full pass, no exponential-smoothing -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.noexp.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish $opts \ --log expsmooth_sync_0.log @@ -36,7 +36,7 @@ cat expsmooth_sync_0.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep # Full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.full.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_sync_f.log @@ -49,7 +49,7 @@ cat expsmooth_sync_f.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep # A first part of batches -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_save --exponential-smoothing $opt_exp $opts \ --log expsmooth_sync_1.log @@ -63,7 +63,7 @@ cat expsmooth_sync_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | grep # Continue training until full pass -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m expsmooth_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --valid-freq $opt_valid --after-batches $opt_finish --exponential-smoothing $opt_exp $opts \ --log expsmooth_sync_2.log diff --git a/tests/training/restoring/multi-gpu/test_async.sh b/tests/training/restoring/multi-gpu/test_async.sh index 0c3fcca..c32050d 100644 --- a/tests/training/restoring/multi-gpu/test_async.sh +++ b/tests/training/restoring/multi-gpu/test_async.sh @@ -18,7 +18,7 @@ opt_disp=1 opt_save=8 opt_finish=16 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m async/model.full.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --after-batches $opt_finish $opts \ --log async_f.log @@ -28,7 +28,7 @@ test -e async_f.log cat async_f.log | $MRT_TOOLS/extract-costs.sh > async.unsorted.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m async/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --after-batches $opt_save $opts \ --log async_1.log @@ -39,7 +39,7 @@ test -e async_1.log cat async_1.log | $MRT_TOOLS/extract-costs.sh > async.unsorted.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m async/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq $opt_disp --after-batches $opt_finish $opts \ --log async_2.log diff --git a/tests/training/restoring/multi-gpu/test_sync.sh b/tests/training/restoring/multi-gpu/test_sync.sh index 78190f6..98aaf96 100644 --- a/tests/training/restoring/multi-gpu/test_sync.sh +++ b/tests/training/restoring/multi-gpu/test_sync.sh @@ -15,7 +15,7 @@ mkdir -p sync opts="--no-shuffle --seed 777 --mini-batch 4 --maxi-batch 1 --maxi-batch-sort none --dim-rnn 64 --dim-emb 32 --optimizer sgd --learn-rate 0.1 --devices 0 1 --sync-sgd" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sync/model.full.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 $opts \ --log sync_f.log @@ -26,7 +26,7 @@ test -e sync_f.log cat sync_f.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sync.expected -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 50 $opts \ --log sync_1.log @@ -37,7 +37,7 @@ test -e sync_1.log cat sync_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > sync.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 $opts \ --log sync_2.log diff --git a/tests/training/restoring/optimizer/setup.sh b/tests/training/restoring/optimizer/setup.sh index 92aadd2..1a15d62 100644 --- a/tests/training/restoring/optimizer/setup.sh +++ b/tests/training/restoring/optimizer/setup.sh @@ -4,7 +4,7 @@ test -f $MRT_DATA/europarl.de-en/corpus.bpe.de || exit 1 test -e $MRT_DATA/train.max50.en test -e $MRT_DATA/train.max50.de -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/restoring/optimizer/test_adagrad_params.sh b/tests/training/restoring/optimizer/test_adagrad_params.sh index dabe9bd..333cc24 100644 --- a/tests/training/restoring/optimizer/test_adagrad_params.sh +++ b/tests/training/restoring/optimizer/test_adagrad_params.sh @@ -7,7 +7,7 @@ set -e rm -rf adagrad adagrad*.log mkdir -p adagrad -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --dim-emb 128 --dim-rnn 256 \ -m adagrad/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 --save-freq 60 --optimizer adagrad \ @@ -20,10 +20,10 @@ test -e adagrad.log $MRT_TOOLS/extract-costs.sh < adagrad.log > adagrad.costs.out $MRT_TOOLS/diff-nums.py adagrad.costs.out adagrad.costs.expected -p 0.2 -o adagrad.costs.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adagrad/model.npz.optimizer.npz > adagrad.keys.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adagrad/model.npz.optimizer.npz > adagrad.keys.out $MRT_TOOLS/diff.sh adagrad.keys.out adagrad.keys.expected > adagrad.keys.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adagrad/model.npz.optimizer.npz -k "adagrad_gt" > adagrad.gt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adagrad/model.npz.optimizer.npz -k "adagrad_gt" > adagrad.gt.out $MRT_TOOLS/diff-nums.py --numpy -p 0.0001 adagrad.gt.out adagrad.gt.expected -o adagrad.gt.diff # Exit with success code diff --git a/tests/training/restoring/optimizer/test_adam_params.sh b/tests/training/restoring/optimizer/test_adam_params.sh index 7c08df1..65c8321 100644 --- a/tests/training/restoring/optimizer/test_adam_params.sh +++ b/tests/training/restoring/optimizer/test_adam_params.sh @@ -7,7 +7,7 @@ set -e rm -rf adam adam.log mkdir -p adam -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --dim-emb 128 --dim-rnn 256 \ -m adam/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 --save-freq 60 \ @@ -20,12 +20,12 @@ test -e adam.log $MRT_TOOLS/extract-costs.sh < adam.log > adam.costs.out $MRT_TOOLS/diff-nums.py adam.costs.out adam.costs.expected -p 0.2 -o adam.costs.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz > adam.keys.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz > adam.keys.out $MRT_TOOLS/diff.sh adam.keys.out adam.keys.expected > adam.keys.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k "adam_mt" > adam.mt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k "adam_mt" > adam.mt.out $MRT_TOOLS/diff-nums.py --numpy -p 0.0001 adam.mt.out adam.mt.expected -o adam.mt.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k "adam_vt" > adam.vt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam/model.npz.optimizer.npz -k "adam_vt" > adam.vt.out $MRT_TOOLS/diff-nums.py --numpy -p 0.000005 adam.vt.out adam.vt.expected -o adam.vt.diff # Exit with success code diff --git a/tests/training/restoring/optimizer/test_adam_params_async.sh b/tests/training/restoring/optimizer/test_adam_params_async.sh index 211c135..108db52 100644 --- a/tests/training/restoring/optimizer/test_adam_params_async.sh +++ b/tests/training/restoring/optimizer/test_adam_params_async.sh @@ -12,7 +12,7 @@ if (( $MRT_NUM_DEVICES < 2 )); then exit 100 fi -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --dim-emb 128 --dim-rnn 256 \ -m adam_async/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 --save-freq 60 \ @@ -25,11 +25,11 @@ test -e adam_async.log $MRT_TOOLS/extract-costs.sh < adam_async.log > adam_async.costs.out $MRT_TOOLS/diff-nums.py adam_async.costs.out adam_async.costs.expected -p 10.00 -n 2 -o adam_async.costs.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz > adam_async.keys.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz > adam_async.keys.out $MRT_TOOLS/diff.sh adam_async.keys.out adam.keys.expected > adam_async.keys.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k "adam_mt" > adam_async.mt.out -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k "adam_vt" > adam_async.vt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k "adam_mt" > adam_async.mt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_async/model.npz.optimizer.npz -k "adam_vt" > adam_async.vt.out $MRT_TOOLS/diff-nums.py --numpy -a -p 0.02 adam_async.mt.out adam_async.mt.expected -o adam_async.mt.diff $MRT_TOOLS/diff-nums.py --numpy -p 0.001 adam_async.vt.out adam_async.vt.expected -o adam_async.vt.diff diff --git a/tests/training/restoring/optimizer/test_adam_params_sync.sh b/tests/training/restoring/optimizer/test_adam_params_sync.sh index e287bff..f5fc3cf 100644 --- a/tests/training/restoring/optimizer/test_adam_params_sync.sh +++ b/tests/training/restoring/optimizer/test_adam_params_sync.sh @@ -12,7 +12,7 @@ if (( $MRT_NUM_DEVICES < 2 )); then exit 100 fi -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 32 --dim-emb 128 --dim-rnn 256 \ -m adam_sync/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 10 --after-batches 100 --save-freq 60 \ @@ -25,11 +25,11 @@ test -e adam_sync.log $MRT_TOOLS/extract-costs.sh < adam_sync.log > adam_sync.costs.out $MRT_TOOLS/diff-nums.py adam_sync.costs.out adam_sync.costs.expected -p 3.00 -n 2 -o adam_sync.costs.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz > adam_sync.keys.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz > adam_sync.keys.out $MRT_TOOLS/diff.sh adam_sync.keys.out adam.keys.expected > adam_sync.keys.diff -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k "adam_mt" > adam_sync.mt.out -python $MRT_MARIAN/scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k "adam_vt" > adam_sync.vt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k "adam_mt" > adam_sync.mt.out +python $MRT_MARIAN/../scripts/contrib/model_info.py -m adam_sync/model.npz.optimizer.npz -k "adam_vt" > adam_sync.vt.out $MRT_TOOLS/diff-nums.py --numpy -p 0.002 adam_sync.mt.out adam_sync.mt.expected -o adam_sync.mt.diff $MRT_TOOLS/diff-nums.py --numpy -p 0.0002 adam_sync.vt.out adam_sync.vt.expected -o adam_sync.vt.diff diff --git a/tests/training/restoring/optimizer/test_loading_adam_params.sh b/tests/training/restoring/optimizer/test_loading_adam_params.sh index 88ddf64..00c0cdc 100644 --- a/tests/training/restoring/optimizer/test_loading_adam_params.sh +++ b/tests/training/restoring/optimizer/test_loading_adam_params.sh @@ -9,7 +9,7 @@ mkdir -p adam_load extra_opts="--no-shuffle --seed 7777 --maxi-batch 1 --maxi-batch-sort none --mini-batch 2 --dim-rnn 64 --dim-emb 32" -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m adam_load/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 1 --after-batches 3 -l 0.1 $extra_opts \ --log adam_load_1.log @@ -20,7 +20,7 @@ test -e adam_load_1.log cat adam_load_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "Ep\. " | sed 's/ : Time.*//' > adam_load.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m adam_load/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ --disp-freq 1 --after-batches 6 -l 0.1 $extra_opts \ --log adam_load_2.log diff --git a/tests/training/restoring/validation/setup.sh b/tests/training/restoring/validation/setup.sh index bbbb893..58c1a93 100644 --- a/tests/training/restoring/validation/setup.sh +++ b/tests/training/restoring/validation/setup.sh @@ -7,7 +7,7 @@ test -e $MRT_DATA/train.max50.de test -e dev.bpe.de || tail -n 100 $MRT_DATA/europarl.de-en/corpus.bpe.de > dev.bpe.de test -e dev.bpe.en || tail -n 100 $MRT_DATA/europarl.de-en/corpus.bpe.en > dev.bpe.en -test -e vocab.de.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml -test -e vocab.en.yml || $MRT_MARIAN/build/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml +test -e vocab.de.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.de > vocab.de.yml +test -e vocab.en.yml || $MRT_MARIAN/marian-vocab < $MRT_DATA/europarl.de-en/corpus.bpe.en > vocab.en.yml test -e vocab.de.yml test -e vocab.en.yml diff --git a/tests/training/restoring/validation/test_adding_validator_after_restart.sh b/tests/training/restoring/validation/test_adding_validator_after_restart.sh index 9a93a6b..e8ef741 100644 --- a/tests/training/restoring/validation/test_adding_validator_after_restart.sh +++ b/tests/training/restoring/validation/test_adding_validator_after_restart.sh @@ -8,7 +8,7 @@ rm -rf valid_add valid_add_?.log mkdir -p valid_add -#$MRT_MARIAN/build/marian \ +#$MRT_MARIAN/marian \ #--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ #--dim-emb 128 --dim-rnn 256 --mini-batch 16 \ #-m valid_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -21,7 +21,7 @@ mkdir -p valid_add #exit 1 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ --dim-emb 128 --dim-rnn 256 --mini-batch 16 \ -m valid_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -37,7 +37,7 @@ test -e valid_add_1.log cp valid_add/model.npz.progress.yml valid_add/model.npz.progress.yml.bac cat valid_add_1.log | $MRT_TOOLS/strip-timestamps.sh > valid_add.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --optimizer sgd \ --dim-emb 128 --dim-rnn 256 --mini-batch 16 \ -m valid_add/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/restoring/validation/test_restoring_newbest_validators.sh b/tests/training/restoring/validation/test_restoring_newbest_validators.sh index 8674944..f1b8dc4 100644 --- a/tests/training/restoring/validation/test_restoring_newbest_validators.sh +++ b/tests/training/restoring/validation/test_restoring_newbest_validators.sh @@ -13,7 +13,7 @@ head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de # Uncomment to re-generate the expected output -#$MRT_MARIAN/build/marian \ +#$MRT_MARIAN/marian \ #--type s2s --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation \ #--dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd \ #-m valid_newbest/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -27,7 +27,7 @@ head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de #exit 1 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --type s2s --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation \ --dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd \ -m valid_newbest/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -45,7 +45,7 @@ cp valid_newbest/model.npz.progress.yml valid_newbest/model.npz.progress.yml.bac cat valid_newbest_1.log | $MRT_TOOLS/strip-timestamps.sh | grep -P "\[valid\]" > valid_newbest.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ -m valid_newbest/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ --after-batches 100 --log valid_newbest_2.log diff --git a/tests/training/restoring/validation/test_restoring_stalled_validators.sh b/tests/training/restoring/validation/test_restoring_stalled_validators.sh index a53e323..358c272 100644 --- a/tests/training/restoring/validation/test_restoring_stalled_validators.sh +++ b/tests/training/restoring/validation/test_restoring_stalled_validators.sh @@ -14,7 +14,7 @@ head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de # Uncomment to re-generate the expected output #mkdir -p valid_stalled_full -#$MRT_MARIAN/build/marian \ +#$MRT_MARIAN/marian \ #--no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation \ #--dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd \ #-m valid_stalled_full/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -33,7 +33,7 @@ head -n 8 $MRT_DATA/europarl.de-en/toy.bpe.de > valid.mini.bpe.de #exit 1 -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation \ --dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd \ -m valid_stalled/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -53,7 +53,7 @@ cat valid_stalled_1.log | $MRT_TOOLS/strip-timestamps.sh \ | head -n -1 > valid_stalled.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none --quiet-translation \ --dim-emb 64 --dim-rnn 128 --mini-batch 16 --optimizer sgd \ -m valid_stalled/model.npz -t $MRT_DATA/europarl.de-en/toy.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/restoring/validation/test_restoring_validation.sh b/tests/training/restoring/validation/test_restoring_validation.sh index 3a1786b..bdc81b7 100644 --- a/tests/training/restoring/validation/test_restoring_validation.sh +++ b/tests/training/restoring/validation/test_restoring_validation.sh @@ -7,7 +7,7 @@ set -e rm -rf valid valid_?.log valid_script.temp mkdir -p valid -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none \ --dim-emb 128 --dim-rnn 256 --mini-batch 16 \ -m valid/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -23,7 +23,7 @@ test -e valid_1.log cp valid/model.npz.progress.yml valid/model.npz.progress.yml.bac cat valid_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "valid-script" > valid.out -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 2222 --maxi-batch 1 --maxi-batch-sort none \ --dim-emb 128 --dim-rnn 256 --mini-batch 16 \ -m valid/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh index 4a6c4a9..74fefa5 100644 --- a/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh +++ b/tests/training/restoring/validation/test_restoring_validation_lower_is_better.sh @@ -8,7 +8,7 @@ rm -rf valid_lowisbet valid_lowisbet_?.log mkdir -p valid_lowisbet # Files for the validation sets are swapped intentionally -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none \ --dim-emb 64 --dim-rnn 128 --mini-batch 32 \ -m valid_lowisbet/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ @@ -24,7 +24,7 @@ cp valid_lowisbet/model.npz.progress.yml valid_lowisbet/model.npz.progress.yml.b cat valid_lowisbet_1.log | $MRT_TOOLS/strip-timestamps.sh | grep "cross-entropy" > valid_lowisbet.out # Files for the validation sets are swapped intentionally -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --maxi-batch 1 --maxi-batch-sort none \ --dim-emb 64 --dim-rnn 128 --mini-batch 32 \ -m valid_lowisbet/model.npz -t $MRT_DATA/train.max50.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/validation/test_empty_valid_set_lm.sh b/tests/training/validation/test_empty_valid_set_lm.sh index c96b717..3c6afcb 100644 --- a/tests/training/validation/test_empty_valid_set_lm.sh +++ b/tests/training/validation/test_empty_valid_set_lm.sh @@ -9,7 +9,7 @@ rm -rf empty_valid_lm empty_valid_lm.log empty_valid.?? mkdir -p empty_valid_lm touch empty_valid.en -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ --model empty_valid_lm/model.npz --type lm \ -t $MRT_DATA/europarl.de-en/corpus.bpe.en -v vocab.en.yml \ diff --git a/tests/training/validation/test_empty_valid_set_source.sh b/tests/training/validation/test_empty_valid_set_source.sh index e4ad04d..8082926 100644 --- a/tests/training/validation/test_empty_valid_set_source.sh +++ b/tests/training/validation/test_empty_valid_set_source.sh @@ -9,7 +9,7 @@ rm -rf empty_src empty_src.log empty_valid.?? mkdir -p empty_src touch empty_valid.en -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ --model empty_src/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/validation/test_empty_valid_set_target.sh b/tests/training/validation/test_empty_valid_set_target.sh index 403fb46..640eb51 100644 --- a/tests/training/validation/test_empty_valid_set_target.sh +++ b/tests/training/validation/test_empty_valid_set_target.sh @@ -9,7 +9,7 @@ rm -rf empty_trg empty_trg.log empty_valid.?? mkdir -p empty_trg touch empty_valid.de -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ --model empty_trg/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/validation/test_empty_valid_sets.sh b/tests/training/validation/test_empty_valid_sets.sh index af6ff7f..9e1e79f 100644 --- a/tests/training/validation/test_empty_valid_sets.sh +++ b/tests/training/validation/test_empty_valid_sets.sh @@ -10,7 +10,7 @@ mkdir -p empty_validsets touch empty_valid.en touch empty_valid.de -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ --model empty_validsets/model.npz \ -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} -v vocab.en.yml vocab.de.yml \ diff --git a/tests/training/validation/test_final_validation_after_batches.sh b/tests/training/validation/test_final_validation_after_batches.sh index d6de7f1..b4ccc3c 100644 --- a/tests/training/validation/test_final_validation_after_batches.sh +++ b/tests/training/validation/test_final_validation_after_batches.sh @@ -7,7 +7,7 @@ set -e rm -rf final_batch final_batch.log vocab.*.yml mkdir -p final_batch -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ -m final_batch/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} \ -v vocab.en.yml vocab.de.yml --dim-vocabs 50000 50000 \ diff --git a/tests/training/validation/test_final_validation_after_batches_match.sh b/tests/training/validation/test_final_validation_after_batches_match.sh index 673c087..fc676cf 100644 --- a/tests/training/validation/test_final_validation_after_batches_match.sh +++ b/tests/training/validation/test_final_validation_after_batches_match.sh @@ -7,7 +7,7 @@ set -e rm -rf final_match final_match.log vocab.*.yml mkdir -p final_match -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ -m final_match/model.npz -t $MRT_DATA/europarl.de-en/corpus.bpe.{en,de} \ -v vocab.en.yml vocab.de.yml --dim-vocabs 50000 50000 \ diff --git a/tests/training/validation/test_final_validation_after_epochs.sh b/tests/training/validation/test_final_validation_after_epochs.sh index 08d391f..e8e606f 100644 --- a/tests/training/validation/test_final_validation_after_epochs.sh +++ b/tests/training/validation/test_final_validation_after_epochs.sh @@ -10,7 +10,7 @@ mkdir -p final_epoch test -e train.bpe.en || head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.en > train.bpe.en test -e train.bpe.de || head -n 3000 $MRT_DATA/europarl.de-en/corpus.bpe.de > train.bpe.de -$MRT_MARIAN/build/marian \ +$MRT_MARIAN/marian \ --no-shuffle --seed 1111 --optimizer sgd --dim-emb 64 --dim-rnn 128 \ -m final_epoch/model.npz -t train.bpe.{en,de} \ -v vocab.small.en.yml vocab.small.de.yml --dim-vocabs 50000 50000 \ |