diff options
Diffstat (limited to 'tests/sentencepiece/test_create_spm_vocab_lm.sh')
| -rw-r--r-- | tests/sentencepiece/test_create_spm_vocab_lm.sh | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/tests/sentencepiece/test_create_spm_vocab_lm.sh b/tests/sentencepiece/test_create_spm_vocab_lm.sh index 58cdaba..93d4738 100644 --- a/tests/sentencepiece/test_create_spm_vocab_lm.sh +++ b/tests/sentencepiece/test_create_spm_vocab_lm.sh @@ -31,7 +31,7 @@ grep -q "Setting vocabulary size .* to 4,\?000" vocab.lm.log grep -q "Loading SentencePiece vocabulary .*vocab.en.spm" vocab.lm.log # Extract a textual vocabulary and compare with the expected output -LC_ALL=C $MRT_MARIAN/spm_export_vocab --model vocab.lm/vocab.en.spm | head -n 3980 | sort > vocab.lm.out +LC_ALL=C $MRT_MARIAN/spm_export_vocab --model vocab.lm/vocab.en.spm | sort > vocab.lm.out $MRT_TOOLS/diff-nums.py vocab.lm.out vocabs.en.expected -o vocab.lm.diff |