Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-07-24 09:22:49 +0300
committerTaku Kudo <taku@google.com>2018-07-24 09:22:49 +0300
commit89831f80c125335a56807a1c738f2f509d03d6d5 (patch)
tree6d7dbfcaf2f195ae2d24017f6eee3049e2d7749f /src/sentencepiece_trainer_test.cc
parentd64cc9ada66c4a601536b94e88132937f8768e9c (diff)
Switched to cmake
Diffstat (limited to 'src/sentencepiece_trainer_test.cc')
-rw-r--r--src/sentencepiece_trainer_test.cc35
1 files changed, 20 insertions, 15 deletions
diff --git a/src/sentencepiece_trainer_test.cc b/src/sentencepiece_trainer_test.cc
index 0c2107d..ead34c1 100644
--- a/src/sentencepiece_trainer_test.cc
+++ b/src/sentencepiece_trainer_test.cc
@@ -13,31 +13,36 @@
// limitations under the License.!
#include "sentencepiece_trainer.h"
+#include "flags.h"
#include "sentencepiece_model.pb.h"
#include "testharness.h"
#include "util.h"
+DECLARE_string(data_dir);
+
namespace sentencepiece {
namespace {
TEST(SentencePieceTrainerTest, TrainFromArgsTest) {
- SentencePieceTrainer::Train(
- "--input=../data/botchan.txt --model_prefix=m --vocab_size=1000");
- SentencePieceTrainer::Train(
- "--input=../data/botchan.txt --model_prefix=m --vocab_size=1000 "
- "--model_type=bpe");
- SentencePieceTrainer::Train(
- "--input=../data/botchan.txt --model_prefix=m --vocab_size=1000 "
- "--model_type=char");
- SentencePieceTrainer::Train(
- "--input=../data/botchan.txt --model_prefix=m --vocab_size=1000 "
- "--model_type=word");
+ std::string input = FLAGS_data_dir + "/botchan.txt";
+ SentencePieceTrainer::Train(std::string("--input=") + input +
+ " --model_prefix=m --vocab_size=1000");
+ SentencePieceTrainer::Train(std::string("--input=") + input +
+ " --model_prefix=m --vocab_size=1000 "
+ "--model_type=bpe");
+ SentencePieceTrainer::Train(std::string("--input=") + input +
+ " --model_prefix=m --vocab_size=1000 "
+ "--model_type=char");
+ SentencePieceTrainer::Train(std::string("--input=") + input +
+ " --model_prefix=m --vocab_size=1000 "
+ "--model_type=word");
}
TEST(SentencePieceTrainerTest, TrainWithCustomNormalizationRule) {
- SentencePieceTrainer::Train(
- "--input=../data/botchan.txt --model_prefix=m --vocab_size=1000 "
- "--normalization_rule_tsv=../data/nfkc.tsv");
+ SentencePieceTrainer::Train("--input=" + FLAGS_data_dir +
+ "/botchan.txt --model_prefix=m --vocab_size=1000 "
+ "--normalization_rule_tsv=" +
+ FLAGS_data_dir + "/nfkc.tsv");
}
TEST(SentencePieceTrainerTest, TrainErrorTest) {
@@ -50,7 +55,7 @@ TEST(SentencePieceTrainerTest, TrainErrorTest) {
TEST(SentencePieceTrainerTest, TrainTest) {
TrainerSpec trainer_spec;
- trainer_spec.add_input("../data/botchan.txt");
+ trainer_spec.add_input(FLAGS_data_dir + "/botchan.txt");
trainer_spec.set_model_prefix("m");
trainer_spec.set_vocab_size(1000);
NormalizerSpec normalizer_spec;