Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-08-03 08:36:46 +0300
committerTaku Kudo <taku@google.com>2018-08-03 08:36:46 +0300
commit1caa5fb76eb41c7ce8bf17a67a1db7024d82e788 (patch)
tree9cd94115261e3927391561957315d194679be887 /src/sentencepiece_trainer_test.cc
parenta0b734a4a2a2259e346f5b602ba807c5deef2f0b (diff)
Added JoinPath and StrCat
Diffstat (limited to 'src/sentencepiece_trainer_test.cc')
-rw-r--r--src/sentencepiece_trainer_test.cc35
1 files changed, 18 insertions, 17 deletions
diff --git a/src/sentencepiece_trainer_test.cc b/src/sentencepiece_trainer_test.cc
index ead34c1..3fa57b8 100644
--- a/src/sentencepiece_trainer_test.cc
+++ b/src/sentencepiece_trainer_test.cc
@@ -24,25 +24,26 @@ namespace sentencepiece {
namespace {
TEST(SentencePieceTrainerTest, TrainFromArgsTest) {
- std::string input = FLAGS_data_dir + "/botchan.txt";
- SentencePieceTrainer::Train(std::string("--input=") + input +
- " --model_prefix=m --vocab_size=1000");
- SentencePieceTrainer::Train(std::string("--input=") + input +
- " --model_prefix=m --vocab_size=1000 "
- "--model_type=bpe");
- SentencePieceTrainer::Train(std::string("--input=") + input +
- " --model_prefix=m --vocab_size=1000 "
- "--model_type=char");
- SentencePieceTrainer::Train(std::string("--input=") + input +
- " --model_prefix=m --vocab_size=1000 "
- "--model_type=word");
+ std::string input = util::JoinPath(FLAGS_data_dir, "botchan.txt");
+ SentencePieceTrainer::Train(string_util::StrCat(
+ "--input=", input, " --model_prefix=m --vocab_size=1000"));
+ SentencePieceTrainer::Train(string_util::StrCat(
+ "--input=", input, " --model_prefix=m --vocab_size=1000 ",
+ "--model_type=bpe"));
+ SentencePieceTrainer::Train(string_util::StrCat(
+ "--input=", input, " --model_prefix=m --vocab_size=1000 ",
+ "--model_type=char"));
+ SentencePieceTrainer::Train(string_util::StrCat(
+ "--input=", input, " --model_prefix=m --vocab_size=1000 ",
+ "--model_type=word"));
}
TEST(SentencePieceTrainerTest, TrainWithCustomNormalizationRule) {
- SentencePieceTrainer::Train("--input=" + FLAGS_data_dir +
- "/botchan.txt --model_prefix=m --vocab_size=1000 "
- "--normalization_rule_tsv=" +
- FLAGS_data_dir + "/nfkc.tsv");
+ std::string input = util::JoinPath(FLAGS_data_dir, "botchan.txt");
+ std::string rule = util::JoinPath(FLAGS_data_dir, "nfkc.tsv");
+ SentencePieceTrainer::Train(string_util::StrCat(
+ "--input=", input, " --model_prefix=m --vocab_size=1000 ",
+ "--normalization_rule_tsv=", rule));
}
TEST(SentencePieceTrainerTest, TrainErrorTest) {
@@ -55,7 +56,7 @@ TEST(SentencePieceTrainerTest, TrainErrorTest) {
TEST(SentencePieceTrainerTest, TrainTest) {
TrainerSpec trainer_spec;
- trainer_spec.add_input(FLAGS_data_dir + "/botchan.txt");
+ trainer_spec.add_input(util::JoinPath(FLAGS_data_dir, "botchan.txt"));
trainer_spec.set_model_prefix("m");
trainer_spec.set_vocab_size(1000);
NormalizerSpec normalizer_spec;