diff options
author | Taku Kudo <taku@google.com> | 2018-05-11 17:24:12 +0300 |
---|---|---|
committer | Taku Kudo <taku@google.com> | 2018-05-11 17:24:12 +0300 |
commit | 2e01a0890ef87312407da71c925fdfa56bdd4d1d (patch) | |
tree | 77dcc98089b9908827f4c1b9dab7c13f5fb9770a | |
parent | 49da7daf913d836edcb84eadb1ea10b277f3e3db (diff) |
s/PopulateNormalizationSpec/PopulateNormalizerSpec/
-rwxr-xr-x | python/setup.py | 2 | ||||
-rw-r--r-- | src/bpe_model_trainer_test.cc | 4 | ||||
-rw-r--r-- | src/builder.cc | 3 | ||||
-rw-r--r-- | src/builder.h | 3 | ||||
-rw-r--r-- | src/builder_test.cc | 4 | ||||
-rw-r--r-- | src/char_model_trainer_test.cc | 2 | ||||
-rw-r--r-- | src/normalizer_test.cc | 2 | ||||
-rw-r--r-- | src/sentencepiece_processor_test.cc | 2 | ||||
-rw-r--r-- | src/sentencepiece_trainer.cc | 2 | ||||
-rw-r--r-- | src/spm_normalize_main.cc | 4 | ||||
-rw-r--r-- | src/unigram_model_trainer_test.cc | 2 | ||||
-rw-r--r-- | src/word_model_trainer_test.cc | 2 |
12 files changed, 15 insertions, 17 deletions
diff --git a/python/setup.py b/python/setup.py index fb5069c..31ba95b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -25,7 +25,7 @@ setup(name = 'sentencepiece', author_email='taku@google.com', description = 'SentencePiece python wrapper', long_description = long_description, - version='0.0.8', + version='0.0.9', url = 'https://github.com/google/sentencepiece', license = 'Apache', platforms = 'Unix', diff --git a/src/bpe_model_trainer_test.cc b/src/bpe_model_trainer_test.cc index 8e4c675..8a45dbe 100644 --- a/src/bpe_model_trainer_test.cc +++ b/src/bpe_model_trainer_test.cc @@ -47,7 +47,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) { NormalizerSpec normalizer_spec; normalizer_spec.set_name("identity"); normalizer_spec.set_add_dummy_prefix(false); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); Trainer trainer(trainer_spec, normalizer_spec); trainer.Train(); @@ -82,7 +82,7 @@ TEST(BPETrainerTest, EndToEndTest) { NormalizerSpec normalizer_spec; normalizer_spec.set_name("nfkc"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); constexpr int kVocabSize = 8000; trainer_spec.set_vocab_size(kVocabSize); diff --git a/src/builder.cc b/src/builder.cc index c94832b..81e2027 100644 --- a/src/builder.cc +++ b/src/builder.cc @@ -220,8 +220,7 @@ util::Status Builder::GetPrecompiledCharsMap(const std::string &name, } // static -util::Status Builder::PopulateNormalizationSpec( - NormalizerSpec *normalizer_spec) { +util::Status Builder::PopulateNormalizerSpec(NormalizerSpec *normalizer_spec) { CHECK_OR_RETURN(normalizer_spec); if (!normalizer_spec->normalization_rule_tsv().empty()) { diff --git a/src/builder.h b/src/builder.h index 6c1d6fe..7a5fdb2 100644 --- a/src/builder.h +++ b/src/builder.h @@ -51,8 +51,7 @@ class Builder { // Populates necessary fields (precompiled_charmap) from // `name` or `normalization_rule_tsv` fields in `normalizer_spec`. - static util::Status PopulateNormalizationSpec( - NormalizerSpec *normalizer_spec); + static util::Status PopulateNormalizerSpec(NormalizerSpec *normalizer_spec); // Makes a normalization mapping based on NFKC. // diff --git a/src/builder_test.cc b/src/builder_test.cc index 48e1aa6..8b549fc 100644 --- a/src/builder_test.cc +++ b/src/builder_test.cc @@ -65,7 +65,7 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) { { NormalizerSpec spec; spec.set_name("nfkc"); - EXPECT_OK(Builder::PopulateNormalizationSpec(&spec)); + EXPECT_OK(Builder::PopulateNormalizerSpec(&spec)); const Normalizer normalizer(spec); EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); EXPECT_EQ(WS "(株)", normalizer.Normalize("㈱")); @@ -75,7 +75,7 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) { { NormalizerSpec spec; spec.set_name("identity"); - EXPECT_OK(Builder::PopulateNormalizationSpec(&spec)); + EXPECT_OK(Builder::PopulateNormalizerSpec(&spec)); const Normalizer normalizer(spec); EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC")); EXPECT_EQ(WS "㈱", normalizer.Normalize("㈱")); diff --git a/src/char_model_trainer_test.cc b/src/char_model_trainer_test.cc index 943dc9a..2fb5097 100644 --- a/src/char_model_trainer_test.cc +++ b/src/char_model_trainer_test.cc @@ -45,7 +45,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) { NormalizerSpec normalizer_spec; normalizer_spec.set_name("identity"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); Trainer trainer(trainer_spec, normalizer_spec); trainer.Train(); diff --git a/src/normalizer_test.cc b/src/normalizer_test.cc index ea68074..c100444 100644 --- a/src/normalizer_test.cc +++ b/src/normalizer_test.cc @@ -26,7 +26,7 @@ namespace { NormalizerSpec MakeDefaultSpec() { NormalizerSpec normalizer_spec; normalizer_spec.set_name("nfkc"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); return normalizer_spec; } } // namespace diff --git a/src/sentencepiece_processor_test.cc b/src/sentencepiece_processor_test.cc index cc766d5..3c5d5fc 100644 --- a/src/sentencepiece_processor_test.cc +++ b/src/sentencepiece_processor_test.cc @@ -102,7 +102,7 @@ std::vector<std::string> GetSpVec(const SentencePieceText &spt) { NormalizerSpec MakeDefaultNormalizerSpec() { NormalizerSpec normalizer_spec; normalizer_spec.set_name("nfkc"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); return normalizer_spec; } diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc index 7513f4c..11e19ed 100644 --- a/src/sentencepiece_trainer.cc +++ b/src/sentencepiece_trainer.cc @@ -37,7 +37,7 @@ util::Status SentencePieceTrainer::Train( const TrainerSpec &trainer_spec, const NormalizerSpec &normalizer_spec) { auto copied_normalizer_spec = normalizer_spec; RETURN_IF_ERROR( - normalizer::Builder::PopulateNormalizationSpec(&copied_normalizer_spec)); + normalizer::Builder::PopulateNormalizerSpec(&copied_normalizer_spec)); auto trainer = TrainerFactory::Create(trainer_spec, copied_normalizer_spec); return trainer->Train(); diff --git a/src/spm_normalize_main.cc b/src/spm_normalize_main.cc index 4267576..f76ba9e 100644 --- a/src/spm_normalize_main.cc +++ b/src/spm_normalize_main.cc @@ -46,10 +46,10 @@ int main(int argc, char *argv[]) { spec = sp.model_proto().normalizer_spec(); } else if (!FLAGS_normalization_rule_tsv.empty()) { spec.set_normalization_rule_tsv(FLAGS_normalization_rule_tsv); - CHECK_OK(Builder::PopulateNormalizationSpec(&spec)); + CHECK_OK(Builder::PopulateNormalizerSpec(&spec)); } else if (!FLAGS_normalization_rule_name.empty()) { spec.set_name(FLAGS_normalization_rule_name); - CHECK_OK(Builder::PopulateNormalizationSpec(&spec)); + CHECK_OK(Builder::PopulateNormalizerSpec(&spec)); } else { LOG(FATAL) << "Sets --model, normalization_rule_tsv, or " "normalization_rule_name flag."; diff --git a/src/unigram_model_trainer_test.cc b/src/unigram_model_trainer_test.cc index 34caf78..232fc33 100644 --- a/src/unigram_model_trainer_test.cc +++ b/src/unigram_model_trainer_test.cc @@ -39,7 +39,7 @@ TEST(UnigramTrainerTest, EndToEndTest) { NormalizerSpec normalizer_spec; normalizer_spec.set_name("identity"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); constexpr int kVocabSize = 8000; trainer_spec.set_vocab_size(kVocabSize); diff --git a/src/word_model_trainer_test.cc b/src/word_model_trainer_test.cc index 87b105a..b509164 100644 --- a/src/word_model_trainer_test.cc +++ b/src/word_model_trainer_test.cc @@ -46,7 +46,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) { NormalizerSpec normalizer_spec; normalizer_spec.set_name("identity"); - EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec)); + EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec)); normalizer_spec.set_add_dummy_prefix(true); Trainer trainer(trainer_spec, normalizer_spec); |