Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-05-11 17:24:12 +0300
committerTaku Kudo <taku@google.com>2018-05-11 17:24:12 +0300
commit2e01a0890ef87312407da71c925fdfa56bdd4d1d (patch)
tree77dcc98089b9908827f4c1b9dab7c13f5fb9770a
parent49da7daf913d836edcb84eadb1ea10b277f3e3db (diff)
s/PopulateNormalizationSpec/PopulateNormalizerSpec/
-rwxr-xr-xpython/setup.py2
-rw-r--r--src/bpe_model_trainer_test.cc4
-rw-r--r--src/builder.cc3
-rw-r--r--src/builder.h3
-rw-r--r--src/builder_test.cc4
-rw-r--r--src/char_model_trainer_test.cc2
-rw-r--r--src/normalizer_test.cc2
-rw-r--r--src/sentencepiece_processor_test.cc2
-rw-r--r--src/sentencepiece_trainer.cc2
-rw-r--r--src/spm_normalize_main.cc4
-rw-r--r--src/unigram_model_trainer_test.cc2
-rw-r--r--src/word_model_trainer_test.cc2
12 files changed, 15 insertions, 17 deletions
diff --git a/python/setup.py b/python/setup.py
index fb5069c..31ba95b 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -25,7 +25,7 @@ setup(name = 'sentencepiece',
author_email='taku@google.com',
description = 'SentencePiece python wrapper',
long_description = long_description,
- version='0.0.8',
+ version='0.0.9',
url = 'https://github.com/google/sentencepiece',
license = 'Apache',
platforms = 'Unix',
diff --git a/src/bpe_model_trainer_test.cc b/src/bpe_model_trainer_test.cc
index 8e4c675..8a45dbe 100644
--- a/src/bpe_model_trainer_test.cc
+++ b/src/bpe_model_trainer_test.cc
@@ -47,7 +47,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("identity");
normalizer_spec.set_add_dummy_prefix(false);
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
Trainer trainer(trainer_spec, normalizer_spec);
trainer.Train();
@@ -82,7 +82,7 @@ TEST(BPETrainerTest, EndToEndTest) {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("nfkc");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
constexpr int kVocabSize = 8000;
trainer_spec.set_vocab_size(kVocabSize);
diff --git a/src/builder.cc b/src/builder.cc
index c94832b..81e2027 100644
--- a/src/builder.cc
+++ b/src/builder.cc
@@ -220,8 +220,7 @@ util::Status Builder::GetPrecompiledCharsMap(const std::string &name,
}
// static
-util::Status Builder::PopulateNormalizationSpec(
- NormalizerSpec *normalizer_spec) {
+util::Status Builder::PopulateNormalizerSpec(NormalizerSpec *normalizer_spec) {
CHECK_OR_RETURN(normalizer_spec);
if (!normalizer_spec->normalization_rule_tsv().empty()) {
diff --git a/src/builder.h b/src/builder.h
index 6c1d6fe..7a5fdb2 100644
--- a/src/builder.h
+++ b/src/builder.h
@@ -51,8 +51,7 @@ class Builder {
// Populates necessary fields (precompiled_charmap) from
// `name` or `normalization_rule_tsv` fields in `normalizer_spec`.
- static util::Status PopulateNormalizationSpec(
- NormalizerSpec *normalizer_spec);
+ static util::Status PopulateNormalizerSpec(NormalizerSpec *normalizer_spec);
// Makes a normalization mapping based on NFKC.
//
diff --git a/src/builder_test.cc b/src/builder_test.cc
index 48e1aa6..8b549fc 100644
--- a/src/builder_test.cc
+++ b/src/builder_test.cc
@@ -65,7 +65,7 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) {
{
NormalizerSpec spec;
spec.set_name("nfkc");
- EXPECT_OK(Builder::PopulateNormalizationSpec(&spec));
+ EXPECT_OK(Builder::PopulateNormalizerSpec(&spec));
const Normalizer normalizer(spec);
EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC"));
EXPECT_EQ(WS "(株)", normalizer.Normalize("㈱"));
@@ -75,7 +75,7 @@ TEST(BuilderTest, GetPrecompiledCharsMapTest) {
{
NormalizerSpec spec;
spec.set_name("identity");
- EXPECT_OK(Builder::PopulateNormalizationSpec(&spec));
+ EXPECT_OK(Builder::PopulateNormalizerSpec(&spec));
const Normalizer normalizer(spec);
EXPECT_EQ(WS "ABC", normalizer.Normalize("ABC"));
EXPECT_EQ(WS "㈱", normalizer.Normalize("㈱"));
diff --git a/src/char_model_trainer_test.cc b/src/char_model_trainer_test.cc
index 943dc9a..2fb5097 100644
--- a/src/char_model_trainer_test.cc
+++ b/src/char_model_trainer_test.cc
@@ -45,7 +45,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("identity");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
Trainer trainer(trainer_spec, normalizer_spec);
trainer.Train();
diff --git a/src/normalizer_test.cc b/src/normalizer_test.cc
index ea68074..c100444 100644
--- a/src/normalizer_test.cc
+++ b/src/normalizer_test.cc
@@ -26,7 +26,7 @@ namespace {
NormalizerSpec MakeDefaultSpec() {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("nfkc");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
return normalizer_spec;
}
} // namespace
diff --git a/src/sentencepiece_processor_test.cc b/src/sentencepiece_processor_test.cc
index cc766d5..3c5d5fc 100644
--- a/src/sentencepiece_processor_test.cc
+++ b/src/sentencepiece_processor_test.cc
@@ -102,7 +102,7 @@ std::vector<std::string> GetSpVec(const SentencePieceText &spt) {
NormalizerSpec MakeDefaultNormalizerSpec() {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("nfkc");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
return normalizer_spec;
}
diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc
index 7513f4c..11e19ed 100644
--- a/src/sentencepiece_trainer.cc
+++ b/src/sentencepiece_trainer.cc
@@ -37,7 +37,7 @@ util::Status SentencePieceTrainer::Train(
const TrainerSpec &trainer_spec, const NormalizerSpec &normalizer_spec) {
auto copied_normalizer_spec = normalizer_spec;
RETURN_IF_ERROR(
- normalizer::Builder::PopulateNormalizationSpec(&copied_normalizer_spec));
+ normalizer::Builder::PopulateNormalizerSpec(&copied_normalizer_spec));
auto trainer = TrainerFactory::Create(trainer_spec, copied_normalizer_spec);
return trainer->Train();
diff --git a/src/spm_normalize_main.cc b/src/spm_normalize_main.cc
index 4267576..f76ba9e 100644
--- a/src/spm_normalize_main.cc
+++ b/src/spm_normalize_main.cc
@@ -46,10 +46,10 @@ int main(int argc, char *argv[]) {
spec = sp.model_proto().normalizer_spec();
} else if (!FLAGS_normalization_rule_tsv.empty()) {
spec.set_normalization_rule_tsv(FLAGS_normalization_rule_tsv);
- CHECK_OK(Builder::PopulateNormalizationSpec(&spec));
+ CHECK_OK(Builder::PopulateNormalizerSpec(&spec));
} else if (!FLAGS_normalization_rule_name.empty()) {
spec.set_name(FLAGS_normalization_rule_name);
- CHECK_OK(Builder::PopulateNormalizationSpec(&spec));
+ CHECK_OK(Builder::PopulateNormalizerSpec(&spec));
} else {
LOG(FATAL) << "Sets --model, normalization_rule_tsv, or "
"normalization_rule_name flag.";
diff --git a/src/unigram_model_trainer_test.cc b/src/unigram_model_trainer_test.cc
index 34caf78..232fc33 100644
--- a/src/unigram_model_trainer_test.cc
+++ b/src/unigram_model_trainer_test.cc
@@ -39,7 +39,7 @@ TEST(UnigramTrainerTest, EndToEndTest) {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("identity");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
constexpr int kVocabSize = 8000;
trainer_spec.set_vocab_size(kVocabSize);
diff --git a/src/word_model_trainer_test.cc b/src/word_model_trainer_test.cc
index 87b105a..b509164 100644
--- a/src/word_model_trainer_test.cc
+++ b/src/word_model_trainer_test.cc
@@ -46,7 +46,7 @@ std::string RunTrainer(const std::vector<std::string> &input, int size) {
NormalizerSpec normalizer_spec;
normalizer_spec.set_name("identity");
- EXPECT_OK(normalizer::Builder::PopulateNormalizationSpec(&normalizer_spec));
+ EXPECT_OK(normalizer::Builder::PopulateNormalizerSpec(&normalizer_spec));
normalizer_spec.set_add_dummy_prefix(true);
Trainer trainer(trainer_spec, normalizer_spec);