diff options
author | Taku Kudo <taku@google.com> | 2018-05-01 03:58:42 +0300 |
---|---|---|
committer | Taku Kudo <taku@google.com> | 2018-05-01 03:58:42 +0300 |
commit | bfce4c842b854e96480887dce02cab19c637b7e3 (patch) | |
tree | 01be9a9a734e0ba9e2b945f3eeb19471512ffa22 /src | |
parent | f228e556b484015cdb29c467416dd95ca38360cd (diff) |
Set normalization_rule in once place
Diffstat (limited to 'src')
-rw-r--r-- | src/sentencepiece_trainer.cc | 4 | ||||
-rw-r--r-- | src/spm_train_main.cc | 8 |
2 files changed, 4 insertions, 8 deletions
diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc index bfc2f8e..be74035 100644 --- a/src/sentencepiece_trainer.cc +++ b/src/sentencepiece_trainer.cc @@ -32,7 +32,6 @@ static constexpr char kDefaultNormalizerName[] = "nfkc"; // static util::Status SentencePieceTrainer::Train(const TrainerSpec &trainer_spec) { NormalizerSpec normalizer_spec; - normalizer_spec.set_name(kDefaultNormalizerName); Train(trainer_spec, normalizer_spec); return util::OkStatus(); } @@ -192,10 +191,7 @@ util::Status SentencePieceTrainer::MergeSpecsFromArgs( util::Status SentencePieceTrainer::Train(const std::string &args) { TrainerSpec trainer_spec; NormalizerSpec normalizer_spec; - normalizer_spec.set_name(kDefaultNormalizerName); - CHECK_OK(MergeSpecsFromArgs(args, &trainer_spec, &normalizer_spec)); - return Train(trainer_spec, normalizer_spec); } diff --git a/src/spm_train_main.cc b/src/spm_train_main.cc index 761cc15..92cc6a1 100644 --- a/src/spm_train_main.cc +++ b/src/spm_train_main.cc @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License.! -#include "builder.h" +#include <map> #include "flags.h" +#include "sentencepiece_model.pb.h" #include "sentencepiece_trainer.h" #include "util.h" using sentencepiece::NormalizerSpec; using sentencepiece::TrainerSpec; -using sentencepiece::normalizer::Builder; namespace { static sentencepiece::TrainerSpec kDefaultTrainerSpec; @@ -141,8 +141,8 @@ int main(int argc, char *argv[]) { {"word", TrainerSpec::WORD}, {"char", TrainerSpec::CHAR}}; - trainer_spec.set_model_type( - sentencepiece::port::FindOrDie(kModelTypeMap, FLAGS_model_type)); + trainer_spec.set_model_type(sentencepiece::port::FindOrDie( + kModelTypeMap, sentencepiece::string_util::ToLower(FLAGS_model_type))); CHECK_OK(sentencepiece::SentencePieceTrainer::Train(trainer_spec, normalizer_spec)); |