Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-05-10 18:40:42 +0300
committerTaku Kudo <taku@google.com>2018-05-10 18:40:42 +0300
commit54210ca31e1489950acbaf7cc4f449c38940b643 (patch)
tree10e7cd886f4c2d9cc3c9bcb03a32cae936fb8ad3 /src/sentencepiece_trainer.cc
parentd9469a14f96150f5bc94a4c159452b2d50618986 (diff)
CHECK to util::Status migration for Builder
Diffstat (limited to 'src/sentencepiece_trainer.cc')
-rw-r--r--src/sentencepiece_trainer.cc30
1 files changed, 3 insertions, 27 deletions
diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc
index c9ac133..7513f4c 100644
--- a/src/sentencepiece_trainer.cc
+++ b/src/sentencepiece_trainer.cc
@@ -25,9 +25,6 @@
#include "util.h"
namespace sentencepiece {
-namespace {
-static constexpr char kDefaultNormalizerName[] = "nfkc";
-} // namespace
// static
util::Status SentencePieceTrainer::Train(const TrainerSpec &trainer_spec) {
@@ -39,32 +36,11 @@ util::Status SentencePieceTrainer::Train(const TrainerSpec &trainer_spec) {
util::Status SentencePieceTrainer::Train(
const TrainerSpec &trainer_spec, const NormalizerSpec &normalizer_spec) {
auto copied_normalizer_spec = normalizer_spec;
-
- if (!copied_normalizer_spec.normalization_rule_tsv().empty()) {
- CHECK_OR_RETURN(copied_normalizer_spec.precompiled_charsmap().empty())
- << "precompiled_charsmap is already defined.";
-
- const auto chars_map = normalizer::Builder::BuildMapFromFile(
- copied_normalizer_spec.normalization_rule_tsv());
- copied_normalizer_spec.set_precompiled_charsmap(
- normalizer::Builder::CompileCharsMap(chars_map));
- copied_normalizer_spec.set_name("user_defined");
- } else {
- if (copied_normalizer_spec.name().empty()) {
- copied_normalizer_spec.set_name(kDefaultNormalizerName);
- }
-
- if (copied_normalizer_spec.precompiled_charsmap().empty()) {
- *(copied_normalizer_spec.mutable_precompiled_charsmap()) =
- normalizer::Builder::GetNormalizerSpec(copied_normalizer_spec.name())
- .precompiled_charsmap();
- }
- }
+ RETURN_IF_ERROR(
+ normalizer::Builder::PopulateNormalizationSpec(&copied_normalizer_spec));
auto trainer = TrainerFactory::Create(trainer_spec, copied_normalizer_spec);
- RETURN_IF_ERROR(trainer->Train());
-
- return util::OkStatus();
+ return trainer->Train();
}
// static