diff options
| author | Taku Kudo <taku@google.com> | 2018-06-06 13:18:53 +0300 |
|---|---|---|
| committer | Taku Kudo <taku@google.com> | 2018-06-06 13:18:53 +0300 |
| commit | aeec94b63ae2e96be762fdfe6f70d934ebd2beb4 (patch) | |
| tree | 05190bdf102a7f760dbcff3342f0290ab220606b /src/model_interface.cc | |
| parent | 93d84a6f3c52fd7407fd8e5e51fb6d1344bf7675 (diff) | |
Support vocab restriction feature in BPE model.
Diffstat (limited to 'src/model_interface.cc')
| -rw-r--r-- | src/model_interface.cc | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/src/model_interface.cc b/src/model_interface.cc index bf8c42e..a938bdb 100644 --- a/src/model_interface.cc +++ b/src/model_interface.cc @@ -76,7 +76,8 @@ void ModelInterface::InitializePieces(bool enable_user_defined) { const bool is_normal_piece = (sp.type() == ModelProto::SentencePiece::NORMAL || - sp.type() == ModelProto::SentencePiece::USER_DEFINED); + sp.type() == ModelProto::SentencePiece::USER_DEFINED || + sp.type() == ModelProto::SentencePiece::UNUSED); if (!port::InsertIfNotPresent( is_normal_piece ? &pieces_ : &reserved_id_map_, sp.piece(), i)) { status_ = util::StatusBuilder(util::error::INTERNAL) |