diff options
author | Taku Kudo <taku@google.com> | 2018-06-18 03:31:16 +0300 |
---|---|---|
committer | Taku Kudo <taku@google.com> | 2018-06-18 03:31:16 +0300 |
commit | 75c18c6e0467b32371309862259dcf6be55ace72 (patch) | |
tree | 347445d637136dd20b23d6c184c24ba1ba1df52c /src/model_interface.cc | |
parent | 6884e10ccd24d5192b0da010ecfdb5081b1a4dc6 (diff) |
Uses abs::string_view instead of StringPiece
Diffstat (limited to 'src/model_interface.cc')
-rw-r--r-- | src/model_interface.cc | 22 |
1 files changed, 12 insertions, 10 deletions
diff --git a/src/model_interface.cc b/src/model_interface.cc index 255d1be..5cbb1a5 100644 --- a/src/model_interface.cc +++ b/src/model_interface.cc @@ -20,7 +20,7 @@ namespace sentencepiece { -PrefixMatcher::PrefixMatcher(const std::set<StringPiece> &dic) { +PrefixMatcher::PrefixMatcher(const std::set<absl::string_view> &dic) { if (dic.empty()) return; std::vector<const char *> key; key.reserve(dic.size()); @@ -30,7 +30,7 @@ PrefixMatcher::PrefixMatcher(const std::set<StringPiece> &dic) { nullptr)); } -int PrefixMatcher::PrefixMatch(StringPiece w, bool *found) const { +int PrefixMatcher::PrefixMatch(absl::string_view w, bool *found) const { if (trie_ == nullptr) { if (found) *found = false; return std::min<int>(w.size(), string_util::OneCharLen(w.data())); @@ -54,7 +54,8 @@ int PrefixMatcher::PrefixMatch(StringPiece w, bool *found) const { return mblen; } -std::string PrefixMatcher::GlobalReplace(StringPiece w, StringPiece out) const { +std::string PrefixMatcher::GlobalReplace(absl::string_view w, + absl::string_view out) const { std::string result; while (!w.empty()) { bool found = false; @@ -73,7 +74,7 @@ ModelInterface::ModelInterface(const ModelProto &model_proto) : model_proto_(&model_proto), status_(util::OkStatus()) {} ModelInterface::~ModelInterface() {} -int ModelInterface::PieceToId(StringPiece piece) const { +int ModelInterface::PieceToId(absl::string_view piece) const { auto it = reserved_id_map_.find(piece); if (it != reserved_id_map_.end()) { return it->second; @@ -119,7 +120,7 @@ void ModelInterface::InitializePieces(bool use_prefix_matcher) { reserved_id_map_.clear(); unk_id_ = -1; - std::set<StringPiece> user_defined_symbols; + std::set<absl::string_view> user_defined_symbols; for (int i = 0; i < model_proto_->pieces_size(); ++i) { const auto &sp = model_proto_->pieces(i); @@ -162,22 +163,23 @@ void ModelInterface::InitializePieces(bool use_prefix_matcher) { } } -std::vector<StringPiece> SplitIntoWords(StringPiece text) { +std::vector<absl::string_view> SplitIntoWords(absl::string_view text) { const char *begin = text.data(); const char *end = text.data() + text.size(); // Space symbol (U+2581) - const StringPiece kSpaceSymbol = "\xe2\x96\x81"; + const absl::string_view kSpaceSymbol = "\xe2\x96\x81"; - std::vector<StringPiece> result; + std::vector<absl::string_view> result; while (begin < end) { const int mblen = std::min<int>(string_util::OneCharLen(begin), end - begin); - if (begin == text.data() || StringPiece(begin, mblen) == kSpaceSymbol) { + if (begin == text.data() || + absl::string_view(begin, mblen) == kSpaceSymbol) { result.emplace_back(begin, 0); // add empty string piece. } result.back() = - StringPiece(result.back().data(), result.back().size() + mblen); + absl::string_view(result.back().data(), result.back().size() + mblen); begin += mblen; } |