Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTaku Kudo <taku@google.com>2018-06-08 12:44:58 +0300
committerTaku Kudo <taku@google.com>2018-06-08 16:10:13 +0300
commitdb1eeac98580e3aa5a9e523ece5cdc1dc839b333 (patch)
tree9b4261d1cadf7a6764d983b83db6ca82a8d323d3 /src/model_interface.cc
parent54ccef78b800625a58cbdbac1245d77c9b744e84 (diff)
Allows to define duplicated user defined symbols
Diffstat (limited to 'src/model_interface.cc')
-rw-r--r--src/model_interface.cc15
1 files changed, 15 insertions, 0 deletions
diff --git a/src/model_interface.cc b/src/model_interface.cc
index 656334f..255d1be 100644
--- a/src/model_interface.cc
+++ b/src/model_interface.cc
@@ -54,6 +54,21 @@ int PrefixMatcher::PrefixMatch(StringPiece w, bool *found) const {
return mblen;
}
+std::string PrefixMatcher::GlobalReplace(StringPiece w, StringPiece out) const {
+ std::string result;
+ while (!w.empty()) {
+ bool found = false;
+ const int mblen = PrefixMatch(w, &found);
+ if (found) {
+ result.append(out.data(), out.size());
+ } else {
+ result.append(w.data(), mblen);
+ }
+ w.remove_prefix(mblen);
+ }
+ return result;
+}
+
ModelInterface::ModelInterface(const ModelProto &model_proto)
: model_proto_(&model_proto), status_(util::OkStatus()) {}
ModelInterface::~ModelInterface() {}