Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/marian-nmt/sentencepiece.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'src/normalizer.h')
-rw-r--r--src/normalizer.h12
1 files changed, 7 insertions, 5 deletions
diff --git a/src/normalizer.h b/src/normalizer.h
index b198722..c31864d 100644
--- a/src/normalizer.h
+++ b/src/normalizer.h
@@ -95,10 +95,6 @@ class Normalizer {
friend class Builder;
- // Swap endian in `compiled_chars_map`. Only called big-endian machine.
- static util::Status MaybeSwapEndian(std::string *compiled_chars_map,
- uint32 trie_blob_size);
-
private:
FRIEND_TEST(NormalizerTest, EncodeDecodePrecompiledCharsMapTest);
@@ -126,7 +122,8 @@ class Normalizer {
// Decodes blob into trie_blob and normalized string.
static util::Status DecodePrecompiledCharsMap(absl::string_view blob,
absl::string_view *trie_blob,
- absl::string_view *normalized);
+ absl::string_view *normalized,
+ std::string *buffer = nullptr);
// Maximum size of the return value of Trie, which corresponds
// to the maximum size of shared common prefix in the chars map.
@@ -149,6 +146,11 @@ class Normalizer {
// "_hello" and "_world".
const bool treat_whitespace_as_suffix_ = false;
+#ifndef __BIG_ENDIAN__
+ // Stores the blob for TRIE encoded in big-endian.
+ std::string precompiled_charsmap_buffer_;
+#endif
+
// Normalizer's status.
util::Status status_;
};