diff options
| author | Kenneth Heafield <github@kheafield.com> | 2014-06-02 21:28:02 +0400 |
|---|---|---|
| committer | Kenneth Heafield <github@kheafield.com> | 2014-06-02 21:29:40 +0400 |
| commit | dd03f9fb69da82a37f1846d25dcc8d8781956e96 (patch) | |
| tree | 12df06240d6149f110982d1f76c9addf40acbce1 /util/tokenize_piece.hh | |
| parent | 31a583b0bc322626356a42965f02534249070525 (diff) | |
KenLM 5a7efd8fe1db88ee0a9f7e9479b24ac3ca348221 with Hieu's patch to exception.hh
Diffstat (limited to 'util/tokenize_piece.hh')
| -rw-r--r-- | util/tokenize_piece.hh | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/util/tokenize_piece.hh b/util/tokenize_piece.hh index 24eae8fbe..908c8dafe 100644 --- a/util/tokenize_piece.hh +++ b/util/tokenize_piece.hh @@ -1,5 +1,5 @@ -#ifndef UTIL_TOKENIZE_PIECE__ -#define UTIL_TOKENIZE_PIECE__ +#ifndef UTIL_TOKENIZE_PIECE_H +#define UTIL_TOKENIZE_PIECE_H #include "util/exception.hh" #include "util/string_piece.hh" @@ -7,7 +7,8 @@ #include <boost/iterator/iterator_facade.hpp> #include <algorithm> -#include <iostream> + +#include <string.h> namespace util { @@ -71,6 +72,13 @@ class BoolCharacter { return StringPiece(in.data() + in.size(), 0); } + template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) { + memset(out, 0, sizeof(out)); + for (const char *i = characters; i != characters + Length; ++i) { + out[static_cast<unsigned char>(*i)] = true; + } + } + private: const bool *delimiter_; }; @@ -140,4 +148,4 @@ template <class Find, bool SkipEmpty = false> class TokenIter : public boost::it } // namespace util -#endif // UTIL_TOKENIZE_PIECE__ +#endif // UTIL_TOKENIZE_PIECE_H |