#pragma once #include "base/buffer_vector.hpp" #include "base/stl_add.hpp" #include "std/algorithm.hpp" #include "std/cstdint.hpp" #include "std/iterator.hpp" #include "std/limits.hpp" #include "std/regex.hpp" #include "std/sstream.hpp" #include "std/string.hpp" #include "std/type_traits.hpp" #include "3party/utfcpp/source/utf8/unchecked.h" /// All methods work with strings in utf-8 format namespace strings { typedef uint32_t UniChar; // typedef buffer_vector UniString; /// Make new type, not typedef. Need to specialize DebugPrint. class UniString : public buffer_vector { typedef buffer_vector BaseT; public: UniString() {} explicit UniString(size_t n, UniChar c = UniChar()) : BaseT(n, c) {} template UniString(IterT b, IterT e) : BaseT(b, e) { } bool IsEqualAscii(char const * s) const; UniString & operator+=(UniString const & rhs) { append(rhs); return *this; } UniString operator+(UniString const & rhs) const { UniString result(*this); result += rhs; return result; } }; /// Performs full case folding for string to make it search-compatible according /// to rules in ftp://ftp.unicode.org/Public/UNIDATA/CaseFolding.txt /// For implementation @see base/lower_case.cpp void MakeLowerCaseInplace(UniString & s); UniString MakeLowerCase(UniString const & s); /// Performs NFKD - Compatibility decomposition for Unicode according /// to rules in ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt /// For implementation @see base/normilize_unicode.cpp void NormalizeInplace(UniString & s); UniString Normalize(UniString const & s); /// Replaces "full width" unicode digits with ascii ones. void NormalizeDigits(string & utf8); void NormalizeDigits(UniString & us); /// Counts number of start symbols in string s (that is not lower and not normalized) that maches /// to lower and normalized string low_s. If s doen't starts with low_s then returns 0; otherwise /// returns number of start symbols in s that equivalent to lowStr /// For implementation @see base/lower_case.cpp size_t CountNormLowerSymbols(UniString const & s, UniString const & lowStr); void AsciiToLower(string & s); // TODO(AlexZ): current boost impl uses default std::locale() to trim. // In general, it does not work for any unicode whitespace except ASCII U+0020 one. void Trim(string & s); /// Remove any characters that contain in "anyOf" on left and right side of string s void Trim(string & s, char const * anyOf); void MakeLowerCaseInplace(string & s); string MakeLowerCase(string const & s); bool EqualNoCase(string const & s1, string const & s2); UniString MakeUniString(string const & utf8s); string ToUtf8(UniString const & s); bool IsASCIIString(string const & str); bool IsASCIIDigit(UniChar c); bool IsASCIILatin(UniChar c); inline string DebugPrint(UniString const & s) { return ToUtf8(s); } template class TokenizeIterator { public: using difference_type = std::ptrdiff_t; using value_type = string; using pointer = void; using reference = string; using iterator_category = std::input_iterator_tag; // *NOTE* |s| must be not temporary! TokenizeIterator(string const & s, TDelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn) { Move(); } // *NOTE* |s| must be not temporary! TokenizeIterator(UniString const & s, TDelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn) { Move(); } string operator*() const { ASSERT(m_start != m_finish, ("Dereferencing of empty iterator.")); return string(m_start.base(), m_end.base()); } UniString GetUniString() const { ASSERT(m_start != m_finish, ("Dereferencing of empty iterator.")); return UniString(m_start, m_end); } operator bool() const { return m_start != m_finish; } TokenizeIterator & operator++() { Move(); return *this; } bool operator==(TokenizeIterator const & rhs) const { if (!*this && !rhs) return true; if (*this && rhs) return m_start == rhs.m_start && m_end == rhs.m_end && m_finish == rhs.m_finish; return false; } bool operator!=(TokenizeIterator const & rhs) const { return !(*this == rhs); } private: void Move() { m_start = m_end; while (m_start != m_finish && m_delimFn(*m_start)) ++m_start; m_end = m_start; while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // Token is defined as a pair (|m_start|, |m_end|), where: // // * m_start < m_end // * m_start == begin or m_delimFn(m_start - 1) // * m_end == m_finish or m_delimFn(m_end) // * for all i from [m_start, m_end): !m_delimFn(i) // // This version of TokenizeIterator iterates over all tokens and // keeps the invariant above. TIt m_start; TIt m_end; // The end of the string the iterator iterates over. TIt m_finish; TDelimFn m_delimFn; }; template class TokenizeIterator { public: using difference_type = std::ptrdiff_t; using value_type = string; using pointer = void; using reference = string; using iterator_category = std::input_iterator_tag; // *NOTE* |s| must be not temporary! TokenizeIterator(string const & s, TDelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn), m_finished(false) { while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // *NOTE* |s| must be not temporary! TokenizeIterator(UniString const & s, TDelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn), m_finished(false) { while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } string operator*() const { ASSERT(!m_finished, ("Dereferencing of empty iterator.")); return string(m_start.base(), m_end.base()); } UniString GetUniString() const { ASSERT(!m_finished, ("Dereferencing of empty iterator.")); return UniString(m_start, m_end); } operator bool() const { return !m_finished; } TokenizeIterator & operator++() { Move(); return *this; } bool operator==(TokenizeIterator const & rhs) const { if (!*this && !rhs) return true; if (*this && rhs) { return m_start == rhs.m_start && m_end == rhs.m_end && m_finish == rhs.m_finish && m_finished == rhs.m_finished; } return false; } bool operator!=(TokenizeIterator const & rhs) const { return !(*this == rhs); } private: void Move() { if (m_end == m_finish) { ASSERT(!m_finished, ()); m_start = m_end = m_finish; m_finished = true; return; } m_start = m_end; ++m_start; m_end = m_start; while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // Token is defined as a pair (|m_start|, |m_end|), where: // // * m_start <= m_end // * m_start == begin or m_delimFn(m_start - 1) // * m_end == m_finish or m_delimFn(m_end) // * for all i from [m_start, m_end): !m_delimFn(i) // // This version of TokenizeIterator iterates over all tokens and // keeps the invariant above. TIt m_start; TIt m_end; // The end of the string the iterator iterates over. TIt m_finish; TDelimFn m_delimFn; // When true, iterator is at the end position and is not valid // anymore. bool m_finished; }; class SimpleDelimiter { UniString m_delims; public: SimpleDelimiter(char const * delims); SimpleDelimiter(char delim); // Returns true iff |c| is a delimiter. bool operator()(UniChar c) const; }; using SimpleTokenizer = TokenizeIterator, false /* KeepEmptyTokens */>; using SimpleTokenizerWithEmptyTokens = TokenizeIterator, true /* KeepEmptyTokens */>; template void Tokenize(string const & str, char const * delims, TFunctor && f) { SimpleTokenizer iter(str, delims); while (iter) { f(*iter); ++iter; } } template