#pragma once #include "base/buffer_vector.hpp" #include "base/macros.hpp" #include "base/stl_helpers.hpp" #include #include #include #include #include #include #include #include #include "3party/utfcpp/source/utf8/unchecked.h" /// All methods work with strings in utf-8 format namespace strings { using UniChar = uint32_t; // typedef buffer_vector UniString; /// Make new type, not typedef. Need to specialize DebugPrint. class UniString : public buffer_vector { using BaseT = buffer_vector; public: using value_type = UniChar; UniString() {} explicit UniString(size_t n, UniChar c = UniChar()) : BaseT(n, c) {} template UniString(Iter b, Iter e) : BaseT(b, e) { } bool IsEqualAscii(char const * s) const; UniString & operator+=(UniString const & rhs) { append(rhs); return *this; } UniString operator+(UniString const & rhs) const { UniString result(*this); result += rhs; return result; } template void Replace(iterator first, iterator last, Iter first2, Iter last2) { auto it = first; auto it2 = first2; for (; it < last && it2 < last2; ++it, ++it2) *it = *it2; if (it == last && it2 == last2) return; if (it == last) { insert(it, it2, last2); return; } erase(it, last); } }; /// Performs full case folding for string to make it search-compatible according /// to rules in ftp://ftp.unicode.org/Public/UNIDATA/CaseFolding.txt /// For implementation @see base/lower_case.cpp void MakeLowerCaseInplace(UniString & s); UniString MakeLowerCase(UniString const & s); /// Performs NFKD - Compatibility decomposition for Unicode according /// to rules in ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt /// For implementation @see base/normalize_unicode.cpp void NormalizeInplace(UniString & s); UniString Normalize(UniString const & s); std::string Normalize(std::string const & s); /// Replaces "full width" unicode digits with ascii ones. void NormalizeDigits(std::string & utf8); void NormalizeDigits(UniString & us); /// Counts number of start symbols in string s (that is not lower and not normalized) that maches /// to lower and normalized string low_s. If s doen't starts with low_s then returns 0; otherwise /// returns number of start symbols in s that equivalent to lowStr /// For implementation @see base/lower_case.cpp size_t CountNormLowerSymbols(UniString const & s, UniString const & lowStr); void AsciiToLower(std::string & s); // TODO(AlexZ): current boost impl uses default std::locale() to trim. // In general, it does not work for any unicode whitespace except ASCII U+0020 one. void Trim(std::string & s); /// Remove any characters that contain in "anyOf" on left and right side of string s void Trim(std::string & s, char const * anyOf); // Replace the first match of the search substring in the input with the format string. // str - An input string // from - A substring to be searched for // to - A substitute string bool ReplaceFirst(std::string & str, std::string const & from, std::string const & to); void MakeLowerCaseInplace(std::string & s); std::string MakeLowerCase(std::string const & s); bool EqualNoCase(std::string const & s1, std::string const & s2); UniString MakeUniString(std::string const & utf8s); std::string ToUtf8(UniString const & s); bool IsASCIIString(std::string const & str); bool IsASCIIDigit(UniChar c); bool IsASCIISpace(UniChar c); bool IsASCIILatin(UniChar c); inline std::string DebugPrint(UniString const & s) { return ToUtf8(s); } template class TokenizeIterator { public: using difference_type = std::ptrdiff_t; using value_type = std::string; using pointer = void; using reference = std::string; using iterator_category = std::input_iterator_tag; // *NOTE* |s| must be not temporary! TokenizeIterator(std::string const & s, DelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn) { Move(); } // *NOTE* |s| must be not temporary! TokenizeIterator(UniString const & s, DelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn) { Move(); } std::string operator*() const { ASSERT(m_start != m_finish, ("Dereferencing of empty iterator.")); return std::string(m_start.base(), m_end.base()); } UniString GetUniString() const { ASSERT(m_start != m_finish, ("Dereferencing of empty iterator.")); return UniString(m_start, m_end); } operator bool() const { return m_start != m_finish; } TokenizeIterator & operator++() { Move(); return *this; } bool operator==(TokenizeIterator const & rhs) const { if (!*this && !rhs) return true; if (*this && rhs) return m_start == rhs.m_start && m_end == rhs.m_end && m_finish == rhs.m_finish; return false; } bool operator!=(TokenizeIterator const & rhs) const { return !(*this == rhs); } private: void Move() { m_start = m_end; while (m_start != m_finish && m_delimFn(*m_start)) ++m_start; m_end = m_start; while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // Token is defined as a pair (|m_start|, |m_end|), where: // // * m_start < m_end // * m_start == begin or m_delimFn(m_start - 1) // * m_end == m_finish or m_delimFn(m_end) // * for all i from [m_start, m_end): !m_delimFn(i) // // This version of TokenizeIterator iterates over all tokens and // keeps the invariant above. Iter m_start; Iter m_end; // The end of the string the iterator iterates over. Iter m_finish; DelimFn m_delimFn; }; template class TokenizeIterator { public: using difference_type = std::ptrdiff_t; using value_type = std::string; using pointer = void; using reference = std::string; using iterator_category = std::input_iterator_tag; // *NOTE* |s| must be not temporary! TokenizeIterator(std::string const & s, DelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn), m_finished(false) { while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // *NOTE* |s| must be not temporary! TokenizeIterator(UniString const & s, DelimFn const & delimFn) : m_start(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFn(delimFn), m_finished(false) { while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } std::string operator*() const { ASSERT(!m_finished, ("Dereferencing of empty iterator.")); return std::string(m_start.base(), m_end.base()); } UniString GetUniString() const { ASSERT(!m_finished, ("Dereferencing of empty iterator.")); return UniString(m_start, m_end); } operator bool() const { return !m_finished; } TokenizeIterator & operator++() { Move(); return *this; } bool operator==(TokenizeIterator const & rhs) const { if (!*this && !rhs) return true; if (*this && rhs) { return m_start == rhs.m_start && m_end == rhs.m_end && m_finish == rhs.m_finish && m_finished == rhs.m_finished; } return false; } bool operator!=(TokenizeIterator const & rhs) const { return !(*this == rhs); } private: void Move() { if (m_end == m_finish) { ASSERT(!m_finished, ()); m_start = m_end = m_finish; m_finished = true; return; } m_start = m_end; ++m_start; m_end = m_start; while (m_end != m_finish && !m_delimFn(*m_end)) ++m_end; } // Token is defined as a pair (|m_start|, |m_end|), where: // // * m_start <= m_end // * m_start == begin or m_delimFn(m_start - 1) // * m_end == m_finish or m_delimFn(m_end) // * for all i from [m_start, m_end): !m_delimFn(i) // // This version of TokenizeIterator iterates over all tokens and // keeps the invariant above. Iter m_start; Iter m_end; // The end of the string the iterator iterates over. Iter m_finish; DelimFn m_delimFn; // When true, iterator is at the end position and is not valid // anymore. bool m_finished; }; class SimpleDelimiter { UniString m_delims; public: SimpleDelimiter(char const * delims); SimpleDelimiter(char delim); // Returns true iff |c| is a delimiter. bool operator()(UniChar c) const; }; using SimpleTokenizer = TokenizeIterator, false /* KeepEmptyTokens */>; using SimpleTokenizerWithEmptyTokens = TokenizeIterator, true /* KeepEmptyTokens */>; template void Tokenize(std::string const & str, char const * delims, TFunctor && f) { SimpleTokenizer iter(str, delims); while (iter) { f(*iter); ++iter; } } template