1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
#pragma once
#include "../base/buffer_vector.hpp"
#include "../std/string.hpp"
#include "../std/stdint.hpp"
#include "../std/sstream.hpp"
#include "../3party/utfcpp/source/utf8/unchecked.h"
/// All methods work with strings in utf-8 format
namespace strings
{
typedef uint32_t UniChar;
//typedef buffer_vector<UniChar, 32> UniString;
/// Make new type, not typedef. Need to specialize DebugPrint.
class UniString : public buffer_vector<UniChar, 32>
{
typedef buffer_vector<UniChar, 32> BaseT;
public:
UniString() {}
explicit UniString(size_t n, UniChar c = UniChar()) : BaseT(n, c) {}
template <class IterT> UniString(IterT b, IterT e) : BaseT(b, e) {}
};
UniString MakeLowerCase(UniString const & s);
/// For implementation @see base/lower_case.cpp
void MakeLowerCase(UniString & s);
UniString Normalize(UniString const & s);
/// For implementation @see base/normilize_unicode.cpp
void Normalize(UniString & s);
void AsciiToLower(string & s);
void Trim(string & s);
void MakeLowerCase(string & s);
string MakeLowerCase(string const & s);
bool EqualNoCase(string const & s1, string const & s2);
UniString MakeUniString(string const & utf8s);
string ToUtf8(UniString const & s);
inline string DebugPrint(UniString const & s)
{
return ToUtf8(s);
}
template <typename DelimFuncT, typename UniCharIterT = UniString::const_iterator>
class TokenizeIterator
{
UniCharIterT m_beg, m_end, m_finish;
DelimFuncT m_delimFunc;
/// Explicitly disabled, because we're storing iterators for string
TokenizeIterator(char const *, DelimFuncT const &);
void move()
{
m_beg = m_end;
while (m_beg != m_finish)
{
if (m_delimFunc(*m_beg))
++m_beg;
else
break;
}
m_end = m_beg;
while (m_end != m_finish)
{
if (m_delimFunc(*m_end))
break;
else
++m_end;
}
}
public:
TokenizeIterator(string const & s, DelimFuncT const & delimFunc)
: m_beg(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFunc(delimFunc)
{
move();
}
TokenizeIterator(UniString const & s, DelimFuncT const & delimFunc)
: m_beg(s.begin()), m_end(s.begin()), m_finish(s.end()), m_delimFunc(delimFunc)
{
move();
}
string operator*() const
{
ASSERT( m_beg != m_finish, ("dereferencing of empty iterator") );
return string(m_beg.base(), m_end.base());
}
operator bool() const { return m_beg != m_finish; }
TokenizeIterator & operator++()
{
move();
return (*this);
}
bool IsLast() const
{
if (!*this)
return false;
TokenizeIterator<DelimFuncT, UniCharIterT> copy(*this);
++copy;
return !copy;
}
UniString GetUniString() const
{
return UniString(m_beg, m_end);
}
};
class SimpleDelimiter
{
UniString m_delims;
public:
SimpleDelimiter(char const * delimChars);
/// @return true if c is delimiter
bool operator()(UniChar c) const;
};
typedef TokenizeIterator<SimpleDelimiter,
::utf8::unchecked::iterator<string::const_iterator> > SimpleTokenizer;
template <typename FunctorT>
void Tokenize(string const & str, char const * delims, FunctorT f)
{
SimpleTokenizer iter(str, delims);
while (iter)
{
f(*iter);
++iter;
}
}
/// @return code of last symbol in string or 0 if s is empty
UniChar LastUniChar(string const & s);
template <class T, size_t N, class TT> bool IsInArray(T (&arr) [N], TT const & t)
{
for (size_t i = 0; i < N; ++i)
if (arr[i] == t)
return true;
return false;
}
bool to_int(char const * s, int & i);
bool to_uint64(char const * s, uint64_t & i);
bool to_int64(char const * s, int64_t & i);
bool to_double(char const * s, double & d);
template <class T>
string to_string(T i)
{
ostringstream ss;
ss << i;
return ss.str();
}
inline bool to_int(string const & s, int & i) { return to_int(s.c_str(), i); }
inline bool to_uint64(string const & s, uint64_t & i) { return to_uint64(s.c_str(), i); }
inline bool to_int64(string const & s, int64_t & i) { return to_int64(s.c_str(), i); }
inline bool to_double(string const & s, double & d) { return to_double(s.c_str(), d); }
/*
template <typename ItT, typename DelimiterT>
typename ItT::value_type JoinStrings(ItT begin, ItT end, DelimiterT const & delimiter)
{
typedef typename ItT::value_type StringT;
if (begin == end) return StringT();
StringT result = *begin++;
for (ItT it = begin; it != end; ++it)
{
result += delimiter;
result += *it;
}
return result;
}
template <typename ContainerT, typename DelimiterT>
typename ContainerT::value_type JoinStrings(ContainerT const & container,
DelimiterT const & delimiter)
{
return JoinStrings(container.begin(), container.end(), delimiter);
}
*/
}
|
