1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
#include "coding/transliteration.hpp"
#include "coding/string_utf8_multilang.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
#include "3party/icu/common/unicode/uclean.h"
#include "3party/icu/common/unicode/unistr.h"
#include "3party/icu/common/unicode/utypes.h"
#include "3party/icu/i18n/unicode/translit.h"
#include "3party/icu/i18n/unicode/utrans.h"
#include <cstring>
#include <mutex>
struct Transliteration::TransliteratorInfo
{
TransliteratorInfo()
: m_initialized(false)
{}
std::atomic<bool> m_initialized;
std::mutex m_mutex;
std::unique_ptr<Transliterator> m_transliterator;
};
Transliteration::Transliteration()
: m_mode(Mode::Enabled)
{}
Transliteration::~Transliteration()
{
// The use of u_cleanup() just before an application terminates is optional,
// but it should be called only once for performance reasons.
// The primary benefit is to eliminate reports of memory or resource leaks originating
// in ICU code from the results generated by heap analysis tools.
m_transliterators.clear();
u_cleanup();
}
Transliteration & Transliteration::Instance()
{
static Transliteration instance;
return instance;
}
void Transliteration::Init(std::string const & icuDataDir)
{
// This function should be called at most once in a process,
// before the first ICU operation that will require the loading of an ICU data file.
// This function is not thread-safe. Use it before calling ICU APIs from multiple threads.
u_setDataDirectory(icuDataDir.c_str());
for (auto const & lang : StringUtf8Multilang::GetSupportedLanguages())
{
if (strlen(lang.m_transliteratorId) == 0 || m_transliterators.count(lang.m_transliteratorId) != 0)
continue;
m_transliterators.emplace(lang.m_transliteratorId, std::make_unique<TransliteratorInfo>());
}
}
void Transliteration::SetMode(Transliteration::Mode mode)
{
m_mode = mode;
}
bool Transliteration::Transliterate(std::string const & str, int8_t langCode, std::string & out) const
{
if (m_mode != Mode::Enabled)
return false;
if (str.empty() || strings::IsASCIIString(str))
return false;
std::string transliteratorId(StringUtf8Multilang::GetTransliteratorIdByCode(langCode));
if (transliteratorId.empty())
return false;
auto it = m_transliterators.find(transliteratorId);
if (it == m_transliterators.end())
{
LOG(LWARNING, ("Transliteration failed, unknown transliterator \"", transliteratorId, "\""));
return false;
}
if (!it->second->m_initialized)
{
std::lock_guard<std::mutex> lock(it->second->m_mutex);
if (!it->second->m_initialized)
{
UErrorCode status = U_ZERO_ERROR;
std::string const removeDiacriticRule = ";NFD;[\u02B9-\u02D3\u0301-\u0358\u00B7\u0027]Remove;NFC";
transliteratorId.append(removeDiacriticRule);
UnicodeString translitId(transliteratorId.c_str());
it->second->m_transliterator.reset(Transliterator::createInstance(translitId, UTRANS_FORWARD, status));
if (it->second->m_transliterator == nullptr)
LOG(LWARNING, ("Cannot create transliterator \"", transliteratorId, "\", icu error =", status));
it->second->m_initialized = true;
}
}
if (it->second->m_transliterator == nullptr)
return false;
UnicodeString ustr(str.c_str());
it->second->m_transliterator->transliterate(ustr);
if (ustr.isEmpty())
return false;
ustr.toUTF8String(out);
return true;
}
|
