1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
#include "coding/transliteration.hpp"
#include "coding/multilang_utf8_string.hpp"
#include "base/logging.hpp"
#include <cstring>
#include "3party/icu/common/unicode/uclean.h"
#include "3party/icu/common/unicode/unistr.h"
#include "3party/icu/common/unicode/utypes.h"
#include "3party/icu/i18n/unicode/translit.h"
#include "3party/icu/i18n/unicode/utrans.h"
Transliteration::~Transliteration()
{
// The use of u_cleanup() just before an application terminates is optional,
// but it should be called only once for performance reasons.
// The primary benefit is to eliminate reports of memory or resource leaks originating
// in ICU code from the results generated by heap analysis tools.
// http://www.icu-project.org/apiref/icu4c/uclean_8h.html#a93f27d0ddc7c196a1da864763f2d8920
m_transliterators.clear();
u_cleanup();
}
Transliteration & Transliteration::Instance()
{
static Transliteration instance;
return instance;
}
void Transliteration::Init(std::string const & icuDataDir)
{
u_setDataDirectory(icuDataDir.c_str());
for (auto const & lang : StringUtf8Multilang::GetSupportedLanguages())
{
if (strlen(lang.m_transliteratorId) == 0 || m_transliterators.count(lang.m_transliteratorId) != 0)
continue;
UErrorCode status = U_ZERO_ERROR;
std::unique_ptr<Transliterator> transliterator(
Transliterator::createInstance(lang.m_transliteratorId, UTRANS_FORWARD, status));
if (transliterator != nullptr)
m_transliterators.emplace(lang.m_transliteratorId, std::move(transliterator));
else
LOG(LWARNING, ("Cannot create transliterator \"", lang.m_transliteratorId, "\", icu error =", status));
}
}
bool Transliteration::Transliterate(std::string const & str, int8_t langCode, std::string & out) const
{
if (str.empty())
return false;
auto const transliteratorId = StringUtf8Multilang::GetTransliteratorIdByCode(langCode);
auto const & it = m_transliterators.find(transliteratorId);
if (it == m_transliterators.end())
{
LOG(LWARNING, ("Transliteration failed, unknown transliterator \"", transliteratorId, "\""));
return false;
}
UnicodeString ustr(str.c_str());
it->second->transliterate(ustr);
if (ustr.isEmpty())
return false;
ustr.toUTF8String(out);
return true;
}
|
