diff options
Diffstat (limited to 'indexer/search_string_utils.hpp')
-rw-r--r-- | indexer/search_string_utils.hpp | 56 |
1 files changed, 53 insertions, 3 deletions
diff --git a/indexer/search_string_utils.hpp b/indexer/search_string_utils.hpp index b729e08327..d3cf48f8f1 100644 --- a/indexer/search_string_utils.hpp +++ b/indexer/search_string_utils.hpp @@ -1,15 +1,65 @@ #pragma once #include "../base/string_utils.hpp" +#include "../std/algorithm.hpp" + namespace search { // This function should be used for all search strings normalization. +// It does some magic text transformation which greatly helps us to improve our search. inline strings::UniString NormalizeAndSimplifyString(string const & s) { - strings::UniString uniS = strings::MakeLowerCase(strings::MakeUniString(s)); - strings::Normalize(uniS); - return uniS; + using namespace strings; + UniString uniString = MakeUniString(s); + for (size_t i = 0; i < uniString.size(); ++i) + { + UniChar & c = uniString[i]; + switch (c) + { + // Replace small turkish dotless 'ı' with dotted 'i'. + // Our own invented hack to avoid well-known Turkish I-letter bug. + case 0x0131: c = 'i'; break; + // Replace capital turkish dotted 'İ' with dotted lowercased 'i'. + // Here we need to handle this case manually too, because default unicode-compliant implementation + // of MakeLowerCase converts 'İ' to 'i' + 0x0307. + case 0x0130: c = 'i'; break; + // Some Danish-specific hacks. + case 0x00d8: // Ø + case 0x00f8: c = 'o'; break; // ø + case 0x0152: // Œ + case 0x0153: // œ + c = 'o'; + uniString.insert(uniString.begin() + (i++) + 1, 'e'); + break; + case 0x00c6: // Æ + case 0x00e6: // æ + c = 'a'; + uniString.insert(uniString.begin() + (i++) + 1, 'e'); + break; + } + } + MakeLowerCase(uniString); + // Just after lower casing is a correct place to avoid normalization for specific chars. + static auto const isSpecificChar = [](UniChar c) -> bool + { + return c == 0x0439; // й + }; + UniString result; + result.reserve(uniString.size()); + for (auto i = uniString.begin(), end = uniString.end(); i != end;) + { + auto j = find_if(i, end, isSpecificChar); + // We don't check if (j != i) because UniString and Normalize handle it correctly. + UniString normString(i, j); + Normalize(normString); + result.insert(result.end(), normString.begin(), normString.end()); + if (j == end) + break; + result.push_back(*j); + i = j + 1; + } + return result; } template <class DelimsT, typename F> |