diff options
author | Maxim Pimenov <m@maps.me> | 2019-05-14 17:17:06 +0300 |
---|---|---|
committer | Arsentiy Milchakov <milcars@mapswithme.com> | 2019-05-15 12:19:00 +0300 |
commit | 06978f4585c463e97db42f96e7140616061939c3 (patch) | |
tree | f6f2029b98e217744064881de78d1c51f16f7602 /indexer | |
parent | 7cb058a8cc64b0567ee429873e07ad306dd19c1f (diff) |
[categories] Removed the '^' symbol.
Now that we have data/strings/types_strings.txt, there is
no need in the special syntax to mark the displayed name in categories.txt.
Diffstat (limited to 'indexer')
-rw-r--r-- | indexer/categories_holder.cpp | 60 | ||||
-rw-r--r-- | indexer/categories_holder.hpp | 13 | ||||
-rw-r--r-- | indexer/indexer_tests/categories_test.cpp | 102 |
3 files changed, 24 insertions, 151 deletions
diff --git a/indexer/categories_holder.cpp b/indexer/categories_holder.cpp index 9156f6c0df..a7514d8c2d 100644 --- a/indexer/categories_holder.cpp +++ b/indexer/categories_holder.cpp @@ -19,23 +19,9 @@ enum State EParseLanguages }; -void ProcessSynonym(CategoriesHolder::Category::Name const & name, - deque<CategoriesHolder::Category::Name> & synonyms) -{ - if (name.m_name[0] != '^') - { - synonyms.push_back(name); - return; - } - - // Name which starts with '^' is readable name for UI and it should be in the beginning. - synonyms.push_front(name); - synonyms.front().m_name = name.m_name.substr(1); -} - -void GroupTranslationsToSynonyms(vector<string> const & groups, - CategoriesHolder::GroupTranslations const & translations, - deque<CategoriesHolder::Category::Name> & synonyms) +void AddGroupTranslationsToSynonyms(vector<string> const & groups, + CategoriesHolder::GroupTranslations const & translations, + vector<CategoriesHolder::Category::Name> & synonyms) { for (string const & group : groups) { @@ -43,19 +29,7 @@ void GroupTranslationsToSynonyms(vector<string> const & groups, if (it == translations.end()) continue; for (auto & synonym : it->second) - ProcessSynonym(synonym, synonyms); - } -} - -void TrimGroupTranslations(CategoriesHolder::GroupTranslations & translations) -{ - for (auto & translation : translations) - { - for (auto & synonym : translation.second) - { - if (synonym.m_name[0] == '^') - synonym.m_name = synonym.m_name.substr(1); - } + synonyms.push_back(synonym); } } @@ -97,7 +71,7 @@ void FillPrefixLengthToSuggest(CategoriesHolder::Category::Name & name) void ProcessName(CategoriesHolder::Category::Name name, vector<string> const & groups, vector<uint32_t> const & types, CategoriesHolder::GroupTranslations & translations, - deque<CategoriesHolder::Category::Name> & synonyms) + vector<CategoriesHolder::Category::Name> & synonyms) { if (name.m_name.empty()) { @@ -111,9 +85,14 @@ void ProcessName(CategoriesHolder::Category::Name name, vector<string> const & g return; if (groups.size() == 1 && types.empty()) - translations[groups[0]].push_back(name); // not a translation, but a category group definition + { + // Not a translation, but a category group definition. + translations[groups[0]].push_back(name); + } else - ProcessSynonym(name, synonyms); + { + synonyms.push_back(name); + } } void ProcessCategory(string const & line, vector<string> & groups, vector<uint32_t> & types) @@ -134,7 +113,7 @@ void ProcessCategory(string const & line, vector<string> & groups, vector<uint32 uint32_t const type = classif().GetTypeByPathSafe(v); if (type == 0) { - LOG(LWARNING, ("Invalid type:", v, "; during parcing the line:", line)); + LOG(LWARNING, ("Invalid type:", v, "; during parsing the line:", line)); return; } @@ -212,11 +191,11 @@ void CategoriesHolder::AddCategory(Category & cat, vector<uint32_t> & types) auto const locale = synonym.m_locale; ASSERT_NOT_EQUAL(locale, kUnsupportedLocaleCode, ()); - auto const localePrefix = String(1, static_cast<strings::UniChar>(locale)); + auto const localePrefix = strings::UniString(1, static_cast<strings::UniChar>(locale)); auto const uniName = search::NormalizeAndSimplifyString(synonym.m_name); - vector<String> tokens; + vector<strings::UniString> tokens; SplitUniString(uniName, base::MakeBackInsertFunctor(tokens), search::Delimiters()); for (auto const & token : tokens) @@ -233,7 +212,7 @@ void CategoriesHolder::AddCategory(Category & cat, vector<uint32_t> & types) types.clear(); } -bool CategoriesHolder::ValidKeyToken(String const & s) +bool CategoriesHolder::ValidKeyToken(strings::UniString const & s) { if (s.size() > 2) return true; @@ -285,10 +264,8 @@ void CategoriesHolder::LoadFromStream(istream & s) if (!types.empty() || currentGroups.size() == 1) { - // Add translations into synonyms first, it will allow to override - // translations for UI by concrete category translation. - GroupTranslationsToSynonyms(currentGroups, m_groupTranslations, cat.m_synonyms); state = EParseLanguages; + AddGroupTranslationsToSynonyms(currentGroups, m_groupTranslations, cat.m_synonyms); } } else if (state == EParseLanguages) @@ -313,9 +290,8 @@ void CategoriesHolder::LoadFromStream(istream & s) } } } - // add last category + // Add the last category. AddCategory(cat, types); - TrimGroupTranslations(m_groupTranslations); } bool CategoriesHolder::GetNameByType(uint32_t type, int8_t locale, string & name) const diff --git a/indexer/categories_holder.hpp b/indexer/categories_holder.hpp index 2f385292df..c114cfe8de 100644 --- a/indexer/categories_holder.hpp +++ b/indexer/categories_holder.hpp @@ -5,7 +5,7 @@ #include "base/string_utils.hpp" #include <algorithm> -#include <deque> +#include <cstdint> #include <iostream> #include <map> #include <memory> @@ -33,7 +33,7 @@ public: uint8_t m_prefixLengthToSuggest; }; - std::deque<Name> m_synonyms; + std::vector<Name> m_synonyms; void Swap(Category & r) { @@ -50,9 +50,8 @@ public: using GroupTranslations = std::unordered_map<std::string, std::vector<Category::Name>>; private: - using String = strings::UniString; using Type2CategoryCont = std::multimap<uint32_t, std::shared_ptr<Category>>; - using Trie = base::MemTrie<String, base::VectorValues<uint32_t>>; + using Trie = base::MemTrie<strings::UniString, base::VectorValues<uint32_t>>; Type2CategoryCont m_type2cat; @@ -119,9 +118,9 @@ public: } template <class ToDo> - void ForEachTypeByName(int8_t locale, String const & name, ToDo && toDo) const + void ForEachTypeByName(int8_t locale, strings::UniString const & name, ToDo && toDo) const { - auto const localePrefix = String(1, static_cast<strings::UniChar>(locale)); + auto const localePrefix = strings::UniString(1, static_cast<strings::UniChar>(locale)); m_name2type.ForEachInNode(localePrefix + name, std::forward<ToDo>(toDo)); } @@ -157,7 +156,7 @@ public: private: void LoadFromStream(std::istream & s); void AddCategory(Category & cat, std::vector<uint32_t> & types); - static bool ValidKeyToken(String const & s); + static bool ValidKeyToken(strings::UniString const & s); }; inline void swap(CategoriesHolder & a, CategoriesHolder & b) diff --git a/indexer/indexer_tests/categories_test.cpp b/indexer/indexer_tests/categories_test.cpp index 1c12f37659..592790d7a7 100644 --- a/indexer/indexer_tests/categories_test.cpp +++ b/indexer/indexer_tests/categories_test.cpp @@ -117,31 +117,6 @@ UNIT_TEST(CategoriesHolder_Smoke) } } -UNIT_TEST(CategoriesHolder_DisplayedNameSmoke) -{ - classificator::Load(); - - auto const & categoriesHolder = GetDefaultCategories(); - auto const & groupTranslations = categoriesHolder.GetGroupTranslations(); - - categoriesHolder.ForEachCategory([](CategoriesHolder::Category const & cat) { - for (auto const & synonym : cat.m_synonyms) - { - TEST_NOT_EQUAL(synonym.m_name[0], '^', ("symbol ^ is used incorrectly in categories.txt " - "and loaded to synonyms.")); - } - }); - - for (auto const & group : groupTranslations) - { - for (auto const & translation : group.second) - { - TEST_NOT_EQUAL(translation.m_name[0], '^', ("symbol ^ is used incorrectly in categories.txt " - "and loaded to group translations")); - } - } -} - UNIT_TEST(CategoriesHolder_LoadDefault) { classificator::Load(); @@ -159,83 +134,6 @@ UNIT_TEST(CategoriesHolder_LoadDefault) TEST_GREATER(counter, 0, ()); } -UNIT_TEST(CategoriesHolder_DisplayedName) -{ - char const kCategories[] = - "@shop\n" - "en:^Shop\n" - "ru:^Mагазин\n" - "\n" - "@meat\n" - "en:Beef|^Meat\n" - "ru:мясо\n" - "de:Schlachter\n" - "\n" - "@butcher\n" - "de:^Metzgerei\n" - "\n" - "shop|@shop\n" - "en:market\n" - "\n" - "shop-alcohol|@shop\n" - "en:Liquor Store|2^Alcostore\n" - "\n" - "shop-bakery|@shop\n" - "en:^buns\n" - "\n" - "shop-butcher|@meat|@butcher\n" - "en:2butcher\n" - "ru:3^Мясная лавка\n" - "de:Geschäft|2Laden\n" - ""; - - classificator::Load(); - CategoriesHolder holder(make_unique<MemReader>(kCategories, ARRAY_SIZE(kCategories) - 1)); - - holder.ForEachTypeAndCategory([](uint32_t const type, CategoriesHolder::Category const & cat) { - auto const readableTypeName = classif().GetReadableObjectName(type); - if (readableTypeName == "shop") - { - TEST_EQUAL(cat.m_synonyms.size(), 3, ()); - TEST_EQUAL(cat.m_synonyms[0].m_name, "Mагазин", ()); - TEST_EQUAL(cat.m_synonyms[1].m_name, "Shop", ()); - TEST_EQUAL(cat.m_synonyms[2].m_name, "market", ()); - } - else if (readableTypeName == "shop-alcohol") - { - TEST_EQUAL(cat.m_synonyms.size(), 4, ()); - TEST_EQUAL(cat.m_synonyms[0].m_name, "Alcostore", ()); - TEST_EQUAL(cat.m_synonyms[1].m_name, "Mагазин", ()); - TEST_EQUAL(cat.m_synonyms[2].m_name, "Shop", ()); - TEST_EQUAL(cat.m_synonyms[3].m_name, "Liquor Store", ()); - } - else if (readableTypeName == "shop-bakery") - { - TEST_EQUAL(cat.m_synonyms.size(), 3, ()); - TEST_EQUAL(cat.m_synonyms[0].m_name, "buns", ()); - TEST_EQUAL(cat.m_synonyms[1].m_name, "Mагазин", ()); - TEST_EQUAL(cat.m_synonyms[2].m_name, "Shop", ()); - } - else if (readableTypeName == "shop-butcher") - { - TEST_EQUAL(cat.m_synonyms.size(), 9, ()); - TEST_EQUAL(cat.m_synonyms[0].m_name, "Мясная лавка", ()); - TEST_EQUAL(cat.m_synonyms[1].m_name, "Metzgerei", ()); - TEST_EQUAL(cat.m_synonyms[2].m_name, "Meat", ()); - TEST_EQUAL(cat.m_synonyms[3].m_name, "Beef", ()); - TEST_EQUAL(cat.m_synonyms[4].m_name, "мясо", ()); - TEST_EQUAL(cat.m_synonyms[5].m_name, "Schlachter", ()); - TEST_EQUAL(cat.m_synonyms[6].m_name, "butcher", ()); - TEST_EQUAL(cat.m_synonyms[7].m_name, "Geschäft", ()); - TEST_EQUAL(cat.m_synonyms[8].m_name, "Laden", ()); - } - else - { - TEST(false, ("Unexpected group name:", readableTypeName)); - } - }); -} - UNIT_TEST(CategoriesHolder_ForEach) { char const kCategories[] = |