#pragma once #include "search/common.hpp" #include "search/query_params.hpp" #include "search/search_index_values.hpp" #include "search/search_trie.hpp" #include "search/token_slice.hpp" #include "indexer/trie.hpp" #include "base/assert.hpp" #include "base/dfa_helpers.hpp" #include "base/levenshtein_dfa.hpp" #include "base/stl_helpers.hpp" #include "base/string_utils.hpp" #include "base/uni_string_dfa.hpp" #include #include #include #include #include #include #include #include namespace search { namespace impl { namespace { template bool FindLangIndex(trie::Iterator const & trieRoot, uint8_t lang, uint32_t & langIx) { ASSERT_LESS(trieRoot.m_edges.size(), std::numeric_limits::max(), ()); uint32_t const numLangs = static_cast(trieRoot.m_edges.size()); for (uint32_t i = 0; i < numLangs; ++i) { auto const & edge = trieRoot.m_edges[i].m_label; ASSERT_GREATER_OR_EQUAL(edge.size(), 1, ()); if (edge[0] == lang) { langIx = i; return true; } } return false; } } // namespace template bool MatchInTrie(trie::Iterator const & trieRoot, strings::UniChar const * rootPrefix, size_t rootPrefixSize, DFA const & dfa, ToDo && toDo) { using TrieDFAIt = std::shared_ptr>; using DFAIt = typename DFA::Iterator; using State = std::pair; std::queue q; { auto it = dfa.Begin(); DFAMove(it, rootPrefix, rootPrefix + rootPrefixSize); if (it.Rejects()) return false; q.emplace(trieRoot.Clone(), it); } bool found = false; while (!q.empty()) { auto const p = q.front(); q.pop(); auto const & trieIt = p.first; auto const & dfaIt = p.second; if (dfaIt.Accepts()) { trieIt->m_values.ForEach(toDo); found = true; } size_t const numEdges = trieIt->m_edges.size(); for (size_t i = 0; i < numEdges; ++i) { auto const & edge = trieIt->m_edges[i]; auto curIt = dfaIt; strings::DFAMove(curIt, edge.m_label.begin(), edge.m_label.end()); if (!curIt.Rejects()) q.emplace(trieIt->GoToEdge(i), curIt); } } return found; } template class OffsetIntersector { using Set = std::unordered_set; Filter const & m_filter; std::unique_ptr m_prevSet; std::unique_ptr m_set; public: explicit OffsetIntersector(Filter const & filter) : m_filter(filter), m_set(std::make_unique()) { } void operator()(Value const & v) { if (m_prevSet && !m_prevSet->count(v)) return; if (m_filter(v)) m_set->insert(v); } void NextStep() { if (!m_prevSet) m_prevSet = std::make_unique(); m_prevSet.swap(m_set); m_set->clear(); } template void ForEachResult(ToDo && toDo) const { if (!m_prevSet) return; for (auto const & value : *m_prevSet) toDo(value); } }; } // namespace impl template struct TrieRootPrefix { using Value = typename ValueList::Value; using Iterator = trie::Iterator; Iterator const & m_root; strings::UniChar const * m_prefix; size_t m_prefixSize; TrieRootPrefix(Iterator const & root, typename Iterator::Edge::EdgeLabel const & edge) : m_root(root) { if (edge.size() == 1) { m_prefix = 0; m_prefixSize = 0; } else { m_prefix = &edge[1]; m_prefixSize = edge.size() - 1; } } }; template class TrieValuesHolder { public: TrieValuesHolder(Filter const & filter) : m_filter(filter) {} void operator()(Value const & v) { if (m_filter(v)) m_values.push_back(v); } template void ForEachValue(ToDo && toDo) const { for (auto const & value : m_values) toDo(value); } private: std::vector m_values; Filter const & m_filter; }; template struct SearchTrieRequest { template void SetLangs(Langs const & langs) { m_langs.clear(); for (auto const lang : langs) { if (lang >= 0 && lang <= std::numeric_limits::max()) m_langs.insert(static_cast(lang)); } } bool HasLang(int8_t lang) const { return m_langs.find(lang) != m_langs.cend(); } void Clear() { m_names.clear(); m_categories.clear(); m_langs.clear(); } std::vector m_names; std::vector m_categories; // Set of languages, will be prepended to all DFAs in |m_names| // during retrieval from a search index. Semantics of this field // depends on the search index, for example this can be a set of // langs from StringUtf8Multilang, or a set of locale indices. std::unordered_set m_langs; }; // Calls |toDo| for each feature accepted by at least one DFA. // // *NOTE* |toDo| may be called several times for the same feature. template void MatchInTrie(std::vector const & dfas, TrieRootPrefix const & trieRoot, ToDo && toDo) { for (auto const & dfa : dfas) impl::MatchInTrie(trieRoot.m_root, trieRoot.m_prefix, trieRoot.m_prefixSize, dfa, toDo); } // Calls |toDo| for each feature in categories branch matching to |request|. // // *NOTE* |toDo| may be called several times for the same feature. template bool MatchCategoriesInTrie(SearchTrieRequest const & request, trie::Iterator const & trieRoot, ToDo && toDo) { uint32_t langIx = 0; if (!impl::FindLangIndex(trieRoot, search::kCategoriesLang, langIx)) return false; auto const & edge = trieRoot.m_edges[langIx].m_label; ASSERT_GREATER_OR_EQUAL(edge.size(), 1, ()); auto const catRoot = trieRoot.GoToEdge(langIx); MatchInTrie(request.m_categories, TrieRootPrefix(*catRoot, edge), toDo); return true; } // Calls |toDo| with trie root prefix and language code on each // language allowed by |request|. template void ForEachLangPrefix(SearchTrieRequest const & request, trie::Iterator const & trieRoot, ToDo && toDo) { ASSERT_LESS(trieRoot.m_edges.size(), std::numeric_limits::max(), ()); uint32_t const numLangs = static_cast(trieRoot.m_edges.size()); for (uint32_t langIx = 0; langIx < numLangs; ++langIx) { auto const & edge = trieRoot.m_edges[langIx].m_label; ASSERT_GREATER_OR_EQUAL(edge.size(), 1, ()); int8_t const lang = static_cast(edge[0]); if (edge[0] < search::kCategoriesLang && request.HasLang(lang)) { auto const langRoot = trieRoot.GoToEdge(langIx); TrieRootPrefix langPrefix(*langRoot, edge); toDo(langPrefix, lang); } } } // Calls |toDo| for each feature whose description matches to // |request|. Each feature will be passed to |toDo| only once. template void MatchFeaturesInTrie(SearchTrieRequest const & request, trie::Iterator const & trieRoot, Filter const & filter, ToDo && toDo) { using Value = typename ValueList::Value; TrieValuesHolder categoriesHolder(filter); bool const categoriesMatched = MatchCategoriesInTrie(request, trieRoot, categoriesHolder); impl::OffsetIntersector intersector(filter); ForEachLangPrefix( request, trieRoot, [&request, &intersector](TrieRootPrefix & langRoot, int8_t /* lang */) { MatchInTrie(request.m_names, langRoot, intersector); }); if (categoriesMatched) categoriesHolder.ForEachValue(intersector); intersector.NextStep(); intersector.ForEachResult(forward(toDo)); } template void MatchPostcodesInTrie(TokenSlice const & slice, trie::Iterator const & trieRoot, Filter const & filter, ToDo && toDo) { using namespace strings; using Value = typename ValueList::Value; uint32_t langIx = 0; if (!impl::FindLangIndex(trieRoot, search::kPostcodesLang, langIx)) return; auto const & edge = trieRoot.m_edges[langIx].m_label; auto const postcodesRoot = trieRoot.GoToEdge(langIx); impl::OffsetIntersector intersector(filter); for (size_t i = 0; i < slice.Size(); ++i) { if (slice.IsPrefix(i)) { std::vector> dfas; slice.Get(i).ForEach([&dfas](UniString const & s) { dfas.emplace_back(UniStringDFA(s)); }); MatchInTrie(dfas, TrieRootPrefix(*postcodesRoot, edge), intersector); } else { std::vector dfas; slice.Get(i).ForEach([&dfas](UniString const & s) { dfas.emplace_back(s); }); MatchInTrie(dfas, TrieRootPrefix(*postcodesRoot, edge), intersector); } intersector.NextStep(); } intersector.ForEachResult(std::forward(toDo)); } } // namespace search