diff options
-rw-r--r-- | generator/dumper.cpp | 17 | ||||
-rw-r--r-- | indexer/features_vector.hpp | 2 | ||||
-rw-r--r-- | indexer/search_index_builder.cpp | 70 | ||||
-rw-r--r-- | indexer/search_trie.hpp | 31 | ||||
-rw-r--r-- | indexer/string_file.cpp | 13 | ||||
-rw-r--r-- | indexer/string_file.hpp | 16 | ||||
-rw-r--r-- | search/search_query.cpp | 85 |
7 files changed, 143 insertions, 91 deletions
diff --git a/generator/dumper.cpp b/generator/dumper.cpp index e6102dca58..d03fb66254 100644 --- a/generator/dumper.cpp +++ b/generator/dumper.cpp @@ -127,6 +127,7 @@ namespace feature { typedef pair<strings::UniString, pair<unsigned int, string> > NameElemT; typedef vector<NameElemT> VecToSortT; + VecToSortT v(container.begin(), container.end()); sort(v.begin(), v.end(), &SortFunc<NameElemT>); @@ -164,7 +165,7 @@ namespace feature SearchTokensCollector() : m_currentS(), m_currentCount(0) {} - void operator() (strings::UniString const & s, search::trie::ValueReader::ValueType) + void operator() (strings::UniString const & s, search::trie::ValueReader::ValueType const &) { if (m_currentS == s) { @@ -197,12 +198,20 @@ namespace feature void DumpSearchTokens(string const & fPath) { FilesContainerR container(new FileReader(fPath)); + + feature::DataHeader header; + header.Load(container.GetReader(HEADER_FILE_TAG)); + + serial::CodingParams cp(search::POINT_CODING_BITS, + header.GetDefCodingParams().GetBasePointUint64()); + scoped_ptr<search::TrieIterator> pTrieRoot( ::trie::reader::ReadTrie(container.GetReader(SEARCH_INDEX_FILE_TAG), - ::search::trie::ValueReader(), - ::search::trie::EdgeValueReader())); + search::trie::ValueReader(cp), + search::trie::EdgeValueReader())); + SearchTokensCollector f; - trie::ForEachRef(*pTrieRoot, f, strings::UniString()); + ::trie::ForEachRef(*pTrieRoot, f, strings::UniString()); f.Finish(); while (!f.tokens.empty()) diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp index c66bbd1bee..59f7edf1c1 100644 --- a/indexer/features_vector.hpp +++ b/indexer/features_vector.hpp @@ -28,6 +28,8 @@ public: m_RecordReader.ForEachRecord(DoGetFeatures<ToDo>(m_LoadInfo, toDo)); } + serial::CodingParams const & GetCodingParams() const { return m_LoadInfo.GetDefCodingParams(); } + private: template <class ToDo> class DoGetFeatures { diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index 89e7f43540..22e839da9d 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -28,25 +28,19 @@ namespace struct FeatureNameInserter { StringsFile & m_names; - uint32_t m_pos; - uint32_t m_rank; + StringsFile::ValueT m_val; - FeatureNameInserter(StringsFile & names, uint32_t pos, uint8_t rank) - : m_names(names), m_pos(pos), m_rank(rank) {} + FeatureNameInserter(StringsFile & names) : m_names(names) {} void AddToken(signed char lang, strings::UniString const & s) const { - AddToken(lang, s, m_rank); - } - - void AddToken(signed char lang, strings::UniString const & s, uint32_t /*rank*/) const - { - m_names.AddString(StringsFile::StringT(s, lang, m_pos)); + m_names.AddString(StringsFile::StringT(s, lang, m_val)); } bool operator()(signed char lang, string const & name) const { - strings::UniString uniName = search::NormalizeAndSimplifyString(name); + strings::UniString const uniName = search::NormalizeAndSimplifyString(name); + buffer_vector<strings::UniString, 32> tokens; SplitUniString(uniName, MakeBackInsertFunctor(tokens), search::Delimiters()); if (tokens.size() > 30) @@ -54,30 +48,59 @@ struct FeatureNameInserter LOG(LWARNING, ("Name has too many tokens:", name)); tokens.resize(30); } + for (size_t i = 0; i < tokens.size(); ++i) - AddToken(lang, tokens[i], /*i < 3 ? m_rank + 10 * (3 - i) : */m_rank); + AddToken(lang, tokens[i]); + return true; } }; -struct FeatureInserter +class FeatureInserter { StringsFile & m_names; - explicit FeatureInserter(StringsFile & names) : m_names(names) {} + typedef StringsFile::ValueT ValueT; + typedef search::trie::ValueReader SaverT; + SaverT m_valueSaver; + + void MakeValue(FeatureType const & f, feature::TypesHolder const & types, + uint64_t pos, ValueT & value) const + { + SaverT::ValueType v; + v.m_featureId = static_cast<uint32_t>(pos); + + // get BEST geometry rect of feature + m2::RectD const rect = f.GetLimitRect(-1); + v.m_pt = rect.Center(); + v.m_rank = feature::GetSearchRank(types, v.m_pt, f.GetPopulation()); + + // write to buffer + PushBackByteSink<ValueT> sink(value); + m_valueSaver.Save(sink, v); + } + +public: + FeatureInserter(StringsFile & names, serial::CodingParams const & cp) + : m_names(names), m_valueSaver(cp) + { + } void operator() (FeatureType const & f, uint64_t pos) const { feature::TypesHolder types(f); - // Add names of the feature. - uint8_t const rank = feature::GetSearchRank(types, f.GetLimitRect(-2).Center(), f.GetPopulation()); - FeatureNameInserter toDo(m_names, static_cast<uint32_t>(pos), rank); - f.ForEachNameRef(toDo); + // init inserter with serialized value + FeatureNameInserter inserter(m_names); + MakeValue(f, types, pos, inserter.m_val); - // Add names of categories of the feature. + // add names of the feature + f.ForEachNameRef(inserter); + + // add names of categories of the feature + inserter.m_val.clear(); for (size_t i = 0; i < types.Size(); ++i) - toDo.AddToken(search::CATEGORIES_LANG, search::FeatureTypeToString(types[i])); + inserter.AddToken(search::CATEGORIES_LANG, search::FeatureTypeToString(types[i])); } }; @@ -88,14 +111,17 @@ void indexer::BuildSearchIndex(FeaturesVector const & featuresVector, Writer & w { { StringsFile names(tmpFilePath); - featuresVector.ForEachOffset(FeatureInserter(names)); + serial::CodingParams cp(search::POINT_CODING_BITS, + featuresVector.GetCodingParams().GetBasePointUint64()); + + featuresVector.ForEachOffset(FeatureInserter(names, cp)); names.EndAdding(); names.OpenForRead(); trie::Build(writer, names.Begin(), names.End(), trie::builder::EmptyEdgeBuilder()); - // at this point all readers should be dead + // at this point all readers of StringsFile should be dead } FileWriter::DeleteFileX(tmpFilePath); diff --git a/indexer/search_trie.hpp b/indexer/search_trie.hpp index 19dd556f9c..cd899f19ed 100644 --- a/indexer/search_trie.hpp +++ b/indexer/search_trie.hpp @@ -1,5 +1,7 @@ #pragma once +#include "geometry_serialization.hpp" + #include "../coding/reader.hpp" #include "../coding/trie.hpp" #include "../coding/trie_reader.hpp" @@ -11,16 +13,32 @@ namespace trie { // Value: feature offset and search rank are stored. -struct ValueReader +class ValueReader { + serial::CodingParams const & m_cp; + +public: + explicit ValueReader(serial::CodingParams const & cp) : m_cp(cp) {} + struct ValueType { - uint32_t m_featureId; // Offset of the featuer. + m2::PointD m_pt; // Center point of feature; + uint32_t m_featureId; // Offset of the feature; + uint8_t m_rank; // Rank of feature; }; - template <typename SourceT> void operator() (SourceT & src, ValueType & value) const + template <typename SourceT> void operator() (SourceT & src, ValueType & v) const + { + v.m_pt = serial::LoadPoint(src, m_cp); + v.m_featureId = ReadPrimitiveFromSource<uint32_t>(src); + v.m_rank = ReadPrimitiveFromSource<uint8_t>(src); + } + + template <class TSink> void Save(TSink & sink, ValueType const & v) const { - value.m_featureId = ReadPrimitiveFromSource<uint32_t>(src); + serial::SavePoint(sink, v.m_pt, m_cp); + WriteToSink(sink, v.m_featureId); + WriteToSink(sink, v.m_rank); } }; @@ -29,9 +47,10 @@ typedef ::trie::reader::EmptyValueReader EdgeValueReader; } // namespace search::trie typedef ::trie::Iterator< - search::trie::ValueReader::ValueType, - search::trie::EdgeValueReader::ValueType> TrieIterator; + trie::ValueReader::ValueType, + trie::EdgeValueReader::ValueType> TrieIterator; static const uint8_t CATEGORIES_LANG = 128; + static const uint8_t POINT_CODING_BITS = 24; } // namespace search diff --git a/indexer/string_file.cpp b/indexer/string_file.cpp index 849d37516a..2283552497 100644 --- a/indexer/string_file.cpp +++ b/indexer/string_file.cpp @@ -17,7 +17,7 @@ StringsFile::IdT StringsFile::StringT::Write(TWriter & writer) const CHECK_EQUAL(static_cast<uint64_t>(pos), writer.Pos(), ()); rw::Write(writer, m_name); - WriteVarUint(writer, m_pos); + rw::WriteRaw(writer, m_val); return pos; } @@ -26,21 +26,20 @@ template <class TReader> void StringsFile::StringT::Read(TReader & src) { rw::Read(src, m_name); - m_pos = ReadVarUint<uint32_t>(src); + rw::ReadRaw(src, m_val); } bool StringsFile::StringT::operator < (StringT const & name) const { if (m_name != name.m_name) - return m_name < name.m_name; - if (GetOffset() != name.GetOffset()) - return GetOffset() < name.GetOffset(); - return false; + return (m_name < name.m_name); + + return (m_val < name.m_val); } bool StringsFile::StringT::operator == (StringT const & name) const { - return (m_name == name.m_name && m_pos == name.m_pos); + return (m_name == name.m_name && m_val == name.m_val); } void StringsFile::AddString(StringT const & s) diff --git a/indexer/string_file.hpp b/indexer/string_file.hpp index 4a872d0ae8..50acece3be 100644 --- a/indexer/string_file.hpp +++ b/indexer/string_file.hpp @@ -14,18 +14,18 @@ class StringsFile { public: - + typedef buffer_vector<uint8_t, 32> ValueT; typedef uint32_t IdT; class StringT { strings::UniString m_name; - uint32_t m_pos; + ValueT m_val; public: StringT() {} - StringT(strings::UniString const & name, signed char lang, uint32_t pos) - : m_pos(pos) + StringT(strings::UniString const & name, signed char lang, ValueT const & val) + : m_val(val) { m_name.reserve(name.size() + 1); m_name.push_back(static_cast<uint8_t>(lang)); @@ -39,13 +39,9 @@ public: template <class TCont> void SerializeValue(TCont & cont) const { - cont.resize(4); - uint32_t const i = SwapIfBigEndian(m_pos); - memcpy(&cont[0], &i, 4); + cont.assign(m_val.begin(), m_val.end()); } - uint32_t GetOffset() const { return m_pos; } - bool operator < (StringT const & name) const; bool operator == (StringT const & name) const; @@ -55,7 +51,7 @@ public: void Swap(StringT & r) { m_name.swap(r.m_name); - swap(m_pos, r.m_pos); + m_val.swap(r.m_val); } }; diff --git a/search/search_query.cpp b/search/search_query.cpp index 57c3d695c4..cae9a99b2c 100644 --- a/search/search_query.cpp +++ b/search/search_query.cpp @@ -531,58 +531,59 @@ void Query::SearchFeatures(vector<vector<strings::UniString> > const & tokens, { if (pMwm->m_cont.IsReaderExist(SEARCH_INDEX_FILE_TAG)) { + feature::DataHeader const & header = pMwm->GetHeader(); + serial::CodingParams cp(POINT_CODING_BITS, + header.GetDefCodingParams().GetBasePointUint64()); + scoped_ptr<TrieIterator> pTrieRoot(::trie::reader::ReadTrie( pMwm->m_cont.GetReader(SEARCH_INDEX_FILE_TAG), - ::search::trie::ValueReader(), - ::search::trie::EdgeValueReader())); - if (pTrieRoot) - { - feature::DataHeader const & header = pMwm->GetHeader(); - bool const isWorld = (header.GetType() == feature::DataHeader::world); + trie::ValueReader(cp), + trie::EdgeValueReader())); + + bool const isWorld = (header.GetType() == feature::DataHeader::world); - FeaturesVector featuresVector(pMwm->m_cont, header); - impl::FeatureLoader emitter(featuresVector, *this, isWorld ? "" : mwmLock.GetCountryName()); + FeaturesVector featuresVector(pMwm->m_cont, header); + impl::FeatureLoader emitter(featuresVector, *this, isWorld ? "" : mwmLock.GetCountryName()); - size_t const count = pTrieRoot->m_edge.size(); + size_t const count = pTrieRoot->m_edge.size(); - // Get categories edge root. - scoped_ptr<TrieIterator> pCategoriesRoot; - TrieIterator::Edge::EdgeStrT categoriesEdge; + // Get categories edge root. + scoped_ptr<TrieIterator> pCategoriesRoot; + TrieIterator::Edge::EdgeStrT categoriesEdge; + + for (size_t i = 0; i < count; ++i) + { + TrieIterator::Edge::EdgeStrT const & edge = pTrieRoot->m_edge[i].m_str; + ASSERT_GREATER_OR_EQUAL(edge.size(), 1, ()); - for (size_t i = 0; i < count; ++i) + if (edge[0] == search::CATEGORIES_LANG) { - TrieIterator::Edge::EdgeStrT const & edge = pTrieRoot->m_edge[i].m_str; - ASSERT_GREATER_OR_EQUAL(edge.size(), 1, ()); - - if (edge[0] == search::CATEGORIES_LANG) - { - categoriesEdge = edge; - pCategoriesRoot.reset(pTrieRoot->GoToEdge(i)); - break; - } + categoriesEdge = edge; + pCategoriesRoot.reset(pTrieRoot->GoToEdge(i)); + break; } - ASSERT_NOT_EQUAL(pCategoriesRoot, 0, ()); + } + ASSERT_NOT_EQUAL(pCategoriesRoot, 0, ()); - // Iterate through first language edges. - for (size_t i = 0; i < count; ++i) + // Iterate through first language edges. + for (size_t i = 0; i < count; ++i) + { + TrieIterator::Edge::EdgeStrT const & edge = pTrieRoot->m_edge[i].m_str; + if (edge[0] < search::CATEGORIES_LANG && langs.count(static_cast<int8_t>(edge[0]))) { - TrieIterator::Edge::EdgeStrT const & edge = pTrieRoot->m_edge[i].m_str; - if (edge[0] < search::CATEGORIES_LANG && langs.count(static_cast<int8_t>(edge[0]))) - { - scoped_ptr<TrieIterator> pLangRoot(pTrieRoot->GoToEdge(i)); - - MatchFeaturesInTrie(tokens, m_prefix, - TrieRootPrefix(*pLangRoot, edge), - TrieRootPrefix(*pCategoriesRoot, categoriesEdge), - FeaturesFilter(m_offsetsInViewport[mwmId], isWorld, m_cancel), emitter); - - LOG(LDEBUG, ("Lang:", - StringUtf8Multilang::GetLangByCode(static_cast<int8_t>(edge[0])), - "Matched: ", - emitter.GetCount())); - - emitter.Reset(); - } + scoped_ptr<TrieIterator> pLangRoot(pTrieRoot->GoToEdge(i)); + + MatchFeaturesInTrie(tokens, m_prefix, + TrieRootPrefix(*pLangRoot, edge), + TrieRootPrefix(*pCategoriesRoot, categoriesEdge), + FeaturesFilter(m_offsetsInViewport[mwmId], isWorld, m_cancel), emitter); + + LOG(LDEBUG, ("Lang:", + StringUtf8Multilang::GetLangByCode(static_cast<int8_t>(edge[0])), + "Matched: ", + emitter.GetCount())); + + emitter.Reset(); } } } |