diff options
author | Sergey Yershov <yershov@corp.mail.ru> | 2015-08-19 20:15:12 +0300 |
---|---|---|
committer | Alex Zolotarev <alex@maps.me> | 2015-09-23 03:02:16 +0300 |
commit | 7216d84b8662790f94479dffe03c7c5393e02f13 (patch) | |
tree | dda4c24b25d910b378875946a1fcd059e9bc13ab /generator | |
parent | 38e66c99d3ee367baf0082f1b4b5da966c52eaf0 (diff) |
Refactor intermediate date readers and writers
Diffstat (limited to 'generator')
-rw-r--r-- | generator/data_cache_file.hpp | 62 | ||||
-rw-r--r-- | generator/osm_decl.cpp | 37 | ||||
-rw-r--r-- | generator/osm_decl.hpp | 93 | ||||
-rw-r--r-- | generator/osm_source.cpp | 159 |
4 files changed, 141 insertions, 210 deletions
diff --git a/generator/data_cache_file.hpp b/generator/data_cache_file.hpp index e40eb0501b..c610386fc4 100644 --- a/generator/data_cache_file.hpp +++ b/generator/data_cache_file.hpp @@ -121,44 +121,29 @@ public: }; } // namespace detail -template <class TStream, class TOffsetFile> -class DataFileBase + +template <bool TModeWrite> +class OSMElementCache { public: using TKey = uint64_t; + using TStream = typename conditional<TModeWrite, FileWriterStream, FileReaderStream>::type; + using TOffsetFile = typename conditional<TModeWrite, FileWriter, FileReader>::type; protected: TStream m_stream; detail::IndexFile<TOffsetFile, uint64_t> m_offsets; public: - DataFileBase(string const & name) : m_stream(name), m_offsets(name + OFFSET_EXT) {} -}; - -class DataFileWriter : public DataFileBase<FileWriterStream, FileWriter> -{ - using TBase = DataFileBase<FileWriterStream, FileWriter>; - -public: - DataFileWriter(string const & name) : TBase(name) {} + OSMElementCache(string const & name) : m_stream(name), m_offsets(name + OFFSET_EXT) {} template <class TValue> void Write(TKey id, TValue const & value) { m_offsets.Add(id, m_stream.Pos()); - m_stream << value; + value.Write(m_stream); } - void SaveOffsets() { m_offsets.WriteAll(); } -}; - -class DataFileReader : public DataFileBase<FileReaderStream, FileReader> -{ - using TBase = DataFileBase<FileReaderStream, FileReader>; - -public: - DataFileReader(string const & name) : TBase(name) {} - template <class TValue> bool Read(TKey id, TValue & value) { @@ -166,7 +151,7 @@ public: if (m_offsets.GetValueByKey(id, pos)) { m_stream.Seek(pos); - m_stream >> value; + value.Read(m_stream); return true; } else @@ -176,34 +161,7 @@ public: } } - void LoadOffsets() { m_offsets.ReadAll(); } -}; - -template <class TNodesHolder, class TData, class TFile> -class BaseFileHolder -{ -protected: - using TKey = typename TData::TKey; - static_assert(is_integral<TKey>::value, "TKey is not integral type"); - - using TIndex = detail::IndexFile<TFile, TKey>; - - TNodesHolder & m_nodes; - - TData m_ways; - TData m_relations; - - TIndex m_nodeToRelations; - TIndex m_wayToRelations; - -public: - BaseFileHolder(TNodesHolder & nodes, string const & dir) - : m_nodes(nodes) - , m_ways(my::JoinFoldersToPath(dir, WAYS_FILE)) - , m_relations(my::JoinFoldersToPath(dir, RELATIONS_FILE)) - , m_nodeToRelations(my::JoinFoldersToPath(dir, string(NODES_FILE) + ID2REL_EXT)) - , m_wayToRelations(my::JoinFoldersToPath(dir, string(WAYS_FILE) + ID2REL_EXT)) - { - } + inline void SaveOffsets() { m_offsets.WriteAll(); } + inline void LoadOffsets() { m_offsets.ReadAll(); } }; } diff --git a/generator/osm_decl.cpp b/generator/osm_decl.cpp index 92ea55308f..0096c3ccce 100644 --- a/generator/osm_decl.cpp +++ b/generator/osm_decl.cpp @@ -1,39 +1,2 @@ #include "generator/osm_decl.hpp" -#include "indexer/classificator.hpp" - -#include "base/macros.hpp" - -#include "std/target_os.hpp" - - -string RelationElement::GetType() const -{ - map<string, string>::const_iterator i = tags.find("type"); - return ((i != tags.end()) ? i->second : string()); -} - -namespace -{ - bool FindRoleImpl(vector<pair<uint64_t, string> > const & cnt, - uint64_t id, string & role) - { - for (size_t i = 0; i < cnt.size(); ++i) - if (cnt[i].first == id) - { - role = cnt[i].second; - return true; - } - return false; - } -} - -bool RelationElement::FindWay(uint64_t id, string & role) const -{ - return FindRoleImpl(ways, id, role); -} - -bool RelationElement::FindNode(uint64_t id, string & role) const -{ - return FindRoleImpl(nodes, id, role); -} diff --git a/generator/osm_decl.hpp b/generator/osm_decl.hpp index 885738765f..8e377bf228 100644 --- a/generator/osm_decl.hpp +++ b/generator/osm_decl.hpp @@ -31,11 +31,9 @@ struct WayElement { if (id == nodes.front()) return nodes.back(); - else - { - ASSERT ( id == nodes.back(), () ); - return nodes.front(); - } + + ASSERT ( id == nodes.back(), () ); + return nodes.front(); } template <class ToDo> void ForEachPoint(ToDo & toDo) const @@ -50,19 +48,48 @@ struct WayElement else for_each(nodes.rbegin(), nodes.rend(), ref(toDo)); } + + + template <class TArchive> + void Write(TArchive & ar) const + { + ar << nodes; + } + + template <class TArchive> + void Read(TArchive & ar) + { + ar >> nodes; + } }; -struct RelationElement +class RelationElement { - typedef vector<pair<uint64_t, string> > ref_vec_t; - ref_vec_t nodes, ways; + using TMembers = vector<pair<uint64_t, string>>; + +public: + TMembers nodes; + TMembers ways; map<string, string> tags; +public: bool IsValid() const { return !(nodes.empty() && ways.empty()); } - string GetType() const; - bool FindWay(uint64_t id, string & role) const; - bool FindNode(uint64_t id, string & role) const; + string GetType() const + { + auto it = tags.find("type"); + return ((it != tags.end()) ? it->second : string()); + } + + bool FindWay(uint64_t id, string & role) const + { + return FindRoleImpl(ways, id, role); + } + + bool FindNode(uint64_t id, string & role) const + { + return FindRoleImpl(nodes, id, role); + } template <class ToDo> void ForEachWay(ToDo & toDo) const { @@ -76,28 +103,28 @@ struct RelationElement ways.swap(rhs.ways); tags.swap(rhs.tags); } -}; - -template <class TArchive> TArchive & operator << (TArchive & ar, WayElement const & e) -{ - ar << e.nodes; - return ar; -} -template <class TArchive> TArchive & operator >> (TArchive & ar, WayElement & e) -{ - ar >> e.nodes; - return ar; -} + template <class TArchive> + void Write(TArchive & ar) const + { + ar << nodes << ways << tags; + } -template <class TArchive> TArchive & operator << (TArchive & ar, RelationElement const & e) -{ - ar << e.nodes << e.ways << e.tags; - return ar; -} + template <class TArchive> + void Read(TArchive & ar) + { + ar >> nodes >> ways >> tags; + } -template <class TArchive> TArchive & operator >> (TArchive & ar, RelationElement & e) -{ - ar >> e.nodes >> e.ways >> e.tags; - return ar; -} +protected: + bool FindRoleImpl(TMembers const & cnt, uint64_t id, string & role) const + { + for (auto const & e : cnt) + if (e.first == id) + { + role = e.second; + return true; + } + return false; + } +}; diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 03e5412f99..e330d1046f 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -48,19 +48,26 @@ uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize) namespace { -template <class TNodesHolder> -class IntermediateDataReader - : public cache::BaseFileHolder<TNodesHolder, cache::DataFileReader, FileReader> +template <class TNodesHolder, bool TModeWrite> +class IntermediateData { - using TReader = cache::DataFileReader; - using TBase = cache::BaseFileHolder<TNodesHolder, TReader, FileReader>; + using TReader = cache::OSMElementCache<TModeWrite>; + + using TFile = typename conditional<TModeWrite, FileWriter, FileReader>::type; + + + using TKey = uint64_t; + static_assert(is_integral<TKey>::value, "TKey is not integral type"); + + using TIndex = cache::detail::IndexFile<TFile, TKey>; + + TNodesHolder & m_nodes; - using typename TBase::TKey; - using TBase::m_nodes; - using TBase::m_nodeToRelations; - using TBase::m_ways; - using TBase::m_wayToRelations; - using TBase::m_relations; + TReader m_ways; + TReader m_relations; + + TIndex m_nodeToRelations; + TIndex m_wayToRelations; template <class TElement, class ToDo> struct ElementProcessorBase @@ -96,13 +103,40 @@ class IntermediateDataReader bool operator()(uint64_t id) { return this->m_toDo(id, this->m_reader); } }; + template <class TIndex, class TContainer> + static void AddToIndex(TIndex & index, TKey relationId, TContainer const & values) + { + for (auto const & v : values) + index.Add(v.first, relationId); + } + public: - IntermediateDataReader(TNodesHolder & holder, string const & dir) : TBase(holder, dir) {} + IntermediateData(TNodesHolder & nodes, string const & dir) + : m_nodes(nodes) + , m_ways(my::JoinFoldersToPath(dir, WAYS_FILE)) + , m_relations(my::JoinFoldersToPath(dir, RELATIONS_FILE)) + , m_nodeToRelations(my::JoinFoldersToPath(dir, string(NODES_FILE) + ID2REL_EXT)) + , m_wayToRelations(my::JoinFoldersToPath(dir, string(WAYS_FILE) + ID2REL_EXT)) + { + } + void AddNode(TKey id, double lat, double lng) { m_nodes.AddPoint(id, lat, lng); } bool GetNode(TKey id, double & lat, double & lng) { return m_nodes.GetPoint(id, lat, lng); } + void AddWay(TKey id, WayElement const & e) { m_ways.Write(id, e); } bool GetWay(TKey id, WayElement & e) { return m_ways.Read(id, e); } + void AddRelation(TKey id, RelationElement const & e) + { + string const & relationType = e.GetType(); + if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary")) + return; + + m_relations.Write(id, e); + AddToIndex(m_nodeToRelations, id, e.nodes); + AddToIndex(m_wayToRelations, id, e.ways); + } + template <class ToDo> void ForEachRelationByWay(TKey id, ToDo && toDo) { @@ -124,54 +158,6 @@ public: m_wayToRelations.ForEachByKey(id, processor); } - void LoadIndex() - { - m_ways.LoadOffsets(); - m_relations.LoadOffsets(); - - m_nodeToRelations.ReadAll(); - m_wayToRelations.ReadAll(); - } -}; - -template <class TNodesHolder> -class IntermediateDataWriter - : public cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter> -{ - using TBase = cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter>; - - using typename TBase::TKey; - using TBase::m_nodes; - using TBase::m_nodeToRelations; - using TBase::m_ways; - using TBase::m_wayToRelations; - using TBase::m_relations; - - template <class TIndex, class TContainer> - static void AddToIndex(TIndex & index, TKey relationId, TContainer const & values) - { - for (auto const & v : values) - index.Add(v.first, relationId); - } - -public: - IntermediateDataWriter(TNodesHolder & nodes, string const & dir) : TBase(nodes, dir) {} - - void AddNode(TKey id, double lat, double lng) { m_nodes.AddPoint(id, lat, lng); } - - void AddWay(TKey id, WayElement const & e) { m_ways.Write(id, e); } - - void AddRelation(TKey id, RelationElement const & e) - { - string const & relationType = e.GetType(); - if (!(relationType == "multipolygon" || relationType == "route" || relationType == "boundary")) - return; - - m_relations.Write(id, e); - AddToIndex(m_nodeToRelations, id, e.nodes); - AddToIndex(m_wayToRelations, id, e.ways); - } - void SaveIndex() { m_ways.SaveOffsets(); @@ -180,6 +166,15 @@ public: m_nodeToRelations.WriteAll(); m_wayToRelations.WriteAll(); } + + void LoadIndex() + { + m_ways.LoadOffsets(); + m_relations.LoadOffsets(); + + m_nodeToRelations.ReadAll(); + m_wayToRelations.ReadAll(); + } }; class MainFeaturesEmitter @@ -392,21 +387,13 @@ void AddElementToCache(TCache & cache, TElement const & em) template <typename TCache> void BuildIntermediateDataFromXML(SourceReader & stream, TCache & cache) { - BaseOSMParser parser([&](XMLElement * e) - { - AddElementToCache(cache, *e); - }); - + BaseOSMParser parser([&](XMLElement * e) { AddElementToCache(cache, *e); }); ParseXMLSequence(stream, parser); } void BuildFeaturesFromXML(SourceReader & stream, function<void(XMLElement *)> processor) { - BaseOSMParser parser([&](XMLElement * e) - { - processor(e); - }); - + BaseOSMParser parser([&](XMLElement * e) { processor(e); }); ParseXMLSequence(stream, parser); } @@ -433,6 +420,17 @@ void BuildFeaturesFromO5M(SourceReader & stream, function<void(XMLElement *)> pr return stream.Read(reinterpret_cast<char *>(buffer), size); }); + auto translate = [](TType t) -> XMLElement::EntityType + { + switch (t) + { + case TType::Node: return XMLElement::EntityType::Node; + case TType::Way: return XMLElement::EntityType::Way; + case TType::Relation: return XMLElement::EntityType::Relation; + default: return XMLElement::EntityType::Unknown; + } + }; + for (auto const & em : dataset) { XMLElement p; @@ -458,22 +456,7 @@ void BuildFeaturesFromO5M(SourceReader & stream, function<void(XMLElement *)> pr { p.type = XMLElement::EntityType::Relation; for (auto const & member : em.Members()) - { - switch (member.type) - { - case TType::Node: - p.AddMember(member.ref, XMLElement::EntityType::Node, member.role); - break; - case TType::Way: - p.AddMember(member.ref, XMLElement::EntityType::Way, member.role); - break; - case TType::Relation: - p.AddMember(member.ref, XMLElement::EntityType::Relation, member.role); - break; - - default: break; - } - } + p.AddMember(member.ref, translate(member.type), member.role); break; } default: break; @@ -497,7 +480,7 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info) { TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - using TDataCache = IntermediateDataReader<TNodesHolder>; + using TDataCache = IntermediateData<TNodesHolder, /*WriteMode=*/false>; TDataCache cache(nodes, info.m_intermediateDir); cache.LoadIndex(); @@ -543,7 +526,7 @@ bool GenerateIntermediateDataImpl(feature::GenerateInfo & info) try { TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, "")); - using TDataCache = IntermediateDataWriter<TNodesHolder>; + using TDataCache = IntermediateData<TNodesHolder, /*WriteMode=*/true>; TDataCache cache(nodes, info.m_intermediateDir); SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); |