From 4ca9859adfd3ffa46d2e33ae662302e41c35792f Mon Sep 17 00:00:00 2001 From: Sergey Yershov Date: Tue, 25 Aug 2015 13:40:53 +0300 Subject: Move all classes for work with intermediate cache to imtermediate_data.hpp --- generator/intermediate_data.hpp | 216 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 2 deletions(-) (limited to 'generator/intermediate_data.hpp') diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp index 585cf2ab7c..ad8c9735bb 100644 --- a/generator/intermediate_data.hpp +++ b/generator/intermediate_data.hpp @@ -3,6 +3,9 @@ #include "generator/osm_decl.hpp" #include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" +#include "coding/mmap_reader.hpp" #include "base/logging.hpp" @@ -126,7 +129,6 @@ public: }; } // namespace detail - template class OSMElementCache { @@ -212,4 +214,214 @@ public: inline void SaveOffsets() { m_offsets.WriteAll(); } inline void LoadOffsets() { m_offsets.ReadAll(); } }; -} // namespace cache + +/// Used to store all world nodes inside temporary index file. +/// To find node by id, just calculate offset inside index file: +/// offset_in_file = sizeof(LatLon) * node_ID +class PointStorage +{ + size_t m_processedPoint = 0; + +public: + struct LatLon + { + int32_t lat; + int32_t lon; + }; + static_assert(sizeof(LatLon) == 8, "Invalid structure size"); + + struct LatLonPos + { + uint64_t pos; + int32_t lat; + int32_t lon; + }; + static_assert(sizeof(LatLonPos) == 16, "Invalid structure size"); + + inline size_t GetProcessedPoint() const { return m_processedPoint; } + inline void IncProcessedPoint() { ++m_processedPoint; } +}; + +template +class RawFilePointStorage : public PointStorage +{ +#ifdef OMIM_OS_WINDOWS + using TFileReader = FileReader; +#else + using TFileReader = MmapReader; +#endif + + typename conditional::type m_file; + + constexpr static double const kValueOrder = 1E+7; + +public: + RawFilePointStorage(string const & name) : m_file(name) {} + + template + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLon ll; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + + m_file.Seek(id * sizeof(ll)); + m_file.Write(&ll, sizeof(ll)); + + IncProcessedPoint(); + } + + template + typename enable_if::type GetPoint(uint64_t id, double & lat, + double & lng) const + { + LatLon ll; + m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); + + // assume that valid coordinate is not (0, 0) + if (ll.lat != 0.0 || ll.lon != 0.0) + { + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; + return true; + } + LOG(LERROR, ("Node with id = ", id, " not found!")); + return false; + } +}; + +template +class RawMemPointStorage : public PointStorage +{ + typename conditional::type m_file; + + constexpr static double const kValueOrder = 1E+7; + + vector m_data; + +public: + RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF) + { + InitStorage(); + } + + ~RawMemPointStorage() { DoneStorage(); } + + template + typename enable_if::type InitStorage() {} + + template + typename enable_if::type InitStorage() + { + m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); + } + + template + typename enable_if::type DoneStorage() + { + m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon)); + } + + template + typename enable_if::type DoneStorage() {} + + template + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLon & ll = m_data[id]; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + + IncProcessedPoint(); + } + + template + typename enable_if::type GetPoint(uint64_t id, double & lat, + double & lng) const + { + LatLon const & ll = m_data[id]; + // assume that valid coordinate is not (0, 0) + if (ll.lat != 0.0 || ll.lon != 0.0) + { + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; + return true; + } + LOG(LERROR, ("Node with id = ", id, " not found!")); + return false; + } +}; + +template +class MapFilePointStorage : public PointStorage +{ + typename conditional::type m_file; + unordered_map> m_map; + + constexpr static double const kValueOrder = 1E+7; + +public: + MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage(); } + + template + typename enable_if::type InitStorage() {} + + template + typename enable_if::type InitStorage() + { + LOG(LINFO, ("Nodes reading is started")); + + uint64_t const count = m_file.Size(); + + uint64_t pos = 0; + while (pos < count) + { + LatLonPos ll; + m_file.Read(pos, &ll, sizeof(ll)); + + m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); + + pos += sizeof(ll); + } + + LOG(LINFO, ("Nodes reading is finished")); + } + + void AddPoint(uint64_t id, double lat, double lng) + { + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; + + LatLonPos ll; + ll.pos = id; + ll.lat = static_cast(lat64); + ll.lon = static_cast(lng64); + CHECK_EQUAL(static_cast(ll.lat), lat64, ("Latitude is out of 32bit boundary!")); + CHECK_EQUAL(static_cast(ll.lon), lng64, ("Longtitude is out of 32bit boundary!")); + m_file.Write(&ll, sizeof(ll)); + + IncProcessedPoint(); + } + + bool GetPoint(uint64_t id, double & lat, double & lng) const + { + auto i = m_map.find(id); + if (i == m_map.end()) + return false; + lat = static_cast(i->second.first) / kValueOrder; + lng = static_cast(i->second.second) / kValueOrder; + return true; + } +}; + +} // namespace cache -- cgit v1.2.3