Move all classes for work with intermediate cache to imtermediate_data.hpp

1 files changed, 214 insertions, 2 deletions

diff --git a/generator/intermediate_data.hpp b/generator/intermediate_data.hpp
index 585cf2ab7c..ad8c9735bb 100644
--- a/generator/intermediate_data.hpp
+++ b/generator/intermediate_data.hpp
@@ -3,6 +3,9 @@
 #include "generator/osm_decl.hpp"
 
 #include "coding/file_name_utils.hpp"
+#include "coding/file_reader.hpp"
+#include "coding/file_writer.hpp"
+#include "coding/mmap_reader.hpp"
 
 #include "base/logging.hpp"
 
@@ -126,7 +129,6 @@ public:
 };
 } // namespace detail
 
-
 template <EMode TMode>
 class OSMElementCache
 {
@@ -212,4 +214,214 @@ public:
   inline void SaveOffsets() { m_offsets.WriteAll(); }
   inline void LoadOffsets() { m_offsets.ReadAll(); }
 };
-} // namespace cache
+
+/// Used to store all world nodes inside temporary index file.
+/// To find node by id, just calculate offset inside index file:
+/// offset_in_file = sizeof(LatLon) * node_ID
+class PointStorage
+{
+  size_t m_processedPoint = 0;
+
+public:
+  struct LatLon
+  {
+    int32_t lat;
+    int32_t lon;
+  };
+  static_assert(sizeof(LatLon) == 8, "Invalid structure size");
+
+  struct LatLonPos
+  {
+    uint64_t pos;
+    int32_t lat;
+    int32_t lon;
+  };
+  static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
+
+  inline size_t GetProcessedPoint() const { return m_processedPoint; }
+  inline void IncProcessedPoint() { ++m_processedPoint; }
+};
+
+template <EMode TMode>
+class RawFilePointStorage : public PointStorage
+{
+#ifdef OMIM_OS_WINDOWS
+  using TFileReader = FileReader;
+#else
+  using TFileReader = MmapReader;
+#endif
+
+  typename conditional<TMode == EMode::Write, FileWriter, TFileReader>::type m_file;
+
+  constexpr static double const kValueOrder = 1E+7;
+
+public:
+  RawFilePointStorage(string const & name) : m_file(name) {}
+
+  template <EMode T = TMode>
+  typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
+  {
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
+
+    LatLon ll;
+    ll.lat = static_cast<int32_t>(lat64);
+    ll.lon = static_cast<int32_t>(lng64);
+    CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
+    CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
+
+    m_file.Seek(id * sizeof(ll));
+    m_file.Write(&ll, sizeof(ll));
+
+    IncProcessedPoint();
+  }
+
+  template <EMode T = TMode>
+  typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
+                                                            double & lng) const
+  {
+    LatLon ll;
+    m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
+
+    // assume that valid coordinate is not (0, 0)
+    if (ll.lat != 0.0 || ll.lon != 0.0)
+    {
+      lat = static_cast<double>(ll.lat) / kValueOrder;
+      lng = static_cast<double>(ll.lon) / kValueOrder;
+      return true;
+    }
+    LOG(LERROR, ("Node with id = ", id, " not found!"));
+    return false;
+  }
+};
+
+template <EMode TMode>
+class RawMemPointStorage : public PointStorage
+{
+  typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
+
+  constexpr static double const kValueOrder = 1E+7;
+
+  vector<LatLon> m_data;
+
+public:
+  RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF)
+  {
+    InitStorage<TMode>();
+  }
+
+  ~RawMemPointStorage() { DoneStorage<TMode>(); }
+
+  template <EMode T>
+  typename enable_if<T == EMode::Write, void>::type InitStorage() {}
+
+  template <EMode T>
+  typename enable_if<T == EMode::Read, void>::type InitStorage()
+  {
+    m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
+  }
+
+  template <EMode T>
+  typename enable_if<T == EMode::Write, void>::type DoneStorage()
+  {
+    m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon));
+  }
+
+  template <EMode T>
+  typename enable_if<T == EMode::Read, void>::type DoneStorage() {}
+
+  template <EMode T = TMode>
+  typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
+  {
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
+
+    LatLon & ll = m_data[id];
+    ll.lat = static_cast<int32_t>(lat64);
+    ll.lon = static_cast<int32_t>(lng64);
+    CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
+    CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
+
+    IncProcessedPoint();
+  }
+
+  template <EMode T = TMode>
+  typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
+                                                            double & lng) const
+  {
+    LatLon const & ll = m_data[id];
+    // assume that valid coordinate is not (0, 0)
+    if (ll.lat != 0.0 || ll.lon != 0.0)
+    {
+      lat = static_cast<double>(ll.lat) / kValueOrder;
+      lng = static_cast<double>(ll.lon) / kValueOrder;
+      return true;
+    }
+    LOG(LERROR, ("Node with id = ", id, " not found!"));
+    return false;
+  }
+};
+
+template <EMode TMode>
+class MapFilePointStorage : public PointStorage
+{
+  typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
+  unordered_map<uint64_t, pair<int32_t, int32_t>> m_map;
+
+  constexpr static double const kValueOrder = 1E+7;
+
+public:
+  MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage<TMode>(); }
+
+  template <EMode T>
+  typename enable_if<T == EMode::Write, void>::type InitStorage() {}
+
+  template <EMode T>
+  typename enable_if<T == EMode::Read, void>::type InitStorage()
+  {
+    LOG(LINFO, ("Nodes reading is started"));
+
+    uint64_t const count = m_file.Size();
+
+    uint64_t pos = 0;
+    while (pos < count)
+    {
+      LatLonPos ll;
+      m_file.Read(pos, &ll, sizeof(ll));
+
+      m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
+
+      pos += sizeof(ll);
+    }
+
+    LOG(LINFO, ("Nodes reading is finished"));
+  }
+
+  void AddPoint(uint64_t id, double lat, double lng)
+  {
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
+
+    LatLonPos ll;
+    ll.pos = id;
+    ll.lat = static_cast<int32_t>(lat64);
+    ll.lon = static_cast<int32_t>(lng64);
+    CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
+    CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
+    m_file.Write(&ll, sizeof(ll));
+
+    IncProcessedPoint();
+  }
+
+  bool GetPoint(uint64_t id, double & lat, double & lng) const
+  {
+    auto i = m_map.find(id);
+    if (i == m_map.end())
+      return false;
+    lat = static_cast<double>(i->second.first) / kValueOrder;
+    lng = static_cast<double>(i->second.second) / kValueOrder;
+    return true;
+  }
+};
+
+}  // namespace cache