Support old format 1.01 (April 2011).

3 files changed, 535 insertions, 0 deletions

diff --git a/indexer/old/feature_loader_101.cpp b/indexer/old/feature_loader_101.cpp
new file mode 100644
index 0000000000..c886c40684
--- /dev/null
+++ b/indexer/old/feature_loader_101.cpp
@@ -0,0 +1,332 @@
+#include "../../base/SRC_FIRST.hpp"
+
+#include "feature_loader_101.hpp"
+
+#include "../feature.hpp"
+#include "../scales.hpp"
+#include "../geometry_serialization.hpp"
+#include "../coding_params.hpp"
+
+#include "../../coding/byte_stream.hpp"
+
+#include "../../base/start_mem_debug.hpp"
+
+
+namespace old_101 { namespace feature {
+
+
+uint8_t LoaderImpl::GetHeader()
+{
+  uint8_t const h = Header();
+
+  uint8_t header = static_cast<uint8_t>((h & 7) - 1);
+
+  if (h & HEADER_HAS_NAME)
+    header |= ::feature::HEADER_HAS_NAME;
+
+  if (h & HEADER_HAS_LAYER)
+    header |= ::feature::HEADER_HAS_LAYER;
+
+  if (h & HEADER_IS_LINE)
+    header |= ::feature::HEADER_GEOM_LINE;
+
+  if (h & HEADER_IS_AREA)
+    header |= ::feature::HEADER_GEOM_AREA;
+
+  return header;
+}
+
+void LoaderImpl::ParseTypes()
+{
+  ArrayByteSource source(DataPtr() + m_TypesOffset);
+
+  size_t const count = m_pF->GetTypesCount();
+  for (size_t i = 0; i < count; ++i)
+    m_pF->m_Types[i] = ReadVarUint<uint32_t>(source);
+
+  m_CommonOffset = CalcOffset(source);
+}
+
+void LoaderImpl::ParseCommon()
+{
+  ArrayByteSource source(DataPtr() + m_CommonOffset);
+
+  uint8_t const h = Header();
+
+  if (h & HEADER_HAS_LAYER)
+    m_pF->m_Params.layer = ReadVarInt<int32_t>(source);
+
+  if (h & HEADER_HAS_NAME)
+  {
+    string name;
+    name.resize(ReadVarUint<uint32_t>(source) + 1);
+    source.Read(&name[0], name.size());
+    m_pF->m_Params.name.AddString(0, name);
+  }
+
+  if (h & HEADER_HAS_POINT)
+  {
+    CoordPointT const center = Int64ToPoint(
+          ReadVarInt<int64_t>(source) + m_Info.GetCodingParams().GetBasePointInt64(), POINT_COORD_BITS);
+
+    m_pF->m_Center = m2::PointD(center.first, center.second);
+    m_pF->m_LimitRect.Add(m_pF->m_Center);
+  }
+
+  m_Header2Offset = CalcOffset(source);
+}
+
+namespace
+{
+    uint32_t const kInvalidOffset = uint32_t(-1);
+}
+
+int LoaderImpl::GetScaleIndex(int scale) const
+{
+  int const count = m_Info.GetScalesCount();
+  if (scale == -1) return count-1;
+
+  for (int i = 0; i < count; ++i)
+    if (scale <= m_Info.GetScale(i))
+      return i;
+  return -1;
+}
+
+int LoaderImpl::GetScaleIndex(int scale, offsets_t const & offsets) const
+{
+  if (scale == -1)
+  {
+    // Choose the best geometry for the last visible scale.
+    int i = offsets.size()-1;
+    while (i >= 0 && offsets[i] == kInvalidOffset) --i;
+    if (i >= 0)
+      return i;
+    else
+      CHECK ( false, ("Feature should have any geometry ...") );
+  }
+  else
+  {
+    for (size_t i = 0; i < m_Info.GetScalesCount(); ++i)
+      if (scale <= m_Info.GetScale(i))
+      {
+        if (offsets[i] != kInvalidOffset)
+          return i;
+        else
+          break;
+      }
+  }
+
+  return -1;
+}
+
+namespace
+{
+  class BitSource
+  {
+    char const * m_ptr;
+    uint8_t m_pos;
+
+  public:
+    BitSource(char const * p) : m_ptr(p), m_pos(0) {}
+
+    uint8_t Read(uint8_t count)
+    {
+      ASSERT_LESS ( count, 9, () );
+
+      uint8_t v = *m_ptr;
+      v >>= m_pos;
+      v &= ((1 << count) - 1);
+
+      m_pos += count;
+      if (m_pos >= 8)
+      {
+        ASSERT_EQUAL ( m_pos, 8, () );
+        ++m_ptr;
+        m_pos = 0;
+      }
+
+      return v;
+    }
+
+    char const * RoundPtr()
+    {
+      if (m_pos > 0)
+      {
+        ++m_ptr;
+        m_pos = 0;
+      }
+      return m_ptr;
+    }
+  };
+
+  template <class TSource> uint8_t ReadByte(TSource & src)
+  {
+    return ReadPrimitiveFromSource<uint8_t>(src);
+  }
+}
+
+void LoaderImpl::ParseHeader2()
+{
+  uint8_t ptsCount, ptsMask, trgCount, trgMask;
+
+  uint8_t const commonH = Header();
+  BitSource bitSource(DataPtr() + m_Header2Offset);
+
+  if (commonH & HEADER_IS_LINE)
+  {
+    ptsCount = bitSource.Read(4);
+    if (ptsCount == 0)
+      ptsMask = bitSource.Read(4);
+    else
+    {
+      ASSERT_GREATER ( ptsCount, 1, () );
+    }
+  }
+
+  if (commonH & HEADER_IS_AREA)
+  {
+    trgCount = bitSource.Read(4);
+    if (trgCount == 0)
+      trgMask = bitSource.Read(4);
+  }
+
+  ArrayByteSource src(bitSource.RoundPtr());
+
+  if (commonH & HEADER_IS_LINE)
+  {
+    if (ptsCount > 0)
+    {
+      int const count = (ptsCount - 2 + 3) / 4;
+      ASSERT_LESS ( count, 4, () );
+
+      for (int i = 0; i < count; ++i)
+      {
+        uint32_t mask = ReadByte(src);
+        m_ptsSimpMask += (mask << (i << 3));
+      }
+
+      char const * start = static_cast<char const *>(src.Ptr());
+
+      src = ArrayByteSource(serial::LoadInnerPath(
+                              src.Ptr(), ptsCount, m_Info.GetCodingParams(), m_pF->m_Points));
+
+      m_pF->m_InnerStats.m_Points = static_cast<char const *>(src.Ptr()) - start;
+    }
+    else
+      ReadOffsets(src, ptsMask, m_ptsOffsets);
+  }
+
+  if (commonH & HEADER_IS_AREA)
+  {
+    if (trgCount > 0)
+    {
+      trgCount += 2;
+
+      char const * start = static_cast<char const *>(src.Ptr());
+
+      FeatureType::points_t points;
+      src = ArrayByteSource(serial::LoadInnerTriangles(
+                              src.Ptr(), trgCount, m_Info.GetCodingParams(), points));
+
+      m_pF->m_InnerStats.m_Strips = static_cast<char const *>(src.Ptr()) - start;
+
+      for (uint8_t i = 2; i < trgCount; ++i)
+      {
+        m_pF->m_Triangles.push_back(points[i-2]);
+        m_pF->m_Triangles.push_back(points[i-1]);
+        m_pF->m_Triangles.push_back(points[i]);
+      }
+    }
+    else
+      ReadOffsets(src, trgMask, m_trgOffsets);
+  }
+
+  m_pF->m_InnerStats.m_Size = static_cast<char const *>(src.Ptr()) - DataPtr();
+}
+
+uint32_t LoaderImpl::ParseGeometry(int scale)
+{
+  uint32_t sz = 0;
+  if (Header() & HEADER_IS_LINE)
+  {
+    if (m_pF->m_Points.empty())
+    {
+      // outer geometry
+      int const ind = GetScaleIndex(scale, m_ptsOffsets);
+      if (ind != -1)
+      {
+        ReaderSource<FilesContainerR::ReaderT> src(m_Info.GetGeometryReader(ind));
+        src.Skip(m_ptsOffsets[ind]);
+        serial::LoadOuterPath(src, m_Info.GetCodingParams(), m_pF->m_Points);
+
+        sz = static_cast<uint32_t>(src.Pos() - m_ptsOffsets[ind]);
+      }
+    }
+    else
+    {
+      // filter inner geometry
+
+      size_t const count = m_pF->m_Points.size();
+      FeatureType::points_t points;
+      points.reserve(count);
+
+      int const scaleIndex = GetScaleIndex(scale);
+      ASSERT_LESS ( scaleIndex, m_Info.GetScalesCount(), () );
+
+      points.push_back(m_pF->m_Points.front());
+      for (size_t i = 1; i < count-1; ++i)
+      {
+        // check for point visibility in needed scaleIndex
+        if (((m_ptsSimpMask >> (2*(i-1))) & 0x3) <= scaleIndex)
+          points.push_back(m_pF->m_Points[i]);
+      }
+      points.push_back(m_pF->m_Points.back());
+
+      m_pF->m_Points.swap(points);
+    }
+
+    ::feature::CalcRect(m_pF->m_Points, m_pF->m_LimitRect);
+  }
+
+  return sz;
+}
+
+uint32_t LoaderImpl::ParseTriangles(int scale)
+{
+  uint32_t sz = 0;
+  if (Header() & HEADER_IS_AREA)
+  {
+    if (m_pF->m_Triangles.empty())
+    {
+      uint32_t const ind = GetScaleIndex(scale, m_trgOffsets);
+      if (ind != -1)
+      {
+        ReaderSource<FilesContainerR::ReaderT> src(m_Info.GetTrianglesReader(ind));
+        src.Skip(m_trgOffsets[ind]);
+        serial::LoadOuterTriangles(src, m_Info.GetCodingParams(), m_pF->m_Triangles);
+
+        sz = static_cast<uint32_t>(src.Pos() - m_trgOffsets[ind]);
+      }
+    }
+
+    ::feature::CalcRect(m_pF->m_Triangles, m_pF->m_LimitRect);
+  }
+
+  return sz;
+}
+
+void LoaderImpl::ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const
+{
+  ASSERT_GREATER ( mask, 0, () );
+
+  int index = 0;
+  while (mask > 0)
+  {
+    ASSERT_LESS ( index, m_Info.GetScalesCount(), () );
+    offsets[index++] = (mask & 0x01) ? ReadVarUint<uint32_t>(src) : kInvalidOffset;
+    mask = mask >> 1;
+  }
+}
+
+}
+}
diff --git a/indexer/old/feature_loader_101.hpp b/indexer/old/feature_loader_101.hpp
new file mode 100644
index 0000000000..d7fe928c10
--- /dev/null
+++ b/indexer/old/feature_loader_101.hpp
@@ -0,0 +1,44 @@
+#pragma once
+#include "../feature_loader_base.hpp"
+
+
+namespace old_101 { namespace feature
+{
+  class LoaderImpl : public ::feature::LoaderBase
+  {
+    typedef ::feature::LoaderBase BaseT;
+
+    void ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const;
+
+    /// Get the index for geometry serialization.
+    /// @param[in]  scale:
+    /// -1 : index for the best geometry
+    /// default : needed geometry
+    //@{
+    int GetScaleIndex(int scale) const;
+    int GetScaleIndex(int scale, offsets_t const & offsets) const;
+    //@}
+
+    enum
+    {
+      HEADER_HAS_LAYER = 1U << 7,
+      HEADER_HAS_NAME = 1U << 6,
+      HEADER_IS_AREA = 1U << 5,
+      HEADER_IS_LINE = 1U << 4,
+      HEADER_HAS_POINT = 1U << 3
+    };
+
+  public:
+    LoaderImpl(::feature::SharedLoadInfo const & info) : BaseT(info) {}
+
+    virtual uint8_t GetHeader();
+
+    virtual void ParseTypes();
+    virtual void ParseCommon();
+    virtual void ParseHeader2();
+    virtual uint32_t ParseGeometry(int scale);
+    virtual uint32_t ParseTriangles(int scale);
+  };
+
+}
+}
diff --git a/indexer/old/interval_index_101.hpp b/indexer/old/interval_index_101.hpp
new file mode 100644
index 0000000000..2a72c841fe
--- /dev/null
+++ b/indexer/old/interval_index_101.hpp
@@ -0,0 +1,159 @@
+#pragma once
+#include "../interval_index_iface.hpp"
+
+#include "../../coding/endianness.hpp"
+
+#include "../../base/assert.hpp"
+#include "../../base/base.hpp"
+#include "../../base/macros.hpp"
+
+#include "../../std/memcpy.hpp"
+
+
+namespace old_101 {
+
+class IntervalIndexBase : public IntervalIndexIFace
+{
+public:
+#pragma pack(push, 1)
+  struct Header
+  {
+    uint8_t m_CellIdLeafBytes;
+  };
+#pragma pack(pop)
+
+  class Index
+  {
+  public:
+    uint32_t GetBaseOffset() const { return UINT32_FROM_UINT16(m_BaseOffsetHi, m_BaseOffsetLo); }
+    void SetBaseOffset(uint32_t baseOffset)
+    {
+      m_BaseOffsetLo = UINT32_LO(baseOffset);
+      m_BaseOffsetHi = UINT32_HI(baseOffset);
+    }
+
+  private:
+    uint16_t m_BaseOffsetLo;
+    uint16_t m_BaseOffsetHi;
+  public:
+    uint16_t m_Count[256];
+  };
+  STATIC_ASSERT(sizeof(Index) == 2 * 258);
+};
+
+// TODO: IntervalIndex shouldn't do SwapIfBigEndian for ValueT.
+template <typename ValueT, class ReaderT>
+class IntervalIndex : public IntervalIndexBase
+{
+  typedef IntervalIndexBase base_t;
+
+public:
+
+  class Query : public base_t::QueryIFace
+  {
+  public:
+    void Clear() {}
+
+  private:
+    friend class IntervalIndex;
+    vector<char> m_IntervalIndexCache;
+  };
+
+  IntervalIndex(ReaderT const & reader, int cellIdBytes = 5)
+    : m_Reader(reader), m_CellIdBytes(cellIdBytes)
+  {
+    m_Reader.Read(0, &m_Header, sizeof(m_Header));
+    ReadIndex(sizeof(m_Header), m_Level0Index);
+  }
+
+  template <typename F>
+  void ForEach(F const & f, uint64_t beg, uint64_t end, Query & query) const
+  {
+    ASSERT_LESS(beg, 1ULL << 8 * m_CellIdBytes, (beg, end));
+    ASSERT_LESS_OR_EQUAL(end, 1ULL << 8 * m_CellIdBytes, (beg, end));
+    // end is inclusive in ForEachImpl().
+    --end;
+    ForEachImpl(f, beg, end, m_Level0Index, m_CellIdBytes - 1, query);
+  }
+
+  template <typename F>
+  void ForEach(F const & f, uint64_t beg, uint64_t end) const
+  {
+    Query query;
+    ForEach(f, beg, end, query);
+  }
+
+  virtual void DoForEach(FunctionT const & f, uint64_t beg, uint64_t end, QueryIFace & /*query*/)
+  {
+    ForEach(f, beg, end);
+  }
+
+private:
+  template <typename F>
+  void ForEachImpl(F const & f, uint64_t beg, uint64_t end, Index const & index, int level,
+                   Query & query) const
+  {
+    uint32_t const beg0 = static_cast<uint32_t>(beg >> (8 * level));
+    uint32_t const end0 = static_cast<uint32_t>(end >> (8 * level));
+    uint32_t cumCount = 0;
+    for (uint32_t i = 0; i < beg0; ++i)
+      cumCount += index.m_Count[i];
+    for (uint32_t i = beg0; i <= end0; ++i)
+    {
+      ASSERT_LESS(i, 256, ());
+      if (index.m_Count[i] != 0)
+      {
+        uint64_t const levelBytesFF = (1ULL << 8 * level) - 1;
+        uint64_t const b1 = (i == beg0) ? (beg & levelBytesFF) : 0;
+        uint64_t const e1 = (i == end0) ? (end & levelBytesFF) : levelBytesFF;
+        if (level > m_Header.m_CellIdLeafBytes)
+        {
+          Index index1;
+          ReadIndex(index.GetBaseOffset() + (cumCount * sizeof(Index)), index1);
+          ForEachImpl(f, b1, e1, index1, level - 1, query);
+        }
+        else
+        {
+          // TODO: Use binary search here if count is very large.
+          uint32_t const step = sizeof(ValueT) + m_Header.m_CellIdLeafBytes;
+          uint32_t const count = index.m_Count[i];
+          uint32_t pos = index.GetBaseOffset() + (cumCount * step);
+          size_t const readSize = step * count;
+          query.m_IntervalIndexCache.assign(readSize, 0);
+          char * pData = &query.m_IntervalIndexCache[0];
+          m_Reader.Read(pos, pData, readSize);
+          for (uint32_t j = 0; j < count; ++j, pData += step)
+          // for (uint32_t j = 0; j < count; ++j, pos += step)
+          {
+            ValueT value;
+            uint32_t cellIdOnDisk = 0;
+            memcpy(&value, pData, sizeof(ValueT));
+            memcpy(&cellIdOnDisk, pData + sizeof(ValueT), m_Header.m_CellIdLeafBytes);
+            // m_Reader.Read(pos, &value, step);
+            uint32_t const cellId = SwapIfBigEndian(cellIdOnDisk);
+            if (b1 <= cellId && cellId <= e1)
+              f(SwapIfBigEndian(value));
+          }
+        }
+        cumCount += index.m_Count[i];
+      }
+    }
+  }
+
+  void ReadIndex(uint64_t pos, Index & index) const
+  {
+    m_Reader.Read(pos, &index, sizeof(Index));
+    if (IsBigEndian())
+    {
+      for (uint32_t i = 0; i < 256; ++i)
+        index.m_Count[i] = SwapIfBigEndian(index.m_Count[i]);
+    }
+  }
+
+  ReaderT m_Reader;
+  Header m_Header;
+  Index m_Level0Index;
+  int m_CellIdBytes;
+};
+
+}