diff options
author | vng <viktor.govako@gmail.com> | 2010-12-19 23:40:52 +0300 |
---|---|---|
committer | Alex Zolotarev <alex@maps.me> | 2015-09-23 01:08:42 +0300 |
commit | 327aa0e7cb1d20ee8bfd333899a8ec7b8cea9c41 (patch) | |
tree | 610e24a6ae45f456675550db56ffdf497bd2c88e /indexer | |
parent | c65e815b69d1eb1ac86636e70d8b66ebcf40052e (diff) |
[Refactoring]
Divide FeatureBuilder in:
- FeatureBuilderGeom (store geometry as-as)
- FeatureBuilderGeomRef (store geometry separately)
Add FeatureGeomRef as feature with separate geometry.
Diffstat (limited to 'indexer')
-rw-r--r-- | indexer/covering.cpp | 6 | ||||
-rw-r--r-- | indexer/covering.hpp | 2 | ||||
-rw-r--r-- | indexer/feature.cpp | 325 | ||||
-rw-r--r-- | indexer/feature.hpp | 178 | ||||
-rw-r--r-- | indexer/feature_processor.hpp | 9 | ||||
-rw-r--r-- | indexer/features_vector.hpp | 14 | ||||
-rw-r--r-- | indexer/indexer_tests/feature_bucketer_test.cpp | 17 | ||||
-rw-r--r-- | indexer/indexer_tests/feature_test.cpp | 23 | ||||
-rw-r--r-- | indexer/indexer_tool/feature_bucketer.hpp | 17 | ||||
-rw-r--r-- | indexer/indexer_tool/feature_generator.cpp | 66 | ||||
-rw-r--r-- | indexer/indexer_tool/feature_generator.hpp | 16 | ||||
-rw-r--r-- | indexer/indexer_tool/feature_sorter.cpp | 29 | ||||
-rw-r--r-- | indexer/indexer_tool/osm_element.hpp | 12 | ||||
-rw-r--r-- | indexer/indexer_tool/tesselator.cpp | 8 | ||||
-rw-r--r-- | indexer/scale_index_builder.hpp | 2 |
15 files changed, 555 insertions, 169 deletions
diff --git a/indexer/covering.cpp b/indexer/covering.cpp index d5c8ab1246..afe5da69d1 100644 --- a/indexer/covering.cpp +++ b/indexer/covering.cpp @@ -68,10 +68,10 @@ namespace }; } -vector<int64_t> covering::CoverFeature(FeatureGeom const & feature) +vector<int64_t> covering::CoverFeature(FeatureGeom const & feature, int level) { vector<CoordPointT> geometry; - feature.ForEachPoint(MakeBackInsertFunctor(geometry)); + feature.ForEachPoint(MakeBackInsertFunctor(geometry), level); ASSERT(!geometry.empty(), ()); if (geometry.empty()) @@ -87,7 +87,7 @@ vector<int64_t> covering::CoverFeature(FeatureGeom const & feature) typedef covering::Covering<RectId> CoveringType; typedef TriangleCoverer<MercatorBounds, CoveringType> CovererType; CovererType coverer = CovererType(CoveringType(ids)); - feature.ForEachTriangleRef(coverer); + feature.ForEachTriangleRef(coverer, level); vector<int64_t> res; coverer.GetCovering().OutputToVector(res); return res; diff --git a/indexer/covering.hpp b/indexer/covering.hpp index 5decac86b6..9cc0b5a781 100644 --- a/indexer/covering.hpp +++ b/indexer/covering.hpp @@ -12,7 +12,7 @@ class FeatureGeom; namespace covering { // Cover feature with RectIds and return their integer representations. - vector<int64_t> CoverFeature(FeatureGeom const & feature); + vector<int64_t> CoverFeature(FeatureGeom const & feature, int level); // Cover viewport with RectIds and append their RectIds as well. vector<pair<int64_t, int64_t> > CoverViewportAndAppendLowerLevels(m2::RectD const & rect); // Given a vector of intervals [a, b), sort them and merge overlapping intervals. diff --git a/indexer/feature.cpp b/indexer/feature.cpp index f7263042b9..5bd0a40aa2 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -1,7 +1,11 @@ #include "feature.hpp" #include "cell_id.hpp" +#include "feature_visibility.hpp" +#include "scales.hpp" #include "../geometry/rect2d.hpp" +#include "../geometry/distance.hpp" +#include "../geometry/simplification.hpp" #include "../coding/byte_stream.hpp" #include "../coding/reader.hpp" @@ -31,25 +35,25 @@ namespace pts } /////////////////////////////////////////////////////////////////////////////////////////////////// -// FeatureBuilder implementation +// FeatureBuilderGeom implementation /////////////////////////////////////////////////////////////////////////////////////////////////// -FeatureBuilder::FeatureBuilder() : m_Layer(0) +FeatureBuilderGeom::FeatureBuilderGeom() : m_Layer(0) { } -bool FeatureBuilder::IsGeometryClosed() const +bool FeatureBuilderGeom::IsGeometryClosed() const { return !m_Geometry.empty() && m_Geometry.front() == m_Geometry.back(); } -void FeatureBuilder::AddPoint(m2::PointD const & p) +void FeatureBuilderGeom::AddPoint(m2::PointD const & p) { m_Geometry.push_back(p); m_LimitRect.Add(p); } -void FeatureBuilder::AddTriangle(m2::PointD const & a, m2::PointD const & b, m2::PointD const & c) +void FeatureBuilderGeom::AddTriangle(m2::PointD const & a, m2::PointD const & b, m2::PointD const & c) { pts::Fpt2id fn; m_Triangles.push_back(fn(a)); @@ -57,13 +61,13 @@ void FeatureBuilder::AddTriangle(m2::PointD const & a, m2::PointD const & b, m2: m_Triangles.push_back(fn(c)); } -void FeatureBuilder::AddName(string const & name) +void FeatureBuilderGeom::AddName(string const & name) { CHECK_EQUAL(m_Name, "", (name)); m_Name = name; } -void FeatureBuilder::AddLayer(int32_t layer) +void FeatureBuilderGeom::AddLayer(int32_t layer) { CHECK_EQUAL(m_Layer, 0, (layer)); @@ -73,7 +77,7 @@ void FeatureBuilder::AddLayer(int32_t layer) m_Layer = layer; } -FeatureBase FeatureBuilder::GetFeatureBase() const +FeatureBase FeatureBuilderGeom::GetFeatureBase() const { FeatureBase f; f.SetHeader(GetHeader()); @@ -105,7 +109,7 @@ namespace } } -bool FeatureBuilder::operator == (FeatureBuilder const & fb) const +bool FeatureBuilderGeom::operator == (FeatureBuilderGeom const & fb) const { if (m_Geometry.size() != fb.m_Geometry.size()) return false; @@ -123,7 +127,7 @@ bool FeatureBuilder::operator == (FeatureBuilder const & fb) const is_equal(m_LimitRect, fb.m_LimitRect); } -uint8_t FeatureBuilder::GetHeader() const +uint8_t FeatureBuilderGeom::GetHeader() const { uint8_t header = static_cast<uint8_t>(m_Types.size()); if (m_Layer != 0) @@ -140,41 +144,70 @@ uint8_t FeatureBuilder::GetHeader() const return header; } -void FeatureBuilder::Serialize(vector<char> & data) const +void FeatureBuilderGeom::SerializeBase(buffer_t & data) const { CHECK(!m_Geometry.empty(), ()); CHECK(m_Geometry.size() > 1 || m_Triangles.empty(), ()); CHECK_LESS(m_Types.size(), 16, ()); - data.clear(); - PushBackByteSink<vector<char> > sink(data); + PushBackByteSink<buffer_t> sink(data); - // Serializing header. + // Serialize header. WriteToSink(sink, GetHeader()); - // Serializing types. + // Serialize types. { for (size_t i = 0; i < m_Types.size(); ++i) WriteVarUint(sink, m_Types[i]); } - // Serializing layer. + // Serialize layer. if (m_Layer != 0) WriteVarInt(sink, m_Layer); - // Serializing name. + // Serialize name. if (!m_Name.empty()) { WriteVarUint(sink, m_Name.size() - 1); sink.Write(&m_Name[0], m_Name.size()); } +} + +void FeatureBuilderGeom::Serialize(buffers_holder_t & data) const +{ + data.clear(); + + SerializeBase(data); + SerializePoints(m_Geometry, data); + SerializeTriangles(data); + + ASSERT ( CheckCorrect(data), () ); +} + +void FeatureBuilderGeom::SerializeTriangles(buffer_t & data) const +{ + if (!m_Triangles.empty()) + { + PushBackByteSink<buffer_t> sink(data); + + ASSERT_EQUAL(m_Triangles.size() % 3, 0, (m_Triangles.size())); + WriteVarUint(sink, m_Triangles.size() / 3 - 1); + for (size_t i = 0; i < m_Triangles.size(); ++i) + WriteVarInt(sink, i == 0 ? m_Triangles[i] : (m_Triangles[i] - m_Triangles[i-1])); + } +} + +void FeatureBuilderGeom::SerializePoints(points_t const & points, buffer_t & data) +{ + uint32_t const ptsCount = points.size(); + ASSERT_GREATER_OR_EQUAL(ptsCount, 1, ()); - size_t const ptsCount = m_Geometry.size(); vector<int64_t> geom; geom.reserve(ptsCount); - transform(m_Geometry.begin(), m_Geometry.end(), back_inserter(geom), pts::Fpt2id()); + transform(points.begin(), points.end(), back_inserter(geom), pts::Fpt2id()); + + PushBackByteSink<buffer_t> sink(data); - // Serializing geometry. if (ptsCount == 1) { WriteVarInt(sink, geom[0]); @@ -185,25 +218,15 @@ void FeatureBuilder::Serialize(vector<char> & data) const for (size_t i = 0; i < ptsCount; ++i) WriteVarInt(sink, i == 0 ? geom[0] : geom[i] - geom[i-1]); } - - // Serializing triangles. - if (!m_Triangles.empty()) - { - ASSERT_EQUAL(m_Triangles.size() % 3, 0, (m_Triangles.size())); - WriteVarUint(sink, m_Triangles.size() / 3 - 1); - for (size_t i = 0; i < m_Triangles.size(); ++i) - WriteVarInt(sink, i == 0 ? m_Triangles[i] : (m_Triangles[i] - m_Triangles[i-1])); - } - - ASSERT ( CheckCorrect(data), () ); } -bool FeatureBuilder::CheckCorrect(vector<char> const & data) const +bool FeatureBuilderGeom::CheckCorrect(vector<char> const & data) const { - vector<char> data1 = data; - FeatureGeom f(data1); + FeatureGeom::read_source_t src; + src.m_data = data; + FeatureGeom f(src); - FeatureBuilder fb; + FeatureBuilderGeom fb; f.InitFeatureBuilder(fb); string const s = f.DebugString(); @@ -217,6 +240,93 @@ bool FeatureBuilder::CheckCorrect(vector<char> const & data) const } /////////////////////////////////////////////////////////////////////////////////////////////////// +// FeatureBuilderGeomRef implementation +/////////////////////////////////////////////////////////////////////////////////////////////////// + +bool FeatureBuilderGeomRef::IsDrawable(int lowS, int highS) const +{ + FeatureBase const fb = GetFeatureBase(); + + while (lowS <= highS) + if (feature::IsDrawableForIndex(fb, lowS++)) + return true; + + return false; +} + +void FeatureBuilderGeomRef::SimplifyPoints(points_t const & in, points_t & out, int level) const +{ + if (in.size() >= 2) + { + SimplifyDP<mn::DistanceToLineSquare<m2::PointD> >(in.begin(), in.end()-1, + my::sq(scales::GetEpsilonForLevel(level)), MakeBackInsertFunctor(out)); + + switch (out.size()) + { + case 0: + out.push_back(in.front()); + case 1: + out.push_back(in.back()); + break; + default: + if (!out.back().EqualDxDy(in.back(), MercatorBounds::GetCellID2PointAbsEpsilon())) + out.push_back(in.back()); + } + } +} + +int g_arrScales[] = { 5, 10, 14, 17 }; // 17 = scales::GetUpperScale() +//int g_arrScales[] = { 17 }; + +void FeatureBuilderGeomRef::Serialize(buffers_holder_t & data) const +{ + data.clear(); + + SerializeBase(data.m_buffers[0]); + + PushBackByteSink<buffer_t> sink(data.m_buffers[0]); + + // for point feature write geometry-point immediately + if (m_Geometry.size() == 1) + { + SerializePoints(m_Geometry, data.m_buffers[0]); + } + else + { + uint32_t mask = 0; + int lowS = 0; + vector<uint32_t> offsets; + for (int i = 0; i < ARRAY_SIZE(g_arrScales); ++i) + { + if (IsDrawable(lowS, g_arrScales[i])) + { + mask |= (1 << i); + offsets.push_back(data.m_buffers[1].size()); + + // serialize points + points_t points; + SimplifyPoints(m_Geometry, points, g_arrScales[i]); + SerializePoints(points, data.m_buffers[1]); + } + lowS = g_arrScales[i]+1; + } + + CHECK(mask > 0, (mask)); // feature should be visible + + // serialize geometry offsets + WriteVarUint(sink, mask); + for (size_t i = 0; i < offsets.size(); ++i) + WriteVarUint(sink, data.m_lineOffset + offsets[i]); + } + + if (!m_Triangles.empty()) + { + WriteVarUint(sink, data.m_trgOffset); + SerializeTriangles(data.m_buffers[2]); + } +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// // FeatureBase implementation /////////////////////////////////////////////////////////////////////////////////////////////////// @@ -302,7 +412,7 @@ string FeatureBase::DebugString() const return res; } -void FeatureBase::InitFeatureBuilder(FeatureBuilder & fb) const +void FeatureBase::InitFeatureBuilder(FeatureBuilderGeom & fb) const { ASSERT(m_bNameParsed, ()); @@ -315,62 +425,78 @@ void FeatureBase::InitFeatureBuilder(FeatureBuilder & fb) const // FeatureGeom implementation /////////////////////////////////////////////////////////////////////////////////////////////////// -FeatureGeom::FeatureGeom(vector<char> & data, uint32_t offset) +FeatureGeom::FeatureGeom(read_source_t & src) { - Deserialize(data, offset); + Deserialize(src); } -void FeatureGeom::Deserialize(vector<char> & data, uint32_t offset) +void FeatureGeom::Deserialize(read_source_t & src) { - base_type::Deserialize(data, offset); + base_type::Deserialize(src.m_data, src.m_offset); m_Geometry.clear(); m_Triangles.clear(); } -void FeatureGeom::ParseGeometry() const +template <class TSource> +void FeatureGeom::ParseGeometryImpl(TSource & src) const { ASSERT(!m_bGeometryParsed, ()); - if (!m_bNameParsed) - ParseName(); - - ArrayByteSource source(DataPtr() + m_GeometryOffset); uint32_t const geometrySize = - (GetFeatureType() == FEATURE_TYPE_POINT ? 1 : ReadVarUint<uint32_t>(source) + 1); + (GetFeatureType() == FEATURE_TYPE_POINT ? 1 : ReadVarUint<uint32_t>(src) + 1); m_Geometry.resize(geometrySize); int64_t id = 0; for (size_t i = 0; i < geometrySize; ++i) - m_LimitRect.Add(m_Geometry[i] = pts::ToPoint(id += ReadVarInt<int64_t>(source))); + m_LimitRect.Add(m_Geometry[i] = pts::ToPoint(id += ReadVarInt<int64_t>(src))); m_bGeometryParsed = true; +} + +void FeatureGeom::ParseGeometry(int) const +{ + if (!m_bNameParsed) + ParseName(); + + ArrayByteSource source(DataPtr() + m_GeometryOffset); + ParseGeometryImpl(source); + m_TrianglesOffset = CalcOffset(source); } -void FeatureGeom::ParseTriangles() const +template <class TSource> +void FeatureGeom::ParseTrianglesImpl(TSource & src) const { ASSERT(!m_bTrianglesParsed, ()); - if (!m_bGeometryParsed) - ParseGeometry(); - - ArrayByteSource source(DataPtr() + m_TrianglesOffset); if (GetFeatureType() == FEATURE_TYPE_AREA) { - uint32_t const trgPoints = (ReadVarUint<uint32_t>(source) + 1) * 3; + uint32_t const trgPoints = (ReadVarUint<uint32_t>(src) + 1) * 3; m_Triangles.resize(trgPoints); int64_t id = 0; for (size_t i = 0; i < trgPoints; ++i) - m_Triangles[i] = pts::ToPoint(id += ReadVarInt<int64_t>(source)); + m_Triangles[i] = pts::ToPoint(id += ReadVarInt<int64_t>(src)); } - m_bTrianglesParsed = true; +} + +void FeatureGeom::ParseTriangles(int scale) const +{ + if (!m_bGeometryParsed) + ParseGeometry(scale); + + ArrayByteSource source(DataPtr() + m_TrianglesOffset); + ParseTrianglesImpl(source); + ASSERT_EQUAL ( CalcOffset(source), m_Data.size() - m_Offset, () ); } void FeatureGeom::ParseAll() const { + if (!m_bGeometryParsed) + ParseGeometry(m_defScale); + if (!m_bTrianglesParsed) - ParseTriangles(); + ParseTriangles(m_defScale); } string FeatureGeom::DebugString() const @@ -382,7 +508,7 @@ string FeatureGeom::DebugString() const return res; } -void FeatureGeom::InitFeatureBuilder(FeatureBuilder & fb) const +void FeatureGeom::InitFeatureBuilder(FeatureBuilderGeom & fb) const { ParseAll(); base_type::InitFeatureBuilder(fb); @@ -399,3 +525,88 @@ void FeatureGeom::InitFeatureBuilder(FeatureBuilder & fb) const /////////////////////////////////////////////////////////////////////////////////////////////////// // FeatureGeomRef implementation /////////////////////////////////////////////////////////////////////////////////////////////////// + +FeatureGeomRef::FeatureGeomRef(read_source_t & src) + : base_type(src), m_gF(&src.m_gF), m_trgF(&src.m_trgF) +{ +} + +void FeatureGeomRef::Deserialize(read_source_t & src) +{ + m_gF = &src.m_gF; + m_trgF = &src.m_trgF; + + m_bOffsetsParsed = false; + m_mask = 0; + m_gOffsets.clear(); + + base_type::Deserialize(src); +} + +uint32_t FeatureGeomRef::GetOffset(int scale) const +{ + for (size_t i = 0; i < ARRAY_SIZE(g_arrScales); ++i) + if (scale <= g_arrScales[i]) + return m_gOffsets[i]; + + return m_invalidOffset; +} + +void FeatureGeomRef::ParseGeometry(int scale) const +{ + if (!m_bOffsetsParsed) + ParseOffsets(); + if (m_bGeometryParsed) + return; + + ReaderSource<FileReader> source(*m_gF); + uint32_t const offset = GetOffset(scale); + CHECK ( offset != m_invalidOffset, (offset) ); + source.Skip(offset); + + ParseGeometryImpl(source); +} + +void FeatureGeomRef::ParseTriangles(int scale) const +{ + if (!m_bOffsetsParsed) + ParseOffsets(); + + ReaderSource<FileReader> source(*m_trgF); + source.Skip(m_trgOffset); + + ParseTrianglesImpl(source); +} + +void FeatureGeomRef::ParseOffsets() const +{ + if (!m_bNameParsed) + ParseName(); + + ArrayByteSource source(DataPtr() + m_GeometryOffset); + FeatureType const type = GetFeatureType(); + + if (type == FEATURE_TYPE_POINT) + { + ParseGeometryImpl(source); + } + else + { + uint32_t mask = m_mask = ReadVarUint<uint32_t>(source); + ASSERT ( mask > 0, () ); + while (mask > 0) + { + if (mask & 0x01) + m_gOffsets.push_back(ReadVarUint<uint32_t>(source)); + else + m_gOffsets.push_back(m_invalidOffset); + + mask = mask >> 1; + } + + if (type == FEATURE_TYPE_AREA) + m_trgOffset = ReadVarUint<uint32_t>(source); + } + + m_bOffsetsParsed = true; +} diff --git a/indexer/feature.hpp b/indexer/feature.hpp index 286fd079e3..2de8e75e19 100644 --- a/indexer/feature.hpp +++ b/indexer/feature.hpp @@ -1,22 +1,25 @@ #pragma once +#include "cell_id.hpp" + #include "../geometry/point2d.hpp" #include "../geometry/rect2d.hpp" +#include "../coding/file_reader.hpp" + #include "../base/base.hpp" #include "../std/string.hpp" #include "../std/vector.hpp" -#include "cell_id.hpp" class ArrayByteSource; class FeatureBase; -class FeatureBuilder +class FeatureBuilderGeom { public: - FeatureBuilder(); + FeatureBuilderGeom(); void AddName(string const & name); void AddPoint(m2::PointD const & p); @@ -37,7 +40,9 @@ public: void AddLayer(int32_t layer); - void Serialize(vector<char> & data) const; + typedef vector<char> buffer_t; + typedef buffer_t buffers_holder_t; + void Serialize(buffers_holder_t & data) const; inline m2::RectD GetLimitRect() const { return m_LimitRect; } @@ -47,9 +52,15 @@ public: bool IsGeometryClosed() const; size_t GetPointsCount() const { return m_Geometry.size(); } - bool operator == (FeatureBuilder const &) const; + bool operator == (FeatureBuilderGeom const &) const; + +protected: + typedef vector<m2::PointD> points_t; + + void SerializeBase(buffer_t & data) const; + static void SerializePoints(points_t const & points, buffer_t & data); + void SerializeTriangles(buffer_t & data) const; -private: uint8_t GetHeader() const; bool CheckCorrect(vector<char> const & data) const; @@ -60,11 +71,38 @@ private: m2::RectD m_LimitRect; - vector<m2::PointD> m_Geometry; // store points as is for further processing + points_t m_Geometry; // store points as is for further processing vector<int64_t> m_Triangles; }; +class FeatureBuilderGeomRef : public FeatureBuilderGeom +{ + typedef FeatureBuilderGeom base_type; + + bool IsDrawable(int lowS, int highS) const; + void SimplifyPoints(points_t const & in, points_t & out, int level) const; + +public: + + struct buffers_holder_t + { + uint32_t m_lineOffset, m_trgOffset; ///< in + base_type::buffer_t m_buffers[3]; ///< out + + void clear() + { + for (int i = 0; i < 3; ++i) + m_buffers[i].clear(); + } + }; + + /// @name Overwrite from base_type. + //@{ + void Serialize(buffers_holder_t & data) const; + //@} +}; + class FeatureBase { public: @@ -82,7 +120,7 @@ public: //@{ void Deserialize(vector<char> & data, uint32_t offset = 0); string DebugString() const; - void InitFeatureBuilder(FeatureBuilder & fb) const; + void InitFeatureBuilder(FeatureBuilderGeom & fb) const; //@} inline FeatureType GetFeatureType() const @@ -154,7 +192,7 @@ protected: vector<char> m_Data; uint32_t m_Offset; - friend class FeatureBuilder; + friend class FeatureBuilderGeom; void SetHeader(uint8_t h); @@ -189,57 +227,80 @@ class FeatureGeom : public FeatureBase typedef FeatureBase base_type; public: + struct read_source_t + { + vector<char> m_data; + uint32_t m_offset; + + read_source_t() : m_offset(0) {} + read_source_t(string const &) : m_offset(0) {} + + void assign(char const * data, uint32_t size) + { + m_data.assign(data, data + size); + } + }; + FeatureGeom() {} - FeatureGeom(vector<char> & data, uint32_t offset = 0); + FeatureGeom(read_source_t & src); /// @name Overwrite from base_type. //@{ - void Deserialize(vector<char> & data, uint32_t offset = 0); + void Deserialize(read_source_t & src); string DebugString() const; - void InitFeatureBuilder(FeatureBuilder & fb) const; + void InitFeatureBuilder(FeatureBuilderGeom & fb) const; //@} inline m2::RectD GetLimitRect() const { if (!m_bGeometryParsed) - ParseGeometry(); + { + // this function use only in index generation, + // so get geometry for upper scale + ParseGeometry(m_defScale); + } return base_type::GetLimitRect(); } - inline uint32_t GetGeometrySize() const - { - if (!m_bGeometryParsed) - ParseGeometry(); - return m_Geometry.size(); - } + /// @name Used only in unit-tests. So remove it. + //@{ + //inline uint32_t GetGeometrySize() const + //{ + // if (!m_bGeometryParsed) + // ParseGeometry(); + // return m_Geometry.size(); + //} + + //inline uint32_t GetTriangleCount() const + //{ + // if (!m_bTrianglesParsed) + // ParseTriangles(); + // return (m_Triangles.size() / 3); + //} + //@} - inline uint32_t GetTriangleCount() const - { - if (!m_bTrianglesParsed) - ParseTriangles(); - return (m_Triangles.size() / 3); - } + static int const m_defScale = 17; template <typename FunctorT> - void ForEachPointRef(FunctorT & f) const + void ForEachPointRef(FunctorT & f, int scale) const { if (!m_bGeometryParsed) - ParseGeometry(); + ParseGeometry(scale); for (size_t i = 0; i < m_Geometry.size(); ++i) f(CoordPointT(m_Geometry[i].x, m_Geometry[i].y)); } template <typename FunctorT> - void ForEachPoint(FunctorT f) const + void ForEachPoint(FunctorT f, int scale) const { - ForEachPointRef(f); + ForEachPointRef(f, scale); } template <typename FunctorT> - void ForEachTriangleRef(FunctorT & f) const + void ForEachTriangleRef(FunctorT & f, int scale) const { if (!m_bTrianglesParsed) - ParseTriangles(); + ParseTriangles(scale); for (size_t i = 0; i < m_Triangles.size();) { f(m_Triangles[i], m_Triangles[i+1], m_Triangles[i+2]); @@ -248,17 +309,19 @@ public: } template <typename FunctorT> - void ForEachTriangleExRef(FunctorT & f) const + void ForEachTriangleExRef(FunctorT & f, int scale) const { f.StartPrimitive(m_Triangles.size()); - ForEachTriangleRef(f); + ForEachTriangleRef(f, scale); f.EndPrimitive(); } protected: + template <class TSource> void ParseGeometryImpl(TSource & src) const; + template <class TSource> void ParseTrianglesImpl(TSource & src) const; - void ParseGeometry() const; - void ParseTriangles() const; + virtual void ParseGeometry(int scale) const; + virtual void ParseTriangles(int scale) const; void ParseAll() const; @@ -271,8 +334,43 @@ class FeatureGeomRef : public FeatureGeom typedef FeatureGeom base_type; public: + struct read_source_t : public base_type::read_source_t + { + FileReader m_gF; + FileReader m_trgF; + + read_source_t(string const & name) + : m_gF(name + ".geom"), m_trgF(name + ".trg") + { + } + }; + FeatureGeomRef() {} - FeatureGeomRef(vector<char> & data, uint32_t offset = 0) : base_type(data, offset) {} + FeatureGeomRef(read_source_t & src); + + void Deserialize(read_source_t & src); + +protected: + /// @name Overwrite from base_type. + //@{ + virtual void ParseGeometry(int scale) const; + virtual void ParseTriangles(int scale) const; + //@} + +private: + FileReader * m_gF; + FileReader * m_trgF; + + void ParseOffsets() const; + mutable bool m_bOffsetsParsed; + + static uint32_t const m_invalidOffset = uint32_t(-1); + + uint32_t GetOffset(int scale) const; + + mutable uint32_t m_mask; + mutable vector<uint32_t> m_gOffsets; + mutable uint32_t m_trgOffset; }; inline string debug_print(FeatureGeom const & f) @@ -280,5 +378,11 @@ inline string debug_print(FeatureGeom const & f) return f.DebugString(); } +inline string debug_print(FeatureGeomRef const & f) +{ + return f.DebugString(); +} + -typedef FeatureGeom FeatureType; +typedef FeatureGeomRef FeatureType; +typedef FeatureBuilderGeomRef FeatureBuilderType; diff --git a/indexer/feature_processor.hpp b/indexer/feature_processor.hpp index 4ef1b23494..1045f6593a 100644 --- a/indexer/feature_processor.hpp +++ b/indexer/feature_processor.hpp @@ -12,11 +12,11 @@ namespace feature { /// Read feature from feature source. template <class TSource, class TFeature> - void ReadFromSource(TSource & src, TFeature & f) + void ReadFromSource(TSource & src, TFeature & f, typename TFeature::read_source_t & buffer) { uint32_t const sz = ReadVarUint<uint32_t>(src); - vector<char> buffer(sz); - src.Read(&buffer[0], sz); + buffer.m_data.resize(sz); + src.Read(&buffer.m_data[0], sz); f.Deserialize(buffer); } @@ -28,6 +28,7 @@ namespace feature FileReader reader(fName); source_t src(reader); + typename TFeature::read_source_t buffer(fName); // skip header uint64_t currPos = feature::GetSkipHeaderSize(reader); @@ -38,7 +39,7 @@ namespace feature while (currPos < fSize) { TFeature f; - ReadFromSource(src, f); + ReadFromSource(src, f, buffer); toDo(f, currPos); currPos = src.Pos(); } diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp index 81d62abdc8..0721366a03 100644 --- a/indexer/features_vector.hpp +++ b/indexer/features_vector.hpp @@ -14,16 +14,15 @@ class FeaturesVector public: typedef ReaderT ReaderType; - explicit FeaturesVector(ReaderT const & reader) : m_RecordReader(reader, 256) + explicit FeaturesVector(ReaderT const & reader) + : m_RecordReader(reader, 256), m_source(reader.GetName()) { } void Get(uint64_t pos, FeatureType & feature) const { - vector<char> record; - uint32_t offset; - m_RecordReader.ReadRecord(pos, record, offset); - feature.Deserialize(record, offset); + m_RecordReader.ReadRecord(pos, m_source.m_data, m_source.m_offset); + feature.Deserialize(m_source); } template <class TDo> void ForEachOffset(TDo const & toDo) const @@ -48,10 +47,11 @@ public: private: FeatureType const & DeserializeFeature(char const * data, uint32_t size, FeatureType * pFeature) const { - vector<char> data1(data, data + size); - pFeature->Deserialize(data1); + m_source.assign(data, size); + pFeature->Deserialize(m_source); return *pFeature; } VarRecordReader<ReaderT, &VarRecordSizeReaderVarint> m_RecordReader; + mutable FeatureType::read_source_t m_source; }; diff --git a/indexer/indexer_tests/feature_bucketer_test.cpp b/indexer/indexer_tests/feature_bucketer_test.cpp index 5783066fa9..d318b51ba1 100644 --- a/indexer/indexer_tests/feature_bucketer_test.cpp +++ b/indexer/indexer_tests/feature_bucketer_test.cpp @@ -19,12 +19,12 @@ namespace PushBackFeatureDebugStringOutput(string const & name, InitDataType const & initData) : m_pContainer(&((*initData)[name])) {} - void operator() (FeatureBuilder const & fb) + void operator() (FeatureBuilderType const & fb) { - vector<char> bytes; - fb.Serialize(bytes); - FeatureType f(bytes); + FeatureType::read_source_t bytes; + fb.Serialize(bytes.m_data); + FeatureType f(bytes); m_pContainer->push_back(f.DebugString()); } @@ -39,10 +39,11 @@ namespace RectId > FeatureBucketer; - FeatureType MakeFeature(FeatureBuilder const & fb) + FeatureType MakeFeature(FeatureBuilderType const & fb) { - vector<char> bytes; - fb.Serialize(bytes); + FeatureType::read_source_t bytes; + fb.Serialize(bytes.m_data); + return FeatureType(bytes); } } @@ -52,7 +53,7 @@ UNIT_TEST(FeatureBucketerSmokeTest) map<string, vector<string> > out, expectedOut; FeatureBucketer bucketer(1, &out); - FeatureBuilder fb; + FeatureBuilderType fb; fb.AddPoint(m2::PointD(10, 10)); fb.AddPoint(m2::PointD(20, 20)); bucketer(fb); diff --git a/indexer/indexer_tests/feature_test.cpp b/indexer/indexer_tests/feature_test.cpp index 14d31d9df2..d40b125cfd 100644 --- a/indexer/indexer_tests/feature_test.cpp +++ b/indexer/indexer_tests/feature_test.cpp @@ -37,7 +37,7 @@ UNIT_TEST(Feature_Deserialize) vector<int> a; a.push_back(1); a.push_back(2); - FeatureBuilder builder; + FeatureBuilderType builder; builder.AddName("name"); @@ -66,9 +66,10 @@ UNIT_TEST(Feature_Deserialize) uint32_t arrTypes[typesCount+1] = { 5, 7, 0 }; builder.AddTypes(arrTypes, arrTypes + typesCount); - vector<char> serial; + FeatureBuilderType::buffers_holder_t serial; builder.Serialize(serial); - vector<char> serial1 = serial; + FeatureType::read_source_t serial1; + serial1.m_data = serial; FeatureType f(serial1); TEST_EQUAL(f.GetFeatureType(), FeatureBase::FEATURE_TYPE_AREA, ()); @@ -79,15 +80,15 @@ UNIT_TEST(Feature_Deserialize) TEST_EQUAL(f.GetLayer(), 3, ()); TEST_EQUAL(f.GetName(), "name", ()); - TEST_EQUAL(f.GetGeometrySize(), 4, ()); - TEST_EQUAL(f.GetTriangleCount(), 1, ()); + //TEST_EQUAL(f.GetGeometrySize(), 4, ()); + //TEST_EQUAL(f.GetTriangleCount(), 1, ()); PointAccumulator featurePoints; - f.ForEachPointRef(featurePoints); + f.ForEachPointRef(featurePoints, FeatureType::m_defScale); TEST_EQUAL(points, featurePoints.m_V, ()); PointAccumulator featureTriangles; - f.ForEachTriangleRef(featureTriangles); + f.ForEachTriangleRef(featureTriangles, FeatureType::m_defScale); TEST_EQUAL(triangles, featureTriangles.m_V, ()); double const eps = MercatorBounds::GetCellID2PointAbsEpsilon(); @@ -96,11 +97,11 @@ UNIT_TEST(Feature_Deserialize) TEST_LESS(fabs(f.GetLimitRect().maxX() - 1.00), eps, ()); TEST_LESS(fabs(f.GetLimitRect().maxY() - 1.00), eps, ()); - vector<char> serial2; - FeatureBuilder builder2; + FeatureType::read_source_t serial2; + FeatureBuilderType builder2; f.InitFeatureBuilder(builder2); - builder2.Serialize(serial2); + builder2.Serialize(serial2.m_data); - TEST_EQUAL(serial, serial2, + TEST_EQUAL(serial, serial2.m_data, (f.DebugString(), FeatureType(serial2).DebugString())); } diff --git a/indexer/indexer_tool/feature_bucketer.hpp b/indexer/indexer_tool/feature_bucketer.hpp index 9b3a35d482..8f291b8716 100644 --- a/indexer/indexer_tool/feature_bucketer.hpp +++ b/indexer/indexer_tool/feature_bucketer.hpp @@ -22,6 +22,8 @@ namespace feature template <class FeatureOutT, class FeatureClipperT, class BoundsT, typename CellIdT> class CellFeatureBucketer { + typedef typename FeatureClipperT::feature_builder_t feature_builder_t; + public: CellFeatureBucketer(int level, typename FeatureOutT::InitDataType const & featureOutInitData, int maxWorldZoom = -1) @@ -42,7 +44,7 @@ public: m_worldBucket.reset(new FeatureOutT(WORLD_FILE_NAME, m_FeatureOutInitData)); } - void operator () (FeatureBuilder const & fb) + void operator () (feature_builder_t const & fb) { // separately store features needed for world map if (m_worldBucket && m_maxWorldZoom >= feature::MinDrawableScaleForFeature(fb.GetFeatureBase())) @@ -57,7 +59,7 @@ public: // Clipper may (or may not) do a better intersection. if (m_Buckets[i].m_Rect.IsIntersect(limitRect)) { - FeatureBuilder clippedFb; + feature_builder_t clippedFb; if (clipper(m_Buckets[i].m_Rect, clippedFb)) { if (!m_Buckets[i].m_pOut) @@ -101,13 +103,18 @@ private: class SimpleFeatureClipper { - FeatureBuilder const & m_Feature; public: - explicit SimpleFeatureClipper(FeatureBuilder const & f) : m_Feature(f) + typedef FeatureBuilderGeom feature_builder_t; + +private: + feature_builder_t const & m_Feature; + +public: + explicit SimpleFeatureClipper(feature_builder_t const & f) : m_Feature(f) { } - bool operator () (m2::RectD const & /*rect*/, FeatureBuilder & clippedF) const + bool operator () (m2::RectD const & /*rect*/, feature_builder_t & clippedF) const { clippedF = m_Feature; return true; diff --git a/indexer/indexer_tool/feature_generator.cpp b/indexer/indexer_tool/feature_generator.cpp index 6a88ef77e7..143474f65a 100644 --- a/indexer/indexer_tool/feature_generator.cpp +++ b/indexer/indexer_tool/feature_generator.cpp @@ -129,37 +129,49 @@ public: } }; +/////////////////////////////////////////////////////////////////////////////////////////////////// +// FeaturesCollector implementation +/////////////////////////////////////////////////////////////////////////////////////////////////// + void FeaturesCollector::Init() { // write empty stub, will be updated in Finish() WriteDataHeader(m_datFile, feature::DataHeader()); } -FeaturesCollector::FeaturesCollector(string const & datFile) : m_datFile(datFile) +FeaturesCollector::FeaturesCollector(string const & fName) +: m_datFile(fName), m_geoFile(fName + ".geom"), m_trgFile(fName + ".trg") { Init(); } -FeaturesCollector::FeaturesCollector(string const & bucketName, - FeaturesCollector::InitDataType const & datFilePrefixSuffix) - : m_datFile(datFilePrefixSuffix.first + bucketName + datFilePrefixSuffix.second) +FeaturesCollector::FeaturesCollector(string const & bucket, + FeaturesCollector::InitDataType const & prefix) +: m_datFile(prefix.first + bucket + prefix.second), + m_geoFile(prefix.first + bucket + prefix.second + ".geom"), + m_trgFile(prefix.first + bucket + prefix.second + ".trg") { Init(); } -void FeaturesCollector::operator() (FeatureBuilder const & fb) +void FeaturesCollector::FilePreCondition(FileWriter const & f) { -#ifdef DEBUG // .dat file should be less than 4Gb - uint64_t const pos = m_datFile.Pos(); - ASSERT_EQUAL ( static_cast<uint64_t>(static_cast<uint32_t>(pos)), pos, + uint64_t const pos = f.Pos(); + CHECK_EQUAL ( static_cast<uint64_t>(static_cast<uint32_t>(pos)), pos, ("Feature offset is out of 32bit boundary!") ); -#endif +} - vector<char> bytes; - fb.Serialize(bytes); +void FeaturesCollector::WriteBuffer(FileWriter & f, vector<char> const & bytes) +{ + if (!bytes.empty()) + f.Write(&bytes[0], bytes.size()); +} + +void FeaturesCollector::WriteFeatureBase(vector<char> const & bytes, FeatureBuilderGeom const & fb) +{ size_t const sz = bytes.size(); - CHECK(sz, ("Empty feature! WTF?")); + CHECK ( sz != 0, ("Empty feature not allowed here!") ); if (sz > 0) { @@ -170,6 +182,32 @@ void FeaturesCollector::operator() (FeatureBuilder const & fb) } } +void FeaturesCollector::operator() (FeatureBuilderGeom const & fb) +{ + FilePreCondition(m_datFile); + + FeatureBuilderGeom::buffers_holder_t bytes; + fb.Serialize(bytes); + WriteFeatureBase(bytes, fb); +} + +void FeaturesCollector::operator() (FeatureBuilderGeomRef const & fb) +{ + FilePreCondition(m_datFile); + FilePreCondition(m_geoFile); + FilePreCondition(m_trgFile); + + FeatureBuilderGeomRef::buffers_holder_t buffers; + buffers.m_lineOffset = static_cast<uint32_t>(m_geoFile.Pos()); + buffers.m_trgOffset = static_cast<uint32_t>(m_trgFile.Pos()); + + fb.Serialize(buffers); + WriteFeatureBase(buffers.m_buffers[0], fb); + + WriteBuffer(m_geoFile, buffers.m_buffers[1]); + WriteBuffer(m_trgFile, buffers.m_buffers[2]); +} + FeaturesCollector::~FeaturesCollector() { // rewrite map information with actual data @@ -179,6 +217,10 @@ FeaturesCollector::~FeaturesCollector() WriteDataHeader(m_datFile, header); } +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Generate functions implementations. +/////////////////////////////////////////////////////////////////////////////////////////////////// + class points_in_file { FileReader m_file; diff --git a/indexer/indexer_tool/feature_generator.hpp b/indexer/indexer_tool/feature_generator.hpp index 6d55c864f3..bf68796394 100644 --- a/indexer/indexer_tool/feature_generator.hpp +++ b/indexer/indexer_tool/feature_generator.hpp @@ -9,7 +9,8 @@ #include "../../std/vector.hpp" #include "../../std/string.hpp" -class FeatureBuilder; +class FeatureBuilderGeom; +class FeatureBuilderGeomRef; namespace feature { @@ -31,10 +32,14 @@ namespace feature // Writes features to dat file. class FeaturesCollector { - FileWriter m_datFile; + FileWriter m_datFile, m_geoFile, m_trgFile; m2::RectD m_bounds; void Init(); + void FilePreCondition(FileWriter const & f); + + static void WriteBuffer(FileWriter & f, vector<char> const & bytes); + void WriteFeatureBase(vector<char> const & bytes, FeatureBuilderGeom const & fb); public: ~FeaturesCollector(); @@ -42,9 +47,10 @@ namespace feature // Stores prefix and suffix of a dat file name. typedef pair<string, string> InitDataType; - explicit FeaturesCollector(string const & datFile); - FeaturesCollector(string const & bucketName, InitDataType const & datFilePrefixSuffix); + explicit FeaturesCollector(string const & fName); + FeaturesCollector(string const & bucket, InitDataType const & prefix); - void operator() (FeatureBuilder const & f); + void operator() (FeatureBuilderGeom const & f); + void operator() (FeatureBuilderGeomRef const & f); }; } diff --git a/indexer/indexer_tool/feature_sorter.cpp b/indexer/indexer_tool/feature_sorter.cpp index 3fafedb703..db068ec262 100644 --- a/indexer/indexer_tool/feature_sorter.cpp +++ b/indexer/indexer_tool/feature_sorter.cpp @@ -34,7 +34,7 @@ namespace m_midX = 0.0; m_midY = 0.0; m_counter = 0; - ft.ForEachPointRef(*this); + ft.ForEachPointRef(*this, FeatureGeom::m_defScale); m_midX /= m_counter; m_midY /= m_counter; @@ -85,12 +85,14 @@ namespace feature // store sorted features { - FileWriter writer(datFilePath); + //FileWriter writer(datFilePath); FileReader reader(tempDatFilePath); - feature::DataHeader header; - feature::ReadDataHeader(tempDatFilePath, header); - feature::WriteDataHeader(writer, header); + //feature::DataHeader header; + //feature::ReadDataHeader(tempDatFilePath, header); + //feature::WriteDataHeader(writer, header); + + FeaturesCollector collector(datFilePath); for (size_t i = 0; i < midPoints.m_vec.size(); ++i) { @@ -101,12 +103,21 @@ namespace feature // read feature bytes uint32_t const sz = ReadVarUint<uint32_t>(src); - vector<char> buffer(sz); - src.Read(&buffer[0], sz); + FeatureGeom::read_source_t buffer; + buffer.m_data.resize(sz); + src.Read(&buffer.m_data[0], sz); + + FeatureGeom f; + f.Deserialize(buffer); + + FeatureBuilderType fb; + f.InitFeatureBuilder(fb); // write feature bytes - WriteVarUint(writer, sz); - writer.Write(&buffer[0], sz); + //WriteVarUint(writer, sz); + //writer.Write(&buffer[0], sz); + + collector(fb); } // at this point files should be closed diff --git a/indexer/indexer_tool/osm_element.hpp b/indexer/indexer_tool/osm_element.hpp index 2712db7907..49ef5d165e 100644 --- a/indexer/indexer_tool/osm_element.hpp +++ b/indexer/indexer_tool/osm_element.hpp @@ -19,7 +19,7 @@ namespace feature { typedef list<vector<m2::PointD> > holes_cont_t; - void TesselateInterior(FeatureBuilder & featureBuilder, feature::holes_cont_t const & holes); + void TesselateInterior(FeatureBuilderGeom & featureBuilder, feature::holes_cont_t const & holes); } /// @param TEmitter Feature accumulating policy @@ -185,12 +185,14 @@ protected: } } m_typeProcessor; + typedef FeatureBuilderGeom feature_builder_t; + bool GetPoint(uint64_t id, m2::PointD & pt) { return m_holder.GetNode(id, pt.y, pt.x); } - void FinishAreaFeature(uint64_t id, FeatureBuilder & ft) + void FinishAreaFeature(uint64_t id, feature_builder_t & ft) { if (ft.IsGeometryClosed()) { @@ -228,7 +230,7 @@ class SecondPassParserJoin : public SecondPassParserBase<TEmitter, THolder> set<uint64_t> m_usedDirect; - bool TryEmitUnited(uint64_t featureID, FeatureBuilder & ft) + bool TryEmitUnited(uint64_t featureID, base_type::feature_builder_t & ft) { // check, if feature already processed early if (m_usedDirect.count(featureID) > 0) @@ -324,7 +326,7 @@ protected: for (typename base_type::value_t::types_t::iterator i = fValue.types.begin(); i != fValue.types.end(); ++i) if (feature::NeedUnite(*i)) { - FeatureBuilder ft; + base_type::feature_builder_t ft; ft.AddName(fValue.name); ft.AddTypes(fValue.types.begin(), fValue.types.end()); ft.AddLayer(fValue.layer); @@ -354,7 +356,7 @@ protected: if (!ParseType(p, id, fValue)) return; - FeatureBuilder ft; + base_type::feature_builder_t ft; ft.AddName(fValue.name); ft.AddTypes(fValue.types.begin(), fValue.types.end()); ft.AddLayer(fValue.layer); diff --git a/indexer/indexer_tool/tesselator.cpp b/indexer/indexer_tool/tesselator.cpp index 642ed346a1..659874307e 100644 --- a/indexer/indexer_tool/tesselator.cpp +++ b/indexer/indexer_tool/tesselator.cpp @@ -17,7 +17,7 @@ namespace feature } }; - void TesselateInterior(FeatureBuilder & fb, feature::holes_cont_t const & holes) + void TesselateInterior(FeatureBuilderGeom & fb, feature::holes_cont_t const & holes) { ASSERT(fb.IsGeometryClosed(), ()); @@ -30,10 +30,10 @@ namespace feature tess.beginContour(); { - vector<char> bytes; - fb.Serialize(bytes); + FeatureGeom::read_source_t bytes; + fb.Serialize(bytes.m_data); FeatureGeom f(bytes); - f.ForEachPoint(AddTessPointF(tess)); + f.ForEachPoint(AddTessPointF(tess), FeatureGeom::m_defScale); } tess.endContour(); diff --git a/indexer/scale_index_builder.hpp b/indexer/scale_index_builder.hpp index 042c3afcac..3be990b440 100644 --- a/indexer/scale_index_builder.hpp +++ b/indexer/scale_index_builder.hpp @@ -57,7 +57,7 @@ public: { if (FeatureShouldBeIndexed(f)) { - vector<int64_t> const cells = covering::CoverFeature(f); + vector<int64_t> const cells = covering::CoverFeature(f, m_ScaleRange.second); for (vector<int64_t>::const_iterator it = cells.begin(); it != cells.end(); ++it) m_Sorter.Add(make_pair(*it, offset)); ++m_NumFeatures; |