diff options
| -rw-r--r-- | defines.hpp | 2 | ||||
| -rw-r--r-- | generator/CMakeLists.txt | 5 | ||||
| -rw-r--r-- | generator/feature_builder.cpp | 67 | ||||
| -rw-r--r-- | generator/feature_builder.hpp | 4 | ||||
| -rw-r--r-- | generator/feature_generator.cpp | 1 | ||||
| -rw-r--r-- | generator/feature_generator.hpp | 2 | ||||
| -rw-r--r-- | generator/feature_helpers.cpp | 58 | ||||
| -rw-r--r-- | generator/feature_helpers.hpp | 101 | ||||
| -rw-r--r-- | generator/feature_sorter.cpp | 317 | ||||
| -rw-r--r-- | generator/feature_sorter.hpp | 61 | ||||
| -rw-r--r-- | generator/generator_tests/coasts_test.cpp | 4 | ||||
| -rw-r--r-- | generator/generator_tool/generator_tool.cpp | 58 | ||||
| -rw-r--r-- | generator/geometry_holder.hpp | 249 | ||||
| -rw-r--r-- | generator/locality_sorter.cpp | 176 | ||||
| -rw-r--r-- | generator/locality_sorter.hpp | 11 | ||||
| -rw-r--r-- | indexer/locality_index_builder.cpp | 8 | ||||
| -rw-r--r-- | indexer/locality_object.cpp | 12 | ||||
| -rw-r--r-- | xcode/generator/generator.xcodeproj/project.pbxproj | 20 |
18 files changed, 772 insertions, 384 deletions
diff --git a/defines.hpp b/defines.hpp index efc0bc6080..9518918c1e 100644 --- a/defines.hpp +++ b/defines.hpp @@ -11,6 +11,8 @@ #define EXTENSION_TMP ".tmp" #define ADDR_FILE_EXTENSION ".addr" #define RAW_GEOM_FILE_EXTENSION ".rawgeom" +#define LOC_IDX_FILE_EXTENSION ".locidx" +#define LOC_DATA_FILE_EXTENSION ".locdata" #define NODES_FILE "nodes.dat" #define WAYS_FILE "ways.dat" diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index f34c76b280..cac764772d 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -30,14 +30,19 @@ set(SRC feature_emitter_iface.hpp feature_generator.cpp feature_generator.hpp + feature_helpers.cpp + feature_helpers.hpp feature_merger.cpp feature_merger.hpp feature_sorter.cpp feature_sorter.hpp gen_mwm_info.hpp generate_info.hpp + geometry_holder.hpp intermediate_data.hpp intermediate_elements.hpp + locality_sorter.cpp + locality_sorter.hpp metalines_builder.cpp metalines_builder.hpp node_mixer.hpp diff --git a/generator/feature_builder.cpp b/generator/feature_builder.cpp index 4e859a920c..4a754684a1 100644 --- a/generator/feature_builder.cpp +++ b/generator/feature_builder.cpp @@ -6,10 +6,10 @@ #include "routing_common/car_model.hpp" #include "routing_common/pedestrian_model.hpp" +#include "indexer/coding_params.hpp" #include "indexer/feature_impl.hpp" #include "indexer/feature_visibility.hpp" #include "indexer/geometry_serialization.hpp" -#include "indexer/coding_params.hpp" #include "geometry/region2d.hpp" @@ -21,8 +21,10 @@ #include <algorithm> #include <cstring> +#include <vector> using namespace feature; +using namespace std; /////////////////////////////////////////////////////////////////////////////////////////////////// // FeatureBuilder1 implementation @@ -79,14 +81,14 @@ void FeatureBuilder1::SetRank(uint8_t rank) m_params.rank = rank; } -void FeatureBuilder1::AddHouseNumber(std::string const & houseNumber) +void FeatureBuilder1::AddHouseNumber(string const & houseNumber) { m_params.AddHouseNumber(houseNumber); } -void FeatureBuilder1::AddStreet(std::string const & streetName) { m_params.AddStreet(streetName); } +void FeatureBuilder1::AddStreet(string const & streetName) { m_params.AddStreet(streetName); } -void FeatureBuilder1::AddPostcode(std::string const & postcode) +void FeatureBuilder1::AddPostcode(string const & postcode) { m_params.GetMetadata().Set(Metadata::FMD_POSTCODE, postcode); } @@ -106,7 +108,7 @@ void FeatureBuilder1::SetLinear(bool reverseGeometry) { auto & cont = m_polygons.front(); ASSERT(!cont.empty(), ()); - std::reverse(cont.begin(), cont.end()); + reverse(cont.begin(), cont.end()); } } @@ -169,7 +171,7 @@ bool FeatureBuilder1::RemoveInvalidTypes() m_params.IsEmptyNames()); } -bool FeatureBuilder1::FormatFullAddress(std::string & res) const +bool FeatureBuilder1::FormatFullAddress(string & res) const { return m_params.FormatFullAddress(m_limitRect.Center(), res); } @@ -491,7 +493,7 @@ bool FeatureBuilder1::HasOsmId(osm::Id const & id) const return false; } -std::string FeatureBuilder1::GetOsmIdsString() const +string FeatureBuilder1::GetOsmIdsString() const { if (m_osmIds.empty()) return "(NOT AN OSM FEATURE)"; @@ -510,14 +512,14 @@ int FeatureBuilder1::GetMinFeatureDrawScale() const return (minScale == -1 ? 1000 : minScale); } -bool FeatureBuilder1::AddName(std::string const & lang, std::string const & name) +bool FeatureBuilder1::AddName(string const & lang, string const & name) { return m_params.AddName(lang, name); } -std::string FeatureBuilder1::GetName(int8_t lang) const +string FeatureBuilder1::GetName(int8_t lang) const { - std::string s; + string s; VERIFY(m_params.name.GetString(lang, s) != s.empty(), ()); return s; } @@ -530,7 +532,7 @@ size_t FeatureBuilder1::GetPointsCount() const return counter; } -std::string DebugPrint(FeatureBuilder1 const & f) +string DebugPrint(FeatureBuilder1 const & f) { ostringstream out; @@ -569,7 +571,7 @@ uint64_t FeatureBuilder1::GetWayIDForRouting() const return 0; } -std::string DebugPrint(FeatureBuilder2 const & f) +string DebugPrint(FeatureBuilder2 const & f) { return DebugPrint(static_cast<FeatureBuilder1 const &>(f)); } @@ -593,6 +595,39 @@ bool FeatureBuilder2::PreSerialize(SupportingData const & data) return TBase::PreSerialize(); } +bool FeatureBuilder2::IsLocalityObject() +{ + return (m_params.GetGeomType() == GEOM_POINT || m_params.GetGeomType() == GEOM_AREA) && + !m_params.house.IsEmpty(); +} + +void FeatureBuilder2::SerializeLocalityObject(serial::CodingParams const & params, + SupportingData & data) +{ + data.m_buffer.clear(); + + PushBackByteSink<TBuffer> sink(data.m_buffer); + WriteToSink(sink, GetMostGenericOsmId().EncodedId()); + + auto const type = m_params.GetGeomType(); + WriteToSink(sink, static_cast<uint8_t>(type)); + + if (type == GEOM_POINT) + { + serial::SavePoint(sink, m_center, params); + return; + } + + CHECK_EQUAL(type, GEOM_AREA, ("Supported types are GEOM_POINT and GEOM_AREA")); + + uint32_t trgCount = base::asserted_cast<uint32_t>(data.m_innerTrg.size()); + CHECK_GREATER(trgCount, 2, ()); + trgCount -= 2; + + WriteToSink(sink, trgCount); + serial::SaveInnerTriangles(data.m_innerTrg, params, sink); +} + void FeatureBuilder2::Serialize(SupportingData & data, serial::CodingParams const & params) { data.m_buffer.clear(); @@ -602,8 +637,8 @@ void FeatureBuilder2::Serialize(SupportingData & data, serial::CodingParams cons PushBackByteSink<TBuffer> sink(data.m_buffer); - uint8_t const ptsCount = static_cast<uint8_t>(data.m_innerPts.size()); - uint8_t trgCount = static_cast<uint8_t>(data.m_innerTrg.size()); + uint8_t const ptsCount = base::asserted_cast<uint8_t>(data.m_innerPts.size()); + uint8_t trgCount = base::asserted_cast<uint8_t>(data.m_innerTrg.size()); if (trgCount > 0) { ASSERT_GREATER ( trgCount, 2, () ); @@ -654,7 +689,7 @@ void FeatureBuilder2::Serialize(SupportingData & data, serial::CodingParams cons serial::SavePoint(sink, GetOuterGeometry()[0], params); // offsets was pushed from high scale index to low - std::reverse(data.m_ptsOffset.begin(), data.m_ptsOffset.end()); + reverse(data.m_ptsOffset.begin(), data.m_ptsOffset.end()); serial::WriteVarUintArray(data.m_ptsOffset, sink); } } @@ -665,7 +700,7 @@ void FeatureBuilder2::Serialize(SupportingData & data, serial::CodingParams cons else { // offsets was pushed from high scale index to low - std::reverse(data.m_trgOffset.begin(), data.m_trgOffset.end()); + reverse(data.m_trgOffset.begin(), data.m_trgOffset.end()); serial::WriteVarUintArray(data.m_trgOffset, sink); } } diff --git a/generator/feature_builder.hpp b/generator/feature_builder.hpp index cffe7361ba..7bd1581263 100644 --- a/generator/feature_builder.hpp +++ b/generator/feature_builder.hpp @@ -9,7 +9,7 @@ #include <functional> #include <list> - +#include <string> namespace serial { class CodingParams; } @@ -254,6 +254,8 @@ public: /// @name Overwrite from base_type. //@{ bool PreSerialize(SupportingData const & data); + bool IsLocalityObject(); + void SerializeLocalityObject(serial::CodingParams const & params, SupportingData & data); void Serialize(SupportingData & data, serial::CodingParams const & params); //@} diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp index 577b86a120..4e6614845d 100644 --- a/generator/feature_generator.cpp +++ b/generator/feature_generator.cpp @@ -37,6 +37,7 @@ FeaturesCollector::~FeaturesCollector() (void)GetFileSize(m_datFile); } +// static uint32_t FeaturesCollector::GetFileSize(FileWriter const & f) { // .dat file should be less than 4Gb diff --git a/generator/feature_generator.hpp b/generator/feature_generator.hpp index a0e1af6b1d..7dc23c9c05 100644 --- a/generator/feature_generator.hpp +++ b/generator/feature_generator.hpp @@ -30,7 +30,6 @@ private: void FlushBuffer(); protected: - static uint32_t GetFileSize(FileWriter const & f); /// @return feature offset in the file, which is used as an ID later uint32_t WriteFeatureBase(std::vector<char> const & bytes, FeatureBuilder1 const & fb); @@ -41,6 +40,7 @@ public: FeaturesCollector(std::string const & fName); virtual ~FeaturesCollector(); + static uint32_t GetFileSize(FileWriter const & f); std::string const & GetFilePath() const { return m_datFile.GetName(); } /// \brief Serializes |f|. /// \returns feature id of serialized feature if |f| is serialized after the call diff --git a/generator/feature_helpers.cpp b/generator/feature_helpers.cpp new file mode 100644 index 0000000000..9b1637ffa9 --- /dev/null +++ b/generator/feature_helpers.cpp @@ -0,0 +1,58 @@ +#include "generator/feature_helpers.hpp" + +#include "generator/feature_builder.hpp" + +#include "indexer/feature_visibility.hpp" + +#include "base/stl_helpers.hpp" + +#include <algorithm> + +using namespace std; + +namespace feature +{ +void CalculateMidPoints::operator()(FeatureBuilder1 const & ft, uint64_t pos) +{ + // Reset state. + m_midLoc = m2::PointD::Zero();; + m_locCount = 0; + + ft.ForEachGeometryPoint(*this); + ASSERT_NOT_EQUAL(m_locCount, 0, ()); + m_midLoc = m_midLoc / m_locCount; + + uint64_t const pointAsInt64 = PointToInt64(m_midLoc, m_coordBits); + int const minScale = feature::GetMinDrawableScale(ft.GetFeatureBase()); + + /// May be invisible if it's small area object with [0-9] scales. + /// @todo Probably, we need to keep that objects if 9 scale (as we do in 17 scale). + if (minScale != -1 || feature::RequireGeometryInIndex(ft.GetFeatureBase())) + { + uint64_t const order = (static_cast<uint64_t>(minScale) << 59) | (pointAsInt64 >> 5); + m_vec.push_back(make_pair(order, pos)); + } +} + +bool CalculateMidPoints::operator()(m2::PointD const & p) +{ + m_midLoc += p; + m_midAll += p; + ++m_locCount; + ++m_allCount; + return true; +} + +m2::PointD CalculateMidPoints::GetCenter() const +{ + if (m_allCount == 0) + return {}; + + return m_midAll / m_allCount; +} + +void CalculateMidPoints::Sort() +{ + sort(m_vec.begin(), m_vec.end(), my::LessBy(&CellAndOffset::first)); +} +} // namespace feature diff --git a/generator/feature_helpers.hpp b/generator/feature_helpers.hpp new file mode 100644 index 0000000000..0ab9879756 --- /dev/null +++ b/generator/feature_helpers.hpp @@ -0,0 +1,101 @@ +#pragma once + +#include "geometry/distance.hpp" +#include "geometry/point2d.hpp" +#include "geometry/simplification.hpp" + +#include "indexer/coding_params.hpp" +#include "indexer/scales.hpp" + +#include "base/assert.hpp" +#include "base/math.hpp" + +#include <cstdint> +#include <limits> +#include <utility> +#include <vector> + +class FeatureBuilder1; + +namespace feature +{ +class CalculateMidPoints +{ +public: + using CellAndOffset = std::pair<uint64_t, uint64_t>; + + CalculateMidPoints() = default; + + void operator()(FeatureBuilder1 const & ft, uint64_t pos); + bool operator()(m2::PointD const & p); + + m2::PointD GetCenter() const; + std::vector<CellAndOffset> const & GetVector() const { return m_vec; } + + void Sort(); + +private: + m2::PointD m_midLoc; + m2::PointD m_midAll; + size_t m_locCount = 0; + size_t m_allCount = 0; + uint32_t m_coordBits = serial::CodingParams().GetCoordBits(); + std::vector<CellAndOffset> m_vec; +}; + +template <class Point> +inline bool ArePointsEqual(Point const & p1, Point const & p2) +{ + return p1 == p2; +} + +template <> +inline bool ArePointsEqual<m2::PointD>(m2::PointD const & p1, m2::PointD const & p2) +{ + return AlmostEqualULPs(p1, p2); +} + +class BoundsDistance : public m2::DistanceToLineSquare<m2::PointD> +{ +public: + explicit BoundsDistance(m2::RectD const & rect) : m_rect(rect) {} + + double GetEpsilon() const { return m_eps; } + + double operator()(m2::PointD const & p) const + { + if (my::AlmostEqualAbs(p.x, m_rect.minX(), m_eps) || + my::AlmostEqualAbs(p.x, m_rect.maxX(), m_eps) || + my::AlmostEqualAbs(p.y, m_rect.minY(), m_eps) || + my::AlmostEqualAbs(p.y, m_rect.maxY(), m_eps)) + { + // Points near rect should be in a result simplified vector. + return std::numeric_limits<double>::max(); + } + + return m2::DistanceToLineSquare<m2::PointD>::operator()(p); + } + +private: + m2::RectD const & m_rect; + // 5.0E-7 is near with minimal epsilon when integer points are different + // PointD2PointU(x, y) != PointD2PointU(x + m_eps, y + m_eps) + double m_eps = 5.0E-7; +}; + +template <class Distance, class PointsContainer> +void SimplifyPoints(Distance dist, int level, PointsContainer const & in, PointsContainer & out) +{ + if (in.size() < 2) + return; + + double const eps = my::sq(scales::GetEpsilonForSimplify(level)); + + SimplifyNearOptimal(20, in.begin(), in.end(), eps, dist, + AccumulateSkipSmallTrg<Distance, m2::PointD>(dist, out, eps)); + + CHECK_GREATER(out.size(), 1, ()); + CHECK(ArePointsEqual(in.front(), out.front()), ()); + CHECK(ArePointsEqual(in.back(), out.back()), ()); +} +} // namespace feature diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index ff27a8d3ad..47c63cf4b8 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -1,7 +1,9 @@ #include "generator/feature_sorter.hpp" + #include "generator/feature_builder.hpp" #include "generator/feature_generator.hpp" #include "generator/gen_mwm_info.hpp" +#include "generator/geometry_holder.hpp" #include "generator/region_meta.hpp" #include "generator/tesselator.hpp" @@ -28,62 +30,11 @@ #include "base/scope_guard.hpp" #include "base/string_utils.hpp" -namespace -{ -typedef pair<uint64_t, uint64_t> CellAndOffsetT; - -class CalculateMidPoints -{ - m2::PointD m_midLoc, m_midAll; - size_t m_locCount, m_allCount; - uint32_t m_coordBits; - -public: - CalculateMidPoints() - : m_midAll(0, 0), m_allCount(0), m_coordBits(serial::CodingParams().GetCoordBits()) - { - } - - vector<CellAndOffsetT> m_vec; - - void operator()(FeatureBuilder1 const & ft, uint64_t pos) - { - // reset state - m_midLoc = m2::PointD(0, 0); - m_locCount = 0; - - ft.ForEachGeometryPoint(*this); - m_midLoc = m_midLoc / m_locCount; - - uint64_t const pointAsInt64 = PointToInt64(m_midLoc, m_coordBits); - int const minScale = feature::GetMinDrawableScale(ft.GetFeatureBase()); - - /// May be invisible if it's small area object with [0-9] scales. - /// @todo Probably, we need to keep that objects if 9 scale (as we do in 17 scale). - if (minScale != -1 || feature::RequireGeometryInIndex(ft.GetFeatureBase())) - { - uint64_t const order = (static_cast<uint64_t>(minScale) << 59) | (pointAsInt64 >> 5); - m_vec.push_back(make_pair(order, pos)); - } - } - - bool operator()(m2::PointD const & p) - { - m_midLoc += p; - m_midAll += p; - ++m_locCount; - ++m_allCount; - return true; - } - - m2::PointD GetCenter() const { return m_midAll / m_allCount; } -}; +#include <list> +#include <memory> +#include <vector> -bool SortMidPointsFunc(CellAndOffsetT const & c1, CellAndOffsetT const & c2) -{ - return c1.first < c2.first; -} -} // namespace +using namespace std; namespace feature { @@ -96,7 +47,7 @@ class FeaturesCollector2 : public FeaturesCollector class TmpFile : public FileWriter { public: - explicit TmpFile(std::string const & filePath) : FileWriter(filePath) {} + explicit TmpFile(string const & filePath) : FileWriter(filePath) {} ~TmpFile() { DeleteFileX(GetName()); } }; @@ -112,7 +63,7 @@ class FeaturesCollector2 : public FeaturesCollector TmpFiles m_helperFile; // Mapping from feature id to offset in file section with the correspondent metadata. - vector<pair<uint32_t, uint32_t>> m_metadataIndex; + vector<pair<uint32_t, uint32_t>> m_metadataOffset; DataHeader m_header; RegionData m_regionData; @@ -121,8 +72,8 @@ class FeaturesCollector2 : public FeaturesCollector gen::OsmID2FeatureID m_osm2ft; public: - FeaturesCollector2(std::string const & fName, DataHeader const & header, - RegionData const & regionData, uint32_t versionDate) + FeaturesCollector2(string const & fName, DataHeader const & header, RegionData const & regionData, + uint32_t versionDate) : FeaturesCollector(fName + DATA_FILE_TAG) , m_writer(fName) , m_header(header) @@ -131,7 +82,7 @@ public: { for (size_t i = 0; i < m_header.GetScalesCount(); ++i) { - std::string const postfix = strings::to_string(i); + string const postfix = strings::to_string(i); m_geoFile.push_back(make_unique<TmpFile>(fName + GEOMETRY_FILE_TAG + postfix)); m_trgFile.push_back(make_unique<TmpFile>(fName + TRIANGLE_FILE_TAG + postfix)); } @@ -168,15 +119,15 @@ public: m_writer.Write(m_datFile.GetName(), DATA_FILE_TAG); // File Writer finalization function with appending to the main mwm file. - auto const finalizeFn = [this](unique_ptr<TmpFile> w, std::string const & tag, - std::string const & postfix = std::string()) { + auto const finalizeFn = [this](unique_ptr<TmpFile> w, string const & tag, + string const & postfix = string()) { w->Flush(); m_writer.Write(w->GetName(), tag + postfix); }; for (size_t i = 0; i < m_header.GetScalesCount(); ++i) { - std::string const postfix = strings::to_string(i); + string const postfix = strings::to_string(i); finalizeFn(move(m_geoFile[i]), GEOMETRY_FILE_TAG, postfix); finalizeFn(move(m_trgFile[i]), TRIANGLE_FILE_TAG, postfix); } @@ -184,7 +135,7 @@ public: { /// @todo Replace this mapping vector with succint structure. FileWriter w = m_writer.GetWriter(METADATA_INDEX_FILE_TAG); - for (auto const & v : m_metadataIndex) + for (auto const & v : m_metadataOffset) { WriteToSink(w, v.first); WriteToSink(w, v.second); @@ -207,211 +158,18 @@ private: typedef vector<m2::PointD> points_t; typedef list<points_t> polygons_t; - class GeometryHolder - { - public: - FeatureBuilder2::SupportingData m_buffer; - - private: - FeaturesCollector2 & m_rMain; - FeatureBuilder2 & m_rFB; - - points_t m_current; - - DataHeader const & m_header; - - void WriteOuterPoints(points_t const & points, int i) - { - // outer path can have 2 points in small scale levels - ASSERT_GREATER(points.size(), 1, ()); - - serial::CodingParams cp = m_header.GetCodingParams(i); - - // Optimization: Store first point once in header for outer linear features. - cp.SetBasePoint(points[0]); - // Can optimize here, but ... Make copy of vector. - points_t toSave(points.begin() + 1, points.end()); - - m_buffer.m_ptsMask |= (1 << i); - m_buffer.m_ptsOffset.push_back(m_rMain.GetFileSize(*m_rMain.m_geoFile[i])); - serial::SaveOuterPath(toSave, cp, *m_rMain.m_geoFile[i]); - } - - void WriteOuterTriangles(polygons_t const & polys, int i) - { - // tesselation - tesselator::TrianglesInfo info; - if (0 == tesselator::TesselateInterior(polys, info)) - { - LOG(LINFO, ("NO TRIANGLES in", polys)); - return; - } - - serial::CodingParams const cp = m_header.GetCodingParams(i); - - serial::TrianglesChainSaver saver(cp); - - // points conversion - tesselator::PointsInfo points; - m2::PointU (*D2U)(m2::PointD const &, uint32_t) = &PointD2PointU; - info.GetPointsInfo(saver.GetBasePoint(), saver.GetMaxPoint(), - std::bind(D2U, std::placeholders::_1, cp.GetCoordBits()), points); - - // triangles processing (should be optimal) - info.ProcessPortions(points, saver, true); - - // check triangles processing (to compare with optimal) - // serial::TrianglesChainSaver checkSaver(cp); - // info.ProcessPortions(points, checkSaver, false); - - // CHECK_LESS_OR_EQUAL(saver.GetBufferSize(), checkSaver.GetBufferSize(), ()); - - // saving to file - m_buffer.m_trgMask |= (1 << i); - m_buffer.m_trgOffset.push_back(m_rMain.GetFileSize(*m_rMain.m_trgFile[i])); - saver.Save(*m_rMain.m_trgFile[i]); - } - - void FillInnerPointsMask(points_t const & points, uint32_t scaleIndex) - { - points_t const & src = m_buffer.m_innerPts; - CHECK(!src.empty(), ()); - - CHECK(are_points_equal(src.front(), points.front()), ()); - CHECK(are_points_equal(src.back(), points.back()), ()); - - size_t j = 1; - for (size_t i = 1; i < points.size() - 1; ++i) - { - for (; j < src.size() - 1; ++j) - { - if (are_points_equal(src[j], points[i])) - { - // set corresponding 2 bits for source point [j] to scaleIndex - uint32_t mask = 0x3; - m_buffer.m_ptsSimpMask &= ~(mask << (2 * (j - 1))); - m_buffer.m_ptsSimpMask |= (scaleIndex << (2 * (j - 1))); - break; - } - } - - CHECK_LESS(j, src.size() - 1, ("Simplified point not found in source point array")); - } - } - - bool m_ptsInner, m_trgInner; - - class strip_emitter - { - points_t const & m_src; - points_t & m_dest; - - public: - strip_emitter(points_t const & src, points_t & dest) : m_src(src), m_dest(dest) - { - m_dest.reserve(m_src.size()); - } - void operator()(size_t i) { m_dest.push_back(m_src[i]); } - }; - - public: - GeometryHolder(FeaturesCollector2 & rMain, FeatureBuilder2 & fb, DataHeader const & header) - : m_rMain(rMain), m_rFB(fb), m_header(header), m_ptsInner(true), m_trgInner(true) - { - } - - points_t const & GetSourcePoints() - { - return (!m_current.empty() ? m_current : m_rFB.GetOuterGeometry()); - } - - void AddPoints(points_t const & points, int scaleIndex) - { - if (m_ptsInner && points.size() < 15) - { - if (m_buffer.m_innerPts.empty()) - m_buffer.m_innerPts = points; - else - FillInnerPointsMask(points, scaleIndex); - m_current = points; - } - else - { - m_ptsInner = false; - WriteOuterPoints(points, scaleIndex); - } - } - - bool NeedProcessTriangles() const { return (!m_trgInner || m_buffer.m_innerTrg.empty()); } - - bool TryToMakeStrip(points_t & points) - { - size_t const count = points.size(); - if (!m_trgInner || count > 15 + 2) - { - // too many points for strip - m_trgInner = false; - return false; - } - - if (m2::robust::CheckPolygonSelfIntersections(points.begin(), points.end())) - { - // polygon has self-intersectios - m_trgInner = false; - return false; - } - - CHECK(m_buffer.m_innerTrg.empty(), ()); - - // make CCW orientation for polygon - if (!IsPolygonCCW(points.begin(), points.end())) - { - reverse(points.begin(), points.end()); - - // Actually this check doesn't work for some degenerate polygons. - // See IsPolygonCCW_DataSet tests for more details. - // ASSERT(IsPolygonCCW(points.begin(), points.end()), (points)); - if (!IsPolygonCCW(points.begin(), points.end())) - return false; - } - - size_t const index = FindSingleStrip( - count, IsDiagonalVisibleFunctor<points_t::const_iterator>(points.begin(), points.end())); - - if (index == count) - { - // can't find strip - m_trgInner = false; - return false; - } - - MakeSingleStripFromIndex(index, count, strip_emitter(points, m_buffer.m_innerTrg)); - - CHECK_EQUAL(count, m_buffer.m_innerTrg.size(), ()); - return true; - } - - void AddTriangles(polygons_t const & polys, int scaleIndex) - { - CHECK(m_buffer.m_innerTrg.empty(), ()); - m_trgInner = false; - - WriteOuterTriangles(polys, scaleIndex); - } - }; - - void SimplifyPoints(points_t const & in, points_t & out, int level, bool isCoast, - m2::RectD const & rect) + void SimplifyPoints(int level, bool isCoast, m2::RectD const & rect, points_t const & in, + points_t & out) { if (isCoast) { BoundsDistance dist(rect); - feature::SimplifyPoints(dist, in, out, level); + feature::SimplifyPoints(dist, level, in, out); } else { m2::DistanceToLineSquare<m2::PointD> dist; - feature::SimplifyPoints(dist, in, out, level); + feature::SimplifyPoints(dist, level, in, out); } } @@ -419,16 +177,16 @@ private: { ASSERT(poly.front() == poly.back(), ()); - double res = 0; + double res = 0.0; for (size_t i = 0; i < poly.size() - 1; ++i) - { res += (poly[i + 1].x - poly[i].x) * (poly[i + 1].y + poly[i].y) / 2.0; - } return fabs(res); } static bool IsGoodArea(points_t const & poly, int level) { + // Area has the same first and last points. That's why minimal number of points for + // area is 4. if (poly.size() < 4) return false; @@ -443,7 +201,8 @@ private: public: uint32_t operator()(FeatureBuilder2 & fb) { - GeometryHolder holder(*this, fb, m_header); + GeometryHolder holder([this](int i) -> FileWriter & { return *m_geoFile[i]; }, + [this](int i) -> FileWriter & { return *m_trgFile[i]; }, fb, m_header); bool const isLine = fb.IsLine(); bool const isArea = fb.IsArea(); @@ -464,7 +223,7 @@ public: if (isLine && i == scalesStart && IsCountry() && fb.IsRoad()) points = holder.GetSourcePoints(); else - SimplifyPoints(holder.GetSourcePoints(), points, level, isCoast, rect); + SimplifyPoints(level, isCoast, rect, holder.GetSourcePoints(), points); if (isLine) holder.AddPoints(points, i); @@ -475,6 +234,7 @@ public: bool const good = isCoast || IsGoodArea(points, level); // At this point we don't need last point equal to first. + CHECK_GREATER(points.size(), 0, ()); points.pop_back(); polygons_t const & polys = fb.GetGeometry(); @@ -493,13 +253,14 @@ public: { simplified.push_back(points_t()); - SimplifyPoints(*iH, simplified.back(), level, isCoast, rect); + SimplifyPoints(level, isCoast, rect, *iH, simplified.back()); // Increment level check for coastline polygons for the first scale level. // This is used for better coastlines quality. if (IsGoodArea(simplified.back(), (isCoast && i == 0) ? level + 1 : level)) { // At this point we don't need last point equal to first. + CHECK_GREATER(simplified.back().size(), 0, ()); simplified.back().pop_back(); } else @@ -516,11 +277,12 @@ public: } uint32_t featureId = kInvalidFeatureId; - if (fb.PreSerialize(holder.m_buffer)) + auto & buffer = holder.GetBuffer(); + if (fb.PreSerialize(buffer)) { - fb.Serialize(holder.m_buffer, m_header.GetDefCodingParams()); + fb.Serialize(buffer, m_header.GetDefCodingParams()); - featureId = WriteFeatureBase(holder.m_buffer.m_buffer, fb); + featureId = WriteFeatureBase(buffer.m_buffer, fb); fb.GetAddressData().Serialize(*(m_helperFile[SEARCH_TOKENS])); @@ -531,7 +293,7 @@ public: uint64_t const offset = w->Pos(); ASSERT_LESS_OR_EQUAL(offset, numeric_limits<uint32_t>::max(), ()); - m_metadataIndex.emplace_back(featureId, static_cast<uint32_t>(offset)); + m_metadataOffset.emplace_back(featureId, static_cast<uint32_t>(offset)); fb.GetMetadata().Serialize(*w); } @@ -551,18 +313,17 @@ FeatureBuilder2 & GetFeatureBuilder2(FeatureBuilder1 & fb) return static_cast<FeatureBuilder2 &>(fb); } -bool GenerateFinalFeatures(feature::GenerateInfo const & info, std::string const & name, - int mapType) +bool GenerateFinalFeatures(feature::GenerateInfo const & info, string const & name, int mapType) { - std::string const srcFilePath = info.GetTmpFileName(name); - std::string const datFilePath = info.GetTargetFileName(name); + string const srcFilePath = info.GetTmpFileName(name); + string const datFilePath = info.GetTargetFileName(name); // stores cellIds for middle points CalculateMidPoints midPoints; ForEachFromDatRawFormat(srcFilePath, midPoints); // sort features by their middle point - sort(midPoints.m_vec.begin(), midPoints.m_vec.end(), &SortMidPointsFunc); + midPoints.Sort(); // store sorted features { @@ -599,10 +360,10 @@ bool GenerateFinalFeatures(feature::GenerateInfo const & info, std::string const { FeaturesCollector2 collector(datFilePath, header, regionData, info.m_versionDate); - for (size_t i = 0; i < midPoints.m_vec.size(); ++i) + for (auto const & point : midPoints.GetVector()) { ReaderSource<FileReader> src(reader); - src.Skip(midPoints.m_vec[i].second); + src.Skip(point.second); FeatureBuilder1 f; ReadFromSourceRowFormat(src, f); diff --git a/generator/feature_sorter.hpp b/generator/feature_sorter.hpp index cd59a86ff0..58fd764816 100644 --- a/generator/feature_sorter.hpp +++ b/generator/feature_sorter.hpp @@ -1,12 +1,6 @@ #pragma once #include "generator/generate_info.hpp" -#include "geometry/distance.hpp" -#include "geometry/point2d.hpp" -#include "geometry/simplification.hpp" - -#include "indexer/scales.hpp" - #include <string> namespace feature @@ -16,59 +10,4 @@ namespace feature /// @param name - name of generated country; bool GenerateFinalFeatures(feature::GenerateInfo const & info, std::string const & name, int mapType); - -template <class PointT> -inline bool are_points_equal(PointT const & p1, PointT const & p2) -{ - return p1 == p2; -} - -template <> -inline bool are_points_equal<m2::PointD>(m2::PointD const & p1, m2::PointD const & p2) -{ - return AlmostEqualULPs(p1, p2); -} - -class BoundsDistance : public m2::DistanceToLineSquare<m2::PointD> -{ - m2::RectD const & m_rect; - double m_eps; - -public: - BoundsDistance(m2::RectD const & rect) : m_rect(rect), m_eps(5.0E-7) - { - // 5.0E-7 is near with minimal epsilon when integer points are different - // PointD2PointU(x, y) != PointD2PointU(x + m_eps, y + m_eps) - } - - double GetEpsilon() const { return m_eps; } - - double operator()(m2::PointD const & p) const - { - if (fabs(p.x - m_rect.minX()) <= m_eps || fabs(p.x - m_rect.maxX()) <= m_eps || - fabs(p.y - m_rect.minY()) <= m_eps || fabs(p.y - m_rect.maxY()) <= m_eps) - { - // points near rect should be in a result simplified vector - return std::numeric_limits<double>::max(); - } - - return m2::DistanceToLineSquare<m2::PointD>::operator()(p); - } -}; - -template <class DistanceT, class PointsContainerT> -void SimplifyPoints(DistanceT dist, PointsContainerT const & in, PointsContainerT & out, int level) -{ - if (in.size() >= 2) - { - double const eps = my::sq(scales::GetEpsilonForSimplify(level)); - - SimplifyNearOptimal(20, in.begin(), in.end(), eps, dist, - AccumulateSkipSmallTrg<DistanceT, m2::PointD>(dist, out, eps)); - - CHECK_GREATER(out.size(), 1, ()); - CHECK(are_points_equal(in.front(), out.front()), ()); - CHECK(are_points_equal(in.back(), out.back()), ()); - } -} } // namespace feature diff --git a/generator/generator_tests/coasts_test.cpp b/generator/generator_tests/coasts_test.cpp index 1a4a7711f1..309cf9a86a 100644 --- a/generator/generator_tests/coasts_test.cpp +++ b/generator/generator_tests/coasts_test.cpp @@ -1,8 +1,8 @@ #include "testing/testing.hpp" #include "generator/feature_builder.hpp" -#include "generator/feature_sorter.hpp" #include "generator/feature_generator.hpp" +#include "generator/feature_helpers.hpp" #include "geometry/mercator.hpp" #include "indexer/cell_id.hpp" @@ -148,7 +148,7 @@ namespace for (PolygonsT::const_iterator i = poly.begin(); i != poly.end(); ++i) { PointsT pts; - feature::SimplifyPoints(dist, *i, pts, level); + feature::SimplifyPoints(dist, level, *i, pts); LOG(LINFO, ("Simplified. Level = ", level, "Points = ", pts)); diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 89c4034eb1..7842957c95 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -8,6 +8,7 @@ #include "generator/feature_generator.hpp" #include "generator/feature_sorter.hpp" #include "generator/generate_info.hpp" +#include "generator/locality_sorter.hpp" #include "generator/metalines_builder.hpp" #include "generator/osm_source.hpp" #include "generator/restriction_generator.hpp" @@ -29,6 +30,7 @@ #include "indexer/features_offsets_table.hpp" #include "indexer/features_vector.hpp" #include "indexer/index_builder.hpp" +#include "indexer/locality_index_builder.hpp" #include "indexer/map_style_reader.hpp" #include "indexer/rank_table.hpp" @@ -40,20 +42,21 @@ #include "base/timer.hpp" -#include "std/unique_ptr.hpp" - #include <cstdlib> +#include <memory> #include <string> #include "defines.hpp" #include "3party/gflags/src/gflags/gflags.h" +using namespace std; + namespace { char const * GetDataPathHelp() { - static std::string const kHelp = + static string const kHelp = "Directory where the generated mwms are put into. Also used as the path for helper " "functions, such as those that calculate statistics and regenerate sections. " "Default: " + @@ -88,6 +91,7 @@ DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry and triangles for features."); DEFINE_bool(generate_index, false, "4rd pass - generate index."); DEFINE_bool(generate_search_index, false, "5th pass - generate search index."); +DEFINE_bool(generate_locality_index, false, "3rd pass - generate locality objects and locality index."); DEFINE_bool(dump_cities_boundaries, false, "Dump cities boundaries to a file"); DEFINE_bool(generate_cities_boundaries, false, "Generate cities boundaries section"); @@ -155,7 +159,7 @@ int main(int argc, char ** argv) pl.SetSettingsDir(FLAGS_user_resource_path); } - std::string const path = + string const path = FLAGS_data_path.empty() ? pl.WritableDir() : my::AddSlashIfNeeded(FLAGS_data_path); // So that stray GetWritablePathForFile calls do not crash the generator. @@ -169,7 +173,7 @@ int main(int argc, char ** argv) /// @todo Probably, it's better to add separate option for .mwm.tmp files. if (!FLAGS_intermediate_data_path.empty()) { - std::string const tmpPath = genInfo.m_intermediateDir + "tmp" + my::GetNativeSeparator(); + string const tmpPath = my::JoinPath(genInfo.m_intermediateDir, "tmp"); if (Platform::MkDir(tmpPath) != Platform::ERR_UNKNOWN) genInfo.m_tmpDir = tmpPath; } @@ -205,7 +209,7 @@ int main(int argc, char ** argv) GetStyleReader().SetCurrentStyle(MapStyleMerged); // Load classificator only when necessary. - if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry || + if (FLAGS_make_coasts || FLAGS_generate_features || FLAGS_generate_geometry || FLAGS_generate_locality_index || FLAGS_generate_index || FLAGS_generate_search_index || FLAGS_generate_cities_boundaries || FLAGS_calc_statistics || FLAGS_type_statistics || FLAGS_dump_types || FLAGS_dump_prefixes || FLAGS_dump_feature_names != "" || FLAGS_check_mwm || FLAGS_srtm_path != "" || @@ -217,7 +221,7 @@ int main(int argc, char ** argv) } // Load mwm tree only if we need it - std::unique_ptr<storage::CountryParentGetter> countryParentGetter; + unique_ptr<storage::CountryParentGetter> countryParentGetter; if (FLAGS_make_routing_index || FLAGS_make_cross_mwm || FLAGS_make_transit_cross_mwm) countryParentGetter = make_unique<storage::CountryParentGetter>(); @@ -259,13 +263,39 @@ int main(int argc, char ** argv) genInfo.m_bucketNames.push_back(FLAGS_output); } + if (FLAGS_generate_locality_index) + { + if (FLAGS_output.empty() || FLAGS_intermediate_data_path.empty()) + { + LOG(LCRITICAL, ("Bad output or intermediate_data_path. Output:", FLAGS_output, + "intermediate_data_path:", FLAGS_intermediate_data_path)); + return -1; + } + + auto const locDataFile = my::JoinPath(path, FLAGS_output + LOC_DATA_FILE_EXTENSION); + auto const outFile = my::JoinPath(path, FLAGS_output + LOC_IDX_FILE_EXTENSION); + if (!feature::GenerateLocalityData(genInfo.m_tmpDir, locDataFile)) + { + LOG(LCRITICAL, ("Error generating locality data.")); + return -1; + } + + LOG(LINFO, ("Saving locality index to", outFile)); + + if (!indexer::BuildLocalityIndexFromDataFile(locDataFile, outFile)) + { + LOG(LCRITICAL, ("Error generating locality index.")); + return -1; + } + } + // Enumerate over all dat files that were created. size_t const count = genInfo.m_bucketNames.size(); for (size_t i = 0; i < count; ++i) { - std::string const & country = genInfo.m_bucketNames[i]; - std::string const datFile = my::JoinFoldersToPath(path, country + DATA_FILE_EXTENSION); - std::string const osmToFeatureFilename = + string const & country = genInfo.m_bucketNames[i]; + string const datFile = my::JoinPath(path, country + DATA_FILE_EXTENSION); + string const osmToFeatureFilename = genInfo.GetTargetFileName(country) + OSM2FEATURE_FILE_EXTENSION; if (FLAGS_generate_geometry) @@ -288,7 +318,7 @@ int main(int argc, char ** argv) if (mapType == feature::DataHeader::country) { - std::string const metalinesFilename = + string const metalinesFilename = genInfo.GetIntermediateFileName(METALINES_FILENAME, "" /* extension */); LOG(LINFO, ("Processing metalines from", metalinesFilename)); @@ -348,9 +378,9 @@ int main(int argc, char ** argv) return -1; } - std::string const restrictionsFilename = + string const restrictionsFilename = genInfo.GetIntermediateFileName(RESTRICTIONS_FILENAME, "" /* extension */); - std::string const roadAccessFilename = + string const roadAccessFilename = genInfo.GetIntermediateFileName(ROAD_ACCESS_FILENAME, "" /* extension */); routing::BuildRoadRestrictions(datFile, restrictionsFilename, osmToFeatureFilename); @@ -393,7 +423,7 @@ int main(int argc, char ** argv) } } - std::string const datFile = my::JoinFoldersToPath(path, FLAGS_output + DATA_FILE_EXTENSION); + string const datFile = my::JoinPath(path, FLAGS_output + DATA_FILE_EXTENSION); if (FLAGS_calc_statistics) { diff --git a/generator/geometry_holder.hpp b/generator/geometry_holder.hpp new file mode 100644 index 0000000000..44aab7b1bf --- /dev/null +++ b/generator/geometry_holder.hpp @@ -0,0 +1,249 @@ +#pragma once + +#include "generator/feature_builder.hpp" +#include "generator/feature_generator.hpp" +#include "generator/feature_helpers.hpp" +#include "generator/tesselator.hpp" + +#include "geometry/distance.hpp" +#include "geometry/point2d.hpp" +#include "geometry/polygon.hpp" +#include "geometry/simplification.hpp" + +#include "indexer/data_header.hpp" +#include "indexer/geometry_serialization.hpp" + +#include <cstdint> +#include <functional> +#include <list> +#include <vector> + +namespace feature +{ +class GeometryHolder +{ +public: + using FileGetter = std::function<FileWriter &(int i)>; + using Points = std::vector<m2::PointD>; + using Polygons = std::list<Points>; + + // For FeatureType serialization maxNumTriangles should be less than numeric_limits<uint8_t>::max + // because FeatureType format uses uint8_t to encode the number of triangles. + GeometryHolder(FileGetter geoFileGetter, FileGetter trgFileGetter, FeatureBuilder2 & fb, + feature::DataHeader const & header, size_t maxNumTriangles = 14) + : m_geoFileGetter(geoFileGetter) + , m_trgFileGetter(trgFileGetter) + , m_fb(fb) + , m_ptsInner(true) + , m_trgInner(true) + , m_header(header) + , m_maxNumTriangles(maxNumTriangles) + { + } + + GeometryHolder(FeatureBuilder2 & fb, feature::DataHeader const & header, + size_t maxNumTriangles = 14) + : m_fb(fb) + , m_ptsInner(true) + , m_trgInner(true) + , m_header(header) + , m_maxNumTriangles(maxNumTriangles) + { + } + + void SetInner() { m_trgInner = true; } + + FeatureBuilder2::SupportingData & GetBuffer() { return m_buffer; } + + Points const & GetSourcePoints() + { + return !m_current.empty() ? m_current : m_fb.GetOuterGeometry(); + } + + void AddPoints(Points const & points, int scaleIndex) + { + if (m_ptsInner && points.size() <= m_maxNumTriangles) + { + if (m_buffer.m_innerPts.empty()) + m_buffer.m_innerPts = points; + else + FillInnerPointsMask(points, scaleIndex); + m_current = points; + } + else + { + m_ptsInner = false; + WriteOuterPoints(points, scaleIndex); + } + } + + bool NeedProcessTriangles() const { return !m_trgInner || m_buffer.m_innerTrg.empty(); } + + bool TryToMakeStrip(Points & points) + { + size_t const count = points.size(); + if (!m_trgInner || (count >= 2 && count - 2 > m_maxNumTriangles)) + { + // too many points for strip + m_trgInner = false; + return false; + } + + if (m2::robust::CheckPolygonSelfIntersections(points.begin(), points.end())) + { + // polygon has self-intersectios + m_trgInner = false; + return false; + } + + CHECK(m_buffer.m_innerTrg.empty(), ()); + + // make CCW orientation for polygon + if (!IsPolygonCCW(points.begin(), points.end())) + { + reverse(points.begin(), points.end()); + + // Actually this check doesn't work for some degenerate polygons. + // See IsPolygonCCW_DataSet tests for more details. + // ASSERT(IsPolygonCCW(points.begin(), points.end()), (points)); + if (!IsPolygonCCW(points.begin(), points.end())) + return false; + } + + size_t const index = FindSingleStrip( + count, IsDiagonalVisibleFunctor<Points::const_iterator>(points.begin(), points.end())); + + if (index == count) + { + // can't find strip + m_trgInner = false; + return false; + } + + MakeSingleStripFromIndex(index, count, StripEmitter(points, m_buffer.m_innerTrg)); + + CHECK_EQUAL(count, m_buffer.m_innerTrg.size(), ()); + return true; + } + + void AddTriangles(Polygons const & polys, int scaleIndex) + { + CHECK(m_buffer.m_innerTrg.empty(), ()); + m_trgInner = false; + + WriteOuterTriangles(polys, scaleIndex); + } + +private: + class StripEmitter + { + public: + StripEmitter(Points const & src, Points & dest) : m_src(src), m_dest(dest) + { + m_dest.reserve(m_src.size()); + } + void operator()(size_t i) { m_dest.push_back(m_src[i]); } + + private: + Points const & m_src; + Points & m_dest; + }; + + void WriteOuterPoints(Points const & points, int i) + { + CHECK(m_geoFileGetter, ("m_geoFileGetter must be set to write outer points.")); + + // outer path can have 2 points in small scale levels + ASSERT_GREATER(points.size(), 1, ()); + + auto cp = m_header.GetCodingParams(i); + + // Optimization: Store first point once in header for outer linear features. + cp.SetBasePoint(points[0]); + // Can optimize here, but ... Make copy of vector. + Points toSave(points.begin() + 1, points.end()); + + m_buffer.m_ptsMask |= (1 << i); + m_buffer.m_ptsOffset.push_back(feature::FeaturesCollector::GetFileSize(m_geoFileGetter(i))); + serial::SaveOuterPath(toSave, cp, m_geoFileGetter(i)); + } + + void WriteOuterTriangles(Polygons const & polys, int i) + { + CHECK(m_trgFileGetter, ("m_trgFileGetter must be set to write outer triangles.")); + + // tesselation + tesselator::TrianglesInfo info; + if (0 == tesselator::TesselateInterior(polys, info)) + { + LOG(LINFO, ("NO TRIANGLES in", polys)); + return; + } + + auto const cp = m_header.GetCodingParams(i); + + serial::TrianglesChainSaver saver(cp); + + // points conversion + tesselator::PointsInfo points; + m2::PointU (*D2U)(m2::PointD const &, uint32_t) = &PointD2PointU; + info.GetPointsInfo(saver.GetBasePoint(), saver.GetMaxPoint(), + std::bind(D2U, std::placeholders::_1, cp.GetCoordBits()), points); + + // triangles processing (should be optimal) + info.ProcessPortions(points, saver, true); + + // check triangles processing (to compare with optimal) + // serial::TrianglesChainSaver checkSaver(cp); + // info.ProcessPortions(points, checkSaver, false); + + // CHECK_LESS_OR_EQUAL(saver.GetBufferSize(), checkSaver.GetBufferSize(), ()); + + // saving to file + m_buffer.m_trgMask |= (1 << i); + m_buffer.m_trgOffset.push_back(feature::FeaturesCollector::GetFileSize(m_trgFileGetter(i))); + saver.Save(m_trgFileGetter(i)); + } + + void FillInnerPointsMask(Points const & points, uint32_t scaleIndex) + { + auto const & src = m_buffer.m_innerPts; + CHECK(!src.empty(), ()); + + CHECK(feature::ArePointsEqual(src.front(), points.front()), ()); + CHECK(feature::ArePointsEqual(src.back(), points.back()), ()); + + size_t j = 1; + for (size_t i = 1; i < points.size() - 1; ++i) + { + for (; j < src.size() - 1; ++j) + { + if (feature::ArePointsEqual(src[j], points[i])) + { + // set corresponding 2 bits for source point [j] to scaleIndex + uint32_t mask = 0x3; + m_buffer.m_ptsSimpMask &= ~(mask << (2 * (j - 1))); + m_buffer.m_ptsSimpMask |= (scaleIndex << (2 * (j - 1))); + break; + } + } + + CHECK_LESS(j, src.size() - 1, ("Simplified point is not found in the source point array.")); + } + } + + FileGetter m_geoFileGetter = {}; + FileGetter m_trgFileGetter = {}; + + FeatureBuilder2 & m_fb; + + FeatureBuilder2::SupportingData m_buffer; + + Points m_current; + bool m_ptsInner, m_trgInner; + + feature::DataHeader const & m_header; + // max triangles number to store in innerTriangles + size_t m_maxNumTriangles; +}; +} // namespace feature diff --git a/generator/locality_sorter.cpp b/generator/locality_sorter.cpp new file mode 100644 index 0000000000..8c4751edb3 --- /dev/null +++ b/generator/locality_sorter.cpp @@ -0,0 +1,176 @@ +#include "generator/locality_sorter.hpp" + +#include "generator/geometry_holder.hpp" + +#include "indexer/data_header.hpp" +#include "indexer/geometry_serialization.hpp" +#include "indexer/scales.hpp" +#include "indexer/scales_patch.hpp" + +#include "coding/file_container.hpp" +#include "coding/file_name_utils.hpp" +#include "coding/internal/file_data.hpp" + +#include "geometry/convex_hull.hpp" + +#include "platform/platform.hpp" + +#include "base/assert.hpp" +#include "base/logging.hpp" +#include "base/scope_guard.hpp" +#include "base/timer.hpp" + +#include "defines.hpp" + +#include <limits> +#include <vector> + +using namespace feature; +using namespace std; + +class LocalityCollector : public FeaturesCollector +{ + DISALLOW_COPY_AND_MOVE(LocalityCollector); + + FilesContainerW m_writer; + DataHeader m_header; + uint32_t m_versionDate; + +public: + LocalityCollector(string const & fName, DataHeader const & header, uint32_t versionDate) + : FeaturesCollector(fName + EXTENSION_TMP) + , m_writer(fName) + , m_header(header) + , m_versionDate(versionDate) + { + } + + void Finish() + { + { + FileWriter w = m_writer.GetWriter(VERSION_FILE_TAG); + version::WriteVersion(w, m_versionDate); + } + + m_header.SetBounds(m_bounds); + { + FileWriter w = m_writer.GetWriter(HEADER_FILE_TAG); + m_header.Save(w); + } + + Flush(); + + m_writer.Write(m_datFile.GetName(), LOCALITY_DATA_FILE_TAG); + m_writer.Finish(); + } + + void operator()(FeatureBuilder2 & fb) + { + if (!fb.IsLocalityObject()) + return; + + // Do not limit inner triangles number to save all geometry without additional sections. + GeometryHolder holder(fb, m_header, numeric_limits<uint32_t>::max() /* maxTrianglesNumber */); + + // Simplify and serialize geometry. + vector<m2::PointD> points; + m2::DistanceToLineSquare<m2::PointD> dist; + + SimplifyPoints(dist, scales::GetUpperScale(), holder.GetSourcePoints(), points); + + // For areas we save outer geometry only. + if (fb.IsArea() && holder.NeedProcessTriangles()) + { + // At this point we don't need last point equal to first. + points.pop_back(); + auto const & polys = fb.GetGeometry(); + if (polys.size() != 1) + { + points.clear(); + for (auto const & poly : polys) + points.insert(points.end(), poly.begin(), poly.end()); + } + + if (points.size() > 2) + { + if (!holder.TryToMakeStrip(points)) + { + m2::ConvexHull hull(points, 1e-16); + vector<m2::PointD> hullPoints = hull.Points(); + holder.SetInner(); + auto const id = fb.GetMostGenericOsmId(); + CHECK(holder.TryToMakeStrip(hullPoints), + ("Error while building tringles for object with OSM Id:", id.OsmId(), + "Type:", id.IsRelation() ? "Relation" : "Way", "points:", points, + "hull:", hull.Points())); + } + } + } + + auto & buffer = holder.GetBuffer(); + if (fb.PreSerialize(buffer)) + { + fb.SerializeLocalityObject(serial::CodingParams(), buffer); + WriteFeatureBase(buffer.m_buffer, fb); + } + } +}; + +// Simplify geometry for the upper scale. +FeatureBuilder2 & GetFeatureBuilder2(FeatureBuilder1 & fb) +{ + return static_cast<FeatureBuilder2 &>(fb); +} + +namespace feature +{ +bool GenerateLocalityData(string const & featuresDir, string const & dataFilePath) +{ + DataHeader header; + header.SetCodingParams(serial::CodingParams()); + header.SetScales({scales::GetUpperScale()}); + + // Transform features from raw format to LocalityObject format. + try + { + LocalityCollector collector(dataFilePath, header, + static_cast<uint32_t>(my::SecondsSinceEpoch())); + + Platform::FilesList files; + Platform::GetFilesByExt(featuresDir, DATA_FILE_EXTENSION_TMP, files); + + for (auto const & fileName : files) + { + auto const file = my::JoinFoldersToPath(featuresDir, fileName); + LOG(LINFO, ("Processing", file)); + + CalculateMidPoints midPoints; + ForEachFromDatRawFormat(file, midPoints); + + // Sort features by their middle point. + midPoints.Sort(); + + FileReader reader(file); + for (auto const & point : midPoints.GetVector()) + { + ReaderSource<FileReader> src(reader); + src.Skip(point.second); + + FeatureBuilder1 f; + ReadFromSourceRowFormat(src, f); + // Emit object. + collector(GetFeatureBuilder2(f)); + } + } + + collector.Finish(); + } + catch (RootException const & ex) + { + LOG(LCRITICAL, ("Locality data writing error:", ex.Msg())); + return false; + } + + return true; +} +} // namespace feature diff --git a/generator/locality_sorter.hpp b/generator/locality_sorter.hpp new file mode 100644 index 0000000000..e8614c1e79 --- /dev/null +++ b/generator/locality_sorter.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include <string> + +namespace feature +{ +/// Generates data for LocalityIndexBuilder from input feature-dat-files. +/// @param featuresDir - path to folder with pregenerated features data; +/// @param out - output file name; +bool GenerateLocalityData(std::string const & featuresDir, std::string const & out); +} // namespace feature diff --git a/indexer/locality_index_builder.cpp b/indexer/locality_index_builder.cpp index 16dce7a3f3..784676cdef 100644 --- a/indexer/locality_index_builder.cpp +++ b/indexer/locality_index_builder.cpp @@ -57,19 +57,19 @@ private: }; } // namespace -bool BuildLocalityIndexFromDataFile(string const & dataFile, string const & tmpFile) +bool BuildLocalityIndexFromDataFile(string const & dataFile, string const & outFileName) { try { - string const idxFileName(tmpFile + LOCALITY_INDEX_TMP_EXT); + string const idxFileName(outFileName + LOCALITY_INDEX_TMP_EXT); { LocalityVectorReader localities(dataFile); FileWriter writer(idxFileName); - covering::BuildLocalityIndex(localities.GetVector(), writer, tmpFile); + covering::BuildLocalityIndex(localities.GetVector(), writer, outFileName); } - FilesContainerW(dataFile, FileWriter::OP_WRITE_EXISTING) + FilesContainerW(outFileName, FileWriter::OP_WRITE_TRUNCATE) .Write(idxFileName, LOCALITY_INDEX_FILE_TAG); FileWriter::DeleteFileX(idxFileName); } diff --git a/indexer/locality_object.cpp b/indexer/locality_object.cpp index c348d31315..3ca1d9bb30 100644 --- a/indexer/locality_object.cpp +++ b/indexer/locality_object.cpp @@ -22,13 +22,11 @@ void LocalityObject::Deserialize(char const * data) } ASSERT_EQUAL(type, feature::GEOM_AREA, ("Only supported types are GEOM_POINT and GEOM_AREA.")); - uint8_t trgCount; + uint32_t trgCount; ReadPrimitiveFromSource(src, trgCount); - if (trgCount > 0) - { - trgCount += 2; - char const * start = static_cast<char const *>(src.PtrC()); - serial::LoadInnerTriangles(start, trgCount, cp, m_triangles); - } + CHECK_GREATER(trgCount, 0, ()); + trgCount += 2; + char const * start = static_cast<char const *>(src.PtrC()); + serial::LoadInnerTriangles(start, trgCount, cp, m_triangles); } } // namespace indexer diff --git a/xcode/generator/generator.xcodeproj/project.pbxproj b/xcode/generator/generator.xcodeproj/project.pbxproj index 426d34e537..a85bb9818a 100644 --- a/xcode/generator/generator.xcodeproj/project.pbxproj +++ b/xcode/generator/generator.xcodeproj/project.pbxproj @@ -35,6 +35,11 @@ 3DFEBF7F1EF2D58900317D5C /* viator_dataset.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DFEBF7A1EF2D58900317D5C /* viator_dataset.cpp */; }; 3DFEBF801EF2D58900317D5C /* viator_dataset.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF7B1EF2D58900317D5C /* viator_dataset.hpp */; }; 3DFEBF821EF423FB00317D5C /* sponsored_object_storage.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF811EF423FB00317D5C /* sponsored_object_storage.hpp */; }; + 40492BC82021DC53008E093A /* locality_sorter.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 40492BC32021DC53008E093A /* locality_sorter.hpp */; }; + 40492BC92021DC53008E093A /* feature_helpers.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 40492BC42021DC53008E093A /* feature_helpers.cpp */; }; + 40492BCA2021DC53008E093A /* feature_helpers.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 40492BC52021DC53008E093A /* feature_helpers.hpp */; }; + 40492BCB2021DC53008E093A /* locality_sorter.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 40492BC62021DC53008E093A /* locality_sorter.cpp */; }; + 40492BCC2021DC53008E093A /* geometry_holder.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 40492BC72021DC53008E093A /* geometry_holder.hpp */; }; 568762601F6A9B18002C22A6 /* transit_generator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 5687625E1F6A9B18002C22A6 /* transit_generator.cpp */; }; 568762611F6A9B18002C22A6 /* transit_generator.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 5687625F1F6A9B18002C22A6 /* transit_generator.hpp */; }; 670B84BC1A8CDB0000CE4492 /* osm_source.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 670B84BA1A8CDB0000CE4492 /* osm_source.cpp */; }; @@ -145,6 +150,11 @@ 3DFEBF7A1EF2D58900317D5C /* viator_dataset.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = viator_dataset.cpp; sourceTree = "<group>"; }; 3DFEBF7B1EF2D58900317D5C /* viator_dataset.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = viator_dataset.hpp; sourceTree = "<group>"; }; 3DFEBF811EF423FB00317D5C /* sponsored_object_storage.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sponsored_object_storage.hpp; sourceTree = "<group>"; }; + 40492BC32021DC53008E093A /* locality_sorter.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = locality_sorter.hpp; sourceTree = "<group>"; }; + 40492BC42021DC53008E093A /* feature_helpers.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = feature_helpers.cpp; sourceTree = "<group>"; }; + 40492BC52021DC53008E093A /* feature_helpers.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = feature_helpers.hpp; sourceTree = "<group>"; }; + 40492BC62021DC53008E093A /* locality_sorter.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = locality_sorter.cpp; sourceTree = "<group>"; }; + 40492BC72021DC53008E093A /* geometry_holder.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = geometry_holder.hpp; sourceTree = "<group>"; }; 5687625E1F6A9B18002C22A6 /* transit_generator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = transit_generator.cpp; sourceTree = "<group>"; }; 5687625F1F6A9B18002C22A6 /* transit_generator.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = transit_generator.hpp; sourceTree = "<group>"; }; 670B84BA1A8CDB0000CE4492 /* osm_source.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = osm_source.cpp; sourceTree = "<group>"; }; @@ -271,6 +281,11 @@ children = ( 9D1B3F1A2034624900278AC8 /* node_mixer.cpp */, 9D1B3F192034624900278AC8 /* node_mixer.hpp */, + 40492BC42021DC53008E093A /* feature_helpers.cpp */, + 40492BC52021DC53008E093A /* feature_helpers.hpp */, + 40492BC72021DC53008E093A /* geometry_holder.hpp */, + 40492BC62021DC53008E093A /* locality_sorter.cpp */, + 40492BC32021DC53008E093A /* locality_sorter.hpp */, 39B2B9761FB468CC00AB85A1 /* cities_boundaries_builder.cpp */, 39B2B9771FB468CC00AB85A1 /* cities_boundaries_builder.hpp */, 3D74EF051F86841C0081202C /* ugc_section_builder.cpp */, @@ -397,6 +412,7 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + 40492BC82021DC53008E093A /* locality_sorter.hpp in Headers */, 34F558881DBF4C9600A4FC11 /* opentable_dataset.hpp in Headers */, 670E7BCB1EF9C29A00A8E9ED /* ugc_translator.hpp in Headers */, 675340881A3F2A7400A0A8C3 /* unpack_mwm.hpp in Headers */, @@ -407,6 +423,7 @@ 670B84BD1A8CDB0000CE4492 /* osm_source.hpp in Headers */, 675340631A3F2A7400A0A8C3 /* coastlines_generator.hpp in Headers */, 675340641A3F2A7400A0A8C3 /* intermediate_data.hpp in Headers */, + 40492BCA2021DC53008E093A /* feature_helpers.hpp in Headers */, 675340781A3F2A7400A0A8C3 /* intermediate_elements.hpp in Headers */, 3DFEBF7E1EF2D58900317D5C /* utils.hpp in Headers */, 6753406B1A3F2A7400A0A8C3 /* feature_emitter_iface.hpp in Headers */, @@ -430,6 +447,7 @@ 670E7BBA1EF9812B00A8E9ED /* ugc_db.hpp in Headers */, 0C5FEC711DDE19E50017688C /* routing_index_generator.hpp in Headers */, 67C79BB01E2CEEAB00C40034 /* restriction_collector.hpp in Headers */, + 40492BCC2021DC53008E093A /* geometry_holder.hpp in Headers */, 3DFEBF801EF2D58900317D5C /* viator_dataset.hpp in Headers */, 675340941C5231BA002CF0D9 /* search_index_builder.hpp in Headers */, 3D51BC591D5E512500F1FA8D /* srtm_parser.hpp in Headers */, @@ -565,6 +583,7 @@ 6753408D1A3F2A7400A0A8C3 /* osm_element.cpp in Sources */, 6726C1D51A4AFEF4005EEA39 /* osm2meta.cpp in Sources */, 34F5588C1DBF4C9600A4FC11 /* sponsored_scoring.cpp in Sources */, + 40492BCB2021DC53008E093A /* locality_sorter.cpp in Sources */, 6753405E1A3F2A7400A0A8C3 /* borders_loader.cpp in Sources */, 675340691A3F2A7400A0A8C3 /* feature_builder.cpp in Sources */, 677E2A171CAACC5F001DC42A /* towns_dumper.cpp in Sources */, @@ -576,6 +595,7 @@ 3D51BC481D5E50F700F1FA8D /* centers_table_builder.cpp in Sources */, 675340671A3F2A7400A0A8C3 /* dumper.cpp in Sources */, E9502E331D34012200CAB86B /* booking_scoring.cpp in Sources */, + 40492BC92021DC53008E093A /* feature_helpers.cpp in Sources */, 670E7BB91EF9812B00A8E9ED /* ugc_db.cpp in Sources */, 675340831A3F2A7400A0A8C3 /* statistics.cpp in Sources */, 670E7BB51EF9812B00A8E9ED /* road_access_generator.cpp in Sources */, |