diff options
author | Anatoly Serdtcev <serdtcev@maps.me> | 2019-06-17 13:55:21 +0300 |
---|---|---|
committer | Anatoly Serdtcev <serdtcev@maps.me> | 2019-06-17 13:55:21 +0300 |
commit | ac5290ee485c6c7460c8bcfce9a9e76d8ab2d4ac (patch) | |
tree | 1a6d52f1bb4b83bf8d9c7f1f9c1d266e85219a22 /generator | |
parent | 86e540688ecaeee12dba930b71cf1f4848b2d751 (diff) |
[generator:streets] Parallel build streets jsonl
Diffstat (limited to 'generator')
-rw-r--r-- | generator/generator_tool/generator_tool.cpp | 3 | ||||
-rw-r--r-- | generator/streets/streets.cpp | 5 | ||||
-rw-r--r-- | generator/streets/streets.hpp | 3 | ||||
-rw-r--r-- | generator/streets/streets_builder.cpp | 17 | ||||
-rw-r--r-- | generator/streets/streets_builder.hpp | 6 |
5 files changed, 25 insertions, 9 deletions
diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 094057eb09..07427cf95f 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -414,7 +414,8 @@ int GeneratorToolMain(int argc, char ** argv) { streets::GenerateStreets(FLAGS_regions_index, FLAGS_regions_key_value, FLAGS_streets_features, FLAGS_geo_objects_features, - FLAGS_streets_key_value, FLAGS_verbose); + FLAGS_streets_key_value, + FLAGS_verbose, threadsCount); } if (!FLAGS_geo_objects_key_value.empty()) diff --git a/generator/streets/streets.cpp b/generator/streets/streets.cpp index d4ad3f142c..aac8ee06ec 100644 --- a/generator/streets/streets.cpp +++ b/generator/streets/streets.cpp @@ -16,7 +16,8 @@ namespace streets void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv, std::string const & pathInStreetsTmpMwm, std::string const & pathInGeoObjectsTmpMwm, - std::string const & pathOutStreetsKv, bool verbose) + std::string const & pathOutStreetsKv, + bool verbose, size_t threadsCount) { LOG(LINFO, ("Start generating streets...")); auto timer = base::Timer(); @@ -27,7 +28,7 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & regions::RegionInfoGetter regionInfoGetter{pathInRegionsIndex, pathInRegionsKv}; LOG(LINFO, ("Size of regions key-value storage:", regionInfoGetter.GetStorage().Size())); - StreetsBuilder streetsBuilder{regionInfoGetter}; + StreetsBuilder streetsBuilder{regionInfoGetter, threadsCount}; streetsBuilder.AssembleStreets(pathInStreetsTmpMwm); LOG(LINFO, ("Streets were built.")); diff --git a/generator/streets/streets.hpp b/generator/streets/streets.hpp index c56e6be54e..d03397f25d 100644 --- a/generator/streets/streets.hpp +++ b/generator/streets/streets.hpp @@ -12,6 +12,7 @@ namespace streets void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv, std::string const & pathInStreetsTmpMwm, std::string const & pathInGeoObjectsTmpMwm, - std::string const & pathOutStreetsKv, bool verbose); + std::string const & pathOutStreetsKv, + bool verbose, size_t threadsCount); } // namespace streets } // namespace generator diff --git a/generator/streets/streets_builder.cpp b/generator/streets/streets_builder.cpp index ceee7a07f7..fd5c1fa775 100644 --- a/generator/streets/streets_builder.cpp +++ b/generator/streets/streets_builder.cpp @@ -19,8 +19,9 @@ namespace generator { namespace streets { -StreetsBuilder::StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter) - : m_regionInfoGetter{regionInfoGetter} +StreetsBuilder::StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter, + size_t threadsCount) + : m_regionInfoGetter{regionInfoGetter}, m_threadsCount{threadsCount} { } void StreetsBuilder::AssembleStreets(std::string const & pathInStreetsTmpMwm) @@ -28,7 +29,7 @@ void StreetsBuilder::AssembleStreets(std::string const & pathInStreetsTmpMwm) auto const transform = [this](FeatureBuilder & fb, uint64_t /* currPos */) { AddStreet(fb); }; - ForEachFromDatRawFormat(pathInStreetsTmpMwm, transform); + ForEachParallelFromDatRawFormat(m_threadsCount, pathInStreetsTmpMwm, transform); } void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm) @@ -38,7 +39,7 @@ void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm if (!streetName.empty()) AddStreetBinding(std::move(streetName), fb); }; - ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, transform); + ForEachParallelFromDatRawFormat(m_threadsCount, pathInGeoObjectsTmpMwm, transform); } void StreetsBuilder::SaveStreetsKv(std::ostream & streamStreetsKv) @@ -84,6 +85,8 @@ void StreetsBuilder::AddStreetHighway(FeatureBuilder & fb) }; StreetRegionsTracing regionsTracing(fb.GetOuterGeometry(), streetRegionInfoGetter); + std::lock_guard<std::mutex> lock{m_updateMutex}; + auto && pathSegments = regionsTracing.StealPathSegments(); for (auto & segment : pathSegments) { @@ -100,6 +103,8 @@ void StreetsBuilder::AddStreetArea(FeatureBuilder & fb) if (!region) return; + std::lock_guard<std::mutex> lock{m_updateMutex}; + auto & street = InsertStreet(region->first, fb.GetName()); auto osmId = fb.GetMostGenericOsmId(); street.AddHighwayArea(osmId, fb.GetOuterGeometry()); @@ -111,6 +116,8 @@ void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb) if (!region) return; + std::lock_guard<std::mutex> lock{m_updateMutex}; + auto osmId = fb.GetMostGenericOsmId(); auto & street = InsertStreet(region->first, fb.GetName()); street.SetPin({fb.GetKeyPoint(), osmId}); @@ -122,6 +129,8 @@ void StreetsBuilder::AddStreetBinding(std::string && streetName, FeatureBuilder if (!region) return; + std::lock_guard<std::mutex> lock{m_updateMutex}; + auto & street = InsertStreet(region->first, std::move(streetName)); street.AddBinding(NextOsmSurrogateId(), fb.GetKeyPoint()); } diff --git a/generator/streets/streets_builder.hpp b/generator/streets/streets_builder.hpp index 26429f33ae..e764046432 100644 --- a/generator/streets/streets_builder.hpp +++ b/generator/streets/streets_builder.hpp @@ -13,6 +13,7 @@ #include "base/geo_object_id.hpp" #include <memory> +#include <mutex> #include <ostream> #include <stdint.h> #include <string> @@ -27,7 +28,8 @@ namespace streets class StreetsBuilder { public: - explicit StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter); + explicit StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter, + size_t threadsCount); void AssembleStreets(std::string const & pathInStreetsTmpMwm); void AssembleBindings(std::string const & pathInGeoObjectsTmpMwm); @@ -60,6 +62,8 @@ private: std::unordered_map<uint64_t, RegionStreets> m_regions; regions::RegionInfoGetter const & m_regionInfoGetter; uint64_t m_osmSurrogateCounter{0}; + size_t m_threadsCount; + std::mutex m_updateMutex; }; } // namespace streets } // namespace generator |