[generator:streets] Parallel build streets jsonl

5 files changed, 25 insertions, 9 deletions

diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp
index 094057eb09..07427cf95f 100644
--- a/generator/generator_tool/generator_tool.cpp
+++ b/generator/generator_tool/generator_tool.cpp
@@ -414,7 +414,8 @@ int GeneratorToolMain(int argc, char ** argv)
   {
     streets::GenerateStreets(FLAGS_regions_index, FLAGS_regions_key_value,
                              FLAGS_streets_features, FLAGS_geo_objects_features,
-                             FLAGS_streets_key_value, FLAGS_verbose);
+                             FLAGS_streets_key_value,
+                             FLAGS_verbose, threadsCount);
   }
 
   if (!FLAGS_geo_objects_key_value.empty())
diff --git a/generator/streets/streets.cpp b/generator/streets/streets.cpp
index d4ad3f142c..aac8ee06ec 100644
--- a/generator/streets/streets.cpp
+++ b/generator/streets/streets.cpp
@@ -16,7 +16,8 @@ namespace streets
 void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv,
                      std::string const & pathInStreetsTmpMwm,
                      std::string const & pathInGeoObjectsTmpMwm,
-                     std::string const & pathOutStreetsKv, bool verbose)
+                     std::string const & pathOutStreetsKv,
+                     bool verbose, size_t threadsCount)
 {
   LOG(LINFO, ("Start generating streets..."));
   auto timer = base::Timer();
@@ -27,7 +28,7 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const &
   regions::RegionInfoGetter regionInfoGetter{pathInRegionsIndex, pathInRegionsKv};
   LOG(LINFO, ("Size of regions key-value storage:", regionInfoGetter.GetStorage().Size()));
 
-  StreetsBuilder streetsBuilder{regionInfoGetter};
+  StreetsBuilder streetsBuilder{regionInfoGetter, threadsCount};
 
   streetsBuilder.AssembleStreets(pathInStreetsTmpMwm);
   LOG(LINFO, ("Streets were built."));
diff --git a/generator/streets/streets.hpp b/generator/streets/streets.hpp
index c56e6be54e..d03397f25d 100644
--- a/generator/streets/streets.hpp
+++ b/generator/streets/streets.hpp
@@ -12,6 +12,7 @@ namespace streets
 void GenerateStreets(std::string const & pathInRegionsIndex, std::string const & pathInRegionsKv,
                      std::string const & pathInStreetsTmpMwm,
                      std::string const & pathInGeoObjectsTmpMwm,
-                     std::string const & pathOutStreetsKv, bool verbose);
+                     std::string const & pathOutStreetsKv,
+                     bool verbose, size_t threadsCount);
 }  // namespace streets
 }  // namespace generator
diff --git a/generator/streets/streets_builder.cpp b/generator/streets/streets_builder.cpp
index ceee7a07f7..fd5c1fa775 100644
--- a/generator/streets/streets_builder.cpp
+++ b/generator/streets/streets_builder.cpp
@@ -19,8 +19,9 @@ namespace generator
 {
 namespace streets
 {
-StreetsBuilder::StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter)
-    : m_regionInfoGetter{regionInfoGetter}
+StreetsBuilder::StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter,
+                               size_t threadsCount)
+  : m_regionInfoGetter{regionInfoGetter}, m_threadsCount{threadsCount}
 { }
 
 void StreetsBuilder::AssembleStreets(std::string const & pathInStreetsTmpMwm)
@@ -28,7 +29,7 @@ void StreetsBuilder::AssembleStreets(std::string const & pathInStreetsTmpMwm)
   auto const transform = [this](FeatureBuilder & fb, uint64_t /* currPos */) {
     AddStreet(fb);
   };
-  ForEachFromDatRawFormat(pathInStreetsTmpMwm, transform);
+  ForEachParallelFromDatRawFormat(m_threadsCount, pathInStreetsTmpMwm, transform);
 }
 
 void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm)
@@ -38,7 +39,7 @@ void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm
     if (!streetName.empty())
       AddStreetBinding(std::move(streetName), fb);
   };
-  ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, transform);
+  ForEachParallelFromDatRawFormat(m_threadsCount, pathInGeoObjectsTmpMwm, transform);
 }
 
 void StreetsBuilder::SaveStreetsKv(std::ostream & streamStreetsKv)
@@ -84,6 +85,8 @@ void StreetsBuilder::AddStreetHighway(FeatureBuilder & fb)
   };
   StreetRegionsTracing regionsTracing(fb.GetOuterGeometry(), streetRegionInfoGetter);
 
+  std::lock_guard<std::mutex> lock{m_updateMutex};
+
   auto && pathSegments = regionsTracing.StealPathSegments();
   for (auto & segment : pathSegments)
   {
@@ -100,6 +103,8 @@ void StreetsBuilder::AddStreetArea(FeatureBuilder & fb)
   if (!region)
     return;
 
+  std::lock_guard<std::mutex> lock{m_updateMutex};
+
   auto & street = InsertStreet(region->first, fb.GetName());
   auto osmId = fb.GetMostGenericOsmId();
   street.AddHighwayArea(osmId, fb.GetOuterGeometry());
@@ -111,6 +116,8 @@ void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb)
   if (!region)
     return;
 
+  std::lock_guard<std::mutex> lock{m_updateMutex};
+
   auto osmId = fb.GetMostGenericOsmId();
   auto & street = InsertStreet(region->first, fb.GetName());
   street.SetPin({fb.GetKeyPoint(), osmId});
@@ -122,6 +129,8 @@ void StreetsBuilder::AddStreetBinding(std::string && streetName, FeatureBuilder
   if (!region)
     return;
 
+  std::lock_guard<std::mutex> lock{m_updateMutex};
+
   auto & street = InsertStreet(region->first, std::move(streetName));
   street.AddBinding(NextOsmSurrogateId(), fb.GetKeyPoint());
 }
diff --git a/generator/streets/streets_builder.hpp b/generator/streets/streets_builder.hpp
index 26429f33ae..e764046432 100644
--- a/generator/streets/streets_builder.hpp
+++ b/generator/streets/streets_builder.hpp
@@ -13,6 +13,7 @@
 #include "base/geo_object_id.hpp"
 
 #include <memory>
+#include <mutex>
 #include <ostream>
 #include <stdint.h>
 #include <string>
@@ -27,7 +28,8 @@ namespace streets
 class StreetsBuilder
 {
 public:
-  explicit StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter);
+  explicit StreetsBuilder(regions::RegionInfoGetter const & regionInfoGetter,
+                          size_t threadsCount);
 
   void AssembleStreets(std::string const & pathInStreetsTmpMwm);
   void AssembleBindings(std::string const & pathInGeoObjectsTmpMwm);
@@ -60,6 +62,8 @@ private:
   std::unordered_map<uint64_t, RegionStreets> m_regions;
   regions::RegionInfoGetter const & m_regionInfoGetter;
   uint64_t m_osmSurrogateCounter{0};
+  size_t m_threadsCount;
+  std::mutex m_updateMutex;
 };
 }  // namespace streets
 }  // namespace generator