[generator:geo_objects] Generate streets for KV

9 files changed, 219 insertions, 18 deletions

diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt
index 1592d7589b..aa3f7a920e 100644
--- a/generator/CMakeLists.txt
+++ b/generator/CMakeLists.txt
@@ -98,6 +98,8 @@ set(SRC
   geo_objects/key_value_storage.hpp
   geo_objects/region_info_getter.cpp
   geo_objects/region_info_getter.hpp
+  geo_objects/streets_builder.cpp
+  geo_objects/streets_builder.hpp
   holes.cpp
   holes.hpp
   intermediate_data.cpp
diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp
index 728d3ae0e2..162d104f6b 100644
--- a/generator/geo_objects/geo_objects.cpp
+++ b/generator/geo_objects/geo_objects.cpp
@@ -1,12 +1,10 @@
 #include "generator/geo_objects/geo_objects.hpp"
 
+#include "generator/feature_builder.hpp"
 #include "generator/geo_objects/geo_object_info_getter.hpp"
 #include "generator/geo_objects/key_value_storage.hpp"
 #include "generator/geo_objects/region_info_getter.hpp"
-
-#include "generator/geo_objects/key_value_storage.hpp"
-
-#include "generator/feature_builder.hpp"
+#include "generator/geo_objects/streets_builder.hpp"
 #include "generator/locality_sorter.hpp"
 #include "generator/regions/region_base.hpp"
 
@@ -189,7 +187,7 @@ void BuildGeoObjectsWithoutAddresses(GeoObjectInfoGetter const & geoObjectInfoGe
 {
   size_t countGeoObjects = 0;
   auto const fn  = [&](FeatureBuilder1 & fb, uint64_t /* currPos */) {
-    if (IsBuilding(fb) || HasHouse(fb))
+    if (IsBuilding(fb) || HasHouse(fb) || StreetsBuilder::IsStreet(fb))
       return;
 
     auto const house = FindHousePoi(fb, geoObjectInfoGetter);
@@ -229,8 +227,11 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex,
   RegionInfoGetter regionInfoGetter{pathInRegionsIndex, pathInRegionsKv};
   LOG(LINFO, ("Size of regions key-value storage:", regionInfoGetter.GetStorage().Size()));
 
-  std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress);
+  StreetsBuilder streetsBuilder{regionInfoGetter};
   std::ofstream streamGeoObjectsKv(pathOutGeoObjectsKv);
+  streetsBuilder.Build(pathInGeoObjectsTmpMwm, streamGeoObjectsKv);
+  LOG(LINFO, ("Streets was built."));
+
   BuildGeoObjectsWithAddresses(regionInfoGetter, pathInGeoObjectsTmpMwm, streamGeoObjectsKv, verbose);
   LOG(LINFO, ("Geo objects with addresses were built."));
 
@@ -244,6 +245,7 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndex,
     return false;
 
   GeoObjectInfoGetter geoObjectInfoGetter{std::move(*geoObjectIndex), std::move(geoObjectsKv)};
+  std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress);
   BuildGeoObjectsWithoutAddresses(geoObjectInfoGetter, pathInGeoObjectsTmpMwm,
                                   streamGeoObjectsKv, streamIdsWithoutAddress, verbose);
   LOG(LINFO, ("Geo objects without addresses were built."));
diff --git a/generator/geo_objects/region_info_getter.cpp b/generator/geo_objects/region_info_getter.cpp
index 1b138546aa..39650551d8 100644
--- a/generator/geo_objects/region_info_getter.cpp
+++ b/generator/geo_objects/region_info_getter.cpp
@@ -15,8 +15,14 @@ RegionInfoGetter::RegionInfoGetter(std::string const & indexPath, std::string co
 
 boost::optional<KeyValue> RegionInfoGetter::FindDeepest(m2::PointD const & point) const
 {
+  return FindDeepest(point, [] (...) { return true; });
+}
+
+boost::optional<KeyValue> RegionInfoGetter::FindDeepest(
+    m2::PointD const & point, Selector const & selector) const
+{
   auto const ids = SearchObjectsInIndex(point);
-  return GetDeepest(ids);
+  return GetDeepest(ids, selector);
 }
 
 std::vector<base::GeoObjectId> RegionInfoGetter::SearchObjectsInIndex(m2::PointD const & point) const
@@ -27,7 +33,8 @@ std::vector<base::GeoObjectId> RegionInfoGetter::SearchObjectsInIndex(m2::PointD
   return ids;
 }
 
-boost::optional<KeyValue> RegionInfoGetter::GetDeepest(std::vector<base::GeoObjectId> const & ids) const
+boost::optional<KeyValue> RegionInfoGetter::GetDeepest(
+    std::vector<base::GeoObjectId> const & ids, Selector const & selector) const
 {
   boost::optional<KeyValue> deepest;
   int deepestRank = 0;
@@ -49,7 +56,7 @@ boost::optional<KeyValue> RegionInfoGetter::GetDeepest(std::vector<base::GeoObje
     }
 
     int tempRank = GetRank(temp);
-    if (!deepest || deepestRank < tempRank)
+    if ((!deepest || deepestRank < tempRank) && selector(temp))
     {
       deepestRank = tempRank;
       deepest = KeyValue(static_cast<int64_t>(id.GetEncodedId()), temp);
diff --git a/generator/geo_objects/region_info_getter.hpp b/generator/geo_objects/region_info_getter.hpp
index 3d422e616c..3c0d4d9a56 100644
--- a/generator/geo_objects/region_info_getter.hpp
+++ b/generator/geo_objects/region_info_getter.hpp
@@ -24,16 +24,20 @@ namespace geo_objects
 class RegionInfoGetter
 {
 public:
+  using Selector = std::function<bool(base::Json const & json)>;
+
   RegionInfoGetter(std::string const & indexPath, std::string const & kvPath);
 
   boost::optional<KeyValue> FindDeepest(m2::PointD const & point) const;
+  boost::optional<KeyValue> FindDeepest(m2::PointD const & point, Selector const & selector) const;
   KeyValueStorage const & GetStorage() const noexcept;
 
 private:
   using IndexReader = ReaderPtr<Reader>;
 
   std::vector<base::GeoObjectId> SearchObjectsInIndex(m2::PointD const & point) const;
-  boost::optional<KeyValue> GetDeepest(std::vector<base::GeoObjectId> const & ids) const;
+  boost::optional<KeyValue> GetDeepest(std::vector<base::GeoObjectId> const & ids,
+                                       Selector const & selector) const;
   int GetRank(base::Json const & json) const;
 
   indexer::RegionsIndex<IndexReader> m_index;
diff --git a/generator/geo_objects/streets_builder.cpp b/generator/geo_objects/streets_builder.cpp
new file mode 100644
index 0000000000..10477a657d
--- /dev/null
+++ b/generator/geo_objects/streets_builder.cpp
@@ -0,0 +1,139 @@
+#include "generator/geo_objects/streets_builder.hpp"
+
+#include "indexer/classificator.hpp"
+
+#include "base/logging.hpp"
+
+#include <utility>
+
+#include "3party/jansson/myjansson.hpp"
+
+namespace generator
+{
+namespace geo_objects
+{
+StreetsBuilder::StreetsBuilder(RegionInfoGetter const & regionInfoGetter)
+    : m_regionInfoGetter{regionInfoGetter}
+{ }
+
+void StreetsBuilder::Build(std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamGeoObjectsKv)
+{
+  auto const transform = [this, &streamGeoObjectsKv](FeatureBuilder1 & fb, uint64_t /* currPos */) {
+    if (!IsStreet(fb))
+      return;
+
+    auto const region = FindStreetRegionOwner(fb);
+    if (!region)
+      return;
+
+    if (!InsertStreet(*region, fb))
+      return;
+
+    auto const value = MakeStreetValue(fb, *region);
+    auto const id = static_cast<int64_t>(fb.GetMostGenericOsmId().GetEncodedId());
+    streamGeoObjectsKv << id << " " << value.get() << "\n";
+  };
+
+  feature::ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, transform);
+}
+
+boost::optional<KeyValue> StreetsBuilder::FindStreetRegionOwner(FeatureBuilder1 & fb)
+{
+  auto const & line = fb.GetOuterGeometry();
+  auto const & startPoint = line.front();
+  auto const & owner = FindStreetRegionOwner(startPoint);
+  if (!owner)
+    return {};
+
+  auto const & finishPoint = line.back();
+  if (startPoint != finishPoint)
+  {
+    auto const & finishPointOwner = FindStreetRegionOwner(finishPoint);
+    if (!finishPointOwner || finishPointOwner->first != owner->first)
+      LOG(LDEBUG, ("Street", fb.GetMostGenericOsmId(), fb.GetName(), "is in several regions"));
+  }
+
+  return owner;
+}
+
+boost::optional<KeyValue> StreetsBuilder::FindStreetRegionOwner(m2::PointD const & point)
+{
+  auto const isStreetAdministrator = [] (base::Json const & region) {
+    auto const && properties = base::GetJSONObligatoryField(region.get(), "properties");
+    auto const && address = base::GetJSONObligatoryField(properties, "address");
+
+    if (base::GetJSONOptionalField(address, "suburb"))
+      return false;
+    if (base::GetJSONOptionalField(address, "sublocality"))
+      return false;
+
+    return true;
+  };
+
+  return m_regionInfoGetter.FindDeepest(point, isStreetAdministrator);
+}
+
+bool StreetsBuilder::InsertStreet(KeyValue const & region, FeatureBuilder1 & fb)
+{
+  auto & regionStreets = m_regionsStreets[region.first];
+  auto emplace = regionStreets.emplace(fb.GetName());
+  return emplace.second;
+}
+
+std::unique_ptr<char, JSONFreeDeleter> StreetsBuilder::MakeStreetValue(
+    FeatureBuilder1 const & fb, KeyValue const & region)
+{
+  auto const && regionProperties = base::GetJSONObligatoryField(region.second.get(), "properties");
+  auto const && regionAddress = base::GetJSONObligatoryField(regionProperties, "address");
+  auto address = base::JSONPtr{json_deep_copy(regionAddress)};
+  ToJSONObject(*address, "street", fb.GetName());
+
+  auto properties = base::NewJSONObject();
+  ToJSONObject(*properties, "address", std::move(address));
+  ToJSONObject(*properties, "name", fb.GetName());
+  ToJSONObject(*properties, "pid", region.first);
+
+  auto streetObject = base::NewJSONObject();
+  ToJSONObject(*streetObject, "properties", std::move(properties));
+
+  auto const value = json_dumps(streetObject.get(), JSON_COMPACT);
+  return std::unique_ptr<char, JSONFreeDeleter>{value};
+}
+
+// static
+bool StreetsBuilder::IsStreet(OsmElement const & element)
+{
+  if (!element.IsWay() && !element.IsRelation())
+    return false;
+
+  auto const & tags = element.Tags();
+
+  auto const isHighway = std::any_of(std::cbegin(tags), std::cend(tags), [] (auto const & tag) {
+    return tag.key == "highway";
+  });
+  if (!isHighway)
+    return false;
+
+  auto const hasName = std::any_of(std::cbegin(tags), std::cend(tags), [] (auto const & tag) {
+    return tag.key == "name";
+  });
+  if (!hasName)
+    return false;
+
+  return true;
+}
+
+// static
+bool StreetsBuilder::IsStreet(FeatureBuilder1 const & fb)
+{
+  if (!fb.IsLine() && !fb.IsArea())
+    return false;
+
+  static auto const highwayType = classif().GetTypeByPath({"highway"});
+  if (fb.FindType(highwayType, 1) == ftype::GetEmptyValue())
+    return false;
+
+  return !fb.GetName().empty();
+}
+}  // namespace geo_objects
+}  // namespace generator
diff --git a/generator/geo_objects/streets_builder.hpp b/generator/geo_objects/streets_builder.hpp
new file mode 100644
index 0000000000..5de7aadc4c
--- /dev/null
+++ b/generator/geo_objects/streets_builder.hpp
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "generator/feature_builder.hpp"
+#include "generator/geo_objects/key_value_storage.hpp"
+#include "generator/geo_objects/region_info_getter.hpp"
+#include "generator/osm_element.hpp"
+
+#include "coding/reader.hpp"
+
+#include "geometry/point2d.hpp"
+
+#include <ostream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include <boost/optional.hpp>
+
+namespace generator
+{
+namespace geo_objects
+{
+class StreetsBuilder
+{
+public:
+  StreetsBuilder(RegionInfoGetter const & regionInfoGetter);
+
+  void Build(std::string const & pathInGeoObjectsTmpMwm, std::ostream & streamGeoObjectsKv);
+
+  static bool IsStreet(OsmElement const & element);
+  static bool IsStreet(FeatureBuilder1 const & fb);
+
+private:
+  boost::optional<KeyValue> FindStreetRegionOwner(FeatureBuilder1 & fb);
+  boost::optional<KeyValue> FindStreetRegionOwner(m2::PointD const & point);
+  bool InsertStreet(KeyValue const & region, FeatureBuilder1 & fb);
+  std::unique_ptr<char, JSONFreeDeleter> MakeStreetValue(FeatureBuilder1 const & fb, KeyValue const & region);
+
+  std::unordered_map<uint64_t, std::unordered_set<std::string>> m_regionsStreets;
+  RegionInfoGetter const & m_regionInfoGetter;
+};
+}  // namespace geo_objects
+}  // namespace generator
diff --git a/generator/osm_element.cpp b/generator/osm_element.cpp
index d44fbe6e17..fddf9a8adf 100644
--- a/generator/osm_element.cpp
+++ b/generator/osm_element.cpp
@@ -32,7 +32,7 @@ std::string DebugPrint(OsmElement::EntityType e)
 }
 
 
-void OsmElement::AddTag(std::string const & k, std::string const & v)
+void OsmElement::AddTag(std::string_view const & k, std::string_view const & v)
 {
   // Seems like source osm data has empty values. They are useless for us.
   if (k.empty() || v.empty())
@@ -65,12 +65,12 @@ void OsmElement::AddTag(std::string const & k, std::string const & v)
   SKIP_KEY("official_name");
 #undef SKIP_KEY
 
-  std::string value = v;
+  std::string value{std::string{v}};
   strings::Trim(value);
-  m_tags.emplace_back(k, value);
+  m_tags.emplace_back(std::string{k}, std::move(value));
 }
 
-bool OsmElement::HasTag(std::string const & k, std::string const & v) const
+bool OsmElement::HasTag(std::string_view const & k, std::string_view const & v) const
 {
   return std::any_of(m_tags.begin(), m_tags.end(), [&](auto const & t) {
     return t.key == k && t.value == v;
diff --git a/generator/osm_element.hpp b/generator/osm_element.hpp
index f4ae77f646..bf9406a5d0 100644
--- a/generator/osm_element.hpp
+++ b/generator/osm_element.hpp
@@ -5,6 +5,8 @@
 #include "base/math.hpp"
 #include "base/string_utils.hpp"
 
+#include "std/string_view.hpp"
+
 #include <exception>
 #include <functional>
 #include <iomanip>
@@ -139,8 +141,8 @@ struct OsmElement
     m_members.emplace_back(ref, type, role);
   }
 
-  void AddTag(std::string const & k, std::string const & v);
-  bool HasTag(std::string const & k, std::string const & v) const;
+  void AddTag(std::string_view const & k, std::string_view const & v);
+  bool HasTag(std::string_view const & k, std::string_view const & v) const;
   bool HasAnyTag(std::unordered_multimap<std::string, std::string> const & tags) const;
 
   template <class Fn>
diff --git a/generator/translator_geo_objects.cpp b/generator/translator_geo_objects.cpp
index b7325c32a4..71df787c4b 100644
--- a/generator/translator_geo_objects.cpp
+++ b/generator/translator_geo_objects.cpp
@@ -2,6 +2,7 @@
 
 #include "generator/feature_maker.hpp"
 #include "generator/filter_interface.hpp"
+#include "generator/geo_objects/streets_builder.hpp"
 #include "generator/intermediate_data.hpp"
 #include "generator/osm_element.hpp"
 #include "generator/osm_element_helpers.hpp"
@@ -16,12 +17,13 @@ public:
   // FilterInterface overrides:
   bool IsAccepted(OsmElement const & element) override
   {
-    return osm_element::IsBuilding(element) || osm_element::IsPoi(element);
+    return osm_element::IsBuilding(element) || osm_element::IsPoi(element) ||
+           geo_objects::StreetsBuilder::IsStreet(element);
   }
 
   bool IsAccepted(FeatureBuilder1 const & feature) override
   {
-    return feature.GetParams().IsValid() && !feature.IsLine();
+    return feature.GetParams().IsValid();
   }
 };
 }  // namespace