Generator tool routing refactoring

7 files changed, 174 insertions, 117 deletions

diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp
index 0e1b33f7c4..233d8f22e3 100644
--- a/generator/generator_tool/generator_tool.cpp
+++ b/generator/generator_tool/generator_tool.cpp
@@ -60,6 +60,8 @@ DEFINE_string(delete_section, "", "Delete specified section (defines.hpp) from c
 DEFINE_bool(fail_on_coasts, false, "Stop and exit with '255' code if some coastlines are not merged.");
 DEFINE_string(address_file_name, "", "Output file name for storing full addresses.");
 DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info");
+DEFINE_bool(make_routing, false, "Make routing info based on osrm file");
+DEFINE_bool(make_cross_section, false, "Make corss section in routing file for cross mwm routing");
 DEFINE_string(osm_file_name, "", "Input osm area file");
 DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]");
 DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc.");
@@ -248,8 +250,11 @@ int main(int argc, char ** argv)
   if (FLAGS_check_mwm)
     check_model::ReadFeatures(datFile);
 
-  if (!FLAGS_osrm_file_name.empty())
+  if (!FLAGS_osrm_file_name.empty() && FLAGS_make_routing)
     routing::BuildRoutingIndex(path, FLAGS_output, FLAGS_osrm_file_name);
 
+  if (!FLAGS_osrm_file_name.empty() && FLAGS_make_cross_section)
+    routing::BuildCrossRoutingIndex(path, FLAGS_output, FLAGS_osrm_file_name);
+
   return 0;
 }
diff --git a/generator/routing_generator.cpp b/generator/routing_generator.cpp
index 7a96af05d0..6a17bbc426 100644
--- a/generator/routing_generator.cpp
+++ b/generator/routing_generator.cpp
@@ -34,55 +34,32 @@ namespace routing
 
 static double const EQUAL_POINT_RADIUS_M = 2.0;
 
-void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile)
+bool LoadIndexes(string const & mwmFile, string const & osrmFile, osrm::NodeDataVectorT & nodeData, gen::OsmID2FeatureID & osm2ft)
 {
-  classificator::Load();
-
-  string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION;
-  Index index;
-  m2::RectD rect;
-  if (!index.Add(mwmFile, rect))
-  {
-    LOG(LCRITICAL, ("MWM file not found"));
-    return;
-  }
-
-  vector<m2::RegionD> regionBorders;
-  routing::CrossRoutingContextWriter crossContext;
-  LOG(LINFO, ("Loading countries borders"));
-  borders::CountriesContainerT m_countries;
-  CHECK(borders::LoadCountriesList(baseDir, m_countries),
-      ("Error loading country polygons files"));
+  if (!osrm::LoadNodeDataFromFile(osrmFile + ".nodeData", nodeData))
   {
-    m_countries.ForEach([&](borders::CountryPolygons const & c)
-    {
-      if (c.m_name == countryName)
-      {
-        c.m_regions.ForEach([&](m2::RegionD const & region)
-        {
-          regionBorders.push_back(region);
-        });
-      }
-    });
+    LOG(LCRITICAL, ("Can't load node data"));
+    return false;
   }
-
-  gen::OsmID2FeatureID osm2ft;
   {
     FileReader reader(mwmFile + OSM2FEATURE_FILE_EXTENSION);
     ReaderSource<FileReader> src(reader);
     osm2ft.Read(src);
   }
+  return true;
+}
 
-  osrm::NodeDataVectorT nodeData;
-  if (!osrm::LoadNodeDataFromFile(osrmFile + ".nodeData", nodeData))
+void FindCrossNodes(osrm::NodeDataVectorT const & nodeData, gen::OsmID2FeatureID const & osm2ft, borders::CountriesContainerT const & m_countries, string const & countryName, routing::CrossRoutingContextWriter & crossContext)
+{
+  vector<m2::RegionD> regionBorders;
+  m_countries.ForEach([&](borders::CountryPolygons const & c)
   {
-    LOG(LCRITICAL, ("Can't load node data"));
-    return;
-  }
-
-  OsrmFtSegMappingBuilder mapping;
-
-  uint32_t found = 0, all = 0, multiple = 0, equal = 0, moreThan1Seg = 0, stored = 0;
+    if (c.m_name == countryName)
+      c.m_regions.ForEach([&regionBorders](m2::RegionD const & region)
+      {
+        regionBorders.push_back(region);
+      });
+  });
 
   for (size_t nodeId = 0; nodeId < nodeData.size(); ++nodeId)
   {
@@ -109,13 +86,15 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin
               break;
           if (intersection == m2::PointD::Zero())
             continue;
-          else
-            // for old format compatibility
-            intersection = m2::PointD(MercatorBounds::XToLon(intersection.x), MercatorBounds::YToLat(intersection.y));
-          if (outStart && !outEnd)
+
+          // for old format compatibility
+          intersection = m2::PointD(MercatorBounds::XToLon(intersection.x), MercatorBounds::YToLat(intersection.y));
+          if (!outStart && outEnd)
+            crossContext.addIngoingNode(nodeId, intersection);
+          else if (outStart && !outEnd)
           {
             string mwmName;
-            m2::PointD const mercatorPoint(MercatorBounds::FromLatLon(endSeg.lat2, endSeg.lon2));
+            m2::PointD const & mercatorPoint = MercatorBounds::FromLatLon(endSeg.lat2, endSeg.lon2);
             m_countries.ForEachInRect(m2::RectD(mercatorPoint, mercatorPoint), [&](borders::CountryPolygons const & c)
             {
               if (c.m_name == countryName)
@@ -123,11 +102,8 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin
               c.m_regions.ForEachInRect(m2::RectD(mercatorPoint, mercatorPoint), [&](m2::RegionD const & region)
               {
                 // Sometimes Contains make errors for cases near the border.
-                if (region.Contains(mercatorPoint))
+                if (region.Contains(mercatorPoint) || region.AtBorder(mercatorPoint, 0.01 /*Near border accuracy. In mercator.*/))
                   mwmName = c.m_name;
-                else
-                  if (region.AtBorder(mercatorPoint, 0.01 /*Near border accuracy. In mercator.*/))
-                      mwmName = c.m_name;
               });
             });
             if (!mwmName.empty())
@@ -135,11 +111,139 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin
             else
               LOG(LINFO, ("Unknowing outgoing edge", endSeg.lat2, endSeg.lon2, startSeg.lat1, startSeg.lon1));
           }
-          else if (!outStart && outEnd)
-            crossContext.addIngoingNode(nodeId, intersection);
         }
       }
     }
+  }
+}
+
+void LoadRoutingFacade(string const & osrmFile, OsrmRawDataFacade<QueryEdge::EdgeData> & facade)
+{
+  ReaderSource<ModelReaderPtr> edgeSource(new FileReader(osrmFile + "." + ROUTING_EDGEDATA_FILE_TAG));
+  vector<char> edgeBuffer(static_cast<size_t>(edgeSource.Size()));
+  edgeSource.Read(&edgeBuffer[0], edgeSource.Size());
+
+  ReaderSource<ModelReaderPtr> edgeIdsSource(new FileReader(osrmFile + "." + ROUTING_EDGEID_FILE_TAG));
+  vector<char> edgeIdsBuffer(static_cast<size_t>(edgeIdsSource.Size()));
+  edgeIdsSource.Read(&edgeIdsBuffer[0], edgeIdsSource.Size());
+
+  ReaderSource<ModelReaderPtr> shortcutsSource(new FileReader(osrmFile + "." + ROUTING_SHORTCUTS_FILE_TAG));
+  vector<char> shortcutsBuffer(static_cast<size_t>(shortcutsSource.Size()));
+  shortcutsSource.Read(&shortcutsBuffer[0], shortcutsSource.Size());
+
+  ReaderSource<ModelReaderPtr> matrixSource(new FileReader(osrmFile + "." + ROUTING_MATRIX_FILE_TAG));
+  vector<char> matrixBuffer(static_cast<size_t>(matrixSource.Size()));
+  matrixSource.Read(&matrixBuffer[0], matrixSource.Size());
+
+  facade.LoadRawData(edgeBuffer.data(), edgeIdsBuffer.data(), shortcutsBuffer.data(), matrixBuffer.data());
+}
+
+void CalculateCrossAdjacency(OsrmRawDataFacade<QueryEdge::EdgeData> & facade, routing::CrossRoutingContextWriter & crossContext)
+{
+  LOG(LINFO, ("Calculating weight map between outgoing nodes"));
+  crossContext.reserveAdjacencyMatrix();
+  auto const & in = crossContext.GetIngoingIterators();
+  auto const & out = crossContext.GetOutgoingIterators();
+  MultiroutingTaskPointT sources(distance(in.first, in.second)), targets(distance(out.first, out.second));
+  size_t index = 0;
+  // Fill sources and targets with start node task for ingoing (true) and target node task
+  // (false) for outgoing nodes
+  for (auto i = in.first; i != in.second; ++i, ++index)
+    OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, true, sources[index]);
+
+  index = 0;
+  for (auto i = out.first; i != out.second; ++i, ++index)
+    OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, false, targets[index]);
+
+  vector<EdgeWeight> costs;
+  OsrmRouter::FindWeightsMatrix(sources, targets, facade, costs);
+  auto res = costs.begin();
+  for (auto i = in.first; i != in.second; ++i)
+    for (auto j = out.first; j != out.second; ++j)
+    {
+      EdgeWeight const & edgeWeigth = *(res++);
+      if (edgeWeigth != INVALID_EDGE_WEIGHT && edgeWeigth > 0)
+        crossContext.setAdjacencyCost(i, j, edgeWeigth);
+    }
+  LOG(LINFO, ("Calculation of weight map between outgoing nodes DONE"));
+}
+
+void WriteCrossSection(routing::CrossRoutingContextWriter const & crossContext, string const & mwmFile)
+{
+  LOG(LINFO, ("Collect all data into one file..."));
+  string const fPath = mwmFile + ROUTING_FILE_EXTENSION;
+
+  FilesContainerW routingCont(fPath, FileWriter::OP_WRITE_EXISTING);
+
+  {
+    // Write version for routing file that is equal to correspondent mwm file.
+    FilesContainerR mwmCont(mwmFile);
+
+    FileWriter w = routingCont.GetWriter(VERSION_FILE_TAG);
+    ReaderSource<ModelReaderPtr> src(mwmCont.GetReader(VERSION_FILE_TAG));
+    rw::ReadAndWrite(src, w);
+    w.WritePaddingByEnd(4);
+  }
+
+  FileWriter w = routingCont.GetWriter(ROUTING_CROSS_CONTEXT_TAG);
+  size_t const start_size = w.Pos();
+  crossContext.Save(w);
+  w.WritePaddingByEnd(4);
+  LOG(LINFO, ("Have written routing info, bytes written:", w.Pos() - start_size, "bytes"));
+
+  routingCont.Finish();
+}
+
+void BuildCrossRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile)
+{
+  LOG(LINFO, ("Cross mwm routing section builder"));
+  string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION;
+  osrm::NodeDataVectorT nodeData;
+  gen::OsmID2FeatureID osm2ft;
+  if (!LoadIndexes(mwmFile, osrmFile, nodeData, osm2ft))
+    return;
+
+  routing::CrossRoutingContextWriter crossContext;
+  LOG(LINFO, ("Loading countries borders"));
+  borders::CountriesContainerT m_countries;
+  CHECK(borders::LoadCountriesList(baseDir, m_countries),
+        ("Error loading country polygons files"));
+
+  FindCrossNodes(nodeData, osm2ft, m_countries, countryName, crossContext);
+
+  // Load routing facade
+  OsrmRawDataFacade<QueryEdge::EdgeData> facade;
+  LoadRoutingFacade(osrmFile, facade);
+  CalculateCrossAdjacency(facade, crossContext);
+
+  WriteCrossSection(crossContext, mwmFile);
+}
+
+void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile)
+{
+  classificator::Load();
+
+  string const mwmFile = baseDir + countryName + DATA_FILE_EXTENSION;
+  Index index;
+  m2::RectD rect;
+  if (!index.Add(mwmFile, rect))
+  {
+    LOG(LCRITICAL, ("MWM file not found"));
+    return;
+  }
+
+  osrm::NodeDataVectorT nodeData;
+  gen::OsmID2FeatureID osm2ft;
+  if (!LoadIndexes(mwmFile, osrmFile, nodeData, osm2ft))
+    return;
+
+  OsrmFtSegMappingBuilder mapping;
+
+  uint32_t found = 0, all = 0, multiple = 0, equal = 0, moreThan1Seg = 0, stored = 0;
+
+  for (size_t nodeId = 0; nodeId < nodeData.size(); ++nodeId)
+  {
+    auto const & data = nodeData[nodeId];
 
     OsrmFtSegMappingBuilder::FtSegVectorT vec;
 
@@ -263,7 +367,7 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin
   LOG(LINFO, ("Collect all data into one file..."));
   string const fPath = mwmFile + ROUTING_FILE_EXTENSION;
 
-  FilesContainerW routingCont(fPath);
+  FilesContainerW routingCont(fPath /*, FileWriter::OP_APPEND*/);
 
   {
     // Write version for routing file that is equal to correspondent mwm file.
@@ -289,65 +393,6 @@ void BuildRoutingIndex(string const & baseDir, string const & countryName, strin
   appendFile(ROUTING_MATRIX_FILE_TAG);
   appendFile(ROUTING_EDGEID_FILE_TAG);
 
-  {
-    // Load routing facade
-    OsrmRawDataFacade<QueryEdge::EdgeData> facade;
-
-    ReaderSource<ModelReaderPtr> edgeSource(new FileReader(osrmFile + "." + ROUTING_EDGEDATA_FILE_TAG));
-    vector<char> edgeBuffer(static_cast<size_t>(edgeSource.Size()));
-    edgeSource.Read(&edgeBuffer[0], edgeSource.Size());
-
-    ReaderSource<ModelReaderPtr> edgeIdsSource(new FileReader(osrmFile + "." + ROUTING_EDGEID_FILE_TAG));
-    vector<char> edgeIdsBuffer(static_cast<size_t>(edgeIdsSource.Size()));
-    edgeIdsSource.Read(&edgeIdsBuffer[0], edgeIdsSource.Size());
-
-    ReaderSource<ModelReaderPtr> shortcutsSource(new FileReader(osrmFile + "." + ROUTING_SHORTCUTS_FILE_TAG));
-    vector<char> shortcutsBuffer(static_cast<size_t>(shortcutsSource.Size()));
-    shortcutsSource.Read(&shortcutsBuffer[0], shortcutsSource.Size());
-
-    ReaderSource<ModelReaderPtr> matrixSource(new FileReader(osrmFile + "." + ROUTING_MATRIX_FILE_TAG));
-    vector<char> matrixBuffer(static_cast<size_t>(matrixSource.Size()));
-    matrixSource.Read(&matrixBuffer[0], matrixSource.Size());
-
-    facade.LoadRawData(edgeBuffer.data(), edgeIdsBuffer.data(), shortcutsBuffer.data(), matrixBuffer.data());
-
-    LOG(LINFO, ("Calculating weight map between outgoing nodes"));
-    crossContext.reserveAdjacencyMatrix();
-    auto in = crossContext.GetIngoingIterators();
-    auto out = crossContext.GetOutgoingIterators();
-    MultiroutingTaskPointT sources(distance(in.first, in.second)), targets(distance(out.first, out.second));
-    size_t index = 0;
-    // Fill sources and targets with start node task for ingoing (true) and target node task
-    // (false) for outgoing nodes
-    for (auto i = in.first; i != in.second; ++i, ++index)
-    {
-      OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, true, sources[index]);
-    }
-    index = 0;
-    for (auto i = out.first; i != out.second; ++i, ++index)
-    {
-      OsrmRouter::GenerateRoutingTaskFromNodeId(i->m_nodeId, false, targets[index]);
-    }
-
-    vector<EdgeWeight> costs;
-    OsrmRouter::FindWeightsMatrix(sources, targets, facade, costs);
-    auto res = costs.begin();
-    for (auto i = in.first; i < in.second; ++i)
-      for (auto j = out.first; j < out.second; ++j)
-      {
-        EdgeWeight const & edgeWeigth = *(res++);
-        if (edgeWeigth != INVALID_EDGE_WEIGHT && edgeWeigth > 0)
-          crossContext.setAdjacencyCost(i, j, edgeWeigth);
-      }
-    LOG(LINFO, ("Calculation of weight map between outgoing nodes DONE"));
-  }
-
-  FileWriter w = routingCont.GetWriter(ROUTING_CROSS_CONTEXT_TAG);
-  size_t const start_size = w.Pos();
-  crossContext.Save(w);
-  w.WritePaddingByEnd(4);
-  LOG(LINFO, ("Have written routing info, bytes written:", w.Pos() - start_size, "bytes"));
-
   routingCont.Finish();
 
   uint64_t sz;
diff --git a/generator/routing_generator.hpp b/generator/routing_generator.hpp
index 3e7fea3525..0fe0fa5e8d 100644
--- a/generator/routing_generator.hpp
+++ b/generator/routing_generator.hpp
@@ -11,6 +11,8 @@ namespace routing
 /// @param[in]  osrmFile  Full path to .osrm file (all prepared osrm files should be there).
 void BuildRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile);
 
-/// @param[in]  baseDir   Full path to .mwm files directory.
-void BuildCrossesRoutingIndex(string const & baseDir);
+/// @param[in]  baseDir      Full path to .mwm files directory.
+/// @param[in]  countryName   Country name same with .mwm and .border file name.
+/// @param[in]  osrmFile  Full path to .osrm file (all prepared osrm files should be there).
+void BuildCrossRoutingIndex(string const & baseDir, string const & countryName, string const & osrmFile);
 }
diff --git a/routing/cross_routing_context.cpp b/routing/cross_routing_context.cpp
index 887078ffa1..b0bcbf53d4 100644
--- a/routing/cross_routing_context.cpp
+++ b/routing/cross_routing_context.cpp
@@ -125,7 +125,7 @@ size_t CrossRoutingContextWriter::GetIndexInAdjMatrix(IngoingEdgeIteratorT ingoi
   return m_outgoingNodes.size() * ingoing_index + outgoing_index;
 }
 
-void CrossRoutingContextWriter::Save(Writer & w)
+void CrossRoutingContextWriter::Save(Writer & w) const
 {
   uint32_t size = static_cast<uint32_t>(m_ingoingNodes.size());
   w.Write(&size, sizeof(size));
diff --git a/routing/cross_routing_context.hpp b/routing/cross_routing_context.hpp
index badffa7140..1990b7b83f 100644
--- a/routing/cross_routing_context.hpp
+++ b/routing/cross_routing_context.hpp
@@ -78,7 +78,7 @@ class CrossRoutingContextWriter
   size_t GetIndexInAdjMatrix(IngoingEdgeIteratorT ingoing, OutgoingEdgeIteratorT outgoing) const;
 
 public:
-  void Save(Writer & w);
+  void Save(Writer & w) const;
 
   void addIngoingNode(size_t const nodeId, m2::PointD const & point);
 
diff --git a/tools/unix/make_routing.sh b/tools/unix/make_routing.sh
index 9cb3ad5bea..c89e6e551c 100755
--- a/tools/unix/make_routing.sh
+++ b/tools/unix/make_routing.sh
@@ -15,4 +15,4 @@ then
   PV=pv
 fi
 
-$PV ../../../omim-maps/$2.osm.bz2 | bzip2 -d | $GENERATOR_TOOL --osrm_file_name=../../../omim-maps/$2.osrm --output=$2
+$PV ../../../omim-maps/$2.osm.bz2 | bzip2 -d | $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name=../../../omim-maps/$2.osrm --output=$2
diff --git a/tools/unix/routes_from_mwm_and_osrm.sh b/tools/unix/routes_from_mwm_and_osrm.sh
index 43670dfcab..c3977d0e94 100755
--- a/tools/unix/routes_from_mwm_and_osrm.sh
+++ b/tools/unix/routes_from_mwm_and_osrm.sh
@@ -12,4 +12,9 @@ else
   GENERATOR_TOOL=~/omim-build-release/out/release/generator_tool
 fi
 
-ls "$OSRM_DIR"/*.osrm | parallel -t -v "V=`basename -s .osrm {}`; $GENERATOR_TOOL --osrm_file_name={} --data_path=$DATA_DIR --output={/.}"
+NUM_INSTANCES=8
+
+OSRM_LIST=${OSRM_LIST-$(ls -1 $OSRM_DIR/*.osrm | xargs -d "\n" basename -s .osrm)}
+
+echo "$OSRM_LIST" |  xargs -d "\n" -P $NUM_INSTANCES -I % $GENERATOR_TOOL --make_routing --make_cross_section --osrm_file_name=$OSRM_DIR/%.osrm --data_path=$DATA_DIR --output=%
+