diff options
author | Anatoly Serdtcev <serdtcev@maps.me> | 2019-04-02 17:01:08 +0300 |
---|---|---|
committer | mpimenov <mpimenov@users.noreply.github.com> | 2019-04-03 16:41:06 +0300 |
commit | 1deefd7bd6796065897c2cb6931b6e53b51c7567 (patch) | |
tree | db846d8de809af0fa66a162210645ec5c57cac61 /generator | |
parent | 6cdd6a8f2a85509deb3215805f14aeed5a707ca2 (diff) |
[generator:regions] Refactor: class RegionsGenerator
Diffstat (limited to 'generator')
-rw-r--r-- | generator/regions/regions.cpp | 237 |
1 files changed, 132 insertions, 105 deletions
diff --git a/generator/regions/regions.cpp b/generator/regions/regions.cpp index 5937ba25a5..04cb744dc2 100644 --- a/generator/regions/regions.cpp +++ b/generator/regions/regions.cpp @@ -40,68 +40,136 @@ namespace regions { namespace { -std::tuple<RegionsBuilder::Regions, PointCitiesMap> -ReadDatasetFromTmpMwm(std::string const & tmpMwmFilename, RegionInfo & collector) +class RegionsGenerator { - RegionsBuilder::Regions regions; - PointCitiesMap pointCitiesMap; - auto const toDo = [&](FeatureBuilder1 const & fb, uint64_t /* currPos */) { - if (fb.IsArea() && fb.IsGeometryClosed()) - { - auto const id = fb.GetMostGenericOsmId(); - auto region = Region(fb, collector.Get(id)); - regions.emplace_back(std::move(region)); - } - else if (fb.IsPoint()) - { - auto const id = fb.GetMostGenericOsmId(); - pointCitiesMap.emplace(id, City(fb, collector.Get(id))); - } - }; - - feature::ForEachFromDatRawFormat(tmpMwmFilename, toDo); - return std::make_tuple(std::move(regions), std::move(pointCitiesMap)); -} - -void FilterRegions(RegionsBuilder::Regions & regions) -{ - auto const pred = [](Region const & region) { - auto const & label = region.GetLabel(); - auto const & name = region.GetName(); - return label.empty() || name.empty(); - }; - - base::EraseIf(regions, pred); -} - -RegionsBuilder::Regions ReadAndFixData(std::string const & tmpMwmFilename, - RegionInfo & regionsInfoCollector) -{ - RegionsBuilder::Regions regions; - PointCitiesMap pointCitiesMap; - std::tie(regions, pointCitiesMap) = ReadDatasetFromTmpMwm(tmpMwmFilename, regionsInfoCollector); - FixRegionsWithPlacePointApproximation(pointCitiesMap, regions); - FilterRegions(regions); - return regions; -} - -void RepackTmpMwm(std::string const & srcFilename, std::string const & repackedFilename, - std::set<base::GeoObjectId> const & ids, RegionInfo const & regionInfo) -{ - feature::FeaturesCollector collector(repackedFilename); - auto const toDo = [&collector, &ids, ®ionInfo](FeatureBuilder1 & fb, uint64_t /* currPos */) { - if (ids.count(fb.GetMostGenericOsmId()) == 0 || - (fb.IsPoint() && !FeatureCityPointToRegion(regionInfo, fb))) - { - return; - } - - CHECK(fb.IsArea(), ()); - collector(fb); - }; - - feature::ForEachFromDatRawFormat(srcFilename, toDo); -} +public: + RegionsGenerator(std::string const & pathInRegionsTmpMwm, std::string const & pathInRegionsCollector, + std::string const & pathOutRegionsKv, std::string const & pathOutRepackedRegionsTmpMwm, + bool verbose, size_t threadsCount) + : m_pathInRegionsTmpMwm{pathInRegionsTmpMwm} + , m_pathOutRegionsKv{pathOutRegionsKv} + , m_pathOutRepackedRegionsTmpMwm{pathOutRepackedRegionsTmpMwm} + , m_verbose{verbose} + , m_regionsInfoCollector{pathInRegionsCollector} + { + LOG(LINFO, ("Start generating regions from ", m_pathInRegionsTmpMwm)); + auto timer = base::Timer(); + Transliteration::Instance().Init(GetPlatform().ResourcesDir()); + + RegionsBuilder::Regions regions = ReadAndFixData(); + auto jsonPolicy = std::make_unique<JsonPolicy>(m_verbose); + RegionsBuilder builder{std::move(regions), std::move(jsonPolicy), threadsCount}; + GenerateRegions(builder); + + LOG(LINFO, ("Finish generating regions.", timer.ElapsedSeconds(), "seconds.")); + } + +private: + void GenerateRegions(RegionsBuilder & builder) + { + std::ofstream regionsKv{m_pathOutRegionsKv, std::ofstream::out}; + std::set<base::GeoObjectId> setIds; + size_t countIds = 0; + builder.ForEachNormalizedCountry([&](std::string const & name, Node::Ptr const & tree) { + if (!tree) + return; + + if (m_verbose) + DebugPrintTree(tree); + + LOG(LINFO, ("Processing country", name)); + auto const idStringList = builder.ToIdStringList(tree); + for (auto const & s : idStringList) + { + regionsKv << static_cast<int64_t>(s.first.GetEncodedId()) << " " << s.second << "\n"; + ++countIds; + if (!setIds.insert(s.first).second) + LOG(LWARNING, ("Id alredy exists:", s.first)); + } + }); + + LOG(LINFO, ("Regions objects key-value for", builder.GetCountryNames().size(), + "countries storage saved to", m_pathOutRegionsKv)); + LOG(LINFO, (countIds, "total ids.", setIds.size(), "unique ids.")); + + // todo(maksimandrianov1): Perhaps this is not the best solution. This is a hot fix. Perhaps it + // is better to transfer this to index generation(function GenerateRegionsData), + // or to combine index generation and key-value storage generation in + // generator_tool(generator_tool.cpp). + RepackTmpMwm(setIds); + } + + std::tuple<RegionsBuilder::Regions, PointCitiesMap> + ReadDatasetFromTmpMwm(std::string const & tmpMwmFilename, RegionInfo & collector) + { + RegionsBuilder::Regions regions; + PointCitiesMap pointCitiesMap; + auto const toDo = [&](FeatureBuilder1 const & fb, uint64_t /* currPos */) { + if (fb.IsArea() && fb.IsGeometryClosed()) + { + auto const id = fb.GetMostGenericOsmId(); + auto region = Region(fb, collector.Get(id)); + regions.emplace_back(std::move(region)); + } + else if (fb.IsPoint()) + { + auto const id = fb.GetMostGenericOsmId(); + pointCitiesMap.emplace(id, City(fb, collector.Get(id))); + } + }; + + feature::ForEachFromDatRawFormat(tmpMwmFilename, toDo); + return std::make_tuple(std::move(regions), std::move(pointCitiesMap)); + } + + RegionsBuilder::Regions ReadAndFixData() + { + RegionsBuilder::Regions regions; + PointCitiesMap pointCitiesMap; + std::tie(regions, pointCitiesMap) = ReadDatasetFromTmpMwm(m_pathInRegionsTmpMwm, m_regionsInfoCollector); + FixRegionsWithPlacePointApproximation(pointCitiesMap, regions); + FilterRegions(regions); + return regions; + } + + void FilterRegions(RegionsBuilder::Regions & regions) + { + auto const pred = [](Region const & region) { + auto const & label = region.GetLabel(); + auto const & name = region.GetName(); + return label.empty() || name.empty(); + }; + + base::EraseIf(regions, pred); + } + + void RepackTmpMwm(std::set<base::GeoObjectId> const & ids) + { + feature::FeaturesCollector collector(m_pathOutRepackedRegionsTmpMwm); + auto const toDo = [this, &collector, &ids](FeatureBuilder1 & fb, uint64_t /* currPos */) { + if (ids.count(fb.GetMostGenericOsmId()) == 0 || + (fb.IsPoint() && !FeatureCityPointToRegion(m_regionsInfoCollector, fb))) + { + return; + } + + CHECK(fb.IsArea(), ()); + collector(fb); + }; + + feature::ForEachFromDatRawFormat(m_pathInRegionsTmpMwm, toDo); + + LOG(LINFO, ("Repacked regions temprory mwm saved to", m_pathOutRepackedRegionsTmpMwm)); + } + + std::string m_pathInRegionsTmpMwm; + std::string m_pathOutRegionsKv; + std::string m_pathOutRepackedRegionsTmpMwm; + + bool m_verbose{false}; + + RegionInfo m_regionsInfoCollector; +}; } // namespace void GenerateRegions(std::string const & pathInRegionsTmpMwm, @@ -111,50 +179,9 @@ void GenerateRegions(std::string const & pathInRegionsTmpMwm, bool verbose, size_t threadsCount) { - using namespace regions; - - LOG(LINFO, ("Start generating regions from ", pathInRegionsTmpMwm)); - auto timer = base::Timer(); - Transliteration::Instance().Init(GetPlatform().ResourcesDir()); - - RegionInfo regionsInfoCollector(pathInRegionsCollector); - RegionsBuilder::Regions regions = ReadAndFixData(pathInRegionsTmpMwm, regionsInfoCollector); - auto jsonPolicy = std::make_unique<JsonPolicy>(verbose); - auto kvBuilder = std::make_unique<RegionsBuilder>(std::move(regions), std::move(jsonPolicy), threadsCount); - - std::ofstream ofs(pathOutRegionsKv, std::ofstream::out); - std::set<base::GeoObjectId> setIds; - size_t countIds = 0; - kvBuilder->ForEachNormalizedCountry([&](std::string const & name, Node::Ptr const & tree) { - if (!tree) - return; - - if (verbose) - DebugPrintTree(tree); - - LOG(LINFO, ("Processing country", name)); - auto const idStringList = kvBuilder->ToIdStringList(tree); - for (auto const & s : idStringList) - { - ofs << static_cast<int64_t>(s.first.GetEncodedId()) << " " << s.second << "\n"; - ++countIds; - if (!setIds.insert(s.first).second) - LOG(LWARNING, ("Id alredy exists:", s.first)); - } - }); - - // todo(maksimandrianov1): Perhaps this is not the best solution. This is a hot fix. Perhaps it - // is better to transfer this to index generation(function GenerateRegionsData), - // or to combine index generation and key-value storage generation in - // generator_tool(generator_tool.cpp). - if (!pathOutRepackedRegionsTmpMwm.empty()) - RepackTmpMwm(pathInRegionsTmpMwm, pathOutRepackedRegionsTmpMwm, setIds, regionsInfoCollector); - - LOG(LINFO, ("Regions objects key-value for", kvBuilder->GetCountryNames().size(), - "countries storage saved to", pathOutRegionsKv)); - LOG(LINFO, ("Repacked regions temprory mwm saved to", pathOutRepackedRegionsTmpMwm)); - LOG(LINFO, (countIds, "total ids.", setIds.size(), "unique ids.")); - LOG(LINFO, ("Finish generating regions.", timer.ElapsedSeconds(), "seconds.")); + RegionsGenerator(pathInRegionsTmpMwm, pathInRegionsCollector, + pathOutRegionsKv, pathOutRepackedRegionsTmpMwm, + verbose, threadsCount); } } // namespace regions } // namespace generator |