diff options
| -rw-r--r-- | generator/borders_generator.cpp | 159 | ||||
| -rw-r--r-- | generator/borders_generator.hpp | 14 | ||||
| -rw-r--r-- | generator/borders_loader.cpp | 80 | ||||
| -rw-r--r-- | generator/borders_loader.hpp | 27 | ||||
| -rw-r--r-- | generator/generator.pro | 8 | ||||
| -rw-r--r-- | generator/generator_tests/generator_tests.pro | 1 | ||||
| -rw-r--r-- | generator/generator_tests/osm_parser_test.cpp | 201 | ||||
| -rw-r--r-- | generator/generator_tool/generator_tool.cpp | 15 | ||||
| -rw-r--r-- | generator/kml_parser.cpp | 64 | ||||
| -rw-r--r-- | generator/kml_parser.hpp | 22 | ||||
| -rw-r--r-- | generator/osm_xml_parser.cpp | 244 | ||||
| -rw-r--r-- | generator/osm_xml_parser.hpp | 138 | ||||
| -rw-r--r-- | generator/polygonizer.hpp | 24 | ||||
| -rwxr-xr-x | tools/unix/polygons.sh | 17 |
14 files changed, 909 insertions, 105 deletions
diff --git a/generator/borders_generator.cpp b/generator/borders_generator.cpp new file mode 100644 index 0000000000..a1801b7b5e --- /dev/null +++ b/generator/borders_generator.cpp @@ -0,0 +1,159 @@ +#include "borders_generator.hpp" +#include "osm_xml_parser.hpp" + +#include "../base/std_serialization.hpp" +#include "../base/logging.hpp" + +#include "../indexer/mercator.hpp" + +#include "../coding/file_reader.hpp" +#include "../coding/file_writer.hpp" +#include "../coding/streams_sink.hpp" +#include "../coding/parse_xml.hpp" + +#include "../std/list.hpp" +#include "../std/vector.hpp" + +namespace osm +{ + class BordersCreator + { + vector<m2::RegionD> & m_borders; + OsmRawData const & m_data; + m2::RegionD m_currentRegion; + + public: + BordersCreator(vector<m2::RegionD> & outBorders, OsmRawData const & data) + : m_borders(outBorders), m_data(data) + { + } + + void CreateFromWays(OsmWays const & ways) + { + for (OsmWays::const_iterator it = ways.begin(); it != ways.end(); ++it) + { + // clear region + m_currentRegion = m2::RegionD(); + it->ForEachPoint(*this); + CHECK(m_currentRegion.IsValid(), ("Invalid region")); + m_borders.push_back(m_currentRegion); + } + } + + void operator()(OsmId nodeId) + { + try + { + OsmNode node = m_data.NodeById(nodeId); + m_currentRegion.AddPoint(m2::PointD(MercatorBounds::LonToX(node.m_lon), + MercatorBounds::LatToY(node.m_lat))); + } + catch (OsmRawData::OsmInvalidIdException const & e) + { + LOG(LWARNING, (e.what())); + } + } + }; + + void GenerateBordersFromOsm(string const & osmFile, string const & outFile) + { + OsmRawData osmData; + { + FileReader file(osmFile); + ReaderSource<FileReader> source(file); + + OsmXmlParser parser(osmData); + CHECK(ParseXML(source, parser), ("Invalid XML")); + } + + // find country borders relation + OsmIds relationIds = osmData.RelationsByKey("ISO3166-1"); + CHECK(!relationIds.empty(), ("No relation with key 'ISO3166-1' found")); + CHECK_EQUAL(relationIds.size(), 1, ("More than one relation with key 'ISO3166-1' found")); + + OsmRelation countryRelation = osmData.RelationById(relationIds[0]); + + // get country name + string countryName; + if (!countryRelation.TagValueByKey("name:en", countryName)) + countryRelation.TagValueByKey("name", countryName); + LOG(LINFO, ("Processing boundaries for country", countryName)); + + // find border ways + OsmIds wayIdsOuterRole = countryRelation.MembersByTypeAndRole("way", "outer"); + OsmIds wayIdsNoRole = countryRelation.MembersByTypeAndRole("way", ""); + // collect all ways + list<OsmWay> ways; + for(size_t i = 0; i < wayIdsOuterRole.size(); ++i) + ways.push_back(osmData.WayById(wayIdsOuterRole[i])); + for(size_t i = 0; i < wayIdsNoRole.size(); ++i) + ways.push_back(osmData.WayById(wayIdsNoRole[i])); + CHECK(!ways.empty(), ("No any ways in country border")); + + // merge all collected ways + OsmWays mergedWays; + do + { + OsmId lastMergedWayId = -1; // for debugging + OsmWay merged = ways.front(); + size_t lastSize = ways.size(); + ways.pop_front(); + // repeat until we merge everything + while (lastSize > ways.size()) + { + lastSize = ways.size(); + for (list<OsmWay>::iterator it = ways.begin(); it != ways.end(); ++it) + { + if (merged.MergeWith(*it)) + { + lastMergedWayId = it->Id(); + ways.erase(it); + break; + } + } + } + if (!merged.IsClosed()) + { + LOG(LERROR, ("Way merge unsuccessful as borders are not closed. Last merged id:", + lastMergedWayId == -1 ? merged.Id() : lastMergedWayId)); + } + else + { + LOG(LINFO, ("Successfully merged boundaries with points count:", merged.PointsCount())); + mergedWays.push_back(merged); + } + } while (!ways.empty()); + + CHECK(!mergedWays.empty(), ("No borders were generated for country:", countryName)); + + // save generated borders + vector<m2::RegionD> borders; + BordersCreator doCreateBorders(borders, osmData); + doCreateBorders.CreateFromWays(mergedWays); + CHECK_EQUAL(mergedWays.size(), borders.size(), ("Can't generate borders from ways")); + + FileWriter writer(outFile); + stream::SinkWriterStream<Writer> stream(writer); + stream << borders; + + LOG(LINFO, ("Saved", borders.size(), "border(s) for", countryName)); + } + + bool LoadBorders(string const & borderFile, vector<m2::RegionD> & outBorders) + { + try + { + FileReader file(borderFile); + ReaderSource<FileReader> source(file); + stream::SinkReaderStream<ReaderSource<FileReader> > stream(source); + + stream >> outBorders; + CHECK(!outBorders.empty(), ("No borders were loaded from", borderFile)); + } + catch (FileReader::OpenException const &) + { + return false; + } + return true; + } +} diff --git a/generator/borders_generator.hpp b/generator/borders_generator.hpp new file mode 100644 index 0000000000..2dedd0e294 --- /dev/null +++ b/generator/borders_generator.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "../geometry/region2d.hpp" +#include "../geometry/point2d.hpp" + +#include "../std/vector.hpp" +#include "../std/string.hpp" + +namespace osm +{ + void GenerateBordersFromOsm(string const & osmFile, string const & outFile); + /// @return false if borderFile can't be opened + bool LoadBorders(string const & borderFile, vector<m2::RegionD> & outBorders); +} diff --git a/generator/borders_loader.cpp b/generator/borders_loader.cpp new file mode 100644 index 0000000000..1e080ad963 --- /dev/null +++ b/generator/borders_loader.cpp @@ -0,0 +1,80 @@ +#include "borders_loader.hpp" +#include "borders_generator.hpp" + +#include "../base/logging.hpp" +#include "../base/string_utils.hpp" + +#include "../std/fstream.hpp" +#include "../std/vector.hpp" + +#define BORDERS_DIR "borders/" +#define BORDERS_EXTENSION ".borders" +#define POLYGONS_FILE "polygons.lst" + +namespace borders +{ + class PolygonLoader + { + string m_baseDir; + // @TODO not used + int m_level; + + CountryPolygons & m_polygons; + m2::RectD & m_rect; + + public: + // @TODO level is not used + PolygonLoader(string const & basePolygonsDir, int level, CountryPolygons & polygons, m2::RectD & rect) + : m_baseDir(basePolygonsDir), m_level(level), m_polygons(polygons), m_rect(rect) + { + } + + void operator()(string const & name) + { + if (m_polygons.m_name.empty()) + m_polygons.m_name = name; + + vector<m2::RegionD> coutryBorders; + if (osm::LoadBorders(m_baseDir + BORDERS_DIR + name + BORDERS_EXTENSION, coutryBorders)) + { + for (size_t i = 0; i < coutryBorders.size(); ++i) + { + m2::RectD const rect(coutryBorders[i].GetRect()); + m_rect.Add(rect); + m_polygons.m_regions.Add(coutryBorders[i], rect); + } + } + } + }; + + bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries, + int simplifyCountriesLevel) + { + if (simplifyCountriesLevel > 0) + { + LOG_SHORT(LINFO, ("Simplificator level for country polygons:", simplifyCountriesLevel)); + } + + countries.Clear(); + ifstream stream((baseDir + POLYGONS_FILE).c_str()); + string line; + LOG(LINFO, ("Loading countries.")); + while (stream.good()) + { + std::getline(stream, line); + if (line.empty()) + continue; + + CountryPolygons country; + m2::RectD rect; + + PolygonLoader loader(baseDir, simplifyCountriesLevel, country, rect); + utils::TokenizeString(line, "|", loader); + if (!country.m_regions.IsEmpty()) + countries.Add(country, rect); + } + LOG(LINFO, ("Countries loaded:", countries.GetSize())); + return !countries.IsEmpty(); + } + +} diff --git a/generator/borders_loader.hpp b/generator/borders_loader.hpp new file mode 100644 index 0000000000..88e51bf968 --- /dev/null +++ b/generator/borders_loader.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "../geometry/region2d.hpp" +#include "../geometry/tree4d.hpp" + +#include "../std/string.hpp" + +namespace borders +{ + typedef m2::RegionD Region; + typedef m4::Tree<Region> RegionsContainerT; + + struct CountryPolygons + { + CountryPolygons(string const & name = "") : m_name(name), m_index(-1) {} + + RegionsContainerT m_regions; + string m_name; + mutable int m_index; + }; + + typedef m4::Tree<CountryPolygons> CountriesContainerT; + + /// @param[in] simplifyCountriesLevel if positive, used as a level for simplificator + bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries, + int simplifyCountriesLevel = -1); +} diff --git a/generator/generator.pro b/generator/generator.pro index 5632536cb3..9aaa71d0eb 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -20,9 +20,11 @@ SOURCES += \ update_generator.cpp \ grid_generator.cpp \ statistics.cpp \ - kml_parser.cpp \ osm2type.cpp \ classif_routine.cpp \ + borders_generator.cpp \ + osm_xml_parser.cpp \ + borders_loader.cpp \ HEADERS += \ feature_merger.hpp \ @@ -37,8 +39,10 @@ HEADERS += \ update_generator.hpp \ grid_generator.hpp \ statistics.hpp \ - kml_parser.hpp \ polygonizer.hpp \ world_map_generator.hpp \ osm2type.hpp \ classif_routine.hpp \ + borders_generator.hpp \ + osm_xml_parser.hpp \ + borders_loader.hpp \ diff --git a/generator/generator_tests/generator_tests.pro b/generator/generator_tests/generator_tests.pro index a9df30da98..5c80cabae1 100644 --- a/generator/generator_tests/generator_tests.pro +++ b/generator/generator_tests/generator_tests.pro @@ -21,3 +21,4 @@ SOURCES += \ ../../testing/testingmain.cpp \ ../../indexer/indexer_tests/feature_routine.cpp \ feature_bucketer_test.cpp \ + osm_parser_test.cpp \ diff --git a/generator/generator_tests/osm_parser_test.cpp b/generator/generator_tests/osm_parser_test.cpp new file mode 100644 index 0000000000..298b1f6d49 --- /dev/null +++ b/generator/generator_tests/osm_parser_test.cpp @@ -0,0 +1,201 @@ +#include "../../testing/testing.hpp" + +#include "../osm_xml_parser.hpp" +#include "../borders_generator.hpp" + +#include "../../coding/reader.hpp" +#include "../../coding/parse_xml.hpp" +#include "../../coding/file_reader.hpp" +#include "../../coding/file_writer.hpp" + +using namespace osm; + +static char const gOsmXml[] = +"<?xml version=\"1.0\" encoding=\"UTF-8\"?>" +"<osm version=\"0.6\" generator=\"OpenStreetMap server\">" +" <node id=\"1\" lat=\"0.0\" lon=\"0.0\" visible=\"true\"/>" +" <node id=\"2\" lat=\"20.0\" lon=\"0.0\"/>" +" <node id=\"3\" lat=\"20.0\" lon=\"20.0\"/>" +" <node id=\"4\" lat=\"0.0\" lon=\"20.0\"/>" +" <node id=\"7\" lat=\"5.0\" lon=\"10.0\"/>" +" <node id=\"8\" lat=\"10.0\" lon=\"10.0\"/>" +" <node id=\"9\" lat=\"10.0\" lon=\"15.0\"/>" +" <node id=\"10\" lat=\"5.555555\" lon=\"4.222222\">" +" <tag k=\"name\" v=\"Andorra la Vella\"/>" +" <tag k=\"capital\" v=\"yes\"/>" +" </node>" +" <node id=\"11\" lat=\"105.0\" lon=\"110.0\"/>" +" <node id=\"12\" lat=\"110.0\" lon=\"110.0\"/>" +" <node id=\"13\" lat=\"110.0\" lon=\"115.0\"/>" +" <way id=\"100\" visible=\"true\">" +" <nd ref=\"1\"/>" +" <nd ref=\"2\"/>" +" <nd ref=\"3\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"admin_level\" v=\"2\"/>" +" </way>" +" <way id=\"101\" visible=\"true\">" +" <nd ref=\"1\"/>" +" <nd ref=\"4\"/>" +" <nd ref=\"3\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"admin_level\" v=\"2\"/>" +" </way>" +" <way id=\"102\" visible=\"true\">" +" <nd ref=\"11\"/>" +" <nd ref=\"12\"/>" +" <nd ref=\"13\"/>" +" <nd ref=\"11\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"admin_level\" v=\"2\"/>" +" </way>" +" <way id=\"121\" visible=\"true\">" +" <nd ref=\"7\"/>" +" <nd ref=\"8\"/>" +" <nd ref=\"9\"/>" +" <nd ref=\"7\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"admin_level\" v=\"2\"/>" +" </way>" +" <relation id=\"444\" visible=\"true\">" +" <member type=\"way\" ref=\"121\" role=\"outer\"/>" +" <tag k=\"admin_level\" v=\"4\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"name\" v=\"Some Region\"/>" +" <tag k=\"type\" v=\"boundary\"/>" +" </relation>" +" <relation id=\"555\" visible=\"true\">" +" <member type=\"node\" ref=\"10\" role=\"capital\"/>" +" <member type=\"way\" ref=\"100\" role=\"\"/>" +" <member type=\"way\" ref=\"101\" role=\"outer\"/>" +" <member type=\"way\" ref=\"102\" role=\"\"/>" +" <member type=\"way\" ref=\"121\" role=\"inner\"/>" +" <member type=\"relation\" ref=\"444\" role=\"subarea\"/>" +" <tag k=\"admin_level\" v=\"2\"/>" +" <tag k=\"boundary\" v=\"administrative\"/>" +" <tag k=\"ISO3166-1\" v=\"ad\"/>" +" <tag k=\"name\" v=\"Andorra\"/>" +" <tag k=\"name:en\" v=\"Andorra\"/>" +" <tag k=\"type\" v=\"boundary\"/>" +" </relation>" +"</osm>"; + +#define TEST_EXCEPTION(exception, expression) do { \ + bool gotException = false; \ + try { expression; } \ + catch (exception const &) { gotException = true; } \ + TEST(gotException, ("Exception should be thrown:", #exception)); \ + } while(0) + + +struct PointsTester +{ + list<OsmId> m_controlPoints; + PointsTester() + { + m_controlPoints.push_back(1); + m_controlPoints.push_back(2); + m_controlPoints.push_back(3); + m_controlPoints.push_back(4); + m_controlPoints.push_back(1); + } + void operator()(OsmId const & ptId) + { + TEST(!m_controlPoints.empty(), ()); + TEST_EQUAL(m_controlPoints.front(), ptId, ()); + m_controlPoints.pop_front(); + } + + bool IsOk() const + { + return m_controlPoints.empty(); + } +}; + +UNIT_TEST(OsmRawData_SmokeTest) +{ + OsmRawData osmData; + + { + // -1 to avoid finishing zero at the end of the string + MemReader xmlBlock(gOsmXml, ARRAY_SIZE(gOsmXml) - 1); + ReaderSource<MemReader> source(xmlBlock); + + OsmXmlParser parser(osmData); + TEST(ParseXML(source, parser), ("Invalid XML")); + } + + string outTagValue; + + TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmNode node = osmData.NodeById(98764)); + + OsmNode node = osmData.NodeById(9); + TEST_EQUAL(node.m_lat, 10.0, ()); + TEST_EQUAL(node.m_lon, 15.0, ()); + + TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmWay way = osmData.WayById(635794)); + + OsmWay way = osmData.WayById(100); + TEST_EQUAL(way.PointsCount(), 3, ()); + TEST(!way.TagValueByKey("invalid_tag", outTagValue), ()); + TEST(way.TagValueByKey("boundary", outTagValue), ()); + TEST_EQUAL(outTagValue, "administrative", ()); + TEST(way.TagValueByKey("admin_level", outTagValue), ()); + TEST_EQUAL(outTagValue, "2", ()); + + OsmWay way2 = osmData.WayById(101); + TEST(way.MergeWith(way2), ()); + TEST_EQUAL(way.PointsCount(), 5, ()); + PointsTester tester; + way.ForEachPoint(tester); + TEST(tester.IsOk(), ()); + TEST(way.IsClosed(), ()); + + TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmRelation relation = osmData.RelationById(64342)); + + OsmRelation rel1 = osmData.RelationById(444); + TEST(rel1.TagValueByKey("admin_level", outTagValue), ()); + TEST_EQUAL(outTagValue, "4", ()); + + OsmIds relations = osmData.RelationsByKey("invalid_tag_key"); + TEST(relations.empty(), ()); + relations = osmData.RelationsByKey("ISO3166-1"); + TEST_EQUAL(relations.size(), 1, ()); + TEST_EQUAL(relations[0], 555, ()); + + OsmRelation rel2 = osmData.RelationById(relations[0]); + OsmIds members = rel2.MembersByTypeAndRole("way", ""); + TEST_EQUAL(members.size(), 2, ()); + TEST_EQUAL(members[0], 100, ()); + members = rel2.MembersByTypeAndRole("way", "outer"); + TEST_EQUAL(members.size(), 1, ()); + TEST_EQUAL(members[0], 101, ()); + members = rel2.MembersByTypeAndRole("relation", "invalid_role"); + TEST(members.empty(), ()); + + relations.clear(); + + relations = osmData.RelationsByTag(OsmTag("boundary_invalid", "administrative")); + TEST(relations.empty(), ()); + relations = osmData.RelationsByTag(OsmTag("boundary", "administrative")); + TEST_EQUAL(relations.size(), 2, ()); +} + +UNIT_TEST(BordersGenerator) +{ + string const inputFile("BordersTestInputFile"); + string const outputFile("BordersTestOutputFile"); + + { // create input file + FileWriter f(inputFile); + f.Write(gOsmXml, ARRAY_SIZE(gOsmXml) - 1); // -1 to skip last zero + } + + GenerateBordersFromOsm(inputFile, outputFile); + vector<m2::RegionD> borders; + LoadBorders(outputFile, borders); + TEST_EQUAL(borders.size(), 2, ()); + + FileWriter::DeleteFileX(inputFile); + FileWriter::DeleteFileX(outputFile); +} diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index a8734bbba0..d672c762e8 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -6,6 +6,7 @@ #include "../grid_generator.hpp" #include "../statistics.hpp" #include "../classif_routine.hpp" +#include "../borders_generator.hpp" #include "../../indexer/features_vector.hpp" #include "../../indexer/index_builder.hpp" @@ -51,6 +52,8 @@ DEFINE_int32(generate_world_scale, -1, "If specified, features for zoomlevels [0 DEFINE_bool(split_by_polygons, false, "Use kml shape files to split planet by regions and countries"); DEFINE_int32(simplify_countries_level, -1, "If positive, simplifies country polygons. Recommended values [10..15]"); DEFINE_bool(merge_coastlines, false, "If defined, tries to merge coastlines when renerating World file"); +DEFINE_bool(generate_borders, false, + "Create binary country .borders file for osm xml file given in 'output' parameter"); string AddSlashIfNeeded(string const & str) { @@ -193,5 +196,17 @@ int main(int argc, char ** argv) update::GenerateFilesList(path); } + if (FLAGS_generate_borders) + { + if (!FLAGS_output.empty()) + { + osm::GenerateBordersFromOsm(path + FLAGS_output + ".osm", path + FLAGS_output + ".borders"); + } + else + { + LOG(LINFO, ("Please specify osm country borders file in 'output' command line parameter.")); + } + } + return 0; } diff --git a/generator/kml_parser.cpp b/generator/kml_parser.cpp index 55f4402354..20ccfe6ca6 100644 --- a/generator/kml_parser.cpp +++ b/generator/kml_parser.cpp @@ -218,7 +218,7 @@ namespace kml } } - bool LoadPolygonsFromKml(string const & kmlFile, PolygonsContainerT & country, int level) + bool LoadPolygons(string const & kmlFile, PolygonsContainerT & country, int level) { KmlParser parser(country, level); try @@ -233,66 +233,4 @@ namespace kml } return false; } - - class PolygonLoader - { - string m_baseDir; - int m_level; - - CountryPolygons & m_polygons; - m2::RectD & m_rect; - - public: - PolygonLoader(string const & basePolygonsDir, int level, CountryPolygons & polygons, m2::RectD & rect) - : m_baseDir(basePolygonsDir), m_level(level), m_polygons(polygons), m_rect(rect) - { - } - - void operator()(string const & name) - { - if (m_polygons.m_name.empty()) - m_polygons.m_name = name; - - PolygonsContainerT current; - if (LoadPolygonsFromKml(m_baseDir + BORDERS_DIR + name + BORDERS_EXTENSION, current, m_level)) - { - for (size_t i = 0; i < current.size(); ++i) - { - m2::RectD const rect(current[i].GetRect()); - m_rect.Add(rect); - m_polygons.m_regions.Add(current[i], rect); - } - } - } - }; - - bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries, - int simplifyCountriesLevel) - { - if (simplifyCountriesLevel > 0) - { - LOG_SHORT(LINFO, ("Simplificator level for country polygons:", simplifyCountriesLevel)); - } - - countries.Clear(); - ifstream stream((baseDir + POLYGONS_FILE).c_str()); - string line; - LOG(LINFO, ("Loading countries.")); - while (stream.good()) - { - std::getline(stream, line); - if (line.empty()) - continue; - - CountryPolygons country; - m2::RectD rect; - - PolygonLoader loader(baseDir, simplifyCountriesLevel, country, rect); - utils::TokenizeString(line, "|", loader); - if (!country.m_regions.IsEmpty()) - countries.Add(country, rect); - } - LOG(LINFO, ("Countries loaded:", countries.GetSize())); - return !countries.IsEmpty(); - } } diff --git a/generator/kml_parser.hpp b/generator/kml_parser.hpp index 25a0e9d5da..0b6ebff541 100644 --- a/generator/kml_parser.hpp +++ b/generator/kml_parser.hpp @@ -1,28 +1,10 @@ #pragma once -#include "../geometry/region2d.hpp" -#include "../geometry/tree4d.hpp" - #include "../std/string.hpp" +class PolygonsContainerT; namespace kml { - typedef m2::RegionD Region; - typedef m4::Tree<Region> RegionsContainerT; - - struct CountryPolygons - { - CountryPolygons(string const & name = "") : m_name(name), m_index(-1) {} - - RegionsContainerT m_regions; - string m_name; - mutable int m_index; - }; - - typedef m4::Tree<CountryPolygons> CountriesContainerT; - - /// @param[in] simplifyCountriesLevel if positive, used as a level for simplificator - bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries, - int simplifyCountriesLevel = -1); + bool LoadPolygons(string const & kmlFile, PolygonsContainerT & country, int level); } diff --git a/generator/osm_xml_parser.cpp b/generator/osm_xml_parser.cpp new file mode 100644 index 0000000000..72da2821fc --- /dev/null +++ b/generator/osm_xml_parser.cpp @@ -0,0 +1,244 @@ +#include "osm_xml_parser.hpp" + +#include "../base/assert.hpp" +#include "../base/string_utils.hpp" + +namespace osm +{ + OsmIdAndTagHolder::OsmIdAndTagHolder(OsmId id, OsmTags const & tags) + : m_id(id), m_tags(tags) + { + } + + bool OsmIdAndTagHolder::TagValueByKey(string const & key, string & outValue) const + { + for (OsmTags::const_iterator it = m_tags.begin(); it != m_tags.end(); ++it) + { + if (it->first == key) + { + outValue = it->second; + return true; + } + } + return false; + } + + ///////////////////////////////////////////////////////// + OsmNode::OsmNode(OsmId id, OsmTags const & tags, double lat, double lon) + : OsmIdAndTagHolder(id, tags), m_lat(lat), m_lon(lon) + { + } + + ///////////////////////////////////////////////////////// + + OsmWay::OsmWay(OsmId id, OsmTags const & tags, OsmIds const & pointIds) + : OsmIdAndTagHolder(id, tags), m_points(pointIds) + { + CHECK_GREATER_OR_EQUAL(m_points.size(), 2, ("Can't construct a way with less than 2 points", id)); + } + + size_t OsmWay::PointsCount() const + { + return m_points.size(); + } + + bool OsmWay::IsClosed() const + { + return m_points.front() == m_points.back(); + } + + bool OsmWay::MergeWith(OsmWay const & way) + { + size_t const oldSize = m_points.size(); + + // first, try to connect our end with their start + if (m_points.back() == way.m_points.front()) + m_points.insert(m_points.end(), ++way.m_points.begin(), way.m_points.end()); + // our end and their end + else if (m_points.back() == way.m_points.back()) + m_points.insert(m_points.end(), ++way.m_points.rbegin(), way.m_points.rend()); + // our start and their end + else if (m_points.front() == way.m_points.back()) + m_points.insert(m_points.begin(), way.m_points.begin(), --way.m_points.end()); + // our start and their start + else if (m_points.front() == way.m_points.front()) + m_points.insert(m_points.begin(), way.m_points.rbegin(), --way.m_points.rend()); + + return m_points.size() > oldSize; + } + + /////////////////////////////////////////////////////////////// + + OsmRelation::OsmRelation(OsmId id, OsmTags const & tags, RelationMembers const & members) + : OsmIdAndTagHolder(id, tags), m_members(members) + { + CHECK(!m_members.empty(), ("Can't construct a relation without members", id)); + } + + OsmIds OsmRelation::MembersByTypeAndRole(string const & type, string const & role) const + { + OsmIds result; + for (RelationMembers::const_iterator it = m_members.begin(); it != m_members.end(); ++it) + if (it->m_type == type && it->m_role == role) + result.push_back(it->m_ref); + return result; + } + + /////////////////////////////////////////////////////////////// + + void OsmRawData::AddNode(OsmId id, OsmTags const & tags, double lat, double lon) + { + m_nodes.insert(nodes_type::value_type(id, OsmNode(id, tags, lat, lon))); + } + + void OsmRawData::AddWay(OsmId id, OsmTags const & tags, OsmIds const & nodeIds) + { + m_ways.insert(ways_type::value_type(id, OsmWay(id, tags, nodeIds))); + } + + void OsmRawData::AddRelation(OsmId id, OsmTags const & tags, RelationMembers const & members) + { + m_relations.insert(relations_type::value_type(id, OsmRelation(id, tags, members))); + } + + OsmNode OsmRawData::NodeById(OsmId id) const throw (OsmInvalidIdException) + { + nodes_type::const_iterator found = m_nodes.find(id); + if (found == m_nodes.end()) + MYTHROW( OsmInvalidIdException, (id, "node not found") ); + return found->second; + } + + OsmWay OsmRawData::WayById(OsmId id) const throw (OsmInvalidIdException) + { + ways_type::const_iterator found = m_ways.find(id); + if (found == m_ways.end()) + MYTHROW( OsmInvalidIdException, (id, "way not found") ); + return found->second; + } + + OsmRelation OsmRawData::RelationById(OsmId id) const throw (OsmInvalidIdException) + { + relations_type::const_iterator found = m_relations.find(id); + if (found == m_relations.end()) + MYTHROW( OsmInvalidIdException, (id, "relation not found") ); + return found->second; + } + + OsmIds OsmRawData::RelationsByKey(string const & key) const + { + OsmIds result; + string value; + for (relations_type::const_iterator it = m_relations.begin(); it != m_relations.end(); ++it) + { + if (it->second.TagValueByKey(key, value)) + result.push_back(it->first); + } + return result; + } + + OsmIds OsmRawData::RelationsByTag(OsmTag const & tag) const + { + OsmIds result; + string value; + for (relations_type::const_iterator it = m_relations.begin(); it != m_relations.end(); ++it) + { + if (it->second.TagValueByKey(tag.first, value) && value == tag.second) + result.push_back(it->first); + } + return result; + } + + ///////////////////////////////////////////////////////// + + OsmXmlParser::OsmXmlParser(OsmRawData & outData) + : m_osmRawData(outData) + { + } + + bool OsmXmlParser::Push(string const & element) + { + m_xmlTags.push_back(element); + + return true; + } + + void OsmXmlParser::Pop(string const & element) + { + if (element == "node") + { + m_osmRawData.AddNode(m_id, m_tags, m_lat, m_lon); + m_tags.clear(); + } + else if (element == "nd") + { + m_nds.push_back(m_ref); + } + else if (element == "way") + { + m_osmRawData.AddWay(m_id, m_tags, m_nds); + m_nds.clear(); + m_tags.clear(); + } + else if (element == "tag") + { + m_tags.push_back(OsmTag(m_k, m_v)); + } + else if (element == "member") + { + m_members.push_back(m_member); + } + else if (element == "relation") + { + m_osmRawData.AddRelation(m_id, m_tags, m_members); + m_members.clear(); + m_tags.clear(); + } + m_xmlTags.pop_back(); + } + + void OsmXmlParser::AddAttr(string const & attr, string const & value) + { + string const & elem = m_xmlTags.back(); + if (attr == "id" && (elem == "node" || elem == "way" || elem == "relation")) + { + uint64_t numVal; + CHECK(utils::to_uint64(value, numVal), ()); + m_id = static_cast<int64_t>(numVal); + } + else if (attr == "lat" && elem == "node") + { + CHECK(utils::to_double(value, m_lat), ()); + } + else if (attr == "lon" && elem == "node") + { + CHECK(utils::to_double(value, m_lon), ()); + } + else if (attr == "ref") + { + uint64_t numVal; + CHECK(utils::to_uint64(value, numVal), ()); + if (elem == "nd") + m_ref = static_cast<int64_t>(numVal); + else if (elem == "member") + m_member.m_ref = numVal; + } + else if (attr == "k" && elem == "tag") + { + m_k = value; + } + else if (attr == "v" && elem == "tag") + { + m_v = value; + } + else if (attr == "type" && elem == "member") + { + m_member.m_type = value; + } + else if (attr == "role" && elem == "member") + { + m_member.m_role = value; + } + } + +} diff --git a/generator/osm_xml_parser.hpp b/generator/osm_xml_parser.hpp new file mode 100644 index 0000000000..09e63076ea --- /dev/null +++ b/generator/osm_xml_parser.hpp @@ -0,0 +1,138 @@ +#pragma once + +#include "../base/exception.hpp" + +#include "../geometry/point2d.hpp" + +#include "../std/vector.hpp" +#include "../std/map.hpp" +#include "../std/set.hpp" +#include "../std/string.hpp" + +namespace osm +{ + typedef int64_t OsmId; + typedef vector<OsmId> OsmIds; + + typedef pair<string, string> OsmTag; + typedef vector<OsmTag> OsmTags; + + struct RelationMember + { + OsmId m_ref; + string m_type; + string m_role; + }; + typedef vector<RelationMember> RelationMembers; + + class OsmIdAndTagHolder + { + OsmId m_id; + OsmTags m_tags; + + public: + OsmIdAndTagHolder(OsmId id, OsmTags const & tags); + OsmId Id() const { return m_id; } + bool TagValueByKey(string const & key, string & outValue) const; + template <class TFunctor> void ForEachTag(TFunctor & functor) const + { + for (OsmTags::const_iterator it = m_tags.begin(); it != m_tags.end(); ++it) + functor(*it); + } + }; + + class OsmNode : public OsmIdAndTagHolder + { + public: + double m_lat; + double m_lon; + OsmNode(OsmId id, OsmTags const & tags, double lat, double lon); + }; + + class OsmWay : public OsmIdAndTagHolder + { + OsmIds m_points; + + public: + OsmWay(OsmId id, OsmTags const & tags, OsmIds const & pointIds); + + size_t PointsCount() const; + + /// checks if first and last points are equal + bool IsClosed() const; + + /// Merges ways if they have one common point + /// @warning do not use it where merged way direction is important! (coastlines for example) + bool MergeWith(OsmWay const & way); + + template <class TFunctor> void ForEachPoint(TFunctor & functor) const + { + for (typename OsmIds::const_iterator it = m_points.begin(); it != m_points.end(); ++it) + functor(*it); + } + }; + + typedef vector<OsmWay> OsmWays; + + class OsmRelation : public OsmIdAndTagHolder + { + RelationMembers m_members; + + public: + OsmRelation(OsmId id, OsmTags const & tags, RelationMembers const & members); + OsmIds MembersByTypeAndRole(string const & type, string const & role) const; + }; + + class OsmRawData + { + typedef map<OsmId, OsmNode> nodes_type; + nodes_type m_nodes; + typedef map<OsmId, OsmWay> ways_type; + ways_type m_ways; + typedef map<OsmId, OsmRelation> relations_type; + relations_type m_relations; + + public: + DECLARE_EXCEPTION(OsmInvalidIdException, RootException); + + void AddNode(OsmId id, OsmTags const & tags, double lat, double lon); + void AddWay(OsmId id, OsmTags const & tags, OsmIds const & nodeIds); + void AddRelation(OsmId id, OsmTags const & tags, RelationMembers const & members); + + OsmNode NodeById(OsmId id) const throw (OsmInvalidIdException); + OsmWay WayById(OsmId id) const throw (OsmInvalidIdException); + OsmRelation RelationById(OsmId id) const throw (OsmInvalidIdException); + + OsmIds RelationsByKey(string const & key) const; + OsmIds RelationsByTag(OsmTag const & tag) const; + }; + + class OsmXmlParser + { + vector<string> m_xmlTags; + + OsmRawData & m_osmRawData; + + OsmId m_id; + double m_lon; + double m_lat; + /// <tag k="..." v="..."/> + string m_k; + string m_v; + OsmTags m_tags; + /// <nd ref="..." /> + OsmId m_ref; + OsmIds m_nds; + RelationMember m_member; + RelationMembers m_members; + + public: + OsmXmlParser(OsmRawData & outData); + + bool Push(string const & element); + void Pop(string const & element); + void AddAttr(string const & attr, string const & value); + void CharData(string const &) {} + }; + +} diff --git a/generator/polygonizer.hpp b/generator/polygonizer.hpp index 8ce1963780..876fcfc966 100644 --- a/generator/polygonizer.hpp +++ b/generator/polygonizer.hpp @@ -1,5 +1,5 @@ #pragma once -#include "kml_parser.hpp" +#include "borders_loader.hpp" #include "world_map_generator.hpp" #include "../indexer/feature.hpp" @@ -42,7 +42,7 @@ namespace feature LOG(LINFO, ("Polygonizer thread pool threads:", m_ThreadPool.maxThreadCount())); #endif - CHECK(kml::LoadCountriesList(info.datFilePrefix, m_countries, info.simplifyCountriesLevel), + CHECK(borders::LoadCountriesList(info.datFilePrefix, m_countries, info.simplifyCountriesLevel), ("Error loading country polygons files")); //LOG_SHORT(LINFO, ("Loaded polygons count for regions:")); @@ -59,10 +59,10 @@ namespace feature struct PointChecker { - kml::RegionsContainerT const & m_regions; + borders::RegionsContainerT const & m_regions; bool m_belongs; - PointChecker(kml::RegionsContainerT const & regions) + PointChecker(borders::RegionsContainerT const & regions) : m_regions(regions), m_belongs(false) {} bool operator()(m2::PointD const & pt) @@ -71,7 +71,7 @@ namespace feature return !m_belongs; } - void operator() (kml::Region const & rgn, kml::Region::value_type const & point) + void operator() (borders::Region const & rgn, borders::Region::value_type const & point) { if (!m_belongs) m_belongs = rgn.Contains(point); @@ -80,12 +80,12 @@ namespace feature class InsertCountriesPtr { - typedef buffer_vector<kml::CountryPolygons const *, 32> vec_type; + typedef buffer_vector<borders::CountryPolygons const *, 32> vec_type; vec_type & m_vec; public: InsertCountriesPtr(vec_type & vec) : m_vec(vec) {} - void operator() (kml::CountryPolygons const & c) + void operator() (borders::CountryPolygons const & c) { m_vec.push_back(&c); } @@ -95,7 +95,7 @@ namespace feature { m_worldMap(fb); - buffer_vector<kml::CountryPolygons const *, 32> vec; + buffer_vector<borders::CountryPolygons const *, 32> vec; m_countries.ForEachInRect(fb.GetLimitRect(), InsertCountriesPtr(vec)); switch (vec.size()) @@ -125,7 +125,7 @@ namespace feature #endif } - void EmitFeature(kml::CountryPolygons const * country, FeatureBuilder1 const & fb) + void EmitFeature(borders::CountryPolygons const * country, FeatureBuilder1 const & fb) { #if PARALLEL_POLYGONIZER QMutexLocker mutexLocker(&m_EmitFeatureMutex); @@ -151,7 +151,7 @@ namespace feature vector<FeatureOutT*> m_Buckets; vector<string> m_Names; - kml::CountriesContainerT m_countries; + borders::CountriesContainerT m_countries; WorldMapGenerator<FeatureOutT> m_worldMap; #if PARALLEL_POLYGONIZER @@ -169,7 +169,7 @@ namespace feature { public: PolygonizerTask(Polygonizer * pPolygonizer, - buffer_vector<kml::CountryPolygons const *, 32> const & countries, + buffer_vector<borders::CountryPolygons const *, 32> const & countries, FeatureBuilder1 const & fb) : m_pPolygonizer(pPolygonizer), m_Countries(countries), m_FB(fb) {} @@ -196,7 +196,7 @@ namespace feature private: Polygonizer * m_pPolygonizer; - buffer_vector<kml::CountryPolygons const *, 32> m_Countries; + buffer_vector<borders::CountryPolygons const *, 32> m_Countries; FeatureBuilder1 m_FB; }; }; diff --git a/tools/unix/polygons.sh b/tools/unix/polygons.sh index b14c36f06d..db94911a1b 100755 --- a/tools/unix/polygons.sh +++ b/tools/unix/polygons.sh @@ -10,6 +10,7 @@ set -e -u -x # global params LIGHT_NODES=false PROCESSORS=4 +SIMPLIFY=-1 # displays usage and exits function Usage { @@ -46,13 +47,13 @@ BUCKETING_LEVEL=$2 #fi # check if we have QT in PATH -if [ ! `which qmake` ]; then - echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:' - echo ' 1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH' - echo ' 2. Set it permanently by adding export... string above to your ~/.bashrc' - echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc' - exit 0 -fi +#if [ ! `which qmake` ]; then +# echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:' +# echo ' 1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH' +# echo ' 2. Set it permanently by adding export... string above to your ~/.bashrc' +# echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc' +# exit 0 +#fi # determine script path MY_PATH=`dirname $0` @@ -115,7 +116,7 @@ fi # 2nd pass - not paralleled $PV $OSM_BZ2 | bzip2 -d | $GENERATOR_TOOL --intermediate_data_path=$TMPDIR \ - --use_light_nodes=$LIGHT_NODES --split_by_polygons -simplify_countries_level=10 \ + --use_light_nodes=$LIGHT_NODES --split_by_polygons --simplify_countries_level=$SIMPLIFY \ --generate_features --generate_world_scale=6 --merge_coastlines=true \ --data_path=$DATA_PATH |