diff options
| -rw-r--r-- | generator/altitude_generator.cpp | 145 | ||||
| -rw-r--r-- | generator/altitude_generator.hpp | 16 | ||||
| -rw-r--r-- | generator/generator_tests/altitude_test.cpp | 4 | ||||
| -rw-r--r-- | generator/generator_tool/generator_tool.cpp | 5 | ||||
| -rw-r--r-- | indexer/altitude_loader.cpp | 33 | ||||
| -rw-r--r-- | indexer/altitude_loader.hpp | 8 | ||||
| -rw-r--r-- | indexer/feature_altitude.hpp | 25 | ||||
| -rw-r--r-- | routing/bicycle_model.cpp | 2 | ||||
| -rw-r--r-- | routing/bicycle_model.hpp | 2 | ||||
| -rw-r--r-- | routing/pedestrian_model.cpp | 2 | ||||
| -rw-r--r-- | routing/pedestrian_model.hpp | 2 | ||||
| -rw-r--r-- | routing/routing_helpers.hpp | 4 |
12 files changed, 117 insertions, 131 deletions
diff --git a/generator/altitude_generator.cpp b/generator/altitude_generator.cpp index cf77e5b239..2a282ece0d 100644 --- a/generator/altitude_generator.cpp +++ b/generator/altitude_generator.cpp @@ -13,6 +13,7 @@ #include "coding/file_name_utils.hpp" #include "coding/read_write_utils.hpp" #include "coding/reader.hpp" +#include "coding/succinct_mapper.hpp" #include "coding/varint.hpp" #include "geometry/latlon.hpp" @@ -42,7 +43,7 @@ namespace { using namespace routing; -AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 1; +AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 0; class SrtmGetter : public IAltitudeGetter { @@ -72,13 +73,16 @@ public: TFeatureAltitudes const & GetFeatureAltitudes() const { return m_featureAltitudes; } - vector<bool> const & GetAltitudeAvailability() const { return m_altitudeAvailability; } + succinct::bit_vector_builder & GetAltitudeAvailabilityBuilder() + { + return m_altitudeAvailabilityBuilder; + } TAltitude GetMinAltitude() const { return m_minAltitude; } void operator()(FeatureType const & f, uint32_t const & id) { - if (id != m_altitudeAvailability.size()) + if (id != m_altitudeAvailabilityBuilder.size()) { LOG(LERROR, ("There's a gap in feature id order.")); return; @@ -87,7 +91,7 @@ public: bool hasAltitude = false; MY_SCOPE_GUARD(removeTmpDir, [&] () { - m_altitudeAvailability.push_back(hasAltitude); + m_altitudeAvailabilityBuilder.push_back(hasAltitude); }); if (!routing::IsRoad(feature::TypesHolder(f))) @@ -134,120 +138,89 @@ public: return !m_featureAltitudes.empty(); } - void SortFeatureAltitudes() + bool IsFeatureAltitudesSorted() { - sort(m_featureAltitudes.begin(), m_featureAltitudes.end(), my::LessBy(&Processor::TFeatureAltitude::first)); + return is_sorted(m_featureAltitudes.begin(), m_featureAltitudes.end(), + my::LessBy(&Processor::TFeatureAltitude::first)); } private: IAltitudeGetter & m_altitudeGetter; TFeatureAltitudes m_featureAltitudes; - vector<bool> m_altitudeAvailability; + succinct::bit_vector_builder m_altitudeAvailabilityBuilder; TAltitude m_minAltitude; }; - -uint32_t GetFileSize(string const & filePath) -{ - uint64_t size; - if (!my::GetFileSize(filePath, size)) - { - LOG(LERROR, (filePath, "Unable to get file size")); - return 0; - } - - LOG(LINFO, (filePath, "size =", size, "bytes")); - return size; -} - -void MoveFileToAltitudeSection(string const & filePath, uint32_t fileSize, FileWriter & w) -{ - { - ReaderSource<FileReader> r = FileReader(filePath); - w.Write(&fileSize, sizeof(fileSize)); - rw::ReadAndWrite(r, w); - LOG(LINFO, (filePath, "size is", fileSize)); - } - FileWriter::DeleteFileX(filePath); -} } // namespace +static_assert(sizeof(AltitudeHeader) == 16, "Wrong header size of altitude section."); + namespace routing { -void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAltitudeGetter & altitudeGetter) +void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter) { try { // Preparing altitude information. - string const mwmPath = my::JoinFoldersToPath(baseDir, countryName + DATA_FILE_EXTENSION); - Processor processor(altitudeGetter); feature::ForEachFromDat(mwmPath, processor); if (!processor.HasAltitudeInfo()) { - LOG(LINFO, ("No altitude information for road features of mwm", countryName)); + LOG(LINFO, ("No altitude information for road features of mwm:", mwmPath)); return; } - processor.SortFeatureAltitudes(); - Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes(); + CHECK(processor.IsFeatureAltitudesSorted(), ()); - // Writing compressed bit vector with features which have altitude information. - succinct::rs_bit_vector altitudeAvailability(processor.GetAltitudeAvailability()); - string const altitudeAvailabilityPath = my::JoinFoldersToPath(baseDir, "altitude_availability.bitvector"); - LOG(LINFO, ("altitudeAvailability succinct::mapper::size_of(altitudeAvailability) =", succinct::mapper::size_of(altitudeAvailability))); - succinct::mapper::freeze(altitudeAvailability, altitudeAvailabilityPath.c_str()); + FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING); + FileWriter w = cont.GetWriter(ALTITUDES_FILE_TAG); + AltitudeHeader header; + header.minAltitude = processor.GetMinAltitude(); + int64_t const startOffset = w.Pos(); + header.Serialize(w); + { + // Altitude availability serialization. + coding::FreezeVisitor<Writer> visitor(w); + succinct::bit_vector_builder & builder = processor.GetAltitudeAvailabilityBuilder(); + succinct::rs_bit_vector(&builder).map(visitor); + } + header.featureTableOffset = w.Pos() - startOffset; - // Writing feature altitude information to a file and memorizing the offsets. - string const altitudeInfoPath = my::JoinFoldersToPath(baseDir, "altitude_info"); vector<uint32_t> offsets; - TAltitude const minAltitude = processor.GetMinAltitude(); - offsets.reserve(featureAltitudes.size()); + vector<uint8_t> deltas; { - FileWriter altitudeInfoW(altitudeInfoPath); + // Altitude info serialization to memory. + MemWriter<vector<uint8_t>> writer(deltas); + Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes(); for (auto const & a : featureAltitudes) { - offsets.push_back(altitudeInfoW.Pos()); - // Feature altitude serializing. - a.second.Serialize(minAltitude, altitudeInfoW); + offsets.push_back(writer.Pos()); + a.second.Serialize(header.minAltitude, writer); } } - LOG(LINFO, ("Altitude was written for", featureAltitudes.size(), "features.")); - - // Writing feature altitude offsets. - CHECK(is_sorted(offsets.begin(), offsets.end()), ()); - CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ()); - LOG(LINFO, ("Max altitude info offset =", offsets.back(), "number of offsets = =", offsets.size())); - succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size()); - for (uint32_t offset : offsets) - builder.push_back(offset); - - succinct::elias_fano featureTable(&builder); - string const featuresTablePath = my::JoinFoldersToPath(baseDir, "altitude_offsets.elias_fano"); - succinct::mapper::freeze(featureTable, featuresTablePath.c_str()); + { + // Altitude offsets serialization. + CHECK(is_sorted(offsets.begin(), offsets.end()), ()); + CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ()); - uint32_t const altitudeAvailabilitySize = GetFileSize(altitudeAvailabilityPath); - uint32_t const altitudeInfoSize = GetFileSize(altitudeInfoPath); - uint32_t const featuresTableSize = GetFileSize(featuresTablePath); + succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size()); + for (uint32_t offset : offsets) + builder.push_back(offset); - FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING); - FileWriter w = cont.GetWriter(ALTITUDES_FILE_TAG); - - // Writing section with altitude information. - // Writing altitude section header. - TAltitudeSectionOffset const headerSize = AltitudeHeader::GetHeaderSize(); - TAltitudeSectionOffset const featuresTableOffset = headerSize + - sizeof(altitudeAvailabilitySize) + altitudeAvailabilitySize; - TAltitudeSectionOffset const altitudeInfoOffset = featuresTableOffset + - sizeof(featuresTableSize) + featuresTableSize; - AltitudeHeader header(kAltitudeSectionVersion, processor.GetMinAltitude(), - altitudeInfoOffset + sizeof(TAltitudeSectionOffset) /* for altitude info size */); + coding::FreezeVisitor<Writer> visitor(w); + succinct::elias_fano(&builder).map(visitor); + } + // Writing altitude info. + header.altitudeInfoOffset = w.Pos() - startOffset; + w.Write(deltas.data(), deltas.size()); + header.endOffset = w.Pos() - startOffset; + + // Rewriting header info. + header.version = kAltitudeSectionVersion; + int64_t const endOffset = w.Pos(); + w.Seek(startOffset); header.Serialize(w); - - // Copying parts of altitude sections to mwm. - MoveFileToAltitudeSection(altitudeAvailabilityPath, altitudeAvailabilitySize, w); - MoveFileToAltitudeSection(featuresTablePath, featuresTableSize, w); - MoveFileToAltitudeSection(altitudeInfoPath, altitudeInfoSize, w); + w.Seek(endOffset); } catch (RootException const & e) { @@ -255,10 +228,10 @@ void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAlt } } -void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, string const & countryName) +void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath) { - LOG(LINFO, ("srtmPath =", srtmPath, "baseDir =", baseDir, "countryName =", countryName)); + LOG(LINFO, ("mwmPath =", mwmPath, "srtmPath =", srtmPath)); SrtmGetter srtmGetter(srtmPath); - BuildRoadAltitudes(baseDir, countryName, srtmGetter); + BuildRoadAltitudes(mwmPath, srtmGetter); } } // namespace routing diff --git a/generator/altitude_generator.hpp b/generator/altitude_generator.hpp index 78cc1c903e..2dbd7cf147 100644 --- a/generator/altitude_generator.hpp +++ b/generator/altitude_generator.hpp @@ -14,8 +14,16 @@ public: virtual feature::TAltitude GetAltitude(m2::PointD const & p) = 0; }; -void BuildRoadAltitudes(string const & baseDir, string const & countryName, - IAltitudeGetter & altitudeGetter); -void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, - string const & countryName); +/// \brief Adds altitude section to mwm. It has the following format: +/// File offset (bytes) Field name Field size (bytes) +/// 0 version 2 +/// 2 min altitude 2 +/// 4 feature table offset 4 +/// 8 altitude info offset 4 +/// 12 end of section 4 +/// 16 altitude availability feat. table offset - 16 +/// feat. table offset feature table alt. info offset - f. table offset +/// alt. info offset altitude info alt. info offset - end of section +void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter); +void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath); } // namespace routing diff --git a/generator/generator_tests/altitude_test.cpp b/generator/generator_tests/altitude_test.cpp index 97d69e14b5..a6f407ef34 100644 --- a/generator/generator_tests/altitude_test.cpp +++ b/generator/generator_tests/altitude_test.cpp @@ -180,7 +180,8 @@ void TestAltitudeSection(vector<TRounded3DGeom> const & roadFeatures) MockAltitudeGetter altitudeGetter(move(altitudes)); // Adding altitude section to mwm. - BuildRoadAltitudes(testDirFullPath, kTestMwm, altitudeGetter); + string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION); + BuildRoadAltitudes(mwmPath, altitudeGetter); // Reading from mwm and testing altitue information. Index index; @@ -190,7 +191,6 @@ void TestAltitudeSection(vector<TRounded3DGeom> const & roadFeatures) MwmSet::MwmHandle mwmHandle = index.GetMwmHandleById(regResult.first); CHECK(mwmHandle.IsAlive(), ()); - string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION); ReadAndTestAltitudeInfo(*mwmHandle.GetValue<MwmValue>(), mwmPath, altitudeGetter); } } // namespace diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 27b4d41767..653b8cea5e 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -250,7 +250,10 @@ int main(int argc, char ** argv) feature::DumpFeatureNames(datFile, FLAGS_dump_feature_names); if (!FLAGS_srtm_path.empty()) - routing::BuildRoadAltitudes(FLAGS_srtm_path, path, FLAGS_output); + { + string const mwmPath = my::JoinFoldersToPath(path, FLAGS_output + DATA_FILE_EXTENSION); + routing::BuildRoadAltitudes(FLAGS_srtm_path, mwmPath); + } if (FLAGS_unpack_mwm) UnpackMwm(datFile); diff --git a/indexer/altitude_loader.cpp b/indexer/altitude_loader.cpp index 84ffc8a72d..6eab7b38e8 100644 --- a/indexer/altitude_loader.cpp +++ b/indexer/altitude_loader.cpp @@ -1,6 +1,7 @@ #include "indexer/altitude_loader.hpp" #include "coding/reader.hpp" +#include "coding/succinct_mapper.hpp" #include "base/logging.hpp" #include "base/stl_helpers.hpp" @@ -12,18 +13,15 @@ namespace { -void ReadBuffer(ReaderSource<FilesContainerR::TReader> & src, vector<char> & buf) +template<class TCont> +void Map(size_t dataSize, ReaderSource<FilesContainerR::TReader> & src, + TCont & cont, unique_ptr<CopiedMemoryRegion> & region) { - uint32_t bufSz = 0; - src.Read(&bufSz, sizeof(bufSz)); - if (bufSz > src.Size() + src.Pos()) - { - ASSERT(false, ()); - return; - } - buf.clear(); - buf.resize(bufSz); - src.Read(buf.data(), bufSz); + vector<uint8_t> data(dataSize); + src.Read(data.data(), data.size()); + region = make_unique<CopiedMemoryRegion>(move(data)); + coding::MapVisitor visitor(region->ImmutableData()); + cont.map(visitor); } } // namespace @@ -43,13 +41,8 @@ AltitudeLoader::AltitudeLoader(MwmValue const & mwmValue) ReaderSource<FilesContainerR::TReader> src(*m_reader); m_header.Deserialize(src); - // Reading src_bit_vector with altitude availability information. - ReadBuffer(src, m_altitudeAvailabilitBuf); - succinct::mapper::map(m_altitudeAvailability, m_altitudeAvailabilitBuf.data()); - - // Reading table with altitude offsets for features. - ReadBuffer(src, m_featureTableBuf); - succinct::mapper::map(m_featureTable, m_featureTableBuf.data()); + Map(m_header.GetAltitudeAvailabilitySize(), src, m_altitudeAvailability, m_altitudeAvailabilityRegion); + Map(m_header.GetFeatureTableSize(), src, m_featureTable, m_featureTableRegion); } catch (Reader::OpenException const & e) { @@ -60,12 +53,12 @@ AltitudeLoader::AltitudeLoader(MwmValue const & mwmValue) bool AltitudeLoader::IsAvailable() const { - return m_header.minAltitude != kInvalidAltitude && m_header.altitudeInfoOffset != 0; + return m_header.minAltitude != kInvalidAltitude; } TAltitudes const & AltitudeLoader::GetAltitudes(uint32_t featureId, size_t pointCount) const { - if (m_header.altitudeInfoOffset == 0) + if (!IsAvailable()) { // The version of mwm is less then version::Format::v8 or there's no altitude section in mwm. return m_dummy; diff --git a/indexer/altitude_loader.hpp b/indexer/altitude_loader.hpp index 4588e620c9..964b19bc5a 100644 --- a/indexer/altitude_loader.hpp +++ b/indexer/altitude_loader.hpp @@ -2,6 +2,8 @@ #include "indexer/feature_altitude.hpp" #include "indexer/index.hpp" +#include "coding/memory_region.hpp" + #include "std/unique_ptr.hpp" #include "std/vector.hpp" @@ -18,10 +20,12 @@ public: bool IsAvailable() const; private: - vector<char> m_altitudeAvailabilitBuf; - vector<char> m_featureTableBuf; + unique_ptr<CopiedMemoryRegion> m_altitudeAvailabilityRegion; + unique_ptr<CopiedMemoryRegion> m_featureTableRegion; + succinct::rs_bit_vector m_altitudeAvailability; succinct::elias_fano m_featureTable; + unique_ptr<FilesContainerR::TReader> m_reader; mutable map<uint32_t, TAltitudes> m_cache; TAltitudes const m_dummy; diff --git a/indexer/feature_altitude.hpp b/indexer/feature_altitude.hpp index 8cb8502501..3b225d04ee 100644 --- a/indexer/feature_altitude.hpp +++ b/indexer/feature_altitude.hpp @@ -1,7 +1,7 @@ #pragma once #include "coding/varint.hpp" -#include "base/logging.hpp" +#include "base/assert.hpp" #include "std/cstdint.hpp" #include "std/limits.hpp" @@ -24,17 +24,14 @@ struct AltitudeHeader Reset(); } - AltitudeHeader(TAltitudeSectionVersion v, TAltitude min, TAltitudeSectionOffset offset) - : version(v), minAltitude(min), altitudeInfoOffset(offset) - { - } - template <class TSink> void Serialize(TSink & sink) const { sink.Write(&version, sizeof(version)); sink.Write(&minAltitude, sizeof(minAltitude)); + sink.Write(&featureTableOffset, sizeof(featureTableOffset)); sink.Write(&altitudeInfoOffset, sizeof(altitudeInfoOffset)); + sink.Write(&endOffset, sizeof(endOffset)); } template <class TSource> @@ -42,24 +39,32 @@ struct AltitudeHeader { src.Read(&version, sizeof(version)); src.Read(&minAltitude, sizeof(minAltitude)); + src.Read(&featureTableOffset, sizeof(featureTableOffset)); src.Read(&altitudeInfoOffset, sizeof(altitudeInfoOffset)); + src.Read(&endOffset, sizeof(endOffset)); } - static size_t GetHeaderSize() - { - return sizeof(version) + sizeof(minAltitude) + sizeof(altitudeInfoOffset); - } + // Methods below return sizes of parts of altitude section in bytes. + static size_t GetHeaderSize() { return sizeof(AltitudeHeader); } + + size_t GetAltitudeAvailabilitySize() const { return featureTableOffset - GetHeaderSize(); } + size_t GetFeatureTableSize() const { return altitudeInfoOffset - featureTableOffset; } + size_t GetAltitudeInfo() const { return endOffset - altitudeInfoOffset; } void Reset() { version = 1; minAltitude = kInvalidAltitude; + featureTableOffset = 0; altitudeInfoOffset = 0; + endOffset = 0; } TAltitudeSectionVersion version; TAltitude minAltitude; + TAltitudeSectionOffset featureTableOffset; TAltitudeSectionOffset altitudeInfoOffset; + TAltitudeSectionOffset endOffset; }; class Altitude diff --git a/routing/bicycle_model.cpp b/routing/bicycle_model.cpp index ac8a867d43..997bcb6543 100644 --- a/routing/bicycle_model.cpp +++ b/routing/bicycle_model.cpp @@ -639,7 +639,7 @@ bool BicycleModel::IsOneWay(FeatureType const & f) const } // static -BicycleModel const & BicycleModel::DefaultInstance() +BicycleModel const & BicycleModel::AllLimitsInstance() { static BicycleModel const instance; return instance; diff --git a/routing/bicycle_model.hpp b/routing/bicycle_model.hpp index 3765abb9f3..9293fcc1cd 100644 --- a/routing/bicycle_model.hpp +++ b/routing/bicycle_model.hpp @@ -17,7 +17,7 @@ public: /// VehicleModel overrides: bool IsOneWay(FeatureType const & f) const override; - static BicycleModel const & DefaultInstance(); + static BicycleModel const & AllLimitsInstance(); protected: RoadAvailability GetRoadAvailability(feature::TypesHolder const & types) const override; diff --git a/routing/pedestrian_model.cpp b/routing/pedestrian_model.cpp index 7947e0b999..285ed91d38 100644 --- a/routing/pedestrian_model.cpp +++ b/routing/pedestrian_model.cpp @@ -643,7 +643,7 @@ IVehicleModel::RoadAvailability PedestrianModel::GetRoadAvailability(feature::Ty } // static -PedestrianModel const & PedestrianModel::DefaultInstance() +PedestrianModel const & PedestrianModel::AllLimitsInstance() { static PedestrianModel const instance; return instance; diff --git a/routing/pedestrian_model.hpp b/routing/pedestrian_model.hpp index db0b392451..f28eba80e1 100644 --- a/routing/pedestrian_model.hpp +++ b/routing/pedestrian_model.hpp @@ -17,7 +17,7 @@ public: /// VehicleModel overrides: bool IsOneWay(FeatureType const &) const override { return false; } - static PedestrianModel const & DefaultInstance(); + static PedestrianModel const & AllLimitsInstance(); protected: RoadAvailability GetRoadAvailability(feature::TypesHolder const & types) const override; diff --git a/routing/routing_helpers.hpp b/routing/routing_helpers.hpp index 49acc0417a..98365f39c4 100644 --- a/routing/routing_helpers.hpp +++ b/routing/routing_helpers.hpp @@ -11,7 +11,7 @@ template <class TTypes> bool IsRoad(TTypes const & types) { return CarModel::Instance().HasRoadType(types) || - PedestrianModel::DefaultInstance().HasRoadType(types) || - BicycleModel::DefaultInstance().HasRoadType(types); + PedestrianModel::AllLimitsInstance().HasRoadType(types) || + BicycleModel::AllLimitsInstance().HasRoadType(types); } } // namespace rouing |