From 979f48bfaf7322320801db215b1d0e3d231f8699 Mon Sep 17 00:00:00 2001 From: Vladimir Byko-Ianko Date: Sat, 23 Jul 2016 21:38:16 +0300 Subject: Writing altitude section without intermediate files and using MemoryRegion for reading it. --- generator/altitude_generator.cpp | 145 +++++++++++----------------- generator/altitude_generator.hpp | 16 ++- generator/generator_tests/altitude_test.cpp | 4 +- generator/generator_tool/generator_tool.cpp | 5 +- 4 files changed, 77 insertions(+), 93 deletions(-) (limited to 'generator') diff --git a/generator/altitude_generator.cpp b/generator/altitude_generator.cpp index cf77e5b239..2a282ece0d 100644 --- a/generator/altitude_generator.cpp +++ b/generator/altitude_generator.cpp @@ -13,6 +13,7 @@ #include "coding/file_name_utils.hpp" #include "coding/read_write_utils.hpp" #include "coding/reader.hpp" +#include "coding/succinct_mapper.hpp" #include "coding/varint.hpp" #include "geometry/latlon.hpp" @@ -42,7 +43,7 @@ namespace { using namespace routing; -AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 1; +AltitudeHeader::TAltitudeSectionVersion constexpr kAltitudeSectionVersion = 0; class SrtmGetter : public IAltitudeGetter { @@ -72,13 +73,16 @@ public: TFeatureAltitudes const & GetFeatureAltitudes() const { return m_featureAltitudes; } - vector const & GetAltitudeAvailability() const { return m_altitudeAvailability; } + succinct::bit_vector_builder & GetAltitudeAvailabilityBuilder() + { + return m_altitudeAvailabilityBuilder; + } TAltitude GetMinAltitude() const { return m_minAltitude; } void operator()(FeatureType const & f, uint32_t const & id) { - if (id != m_altitudeAvailability.size()) + if (id != m_altitudeAvailabilityBuilder.size()) { LOG(LERROR, ("There's a gap in feature id order.")); return; @@ -87,7 +91,7 @@ public: bool hasAltitude = false; MY_SCOPE_GUARD(removeTmpDir, [&] () { - m_altitudeAvailability.push_back(hasAltitude); + m_altitudeAvailabilityBuilder.push_back(hasAltitude); }); if (!routing::IsRoad(feature::TypesHolder(f))) @@ -134,120 +138,89 @@ public: return !m_featureAltitudes.empty(); } - void SortFeatureAltitudes() + bool IsFeatureAltitudesSorted() { - sort(m_featureAltitudes.begin(), m_featureAltitudes.end(), my::LessBy(&Processor::TFeatureAltitude::first)); + return is_sorted(m_featureAltitudes.begin(), m_featureAltitudes.end(), + my::LessBy(&Processor::TFeatureAltitude::first)); } private: IAltitudeGetter & m_altitudeGetter; TFeatureAltitudes m_featureAltitudes; - vector m_altitudeAvailability; + succinct::bit_vector_builder m_altitudeAvailabilityBuilder; TAltitude m_minAltitude; }; - -uint32_t GetFileSize(string const & filePath) -{ - uint64_t size; - if (!my::GetFileSize(filePath, size)) - { - LOG(LERROR, (filePath, "Unable to get file size")); - return 0; - } - - LOG(LINFO, (filePath, "size =", size, "bytes")); - return size; -} - -void MoveFileToAltitudeSection(string const & filePath, uint32_t fileSize, FileWriter & w) -{ - { - ReaderSource r = FileReader(filePath); - w.Write(&fileSize, sizeof(fileSize)); - rw::ReadAndWrite(r, w); - LOG(LINFO, (filePath, "size is", fileSize)); - } - FileWriter::DeleteFileX(filePath); -} } // namespace +static_assert(sizeof(AltitudeHeader) == 16, "Wrong header size of altitude section."); + namespace routing { -void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAltitudeGetter & altitudeGetter) +void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter) { try { // Preparing altitude information. - string const mwmPath = my::JoinFoldersToPath(baseDir, countryName + DATA_FILE_EXTENSION); - Processor processor(altitudeGetter); feature::ForEachFromDat(mwmPath, processor); if (!processor.HasAltitudeInfo()) { - LOG(LINFO, ("No altitude information for road features of mwm", countryName)); + LOG(LINFO, ("No altitude information for road features of mwm:", mwmPath)); return; } - processor.SortFeatureAltitudes(); - Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes(); + CHECK(processor.IsFeatureAltitudesSorted(), ()); - // Writing compressed bit vector with features which have altitude information. - succinct::rs_bit_vector altitudeAvailability(processor.GetAltitudeAvailability()); - string const altitudeAvailabilityPath = my::JoinFoldersToPath(baseDir, "altitude_availability.bitvector"); - LOG(LINFO, ("altitudeAvailability succinct::mapper::size_of(altitudeAvailability) =", succinct::mapper::size_of(altitudeAvailability))); - succinct::mapper::freeze(altitudeAvailability, altitudeAvailabilityPath.c_str()); + FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING); + FileWriter w = cont.GetWriter(ALTITUDES_FILE_TAG); + AltitudeHeader header; + header.minAltitude = processor.GetMinAltitude(); + int64_t const startOffset = w.Pos(); + header.Serialize(w); + { + // Altitude availability serialization. + coding::FreezeVisitor visitor(w); + succinct::bit_vector_builder & builder = processor.GetAltitudeAvailabilityBuilder(); + succinct::rs_bit_vector(&builder).map(visitor); + } + header.featureTableOffset = w.Pos() - startOffset; - // Writing feature altitude information to a file and memorizing the offsets. - string const altitudeInfoPath = my::JoinFoldersToPath(baseDir, "altitude_info"); vector offsets; - TAltitude const minAltitude = processor.GetMinAltitude(); - offsets.reserve(featureAltitudes.size()); + vector deltas; { - FileWriter altitudeInfoW(altitudeInfoPath); + // Altitude info serialization to memory. + MemWriter> writer(deltas); + Processor::TFeatureAltitudes const & featureAltitudes = processor.GetFeatureAltitudes(); for (auto const & a : featureAltitudes) { - offsets.push_back(altitudeInfoW.Pos()); - // Feature altitude serializing. - a.second.Serialize(minAltitude, altitudeInfoW); + offsets.push_back(writer.Pos()); + a.second.Serialize(header.minAltitude, writer); } } - LOG(LINFO, ("Altitude was written for", featureAltitudes.size(), "features.")); - - // Writing feature altitude offsets. - CHECK(is_sorted(offsets.begin(), offsets.end()), ()); - CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ()); - LOG(LINFO, ("Max altitude info offset =", offsets.back(), "number of offsets = =", offsets.size())); - succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size()); - for (uint32_t offset : offsets) - builder.push_back(offset); - - succinct::elias_fano featureTable(&builder); - string const featuresTablePath = my::JoinFoldersToPath(baseDir, "altitude_offsets.elias_fano"); - succinct::mapper::freeze(featureTable, featuresTablePath.c_str()); + { + // Altitude offsets serialization. + CHECK(is_sorted(offsets.begin(), offsets.end()), ()); + CHECK(adjacent_find(offsets.begin(), offsets.end()) == offsets.end(), ()); - uint32_t const altitudeAvailabilitySize = GetFileSize(altitudeAvailabilityPath); - uint32_t const altitudeInfoSize = GetFileSize(altitudeInfoPath); - uint32_t const featuresTableSize = GetFileSize(featuresTablePath); + succinct::elias_fano::elias_fano_builder builder(offsets.back(), offsets.size()); + for (uint32_t offset : offsets) + builder.push_back(offset); - FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING); - FileWriter w = cont.GetWriter(ALTITUDES_FILE_TAG); - - // Writing section with altitude information. - // Writing altitude section header. - TAltitudeSectionOffset const headerSize = AltitudeHeader::GetHeaderSize(); - TAltitudeSectionOffset const featuresTableOffset = headerSize + - sizeof(altitudeAvailabilitySize) + altitudeAvailabilitySize; - TAltitudeSectionOffset const altitudeInfoOffset = featuresTableOffset + - sizeof(featuresTableSize) + featuresTableSize; - AltitudeHeader header(kAltitudeSectionVersion, processor.GetMinAltitude(), - altitudeInfoOffset + sizeof(TAltitudeSectionOffset) /* for altitude info size */); + coding::FreezeVisitor visitor(w); + succinct::elias_fano(&builder).map(visitor); + } + // Writing altitude info. + header.altitudeInfoOffset = w.Pos() - startOffset; + w.Write(deltas.data(), deltas.size()); + header.endOffset = w.Pos() - startOffset; + + // Rewriting header info. + header.version = kAltitudeSectionVersion; + int64_t const endOffset = w.Pos(); + w.Seek(startOffset); header.Serialize(w); - - // Copying parts of altitude sections to mwm. - MoveFileToAltitudeSection(altitudeAvailabilityPath, altitudeAvailabilitySize, w); - MoveFileToAltitudeSection(featuresTablePath, featuresTableSize, w); - MoveFileToAltitudeSection(altitudeInfoPath, altitudeInfoSize, w); + w.Seek(endOffset); } catch (RootException const & e) { @@ -255,10 +228,10 @@ void BuildRoadAltitudes(string const & baseDir, string const & countryName, IAlt } } -void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, string const & countryName) +void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath) { - LOG(LINFO, ("srtmPath =", srtmPath, "baseDir =", baseDir, "countryName =", countryName)); + LOG(LINFO, ("mwmPath =", mwmPath, "srtmPath =", srtmPath)); SrtmGetter srtmGetter(srtmPath); - BuildRoadAltitudes(baseDir, countryName, srtmGetter); + BuildRoadAltitudes(mwmPath, srtmGetter); } } // namespace routing diff --git a/generator/altitude_generator.hpp b/generator/altitude_generator.hpp index 78cc1c903e..2dbd7cf147 100644 --- a/generator/altitude_generator.hpp +++ b/generator/altitude_generator.hpp @@ -14,8 +14,16 @@ public: virtual feature::TAltitude GetAltitude(m2::PointD const & p) = 0; }; -void BuildRoadAltitudes(string const & baseDir, string const & countryName, - IAltitudeGetter & altitudeGetter); -void BuildRoadAltitudes(string const & srtmPath, string const & baseDir, - string const & countryName); +/// \brief Adds altitude section to mwm. It has the following format: +/// File offset (bytes) Field name Field size (bytes) +/// 0 version 2 +/// 2 min altitude 2 +/// 4 feature table offset 4 +/// 8 altitude info offset 4 +/// 12 end of section 4 +/// 16 altitude availability feat. table offset - 16 +/// feat. table offset feature table alt. info offset - f. table offset +/// alt. info offset altitude info alt. info offset - end of section +void BuildRoadAltitudes(string const & mwmPath, IAltitudeGetter & altitudeGetter); +void BuildRoadAltitudes(string const & mwmPath, string const & srtmPath); } // namespace routing diff --git a/generator/generator_tests/altitude_test.cpp b/generator/generator_tests/altitude_test.cpp index 97d69e14b5..a6f407ef34 100644 --- a/generator/generator_tests/altitude_test.cpp +++ b/generator/generator_tests/altitude_test.cpp @@ -180,7 +180,8 @@ void TestAltitudeSection(vector const & roadFeatures) MockAltitudeGetter altitudeGetter(move(altitudes)); // Adding altitude section to mwm. - BuildRoadAltitudes(testDirFullPath, kTestMwm, altitudeGetter); + string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION); + BuildRoadAltitudes(mwmPath, altitudeGetter); // Reading from mwm and testing altitue information. Index index; @@ -190,7 +191,6 @@ void TestAltitudeSection(vector const & roadFeatures) MwmSet::MwmHandle mwmHandle = index.GetMwmHandleById(regResult.first); CHECK(mwmHandle.IsAlive(), ()); - string const mwmPath = my::JoinFoldersToPath(testDirFullPath, kTestMwm + DATA_FILE_EXTENSION); ReadAndTestAltitudeInfo(*mwmHandle.GetValue(), mwmPath, altitudeGetter); } } // namespace diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 27b4d41767..653b8cea5e 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -250,7 +250,10 @@ int main(int argc, char ** argv) feature::DumpFeatureNames(datFile, FLAGS_dump_feature_names); if (!FLAGS_srtm_path.empty()) - routing::BuildRoadAltitudes(FLAGS_srtm_path, path, FLAGS_output); + { + string const mwmPath = my::JoinFoldersToPath(path, FLAGS_output + DATA_FILE_EXTENSION); + routing::BuildRoadAltitudes(FLAGS_srtm_path, mwmPath); + } if (FLAGS_unpack_mwm) UnpackMwm(datFile); -- cgit v1.2.3