Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/mapsme/omim.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLev Dragunov <l.dragunov@corp.mail.ru>2015-03-05 19:14:29 +0300
committerAlex Zolotarev <alex@maps.me>2015-09-23 02:38:23 +0300
commitc30ae8eab34ab5de5e7684400f5e9032fa8f3736 (patch)
tree90fb41dd723229f9f761ac5f17f0e28013060845
parent3a3028d0599fd329dc0068f3487ae63a49380a4b (diff)
different file index pathes
-rw-r--r--android/jni/Android.mk7
-rw-r--r--android/jni/com/mapswithme/platform/Platform.cpp9
-rw-r--r--defines.hpp8
-rw-r--r--indexer/features_offsets_table.cpp53
-rw-r--r--indexer/features_offsets_table.hpp28
-rw-r--r--indexer/indexer_tests/features_offsets_table_test.cpp24
-rw-r--r--map/active_maps_layout.cpp1
-rw-r--r--map/framework.cpp14
-rw-r--r--map/framework.hpp2
-rw-r--r--platform/platform.hpp2
-rw-r--r--platform/platform_ios.mm10
-rw-r--r--platform/platform_linux.cpp9
-rw-r--r--platform/platform_mac.mm9
-rw-r--r--platform/platform_tizen.cpp9
-rw-r--r--platform/platform_win.cpp7
-rw-r--r--routing/osrm2feature_map.cpp68
-rw-r--r--routing/osrm2feature_map.hpp41
17 files changed, 191 insertions, 110 deletions
diff --git a/android/jni/Android.mk b/android/jni/Android.mk
index 9d30d9bba0..95769e01c6 100644
--- a/android/jni/Android.mk
+++ b/android/jni/Android.mk
@@ -29,6 +29,11 @@ LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libprotobuf.a
include $(PREBUILT_STATIC_LIBRARY)
include $(CLEAR_VARS)
+LOCAL_MODULE := succinct
+LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libsuccinct.a
+include $(PREBUILT_STATIC_LIBRARY)
+
+include $(CLEAR_VARS)
LOCAL_MODULE := tomcrypt
LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libtomcrypt.a
include $(PREBUILT_STATIC_LIBRARY)
@@ -137,7 +142,7 @@ include $(CLEAR_VARS)
LOCAL_CPP_FEATURES += exceptions rtti
LOCAL_MODULE := mapswithme
-LOCAL_STATIC_LIBRARIES := map gui routing search storage indexer graphics platform anim geometry coding base expat freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm stats_client
+LOCAL_STATIC_LIBRARIES := map gui routing search storage indexer graphics platform anim geometry coding base expat freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm stats_client succinct
LOCAL_CFLAGS := -ffunction-sections -fdata-sections -Wno-extern-c-compat
ifneq ($(NDK_DEBUG),1)
diff --git a/android/jni/com/mapswithme/platform/Platform.cpp b/android/jni/com/mapswithme/platform/Platform.cpp
index 2117cdde8a..7aaa4b8b6b 100644
--- a/android/jni/com/mapswithme/platform/Platform.cpp
+++ b/android/jni/com/mapswithme/platform/Platform.cpp
@@ -11,6 +11,7 @@
#include "../../../../../std/algorithm.hpp"
+#include <sys/stat.h>
string Platform::UniqueClientId() const
{
@@ -55,6 +56,14 @@ string Platform::UniqueClientId() const
return res;
}
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+ string dir = WritableDir() + country_name + '/';
+ if (!IsFileExistsByFullPath(dir))
+ mkdir(dir.c_str(), 0755);
+ return dir;
+}
+
void Platform::RunOnGuiThread(TFunctor const & fn)
{
android::Platform::RunOnGuiThreadImpl(fn);
diff --git a/defines.hpp b/defines.hpp
index 4a2000c7df..554df8e6cd 100644
--- a/defines.hpp
+++ b/defines.hpp
@@ -4,6 +4,7 @@
#define DATA_FILE_EXTENSION_TMP ".mwm.tmp"
#define FONT_FILE_EXTENSION ".ttf"
#define OSM2FEATURE_FILE_EXTENSION ".osm2ft"
+#define EXTENSION_TMP ".tmp"
#define DATA_FILE_TAG "dat"
#define GEOMETRY_FILE_TAG "geom"
@@ -14,7 +15,6 @@
#define VERSION_FILE_TAG "version"
#define METADATA_FILE_TAG "meta"
#define METADATA_INDEX_FILE_TAG "metaidx"
-#define FEATURES_OFFSETS_TABLE_FILE_TAG "offsets"
#define COMPRESSED_SEARCH_INDEX_FILE_TAG "csdx"
#define ROUTING_MATRIX_FILE_TAG "mercedes"
@@ -25,8 +25,10 @@
#define ROUTING_FTSEG_FILE_TAG "ftseg"
#define ROUTING_NODEIND_TO_FTSEGIND_FILE_TAG "node2ftseg"
-#define FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG "bftsegnodes"
-#define FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG "bftsegbits"
+
+#define FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT ".bftsegnodes"
+#define FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT ".bftsegbits"
+#define FEATURES_OFFSETS_TABLE_FILE_EXT ".offsets"
//Secret word to unlock experimental features in production builds
#define ROUTING_SECRET_UNLOCKING_WORD "?ariadna"
diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp
index c39a691834..96eda2fbb8 100644
--- a/indexer/features_offsets_table.cpp
+++ b/indexer/features_offsets_table.cpp
@@ -3,10 +3,12 @@
#include "../indexer/data_header.hpp"
#include "../indexer/features_vector.hpp"
#include "../coding/file_writer.hpp"
+#include "../coding/internal/file_data.hpp"
+#include "../platform/platform.hpp"
#include "../base/assert.hpp"
#include "../base/scope_guard.hpp"
#include "../std/string.hpp"
-#include "../defines.hpp"
+
namespace feature
{
@@ -21,10 +23,10 @@ namespace feature
{
}
- FeaturesOffsetsTable::FeaturesOffsetsTable(FilesMappingContainer::Handle && handle)
- : m_handle(move(handle))
+ FeaturesOffsetsTable::FeaturesOffsetsTable(string const & fileName)
{
- succinct::mapper::map(m_table, m_handle.GetData<char>());
+ m_pSrc = unique_ptr<MmapReader>(new MmapReader(fileName));
+ succinct::mapper::map(m_table, reinterpret_cast<char const *>(m_pSrc->Data()));
}
// static
@@ -43,26 +45,29 @@ namespace feature
}
// static
- unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(
- FilesMappingContainer const & container)
+ unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(string const & countryName)
{
- FilesMappingContainer::Handle handle(container.Map(FEATURES_OFFSETS_TABLE_FILE_TAG));
- if (!handle.IsValid())
+ string const fileName = GetIndexFileName(countryName);
+ uint64_t size;
+ if (!GetPlatform().GetFileSizeByFullPath(fileName, size))
return unique_ptr<FeaturesOffsetsTable>();
- return unique_ptr<FeaturesOffsetsTable>(new FeaturesOffsetsTable(std::move(handle)));
+ return unique_ptr<FeaturesOffsetsTable>(new FeaturesOffsetsTable(fileName));
}
// static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::CreateIfNotExistsAndLoad(
- FilesMappingContainer const & container)
+ string const & countryName)
{
- if (container.IsExist(FEATURES_OFFSETS_TABLE_FILE_TAG))
- return Load(container);
-
- if (!container.IsExist(HEADER_FILE_TAG))
+ string const fileName = GetIndexFileName(countryName);
+ uint64_t size;
+ if (GetPlatform().GetFileSizeByFullPath(fileName,size))
+ return Load(countryName);
+
+ string const mwmName = GetPlatform().WritablePathForFile(countryName + DATA_FILE_EXTENSION);
+ FilesContainerR cont(mwmName);
+ if (!cont.IsExist(HEADER_FILE_TAG))
return unique_ptr<FeaturesOffsetsTable>();
- FilesContainerR cont(container.GetName());
DataHeader header;
header.Load(cont.GetReader(HEADER_FILE_TAG));
@@ -72,17 +77,16 @@ namespace feature
builder.PushOffset(offset);
});
unique_ptr<FeaturesOffsetsTable> table(Build(builder));
- FilesContainerW writeCont(container.GetName(), FileWriter::OP_WRITE_EXISTING);
- table->Save(writeCont);
+ table->Save(countryName);
return table;
}
- void FeaturesOffsetsTable::Save(FilesContainerW & container)
+ void FeaturesOffsetsTable::Save(string const & countryName)
{
- string const fileName = container.GetFileName() + "." FEATURES_OFFSETS_TABLE_FILE_TAG;
- MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(fileName)));
- succinct::mapper::freeze(m_table, fileName.c_str());
- container.Write(fileName, FEATURES_OFFSETS_TABLE_FILE_TAG);
+ string const fileName = GetIndexFileName(countryName);
+ string const fileNameTmp = fileName + EXTENSION_TMP;
+ succinct::mapper::freeze(m_table, fileNameTmp.c_str());
+ my::RenameFileX(fileNameTmp, fileName);
}
uint64_t FeaturesOffsetsTable::GetFeatureOffset(size_t index) const
@@ -115,4 +119,9 @@ namespace feature
}
+ string FeaturesOffsetsTable::GetIndexFileName(string const & countryName)
+ {
+ return GetPlatform().WritablePathForFileIndexes(countryName) + countryName + FEATURES_OFFSETS_TABLE_FILE_EXT;
+ }
+
} // namespace feature
diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp
index ceff65f7e0..eb0537c6ba 100644
--- a/indexer/features_offsets_table.hpp
+++ b/indexer/features_offsets_table.hpp
@@ -1,6 +1,8 @@
#pragma once
#include "../coding/file_container.hpp"
+#include "../coding/mmap_reader.hpp"
+#include "../defines.hpp"
#include "../std/stdint.hpp"
#include "../std/unique_ptr.hpp"
#include "../std/vector.hpp"
@@ -51,11 +53,10 @@ namespace feature
/// mapped to the memory and used by internal structures of
/// FeaturesOffsetsTable.
///
- /// \param container a container with a section devoted to
- /// FeaturesOffsetsTable
+ /// \param countryName a countryName to save index file to
/// \return a pointer to an instance of FeaturesOffsetsTable or nullptr
/// when it's not possible to load FeaturesOffsetsTable.
- static unique_ptr<FeaturesOffsetsTable> Load(FilesMappingContainer const & container);
+ static unique_ptr<FeaturesOffsetsTable> Load(string const & countryName);
/// Loads FeaturesOffsetsTable from FilesMappingContainer. Note
/// that some part of a file referenced by container will be
@@ -66,19 +67,18 @@ namespace feature
///
/// \warning May take a lot of time if there is no precomputed section
///
- /// \param container a container with a section devoted to
- /// FeaturesOffsetsTable
+ /// \param countryName a country to create index to
/// \return a pointer to an instance of FeaturesOffsetsTable or nullptr
/// when it's not possible to create FeaturesOffsetsTable.
- static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(FilesMappingContainer const & container);
+ static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(string const & countryName);
FeaturesOffsetsTable(FeaturesOffsetsTable const &) = delete;
FeaturesOffsetsTable const & operator=(FeaturesOffsetsTable const &) = delete;
/// Serializes current instance to a section in container.
///
- /// \param container a container current instance will be serialized to
- void Save(FilesContainerW & container);
+ /// \param countryName a name of the country to create data
+ void Save(string const & countryName);
/// \param index index of a feature
/// \return offset a feature
@@ -102,13 +102,21 @@ namespace feature
return succinct::mapper::size_of(m_table);
}
+ /// Delete temporary index file
+ static void CleanIndexFiles(string const & countryName)
+ {
+ FileWriter::DeleteFileX(GetIndexFileName(countryName));
+ }
+
private:
+ static string GetIndexFileName(string const & countryName);
+
FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder);
- FeaturesOffsetsTable(FilesMappingContainer::Handle && handle);
+ FeaturesOffsetsTable(string const &);
succinct::elias_fano m_table;
- FilesMappingContainer::Handle m_handle;
+ unique_ptr<MmapReader> m_pSrc;
};
} // namespace feature
diff --git a/indexer/indexer_tests/features_offsets_table_test.cpp b/indexer/indexer_tests/features_offsets_table_test.cpp
index fc321bb12d..50e34e27a0 100644
--- a/indexer/indexer_tests/features_offsets_table_test.cpp
+++ b/indexer/indexer_tests/features_offsets_table_test.cpp
@@ -58,12 +58,11 @@ namespace feature
UNIT_TEST(FeaturesOffsetsTable_CreateIfNotExistsAndLoad)
{
+ string const testFileName = "minsk-pass";
Platform & p = GetPlatform();
- FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION));
- if (baseContainer.IsExist(FEATURES_OFFSETS_TABLE_FILE_TAG))
- FilesContainerW(baseContainer.GetFileName(), FileWriter::OP_WRITE_EXISTING).DeleteSection(FEATURES_OFFSETS_TABLE_FILE_TAG);
- FilesMappingContainer mappingContainer(baseContainer.GetFileName());
- unique_ptr<FeaturesOffsetsTable> table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(mappingContainer));
+ FilesContainerR baseContainer(p.GetReader(testFileName + DATA_FILE_EXTENSION));
+ FeaturesOffsetsTable::CleanIndexFiles(testFileName);
+ unique_ptr<FeaturesOffsetsTable> table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(testFileName));
TEST(table.get(), ());
feature::DataHeader header;
@@ -77,13 +76,13 @@ namespace feature
TEST_EQUAL(builderSize, table->size(), ());
table = unique_ptr<FeaturesOffsetsTable>();
- FilesMappingContainer newReadContainer(baseContainer.GetFileName());
- table = unique_ptr<FeaturesOffsetsTable>(FeaturesOffsetsTable::Load(newReadContainer));
+ table = unique_ptr<FeaturesOffsetsTable>(FeaturesOffsetsTable::Load(testFileName));
TEST_EQUAL(builderSize, table->size(), ());
}
UNIT_TEST(FeaturesOffsetsTable_ReadWrite)
{
+ string const testFileName = "test_file";
Platform & p = GetPlatform();
FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION));
@@ -100,7 +99,7 @@ namespace feature
TEST(table.get(), ());
TEST_EQUAL(builder.size(), table->size(), ());
- string const testFile = p.WritablePathForFile("test_file" DATA_FILE_EXTENSION);
+ string const testFile = p.WritablePathForFile(testFileName + DATA_FILE_EXTENSION);
MY_SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, cref(testFile)));
// Store table in a temporary data file.
@@ -111,19 +110,16 @@ namespace feature
// table section.
baseContainer.ForEachTag([&baseContainer, &testContainer](string const & tag)
{
- if (tag != FEATURES_OFFSETS_TABLE_FILE_TAG)
testContainer.Write(baseContainer.GetReader(tag), tag);
});
- table->Save(testContainer);
+ table->Save("test_file");
testContainer.Finish();
}
// Load table from the temporary data file.
{
- FilesMappingContainer testContainer(testFile);
- MY_SCOPE_GUARD(testContainerGuard, bind(&FilesMappingContainer::Close, &testContainer));
-
- unique_ptr<FeaturesOffsetsTable> loadedTable(FeaturesOffsetsTable::Load(testContainer));
+ MY_SCOPE_GUARD(testTableGuard, [&testFileName](){FeaturesOffsetsTable::CleanIndexFiles(testFileName);});
+ unique_ptr<FeaturesOffsetsTable> loadedTable(FeaturesOffsetsTable::Load(testFileName));
TEST(loadedTable.get(), ());
TEST_EQUAL(table->size(), loadedTable->size(), ());
diff --git a/map/active_maps_layout.cpp b/map/active_maps_layout.cpp
index f1e271fecc..58882fb194 100644
--- a/map/active_maps_layout.cpp
+++ b/map/active_maps_layout.cpp
@@ -355,6 +355,7 @@ void ActiveMapsLayout::StatusChangedCallback(TIndex const & index)
if (newStatus == TStatus::EOnDisk)
{
+ m_framework.DeleteCountryIndexes(index);
if (group != TGroup::EUpToDate)
{
// Here we handle
diff --git a/map/framework.cpp b/map/framework.cpp
index 9a450b416f..64dff2ab42 100644
--- a/map/framework.cpp
+++ b/map/framework.cpp
@@ -307,6 +307,20 @@ void Framework::DeleteCountry(TIndex const & index, TMapOptions opt)
m_storage.NotifyStatusChanged(index);
}
+
+ DeleteCountryIndexes(index);
+}
+
+void Framework::DeleteCountryIndexes(TIndex const & index)
+{
+ string const & file = m_storage.CountryByIndex(index).Name();
+ //Remove all indexes
+ m_routingSession.Reset();
+ Platform::FilesList files;
+ Platform const & pl = GetPlatform();
+ pl.GetFilesByRegExp(pl.WritablePathForFileIndexes(file), "*", files);
+ for (auto const & file : files)
+ my::DeleteFileX(file);
}
void Framework::DownloadCountry(TIndex const & index, TMapOptions opt)
diff --git a/map/framework.hpp b/map/framework.hpp
index c1c0d256bb..4040a43b46 100644
--- a/map/framework.hpp
+++ b/map/framework.hpp
@@ -170,6 +170,8 @@ public:
void DeleteCountry(storage::TIndex const & index, storage::TMapOptions opt);
/// options - flags that signal about parts of map that must be downloaded
void DownloadCountry(storage::TIndex const & index, storage::TMapOptions opt);
+ /// Deletes user calculated indexes on country updates
+ void DeleteCountryIndexes(storage::TIndex const & index);
storage::TStatus GetCountryStatus(storage::TIndex const & index) const;
string GetCountryName(storage::TIndex const & index) const;
diff --git a/platform/platform.hpp b/platform/platform.hpp
index e1d26d430b..6605e7b852 100644
--- a/platform/platform.hpp
+++ b/platform/platform.hpp
@@ -60,6 +60,8 @@ public:
string WritableDir() const { return m_writableDir; }
/// @return full path to file in user's writable directory
string WritablePathForFile(string const & file) const { return WritableDir() + file; }
+ /// @return full path to indexes directory for country file. Creates directory if it's not exists.
+ string WritablePathForFileIndexes(string const & country_name) const;
/// @return resource dir (on some platforms it's differ from Writable dir)
string ResourcesDir() const { return m_resourcesDir; }
diff --git a/platform/platform_ios.mm b/platform/platform_ios.mm
index c83e10c29c..302013ab53 100644
--- a/platform/platform_ios.mm
+++ b/platform/platform_ios.mm
@@ -3,6 +3,7 @@
#include "constants.hpp"
#include "../coding/file_reader.hpp"
+#include "../coding/file_name_utils.hpp"
#include <sys/types.h>
#include <sys/socket.h>
@@ -49,6 +50,15 @@ Platform::Platform()
[pool release];
}
+
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+ string dir = m_writableDir + country_name.c_str() + '/';
+ if (!IsFileExistsByFullPath(dir))
+ ::mkdir(dir.c_str(), 0755);
+ return dir;
+}
+
void Platform::GetFilesByRegExp(string const & directory, string const & regexp, FilesList & res)
{
pl::EnumerateFilesByRegExp(directory, regexp, res);
diff --git a/platform/platform_linux.cpp b/platform/platform_linux.cpp
index f99bdb027f..52d054d4c8 100644
--- a/platform/platform_linux.cpp
+++ b/platform/platform_linux.cpp
@@ -2,6 +2,7 @@
#include "../base/logging.hpp"
#include "../coding/file_reader.hpp"
+#include "../coding/file_name_utils.hpp"
#include <stdlib.h>
#include <unistd.h>
@@ -90,6 +91,14 @@ Platform::Platform()
LOG(LDEBUG, ("Client ID:", UniqueClientId()));
}
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+ string dir = WritableDir() + country_name + my::GetNativeSeparator();
+ if (!IsFileExistsByFullPath(dir))
+ mkdir(dir.c_str(), 0755);
+ return dir;
+}
+
int Platform::CpuCores() const
{
const long numCPU = sysconf(_SC_NPROCESSORS_ONLN);
diff --git a/platform/platform_mac.mm b/platform/platform_mac.mm
index fcbdc6d081..cd5eea67c7 100644
--- a/platform/platform_mac.mm
+++ b/platform/platform_mac.mm
@@ -78,6 +78,15 @@ Platform::Platform()
[pool release];
}
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+
+ string dir = m_writableDir + country_name.c_str() + '/';
+ if (!IsFileExistsByFullPath(dir))
+ ::mkdir(dir.c_str(), 0755);
+ return dir;
+}
+
int Platform::CpuCores() const
{
int mib[2], numCPU = 0;
diff --git a/platform/platform_tizen.cpp b/platform/platform_tizen.cpp
index 56bd4aaa93..47a8f01b60 100644
--- a/platform/platform_tizen.cpp
+++ b/platform/platform_tizen.cpp
@@ -10,6 +10,7 @@
#include <sys/types.h>
#include "../coding/file_reader.hpp"
+#include "../coding/file_name_utils.hpp"
#include "../base/logging.hpp"
@@ -43,6 +44,14 @@ Platform::Platform()
m_flags[HAS_ROUTING] = true;
}
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+ string dir = WritableDir() + country_name + my::GetNativeSeparator();
+ if (!IsFileExistsByFullPath(dir))
+ Tizen::Io::Directory::Create(dir.c_str(), true);
+ return dir;
+}
+
int Platform::CpuCores() const
{
/// @todo
diff --git a/platform/platform_win.cpp b/platform/platform_win.cpp
index 271a5f1a07..ddddc4ebfa 100644
--- a/platform/platform_win.cpp
+++ b/platform/platform_win.cpp
@@ -4,6 +4,7 @@
#include "../base/logging.hpp"
#include "../coding/file_writer.hpp"
+#include "../coding/file_name_utils.hpp"
#include "../std/windows.hpp"
#include "../std/bind.hpp"
@@ -86,6 +87,12 @@ Platform::Platform()
LOG(LDEBUG, ("Settings Directory:", m_settingsDir));
}
+string Platform::WritablePathForFileIndexes(string const & country_name) const
+{
+ string dir = WritableDir() + country_name + my::GetNativeSeparator();
+ return dir;
+}
+
bool Platform::IsFileExistsByFullPath(string const & filePath)
{
return ::GetFileAttributesA(filePath.c_str()) != INVALID_FILE_ATTRIBUTES;
diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp
index 924cd20d75..865a6d4361 100644
--- a/routing/osrm2feature_map.cpp
+++ b/routing/osrm2feature_map.cpp
@@ -3,6 +3,8 @@
#include "../defines.hpp"
#include "../coding/varint.hpp"
+#include "../coding/internal/file_data.hpp"
+#include "../coding/file_name_utils.hpp"
#include "../base/assert.hpp"
#include "../base/logging.hpp"
@@ -320,17 +322,49 @@ void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const
cont.Write(fName, ROUTING_FTSEG_FILE_TAG);
}
+void OsrmFtSegBackwardIndex::Save(string const & countryName)
+{
+ string dir = GetPlatform().WritablePathForFileIndexes(countryName);
+ {
+ string const nodesFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT;
+ string const nodesFileNameTmp = nodesFileName + EXTENSION_TMP;
+ succinct::mapper::freeze(m_nodeIds, nodesFileNameTmp.c_str());
+ my::RenameFileX(nodesFileNameTmp, nodesFileName);
+ }
+ {
+ string const bitsFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT;
+ string const bitsFileNameTmp = bitsFileName + EXTENSION_TMP;
+ succinct::mapper::freeze(m_rankIndex, bitsFileNameTmp.c_str());
+ my::RenameFileX(bitsFileNameTmp, bitsFileName);
+ }
+}
+
+bool OsrmFtSegBackwardIndex::Load(string const & countryName)
+{
+ string dir = GetPlatform().WritablePathForFileIndexes(countryName);
+ string const nodesName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT;
+ string const bitsName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT;
+ uint64_t size;
+ if (! GetPlatform().GetFileSizeByFullPath(nodesName, size) || ! GetPlatform().GetFileSizeByFullPath(bitsName, size))
+ return false;
+ m_pMappedNodes = unique_ptr<MmapReader>(new MmapReader(nodesName));
+ m_pMappedBits = unique_ptr<MmapReader>(new MmapReader(bitsName));
+
+ succinct::mapper::map(m_nodeIds, reinterpret_cast<char const *>(m_pMappedNodes->Data()));
+ succinct::mapper::map(m_rankIndex, reinterpret_cast<char const *>(m_pMappedBits->Data()));
+
+ return true;
+}
+
void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const uint32_t maxNodeId, FilesMappingContainer & routingFile)
{
Clear();
// Calculate data file pathes
string const routingName = routingFile.GetName();
- string const mwmName(routingName, 0, routingName.find(ROUTING_FILE_EXTENSION));
- FilesMappingContainer mwmContainer(mwmName);
- mwmContainer.Open(mwmName);
- m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(mwmContainer);
+ string const name(routingName, routingName.rfind(my::GetNativeSeparator())+1, routingName.find(DATA_FILE_EXTENSION) -routingName.rfind(my::GetNativeSeparator())-1);
+ m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(name);
- if (Load(routingFile))
+ if (Load(name))
return;
// Generate temporary index to speedup processing
@@ -365,28 +399,8 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u
succinct::elias_fano_compressed_list(nodeIds).swap(m_nodeIds);
succinct::rs_bit_vector(inIndex).swap(m_rankIndex);
LOG(LINFO, ("Writing section to data file", routingName));
- routingFile.Close();
- FilesContainerW writer(routingName);
- Save(writer);
- writer.Finish();
- routingFile.Open(routingName);
- LOG(LINFO, ("Writing section to data file DONE", routingName));
-
-
- // if (Load(parentCont))
- // return;
- // size_t const count = segments.size();
- // m_index.resize(count);
- // for (size_t i = 0; i < count; ++i)
- // {
- // OsrmMappingTypes::FtSeg s(segments[i]);
- // m_index[i] = make_pair(s.m_fid, i);
- // }
- // sort(m_index.begin(), m_index.end(), [](IndexRecordTypeT const & a, IndexRecordTypeT const & b)
- // {
- // return a.first < b.first;
- // });
- // Save(parentCont);
+
+ Save(name);
}
}
diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp
index fb3139a8ec..33c66a0a65 100644
--- a/routing/osrm2feature_map.hpp
+++ b/routing/osrm2feature_map.hpp
@@ -1,9 +1,12 @@
#pragma once
#include "../coding/file_container.hpp"
+#include "../coding/mmap_reader.hpp"
#include "../base/scope_guard.hpp"
+#include "../platform/platform.hpp"
+
#include "../indexer/features_offsets_table.hpp"
#include "../3party/succinct/rs_bit_vector.hpp"
@@ -97,44 +100,16 @@ class OsrmFtSegBackwardIndex
succinct::elias_fano_compressed_list m_nodeIds;
unique_ptr<feature::FeaturesOffsetsTable> m_table;
- FilesMappingContainer::Handle m_handleNodes;
- FilesMappingContainer::Handle m_handleBits;
+ unique_ptr<MmapReader> m_pMappedNodes, m_pMappedBits;
template <class T> void ClearContainer(T & t)
{
T().swap(t);
}
- void Save(FilesContainerW & container)
- {
- {
- string const nodesFileName = container.GetFileName() + "." FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG;
- MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(nodesFileName)));
- succinct::mapper::freeze(m_nodeIds, nodesFileName.c_str());
- container.Write(nodesFileName, FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG);
- }
- {
- string const bitsFileName = container.GetFileName() + "." FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG;
- MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(bitsFileName)));
- succinct::mapper::freeze(m_nodeIds, bitsFileName.c_str());
- container.Write(bitsFileName, FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG);
- }
- }
-
- bool Load(FilesMappingContainer const & container)
- {
- if (!container.IsExist(FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG) || !container.IsExist(FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG))
- return false;
- m_handleNodes.Assign(container.Map(FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG));
- ASSERT(m_handleNodes.IsValid(), ());
- succinct::mapper::map(m_nodeIds, m_handleNodes.GetData<char>());
+ void Save(string const & countryName);
- m_handleBits.Assign(container.Map(FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG));
- ASSERT(m_handleBits.IsValid(), ());
- succinct::mapper::map(m_rankIndex, m_handleBits.GetData<char>());
-
- return true;
- }
+ bool Load(string const & container);
public:
void Construct(OsrmFtSegMapping const & mapping, uint32_t const maxNodeId, FilesMappingContainer & routingFile);
@@ -156,8 +131,8 @@ public:
ClearContainer(m_nodeIds);
ClearContainer(m_rankIndex);
m_table = unique_ptr<feature::FeaturesOffsetsTable>();
- m_handleNodes.Unmap();
- m_handleBits.Unmap();
+ m_pMappedBits = nullptr;
+ m_pMappedNodes = nullptr;
}
};