Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/mapsme/omim.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--coding/file_container.cpp127
-rw-r--r--coding/file_container.hpp52
-rw-r--r--coding/file_reader.hpp2
-rw-r--r--defines.hpp1
-rw-r--r--generator/generator_tool/generator_tool.cpp24
-rw-r--r--indexer/features_offsets_table.cpp19
-rw-r--r--indexer/features_offsets_table.hpp11
-rw-r--r--indexer/features_vector.cpp5
-rw-r--r--indexer/features_vector.hpp2
-rw-r--r--indexer/index.cpp10
-rw-r--r--map/map_tests/mwm_set_test.cpp3
-rw-r--r--platform/mwm_version.hpp3
12 files changed, 191 insertions, 68 deletions
diff --git a/coding/file_container.cpp b/coding/file_container.cpp
index dedcd2a13a..52d189679e 100644
--- a/coding/file_container.cpp
+++ b/coding/file_container.cpp
@@ -81,7 +81,20 @@ FilesContainerR::ReaderT FilesContainerR::GetReader(Tag const & tag) const
if (p)
return m_source.SubReader(p->m_offset, p->m_size);
else
- MYTHROW(Reader::OpenException, (tag));
+ MYTHROW(Reader::OpenException, ("Can't find section:", tag));
+}
+
+pair<uint64_t, uint64_t> FilesContainerR::GetAbsoluteOffsetAndSize(Tag const & tag) const
+{
+ Info const * p = GetInfo(tag);
+ if (p)
+ {
+ auto reader = dynamic_cast<FileReader const *>(m_source.GetPtr());
+ uint64_t const offset = reader ? reader->GetOffset() : 0;
+ return make_pair(offset + p->m_offset, p->m_size);
+ }
+ else
+ MYTHROW(Reader::OpenException, ("Can't find section:", tag));
}
FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) const
@@ -95,28 +108,14 @@ FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) co
return 0;
}
+namespace detail
+{
/////////////////////////////////////////////////////////////////////////////
-// FilesMappingContainer
+// MappedFile
/////////////////////////////////////////////////////////////////////////////
-
-FilesMappingContainer::FilesMappingContainer(string const & fName)
-{
- Open(fName);
-}
-
-FilesMappingContainer::~FilesMappingContainer()
+void MappedFile::Open(string const & fName)
{
Close();
-}
-
-void FilesMappingContainer::Open(string const & fName)
-{
- Close();
-
- {
- FileReader reader(fName);
- ReadInfo(reader);
- }
#ifdef OMIM_OS_WINDOWS
m_hFile = CreateFileA(fName.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL);
@@ -130,11 +129,9 @@ void FilesMappingContainer::Open(string const & fName)
if (m_fd == -1)
MYTHROW(Reader::OpenException, ("Can't open file:", fName));
#endif
-
- m_name = fName;
}
-void FilesMappingContainer::Close()
+void MappedFile::Close()
{
#ifdef OMIM_OS_WINDOWS
if (m_hMapping != INVALID_HANDLE_VALUE)
@@ -148,42 +145,80 @@ void FilesMappingContainer::Close()
m_fd = -1;
}
#endif
-
- m_name.clear();
}
-FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const
+MappedFile::Handle MappedFile::Map(uint64_t off, uint64_t size, string const & tag) const
{
- Info const * p = GetInfo(tag);
- if (p)
- {
#ifdef OMIM_OS_WINDOWS
- SYSTEM_INFO sysInfo;
- memset(&sysInfo, 0, sizeof(sysInfo));
- GetSystemInfo(&sysInfo);
- long const offsetAlign = sysInfo.dwAllocationGranularity;
+ SYSTEM_INFO sysInfo;
+ memset(&sysInfo, 0, sizeof(sysInfo));
+ GetSystemInfo(&sysInfo);
+ long const offsetAlign = sysInfo.dwAllocationGranularity;
#else
- long const offsetAlign = sysconf(_SC_PAGE_SIZE);
+ long const offsetAlign = sysconf(_SC_PAGE_SIZE);
#endif
- uint64_t const offset = (p->m_offset / offsetAlign) * offsetAlign;
- ASSERT_LESS_OR_EQUAL(offset, p->m_offset, ());
- uint64_t const length = p->m_size + (p->m_offset - offset);
- ASSERT_GREATER_OR_EQUAL(length, p->m_size, ());
+ uint64_t const offset = (off / offsetAlign) * offsetAlign;
+ ASSERT_LESS_OR_EQUAL(offset, off, ());
+ uint64_t const length = size + (off - offset);
+ ASSERT_GREATER_OR_EQUAL(length, size, ());
#ifdef OMIM_OS_WINDOWS
- void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, offset >> (sizeof(DWORD) * 8), DWORD(offset), length);
- if (pMap == NULL)
- MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p, "win last error:", GetLastError()));
+ void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, offset >> (sizeof(DWORD) * 8), DWORD(offset), length);
+ if (pMap == NULL)
+ MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", off, size, "win last error:", GetLastError()));
#else
- void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset);
- if (pMap == MAP_FAILED)
- MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p));
+ void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset);
+ if (pMap == MAP_FAILED)
+ MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", off, size));
#endif
- char const * data = reinterpret_cast<char const *>(pMap);
- char const * d = data + (p->m_offset - offset);
- return Handle(d, data, p->m_size, length);
+ char const * data = reinterpret_cast<char const *>(pMap);
+ char const * d = data + (off - offset);
+ return Handle(d, data, size, length);
+}
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// FilesMappingContainer
+/////////////////////////////////////////////////////////////////////////////
+
+FilesMappingContainer::FilesMappingContainer(string const & fName)
+{
+ Open(fName);
+}
+
+FilesMappingContainer::~FilesMappingContainer()
+{
+ Close();
+}
+
+void FilesMappingContainer::Open(string const & fName)
+{
+ {
+ FileReader reader(fName);
+ ReadInfo(reader);
+ }
+
+ m_file.Open(fName);
+
+ m_name = fName;
+}
+
+void FilesMappingContainer::Close()
+{
+ m_file.Close();
+
+ m_name.clear();
+}
+
+FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const
+{
+ Info const * p = GetInfo(tag);
+ if (p)
+ {
+ ASSERT_EQUAL(tag, p->m_tag, ());
+ return m_file.Map(p->m_offset, p->m_size, tag);
}
else
MYTHROW(Reader::OpenException, ("Can't find section:", tag));
diff --git a/coding/file_container.hpp b/coding/file_container.hpp
index 406cdf95f9..5c5e9c0e19 100644
--- a/coding/file_container.hpp
+++ b/coding/file_container.hpp
@@ -123,24 +123,30 @@ public:
inline uint64_t GetFileSize() const { return m_source.Size(); }
inline string const & GetFileName() const { return m_source.GetName(); }
+ pair<uint64_t, uint64_t> GetAbsoluteOffsetAndSize(Tag const & tag) const;
+
private:
ReaderT m_source;
};
-class FilesMappingContainer : public FilesContainerBase
+namespace detail
{
-public:
- /// Do nothing by default, call Open to attach to file.
- FilesMappingContainer() = default;
- explicit FilesMappingContainer(string const & fName);
- ~FilesMappingContainer();
+class MappedFile
+{
+ DISALLOW_COPY(MappedFile);
+
+public:
+ MappedFile() = default;
+ ~MappedFile() { Close(); }
void Open(string const & fName);
void Close();
- class Handle : private noncopyable
+ class Handle
{
+ DISALLOW_COPY(Handle);
+
void Reset();
public:
@@ -183,13 +189,9 @@ public:
uint64_t m_origSize;
};
- Handle Map(Tag const & tag) const;
- FileReader GetReader(Tag const & tag) const;
-
- string const & GetName() const { return m_name; }
+ Handle Map(uint64_t off, uint64_t size, string const & tag) const;
private:
- string m_name;
#ifdef OMIM_OS_WINDOWS
void * m_hFile = (void *)-1;
void * m_hMapping = (void *)-1;
@@ -198,6 +200,32 @@ private:
#endif
};
+}
+
+class FilesMappingContainer : public FilesContainerBase
+{
+public:
+ /// Do nothing by default, call Open to attach to file.
+ FilesMappingContainer() = default;
+ explicit FilesMappingContainer(string const & fName);
+
+ ~FilesMappingContainer();
+
+ void Open(string const & fName);
+ void Close();
+
+ typedef detail::MappedFile::Handle Handle;
+
+ Handle Map(Tag const & tag) const;
+ FileReader GetReader(Tag const & tag) const;
+
+ string const & GetName() const { return m_name; }
+
+private:
+ string m_name;
+ detail::MappedFile m_file;
+};
+
class FilesContainerW : public FilesContainerBase
{
public:
diff --git a/coding/file_reader.hpp b/coding/file_reader.hpp
index 659079affc..6008452f33 100644
--- a/coding/file_reader.hpp
+++ b/coding/file_reader.hpp
@@ -22,6 +22,8 @@ public:
FileReader SubReader(uint64_t pos, uint64_t size) const;
FileReader * CreateSubReader(uint64_t pos, uint64_t size) const;
+ inline uint64_t GetOffset() const { return m_Offset; }
+
protected:
/// Make assertion that pos + size in FileReader bounds.
bool AssertPosAndSize(uint64_t pos, uint64_t size) const;
diff --git a/defines.hpp b/defines.hpp
index 06ac0198ad..f955343247 100644
--- a/defines.hpp
+++ b/defines.hpp
@@ -24,6 +24,7 @@
#define METADATA_FILE_TAG "meta"
#define METADATA_INDEX_FILE_TAG "metaidx"
#define COMPRESSED_SEARCH_INDEX_FILE_TAG "csdx"
+#define FEATURE_OFFSETS_FILE_TAG "offs"
#define ROUTING_MATRIX_FILE_TAG "mercedes"
#define ROUTING_EDGEDATA_FILE_TAG "daewoo"
diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp
index 3339ac991a..1d103ab742 100644
--- a/generator/generator_tool/generator_tool.cpp
+++ b/generator/generator_tool/generator_tool.cpp
@@ -15,6 +15,7 @@
#include "indexer/classificator_loader.hpp"
#include "indexer/classificator.hpp"
#include "indexer/data_header.hpp"
+#include "indexer/features_offsets_table.hpp"
#include "indexer/features_vector.hpp"
#include "indexer/index_builder.hpp"
#include "indexer/search_index_builder.hpp"
@@ -165,10 +166,11 @@ int main(int argc, char ** argv)
for (size_t i = 0; i < count; ++i)
{
string const & country = genInfo.m_bucketNames[i];
+ string const datFile = path + country + DATA_FILE_EXTENSION;
if (FLAGS_generate_geometry)
{
- LOG(LINFO, ("Generating result features for file", country));
+ LOG(LINFO, ("Generating result features for", country));
int mapType = feature::DataHeader::country;
if (country == WORLD_FILE_NAME)
@@ -181,13 +183,27 @@ int main(int argc, char ** argv)
// If error - move to next bucket without index generation
continue;
}
- }
- string const datFile = path + country + DATA_FILE_EXTENSION;
+ LOG(LINFO, ("Generating offsets table for", datFile));
+
+ try
+ {
+ string const destPath = datFile + ".offsets";
+
+ (void)feature::FeaturesOffsetsTable::Build(FilesContainerR(datFile), destPath);
+ FilesContainerW(datFile, FileWriter::OP_WRITE_EXISTING).Write(destPath, FEATURE_OFFSETS_FILE_TAG);
+
+ FileWriter::DeleteFileX(destPath);
+ }
+ catch (RootException const & ex)
+ {
+ LOG(LERROR, ("Generating offsets table failed for", datFile, "Reason", ex.Msg()));
+ }
+ }
if (FLAGS_generate_index)
{
- LOG(LINFO, ("Generating index for ", datFile));
+ LOG(LINFO, ("Generating index for", datFile));
if (!indexer::BuildIndexFromDatFile(datFile, FLAGS_intermediate_data_path + country))
LOG(LCRITICAL, ("Error generating index."));
diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp
index 2b6d9e19fe..a16d88f2cb 100644
--- a/indexer/features_offsets_table.cpp
+++ b/indexer/features_offsets_table.cpp
@@ -65,6 +65,19 @@ namespace feature
}
// static
+ unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(FilesContainerR const & cont)
+ {
+ unique_ptr<FeaturesOffsetsTable> ptr(new FeaturesOffsetsTable());
+
+ ptr->m_file.Open(cont.GetFileName());
+ auto p = cont.GetAbsoluteOffsetAndSize(FEATURE_OFFSETS_FILE_TAG);
+ ptr->m_handle.Assign(ptr->m_file.Map(p.first, p.second, FEATURE_OFFSETS_FILE_TAG));
+
+ succinct::mapper::map(ptr->m_table, ptr->m_handle.GetData<char>());
+ return ptr;
+ }
+
+ // static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::CreateImpl(
platform::LocalCountryFile const & localFile,
FilesContainerR const & cont, string const & storePath)
@@ -73,6 +86,12 @@ namespace feature
CountryIndexes::PreparePlaceOnDisk(localFile);
+ return Build(cont, storePath);
+ }
+
+ unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Build(FilesContainerR const & cont,
+ string const & storePath)
+ {
Builder builder;
FeaturesVector::ForEachOffset(cont.GetReader(DATA_FILE_TAG), [&builder] (uint32_t offset)
{
diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp
index b1b52f4ddb..d7862657fa 100644
--- a/indexer/features_offsets_table.hpp
+++ b/indexer/features_offsets_table.hpp
@@ -1,5 +1,6 @@
#pragma once
+#include "coding/file_container.hpp"
#include "coding/mmap_reader.hpp"
#include "defines.hpp"
@@ -12,7 +13,6 @@
#include "3party/succinct/mapper.hpp"
-class FilesContainerR;
namespace platform
{
class LocalCountryFile;
@@ -57,6 +57,10 @@ namespace feature
/// Load table by full path to the table file.
static unique_ptr<FeaturesOffsetsTable> Load(string const & filePath);
+ static unique_ptr<FeaturesOffsetsTable> Load(FilesContainerR const & cont);
+ static unique_ptr<FeaturesOffsetsTable> Build(FilesContainerR const & cont,
+ string const & storePath);
+
/// Get table for the MWM map, represented by localFile and cont.
static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(
platform::LocalCountryFile const & localFile, FilesContainerR const & cont);
@@ -94,6 +98,7 @@ namespace feature
private:
FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder);
FeaturesOffsetsTable(string const & filePath);
+ FeaturesOffsetsTable() = default;
static unique_ptr<FeaturesOffsetsTable> LoadImpl(string const & filePath);
static unique_ptr<FeaturesOffsetsTable> CreateImpl(platform::LocalCountryFile const & localFile,
@@ -101,7 +106,9 @@ namespace feature
string const & storePath);
succinct::elias_fano m_table;
-
unique_ptr<MmapReader> m_pReader;
+
+ detail::MappedFile m_file;
+ detail::MappedFile::Handle m_handle;
};
} // namespace feature
diff --git a/indexer/features_vector.cpp b/indexer/features_vector.cpp
index 88627061fa..5672088e31 100644
--- a/indexer/features_vector.cpp
+++ b/indexer/features_vector.cpp
@@ -23,8 +23,11 @@ FeaturesVectorTest::FeaturesVectorTest(string const & filePath)
FeaturesVectorTest::FeaturesVectorTest(FilesContainerR const & cont)
: m_cont(cont), m_header(m_cont), m_vector(m_cont, m_header, 0)
{
- if (m_header.GetFormat() >= version::v5)
+ auto const ver = m_header.GetFormat();
+ if (ver == version::v5)
m_vector.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_cont).release();
+ else if (ver >= version::v6)
+ m_vector.m_table = feature::FeaturesOffsetsTable::Load(m_cont).release();
}
FeaturesVectorTest::~FeaturesVectorTest()
diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp
index 53d8c46d66..be38523a73 100644
--- a/indexer/features_vector.hpp
+++ b/indexer/features_vector.hpp
@@ -55,6 +55,8 @@ private:
/// Used in generator_tool and unit tests.
class FeaturesVectorTest
{
+ DISALLOW_COPY(FeaturesVectorTest);
+
FilesContainerR m_cont;
feature::DataHeader m_header;
FeaturesVector m_vector;
diff --git a/indexer/index.cpp b/indexer/index.cpp
index 1592715dff..cafe3c4213 100644
--- a/indexer/index.cpp
+++ b/indexer/index.cpp
@@ -24,11 +24,17 @@ MwmValue::MwmValue(LocalCountryFile const & localFile)
void MwmValue::SetTable(MwmInfoEx & info)
{
- if (GetHeader().GetFormat() < version::v5)
+ auto const ver = GetHeader().GetFormat();
+ if (ver < version::v5)
return;
if (!info.m_table)
- info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont);
+ {
+ if (ver == version::v5)
+ info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont);
+ else
+ info.m_table = feature::FeaturesOffsetsTable::Load(m_cont);
+ }
m_table = info.m_table.get();
}
diff --git a/map/map_tests/mwm_set_test.cpp b/map/map_tests/mwm_set_test.cpp
index d3919c19c8..0c2b0783a8 100644
--- a/map/map_tests/mwm_set_test.cpp
+++ b/map/map_tests/mwm_set_test.cpp
@@ -15,6 +15,8 @@
using namespace platform;
using namespace my;
+/*
+ * This test is useless because of we don't build offsets index from now.
#ifndef OMIM_OS_WINDOWS
UNIT_TEST(MwmSet_FileSystemErrors)
{
@@ -56,3 +58,4 @@ UNIT_TEST(MwmSet_FileSystemErrors)
TEST(infos.empty(), ());
}
#endif
+*/
diff --git a/platform/mwm_version.hpp b/platform/mwm_version.hpp
index ee1f393976..0fcd02623f 100644
--- a/platform/mwm_version.hpp
+++ b/platform/mwm_version.hpp
@@ -17,7 +17,8 @@ enum Format
v3, // March 2013 (store type index, instead of raw type in search data)
v4, // April 2015 (distinguish и and й in search index)
v5, // July 2015 (feature id is the index in vector now).
- lastFormat = v5
+ v6, // October 2015 (offsets vector is in mwm now).
+ lastFormat = v6
};
struct MwmVersion
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-18 14:15:07 +0300