1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
#include "generator/wiki_url_dumper.hpp"
#include "generator/feature_builder.hpp"
#include "indexer/classificator.hpp"
#include "indexer/feature_processor.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "base/assert.hpp"
#include "base/thread_pool_computational.hpp"
#include <cstdint>
#include <fstream>
#include <sstream>
#include <utility>
namespace generator
{
WikiUrlDumper::WikiUrlDumper(std::string const & path, std::vector<std::string> const & dataFiles)
: m_path(path), m_dataFiles(dataFiles) {}
void WikiUrlDumper::Dump(size_t cpuCount) const
{
CHECK_GREATER(cpuCount, 0, ());
base::thread_pool::computational::ThreadPool threadPool(cpuCount);
std::vector<std::future<std::string>> futures;
futures.reserve(m_dataFiles.size());
auto const fn = [](std::string const & filename) {
std::stringstream stringStream;
DumpOne(filename, stringStream);
return stringStream.str();
};
for (auto const & path : m_dataFiles)
{
auto result = threadPool.Submit(fn, path);
futures.emplace_back(std::move(result));
}
std::ofstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(m_path);
stream << "MwmPath\tosmId\twikipediaUrl\n";
for (auto & f : futures)
{
auto lines = f.get();
stream << lines;
}
}
// static
void WikiUrlDumper::DumpOne(std::string const & path, std::ostream & stream)
{
auto const & needWikiUrl = ftypes::WikiChecker::Instance();
feature::ForEachFromDatRawFormat(path, [&](FeatureBuilder1 const & feature, uint64_t /* pos */) {
if (!needWikiUrl(feature.GetTypesHolder()))
return;
auto const wikiUrl = feature.GetMetadata().GetWikiURL();
if (wikiUrl.empty())
return;
stream << path << "\t" << feature.GetMostGenericOsmId() << "\t" << wikiUrl << "\n";
});
}
WikiDataFilter::WikiDataFilter(std::string const & path, std::vector<std::string> const & dataFiles)
: m_path(path), m_dataFiles(dataFiles)
{
std::ifstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(m_path);
stream.exceptions(std::fstream::badbit);
uint64_t id;
std::string wikidata;
while (stream)
{
stream >> id >> wikidata;
m_idToWikiData.emplace(base::GeoObjectId(id), wikidata);
}
}
// static
void WikiDataFilter::FilterOne(std::string const & path, std::map<base::GeoObjectId, std::string> const & idToWikiData,
std::ostream & stream)
{
auto const & needWikiUrl = ftypes::WikiChecker::Instance();
feature::ForEachFromDatRawFormat(path, [&](FeatureBuilder1 const & feature, uint64_t /* pos */) {
if (!needWikiUrl(feature.GetTypesHolder()))
return;
auto const it = idToWikiData.find(feature.GetMostGenericOsmId());
if (it == std::end(idToWikiData))
return;
stream << it->first.GetEncodedId() << "\t" << it->second << "\n";
});
}
void WikiDataFilter::Filter(size_t cpuCount)
{
CHECK_GREATER(cpuCount, 0, ());
base::thread_pool::computational::ThreadPool threadPool(cpuCount);
std::vector<std::future<std::string>> futures;
futures.reserve(m_dataFiles.size());
auto const fn = [&](std::string const & filename) {
std::stringstream stringStream;
FilterOne(filename, m_idToWikiData, stringStream);
return stringStream.str();
};
for (auto const & path : m_dataFiles)
{
auto result = threadPool.Submit(fn, path);
futures.emplace_back(std::move(result));
}
std::ofstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(m_path);
for (auto & f : futures)
{
auto lines = f.get();
stream << lines;
}
}
} // namespace generator
|
