1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
#include "string_file.hpp"
#include "../coding/read_write_utils.hpp"
#include "../coding/file_reader.hpp"
#include "../coding/file_writer.hpp"
#include "../base/logging.hpp"
#include "../std/algorithm.hpp"
#include "../std/bind.hpp"
template <class TWriter>
StringsFile::IdT StringsFile::StringT::Write(TWriter & writer) const
{
IdT const pos = static_cast<IdT>(writer.Pos());
CHECK_EQUAL(static_cast<uint64_t>(pos), writer.Pos(), ());
rw::Write(writer, m_name);
rw::WriteVectorOfPOD(writer, m_val);
return pos;
}
template <class TReader>
void StringsFile::StringT::Read(TReader & src)
{
rw::Read(src, m_name);
rw::ReadVectorOfPOD(src, m_val);
}
bool StringsFile::StringT::operator < (StringT const & name) const
{
if (m_name != name.m_name)
return (m_name < name.m_name);
return (m_val < name.m_val);
}
bool StringsFile::StringT::operator == (StringT const & name) const
{
return (m_name == name.m_name && m_val == name.m_val);
}
void StringsFile::AddString(StringT const & s)
{
#ifdef OMIM_OS_DESKTOP
size_t const maxSize = 1000000;
#else
size_t const maxSize = 30000;
#endif
if (m_strings.size() >= maxSize)
Flush();
m_strings.push_back(s);
}
bool StringsFile::IteratorT::IsEnd() const
{
return m_file.m_queue.empty();
}
StringsFile::StringT StringsFile::IteratorT::dereference() const
{
ASSERT ( IsValid(), () );
return m_file.m_queue.top().m_string;
}
void StringsFile::IteratorT::increment()
{
ASSERT ( IsValid(), () );
int const index = m_file.m_queue.top().m_index;
m_file.m_queue.pop();
if (!m_file.PushNextValue(index))
m_end = IsEnd();
}
StringsFile::StringsFile(string const & fPath)
{
m_writer.reset(new FileWriter(fPath));
}
void StringsFile::Flush()
{
// store starting offset
uint64_t const pos = m_writer->Pos();
m_offsets.push_back(make_pair(pos, pos));
// sort strings
sort(m_strings.begin(), m_strings.end());
// write strings to file
for_each(m_strings.begin(), m_strings.end(), bind(&StringT::Write<FileWriter>, _1, ref(*m_writer)));
// store end offset
m_offsets.back().second = m_writer->Pos();
m_strings.clear();
}
bool StringsFile::PushNextValue(size_t i)
{
// reach the end of the portion file
if (m_offsets[i].first >= m_offsets[i].second)
return false;
// init source to needed offset
ReaderSource<FileReader> src(*m_reader);
src.Skip(m_offsets[i].first);
// read string
StringT s;
s.Read(src);
// update offset
m_offsets[i].first = src.Pos();
// push value to queue
m_queue.push(QValue(s, i));
return true;
}
void StringsFile::EndAdding()
{
Flush();
m_writer->Flush();
}
void StringsFile::OpenForRead()
{
string const fPath = m_writer->GetName();
m_writer.reset();
m_reader.reset(new FileReader(fPath));
for (size_t i = 0; i < m_offsets.size(); ++i)
PushNextValue(i);
}
|
