1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#include "data/shortlist.h"
#include "microsoft/shortlist/utils/ParameterTree.h"
namespace marian {
namespace data {
// cast current void pointer to T pointer and move forward by num elements
template <typename T>
const T* get(const void*& current, size_t num = 1) {
const T* ptr = (const T*)current;
current = (const T*)current + num;
return ptr;
}
QuicksandShortlistGenerator::QuicksandShortlistGenerator(Ptr<Options> options,
Ptr<const Vocab> srcVocab,
Ptr<const Vocab> trgVocab,
size_t srcIdx,
size_t /*trgIdx*/,
bool /*shared*/)
: options_(options),
srcVocab_(srcVocab),
trgVocab_(trgVocab),
srcIdx_(srcIdx) {
std::vector<std::string> vals = options_->get<std::vector<std::string>>("shortlist");
ABORT_IF(vals.empty(), "No path to filter path given");
std::string fname = vals[0];
auto firstNum = vals.size() > 1 ? std::stoi(vals[1]) : 0;
auto bestNum = vals.size() > 2 ? std::stoi(vals[2]) : 0;
float threshold = vals.size() > 3 ? std::stof(vals[3]) : 0;
if(firstNum != 0 || bestNum != 0 || threshold != 0) {
LOG(warn, "You have provided additional parameters for the Quicksand shortlist, but they are ignored.");
}
mmap_ = mio::mmap_source(fname); // memory-map the binary file once
const void* current = mmap_.data(); // pointer iterator over binary file
// compare magic number in binary file to make sure we are reading the right thing
const int32_t MAGIC_NUMBER = 1234567890;
int32_t header_magic_number = *get<int32_t>(current);
ABORT_IF(header_magic_number != MAGIC_NUMBER, "Trying to mmap Quicksand shortlist but encountered wrong magic number");
auto config = ::quicksand::ParameterTree::FromBinaryReader(current);
use16bit_ = config->GetBoolReq("use_16_bit");
LOG(info, "[data] Mapping Quicksand shortlist from {}", fname);
idSize_ = sizeof(int32_t);
if (use16bit_) {
idSize_ = sizeof(uint16_t);
}
// mmap the binary shortlist pieces
numDefaultIds_ = *get<int32_t>(current);
defaultIds_ = get<int32_t>(current, numDefaultIds_);
numSourceIds_ = *get<int32_t>(current);
sourceLengths_ = get<int32_t>(current, numSourceIds_);
sourceOffsets_ = get<int32_t>(current, numSourceIds_);
numShortlistIds_ = *get<int32_t>(current);
sourceToShortlistIds_ = get<uint8_t>(current, idSize_ * numShortlistIds_);
// display parameters
LOG(info,
"[data] Quicksand shortlist has {} source ids, {} default ids and {} shortlist ids",
numSourceIds_,
numDefaultIds_,
numShortlistIds_);
}
Ptr<Shortlist> QuicksandShortlistGenerator::generate(Ptr<data::CorpusBatch> batch) const {
auto srcBatch = (*batch)[srcIdx_];
auto maxShortlistSize = trgVocab_->size();
std::unordered_set<int32_t> indexSet;
for(int32_t i = 0; i < numDefaultIds_ && i < maxShortlistSize; ++i) {
int32_t id = defaultIds_[i];
indexSet.insert(id);
}
// State
std::vector<std::pair<const uint8_t*, int32_t>> curShortlists(maxShortlistSize);
auto curShortlistIt = curShortlists.begin();
// Because we might fill up our shortlist before reaching max_shortlist_size, we fill the shortlist in order of rank.
// E.g., first rank of word 0, first rank of word 1, ... second rank of word 0, ...
int32_t maxLength = 0;
for (Word word : srcBatch->data()) {
int32_t sourceId = (int32_t)word.toWordIndex();
srcVocab_->transcodeToShortlistInPlace((WordIndex*)&sourceId, 1);
if (sourceId < numSourceIds_) { // if it's a valid source id
const uint8_t* curShortlistIds = sourceToShortlistIds_ + idSize_ * sourceOffsets_[sourceId]; // start position for mapping
int32_t length = sourceLengths_[sourceId]; // how many mappings are there
curShortlistIt->first = curShortlistIds;
curShortlistIt->second = length;
curShortlistIt++;
if (length > maxLength)
maxLength = length;
}
}
// collect the actual shortlist mappings
for (int32_t i = 0; i < maxLength && indexSet.size() < maxShortlistSize; i++) {
for (int32_t j = 0; j < curShortlists.size() && indexSet.size() < maxShortlistSize; j++) {
int32_t length = curShortlists[j].second;
if (i < length) {
const uint8_t* source_shortlist_ids_bytes = curShortlists[j].first;
int32_t id = 0;
if (use16bit_) {
const uint16_t* source_shortlist_ids = reinterpret_cast<const uint16_t*>(source_shortlist_ids_bytes);
id = (int32_t)source_shortlist_ids[i];
}
else {
const int32_t* source_shortlist_ids = reinterpret_cast<const int32_t*>(source_shortlist_ids_bytes);
id = source_shortlist_ids[i];
}
indexSet.insert(id);
}
}
}
// turn into vector and sort (selected indices)
std::vector<WordIndex> indices;
indices.reserve(indexSet.size());
for(auto i : indexSet)
indices.push_back((WordIndex)i);
std::sort(indices.begin(), indices.end());
return New<Shortlist>(indices);
}
Ptr<ShortlistGenerator> createShortlistGenerator(Ptr<Options> options,
Ptr<const Vocab> srcVocab,
Ptr<const Vocab> trgVocab,
size_t srcIdx,
size_t trgIdx,
bool shared) {
std::vector<std::string> vals = options->get<std::vector<std::string>>("shortlist");
ABORT_IF(vals.empty(), "No path to shortlist given");
std::string fname = vals[0];
if(filesystem::Path(fname).extension().string() == ".bin") {
return New<QuicksandShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
} else {
return New<LexicalShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
}
}
} // namespace data
} // namespace marian
|
