1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
|
// $Id$
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <limits>
#include <iostream>
#include <fstream>
#include "dictionary.h"
#include "n_gram.h"
#include "lmContainer.h"
using namespace irstlm;
#include "IRST.h"
#include "moses/LM/PointerState.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/FactorCollection.h"
#include "moses/Phrase.h"
#include "moses/InputFileStream.h"
#include "moses/StaticData.h"
#include "moses/TranslationTask.h"
using namespace std;
namespace Moses
{
class IRSTLMState : public PointerState
{
public:
IRSTLMState():PointerState(NULL) {}
IRSTLMState(const void* lms):PointerState(lms) {}
IRSTLMState(const IRSTLMState& copy_from):PointerState(copy_from.lmstate) {}
IRSTLMState& operator=( const IRSTLMState& rhs ) {
lmstate = rhs.lmstate;
return *this;
}
const void* GetState() const {
return lmstate;
}
};
LanguageModelIRST::LanguageModelIRST(const std::string &line)
:LanguageModelSingleFactor(line)
,m_lmtb_dub(0), m_lmtb_size(0)
{
const StaticData &staticData = StaticData::Instance();
int threadCount = staticData.ThreadCount();
if (threadCount != 1) {
throw runtime_error("Error: " + SPrint(threadCount) + " number of threads specified but IRST LM is not threadsafe.");
}
ReadParameters();
VERBOSE(4, GetScoreProducerDescription() << " LanguageModelIRST::LanguageModelIRST() m_lmtb_dub:|" << m_lmtb_dub << "|" << std::endl);
VERBOSE(4, GetScoreProducerDescription() << " LanguageModelIRST::LanguageModelIRST() m_filePath:|" << m_filePath << "|" << std::endl);
VERBOSE(4, GetScoreProducerDescription() << " LanguageModelIRST::LanguageModelIRST() m_factorType:|" << m_factorType << "|" << std::endl);
VERBOSE(4, GetScoreProducerDescription() << " LanguageModelIRST::LanguageModelIRST() m_lmtb_size:|" << m_lmtb_size << "|" << std::endl);
}
LanguageModelIRST::~LanguageModelIRST()
{
#ifndef WIN32
TRACE_ERR( "reset mmap\n");
if (m_lmtb) m_lmtb->reset_mmap();
#endif
delete m_lmtb;
}
bool LanguageModelIRST::IsUseable(const FactorMask &mask) const
{
bool ret = mask[m_factorType];
return ret;
}
void LanguageModelIRST::Load(AllOptions::ptr const& opts)
{
FactorCollection &factorCollection = FactorCollection::Instance();
m_lmtb = m_lmtb->CreateLanguageModel(m_filePath);
if (m_lmtb_size > 0) m_lmtb->setMaxLoadedLevel(m_lmtb_size);
m_lmtb->load(m_filePath);
d=m_lmtb->getDict();
d->incflag(1);
m_nGramOrder = m_lmtb_size = m_lmtb->maxlevel();
// LM can be ok, just outputs warnings
// Mauro: in the original, the following two instructions are wrongly switched:
m_unknownId = d->oovcode(); // at the level of micro tags
m_empty = -1; // code for an empty position
CreateFactors(factorCollection);
VERBOSE(1, GetScoreProducerDescription() << " LanguageModelIRST::Load() m_unknownId=" << m_unknownId << std::endl);
//install caches to save time (only if PS_CACHE_ENABLE is defined through compilation flags)
m_lmtb->init_caches(m_lmtb_size>2?m_lmtb_size-1:2);
if (m_lmtb_dub > 0) m_lmtb->setlogOOVpenalty(m_lmtb_dub);
}
void LanguageModelIRST::CreateFactors(FactorCollection &factorCollection)
{
// add factors which have srilm id
// code copied & paste from SRI LM class. should do template function
std::map<size_t, int> lmIdMap;
size_t maxFactorId = 0; // to create lookup vector later on
m_empty = -1; // code for an empty position
dict_entry *entry;
dictionary_iter iter(d); // at the level of micro tags
while ( (entry = iter.next()) != NULL) {
size_t factorId = factorCollection.AddFactor(Output, m_factorType, entry->word)->GetId();
lmIdMap[factorId] = entry->code;
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
}
size_t factorId;
m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, BOS_);
factorId = m_sentenceStart->GetId();
const std::string bs = BOS_;
const std::string es = EOS_;
m_lmtb_sentenceStart=lmIdMap[factorId] = GetLmID(BOS_);
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
m_sentenceStartWord[m_factorType] = m_sentenceStart;
m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, EOS_);
factorId = m_sentenceEnd->GetId();
m_lmtb_sentenceEnd=lmIdMap[factorId] = GetLmID(EOS_);
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
m_sentenceEndWord[m_factorType] = m_sentenceEnd;
// add to lookup vector in object
m_lmIdLookup.resize(maxFactorId+1);
fill(m_lmIdLookup.begin(), m_lmIdLookup.end(), m_empty);
map<size_t, int>::iterator iterMap;
for (iterMap = lmIdMap.begin() ; iterMap != lmIdMap.end() ; ++iterMap) {
m_lmIdLookup[iterMap->first] = iterMap->second;
}
}
int LanguageModelIRST::GetLmID( const std::string &str ) const
{
return d->encode( str.c_str() ); // at the level of micro tags
}
int LanguageModelIRST::GetLmID( const Word &word ) const
{
return GetLmID( word.GetFactor(m_factorType) );
}
int LanguageModelIRST::GetLmID( const Factor *factor ) const
{
size_t factorId = factor->GetId();
if ((factorId >= m_lmIdLookup.size()) || (m_lmIdLookup[factorId] == m_empty)) {
if (d->incflag()==1) {
std::string s = factor->GetString().as_string();
int code = d->encode(s.c_str());
//////////
///poiche' non c'e' distinzione tra i factorIDs delle parole sorgenti
///e delle parole target in Moses, puo' accadere che una parola target
///di cui non sia stato ancora calcolato il suo codice target abbia
///comunque un factorID noto (e quindi minore di m_lmIdLookup.size())
///E' necessario dunque identificare questi casi di indeterminatezza
///del codice target. Attualmente, questo controllo e' stato implementato
///impostando a m_empty tutti i termini che non hanno ancora
//ricevuto un codice target effettivo
///////////
///OLD PROBLEM - SOLVED
////////////
/// IL PPROBLEMA ERA QUI
/// m_lmIdLookup.push_back(code);
/// PERCHE' USANDO PUSH_BACK IN REALTA' INSEREVIVAMO L'ELEMENTO NUOVO
/// IN POSIZIONE (factorID-1) invece che in posizione factrID dove dopo andiamo a leggerlo (vedi caso C
/// Cosi' funziona ....
/// ho un dubbio su cosa c'e' nelle prime posizioni di m_lmIdLookup
/// quindi
/// e scopro che rimane vuota una entry ogni due
/// perche' factorID cresce di due in due (perche' codifica sia source che target) "vuota" la posizione (factorID-1)
/// non da problemi di correttezza, ma solo di "spreco" di memoria
/// potremmo sostituirerendere m_lmIdLookup una std:map invece che un std::vector,
/// ma si perde in efficienza nell'accesso perche' non e' piu' possibile quello random dei vettori
/// a te la scelta!!!!
////////////////
if (factorId >= m_lmIdLookup.size()) {
//resize and fill with m_empty
//increment the array more than needed to avoid too many resizing operation.
m_lmIdLookup.resize(factorId+10, m_empty);
}
//insert new code
m_lmIdLookup[factorId] = code;
return code;
} else {
return m_unknownId;
}
} else {
return m_lmIdLookup[factorId];
}
}
const FFState* LanguageModelIRST::EmptyHypothesisState(const InputType &/*input*/) const
{
std::auto_ptr<IRSTLMState> ret(new IRSTLMState());
return ret.release();
}
void LanguageModelIRST::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const
{
fullScore = 0;
ngramScore = 0;
oovCount = 0;
if ( !phrase.GetSize() ) return;
int _min = min(m_lmtb_size - 1, (int) phrase.GetSize());
int codes[m_lmtb_size];
int idx = 0;
codes[idx] = m_lmtb_sentenceStart;
++idx;
int position = 0;
char* msp = NULL;
float before_boundary = 0.0;
for (; position < _min; ++position) {
codes[idx] = GetLmID(phrase.GetWord(position));
if (codes[idx] == m_unknownId) ++oovCount;
before_boundary += m_lmtb->clprob(codes,idx+1,NULL,NULL,&msp);
++idx;
}
ngramScore = 0.0;
int end_loop = (int) phrase.GetSize();
for (; position < end_loop; ++position) {
for (idx = 1; idx < m_lmtb_size; ++idx) {
codes[idx-1] = codes[idx];
}
codes[idx-1] = GetLmID(phrase.GetWord(position));
if (codes[idx-1] == m_unknownId) ++oovCount;
ngramScore += m_lmtb->clprob(codes,idx,NULL,NULL,&msp);
}
before_boundary = TransformLMScore(before_boundary);
ngramScore = TransformLMScore(ngramScore);
fullScore = ngramScore + before_boundary;
}
FFState* LanguageModelIRST::EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const
{
if (!hypo.GetCurrTargetLength()) {
std::auto_ptr<IRSTLMState> ret(new IRSTLMState(ps));
return ret.release();
}
//[begin, end) in STL-like fashion.
const int begin = (const int) hypo.GetCurrTargetWordsRange().GetStartPos();
const int end = (const int) hypo.GetCurrTargetWordsRange().GetEndPos() + 1;
const int adjust_end = (const int) std::min(end, begin + m_lmtb_size - 1);
//set up context
//fill the farthest positions with sentenceStart symbols, if "empty" positions are available
//so that the vector looks like = "<s> <s> context_word context_word" for a two-word context and a LM of order 5
int codes[m_lmtb_size];
int idx=m_lmtb_size-1;
int position = (const int) begin;
while (position >= 0) {
codes[idx] = GetLmID(hypo.GetWord(position));
--idx;
--position;
}
while (idx>=0) {
codes[idx] = m_lmtb_sentenceStart;
--idx;
}
char* msp = NULL;
float score = m_lmtb->clprob(codes,m_lmtb_size,NULL,NULL,&msp);
position = (const int) begin+1;
while (position < adjust_end) {
for (idx=1; idx<m_lmtb_size; idx++) {
codes[idx-1] = codes[idx];
}
codes[idx-1] = GetLmID(hypo.GetWord(position));
score += m_lmtb->clprob(codes,m_lmtb_size,NULL,NULL,&msp);
++position;
}
//adding probability of having sentenceEnd symbol, after this phrase;
//this could happen only when all source words are covered
if (hypo.IsSourceCompleted()) {
idx=m_lmtb_size-1;
codes[idx] = m_lmtb_sentenceEnd;
--idx;
position = (const int) end - 1;
while (position >= 0 && idx >= 0) {
codes[idx] = GetLmID(hypo.GetWord(position));
--idx;
--position;
}
while (idx>=0) {
codes[idx] = m_lmtb_sentenceStart;
--idx;
}
score += m_lmtb->clprob(codes,m_lmtb_size,NULL,NULL,&msp);
} else {
// need to set the LM state
if (adjust_end < end) { //the LMstate of this target phrase refers to the last m_lmtb_size-1 words
position = (const int) end - 1;
for (idx=m_lmtb_size-1; idx>0; --idx) {
codes[idx] = GetLmID(hypo.GetWord(position));
}
codes[idx] = m_lmtb_sentenceStart;
msp = (char *) m_lmtb->cmaxsuffptr(codes,m_lmtb_size);
}
}
score = TransformLMScore(score);
out->PlusEquals(this, score);
std::auto_ptr<IRSTLMState> ret(new IRSTLMState(msp));
return ret.release();
}
LMResult LanguageModelIRST::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
// set up context
size_t count = contextFactor.size();
if (count < 0) {
cerr << "ERROR count < 0\n";
exit(100);
};
// set up context
int codes[MAX_NGRAM_SIZE];
size_t idx=0;
//fill the farthest positions with at most ONE sentenceEnd symbol and at most ONE sentenceEnd symbol, if "empty" positions are available
//so that the vector looks like = "</s> <s> context_word context_word" for a two-word context and a LM of order 5
if (count < (size_t) (m_lmtb_size-1)) codes[idx++] = m_lmtb_sentenceEnd;
if (count < (size_t) m_lmtb_size) codes[idx++] = m_lmtb_sentenceStart;
for (size_t i = 0 ; i < count ; i++) {
codes[idx] = GetLmID(*contextFactor[i]);
++idx;
}
LMResult result;
result.unknown = (codes[idx - 1] == m_unknownId);
char* msp = NULL;
result.score = m_lmtb->clprob(codes,idx,NULL,NULL,&msp);
if (finalState) *finalState=(State *) msp;
result.score = TransformLMScore(result.score);
return result;
}
bool LMCacheCleanup(const int sentences_done, const size_t m_lmcache_cleanup_threshold)
{
if (sentences_done==-1) return true;
if (m_lmcache_cleanup_threshold)
if (sentences_done % m_lmcache_cleanup_threshold == 0)
return true;
return false;
}
void LanguageModelIRST::InitializeForInput(ttasksptr const& ttask)
{
//nothing to do
#ifdef TRACE_CACHE
m_lmtb->sentence_id++;
#endif
}
void LanguageModelIRST::CleanUpAfterSentenceProcessing(const InputType& source)
{
const StaticData &staticData = StaticData::Instance();
static int sentenceCount = 0;
sentenceCount++;
size_t lmcache_cleanup_threshold = staticData.GetLMCacheCleanupThreshold();
if (LMCacheCleanup(sentenceCount, lmcache_cleanup_threshold)) {
TRACE_ERR( "reset caches\n");
m_lmtb->reset_caches();
}
}
void LanguageModelIRST::SetParameter(const std::string& key, const std::string& value)
{
if (key == "dub") {
m_lmtb_dub = Scan<unsigned int>(value);
} else {
LanguageModelSingleFactor::SetParameter(key, value);
}
m_lmtb_size = m_nGramOrder;
}
}
|