blob: 0c9a435399efa75b06e2fbd434ba1775220a2b50 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
#include <boost/foreach.hpp>
#include "InMemoryPerSentenceOnDemandLM.h"
#include "moses/FactorCollection.h"
#include "moses/Util.h"
#include "moses/StaticData.h"
#include "moses/TranslationTask.h"
#include "moses/ContextScope.h"
#include "moses/LM/Ken.h"
#include "lm/model.hh"
#include "util/mmap.hh"
#include <cstdio>
#include <iostream>
#include <fstream>
using namespace std;
namespace Moses
{
InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line)
{
ReadParameters();
}
InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM()
{
}
void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask)
{
// The context scope object for this translation task
// contains a map of translation task-specific data
boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
// The key to the map is this object
void const* key = static_cast<void const*>(this);
// The value stored in the map is a string representing a phrase table
boost::shared_ptr<string> value = contextScope->get<string>(key);
// Create a stream to read the phrase table data
stringstream strme(*(value.get()));
char * nullpointer = (char *) 0;
const char * filename = std::tmpnam(nullpointer);
ofstream tmp;
tmp.open(filename);
// Read the phrase table data, one line at a time
string line;
while (getline(strme, line)) {
tmp << line << "\n";
}
tmp.close();
// m_tmpFilename.reset(new std::string("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm"));
m_tmpFilename.reset(new std::string(filename));
//LanguageModelKen<lm::ngram::ProbingModel> & lm =
GetPerThreadLM();
// std::remove(filename);
}
LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPerThreadLM() const
{
LanguageModelKen<lm::ngram::ProbingModel> *lm;
lm = m_perThreadLM.get();
if (lm == NULL) {
lm = new LanguageModelKen<lm::ngram::ProbingModel>();
string* filename = m_tmpFilename.get();
if (filename == NULL) {
UTIL_THROW(util::Exception, "Can't get a thread-specific LM because no temporary filename has been set for this thread\n");
} else {
lm->LoadModel(*filename, util::POPULATE_OR_READ);
}
VERBOSE(1, filename);
VERBOSE(1, "\tLM initialized\n");
m_perThreadLM.reset(lm);
}
assert(lm);
return *lm;
}
}
|
