1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
//
#include "StaticData.h"
#include "SyntacticLanguageModel.h"
#include "HHMMLangModel-gf.h"
#include "TextObsModel.h"
#include "SyntacticLanguageModelFiles.h"
#include "SyntacticLanguageModelState.h"
namespace Moses
{
// asnteousntaoheisnthaoesntih
SyntacticLanguageModel::SyntacticLanguageModel(const std::vector<std::string>& filePath,
const std::vector<float>& weights,
const FactorType factorType,
size_t beamWidth)
// Initialize member variables
: m_NumScoreComponents(weights.size())
, m_beamWidth(beamWidth)
, m_factorType(factorType)
, m_files(new SyntacticLanguageModelFiles<YModel,XModel>(filePath)) {
// Inform Moses score manager of this feature and its weight(s)
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
VERBOSE(3,"Constructed SyntacticLanguageModel" << endl);
}
SyntacticLanguageModel::~SyntacticLanguageModel() {
VERBOSE(3,"Destructing SyntacticLanguageModel" << std::endl);
// delete m_files;
}
size_t SyntacticLanguageModel::GetNumScoreComponents() const {
return m_NumScoreComponents;
}
std::string SyntacticLanguageModel::GetScoreProducerDescription(unsigned) const {
return "Syntactic Language Model";
}
std::string SyntacticLanguageModel::GetScoreProducerWeightShortName(unsigned) const {
return "slm";
}
const FFState* SyntacticLanguageModel::EmptyHypothesisState(const InputType &input) const {
return new SyntacticLanguageModelState<YModel,XModel,S,R>(m_files,m_beamWidth);
}
/*
double SyntacticLanguageModel::perplexity() {
SyntacticLanguageModelState<YModel,XModel,S,R> *prev =
new SyntacticLanguageModelState<YModel,XModel,S,R>(m_files,m_beamWidth);
std::cerr << "Initial prob:" << "\t" << prev->getProb() <<std::endl;
std::vector<std::string> words(3);
words[0] = "no";
words[1] = ",";
words[2] = "zxvth";
for (std::vector<std::string>::iterator i=words.begin();
i != words.end();
i++) {
prev = new SyntacticLanguageModelState<YModel,XModel,S,R>(prev, *i);
std::cerr << *i << "\t" << prev->getProb() <<std::endl;
}
if (true) exit(-1);
return prev->getProb();
}
*/
FFState* SyntacticLanguageModel::Evaluate(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const {
VERBOSE(3,"Evaluating SyntacticLanguageModel for a hypothesis" << endl);
const SyntacticLanguageModelState<YModel,XModel,S,R>& prev =
static_cast<const SyntacticLanguageModelState<YModel,XModel,S,R>&>(*prev_state);
const SyntacticLanguageModelState<YModel,XModel,S,R>* currentState = &prev;
SyntacticLanguageModelState<YModel,XModel,S,R>* nextState = NULL;
const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();
for (size_t i=0, n=targetPhrase.GetSize(); i<n; i++) {
const Word& word = targetPhrase.GetWord(i);
const Factor* factor = word.GetFactor(m_factorType);
const std::string& string = factor->GetString();
if (i==0) {
nextState = new SyntacticLanguageModelState<YModel,XModel,S,R>(&prev, string);
} else {
currentState = nextState;
nextState = new SyntacticLanguageModelState<YModel,XModel,S,R>(currentState, string);
}
double score = nextState->getScore();
VERBOSE(3,"SynLM evaluated a score of " << score << endl);
accumulator->Assign( this, score );
}
return nextState;
}
}
|
