1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
|
//
// REFERENCE
// ---------
// When using this feature, please cite:
//
// Matthias Huck, Joern Wuebker, Felix Rietig, and Hermann Ney.
// A Phrase Orientation Model for Hierarchical Machine Translation.
// In ACL 2013 Eighth Workshop on Statistical Machine Translation (WMT 2013), pages 452-463, Sofia, Bulgaria, August 2013.
//
#pragma once
#include <bitset>
#include <string>
#include <vector>
#include "StatefulFeatureFunction.h"
#include "FFState.h"
#include "moses/Factor.h"
#include "phrase-extract/extract-ghkm/PhraseOrientation.h"
#include "moses/PP/OrientationPhraseProperty.h"
#include <boost/unordered_set.hpp>
namespace Moses
{
class PhraseOrientationFeatureState : public FFState
{
public:
friend class PhraseOrientationFeature;
PhraseOrientationFeatureState(bool distinguishStates, bool useSparseWord, bool useSparseNT)
: m_leftBoundaryNonTerminalL2RScores(3,0)
, m_rightBoundaryNonTerminalR2LScores(3,0)
, m_leftBoundaryNonTerminalL2RPossibleFutureOrientations(0x7)
, m_rightBoundaryNonTerminalR2LPossibleFutureOrientations(0x7)
, m_leftBoundaryRecursionGuard(false)
, m_rightBoundaryRecursionGuard(false)
, m_leftBoundaryIsSet(false)
, m_rightBoundaryIsSet(false)
, m_distinguishStates(distinguishStates)
, m_useSparseWord(useSparseWord)
, m_useSparseNT(useSparseNT)
{}
void SetLeftBoundaryL2R(const std::vector<float> &scores,
size_t heuristicScoreIndex,
std::bitset<3> &possibleFutureOrientations,
const Factor* leftBoundaryNonTerminalSymbol,
const PhraseOrientationFeatureState* prevState) {
for (size_t i=0; i<3; ++i) {
m_leftBoundaryNonTerminalL2RScores[i] = scores[i];
m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i] = possibleFutureOrientations[i];
}
m_leftBoundaryNonTerminalL2RHeuristicScoreIndex = heuristicScoreIndex;
m_leftBoundaryNonTerminalSymbol = leftBoundaryNonTerminalSymbol;
m_leftBoundaryPrevState = prevState;
m_leftBoundaryIsSet = true;
}
void SetRightBoundaryR2L(const std::vector<float> &scores,
size_t heuristicScoreIndex,
std::bitset<3> &possibleFutureOrientations,
const Factor* rightBoundaryNonTerminalSymbol,
const PhraseOrientationFeatureState* prevState) {
for (size_t i=0; i<3; ++i) {
m_rightBoundaryNonTerminalR2LScores[i] = scores[i];
m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i] = possibleFutureOrientations[i];
}
m_rightBoundaryNonTerminalR2LHeuristicScoreIndex = heuristicScoreIndex;
m_rightBoundaryNonTerminalSymbol = rightBoundaryNonTerminalSymbol;
m_rightBoundaryPrevState = prevState;
m_rightBoundaryIsSet = true;
}
float GetLeftBoundaryL2RScoreMono() const {
return m_leftBoundaryNonTerminalL2RScores[0];
}
float GetLeftBoundaryL2RScoreSwap() const {
return m_leftBoundaryNonTerminalL2RScores[1];
}
float GetLeftBoundaryL2RScoreDiscontinuous() const {
return m_leftBoundaryNonTerminalL2RScores[2];
}
float GetRightBoundaryR2LScoreMono() const {
return m_rightBoundaryNonTerminalR2LScores[0];
}
float GetRightBoundaryR2LScoreSwap() const {
return m_rightBoundaryNonTerminalR2LScores[1];
}
float GetRightBoundaryR2LScoreDiscontinuous() const {
return m_rightBoundaryNonTerminalR2LScores[2];
}
int Compare(const FFState& other) const {
if (!m_distinguishStates) {
return 0;
}
const PhraseOrientationFeatureState &otherState = static_cast<const PhraseOrientationFeatureState&>(other);
if (!m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet &&
!m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 0;
}
if (m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 1;
}
if (!m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
return -1;
}
if (m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 1;
}
if (!m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
return -1;
}
if (m_leftBoundaryIsSet) {
int compareLeft = CompareLeftBoundaryRecursive(*this, otherState, m_useSparseNT);
if (compareLeft != 0) {
return compareLeft;
}
}
if (m_rightBoundaryIsSet) {
int compareRight = CompareRightBoundaryRecursive(*this, otherState, m_useSparseNT);
if (compareRight != 0) {
return compareRight;
}
}
return 0;
};
virtual size_t hash() const;
virtual bool operator==(const FFState& other) const
{
UTIL_THROW2("TODO:Haven't figure this out yet");
}
protected:
static int CompareLeftBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState, bool useSparseNT) {
if (!state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 0;
}
if (state.m_leftBoundaryIsSet && !otherState.m_leftBoundaryIsSet) {
return 1;
}
if (!state.m_leftBoundaryIsSet && otherState.m_leftBoundaryIsSet) {
return -1;
}
if (useSparseNT) {
if ( otherState.m_leftBoundaryNonTerminalSymbol < state.m_leftBoundaryNonTerminalSymbol ) {
return 1;
}
if ( state.m_leftBoundaryNonTerminalSymbol < otherState.m_leftBoundaryNonTerminalSymbol ) {
return -1;
}
}
if ( otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
return 1;
}
if ( state.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex < otherState.m_leftBoundaryNonTerminalL2RHeuristicScoreIndex ) {
return -1;
}
if ( Smaller(otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
return 1;
}
if ( Smaller(state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations, otherState.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations) ) {
return -1;
}
for (size_t i=0; i<state.m_leftBoundaryNonTerminalL2RScores.size(); ++i) {
// compare only for possible future orientations
// (possible future orientations of state and otherState are the same at this point due to the previous two conditional blocks)
if (state.m_leftBoundaryNonTerminalL2RPossibleFutureOrientations[i]) {
if (state.m_leftBoundaryNonTerminalL2RScores[i] > otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
return 1;
}
if (state.m_leftBoundaryNonTerminalL2RScores[i] < otherState.m_leftBoundaryNonTerminalL2RScores[i]) {
return -1;
}
}
}
if (state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
return 0;
}
if (state.m_leftBoundaryRecursionGuard && !otherState.m_leftBoundaryRecursionGuard) {
return 1;
}
if (!state.m_leftBoundaryRecursionGuard && otherState.m_leftBoundaryRecursionGuard) {
return -1;
}
const PhraseOrientationFeatureState *prevState = state.m_leftBoundaryPrevState;
const PhraseOrientationFeatureState *otherPrevState = otherState.m_leftBoundaryPrevState;
return CompareLeftBoundaryRecursive(*prevState, *otherPrevState, useSparseNT);
};
static int CompareRightBoundaryRecursive(const PhraseOrientationFeatureState& state, const PhraseOrientationFeatureState& otherState, bool useSparseNT) {
if (!state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 0;
}
if (state.m_rightBoundaryIsSet && !otherState.m_rightBoundaryIsSet) {
return 1;
}
if (!state.m_rightBoundaryIsSet && otherState.m_rightBoundaryIsSet) {
return -1;
}
if (useSparseNT) {
if ( otherState.m_rightBoundaryNonTerminalSymbol < state.m_rightBoundaryNonTerminalSymbol ) {
return 1;
}
if ( state.m_rightBoundaryNonTerminalSymbol < otherState.m_rightBoundaryNonTerminalSymbol ) {
return -1;
}
}
if ( otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
return 1;
}
if ( state.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex < otherState.m_rightBoundaryNonTerminalR2LHeuristicScoreIndex ) {
return -1;
}
if ( Smaller(otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
return 1;
}
if ( Smaller(state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations, otherState.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations) ) {
return -1;
}
for (size_t i=0; i<state.m_rightBoundaryNonTerminalR2LScores.size(); ++i) {
// compare only for possible future orientations
// (possible future orientations of state and otherState are the same at this point due to the previous two conditional blocks)
if ( state.m_rightBoundaryNonTerminalR2LPossibleFutureOrientations[i]) {
if (state.m_rightBoundaryNonTerminalR2LScores[i] > otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
return 1;
}
if (state.m_rightBoundaryNonTerminalR2LScores[i] < otherState.m_rightBoundaryNonTerminalR2LScores[i]) {
return -1;
}
}
}
if (state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
return 0;
}
if (state.m_rightBoundaryRecursionGuard && !otherState.m_rightBoundaryRecursionGuard) {
return 1;
}
if (!state.m_rightBoundaryRecursionGuard && otherState.m_rightBoundaryRecursionGuard) {
return -1;
}
const PhraseOrientationFeatureState *prevState = state.m_rightBoundaryPrevState;
const PhraseOrientationFeatureState *otherPrevState = otherState.m_rightBoundaryPrevState;
return CompareRightBoundaryRecursive(*prevState, *otherPrevState, useSparseNT);
};
template<std::size_t N> static bool Smaller(const std::bitset<N>& x, const std::bitset<N>& y) {
for (size_t i=0; i<N; ++i) {
if (x[i] ^ y[i])
return y[i];
}
return false;
}
std::vector<float> m_leftBoundaryNonTerminalL2RScores;
std::vector<float> m_rightBoundaryNonTerminalR2LScores;
size_t m_leftBoundaryNonTerminalL2RHeuristicScoreIndex;
size_t m_rightBoundaryNonTerminalR2LHeuristicScoreIndex;
std::bitset<3> m_leftBoundaryNonTerminalL2RPossibleFutureOrientations;
std::bitset<3> m_rightBoundaryNonTerminalR2LPossibleFutureOrientations;
bool m_leftBoundaryRecursionGuard;
bool m_rightBoundaryRecursionGuard;
bool m_leftBoundaryIsSet;
bool m_rightBoundaryIsSet;
const PhraseOrientationFeatureState* m_leftBoundaryPrevState;
const PhraseOrientationFeatureState* m_rightBoundaryPrevState;
const bool m_distinguishStates;
const bool m_useSparseWord;
const bool m_useSparseNT;
const Factor* m_leftBoundaryNonTerminalSymbol;
const Factor* m_rightBoundaryNonTerminalSymbol;
};
class PhraseOrientationFeature : public StatefulFeatureFunction
{
public:
struct ReoClassData {
public:
std::vector<MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS> nonTerminalReoClassL2R;
std::vector<MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS> nonTerminalReoClassR2L;
bool firstNonTerminalIsBoundary;
bool firstNonTerminalPreviousSourceSpanIsAligned;
bool firstNonTerminalFollowingSourceSpanIsAligned;
bool lastNonTerminalIsBoundary;
bool lastNonTerminalPreviousSourceSpanIsAligned;
bool lastNonTerminalFollowingSourceSpanIsAligned;
};
PhraseOrientationFeature(const std::string &line);
~PhraseOrientationFeature() {
}
bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new PhraseOrientationFeatureState(m_distinguishStates,m_useSparseWord,m_useSparseNT);
}
void SetParameter(const std::string& key, const std::string& value);
void Load();
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{};
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const {
UTIL_THROW2(GetScoreProducerDescription()
<< ": EvaluateWhenApplied(const Hypothesis&, ...) not implemented");
return new PhraseOrientationFeatureState(m_distinguishStates,m_useSparseWord,m_useSparseNT);
};
FFState* EvaluateWhenApplied(
const ChartHypothesis& cur_hypo,
int featureID, // used to index the state in the previous hypotheses
ScoreComponentCollection* accumulator) const;
protected:
void LoadWordList(const std::string& filename,
boost::unordered_set<const Factor*>& list);
void LookaheadScore(const OrientationPhraseProperty *orientationPhraseProperty,
ScoreComponentCollection &scoreBreakdown,
bool subtract=false) const;
size_t GetHeuristicScoreIndex(const std::vector<float>& scores,
size_t weightsVectorOffset,
const std::bitset<3> possibleFutureOrientations = 0x7) const;
void LeftBoundaryL2RScoreRecursive(int featureID,
const PhraseOrientationFeatureState *state,
const std::bitset<3> orientation,
std::vector<float>& newScores,
ScoreComponentCollection* scoreBreakdown) const;
void RightBoundaryR2LScoreRecursive(int featureID,
const PhraseOrientationFeatureState *state,
const std::bitset<3> orientation,
std::vector<float>& newScores,
ScoreComponentCollection* scoreBreakdown) const;
void SparseWordL2RScore(const ChartHypothesis* hypo,
ScoreComponentCollection* scoreBreakdown,
const std::string* o) const;
void SparseWordR2LScore(const ChartHypothesis* hypo,
ScoreComponentCollection* scoreBreakdown,
const std::string* o) const;
void SparseNonTerminalL2RScore(const Factor* nonTerminalSymbol,
ScoreComponentCollection* scoreBreakdown,
const std::string* o) const;
void SparseNonTerminalR2LScore(const Factor* nonTerminalSymbol,
ScoreComponentCollection* scoreBreakdown,
const std::string* o) const;
const std::string* ToString(const MosesTraining::Syntax::GHKM::PhraseOrientation::REO_CLASS o) const;
static const std::string MORIENT;
static const std::string SORIENT;
static const std::string DORIENT;
std::string m_glueTargetLHSStr;
const Factor* m_glueTargetLHS;
bool m_distinguishStates;
bool m_useSparseWord;
bool m_useSparseNT;
size_t m_offsetR2LScores;
mutable std::vector<float> m_weightsVector;
std::string m_filenameTargetWordList;
boost::unordered_set<const Factor*> m_targetWordList;
bool m_useTargetWordList;
std::string m_filenameSourceWordList;
boost::unordered_set<const Factor*> m_sourceWordList;
bool m_useSourceWordList;
};
}
|