1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "ChartRuleLookupManagerMemoryPerSentence.h"
#include "RuleTable/PhraseDictionaryFuzzyMatch.h"
#include "InputType.h"
#include "ChartTranslationOptionList.h"
#include "DotChartInMemory.h"
#include "StaticData.h"
#include "NonTerminal.h"
#include "ChartCellCollection.h"
namespace Moses
{
ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence(
const InputType &src,
const ChartCellCollection &cellColl,
const PhraseDictionaryFuzzyMatch &ruleTable)
: ChartRuleLookupManagerCYKPlus(src, cellColl)
, m_ruleTable(ruleTable)
{
CHECK(m_dottedRuleColls.size() == 0);
size_t sourceSize = src.GetSize();
m_dottedRuleColls.resize(sourceSize);
const PhraseDictionaryNodeSCFG &rootNode = m_ruleTable.GetRootNode(src);
for (size_t ind = 0; ind < m_dottedRuleColls.size(); ++ind) {
#ifdef USE_BOOST_POOL
DottedRuleInMemory *initDottedRule = m_dottedRulePool.malloc();
new (initDottedRule) DottedRuleInMemory(rootNode);
#else
DottedRuleInMemory *initDottedRule = new DottedRuleInMemory(rootNode);
#endif
DottedRuleColl *dottedRuleColl = new DottedRuleColl(sourceSize - ind + 1);
dottedRuleColl->Add(0, initDottedRule); // init rule. stores the top node in tree
m_dottedRuleColls[ind] = dottedRuleColl;
}
}
ChartRuleLookupManagerMemoryPerSentence::~ChartRuleLookupManagerMemoryPerSentence()
{
RemoveAllInColl(m_dottedRuleColls);
}
void ChartRuleLookupManagerMemoryPerSentence::GetChartRuleCollection(
const WordsRange &range,
ChartTranslationOptionList &outColl)
{
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
size_t absEndPos = range.GetEndPos();
// MAIN LOOP. create list of nodes of target phrases
// get list of all rules that apply to spans at same starting position
DottedRuleColl &dottedRuleCol = *m_dottedRuleColls[range.GetStartPos()];
const DottedRuleList &expandableDottedRuleList = dottedRuleCol.GetExpandableDottedRuleList();
const ChartCellLabel &sourceWordLabel = GetCellCollection().Get(WordsRange(absEndPos, absEndPos)).GetSourceWordLabel();
// loop through the rules
// (note that expandableDottedRuleList can be expanded as the loop runs
// through calls to ExtendPartialRuleApplication())
for (size_t ind = 0; ind < expandableDottedRuleList.size(); ++ind) {
// rule we are about to extend
const DottedRuleInMemory &prevDottedRule = *expandableDottedRuleList[ind];
// we will now try to extend it, starting after where it ended
size_t startPos = prevDottedRule.IsRoot()
? range.GetStartPos()
: prevDottedRule.GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
// (if only one more word position needs to be covered)
if (startPos == absEndPos) {
// look up in rule dictionary, if the current rule can be extended
// with the source word in the last position
const Word &sourceWord = sourceWordLabel.GetLabel();
const PhraseDictionaryNodeSCFG *node = prevDottedRule.GetLastNode().GetChild(sourceWord);
// if we found a new rule -> create it and add it to the list
if (node != NULL) {
// create the rule
#ifdef USE_BOOST_POOL
DottedRuleInMemory *dottedRule = m_dottedRulePool.malloc();
new (dottedRule) DottedRuleInMemory(*node, sourceWordLabel,
prevDottedRule);
#else
DottedRuleInMemory *dottedRule = new DottedRuleInMemory(*node,
sourceWordLabel,
prevDottedRule);
#endif
dottedRuleCol.Add(relEndPos+1, dottedRule);
}
}
// search for non-terminals
size_t endPos, stackInd;
// span is already complete covered? nothing can be done
if (startPos > absEndPos)
continue;
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos()) {
// We're at the root of the prefix tree so won't try to cover the full
// span (i.e. we don't allow non-lexical unary rules). However, we need
// to match non-unary rules that begin with a non-terminal child, so we
// do that in two steps: during this iteration we search for non-terminals
// that cover all but the last source word in the span (there won't
// already be running nodes for these because that would have required a
// non-lexical unary rule match for an earlier span). Any matches will
// result in running nodes being appended to the list and on subsequent
// iterations (for this same span), we'll extend them to cover the final
// word.
endPos = absEndPos - 1;
stackInd = relEndPos;
}
else
{
endPos = absEndPos;
stackInd = relEndPos + 1;
}
ExtendPartialRuleApplication(prevDottedRule, startPos, endPos, stackInd,
dottedRuleCol);
}
// list of rules that that cover the entire span
DottedRuleList &rules = dottedRuleCol.Get(relEndPos + 1);
// look up target sides for the rules
DottedRuleList::const_iterator iterRule;
for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) {
const DottedRuleInMemory &dottedRule = **iterRule;
const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode();
// look up target sides
const TargetPhraseCollection *tpc = node.GetTargetPhraseCollection();
// add the fully expanded rule (with lexical target side)
if (tpc != NULL) {
AddCompletedRule(dottedRule, *tpc, range, outColl);
}
}
dottedRuleCol.Clear(relEndPos+1);
outColl.ShrinkToLimit();
}
// Given a partial rule application ending at startPos-1 and given the sets of
// source and target non-terminals covering the span [startPos, endPos],
// determines the full or partial rule applications that can be produced through
// extending the current rule application by a single non-terminal.
void ChartRuleLookupManagerMemoryPerSentence::ExtendPartialRuleApplication(
const DottedRuleInMemory &prevDottedRule,
size_t startPos,
size_t endPos,
size_t stackInd,
DottedRuleColl & dottedRuleColl)
{
// source non-terminal labels for the remainder
const NonTerminalSet &sourceNonTerms =
GetSentence().GetLabelSet(startPos, endPos);
// target non-terminal labels for the remainder
const ChartCellLabelSet &targetNonTerms =
GetCellCollection().Get(WordsRange(startPos, endPos)).GetTargetLabelSet();
// note where it was found in the prefix tree of the rule dictionary
const PhraseDictionaryNodeSCFG &node = prevDottedRule.GetLastNode();
const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap =
node.GetNonTerminalMap();
const size_t numChildren = nonTermMap.size();
if (numChildren == 0) {
return;
}
const size_t numSourceNonTerms = sourceNonTerms.size();
const size_t numTargetNonTerms = targetNonTerms.GetSize();
const size_t numCombinations = numSourceNonTerms * numTargetNonTerms;
// We can search by either:
// 1. Enumerating all possible source-target NT pairs that are valid for
// the span and then searching for matching children in the node,
// or
// 2. Iterating over all the NT children in the node, searching
// for each source and target NT in the span's sets.
// We'll do whichever minimises the number of lookups:
if (numCombinations <= numChildren*2) {
// loop over possible source non-terminal labels (as found in input tree)
NonTerminalSet::const_iterator p = sourceNonTerms.begin();
NonTerminalSet::const_iterator sEnd = sourceNonTerms.end();
for (; p != sEnd; ++p) {
const Word & sourceNonTerm = *p;
// loop over possible target non-terminal labels (as found in chart)
ChartCellLabelSet::const_iterator q = targetNonTerms.begin();
ChartCellLabelSet::const_iterator tEnd = targetNonTerms.end();
for (; q != tEnd; ++q) {
const ChartCellLabel &cellLabel = q->second;
// try to match both source and target non-terminal
const PhraseDictionaryNodeSCFG * child =
node.GetChild(sourceNonTerm, cellLabel.GetLabel());
// nothing found? then we are done
if (child == NULL) {
continue;
}
// create new rule
#ifdef USE_BOOST_POOL
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
new (rule) DottedRuleInMemory(*child, cellLabel, prevDottedRule);
#else
DottedRuleInMemory *rule = new DottedRuleInMemory(*child, cellLabel,
prevDottedRule);
#endif
dottedRuleColl.Add(stackInd, rule);
}
}
}
else
{
// loop over possible expansions of the rule
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator p;
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator end =
nonTermMap.end();
for (p = nonTermMap.begin(); p != end; ++p) {
// does it match possible source and target non-terminals?
const PhraseDictionaryNodeSCFG::NonTerminalMapKey &key = p->first;
const Word &sourceNonTerm = key.first;
if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end()) {
continue;
}
const Word &targetNonTerm = key.second;
const ChartCellLabel *cellLabel = targetNonTerms.Find(targetNonTerm);
if (!cellLabel) {
continue;
}
// create new rule
const PhraseDictionaryNodeSCFG &child = p->second;
#ifdef USE_BOOST_POOL
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
new (rule) DottedRuleInMemory(child, *cellLabel, prevDottedRule);
#else
DottedRuleInMemory *rule = new DottedRuleInMemory(child, *cellLabel,
prevDottedRule);
#endif
dottedRuleColl.Add(stackInd, rule);
}
}
}
} // namespace Moses
|