blob: 9358ee4bf5a318d0408a18ac053a9b583e9a372c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
#include "moses/PP/TargetPreferencesPhraseProperty.h"
#include <iostream>
#include <cstdio>
#include <cstdlib>
#include <sstream>
#include <string>
#include <queue>
#include <assert.h>
#include <limits>
namespace Moses
{
void TargetPreferencesPhraseProperty::ProcessValue(const std::string &value)
{
std::istringstream tokenizer(value);
if (! (tokenizer >> m_nNTs)) { // first token: number of non-terminals (incl. left-hand side)
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read number of non-terminals. Flawed property?");
}
assert( m_nNTs > 0 );
if (! (tokenizer >> m_totalCount)) { // second token: overall rule count
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read overall rule count. Flawed property?");
}
assert( m_totalCount > 0.0 );
// read labelled rule items
std::priority_queue<float> ruleLabelledCountsPQ;
while (tokenizer.peek() != EOF) {
try {
TargetPreferencesPhrasePropertyItem item;
size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max();
if (m_nNTs == 1) {
item.m_labelsRHSCount = m_totalCount;
} else { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule
for (size_t i=0; i<m_nNTs-1; ++i) { // RHS non-terminal labels
size_t labelRHS;
if (! (tokenizer >> labelRHS) ) { // RHS non-terminal label
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read right-hand side label index. Flawed property?");
}
item.m_labelsRHS.push_back(labelRHS);
}
if (! (tokenizer >> item.m_labelsRHSCount)) {
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read right-hand side count. Flawed property?");
}
if (! (tokenizer >> numberOfLHSsGivenRHS)) {
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read number of left-hand sides. Flawed property?");
}
}
for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS non-terminal labels seen with this RHS
size_t labelLHS;
if (! (tokenizer >> labelLHS)) { // LHS non-terminal label
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read left-hand side label index. Flawed property?");
}
float ruleLabelledCount;
if (! (tokenizer >> ruleLabelledCount)) {
UTIL_THROW2("TargetPreferencesPhraseProperty: Not able to read count. Flawed property?");
}
item.m_labelsLHSList.push_back( std::make_pair(labelLHS,ruleLabelledCount) );
ruleLabelledCountsPQ.push(ruleLabelledCount);
}
m_labelItems.push_back(item);
} catch (const std::exception &e) {
UTIL_THROW2("TargetPreferencesPhraseProperty: Read error. Flawed property?");
}
}
// keep only top N label vectors
const size_t N=50;
if (ruleLabelledCountsPQ.size() > N) {
float topNRuleLabelledCount = std::numeric_limits<int>::max();
for (size_t i=0; !ruleLabelledCountsPQ.empty() && i<N; ++i) {
topNRuleLabelledCount = ruleLabelledCountsPQ.top();
ruleLabelledCountsPQ.pop();
}
size_t nKept=0;
std::list<TargetPreferencesPhrasePropertyItem>::iterator itemIter=m_labelItems.begin();
while (itemIter!=m_labelItems.end()) {
if (itemIter->m_labelsRHSCount < topNRuleLabelledCount) {
itemIter = m_labelItems.erase(itemIter);
} else {
std::list< std::pair<size_t,float> >::iterator itemLHSIter=(itemIter->m_labelsLHSList).begin();
while (itemLHSIter!=(itemIter->m_labelsLHSList).end()) {
if (itemLHSIter->second < topNRuleLabelledCount) {
itemLHSIter = (itemIter->m_labelsLHSList).erase(itemLHSIter);
} else {
if (nKept >= N) {
itemLHSIter = (itemIter->m_labelsLHSList).erase(itemLHSIter,(itemIter->m_labelsLHSList).end());
} else {
++nKept;
++itemLHSIter;
}
}
}
if ((itemIter->m_labelsLHSList).empty()) {
itemIter = m_labelItems.erase(itemIter);
} else {
++itemIter;
}
}
}
}
};
} // namespace Moses
|
