1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
/*
* score_reordering.cpp
*
* Created by: Sara Stymne - Linköping University
* Machine Translation Marathon 2010, Dublin
*/
#include <string>
#include <vector>
#include <map>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstdlib>
#include <cstring>
#include "reordering_classes.h"
using namespace std;
void split_line(const string& line, string& foreign, string& english, string& wbe, string& phrase, string& hier);
void get_orientations(const string& pair, string& previous, string& next);
int main(int argc, char* argv[])
{
cerr << "Lexical Reordering Scorer\n"
<< "scores lexical reordering models of several types (hierarchical, phrase-based and word-based-extraction\n";
if (argc < 3) {
cerr << "syntax: score_reordering extractFile smoothingValue filepath (--model \"type max-orientation (specification-strings)\" )+\n";
exit(1);
}
char* extractFileName = argv[1];
double smoothingValue = atof(argv[2]);
string filepath = argv[3];
ifstream eFile(extractFileName);
if (!eFile) {
cerr << "Could not open the extract file " << extractFileName <<"for scoring of lexical reordering models\n";
exit(1);
}
bool smoothWithCounts = false;
map<string,ModelScore*> modelScores;
vector<Model*> models;
bool hier = false;
bool phrase = false;
bool wbe = false;
string e,f,w,p,h;
string prev, next;
int i = 4;
while (i<argc) {
if (strcmp(argv[i],"--SmoothWithCounts") == 0) {
smoothWithCounts = true;
} else if (strcmp(argv[i],"--model") == 0) {
if (i+1 >= argc) {
cerr << "score: syntax error, no model information provided to the option" << argv[i] << endl;
exit(1);
}
istringstream is(argv[++i]);
string m,t;
is >> m >> t;
modelScores[m] = ModelScore::createModelScore(t);
if (m.compare("hier") == 0) {
hier = true;
} else if (m.compare("phrase") == 0) {
phrase = true;
}
if (m.compare("wbe") == 0) {
wbe = true;
}
if (!hier && !phrase && !wbe) {
cerr << "WARNING: No models specified for lexical reordering. No lexical reordering table will be trained.\n";
return 0;
}
string config;
//Store all models
while (is >> config) {
models.push_back(Model::createModel(modelScores[m],config,filepath));
}
} else {
cerr << "illegal option given to lexical reordering model score\n";
exit(1);
}
i++;
}
////////////////////////////////////
//calculate smoothing
if (smoothWithCounts) {
string line;
while (getline(eFile,line)) {
split_line(line,e,f,w,p,h);
if (hier) {
get_orientations(h, prev, next);
modelScores["hier"]->add_example(prev,next);
}
if (phrase) {
get_orientations(p, prev, next);
modelScores["phrase"]->add_example(prev,next);
}
if (wbe) {
get_orientations(w, prev, next);
modelScores["wbe"]->add_example(prev,next);
}
}
// calculate smoothing for each model
for (int i=0; i<models.size(); ++i) {
models[i]->createSmoothing(smoothingValue);
}
//reopen eFile
eFile.close();
eFile.open(extractFileName);
} else {
//constant smoothing
for (int i=0; i<models.size(); ++i) {
models[i]->createConstSmoothing(smoothingValue);
}
}
////////////////////////////////////
//calculate scores for reordering table
string line,f_current,e_current;
bool first = true;
while (getline(eFile, line)) {
split_line(line,f,e,w,p,h);
if (first) {
f_current = f;
e_current = e;
first = false;
} else if (f.compare(f_current) != 0 || e.compare(e_current) != 0) {
//fe - score
for (int i=0; i<models.size(); ++i) {
models[i]->score_fe(f_current,e_current);
}
//reset
for(map<string,ModelScore*>::const_iterator it = modelScores.begin(); it != modelScores.end(); ++it) {
it->second->reset_fe();
}
if (f.compare(f_current) != 0) {
//f - score
for (int i=0; i<models.size(); ++i) {
models[i]->score_f(f_current);
}
//reset
for(map<string,ModelScore*>::const_iterator it = modelScores.begin(); it != modelScores.end(); ++it) {
it->second->reset_f();
}
}
f_current = f;
e_current = e;
}
// uppdate counts
if (hier) {
get_orientations(h, prev, next);
modelScores["hier"]->add_example(prev,next);
}
if (phrase) {
get_orientations(p, prev, next);
modelScores["phrase"]->add_example(prev,next);
}
if (wbe) {
get_orientations(w, prev, next);
modelScores["wbe"]->add_example(prev,next);
}
}
//Score the last phrases
for (int i=0; i<models.size(); ++i) {
models[i]->score_fe(f,e);
}
for (int i=0; i<models.size(); ++i) {
models[i]->score_f(f);
}
//Zip all files
for (int i=0; i<models.size(); ++i) {
models[i]->zipFile();
}
return 0;
}
void split_line(const string& line, string& foreign, string& english, string& wbe, string& phrase, string& hier)
{
int begin = 0;
int end = line.find(" ||| ");
foreign = line.substr(begin, end - begin);
begin = end+5;
end = line.find(" ||| ", begin);
english = line.substr(begin, end - begin);
begin = end+5;
end = line.find(" | ", begin);
wbe = line.substr(begin, end - begin);
begin = end+3;
end = line.find(" | ", begin);
phrase = line.substr(begin, end - begin);
begin = end+3;
hier = line.substr(begin, line.size() - begin);
}
void get_orientations(const string& pair, string& previous, string& next)
{
istringstream is(pair);
is >> previous >> next;
}
|
