1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
#ifndef MERT_SCORER_H_
#define MERT_SCORER_H_
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
#include "Types.h"
#include "ScoreData.h"
using namespace std;
class ScoreStats;
namespace mert {
class Vocabulary;
} // namespace mert
/**
* Superclass of all scorers and dummy implementation.
*
* In order to add a new scorer it should be sufficient to override the members
* prepareStats(), setReferenceFiles() and score() (or calculateScore()).
*/
class Scorer
{
public:
Scorer(const string& name, const string& config);
virtual ~Scorer();
/**
* Return the number of statistics needed for the computation of the score.
*/
virtual size_t NumberOfScores() const = 0;
/**
* Set the reference files. This must be called before prepareStats().
*/
virtual void setReferenceFiles(const vector<string>& referenceFiles) {
// do nothing
}
/**
* Process the given guessed text, corresponding to the given reference sindex
* and add the appropriate statistics to the entry.
*/
virtual void prepareStats(size_t sindex, const string& text, ScoreStats& entry) {
// do nothing.
}
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry) {
this->prepareStats(static_cast<size_t>(atoi(sindex.c_str())), text, entry);
}
/**
* Score using each of the candidate index, then go through the diffs
* applying each in turn, and calculating a new score each time.
*/
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) const = 0;
/*
{
//dummy impl
if (!m_score_data) {
throw runtime_error("score data not loaded");
}
scores.push_back(0);
for (size_t i = 0; i < diffs.size(); ++i) {
scores.push_back(0);
}
}
*/
/**
* Calculate the score of the sentences corresponding to the list of candidate
* indices. Each index indicates the 1-best choice from the n-best list.
*/
float score(const candidates_t& candidates) const {
diffs_t diffs;
statscores_t scores;
score(candidates, diffs, scores);
return scores[0];
}
const string& getName() const {
return m_name;
}
size_t getReferenceSize() const {
if (m_score_data) {
return m_score_data->size();
}
return 0;
}
/**
* Set the score data, prior to scoring.
*/
virtual void setScoreData(ScoreData* data) {
m_score_data = data;
}
/**
* Set the factors, which should be used for this metric
*/
virtual void setFactors(const string& factors);
/**
* Take the factored sentence and return the desired factors
*/
virtual string applyFactors(const string& sentece) const;
mert::Vocabulary* GetVocab() const { return m_vocab; }
private:
void InitConfig(const string& config);
string m_name;
mert::Vocabulary* m_vocab;
map<string, string> m_config;
vector<int> m_factors;
protected:
ScoreData* m_score_data;
bool m_enable_preserve_case;
/**
* Get value of config variable. If not provided, return default.
*/
string getConfig(const string& key, const string& def="") const {
map<string,string>::const_iterator i = m_config.find(key);
if (i == m_config.end()) {
return def;
} else {
return i->second;
}
}
/**
* Tokenise line and encode.
* Note: We assume that all tokens are separated by whitespaces.
*/
void TokenizeAndEncode(const string& line, vector<int>& encoded);
void ClearVocabulary();
};
/**
* Abstract base class for Scorers that work by adding statistics across all
* outout sentences, then apply some formula, e.g., BLEU, PER.
*/
class StatisticsBasedScorer : public Scorer
{
public:
StatisticsBasedScorer(const string& name, const string& config);
virtual ~StatisticsBasedScorer() {}
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) const;
protected:
enum RegularisationType {
NONE,
AVERAGE,
MINIMUM,
};
/**
* Calculate the actual score.
*/
virtual statscore_t calculateScore(const vector<int>& totals) const = 0;
// regularisation
RegularisationType m_regularization_type;
size_t m_regularization_window;
};
#endif // MERT_SCORER_H_
|
