1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#include <vector>
#include <boost/scoped_ptr.hpp>
#include <boost/shared_ptr.hpp>
#include "ForestRescore.h"
#include "Hypergraph.h"
#include "HypPackEnumerator.h"
#include "MiraFeatureVector.h"
#include "MiraWeightVector.h"
//
// Used by batch mira to get the hope, fear and model hypothesis. This wraps
// the n-best list and lattice/hypergraph implementations
//
namespace MosesTuning
{
/** Initialise weights from files. Returns weight vector and number of dense features */
std::pair<MiraWeightVector*,size_t>
InitialiseWeights(const std::string& denseInitFile, const std::string& sparseInitFile,
const std::string& type, bool verbose);
class Scorer;
/** To be filled in by the decoder */
struct HopeFearData {
MiraFeatureVector modelFeatures;
MiraFeatureVector hopeFeatures;
MiraFeatureVector fearFeatures;
std::vector<float> modelStats;
std::vector<float> hopeStats;
ValType hopeBleu;
ValType fearBleu;
bool hopeFearEqual;
};
//Abstract base class
class HopeFearDecoder
{
public:
//iterator methods
virtual void reset() = 0;
virtual void next() = 0;
virtual bool finished() = 0;
virtual ~HopeFearDecoder() {};
/**
* Calculate hope, fear and model hypotheses
**/
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) = 0;
/** Max score decoding */
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats)
= 0;
/** Calculate bleu on training set */
ValType Evaluate(const AvgWeightVector& wv);
protected:
Scorer* scorer_;
};
/** Gets hope-fear from nbest lists */
class NbestHopeFearDecoder : public virtual HopeFearDecoder
{
public:
NbestHopeFearDecoder(const std::vector<std::string>& featureFiles,
const std::vector<std::string>& scoreFiles,
bool streaming,
bool no_shuffle,
bool safe_hope,
Scorer* scorer
);
virtual void reset();
virtual void next();
virtual bool finished();
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
);
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats);
private:
boost::scoped_ptr<HypPackEnumerator> train_;
bool safe_hope_;
};
/** Gets hope-fear from hypergraphs */
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder
{
public:
HypergraphHopeFearDecoder(
const std::string& hypergraphDir,
const std::vector<std::string>& referenceFiles,
size_t num_dense,
bool streaming,
bool no_shuffle,
bool safe_hope,
size_t hg_pruning,
const MiraWeightVector& wv,
Scorer* scorer_
);
virtual void reset();
virtual void next();
virtual bool finished();
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
);
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats);
private:
size_t num_dense_;
//maps sentence Id to graph ptr
typedef std::map<size_t, boost::shared_ptr<Graph> > GraphColl;
GraphColl graphs_;
std::vector<size_t> sentenceIds_;
std::vector<size_t>::const_iterator sentenceIdIter_;
ReferenceSet references_;
Vocab vocab_;
};
};
|
