1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
#ifndef moses_LexicalReorderingTable_h
#define moses_LexicalReorderingTable_h
//stdlib dependencies:
#include <vector>
#include <map>
#include <memory>
#include <string>
#include <iostream>
#ifdef WITH_THREADS
#include <boost/thread/tss.hpp>
#endif
//moses dependencies:
#include "TypeDef.h"
#include "Phrase.h"
#include "InputType.h"
#include "ConfusionNet.h"
#include "Sentence.h"
#include "PrefixTreeMap.h"
namespace Moses
{
class Phrase;
class InputType;
class ConfusionNet;
//additional types
class LexicalReorderingTable
{
public:
LexicalReorderingTable(const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors)
: m_FactorsF(f_factors), m_FactorsE(e_factors), m_FactorsC(c_factors) {
}
virtual ~LexicalReorderingTable() {
}
public:
static LexicalReorderingTable* LoadAvailable(const std::string& filePath, const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors);
public:
virtual Scores GetScore(const Phrase& f, const Phrase& e, const Phrase& c) = 0;
virtual void InitializeForInput(const InputType&) {
/* override for on-demand loading */
};
virtual void InitializeForInputPhrase(const Phrase&) {
};
/*
int GetNumScoreComponents() const {
return m_NumScores;
}
*/
const FactorList& GetFFactorMask() const {
return m_FactorsF;
}
const FactorList& GetEFactorMask() const {
return m_FactorsE;
}
const FactorList& GetCFactorMask() const {
return m_FactorsC;
}
virtual void DbgDump(std::ostream* out) const {
*out << "Overwrite in subclass...\n";
};
protected:
FactorList m_FactorsF;
FactorList m_FactorsE;
FactorList m_FactorsC;
};
class LexicalReorderingTableMemory : public LexicalReorderingTable
{
//implements LexicalReorderingTable saving all scores in one large std::map<> thingy
//to be used for non binary tables... uses a LOT of memory
public:
LexicalReorderingTableMemory( const std::string& filePath,
const std::vector<FactorType>& f_factors,
const std::vector<FactorType>& e_factors,
const std::vector<FactorType>& c_factors);
virtual ~LexicalReorderingTableMemory();
public:
virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c);
void DbgDump(std::ostream* out) const;
private:
std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const;
std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const;
void LoadFromFile(const std::string& filePath);
private:
typedef std::map< std::string, std::vector<float> > TableType;
TableType m_Table;
};
class LexicalReorderingTableTree : public LexicalReorderingTable
{
//implements LexicalReorderingTable using the crafty PDT code...
public:
LexicalReorderingTableTree(const std::string& filePath,
const std::vector<FactorType>& f_factors,
const std::vector<FactorType>& e_factors,
const std::vector<FactorType>& c_factors);
~LexicalReorderingTableTree();
public:
bool IsCacheEnabled() const {
return m_UseCache;
};
void EnableCache() {
m_UseCache = true;
};
void DisableCache() {
m_UseCache = false;
};
void ClearCache() {
if (m_UseCache) {
m_Cache.clear();
}
};
virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c);
virtual void InitializeForInput(const InputType& input);
virtual void InitializeForInputPhrase(const Phrase& f) {
ClearCache();
auxCacheForSrcPhrase(f);
}
public:
static bool Create(std::istream& inFile, const std::string& outFileName);
private:
std::string MakeCacheKey(const Phrase& f, const Phrase& e) const;
IPhrase MakeTableKey(const Phrase& f, const Phrase& e) const;
void Cache(const ConfusionNet& input);
void Cache(const Sentence& input);
void auxCacheForSrcPhrase(const Phrase& f);
Scores auxFindScoreForContext(const Candidates& cands, const Phrase& contex);
private:
//typedef LexicalReorderingCand CandType;
typedef std::map< std::string, Candidates > CacheType;
#ifdef WITH_THREADS
typedef boost::thread_specific_ptr<PrefixTreeMap> TableType;
#else
typedef std::auto_ptr<PrefixTreeMap> TableType;
#endif
static const int SourceVocId = 0;
static const int TargetVocId = 1;
bool m_UseCache;
std::string m_FilePath;
CacheType m_Cache;
TableType m_Table;
};
}
#endif
|
