1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
#ifndef LEXICAL_REORDERING_TABLE_H
#define LEXICAL_REORDERING_TABLE_H
//stdlib dependencies:
#include <vector>
#include <map>
#include <string>
#include <iostream>
//moses dependencies:
#include "TypeDef.h"
#include "Phrase.h"
#include "InputType.h"
#include "ConfusionNet.h"
#include "Sentence.h"
#include "PrefixTreeMap.h"
class Phrase;
class InputType;
class ConfusionNet;
//additional types
typedef std::vector<float> Score;
typedef std::vector<FactorType> FactorList;
class LexicalReorderingTable {
public:
LexicalReorderingTable(const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors)
: m_FactorsF(f_factors), m_FactorsE(e_factors), m_FactorsC(c_factors) {
}
virtual ~LexicalReorderingTable(){
}
public:
static LexicalReorderingTable* LoadAvailable(const std::string& filePath, const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors);
public:
virtual Score GetScore(const Phrase& f, const Phrase& e, const Phrase& c) = 0;
virtual void InitializeForInput(const InputType&){
/* override for on-demand loading */
};
virtual void InitializeForInputPhrase(const Phrase&){
};
/*
int GetNumScoreComponents() const {
return m_NumScores;
}
*/
const FactorList& GetFFactorMask() const {
return m_FactorsF;
}
const FactorList& GetEFactorMask() const {
return m_FactorsE;
}
const FactorList& GetCFactorMask() const {
return m_FactorsC;
}
virtual void DbgDump(std::ostream* out) const{
*out << "Overwrite in subclass...\n";
};
protected:
FactorList m_FactorsF;
FactorList m_FactorsE;
FactorList m_FactorsC;
};
class LexicalReorderingTableMemory : public LexicalReorderingTable {
//implements LexicalReorderingTable saving all scores in one large std::map<> thingy
//to be used for non binary tables... uses a LOT of memory
public:
LexicalReorderingTableMemory( const std::string& filePath,
const std::vector<FactorType>& f_factors,
const std::vector<FactorType>& e_factors,
const std::vector<FactorType>& c_factors);
virtual ~LexicalReorderingTableMemory();
public:
virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c);
void DbgDump(std::ostream* out) const;
private:
std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const;
std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const;
void LoadFromFile(const std::string& filePath);
private:
typedef std::map< std::string, std::vector<float> > TableType;
TableType m_Table;
};
class LexicalReorderingTableTree : public LexicalReorderingTable {
//implements LexicalReorderingTable using the crafty PDT code...
public:
LexicalReorderingTableTree(const std::string& filePath,
const std::vector<FactorType>& f_factors,
const std::vector<FactorType>& e_factors,
const std::vector<FactorType>& c_factors);
~LexicalReorderingTableTree();
public:
bool IsCacheEnabled() const {
return m_UseCache;
};
void EnableCache() {
m_UseCache = true;
};
void DisableCache() {
m_UseCache = false;
};
void ClearCache(){
m_Cache.clear();
};
virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c);
virtual void InitializeForInput(const InputType& input);
virtual void InitializeForInputPhrase(const Phrase& f){
ClearCache();
auxCacheForSrcPhrase(f);
}
public:
static bool Create(std::istream& inFile, const std::string& outFileName);
private:
std::string MakeCacheKey(const Phrase& f, const Phrase& e) const;
IPhrase MakeTableKey(const Phrase& f, const Phrase& e) const;
void Cache(const ConfusionNet& input);
void Cache(const Sentence& input);
void auxCacheForSrcPhrase(const Phrase& f);
Score auxFindScoreForContext(const Candidates& cands, const Phrase& contex);
private:
//typedef LexicalReorderingCand CandType;
typedef std::map< std::string, Candidates > CacheType;
typedef PrefixTreeMap TableType;
static const int SourceVocId = 0;
static const int TargetVocId = 1;
bool m_UseCache;
CacheType m_Cache;
TableType m_Table;
};
#endif //LEXICAL_REORDERING_TABLE_H
|