1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
// $Id$
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef moses_Word_h
#define moses_Word_h
#include <cstring>
#include <iostream>
#include <vector>
#include <list>
#include "util/murmur_hash.hh"
#include "TypeDef.h"
#include "Util.h"
#include "util/string_piece.hh"
namespace Moses
{
class Factor;
class FactorMask;
/** Represent a word (terminal or non-term)
* Wrapper around hold a set of factors for a single word
*/
class Word
{
friend std::ostream& operator<<(std::ostream&, const Word&);
protected:
typedef const Factor * FactorArray[MAX_NUM_FACTORS];
FactorArray m_factorArray; /**< set of factors */
bool m_isNonTerminal;
bool m_isOOV;
public:
/** deep copy */
Word(const Word ©)
:m_isNonTerminal(copy.m_isNonTerminal)
,m_isOOV(copy.m_isOOV) {
std::memcpy(m_factorArray, copy.m_factorArray, sizeof(FactorArray));
}
/** empty word */
explicit Word(bool isNonTerminal = false) {
std::memset(m_factorArray, 0, sizeof(FactorArray));
m_isNonTerminal = isNonTerminal;
m_isOOV = false;
}
~Word() {}
//! returns Factor pointer for particular FactorType
const Factor*& operator[](FactorType index) {
return m_factorArray[index];
}
const Factor * const & operator[](FactorType index) const {
return m_factorArray[index];
}
//! Deprecated. should use operator[]
inline const Factor* GetFactor(FactorType factorType) const {
return m_factorArray[factorType];
}
inline void SetFactor(FactorType factorType, const Factor *factor) {
m_factorArray[factorType] = factor;
}
inline bool IsNonTerminal() const {
return m_isNonTerminal;
}
inline void SetIsNonTerminal(bool val) {
m_isNonTerminal = val;
}
inline bool IsOOV() const {
return m_isOOV;
}
inline void SetIsOOV(bool val) {
m_isOOV = val;
}
bool IsEpsilon() const;
/** add the factors from sourceWord into this representation,
* NULL elements in sourceWord will be skipped */
void Merge(const Word &sourceWord);
/** get string representation of list of factors. Used by PDTimp so supposed
* to be invariant to changes in format of debuggin output, therefore, doesn't
* use streaming output or ToString() from any class so not dependant on
* these debugging functions.
*/
std::string GetString(const std::vector<FactorType> factorType,bool endWithBlank) const;
StringPiece GetString(FactorType factorType) const;
TO_STRING();
bool operator== (const Word &compare) const;
inline bool operator!= (const Word &compare) const {
return !(*this == compare);
}
/* static functions */
/** transitive comparison of 2 word objects. Used by operator<.
* Only compare the co-joined factors, ie. where factor exists for both words.
* Should make it non-static
*/
static int Compare(const Word &targetWord, const Word &sourceWord);
void CreateFromString(FactorDirection direction
, const std::vector<FactorType> &factorOrder
, const StringPiece &str
, bool isNonTerminal
, bool strict = true);
void CreateUnknownWord(const Word &sourceWord);
void OnlyTheseFactors(const FactorMask &factors);
inline size_t hash() const {
return util::MurmurHashNative(m_factorArray, MAX_NUM_FACTORS*sizeof(Factor*), m_isNonTerminal);
}
};
inline size_t hash_value(const Word& word)
{
return word.hash();
}
}
#endif
|
