diff options
author | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-02 23:06:28 +0400 |
---|---|---|
committer | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-02 23:06:28 +0400 |
commit | 6ff37224f1a6547d5bbcdab7982ffe570b69cecd (patch) | |
tree | 588bbb4b6315efa6e27aa4967ee132702768c8b7 | |
parent | e971edc1d495c4b53716e0eace540cd58ee672ff (diff) |
multiple LM per factor
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@464 1f5c12ca-751b-0410-a591-d2e778427230
-rwxr-xr-x | moses/src/Factor.cpp | 3 | ||||
-rwxr-xr-x | moses/src/Factor.h | 13 | ||||
-rwxr-xr-x | moses/src/FactorCollection.cpp | 21 | ||||
-rwxr-xr-x | moses/src/FactorCollection.h | 2 | ||||
-rwxr-xr-x | moses/src/LanguageModel.cpp | 1 | ||||
-rwxr-xr-x | moses/src/LanguageModel.h | 2 | ||||
-rwxr-xr-x | moses/src/LanguageModel_IRST.cpp | 16 | ||||
-rwxr-xr-x | moses/src/LanguageModel_SRI.cpp | 6 | ||||
-rwxr-xr-x | moses/src/TypeDef.h | 11 |
9 files changed, 16 insertions, 59 deletions
diff --git a/moses/src/Factor.cpp b/moses/src/Factor.cpp index cf75d3081..ec60e0aa1 100755 --- a/moses/src/Factor.cpp +++ b/moses/src/Factor.cpp @@ -25,11 +25,10 @@ using namespace std; size_t Factor::s_id = 0; -Factor::Factor(FactorDirection direction, FactorType factorType, const std::string *factorString, LmId lmId) +Factor::Factor(FactorDirection direction, FactorType factorType, const std::string *factorString) :m_direction(direction) ,m_factorType(factorType) ,m_ptrString(factorString) -,m_lmId(lmId) ,m_id(s_id++) { } diff --git a/moses/src/Factor.h b/moses/src/Factor.h index 6e184e022..a58bd12d2 100755 --- a/moses/src/Factor.h +++ b/moses/src/Factor.h @@ -59,15 +59,9 @@ protected: FactorType m_factorType; const std::string *m_ptrString; const size_t m_id; - LmId m_lmId; - Factor(FactorDirection direction, FactorType factorType, const std::string *factorString, LmId lmId); + Factor(FactorDirection direction, FactorType factorType, const std::string *factorString); - inline void SetLmId(LmId lmId) - { - m_lmId = lmId; - } - public: inline size_t hash() const { @@ -93,11 +87,6 @@ public: { return m_id; } - inline LmId GetLmId() const - { - return m_lmId; - } - // do it properly. needed for insert & finding of words in dictionary inline int Compare(const Factor &compare) const { diff --git a/moses/src/FactorCollection.cpp b/moses/src/FactorCollection.cpp index 5cb022e3b..1a276c2ba 100755 --- a/moses/src/FactorCollection.cpp +++ b/moses/src/FactorCollection.cpp @@ -53,7 +53,7 @@ bool FactorCollection::Exists(FactorDirection direction, FactorType factorType, const string *ptrString=&(*m_factorStringCollection.insert(factorString).first); FactorSet::const_iterator iterFactor; - Factor search(direction, factorType, ptrString, LanguageModel::UNKNOWN_LM_ID); + Factor search(direction, factorType, ptrString); iterFactor = m_collection.find(search); return iterFactor != m_collection.end(); @@ -61,26 +61,11 @@ bool FactorCollection::Exists(FactorDirection direction, FactorType factorType, const Factor *FactorCollection::AddFactor(FactorDirection direction , FactorType factorType - , const string &factorString - , LmId lmId) + , const string &factorString) { // find string id const string *ptrString=&(*m_factorStringCollection.insert(factorString).first); -// Factor findFactor(direction, factorType, ptr, lmId); - return &(*m_collection.insert(Factor(direction, factorType, ptrString, lmId)).first); -} - -const Factor *FactorCollection::AddFactor(FactorDirection direction - , FactorType factorType - , const string &factorString) -{ - return AddFactor(direction, factorType, factorString, LanguageModel::UNKNOWN_LM_ID); -} - -void FactorCollection::SetFactorLmId(const Factor *factor, LmId lmId) -{ // only used by non-srilm code - Factor *changeFactor = const_cast<Factor *>(factor); - changeFactor->SetLmId(lmId); + return &(*m_collection.insert(Factor(direction, factorType, ptrString)).first); } FactorCollection::~FactorCollection() diff --git a/moses/src/FactorCollection.h b/moses/src/FactorCollection.h index f3d20aff2..39122be46 100755 --- a/moses/src/FactorCollection.h +++ b/moses/src/FactorCollection.h @@ -65,8 +65,6 @@ public: bool Exists(FactorDirection direction, FactorType factorType, const std::string &factorString); const Factor *AddFactor(FactorDirection direction, FactorType factorType, const std::string &factorString); - const Factor *AddFactor(FactorDirection direction, FactorType factorType, const std::string &factorString, LmId lmId); - void SetFactorLmId(const Factor *factor, LmId lmId); void LoadVocab(FactorDirection direction, FactorType factorType, const std::string &fileName); diff --git a/moses/src/LanguageModel.cpp b/moses/src/LanguageModel.cpp index 7e9f2f959..df3f92d97 100755 --- a/moses/src/LanguageModel.cpp +++ b/moses/src/LanguageModel.cpp @@ -36,7 +36,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using namespace std; // static variable init -const LmId LanguageModel::UNKNOWN_LM_ID(-1); LanguageModel::State LanguageModel::UnknownState=0; LanguageModel::LanguageModel() diff --git a/moses/src/LanguageModel.h b/moses/src/LanguageModel.h index b6090307f..93a2ecbfc 100755 --- a/moses/src/LanguageModel.h +++ b/moses/src/LanguageModel.h @@ -44,8 +44,6 @@ public: typedef const void* State; static State UnknownState; - static const LmId UNKNOWN_LM_ID; - LanguageModel(); virtual ~LanguageModel(); virtual void Load(const std::string &fileName diff --git a/moses/src/LanguageModel_IRST.cpp b/moses/src/LanguageModel_IRST.cpp index f30ce2daa..a5cbd08c5 100755 --- a/moses/src/LanguageModel_IRST.cpp +++ b/moses/src/LanguageModel_IRST.cpp @@ -75,19 +75,21 @@ void LanguageModel_IRST::CreateFactors(FactorCollection &factorCollection) { // add factors which have srilm id dict_entry *entry; - LmId lmId; dictionary_iter iter(m_lmtb->dict); while ( (entry = iter.next()) != NULL) { - LmId lmId; - lmId.irst = entry->code; - factorCollection.AddFactor(Output, m_factorType, entry->word, lmId); + const Factor *factor = factorCollection.AddFactor(Output, m_factorType, entry->word); + m_lmIdLookup[factor] = entry->code; } + int lmId; lmId = GetLmID(BOS_); - m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, SENTENCE_START, lmId); + m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, SENTENCE_START); + m_lmIdLookup[m_sentenceStart] = lmId; + lmId = GetLmID(EOS_); - m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, SENTENCE_END, lmId); + m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, SENTENCE_END); + m_lmIdLookup[m_sentenceEnd] = lmId; } int LanguageModel_IRST::GetLmID( const std::string &str ) const @@ -112,7 +114,7 @@ float LanguageModel_IRST::GetValue(const vector<const Factor*> &contextFactor, S std::cout << i <<"="<<contextFactor[i]->GetLmId().irst <<"," << contextFactor[i]->GetString()<<" "; #endif int lmId = GetLmID(contextFactor[i]); - ng.pushc( (lmId == UNKNOWN_LM_ID.irst) ? m_unknownId : lmId); + ng.pushc(lmId); } #ifdef CDYER_DEBUG_LMSCORE std::cout <<" (ng='" << ng << "')\n"; diff --git a/moses/src/LanguageModel_SRI.cpp b/moses/src/LanguageModel_SRI.cpp index efea9392f..0055ce0e0 100755 --- a/moses/src/LanguageModel_SRI.cpp +++ b/moses/src/LanguageModel_SRI.cpp @@ -120,14 +120,12 @@ float LanguageModel_SRI::GetValue(const vector<const Factor*> &contextFactor, St size_t count = contextFactor.size(); for (size_t i = 0 ; i < count - 1 ; i++) { - VocabIndex lmId = GetLmID(contextFactor[count-2-i]); - context[i] = (lmId == UNKNOWN_LM_ID.sri) ? m_unknownId : lmId; + context[i] = GetLmID(contextFactor[count-2-i]); } context[count-1] = Vocab_None; // call sri lm fn - VocabIndex lmId = GetLmID(contextFactor[count-1]); - lmId= (lmId == UNKNOWN_LM_ID.sri) ? m_unknownId : lmId; + VocabIndex lmId= GetLmID(contextFactor[count-1]); float ret = GetValue(lmId, context); if (finalState) { diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index 67b901835..16f696e14 100755 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h @@ -66,17 +66,6 @@ const size_t DEFAULT_VERBOSE_LEVEL = 1; #endif ///////////////////////////////////////////////// -class NGramNode; -union LmId { - unsigned int sri; - const NGramNode* internal; - int irst; - public: - LmId() {} - LmId(int i) { irst = i; }; - LmId(const NGramNode *node) { internal = node; }; -}; - // enums. // must be 0, 1, 2, ..., unless otherwise stated |