diff options
author | redpony <redpony@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-07-18 17:07:31 +0400 |
---|---|---|
committer | redpony <redpony@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-07-18 17:07:31 +0400 |
commit | 9257c64c07dbb1aa92dd8daef446a1b159e7fb26 (patch) | |
tree | 310333ed85ee875a53435a5f109a5073c031ba8b /moses | |
parent | f64cc96d7b16f26eb6dd8d8d9fb72b816e791a7e (diff) |
more language model cleanup:
* move n-best functionality into client, so that LanguageModel interface
doesn't need to take a reference to a NULL list
* fix performance bug with IRST implementation that was replicated from SRI
* make context array that is used in SRI live in the class rather than local
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@166 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'moses')
-rwxr-xr-x | moses/src/LanguageModel.cpp | 9 | ||||
-rwxr-xr-x | moses/src/LanguageModel.h | 3 | ||||
-rwxr-xr-x | moses/src/LanguageModel_IRST.cpp | 2 | ||||
-rwxr-xr-x | moses/src/LanguageModel_IRST.h | 2 | ||||
-rwxr-xr-x | moses/src/LanguageModel_SRI.cpp | 7 | ||||
-rwxr-xr-x | moses/src/LanguageModel_SRI.h | 14 | ||||
-rw-r--r-- | moses/src/TargetPhrase.cpp | 14 |
7 files changed, 24 insertions, 27 deletions
diff --git a/moses/src/LanguageModel.cpp b/moses/src/LanguageModel.cpp index fa032e19f..d2f0e7647 100755 --- a/moses/src/LanguageModel.cpp +++ b/moses/src/LanguageModel.cpp @@ -42,8 +42,7 @@ LanguageModel::~LanguageModel() {} void LanguageModel::CalcScore(const Phrase &phrase , float &fullScore - , float &ngramScore - , list< std::pair<size_t, float> > &ngramComponent) const + , float &ngramScore) const { fullScore = 0; ngramScore = 0; @@ -79,11 +78,5 @@ void LanguageModel::CalcScore(const Phrase &phrase ngramScore += GetValue(contextFactor); } fullScore += ngramScore; - -#ifdef N_BEST - size_t lmId = GetId(); - pair<size_t, float> store(lmId, ngramScore); - ngramComponent.push_back(store); -#endif } diff --git a/moses/src/LanguageModel.h b/moses/src/LanguageModel.h index 5170363e1..39eabef45 100755 --- a/moses/src/LanguageModel.h +++ b/moses/src/LanguageModel.h @@ -65,8 +65,7 @@ public: } void CalcScore(const Phrase &phrase , float &fullScore - , float &ngramScore - , std::list< std::pair<size_t, float> > &ngramComponent) const; + , float &ngramScore) const; FactorType GetFactorType() const { return m_factorType; diff --git a/moses/src/LanguageModel_IRST.cpp b/moses/src/LanguageModel_IRST.cpp index 3bcc95270..0729791dc 100755 --- a/moses/src/LanguageModel_IRST.cpp +++ b/moses/src/LanguageModel_IRST.cpp @@ -110,7 +110,7 @@ float LanguageModel_IRST::GetValue(const vector<const Factor*> &contextFactor) c ngram ng(m_lmtb->dict); for (size_t i = 0 ; i < count ; i++) { - ng.pushc(GetLmID(contextFactor[i]->GetString()).irst); + ng.pushc(contextFactor[i]->GetLmId().irst); } return TransformScore(m_lmtb->prob(ng)); } diff --git a/moses/src/LanguageModel_IRST.h b/moses/src/LanguageModel_IRST.h index aa52d9a4f..42bbe3558 100755 --- a/moses/src/LanguageModel_IRST.h +++ b/moses/src/LanguageModel_IRST.h @@ -48,7 +48,7 @@ public: , FactorType factorType , float weight , size_t nGramOrder); - + protected: lmtable* m_lmtb; diff --git a/moses/src/LanguageModel_SRI.cpp b/moses/src/LanguageModel_SRI.cpp index 30346e340..a294a5b6b 100755 --- a/moses/src/LanguageModel_SRI.cpp +++ b/moses/src/LanguageModel_SRI.cpp @@ -114,15 +114,14 @@ float LanguageModel_SRI::GetValue(const vector<const Factor*> &contextFactor) co { // set up context size_t count = contextFactor.size(); - VocabIndex context[MAX_NGRAM_SIZE]; for (size_t i = 0 ; i < count - 1 ; i++) { - context[i] = contextFactor[count-2-i]->GetLmId().sri; + m_context[i] = contextFactor[count-2-i]->GetLmId().sri; } - context[count-1] = Vocab_None; + m_context[count-1] = Vocab_None; // call sri lm fn - float ret = GetValue(contextFactor[count-1]->GetLmId().sri, context); + float ret = GetValue(contextFactor[count-1]->GetLmId().sri, m_context); return ret; } diff --git a/moses/src/LanguageModel_SRI.h b/moses/src/LanguageModel_SRI.h index f1a0acbd5..a8334da65 100755 --- a/moses/src/LanguageModel_SRI.h +++ b/moses/src/LanguageModel_SRI.h @@ -51,17 +51,17 @@ public: , FactorType factorType , float weight , size_t nGramOrder); - + +private: + // not thread-safe, but not much is + mutable VocabIndex m_context[MAX_NGRAM_SIZE]; + protected: - Vocab *m_srilmVocab; // TODO - make this a ptr, remove #include from header - Ngram *m_srilmModel; // " " + Vocab *m_srilmVocab; + Ngram *m_srilmModel; float GetValue(VocabIndex wordId, VocabIndex *context) const; - LmId GetLmID( const Factor *factor ) const - { - return GetLmID(factor->GetString()); - } void CreateFactors(FactorCollection &factorCollection); public: LmId GetLmID( const std::string &str ) const; diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index f81dc7b22..2e56ca9ab 100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -52,11 +52,14 @@ void TargetPhrase::SetScore(const LMList &languageModels, float weightWP) float fullScore, nGramScore; #ifdef N_BEST - (*lmIter)->CalcScore(*this, fullScore, nGramScore, m_ngramComponent); + (*lmIter)->CalcScore(*this, fullScore, nGramScore); + size_t lmId = (*lmIter)->GetId(); + pair<size_t, float> store(lmId, nGramScore); + m_ngramComponent.push_back(store); #else // this is really, really ugly (a reference to an object at NULL // is asking for trouble). TODO - (*lmIter)->CalcScore(*this, fullScore, nGramScore, *static_cast< list< pair<size_t, float> >* > (NULL)); + (*lmIter)->CalcScore(*this, fullScore, nGramScore); #endif m_fullScore += fullScore * weightLM; @@ -97,11 +100,14 @@ void TargetPhrase::SetScore(const vector<float> &scoreVector, const vector<float const float weightLM = lm.GetWeight(); float fullScore, nGramScore; #ifdef N_BEST - lm.CalcScore(*this, fullScore, nGramScore, m_ngramComponent); + lm.CalcScore(*this, fullScore, nGramScore); + size_t lmId = lm.GetId(); + pair<size_t, float> store(lmId, nGramScore); + m_ngramComponent.push_back(store); #else // this is really, really ugly (a reference to an object at NULL // is asking for trouble). TODO - lm.CalcScore(*this, fullScore, nGramScore, *static_cast< list< pair<size_t, float> >* > (NULL)); + lm.CalcScore(*this, fullScore, nGramScore); #endif // total LM score so far |