diff options
author | redpony <redpony@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-07-17 17:52:01 +0400 |
---|---|---|
committer | redpony <redpony@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-07-17 17:52:01 +0400 |
commit | 79a6b10602030e7574b0c52c63f45fff029c3b47 (patch) | |
tree | 4c755601a7afb39aab37257abc5dc49a28880395 /moses | |
parent | 87e732bc079be53359477e6398cc88f3ccd998f0 (diff) |
more LM clean-up
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@144 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'moses')
-rwxr-xr-x | moses/src/LanguageModel.cpp | 1 | ||||
-rwxr-xr-x | moses/src/LanguageModel.h | 5 | ||||
-rwxr-xr-x | moses/src/LanguageModel_Internal.cpp | 5 | ||||
-rwxr-xr-x | moses/src/LanguageModel_Internal.h | 7 | ||||
-rwxr-xr-x | moses/src/LanguageModel_SRI.cpp | 37 | ||||
-rwxr-xr-x | moses/src/LanguageModel_SRI.h | 28 |
6 files changed, 53 insertions, 30 deletions
diff --git a/moses/src/LanguageModel.cpp b/moses/src/LanguageModel.cpp index 987962e90..3513810ae 100755 --- a/moses/src/LanguageModel.cpp +++ b/moses/src/LanguageModel.cpp @@ -38,6 +38,7 @@ using namespace std; const LmId LanguageModel::UNKNOWN_LM_ID(0); LanguageModel::LanguageModel() {} +LanguageModel::~LanguageModel() {} void LanguageModel::CalcScore(const Phrase &phrase , float &fullScore diff --git a/moses/src/LanguageModel.h b/moses/src/LanguageModel.h index e78cd61c5..5170363e1 100755 --- a/moses/src/LanguageModel.h +++ b/moses/src/LanguageModel.h @@ -43,6 +43,7 @@ public: static const LmId UNKNOWN_LM_ID; LanguageModel(); + virtual ~LanguageModel(); virtual void Load(size_t id , const std::string &fileName , FactorCollection &factorCollection @@ -83,5 +84,9 @@ public: return m_id; } virtual float GetValue(const std::vector<const Factor*> &contextFactor) const = 0; + + // one of the following should probably be made available + // virtual LmId GetLmID( const Factor *factor ) const = 0; + // virtual LmId GetLmID( const std::string &factor ) const = 0; }; diff --git a/moses/src/LanguageModel_Internal.cpp b/moses/src/LanguageModel_Internal.cpp index 4dcbf89d4..3cca42784 100755 --- a/moses/src/LanguageModel_Internal.cpp +++ b/moses/src/LanguageModel_Internal.cpp @@ -235,3 +235,8 @@ float LanguageModel_Internal::GetValue(const Factor *factor0, const Factor *fact } +LmId LanguageModel_Internal::GetLmID( const Factor *factor ) const +{ + return factor->GetLmId(); +} + diff --git a/moses/src/LanguageModel_Internal.h b/moses/src/LanguageModel_Internal.h index b13037228..e445845b2 100755 --- a/moses/src/LanguageModel_Internal.h +++ b/moses/src/LanguageModel_Internal.h @@ -57,10 +57,7 @@ protected: float GetValue(const Factor *factor0, const Factor *factor1, const Factor *factor2) const; public: - LmId GetLmID( const Factor *factor ) const - { - return factor->GetLmId(); - } - + LmId GetLmID( const Factor *factor ) const; + }; diff --git a/moses/src/LanguageModel_SRI.cpp b/moses/src/LanguageModel_SRI.cpp index d1fea8832..341e94957 100755 --- a/moses/src/LanguageModel_SRI.cpp +++ b/moses/src/LanguageModel_SRI.cpp @@ -24,7 +24,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <iostream> #include <fstream> -#include "NGramNode.h" +#include "Ngram.h" +#include "Vocab.h" #include "LanguageModel_SRI.h" #include "TypeDef.h" @@ -35,10 +36,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using namespace std; LanguageModel_SRI::LanguageModel_SRI() -:m_srilmVocab() -,m_srilmModel(m_srilmVocab, 3) +:m_srilmVocab(0) +,m_srilmModel(0) { - m_srilmModel.skipOOVs() = false; +} + +LanguageModel_SRI::~LanguageModel_SRI() +{ + delete m_srilmModel; + delete m_srilmVocab; } void LanguageModel_SRI::Load(size_t id @@ -48,13 +54,17 @@ void LanguageModel_SRI::Load(size_t id , float weight , size_t nGramOrder) { + m_srilmVocab = new Vocab(); + m_srilmModel = new Ngram(*m_srilmVocab, nGramOrder); m_id = id; m_factorType = factorType; m_weight = weight; m_nGramOrder = nGramOrder; + m_srilmModel->skipOOVs() = false; + File file( fileName.c_str(), "r" ); - if (m_srilmModel.read(file)) + if (m_srilmModel->read(file)) { } else @@ -71,7 +81,7 @@ void LanguageModel_SRI::CreateFactors(FactorCollection &factorCollection) VocabString str; LmId lmId; - VocabIter iter(m_srilmVocab); + VocabIter iter(*m_srilmVocab); while ( (str = iter.next()) != NULL) { LmId lmId = GetLmID(str); @@ -85,6 +95,21 @@ void LanguageModel_SRI::CreateFactors(FactorCollection &factorCollection) } +LmId LanguageModel_SRI::GetLmID( const std::string &str ) const +{ + LanguageModel_SRI *lm = const_cast<LanguageModel_SRI*>(this); // hack. not sure if supposed to cast this + LmId res; + res.sri = lm->m_srilmVocab->getIndex( str.c_str(), lm->m_srilmVocab->unkIndex() ); + return res; +} + +float LanguageModel_SRI::GetValue(LmId wordId, VocabIndex *context) const +{ + LanguageModel_SRI *lm = const_cast<LanguageModel_SRI*>(this); // hack. not sure if supposed to cast this + float p = lm->m_srilmModel->wordProb( wordId.sri, context ); + return FloorSRIScore(TransformSRIScore(p)); // log10->log +} + float LanguageModel_SRI::GetValue(const vector<const Factor*> &contextFactor) const { // set up context diff --git a/moses/src/LanguageModel_SRI.h b/moses/src/LanguageModel_SRI.h index bcd11537d..32e9b1b30 100755 --- a/moses/src/LanguageModel_SRI.h +++ b/moses/src/LanguageModel_SRI.h @@ -29,19 +29,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "Util.h" #include "LanguageModel.h" -#include "Ngram.h" -#include "Vocab.h" class FactorCollection; class Factor; class Phrase; +#include "Vocab.h" +class Vocab; // SRI forward decl +class Ngram; // SRI forward decl + class LanguageModel_SRI : public LanguageModel { public: LanguageModel_SRI(); + ~LanguageModel_SRI(); void Load(size_t id , const std::string &fileName , FactorCollection &factorCollection @@ -50,15 +53,10 @@ public: , size_t nGramOrder); protected: - Vocab m_srilmVocab; // TODO - make this a ptr, remove #include from header - Ngram m_srilmModel; // " " + Vocab *m_srilmVocab; // TODO - make this a ptr, remove #include from header + Ngram *m_srilmModel; // " " - float GetValue(LmId wordId, VocabIndex *context) const - { - LanguageModel_SRI *lm = const_cast<LanguageModel_SRI*>(this); // hack. not sure if supposed to cast this - float p = lm->m_srilmModel.wordProb( wordId.sri, context ); - return FloorSRIScore(TransformSRIScore(p)); // log10->log - } + float GetValue(LmId wordId, VocabIndex *context) const; LmId GetLmID( const Factor *factor ) const { @@ -66,15 +64,7 @@ protected: } void CreateFactors(FactorCollection &factorCollection); public: - static const LmId UNKNOWN_LM_ID; - - LmId GetLmID( const std::string &str ) const - { - LanguageModel_SRI *lm = const_cast<LanguageModel_SRI*>(this); // hack. not sure if supposed to cast this - LmId res; - res.sri = lm->m_srilmVocab.getIndex( str.c_str(), lm->m_srilmVocab.unkIndex() ); - return res; - } + LmId GetLmID( const std::string &str ) const; virtual float GetValue(const vector<const Factor*> &contextFactor) const; }; |