diff options
author | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-01 04:10:28 +0400 |
---|---|---|
committer | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-01 04:10:28 +0400 |
commit | b2339c85fe0782cb39ad537768d48974d6811b85 (patch) | |
tree | a456578c240a8bdcc073e12c9a0f98c44fb89a75 | |
parent | 7910b65cdf39554af8f02ca8511d559945561cdd (diff) |
remove LM types
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@432 1f5c12ca-751b-0410-a591-d2e778427230
-rwxr-xr-x | moses/src/Hypothesis.cpp | 57 | ||||
-rwxr-xr-x | moses/src/Hypothesis.h | 2 | ||||
-rwxr-xr-x | moses/src/StaticData.cpp | 47 | ||||
-rwxr-xr-x | moses/src/StaticData.h | 20 | ||||
-rw-r--r-- | moses/src/TargetPhrase.cpp | 1 | ||||
-rwxr-xr-x | moses/src/TypeDef.h | 6 |
6 files changed, 20 insertions, 113 deletions
diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 0a0edf34e..e274b4d73 100755 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp @@ -277,65 +277,14 @@ int Hypothesis::NGramCompare(const Hypothesis &compare) const * /param lmListInitial todo - describe this parameter * /param lmListEnd todo - describe this parameter */ -void Hypothesis::CalcLMScore(const LMList &lmListInitial, const LMList &lmListEnd) +void Hypothesis::CalcLMScore(const LMList &languageModels) { const size_t startPos = m_currTargetWordsRange.GetStartPos(); LMList::const_iterator iterLM; - // for LM which are not in PossTran - // must go through each trigram in current phrase - for (iterLM = lmListEnd.begin() ; iterLM != lmListEnd.end() ; ++iterLM) - { - const LanguageModel &languageModel = **iterLM; - FactorType factorType = languageModel.GetFactorType(); - size_t nGramOrder = languageModel.GetNGramOrder(); - float lmScore; - - // 1st n-gram - vector<const Factor*> contextFactor(nGramOrder); - size_t index = 0; - for (int currPos = (int) startPos - (int) nGramOrder + 1 ; currPos <= (int) startPos ; currPos++) - { - if (currPos >= 0) - contextFactor[index++] = GetFactor(currPos, factorType); - else - contextFactor[index++] = languageModel.GetSentenceStart(); - } - lmScore = languageModel.GetValue(contextFactor); - //cout<<"context factor: "<<languageModel.GetValue(contextFactor)<<endl; - - // main loop - for (size_t currPos = startPos + 1 ; currPos <= m_currTargetWordsRange.GetEndPos() ; currPos++) - { - // shift all args down 1 place - for (size_t i = 0 ; i < nGramOrder - 1 ; i++) - contextFactor[i] = contextFactor[i + 1]; - - // add last factor - contextFactor.back() = GetFactor(currPos, factorType); - - lmScore += languageModel.GetValue(contextFactor); - //cout<<"context factor: "<<languageModel.GetValue(contextFactor)<<endl; - - } - - // end of sentence - if (m_sourceCompleted.IsComplete()) - { - // shift all args down 1 place - for (size_t i = 0 ; i < nGramOrder - 1 ; i++) - contextFactor[i] = contextFactor[i + 1]; - - contextFactor.back() = languageModel.GetSentenceEnd(); - lmScore += languageModel.GetValue(contextFactor); - } - m_scoreBreakdown.PlusEquals(&languageModel, lmScore); - } - - // for LM which are in possTran // already have LM scores from previous and trigram score of poss trans. // just need trigram score of the words of the start of current phrase - for (iterLM = lmListInitial.begin() ; iterLM != lmListInitial.end() ; ++iterLM) + for (iterLM = languageModels.begin() ; iterLM != languageModels.end() ; ++iterLM) { const LanguageModel &languageModel = **iterLM; FactorType factorType = languageModel.GetFactorType(); @@ -422,7 +371,7 @@ void Hypothesis::CalcScore(const StaticData& staticData, const SquareMatrix &fut CalcDistortionScore(); // LANGUAGE MODEL COST - CalcLMScore(staticData.GetLanguageModel(Initial), staticData.GetLanguageModel(Other)); + CalcLMScore(staticData.GetAllLM()); // WORD PENALTY m_scoreBreakdown.PlusEquals(staticData.GetWordPenaltyProducer(), - (float) m_currTargetWordsRange.GetWordsCount()); diff --git a/moses/src/Hypothesis.h b/moses/src/Hypothesis.h index 59c5ee3fb..a27401abe 100755 --- a/moses/src/Hypothesis.h +++ b/moses/src/Hypothesis.h @@ -76,7 +76,7 @@ protected: void CalcFutureScore(const SquareMatrix &futureScore); //void CalcFutureScore(float futureScore[256][256]); - void CalcLMScore(const LMList &lmListInitial, const LMList &lmListEnd); + void CalcLMScore(const LMList &languageModels); void CalcDistortionScore(); //TODO: add appropriate arguments to score calculator diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp index 849e09811..b42d4f34a 100755 --- a/moses/src/StaticData.cpp +++ b/moses/src/StaticData.cpp @@ -48,8 +48,7 @@ extern Timer timer; StaticData* StaticData::s_instance(0); StaticData::StaticData() -:m_languageModel(2) -,m_lexReorder(NULL) +:m_lexReorder(NULL) ,m_inputOutput(NULL) ,m_fLMsLoaded(false) ,m_inputType(0) @@ -239,8 +238,7 @@ bool StaticData::LoadParameters(int argc, char* argv[]) return false; } // type = whether or not to use in future cost calcs - // (DEPRECATED, asked hieu) - LMListType type = static_cast<LMListType>(Scan<int>(token[0])); + // factorType = (see TypeDef.h) // 0 = Surface, 1 = POS, 2 = Stem, 3 = Morphology, etc FactorType factorType = Scan<FactorType>(token[1]); @@ -263,7 +261,7 @@ bool StaticData::LoadParameters(int argc, char* argv[]) // error handling here? lm->Load(i, languageModelFile, m_factorCollection, factorType, weightAll[i], nGramOrder); timer.check(("Finished loading LanguageModel " + languageModelFile).c_str()); - m_languageModel[type].push_back(lm); + m_languageModel.push_back(lm); HypothesisRecombinationOrderer::SetMaxNGramOrder(factorType, nGramMaxOrder); } @@ -372,18 +370,11 @@ StaticData::~StaticData() delete m_generationDictionary[i]; } - LMList &lmList = m_languageModel[0]; LMList::const_iterator iterLM; - for (iterLM = lmList.begin() ; iterLM != lmList.end() ; ++iterLM) + for (iterLM = m_languageModel.begin() ; iterLM != m_languageModel.end() ; ++iterLM) { delete *iterLM; } - lmList = m_languageModel[1]; - for (iterLM = lmList.begin() ; iterLM != lmList.end() ; ++iterLM) - { - delete *iterLM; - } - // small score producers delete m_distortionScoreProducer; delete m_wpProducer; @@ -419,20 +410,15 @@ void StaticData::SetWeightTransModel(const vector<float> &weight) void StaticData::SetWeightLM(const std::vector<float> &weight) { - assert(weight.size() == m_languageModel[Initial].size() + m_languageModel[Other].size()); + assert(weight.size() == m_languageModel.size()); size_t currIndex = 0; LMList::iterator iter; - for (iter = m_languageModel[Initial].begin() ; iter != m_languageModel[Initial].end() ; ++iter) + for (iter = m_languageModel.begin() ; iter != m_languageModel.end() ; ++iter) { LanguageModel *languageModel = *iter; languageModel->SetWeight(weight[currIndex++]); } - for (iter = m_languageModel[Other].begin() ; iter != m_languageModel[Other].end() ; ++iter) - { - LanguageModel *languageModel = *iter; - languageModel->SetWeight(weight[currIndex++]); - } } void StaticData::SetWeightGeneration(const std::vector<float> &weight) @@ -448,17 +434,6 @@ void StaticData::SetWeightGeneration(const std::vector<float> &weight) } } -const LMList StaticData::GetAllLM() const -{ - LMList allLM; - std::copy(m_languageModel[Initial].begin(), m_languageModel[Initial].end() - , std::inserter(allLM, allLM.end())); - std::copy(m_languageModel[Other].begin(), m_languageModel[Other].end() - , std::inserter(allLM, allLM.end())); - - return allLM; -} - void StaticData::LoadPhraseTables(bool filter , const string &inputFileHash , const list< Phrase > &inputPhraseList) @@ -560,8 +535,8 @@ void StaticData::LoadPhraseTables(bool filter , maxTargetPhrase[index] , filterPhrase , inputPhraseList - , this->GetLanguageModel(Initial) - , this->GetWeightWordPenalty() + , GetAllLM() + , GetWeightWordPenalty() , *this); m_phraseDictionary.push_back(pd); } @@ -575,8 +550,8 @@ void StaticData::LoadPhraseTables(bool filter PhraseDictionaryTreeAdaptor *pd=new PhraseDictionaryTreeAdaptor(noScoreComponent,(currDict==0 ? m_numInputScores : 0)); pd->Create(input,output,m_factorCollection,filePath,weight, maxTargetPhrase[index], - this->GetLanguageModel(Initial), - this->GetWeightWordPenalty()); + GetAllLM(), + GetWeightWordPenalty()); m_phraseDictionary.push_back(pd); #endif } @@ -613,6 +588,7 @@ void StaticData::CleanUpAfterSentenceProcessing() for(size_t i=0;i<m_generationDictionary.size();++i) m_generationDictionary[i]->CleanUp(); } + void StaticData::InitializeBeforeSentenceProcessing(InputType const& in) { for(size_t i=0;i<m_phraseDictionary.size();++i) @@ -631,4 +607,3 @@ void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std:: for (size_t i = begin; i < end; i++) m_allWeights[i] = *weightIter++; } - diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h index ac3ada6f7..d1a492a4c 100755 --- a/moses/src/StaticData.h +++ b/moses/src/StaticData.h @@ -54,7 +54,7 @@ protected: Parameter m_parameter; std::vector<FactorType> m_inputFactorOrder; // boost::shared_ptr<UnknownWordHandler> m_unknownWordHandler; //defaults to NULL; pointer allows polymorphism - std::vector<LMList> m_languageModel; + LMList m_languageModel; std::vector<float> m_lexWeights; ScoreIndexManager m_scoreIndexManager; std::vector<float> m_allWeights; @@ -158,12 +158,6 @@ public: { return m_maxNoTransOptPerCoverage; } -/* - boost::shared_ptr<UnknownWordHandler> GetUnknownWordHandler() - { - return m_unknownWordHandler; - } -*/ FactorCollection &GetFactorCollection() { return m_factorCollection; @@ -173,11 +167,6 @@ public: if (factorType >= m_maxNgramOrderForFactor.size()) return 0; return m_maxNgramOrderForFactor[factorType]; } - const LMList &GetLanguageModel(LMListType type) const - { - return m_languageModel[type]; - } - LexicalReordering *GetLexReorder() const { return m_lexReorder; @@ -223,13 +212,12 @@ public: size_t GetLMSize() const { - return m_languageModel[Initial].size() + m_languageModel[Other].size(); + return m_languageModel.size(); } - size_t GetNumLanguageModels() const + const LMList &GetAllLM() const { - return GetLMSize(); + return m_languageModel; } - const LMList GetAllLM() const; size_t GetPhraseDictionarySize() const { return m_phraseDictionary.size(); diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index 4f5eb8c09..68ce682e4 100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -64,6 +64,7 @@ void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, if (GetSize() > 0 && GetFactor(0, lmFactorType) != NULL) { // contains factors used by this LM + TRACE_ERR(*GetFactor(0, lmFactorType) << endl); const float weightLM = lm.GetWeight(); float fullScore, nGramScore; diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index 3a1c930fd..51a0265c9 100755 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h @@ -139,12 +139,6 @@ enum IOMethod ,IOMethodMySQL }; -enum LMListType -{ - Initial - ,Other -}; - // typedef class Factor; typedef const Factor * FactorArray[NUM_FACTORS]; |