diff options
author | Hieu Hoang <hieu@hoang.co.uk> | 2013-05-27 19:50:21 +0400 |
---|---|---|
committer | Hieu Hoang <hieu@hoang.co.uk> | 2013-05-27 19:50:21 +0400 |
commit | 06a5c8669535bbc2324a9288ca770b1d1e9d6071 (patch) | |
tree | bc3dcfcf2cd4b9a5539947ca5aeb4089d58d9517 | |
parent | 9006ee98be01850cdb37967a392b176087f4c8c0 (diff) |
delete LMList
-rw-r--r-- | mira/Main.cpp | 84 | ||||
-rw-r--r-- | mira/Main.h | 4 | ||||
-rw-r--r-- | misc/queryPhraseTableMin.cpp | 4 | ||||
-rw-r--r-- | moses-cmd/TranslationAnalysis.cpp | 23 | ||||
-rw-r--r-- | moses/Manager.cpp | 1 | ||||
-rw-r--r-- | moses/StaticData.cpp | 7 | ||||
-rw-r--r-- | moses/StaticData.h | 4 |
7 files changed, 71 insertions, 56 deletions
diff --git a/mira/Main.cpp b/mira/Main.cpp index ecaa6a2c3..2c62256d9 100644 --- a/mira/Main.cpp +++ b/mira/Main.cpp @@ -527,12 +527,17 @@ int main(int argc, char** argv) { ScoreComponentCollection initialWeights = decoder->getWeights(); if (add2lm != 0) { - const LMList& lmList_new = staticData.GetLMList(); - for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) { - float lmWeight = initialWeights.GetScoreForProducer(*i) + add2lm; - initialWeights.Assign(*i, lmWeight); - cerr << "Rank " << rank << ", add " << add2lm << " to lm weight." << endl; - } + const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + + if (lm) { + float lmWeight = initialWeights.GetScoreForProducer(lm) + add2lm; + initialWeights.Assign(lm, lmWeight); + cerr << "Rank " << rank << ", add " << add2lm << " to lm weight." << endl; + } + } } if (normaliseWeights) { @@ -543,11 +548,18 @@ int main(int argc, char** argv) { decoder->setWeights(initialWeights); // set bleu weight to twice the size of the language model weight(s) - const LMList& lmList = staticData.GetLMList(); if (bleu_weight_lm) { float lmSum = 0; - for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) - lmSum += abs(initialWeights.GetScoreForProducer(*i)); + const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + + if (lm) { + lmSum += abs(initialWeights.GetScoreForProducer(lm)); + } + } + bleuWeight = lmSum * bleu_weight_lm_factor; cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl; } @@ -766,15 +778,20 @@ int main(int argc, char** argv) { } // check LM weight - const LMList& lmList_new = staticData.GetLMList(); - for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) { - float lmWeight = mosesWeights.GetScoreForProducer(*i); - cerr << "Rank " << rank << ", epoch " << epoch << ", lm weight: " << lmWeight << endl; - if (lmWeight <= 0) { - cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: language model weight should never be <= 0." << endl; - mosesWeights.Assign(*i, 0.1); - cerr << "Rank " << rank << ", epoch " << epoch << ", assign lm weights of 0.1" << endl; - } + const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + + if (lm) { + float lmWeight = mosesWeights.GetScoreForProducer(lm); + cerr << "Rank " << rank << ", epoch " << epoch << ", lm weight: " << lmWeight << endl; + if (lmWeight <= 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: language model weight should never be <= 0." << endl; + mosesWeights.Assign(lm, 0.1); + cerr << "Rank " << rank << ", epoch " << epoch << ", assign lm weights of 0.1" << endl; + } + } } // select inference scheme @@ -1168,17 +1185,22 @@ int main(int argc, char** argv) { // scale LM feature (to avoid rapid changes) if (scale_lm) { cerr << "scale lm" << endl; - const LMList& lmList_new = staticData.GetLMList(); - for (LMList::const_iterator iter = lmList_new.begin(); iter != lmList_new.end(); ++iter) { - // scale down score - if (model_hope_fear) { - scaleFeatureScore(*iter, scale_lm_factor, featureValues, rank, epoch); - } - else { - scaleFeatureScore(*iter, scale_lm_factor, featureValuesHope, rank, epoch); - scaleFeatureScore(*iter, scale_lm_factor, featureValuesFear, rank, epoch); - } - } + const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + + if (lm) { + // scale down score + if (model_hope_fear) { + scaleFeatureScore(lm, scale_lm_factor, featureValues, rank, epoch); + } + else { + scaleFeatureScore(lm, scale_lm_factor, featureValuesHope, rank, epoch); + scaleFeatureScore(lm, scale_lm_factor, featureValuesFear, rank, epoch); + } + } + } } // scale WP @@ -1849,7 +1871,7 @@ void applyPerFeatureLearningRates(vector<vector<ScoreComponentCollection> > &fea featureValues[i][j].MultiplyEqualsBackoff(featureLearningRates, sparse_r0); } -void scaleFeatureScore(FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { +void scaleFeatureScore(const FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { string name = sp->GetScoreProducerDescription(); // scale down score @@ -1863,7 +1885,7 @@ void scaleFeatureScore(FeatureFunction *sp, float scaling_factor, vector<vector< } } -void scaleFeatureScores(FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { +void scaleFeatureScores(const FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { string name = sp->GetScoreProducerDescription(); // scale down score diff --git a/mira/Main.h b/mira/Main.h index 93b7c3dd9..6ba375c2c 100644 --- a/mira/Main.h +++ b/mira/Main.h @@ -52,7 +52,7 @@ void deleteTranslations(std::vector<std::vector<const Moses::Word*> > &translati void decodeHopeOrFear(size_t rank, size_t size, size_t decode, std::string decode_filename, std::vector<std::string> &inputSentences, Mira::MosesDecoder* decoder, size_t n, float bleuWeight); void applyLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, float core_r0, float sparse_r0); void applyPerFeatureLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, Moses::ScoreComponentCollection featureLearningRates, float sparse_r0); -void scaleFeatureScore(Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); -void scaleFeatureScores(Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); +void scaleFeatureScore(const Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); +void scaleFeatureScores(const Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); #endif /* MAIN_H_ */ diff --git a/misc/queryPhraseTableMin.cpp b/misc/queryPhraseTableMin.cpp index 01888dbb4..f4dca8b6b 100644 --- a/misc/queryPhraseTableMin.cpp +++ b/misc/queryPhraseTableMin.cpp @@ -47,9 +47,7 @@ int main(int argc, char **argv) std::vector<FactorType> input(1, 0); std::vector<FactorType> output(1, 0); std::vector<float> weight(nscores, 0); - - LMList lmList; - + Parameter *parameter = new Parameter(); const_cast<std::vector<std::string>&>(parameter->GetParam("factor-delimiter")).resize(1, "||dummy_string||"); const_cast<std::vector<std::string>&>(parameter->GetParam("input-factors")).resize(1, "0"); diff --git a/moses-cmd/TranslationAnalysis.cpp b/moses-cmd/TranslationAnalysis.cpp index 186378ca5..4231001e9 100644 --- a/moses-cmd/TranslationAnalysis.cpp +++ b/moses-cmd/TranslationAnalysis.cpp @@ -101,14 +101,21 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo) } os << std::endl << std::endl; if (doLMStats && lmCalls > 0) { - std::vector<unsigned int>::iterator acc = lmAcc.begin(); - const LMList &lmList = StaticData::Instance().GetLMList(); - LMList::const_iterator i = lmList.begin(); - for (; acc != lmAcc.end(); ++acc, ++i) { - char buf[256]; - sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls); - os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl; - } + std::vector<unsigned int>::iterator acc = lmAcc.begin(); + + const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); + + if (lm) { + char buf[256]; + sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls); + os << lm->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl; + + ++acc; + } + } } if (droppedWords.size() > 0) { diff --git a/moses/Manager.cpp b/moses/Manager.cpp index 8a48d71ad..b8e958d04 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -38,7 +38,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "TrellisPathCollection.h" #include "TranslationOption.h" #include "LexicalReordering.h" -#include "LMList.h" #include "TranslationOptionCollection.h" #include "Timer.h" #include "moses/FF/DistortionScoreProducer.h" diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 727684bfb..c88da63cf 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -1173,13 +1173,6 @@ void StaticData::CollectFeatureFunctions() for (iter = ffs.begin(); iter != ffs.end(); ++iter) { const FeatureFunction *ff = *iter; - const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff); - if (lm) { - LanguageModel *lmNonConst = const_cast<LanguageModel*>(lm); - m_languageModel.Add(lmNonConst); - continue; - } - const GenerationDictionary *generation = dynamic_cast<const GenerationDictionary*>(ff); if (generation) { m_generationDictionary.push_back(generation); diff --git a/moses/StaticData.h b/moses/StaticData.h index c037a286f..01a0a19df 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -41,7 +41,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "FactorCollection.h" #include "Parameter.h" #include "LM/Base.h" -#include "LMList.h" #include "SentenceStats.h" #include "DecodeGraph.h" #include "TranslationOptionList.h" @@ -76,7 +75,6 @@ protected: std::vector<const GenerationDictionary*> m_generationDictionary; Parameter *m_parameter; std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder; - LMList m_languageModel; ScoreComponentCollection m_allWeights; std::vector<DecodeGraph*> m_decodeGraphs; @@ -423,8 +421,6 @@ public: bool IsChart() const { return m_searchAlgorithm == ChartDecoding || m_searchAlgorithm == ChartIncremental; } - const LMList &GetLMList() const - { return m_languageModel; } const WordPenaltyProducer *GetWordPenaltyProducer() const { return m_wpProducer; } WordPenaltyProducer *GetWordPenaltyProducer() // for mira |