diff options
author | Lane Schwartz <dowobeha@gmail.com> | 2016-01-12 23:36:14 +0300 |
---|---|---|
committer | Lane Schwartz <dowobeha@gmail.com> | 2016-01-12 23:36:14 +0300 |
commit | d58db287c3e348b6600aa72c3b48b9b14ce653b9 (patch) | |
tree | 4f5f043e218f4323fbbb0380f4474d9f07d01dff /moses/LM | |
parent | 70b20909b171cfe6601c7edea43da719746db159 (diff) |
ReloadingLM compiles and runs, but gives double the LM score it should
Diffstat (limited to 'moses/LM')
-rw-r--r-- | moses/LM/Jamfile | 2 | ||||
-rw-r--r-- | moses/LM/Reloading.cpp | 22 | ||||
-rw-r--r-- | moses/LM/Reloading.h | 60 |
3 files changed, 48 insertions, 36 deletions
diff --git a/moses/LM/Jamfile b/moses/LM/Jamfile index c01f9cd17..75b66603c 100644 --- a/moses/LM/Jamfile +++ b/moses/LM/Jamfile @@ -138,7 +138,7 @@ if $(with-dalm) { #Top-level LM library. If you've added a file that doesn't depend on external #libraries, put it here. -alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Reloading.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp +alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp ../../lm//kenlm ..//headers $(dependencies) ; alias macros : : : : <define>$(lmmacros) ; diff --git a/moses/LM/Reloading.cpp b/moses/LM/Reloading.cpp index 382492d45..dc4eaadb6 100644 --- a/moses/LM/Reloading.cpp +++ b/moses/LM/Reloading.cpp @@ -35,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA //#include "moses/Util.h" //#include "moses/StaticData.h" //#include <iostream> - +/* namespace Moses { namespace @@ -53,21 +53,14 @@ struct ReloadingLMState : public FFState { }; } // namespace -/** Constructs a new reloading language model. */ + template <class Model> ReloadingLanguageModel<Model>::ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line,file,factorType,lazy) { // // This space intentionally left blank // } - -/** - * Constructs an empty reloading language model state. - * - * This state will correspond with a translation hypothesis - * where no source words have been translated. - */ -template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &/*input*/) const +template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &input) const { ReloadingLMState *ret = new ReloadingLMState(); ret->state = m_ngram->BeginSentenceState(); @@ -77,14 +70,6 @@ template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypoth template <class Model> FFState *ReloadingLanguageModel<Model>::EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const { - /* - const lm::ngram::State &in_state = static_cast<const ReloadingLMState&>(*ps).state; - - std::auto_ptr<KenLMState> kenlmState(new KenLMState()); - kenlmState->state = in_state; - - std::auto_ptr<KenLMState> kenlmReturn(LanguageModelKen::EvaluateWhenApplied(hypo, kenlmState, out)); - */ std::auto_ptr<FFState> kenlmState(LanguageModelKen<Model>::EvaluateWhenApplied(hypo, ps, out)); const lm::ngram::State &out_state = static_cast<const ReloadingLMState&>(*kenlmState).state; @@ -124,3 +109,4 @@ LanguageModel *ConstructReloadingLM(const std::string &line, const std::string & } } // namespace Moses +*/ diff --git a/moses/LM/Reloading.h b/moses/LM/Reloading.h index c55bac44b..50ef37b07 100644 --- a/moses/LM/Reloading.h +++ b/moses/LM/Reloading.h @@ -24,38 +24,64 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <string> +#include "moses/LM/Base.h" #include "moses/LM/Ken.h" -#include "lm/state.hh" - +#include <iostream> namespace Moses { -//! This will also load. Returns a templated reloading LM. -LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy); - class FFState; -/* - * An implementation of single factor reloading LM using Kenneth's code. - */ -template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model> +class ReloadingLanguageModel : public LanguageModel { public: - ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy); - virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const; + ReloadingLanguageModel(const std::string &line) : LanguageModel(line), m_lm(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) { + std::cout << "ReloadingLM constructor" << std::endl; + std::cout << std::string(line).replace(0,11,"KENLM") << std::endl; + } + + ~ReloadingLanguageModel() { + delete m_lm; + } + + virtual const FFState *EmptyHypothesisState(const InputType &input) const { + return m_lm->EmptyHypothesisState(input); + } + + virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const { + m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount); + } + + virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const { + return m_lm->EvaluateWhenApplied(hypo, ps, out); + } + + virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const { + return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator); + } + + virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const { + return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator); + } + + virtual void IncrementalCallback(Incremental::Manager &manager) const { + m_lm->IncrementalCallback(manager); + } + + virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const { + m_lm->ReportHistoryOrder(out, phrase); + } - virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const; + virtual bool IsUseable(const FactorMask &mask) const { + return m_lm->IsUseable(mask); + } private: - // These lines are required to make the parent class's protected members visible to this class - using LanguageModelKen<Model>::m_ngram; - // using LanguageModelKen<Model>::m_beginSentenceFactor; - //using LanguageModelKen<Model>::m_factorType; - //using LanguageModelKen<Model>::TranslateID; + LanguageModel *m_lm; }; |