diff options
Diffstat (limited to 'moses')
-rw-r--r-- | moses/src/Jamfile | 8 | ||||
-rw-r--r-- | moses/src/LM/Factory.cpp | 4 | ||||
-rw-r--r-- | moses/src/LM/Jamfile | 22 | ||||
-rw-r--r-- | moses/src/LM/Rand.cpp (renamed from moses/src/LM/RandLM.cpp) | 47 | ||||
-rw-r--r-- | moses/src/LM/Rand.h | 30 | ||||
-rw-r--r-- | moses/src/LM/RandLM.h | 71 | ||||
-rw-r--r-- | moses/src/Makefile.am | 4 |
7 files changed, 100 insertions, 86 deletions
diff --git a/moses/src/Jamfile b/moses/src/Jamfile index f70cbb849..a5b0e2210 100644 --- a/moses/src/Jamfile +++ b/moses/src/Jamfile @@ -1,10 +1,12 @@ +alias headers : ../../util//util : : : <include>. ; + lib z ; -alias InputFileStream : InputFileStream.cpp z ; +alias InputFileStream : InputFileStream.cpp z headers ; if [ option.get "with-synlm" : no : yes ] = yes { lib m ; - obj SyntacticLanguageModel.o : SyntacticLanguageModel.cpp ../../util//util : <include>$(TOP)/synlm/hhmm/rvtl/include <include>$(TOP)/synlm/hhmm/wsjparse/include ; + obj SyntacticLanguageModel.o : SyntacticLanguageModel.cpp headers : <include>$(TOP)/synlm/hhmm/rvtl/include <include>$(TOP)/synlm/hhmm/wsjparse/include ; alias synlm : SyntacticLanguageModel.o m : : : <define>HAVE_SYNLM ; } else { alias synlm ; @@ -121,4 +123,4 @@ WordsRange.cpp XmlOption.cpp synlm -InputFileStream LM//LM ../../util//util : <include>. : : <include>. ; +InputFileStream LM//LM headers ; diff --git a/moses/src/LM/Factory.cpp b/moses/src/LM/Factory.cpp index df1e6bc01..2815c2694 100644 --- a/moses/src/LM/Factory.cpp +++ b/moses/src/LM/Factory.cpp @@ -64,7 +64,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # include "LM/IRST.h" #endif #ifdef LM_RAND -# include "LM/RandLM.h" +# include "LM/Rand.h" #endif #ifdef LM_ORLM # include "LM/ORLM.h" @@ -102,7 +102,7 @@ LanguageModel* CreateLanguageModel(LMImplementation lmImplementation switch (lmImplementation) { case RandLM: #ifdef LM_RAND - lm = new LanguageModelRandLM(); + lm = NewRandLM(); #endif break; case ORLM: diff --git a/moses/src/LM/Jamfile b/moses/src/LM/Jamfile index bb8944e12..dd25775a2 100644 --- a/moses/src/LM/Jamfile +++ b/moses/src/LM/Jamfile @@ -15,7 +15,7 @@ with-irstlm = [ option.get "with-irstlm" ] ; if $(with-irstlm) != "" { lib irstlm : : <search>$(with-irstlm)/lib ; - obj IRST.o : IRST.cpp ../../../util//util : <include>$(with-irstlm)/include <include>.. ; + obj IRST.o : IRST.cpp ..//headers : <include>$(with-irstlm)/include ; alias irst : IRST.o irstlm : : : <define>LM_IRST ; echo "" ; echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ; @@ -29,7 +29,7 @@ if $(with-irstlm) != "" with-srilm = [ option.get "with-srilm" ] ; if $(with-srilm) != "" { - if [ option.get "with-srilm-dynamic" ] != "" + if [ option.get "with-srilm-dynamic" : no : yes ] = yes { alias sri-libs : srilm ; } else { @@ -45,14 +45,22 @@ if $(with-srilm) != "" alias sri-libs : oolm dstruct misc flm ; } - obj SRI.o : SRI.cpp ../../../util//util : <include>$(with-srilm)/include <include>.. ; - obj ParallelBackoff.o : ParallelBackoff.cpp ../../../util//util : <include>$(with-srilm)/include <include>.. ; + obj SRI.o : SRI.cpp ..//headers : <include>$(with-srilm)/include ; + obj ParallelBackoff.o : ParallelBackoff.cpp ..//headers : <include>$(with-srilm)/include ; alias sri : SRI.o ParallelBackoff.o sri-libs : : : <define>LM_SRI ; } else { alias sri ; } +with-randlm = [ option.get "with-randlm" ] ; +if $(with-randlm) != "" +{ + lib randlm : : <search>$(with-randlm)/lib ; + obj Rand.o : Rand.cpp randlm ..//headers : <include>$(with-randlm)/include ; + alias rand : Rand.o : : : <define>LM_RAND ; +} else { + alias rand ; +} + lib LM : Base.cpp Factory.cpp Implementation.cpp Joint.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp - ../../../lm//lm - irst sri -: <include>.. $(requirements) : : <include>.. ; + ../../../lm//lm irst sri rand ..//headers ; diff --git a/moses/src/LM/RandLM.cpp b/moses/src/LM/Rand.cpp index 232f2bf94..93411b8d3 100644 --- a/moses/src/LM/RandLM.cpp +++ b/moses/src/LM/Rand.cpp @@ -21,8 +21,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <limits> #include <iostream> #include <fstream> +#include <string> +#include <vector> +#include "Factor.h" +#include "Util.h" +#include "LM/SingleFactor.h" +#include "RandLM.h" -#include "LM/RandLM.h" + +#include "LM/Rand.h" #include "FactorCollection.h" #include "Phrase.h" #include "InputFileStream.h" @@ -31,8 +38,41 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { +namespace +{ using namespace std; +class LanguageModelRandLM : public LanguageModelPointerState +{ +public: + LanguageModelRandLM() + : m_lm(0) {} + bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder); + virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const; + ~LanguageModelRandLM() { + delete m_lm; + } + void CleanUpAfterSentenceProcessing() { + m_lm->clearCaches(); // clear caches + } + void InitializeBeforeSentenceProcessing() { + m_lm->initThreadSpecificData(); // Creates thread specific data iff + // compiled with multithreading. + } +protected: + std::vector<randlm::WordID> m_randlm_ids_vec; + randlm::RandLM* m_lm; + randlm::WordID m_oov_id; + void CreateFactors(FactorCollection &factorCollection); + randlm::WordID GetLmID( const std::string &str ) const; + randlm::WordID GetLmID( const Factor *factor ) const { + size_t factorId = factor->GetId(); + return ( factorId >= m_randlm_ids_vec.size()) ? m_oov_id : m_randlm_ids_vec[factorId]; + }; + +}; + + bool LanguageModelRandLM::Load(const std::string &filePath, FactorType factorType, size_t nGramOrder) { @@ -116,4 +156,9 @@ LMResult LanguageModelRandLM::GetValue(const vector<const Word*> &contextFactor, } +LanguageModelPointerState *NewRandLM() { + return new LanguageModelRandLM(); +} + +} diff --git a/moses/src/LM/Rand.h b/moses/src/LM/Rand.h new file mode 100644 index 000000000..c4ef99947 --- /dev/null +++ b/moses/src/LM/Rand.h @@ -0,0 +1,30 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#ifndef moses_LM_Rand_h +#define moses_LM_Rand_h + +namespace Moses +{ +class LanguageModelPointerState; +LanguageModelPointerState *NewRandLM(); + +} + +#endif diff --git a/moses/src/LM/RandLM.h b/moses/src/LM/RandLM.h deleted file mode 100644 index 8de5f3bb1..000000000 --- a/moses/src/LM/RandLM.h +++ /dev/null @@ -1,71 +0,0 @@ -/*********************************************************************** -Moses - factored phrase-based language decoder -Copyright (C) 2006 University of Edinburgh - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -***********************************************************************/ - -#ifndef moses_LanguageModelRandLM_h -#define moses_LanguageModelRandLM_h - -#include <string> -#include <vector> -#include "Factor.h" -#include "Util.h" -#include "LM/SingleFactor.h" -#include "RandLM.h" - -class randlm::RandLM; - -namespace Moses -{ -class Factor; -class Phrase; - -// RandLM wrapper (single factor LM) - -class LanguageModelRandLM : public LanguageModelPointerState -{ -public: - LanguageModelRandLM() - : m_lm(0) {} - bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder); - virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const; - ~LanguageModelRandLM() { - delete m_lm; - } - void CleanUpAfterSentenceProcessing() { - m_lm->clearCaches(); // clear caches - } - void InitializeBeforeSentenceProcessing() { - m_lm->initThreadSpecificData(); // Creates thread specific data iff - // compiled with multithreading. - } -protected: - std::vector<randlm::WordID> m_randlm_ids_vec; - randlm::RandLM* m_lm; - randlm::WordID m_oov_id; - void CreateFactors(FactorCollection &factorCollection); - randlm::WordID GetLmID( const std::string &str ) const; - randlm::WordID GetLmID( const Factor *factor ) const { - size_t factorId = factor->GetId(); - return ( factorId >= m_randlm_ids_vec.size()) ? m_oov_id : m_randlm_ids_vec[factorId]; - }; - -}; - -} - -#endif diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am index 9e3e65cb9..96568da5b 100644 --- a/moses/src/Makefile.am +++ b/moses/src/Makefile.am @@ -148,7 +148,7 @@ libmoses_la_HEADERS += LM/IRST.h endif if RAND_LM -libmoses_la_HEADERS += LM/RandLM.h +libmoses_la_HEADERS += LM/Rand.h endif if ORLM_LM @@ -318,7 +318,7 @@ libmoses_la_SOURCES += LM/IRST.cpp endif if RAND_LM -libmoses_la_SOURCES += LM/RandLM.cpp +libmoses_la_SOURCES += LM/Rand.cpp endif if ORLM_LM |