#include "moses/StaticData.h" #include "moses/FactorCollection.h" #include #include "NeuralLMWrapper.h" #include "neuralLM.h" using namespace std; namespace Moses { NeuralLMWrapper::NeuralLMWrapper(const std::string &line) :LanguageModelSingleFactor(line) { ReadParameters(); } NeuralLMWrapper::~NeuralLMWrapper() { delete m_neuralLM_shared; } void NeuralLMWrapper::Load(AllOptions::ptr const& opts) { // Set parameters required by ancestor classes FactorCollection &factorCollection = FactorCollection::Instance(); m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, BOS_); m_sentenceStartWord[m_factorType] = m_sentenceStart; m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, EOS_); m_sentenceEndWord[m_factorType] = m_sentenceEnd; m_neuralLM_shared = new nplm::neuralLM(); m_neuralLM_shared->read(m_filePath); m_neuralLM_shared->premultiply(); //TODO: config option? m_neuralLM_shared->set_cache(1000000); m_unk = m_neuralLM_shared->lookup_word(""); UTIL_THROW_IF2(m_nGramOrder != m_neuralLM_shared->get_order(), "Wrong order of neuralLM: LM has " << m_neuralLM_shared->get_order() << ", but Moses expects " << m_nGramOrder); } LMResult NeuralLMWrapper::GetValue(const vector &contextFactor, State* finalState) const { if (!m_neuralLM.get()) { m_neuralLM.reset(new nplm::neuralLM(*m_neuralLM_shared)); //TODO: config option? m_neuralLM->set_cache(1000000); } vector words(contextFactor.size()); const size_t n = contextFactor.size(); for (size_t i=0; iGetFactor(m_factorType); const std::string string = factor->GetString().as_string(); int neuralLM_wordID = m_neuralLM->lookup_word(string); words[i] = neuralLM_wordID; } // Generate hashCode for only the last n-1 words, that represents the next LM // state size_t hashCode = 0; for (size_t i=1; ilookup_ngram(words); // Create a new struct to hold the result LMResult ret; ret.score = FloorScore(value); ret.unknown = (words.back() == m_unk); (*finalState) = (State*) hashCode; return ret; } }