diff options
author | Prashant Mathur <pramathur@ebay.com> | 2017-07-20 15:04:32 +0300 |
---|---|---|
committer | Prashant Mathur <pramathur@ebay.com> | 2017-07-20 15:04:32 +0300 |
commit | f07e60aece8c0bbd144b2e4028f2ed5114a521a5 (patch) | |
tree | a7d9bcca6234705f547f3ba387d2b18675a6882f /moses | |
parent | 29bd94f94251fbe7c7d213404a1dd897ab054a98 (diff) | |
parent | e32b8f580533e2b1cdb5b8496a8658f277409a69 (diff) |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'moses')
65 files changed, 984 insertions, 1628 deletions
diff --git a/moses/FF/SkeletonStatefulFF.cpp b/moses/FF/ExampleStatefulFF.cpp index 2acaf2d2e..5a53c4f87 100644 --- a/moses/FF/SkeletonStatefulFF.cpp +++ b/moses/FF/ExampleStatefulFF.cpp @@ -1,5 +1,5 @@ #include <vector> -#include "SkeletonStatefulFF.h" +#include "ExampleStatefulFF.h" #include "moses/ScoreComponentCollection.h" #include "moses/Hypothesis.h" @@ -9,7 +9,7 @@ namespace Moses { //////////////////////////////////////////////////////////////// -SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line) +ExampleStatefulFF::ExampleStatefulFF(const std::string &line) :StatefulFeatureFunction(3, line) { ReadParameters(); @@ -19,7 +19,7 @@ SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line) // An empty implementation of this function is provided by StatefulFeatureFunction. // Unless you are actually implementing this, please remove it from your // implementation (and the declaration in the header file to reduce code clutter. -void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source +void ExampleStatefulFF::EvaluateInIsolation(const Phrase &source , const TargetPhrase &targetPhrase , ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &estimatedScores) const @@ -28,7 +28,7 @@ void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source // An empty implementation of this function is provided by StatefulFeatureFunction. // Unless you are actually implementing this, please remove it from your // implementation (and the declaration in the header file to reduce code clutter. -void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input +void ExampleStatefulFF::EvaluateWithSourceContext(const InputType &input , const InputPath &inputPath , const TargetPhrase &targetPhrase , const StackVec *stackVec @@ -39,11 +39,11 @@ void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input // An empty implementation of this function is provided by StatefulFeatureFunction. // Unless you are actually implementing this, please remove it from your // implementation (and the declaration in the header file to reduce code clutter. -void SkeletonStatefulFF::EvaluateTranslationOptionListWithSourceContext +void ExampleStatefulFF::EvaluateTranslationOptionListWithSourceContext (const InputType &input, const TranslationOptionList &translationOptionList) const {} -FFState* SkeletonStatefulFF::EvaluateWhenApplied( +FFState* ExampleStatefulFF::EvaluateWhenApplied( const Hypothesis& cur_hypo, const FFState* prev_state, ScoreComponentCollection* accumulator) const @@ -59,18 +59,18 @@ FFState* SkeletonStatefulFF::EvaluateWhenApplied( accumulator->PlusEquals(this, "sparse-name", 2.4); // int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize(); // ??? [UG] - return new SkeletonState(0); + return new ExampleState(0); } -FFState* SkeletonStatefulFF::EvaluateWhenApplied( +FFState* ExampleStatefulFF::EvaluateWhenApplied( const ChartHypothesis& /* cur_hypo */, int /* featureID - used to index the state in the previous hypotheses */, ScoreComponentCollection* accumulator) const { - return new SkeletonState(0); + return new ExampleState(0); } -void SkeletonStatefulFF::SetParameter(const std::string& key, const std::string& value) +void ExampleStatefulFF::SetParameter(const std::string& key, const std::string& value) { if (key == "arg") { // set value here diff --git a/moses/FF/SkeletonStatefulFF.h b/moses/FF/ExampleStatefulFF.h index 7544ddd30..d66274295 100644 --- a/moses/FF/SkeletonStatefulFF.h +++ b/moses/FF/ExampleStatefulFF.h @@ -7,11 +7,11 @@ namespace Moses { -class SkeletonState : public FFState +class ExampleState : public FFState { int m_targetLen; public: - SkeletonState(int targetLen) + ExampleState(int targetLen) :m_targetLen(targetLen) { } @@ -19,22 +19,22 @@ public: return (size_t) m_targetLen; } virtual bool operator==(const FFState& o) const { - const SkeletonState& other = static_cast<const SkeletonState&>(o); + const ExampleState& other = static_cast<const ExampleState&>(o); return m_targetLen == other.m_targetLen; } }; -class SkeletonStatefulFF : public StatefulFeatureFunction +class ExampleStatefulFF : public StatefulFeatureFunction { public: - SkeletonStatefulFF(const std::string &line); + ExampleStatefulFF(const std::string &line); bool IsUseable(const FactorMask &mask) const { return true; } virtual const FFState* EmptyHypothesisState(const InputType &input) const { - return new SkeletonState(0); + return new ExampleState(0); } // An empty implementation of this function is provided by StatefulFeatureFunction. diff --git a/moses/FF/SkeletonStatelessFF.cpp b/moses/FF/ExampleStatelessFF.cpp index 8474efe76..0e62ad0ad 100644 --- a/moses/FF/SkeletonStatelessFF.cpp +++ b/moses/FF/ExampleStatelessFF.cpp @@ -1,5 +1,5 @@ #include <vector> -#include "SkeletonStatelessFF.h" +#include "ExampleStatelessFF.h" #include "moses/ScoreComponentCollection.h" #include "moses/TargetPhrase.h" @@ -7,13 +7,13 @@ using namespace std; namespace Moses { -SkeletonStatelessFF::SkeletonStatelessFF(const std::string &line) +ExampleStatelessFF::ExampleStatelessFF(const std::string &line) :StatelessFeatureFunction(2, line) { ReadParameters(); } -void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source +void ExampleStatelessFF::EvaluateInIsolation(const Phrase &source , const TargetPhrase &targetPhrase , ScoreComponentCollection &scoreBreakdown , ScoreComponentCollection &estimatedScores) const @@ -29,7 +29,7 @@ void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source } -void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input +void ExampleStatelessFF::EvaluateWithSourceContext(const InputType &input , const InputPath &inputPath , const TargetPhrase &targetPhrase , const StackVec *stackVec @@ -43,20 +43,20 @@ void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input } } -void SkeletonStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input +void ExampleStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input , const TranslationOptionList &translationOptionList) const {} -void SkeletonStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo, +void ExampleStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo, ScoreComponentCollection* accumulator) const {} -void SkeletonStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo, +void ExampleStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo, ScoreComponentCollection* accumulator) const {} -void SkeletonStatelessFF::SetParameter(const std::string& key, const std::string& value) +void ExampleStatelessFF::SetParameter(const std::string& key, const std::string& value) { if (key == "arg") { // set value here diff --git a/moses/FF/SkeletonStatelessFF.h b/moses/FF/ExampleStatelessFF.h index 0dc46e214..e1f007d21 100644 --- a/moses/FF/SkeletonStatelessFF.h +++ b/moses/FF/ExampleStatelessFF.h @@ -6,10 +6,10 @@ namespace Moses { -class SkeletonStatelessFF : public StatelessFeatureFunction +class ExampleStatelessFF : public StatelessFeatureFunction { public: - SkeletonStatelessFF(const std::string &line); + ExampleStatelessFF(const std::string &line); bool IsUseable(const FactorMask &mask) const { return true; diff --git a/moses/FF/SkeletonTranslationOptionListFeature.h b/moses/FF/ExampleTranslationOptionListFeature.h index e47e691aa..7686eb3ff 100644 --- a/moses/FF/SkeletonTranslationOptionListFeature.h +++ b/moses/FF/ExampleTranslationOptionListFeature.h @@ -6,10 +6,10 @@ namespace Moses { -class SkeletonTranslationOptionListFeature : public StatelessFeatureFunction +class ExampleTranslationOptionListFeature : public StatelessFeatureFunction { public: - SkeletonTranslationOptionListFeature(const std::string &line) + ExampleTranslationOptionListFeature(const std::string &line) :StatelessFeatureFunction(1, line) { ReadParameters(); } diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index 2c4668377..00dcac27d 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -15,8 +15,9 @@ #include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h" #include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h" #include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h" -#include "moses/TranslationModel/ProbingPT/ProbingPT.h" +#include "moses/TranslationModel/ProbingPT.h" #include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h" +#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h" #include "moses/FF/LexicalReordering/LexicalReordering.h" @@ -65,15 +66,16 @@ #include "SyntaxRHS.h" #include "DeleteRules.h" -#include "moses/FF/SkeletonStatelessFF.h" -#include "moses/FF/SkeletonStatefulFF.h" -#include "moses/LM/SkeletonLM.h" -#include "moses/FF/SkeletonTranslationOptionListFeature.h" +#include "moses/FF/ExampleStatelessFF.h" +#include "moses/FF/ExampleStatefulFF.h" +#include "moses/LM/ExampleLM.h" +#include "moses/FF/ExampleTranslationOptionListFeature.h" #include "moses/LM/BilingualLM.h" -#include "moses/TranslationModel/SkeletonPT.h" +#include "moses/TranslationModel/ExamplePT.h" #include "moses/Syntax/InputWeightFF.h" #include "moses/Syntax/RuleTableFF.h" +#include "moses/LM/InMemoryPerSentenceOnDemandLM.h" #include "moses/FF/EditOps.h" #include "moses/FF/CorrectionPattern.h" @@ -246,6 +248,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(PhraseDictionaryFuzzyMatch); MOSES_FNAME(ProbingPT); MOSES_FNAME(PhraseDictionaryMemoryPerSentence); + MOSES_FNAME(PhraseDictionaryMemoryPerSentenceOnDemand); MOSES_FNAME2("RuleTable", Syntax::RuleTableFF); MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF); @@ -296,12 +299,13 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(UnalignedWordCountFeature); MOSES_FNAME(DeleteRules); - MOSES_FNAME(SkeletonStatelessFF); - MOSES_FNAME(SkeletonStatefulFF); - MOSES_FNAME(SkeletonLM); - MOSES_FNAME(SkeletonTranslationOptionListFeature); - MOSES_FNAME(SkeletonPT); + MOSES_FNAME(ExampleStatelessFF); + MOSES_FNAME(ExampleStatefulFF); + MOSES_FNAME(ExampleLM); + MOSES_FNAME(ExampleTranslationOptionListFeature); + MOSES_FNAME(ExamplePT); + MOSES_FNAME(InMemoryPerSentenceOnDemandLM); MOSES_FNAME(EditOps); MOSES_FNAME(CorrectionPattern); diff --git a/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp b/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp index 22f550ba8..5d264e4c8 100644 --- a/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp +++ b/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp @@ -15,7 +15,7 @@ size_t BidirectionalReorderingState::hash() const bool BidirectionalReorderingState::operator==(const FFState& o) const { - if (&o == this) return 0; + if (&o == this) return true; BidirectionalReorderingState const &other = static_cast<BidirectionalReorderingState const&>(o); diff --git a/moses/GenerationDictionary.cpp b/moses/GenerationDictionary.cpp index 29a4fa2b3..35546e62c 100644 --- a/moses/GenerationDictionary.cpp +++ b/moses/GenerationDictionary.cpp @@ -120,7 +120,14 @@ const OutputWordCollection *GenerationDictionary::FindWord(const Word &word) con { const OutputWordCollection *ret; - Collection::const_iterator iter = m_collection.find(&word); + Word wordInput; + const std::vector<FactorType> &inputFactors = GetInput(); + for (size_t i = 0; i < inputFactors.size(); ++i) { + FactorType factorType = inputFactors[i]; + wordInput[factorType] = word[factorType]; + } + + Collection::const_iterator iter = m_collection.find(&wordInput); if (iter == m_collection.end()) { // can't find source phrase ret = NULL; diff --git a/moses/Jamfile b/moses/Jamfile index 49aab9025..5200029fb 100644 --- a/moses/Jamfile +++ b/moses/Jamfile @@ -122,10 +122,10 @@ vwfiles synlm mmlib mserver headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT -TranslationModel/ProbingPT//ProbingPT ThreadPool ..//search ../util/double-conversion//double-conversion +../probingpt//probingpt ..//z ../OnDiskPt//OnDiskPt $(TOP)//boost_filesystem @@ -139,5 +139,5 @@ alias headers-to-install : [ glob-tree *.h ] ; import testing ; -unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ; +unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ../probingpt//probingpt ..//boost_unit_test_framework ; diff --git a/moses/LM/SkeletonLM.cpp b/moses/LM/ExampleLM.cpp index 23958e688..034afef2e 100644 --- a/moses/LM/SkeletonLM.cpp +++ b/moses/LM/ExampleLM.cpp @@ -1,16 +1,19 @@ -#include "SkeletonLM.h" +#include "ExampleLM.h" #include "moses/FactorCollection.h" using namespace std; namespace Moses { -SkeletonLM::SkeletonLM(const std::string &line) +ExampleLM::ExampleLM(const std::string &line) :LanguageModelSingleFactor(line) { ReadParameters(); + UTIL_THROW_IF2(m_nGramOrder == NOT_FOUND, "Must set order"); + UTIL_THROW_IF2(m_nGramOrder <= 1, "Ngram order must be more than 1"); + FactorCollection &factorCollection = FactorCollection::Instance(); // needed by parent language model classes. Why didn't they set these themselves? @@ -21,11 +24,11 @@ SkeletonLM::SkeletonLM(const std::string &line) m_sentenceEndWord[m_factorType] = m_sentenceEnd; } -SkeletonLM::~SkeletonLM() +ExampleLM::~ExampleLM() { } -LMResult SkeletonLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const +LMResult ExampleLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const { LMResult ret; ret.score = contextFactor.size(); diff --git a/moses/LM/SkeletonLM.h b/moses/LM/ExampleLM.h index 988c9def9..292462917 100644 --- a/moses/LM/SkeletonLM.h +++ b/moses/LM/ExampleLM.h @@ -7,13 +7,13 @@ namespace Moses { -class SkeletonLM : public LanguageModelSingleFactor +class ExampleLM : public LanguageModelSingleFactor { protected: public: - SkeletonLM(const std::string &line); - ~SkeletonLM(); + ExampleLM(const std::string &line); + ~ExampleLM(); virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const; }; diff --git a/moses/LM/Implementation.cpp b/moses/LM/Implementation.cpp index eb67100ca..399a270d7 100644 --- a/moses/LM/Implementation.cpp +++ b/moses/LM/Implementation.cpp @@ -42,6 +42,7 @@ namespace Moses { LanguageModelImplementation::LanguageModelImplementation(const std::string &line) :LanguageModel(line) + ,m_nGramOrder(NOT_FOUND) { } @@ -61,7 +62,7 @@ void LanguageModelImplementation::ShiftOrPush(std::vector<const Word*> &contextF { if (contextFactor.size() < GetNGramOrder()) { contextFactor.push_back(&word); - } else { + } else if (GetNGramOrder() > 0) { // shift for (size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder() - 1 ; currNGramOrder++) { contextFactor[currNGramOrder] = contextFactor[currNGramOrder + 1]; diff --git a/moses/LM/InMemoryPerSentenceOnDemandLM.cpp b/moses/LM/InMemoryPerSentenceOnDemandLM.cpp new file mode 100644 index 000000000..0c9a43539 --- /dev/null +++ b/moses/LM/InMemoryPerSentenceOnDemandLM.cpp @@ -0,0 +1,101 @@ +#include <boost/foreach.hpp> +#include "InMemoryPerSentenceOnDemandLM.h" +#include "moses/FactorCollection.h" +#include "moses/Util.h" +#include "moses/StaticData.h" +#include "moses/TranslationTask.h" +#include "moses/ContextScope.h" +#include "moses/LM/Ken.h" +#include "lm/model.hh" +#include "util/mmap.hh" + +#include <cstdio> +#include <iostream> +#include <fstream> + +using namespace std; + +namespace Moses +{ +InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line) +{ + ReadParameters(); +} + +InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM() +{ +} + +void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask) +{ + + // The context scope object for this translation task + // contains a map of translation task-specific data + boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope(); + + // The key to the map is this object + void const* key = static_cast<void const*>(this); + + // The value stored in the map is a string representing a phrase table + boost::shared_ptr<string> value = contextScope->get<string>(key); + + // Create a stream to read the phrase table data + stringstream strme(*(value.get())); + + char * nullpointer = (char *) 0; + const char * filename = std::tmpnam(nullpointer); + ofstream tmp; + tmp.open(filename); + + // Read the phrase table data, one line at a time + string line; + while (getline(strme, line)) { + + tmp << line << "\n"; + + } + + tmp.close(); + + // m_tmpFilename.reset(new std::string("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm")); + m_tmpFilename.reset(new std::string(filename)); + + //LanguageModelKen<lm::ngram::ProbingModel> & lm = + GetPerThreadLM(); + + // std::remove(filename); + +} + +LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPerThreadLM() const +{ + + LanguageModelKen<lm::ngram::ProbingModel> *lm; + lm = m_perThreadLM.get(); + if (lm == NULL) { + lm = new LanguageModelKen<lm::ngram::ProbingModel>(); + + string* filename = m_tmpFilename.get(); + if (filename == NULL) { + UTIL_THROW(util::Exception, "Can't get a thread-specific LM because no temporary filename has been set for this thread\n"); + } else { + lm->LoadModel(*filename, util::POPULATE_OR_READ); + } + + VERBOSE(1, filename); + VERBOSE(1, "\tLM initialized\n"); + + m_perThreadLM.reset(lm); + } + assert(lm); + + return *lm; + +} + + + +} + + + diff --git a/moses/LM/InMemoryPerSentenceOnDemandLM.h b/moses/LM/InMemoryPerSentenceOnDemandLM.h new file mode 100644 index 000000000..7f4b60a5f --- /dev/null +++ b/moses/LM/InMemoryPerSentenceOnDemandLM.h @@ -0,0 +1,166 @@ +// $Id$ +#pragma once + +#include <string> +#include <vector> +#include "SingleFactor.h" +#include <boost/thread/tss.hpp> +#include "lm/model.hh" +#include "moses/LM/Ken.h" +#include "moses/FF/FFState.h" + +namespace Moses +{ + +struct InMemoryPerSentenceOnDemandLMState : public FFState { + lm::ngram::State state; + virtual size_t hash() const { + size_t ret = hash_value(state); + return ret; + } + virtual bool operator==(const FFState& o) const { + const InMemoryPerSentenceOnDemandLMState &other = static_cast<const InMemoryPerSentenceOnDemandLMState &>(o); + bool ret = state == other.state; + return ret; + } + +}; + +class InMemoryPerSentenceOnDemandLM : public LanguageModel +{ +public: + InMemoryPerSentenceOnDemandLM(const std::string &line); + ~InMemoryPerSentenceOnDemandLM(); + + void InitializeForInput(ttasksptr const& ttask); + + virtual void SetParameter(const std::string& key, const std::string& value) { + GetPerThreadLM().SetParameter(key, value); + } + + virtual const FFState* EmptyHypothesisState(const InputType &input) const { + if (isInitialized()) { + return GetPerThreadLM().EmptyHypothesisState(input); + } else { + return new InMemoryPerSentenceOnDemandLMState(); + } + } + + virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const { + if (isInitialized()) { + return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out); + } else { + UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n"); + } + } + + virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const { + if (isInitialized()) { + return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator); + } else { + UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n"); + } + } + + virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const { + if (isInitialized()) { + return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator); + } else { + UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n"); + } + } + + + virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const { + if (isInitialized()) { + GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScore called prior to being initialized"); + } + } + + virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const { + if (isInitialized()) { + GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScoreFromCache called prior to being initialized"); + } + } + + virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) { + if (isInitialized()) { + GetPerThreadLM().IssueRequestsFor(hypo, input_state); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IssueRequestsFor called prior to being initialized"); + } + } + + virtual void sync() { + if (isInitialized()) { + GetPerThreadLM().sync(); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::sync called prior to being initialized"); + } + } + + virtual void SetFFStateIdx(int state_idx) { + if (isInitialized()) { + GetPerThreadLM().SetFFStateIdx(state_idx); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::SetFFStateIdx called prior to being initialized"); + } + } + + virtual void IncrementalCallback(Incremental::Manager &manager) const { + if (isInitialized()) { + GetPerThreadLM().IncrementalCallback(manager); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IncrementalCallback called prior to being initialized"); + } + } + + virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const { + if (isInitialized()) { + GetPerThreadLM().ReportHistoryOrder(out, phrase); + } else { + UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::ReportHistoryOrder called prior to being initialized"); + } + } + + virtual void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedScores) const { + if (isInitialized()) { + GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores); + } else { + // UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::EvaluateInIsolation called prior to being initialized"); + } + } + + bool IsUseable(const FactorMask &mask) const { + bool ret = mask[m_factorType]; + return ret; + } + + +protected: + LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const; + + mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM; + mutable boost::thread_specific_ptr<std::string> m_tmpFilename; + + FactorType m_factorType; + + bool isInitialized() const { + if (m_tmpFilename.get() == NULL) { + return false; + } else { + return true; + } + } + +}; + + +} diff --git a/moses/LM/Jamfile b/moses/LM/Jamfile index 75b66603c..0c152d555 100644 --- a/moses/LM/Jamfile +++ b/moses/LM/Jamfile @@ -138,7 +138,7 @@ if $(with-dalm) { #Top-level LM library. If you've added a file that doesn't depend on external #libraries, put it here. -alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp +alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ExampleLM.cpp ../../lm//kenlm ..//headers $(dependencies) ; alias macros : : : : <define>$(lmmacros) ; diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index c7ac663cc..e42e60274 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -105,6 +105,7 @@ template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string config.load_method = load_method; m_ngram.reset(new Model(file.c_str(), config)); + VERBOSE(2, "LanguageModelKen " << m_description << " reset to " << file << "\n"); } template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method) @@ -116,6 +117,15 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::stri LoadModel(file, load_method); } +template <class Model> LanguageModelKen<Model>::LanguageModelKen() + :LanguageModel("KENLM") + ,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_)) + ,m_factorType(0) +{ + ReadParameters(); +} + + template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> ©_from) :LanguageModel(copy_from.GetArgLine()), m_ngram(copy_from.m_ngram), diff --git a/moses/LM/Ken.h b/moses/LM/Ken.h index 4934228c2..33590d659 100644 --- a/moses/LM/Ken.h +++ b/moses/LM/Ken.h @@ -33,11 +33,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "moses/TypeDef.h" #include "moses/Word.h" + + namespace Moses { //class LanguageModel; class FFState; +class InMemoryPerSentenceOnDemandLM; LanguageModel *ConstructKenLM(const std::string &line); @@ -67,6 +70,8 @@ public: virtual bool IsUseable(const FactorMask &mask) const; + friend class InMemoryPerSentenceOnDemandLM; + protected: boost::shared_ptr<Model> m_ngram; @@ -84,6 +89,7 @@ protected: std::vector<lm::WordIndex> m_lmIdLookup; private: + LanguageModelKen(); LanguageModelKen(const LanguageModelKen<Model> ©_from); // Convert last words of hypothesis into vocab ids, returning an end pointer. diff --git a/moses/LM/Reloading.h b/moses/LM/Reloading.h index d5ae83d17..88f8e8869 100644 --- a/moses/LM/Reloading.h +++ b/moses/LM/Reloading.h @@ -35,113 +35,48 @@ namespace Moses { class FFState; - -//LanguageModel *ConstructReloadingLM(const std::string &line); -//LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy); -/* - namespace { -class MappingBuilder : public lm::EnumerateVocab -{ -public: - MappingBuilder(FactorCollection &factorCollection, std::vector<lm::WordIndex> &mapping) - : m_factorCollection(factorCollection), m_mapping(mapping) {} - - void Add(lm::WordIndex index, const StringPiece &str) { - std::size_t factorId = m_factorCollection.AddFactor(str)->GetId(); - if (m_mapping.size() <= factorId) { - // 0 is <unk> :-) - m_mapping.resize(factorId + 1); - } - m_mapping[factorId] = index; - } - -private: - FactorCollection &m_factorCollection; - std::vector<lm::WordIndex> &m_mapping; -}; - } -*/ template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model> { public: - // TODO(Lane) copy less code, update to load_method + ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line, file, factorType, lazy ? util::LAZY : util::POPULATE_OR_READ), m_file(file), m_lazy(lazy) { - std::cerr << "ReloadingLM constructor: " << m_file << std::endl; - // std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl; + VERBOSE(1, "ReloadingLM constructor: " << m_file << std::endl); } virtual void InitializeForInput(ttasksptr const& ttask) { - std::cerr << "ReloadingLM InitializeForInput" << std::endl; - // TODO(lane): load_method - LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ); - /* - lm::ngram::Config config; - if(this->m_verbosity >= 1) { - config.messages = &std::cerr; - } else { - config.messages = NULL; - } - FactorCollection &collection = FactorCollection::Instance(); - MappingBuilder builder(collection, m_lmIdLookup); - config.enumerate_vocab = &builder; - config.load_method = m_lazy ? util::LAZY : util::POPULATE_OR_READ; - - m_ngram.reset(new Model(m_file.c_str(), config)); - - m_beginSentenceFactor = collection.AddFactor(BOS_); - */ - }; + VERBOSE(1, "ReloadingLM InitializeForInput" << std::endl); - /* - ReloadingLanguageModel(const std::string &line) : LanguageModelKen<Model>(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) { - std::cerr << "ReloadingLM constructor" << std::endl; - std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl; - } - */ - /* - ~ReloadingLanguageModel() { - delete m_lm; - } + // The context scope object for this translation task + // contains a map of translation task-specific data + boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope(); - virtual const FFState *EmptyHypothesisState(const InputType &input) const { - return m_lm->EmptyHypothesisState(input); - } + // The key to the map is this object + void const* key = static_cast<void const*>(this); - virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const { - m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount); - } + // The value stored in the map is a string representing a phrase table + boost::shared_ptr<string> value = contextScope->get<string>(key); - virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const { - return m_lm->EvaluateWhenApplied(hypo, ps, out); - } - - virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const { - return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator); - } + // Create a stream to read the phrase table data + stringstream strme(*(value.get())); - virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const { - return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator); - } + ofstream tmp; + tmp.open(m_file.c_str()); - virtual void IncrementalCallback(Incremental::Manager &manager) const { - m_lm->IncrementalCallback(manager); - } + // Read the phrase table data, one line at a time + string line; + while (getline(strme, line)) { - virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const { - m_lm->ReportHistoryOrder(out, phrase); - } + tmp << line << "\n"; - virtual bool IsUseable(const FactorMask &mask) const { - return m_lm->IsUseable(mask); - } + } + tmp.close(); - private: + LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ); + }; - LanguageModel *m_lm; - */ protected: diff --git a/moses/Manager.cpp b/moses/Manager.cpp index 3650baabb..0cd540991 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -1724,7 +1724,7 @@ OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) c for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) { const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]); if (factor) out << fd << *factor; - else out << fd << UNKNOWN_FACTOR; + //else out << fd << UNKNOWN_FACTOR; } if(markUnknown && word.IsOOV()) { diff --git a/moses/OutputCollector.h b/moses/OutputCollector.h index 0d6f37472..797cc85cf 100644 --- a/moses/OutputCollector.h +++ b/moses/OutputCollector.h @@ -32,6 +32,7 @@ #include <iostream> #include <map> #include <ostream> +#include <fstream> #include <string> #include "Util.h" #include "util/exception.hh" diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 4d3e96000..b65c22eb4 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -423,8 +423,13 @@ LoadDecodeGraphsOld(const vector<string> &mappingVector, if (m_decodeGraphs.size() < decodeGraphInd + 1) { DecodeGraph *decodeGraph; if (is_syntax(m_options->search.algo)) { - size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; - VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); + size_t maxChartSpan; + if (decodeGraphInd < maxChartSpans.size()) { + maxChartSpan = maxChartSpans[decodeGraphInd]; + VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl); + } else { + maxChartSpan = DEFAULT_MAX_CHART_SPAN; + } decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); } else { decodeGraph = new DecodeGraph(m_decodeGraphs.size()); diff --git a/moses/TrainingTask.h b/moses/TrainingTask.h index 4d2152920..83933691d 100644 --- a/moses/TrainingTask.h +++ b/moses/TrainingTask.h @@ -39,6 +39,18 @@ public: boost::shared_ptr<IOWrapper> const& ioWrapper) { boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper)); ret->m_self = ret; + ret->m_scope.reset(new ContextScope); + return ret; + } + + // factory function + static boost::shared_ptr<TrainingTask> + create(boost::shared_ptr<InputType> const& source, + boost::shared_ptr<IOWrapper> const& ioWrapper, + boost::shared_ptr<ContextScope> const& scope) { + boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper)); + ret->m_self = ret; + ret->m_scope = scope; return ret; } diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp index ca219f249..6c80e30af 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp @@ -18,7 +18,7 @@ ***********************************************************************/ #include <iostream> -#include "ChartRuleLookupManagerSkeleton.h" +#include "ChartRuleLookupManagerExample.h" #include "DotChartInMemory.h" #include "moses/Util.h" @@ -29,29 +29,29 @@ #include "moses/NonTerminal.h" #include "moses/ChartCellCollection.h" #include "moses/TranslationModel/PhraseDictionaryMemory.h" -#include "moses/TranslationModel/SkeletonPT.h" +#include "moses/TranslationModel/ExamplePT.h" using namespace std; namespace Moses { -ChartRuleLookupManagerSkeleton::ChartRuleLookupManagerSkeleton( +ChartRuleLookupManagerExample::ChartRuleLookupManagerExample( const ChartParser &parser, const ChartCellCollectionBase &cellColl, - const SkeletonPT &skeletonPt) + const ExamplePT &skeletonPt) : ChartRuleLookupManager(parser, cellColl) , m_skeletonPT(skeletonPt) { - cerr << "starting ChartRuleLookupManagerSkeleton" << endl; + cerr << "starting ChartRuleLookupManagerExample" << endl; } -ChartRuleLookupManagerSkeleton::~ChartRuleLookupManagerSkeleton() +ChartRuleLookupManagerExample::~ChartRuleLookupManagerExample() { // RemoveAllInColl(m_tpColl); } -void ChartRuleLookupManagerSkeleton::GetChartRuleCollection( +void ChartRuleLookupManagerExample::GetChartRuleCollection( const InputPath &inputPath, size_t last, ChartParserCallback &outColl) @@ -74,12 +74,12 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection( } TargetPhrase * -ChartRuleLookupManagerSkeleton:: +ChartRuleLookupManagerExample:: CreateTargetPhrase(const Word &sourceWord) const { - // create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:' + // create a target phrase from the 1st word of the source, prefix with 'ChartManagerExample:' string str = sourceWord.GetFactor(0)->GetString().as_string(); - str = "ChartManagerSkeleton:" + str; + str = "ChartManagerExample:" + str; TargetPhrase *tp = new TargetPhrase(&m_skeletonPT); Word &word = tp->AddWord(); diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h index d01f3b9bd..3b3f59ace 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h @@ -29,16 +29,16 @@ class TargetPhraseCollection; class ChartParserCallback; class DottedRuleColl; class Range; -class SkeletonPT; +class ExamplePT; -class ChartRuleLookupManagerSkeleton : public ChartRuleLookupManager +class ChartRuleLookupManagerExample : public ChartRuleLookupManager { public: - ChartRuleLookupManagerSkeleton(const ChartParser &parser, - const ChartCellCollectionBase &cellColl, - const SkeletonPT &skeletonPt); + ChartRuleLookupManagerExample(const ChartParser &parser, + const ChartCellCollectionBase &cellColl, + const ExamplePT &skeletonPt); - ~ChartRuleLookupManagerSkeleton(); + ~ChartRuleLookupManagerExample(); virtual void GetChartRuleCollection( const InputPath &inputPath, @@ -50,7 +50,7 @@ private: StackVec m_stackVec; std::vector<TargetPhraseCollection::shared_ptr > m_tpColl; - const SkeletonPT &m_skeletonPT; + const ExamplePT &m_skeletonPT; }; } // namespace Moses diff --git a/moses/TranslationModel/SkeletonPT.cpp b/moses/TranslationModel/ExamplePT.cpp index 6b42212f9..198ce2814 100644 --- a/moses/TranslationModel/SkeletonPT.cpp +++ b/moses/TranslationModel/ExamplePT.cpp @@ -1,29 +1,29 @@ // vim:tabstop=2 -#include "SkeletonPT.h" -#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" +#include "ExamplePT.h" +#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h" using namespace std; namespace Moses { -SkeletonPT::SkeletonPT(const std::string &line) +ExamplePT::ExamplePT(const std::string &line) : PhraseDictionary(line, true) { ReadParameters(); } -void SkeletonPT::Load(AllOptions::ptr const& opts) +void ExamplePT::Load(AllOptions::ptr const& opts) { m_options = opts; SetFeaturesToApply(); } -void SkeletonPT::InitializeForInput(ttasksptr const& ttask) +void ExamplePT::InitializeForInput(ttasksptr const& ttask) { ReduceCache(); } -void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const +void ExamplePT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const { CacheColl &cache = GetCache(); @@ -46,14 +46,14 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQu } } -TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const +TargetPhrase *ExamplePT::CreateTargetPhrase(const Phrase &sourcePhrase) const { - // create a target phrase from the 1st word of the source, prefix with 'SkeletonPT:' + // create a target phrase from the 1st word of the source, prefix with 'ExamplePT:' assert(sourcePhrase.GetSize()); assert(m_output.size() == 1); string str = sourcePhrase.GetWord(0).GetFactor(0)->GetString().as_string(); - str = "SkeletonPT:" + str; + str = "ExamplePT:" + str; TargetPhrase *tp = new TargetPhrase(this); Word &word = tp->AddWord(); @@ -69,17 +69,17 @@ TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const return tp; } -ChartRuleLookupManager* SkeletonPT::CreateRuleLookupManager(const ChartParser &parser, +ChartRuleLookupManager* ExamplePT::CreateRuleLookupManager(const ChartParser &parser, const ChartCellCollectionBase &cellCollection, std::size_t /*maxChartSpan*/) { - return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this); + return new ChartRuleLookupManagerExample(parser, cellCollection, *this); } -TO_STRING_BODY(SkeletonPT); +TO_STRING_BODY(ExamplePT); // friend -ostream& operator<<(ostream& out, const SkeletonPT& phraseDict) +ostream& operator<<(ostream& out, const ExamplePT& phraseDict) { return out; } diff --git a/moses/TranslationModel/SkeletonPT.h b/moses/TranslationModel/ExamplePT.h index 443f1cc8e..6ec7764c9 100644 --- a/moses/TranslationModel/SkeletonPT.h +++ b/moses/TranslationModel/ExamplePT.h @@ -9,12 +9,12 @@ class ChartParser; class ChartCellCollectionBase; class ChartRuleLookupManager; -class SkeletonPT : public PhraseDictionary +class ExamplePT : public PhraseDictionary { - friend std::ostream& operator<<(std::ostream&, const SkeletonPT&); + friend std::ostream& operator<<(std::ostream&, const ExamplePT&); public: - SkeletonPT(const std::string &line); + ExamplePT(const std::string &line); void Load(AllOptions::ptr const& opts); diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp b/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp index 36a28089b..fc62f0679 100644 --- a/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp +++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp @@ -1,6 +1,6 @@ // vim:tabstop=2 #include "PhraseDictionaryMemoryPerSentence.h" -#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" +#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h" using namespace std; diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp new file mode 100644 index 000000000..acf834cbd --- /dev/null +++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp @@ -0,0 +1,149 @@ +// vim:tabstop=2 +#include "PhraseDictionaryMemoryPerSentenceOnDemand.h" +#include <sstream> + +using namespace std; + +namespace Moses +{ +PhraseDictionaryMemoryPerSentenceOnDemand::PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line) + : PhraseDictionary(line, true), m_valuesAreProbabilities(true) +{ + ReadParameters(); +} + +void PhraseDictionaryMemoryPerSentenceOnDemand::Load(AllOptions::ptr const& opts) +{ + m_options = opts; + SetFeaturesToApply(); + + // don't load anything. Load when we have the input +} + + +TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const +{ + + Coll &coll = GetColl(); + + return coll[source]; + +} + + +void PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(ttasksptr const& ttask) +{ + Coll &coll = GetColl(); + coll.clear(); + + VERBOSE(2, "Initializing PhraseDictionaryMemoryPerSentenceOnDemand " << m_description << "\n"); + + // The context scope object for this translation task + // contains a map of translation task-specific data + boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope(); + + // The key to the map is this object + void const* key = static_cast<void const*>(this); + + // The value stored in the map is a string representing a phrase table + boost::shared_ptr<string> value = contextScope->get<string>(key); + + // Create a stream to read the phrase table data + stringstream strme(*(value.get())); + + // Read the phrase table data, one line at a time + string line; + while (getline(strme, line)) { + + VERBOSE(3, "\t" << line); + + vector<string> toks = TokenizeMultiCharSeparator(line, "|||"); + Phrase source; + source.CreateFromString(Input, m_input, toks[0], NULL); + + TargetPhrase *target = new TargetPhrase(this); + target->CreateFromString(Output, m_output, toks[1], NULL); + + // score for this phrase table + vector<float> scores = Tokenize<float>(toks[2]); + if (m_valuesAreProbabilities) { + std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore); + std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore); + } + target->GetScoreBreakdown().PlusEquals(this, scores); + + // score of all other ff when this rule is being loaded + target->EvaluateInIsolation(source, GetFeaturesToApply()); + + // add to coll + TargetPhraseCollection::shared_ptr &tpsPtr = coll[source]; + TargetPhraseCollection *tps = tpsPtr.get(); + if (tps == NULL) { + tps = new TargetPhraseCollection(); + tpsPtr.reset(tps); + } + tps->Add(target); + } +} + +void PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const +{ + InputPathList::const_iterator iter; + for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) { + InputPath &inputPath = **iter; + const Phrase &source = inputPath.GetPhrase(); + + Coll &coll = GetColl(); + Coll::const_iterator iter = coll.find(source); + if (iter == coll.end()) { + TargetPhraseCollection::shared_ptr tprPtr; + inputPath.SetTargetPhrases(*this, tprPtr, NULL); + } else { + const TargetPhraseCollection::shared_ptr &tprPtr = iter->second; + inputPath.SetTargetPhrases(*this, tprPtr, NULL); + } + } +} + + +ChartRuleLookupManager* PhraseDictionaryMemoryPerSentenceOnDemand::CreateRuleLookupManager(const ChartParser &parser, + const ChartCellCollectionBase &cellCollection, + std::size_t /*maxChartSpan*/) +{ + abort(); +} + +PhraseDictionaryMemoryPerSentenceOnDemand::Coll &PhraseDictionaryMemoryPerSentenceOnDemand::GetColl() const +{ + Coll *coll; + coll = m_coll.get(); + if (coll == NULL) { + coll = new Coll; + m_coll.reset(coll); + } + assert(coll); + return *coll; +} + +void +PhraseDictionaryMemoryPerSentenceOnDemand::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "path") { + UTIL_THROW(util::Exception, "PhraseDictionaryMemoryPerSentenceOnDemand does not support key \"path\"."); + } else if (key == "valuesAreProbabilities") { + m_valuesAreProbabilities = Scan<bool>(value); + } else { + PhraseDictionary::SetParameter(key, value); + } +} + + +TO_STRING_BODY(PhraseDictionaryMemoryPerSentenceOnDemand); + +// friend +ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentenceOnDemand& phraseDict) +{ + return out; +} + +} diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h new file mode 100644 index 000000000..e3fe6514a --- /dev/null +++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h @@ -0,0 +1,48 @@ + +#pragma once + +#include "PhraseDictionary.h" +#include "moses/TypeDef.h" +#include "moses/TranslationTask.h" + +namespace Moses +{ +class ChartParser; +class ChartCellCollectionBase; +class ChartRuleLookupManager; + +class PhraseDictionaryMemoryPerSentenceOnDemand : public PhraseDictionary +{ + friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryMemoryPerSentenceOnDemand&); + +public: + PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line); + + void Load(AllOptions::ptr const& opts); + + void InitializeForInput(ttasksptr const& ttask); + + // for phrase-based model + void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; + + // for syntax/hiero model (CKY+ decoding) + ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, const ChartCellCollectionBase&, std::size_t); + + void SetParameter(const std::string& key, const std::string& value); + + TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const; + + TO_STRING(); + + +protected: + typedef boost::unordered_map<Phrase, TargetPhraseCollection::shared_ptr> Coll; + mutable boost::thread_specific_ptr<Coll> m_coll; + + bool m_valuesAreProbabilities; + + Coll &GetColl() const; + +}; + +} // namespace Moses diff --git a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp index 3d1664822..2ffe880c7 100644 --- a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp +++ b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp @@ -2,7 +2,6 @@ #include <cstdlib> #include "PhraseDictionaryTransliteration.h" -#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" #include "moses/DecodeGraph.h" #include "moses/DecodeStep.h" #include "util/tempfile.hh" diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.cpp b/moses/TranslationModel/ProbingPT.cpp index 1ae0c67c3..dca7835f5 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp +++ b/moses/TranslationModel/ProbingPT.cpp @@ -4,8 +4,8 @@ #include "moses/FactorCollection.h" #include "moses/TargetPhraseCollection.h" #include "moses/InputFileStream.h" -#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h" -#include "querying.hh" +#include "probingpt/querying.h" +#include "probingpt/probing_hash_utils.h" using namespace std; @@ -14,6 +14,7 @@ namespace Moses ProbingPT::ProbingPT(const std::string &line) : PhraseDictionary(line,true) ,m_engine(NULL) + ,load_method(util::POPULATE_OR_READ) { ReadParameters(); @@ -31,7 +32,7 @@ void ProbingPT::Load(AllOptions::ptr const& opts) m_options = opts; SetFeaturesToApply(); - m_engine = new QueryEngine(m_filePath.c_str()); + m_engine = new probingpt::QueryEngine(m_filePath.c_str(), load_method); m_unkId = 456456546456; @@ -116,6 +117,28 @@ void ProbingPT::CreateAlignmentMap(const std::string path) } } +void ProbingPT::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "load") { + if (value == "lazy") { + load_method = util::LAZY; + } else if (value == "populate_or_lazy") { + load_method = util::POPULATE_OR_LAZY; + } else if (value == "populate_or_read" || value == "populate") { + load_method = util::POPULATE_OR_READ; + } else if (value == "read") { + load_method = util::READ; + } else if (value == "parallel_read") { + load_method = util::PARALLEL_READ; + } else { + UTIL_THROW2("load method not supported" << value); + } + } else { + PhraseDictionary::SetParameter(key, value); + } + +} + void ProbingPT::InitializeForInput(ttasksptr const& ttask) { @@ -256,12 +279,12 @@ TargetPhraseCollection *ProbingPT::CreateTargetPhrases( TargetPhrase *ProbingPT::CreateTargetPhrase( const char *&offset) const { - TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; + probingpt::TargetPhraseInfo *tpInfo = (probingpt::TargetPhraseInfo*) offset; size_t numRealWords = tpInfo->numWords / m_output.size(); TargetPhrase *tp = new TargetPhrase(this); - offset += sizeof(TargetPhraseInfo); + offset += sizeof(probingpt::TargetPhraseInfo); // scores float *scores = (float*) offset; diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.h b/moses/TranslationModel/ProbingPT.h index 953a2dc2f..1c996f5fa 100644 --- a/moses/TranslationModel/ProbingPT/ProbingPT.h +++ b/moses/TranslationModel/ProbingPT.h @@ -3,16 +3,20 @@ #include <boost/iostreams/device/mapped_file.hpp> #include <boost/bimap.hpp> #include <boost/unordered_map.hpp> -#include "../PhraseDictionary.h" +#include "PhraseDictionary.h" +#include "util/mmap.hh" +namespace probingpt +{ +class QueryEngine; +class target_text; +} namespace Moses { class ChartParser; class ChartCellCollectionBase; class ChartRuleLookupManager; -class QueryEngine; -class target_text; class ProbingPT : public PhraseDictionary { @@ -26,6 +30,8 @@ public: void InitializeForInput(ttasksptr const& ttask); + void SetParameter(const std::string& key, const std::string& value); + // for phrase-based model void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const; @@ -39,12 +45,13 @@ public: protected: - QueryEngine *m_engine; + probingpt::QueryEngine *m_engine; uint64_t m_unkId; std::vector<uint64_t> m_sourceVocab; // factor id -> pt id std::vector<const Factor*> m_targetVocab; // pt id -> factor* std::vector<const AlignmentInfo*> m_aligns; + util::LoadMethod load_method; boost::iostreams::mapped_file_source file; const char *data; diff --git a/moses/TranslationModel/ProbingPT/Jamfile b/moses/TranslationModel/ProbingPT/Jamfile deleted file mode 100644 index 29c6ec41d..000000000 --- a/moses/TranslationModel/ProbingPT/Jamfile +++ /dev/null @@ -1,8 +0,0 @@ -local current = "" ; -local includes = ; - -fakelib ProbingPT : [ glob *.cpp ] ../..//headers : $(includes) <dependency>$(PT-LOG) : : $(includes) ; - -path-constant PT-LOG : bin/pt.log ; -update-if-changed $(PT-LOG) $(current) ; - diff --git a/moses/TranslationModel/ProbingPT/StoreTarget.cpp b/moses/TranslationModel/ProbingPT/StoreTarget.cpp deleted file mode 100644 index f586a26b9..000000000 --- a/moses/TranslationModel/ProbingPT/StoreTarget.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/* - * StoreTarget.cpp - * - * Created on: 19 Jan 2016 - * Author: hieu - */ -#include <boost/foreach.hpp> -#include "StoreTarget.h" -#include "line_splitter.hh" -#include "probing_hash_utils.hh" -#include "moses/OutputFileStream.h" -#include "moses/Util.h" - -using namespace std; - -namespace Moses -{ - -StoreTarget::StoreTarget(const std::string &basepath) - :m_basePath(basepath) - ,m_vocab(basepath + "/TargetVocab.dat") -{ - std::string path = basepath + "/TargetColl.dat"; - m_fileTargetColl.open(path.c_str(), - std::ios::out | std::ios::binary | std::ios::ate | std::ios::trunc); - if (!m_fileTargetColl.is_open()) { - throw "can't create file "; - } - -} - -StoreTarget::~StoreTarget() -{ - assert(m_coll.empty()); - m_fileTargetColl.close(); - - // vocab - m_vocab.Save(); -} - -uint64_t StoreTarget::Save() -{ - uint64_t ret = m_fileTargetColl.tellp(); - - // save to disk - uint64_t numTP = m_coll.size(); - m_fileTargetColl.write((char*) &numTP, sizeof(uint64_t)); - - for (size_t i = 0; i < m_coll.size(); ++i) { - Save(*m_coll[i]); - } - - // clear coll - RemoveAllInColl(m_coll); - m_coll.clear(); - - // starting position of coll - return ret; -} - -void StoreTarget::Save(const target_text &rule) -{ - // metadata for each tp - TargetPhraseInfo tpInfo; - tpInfo.alignTerm = GetAlignId(rule.word_align_term); - tpInfo.alignNonTerm = GetAlignId(rule.word_align_non_term); - tpInfo.numWords = rule.target_phrase.size(); - tpInfo.propLength = rule.property.size(); - - //cerr << "TPInfo=" << sizeof(TPInfo); - m_fileTargetColl.write((char*) &tpInfo, sizeof(TargetPhraseInfo)); - - // scores - for (size_t i = 0; i < rule.prob.size(); ++i) { - float prob = rule.prob[i]; - m_fileTargetColl.write((char*) &prob, sizeof(prob)); - } - - // tp - for (size_t i = 0; i < rule.target_phrase.size(); ++i) { - uint32_t vocabId = rule.target_phrase[i]; - m_fileTargetColl.write((char*) &vocabId, sizeof(vocabId)); - } - - // prop TODO - -} - -void StoreTarget::SaveAlignment() -{ - std::string path = m_basePath + "/Alignments.dat"; - OutputFileStream file(path); - - BOOST_FOREACH(Alignments::value_type &valPair, m_aligns) { - file << valPair.second << "\t"; - - const std::vector<size_t> &aligns = valPair.first; - BOOST_FOREACH(size_t align, aligns) { - file << align << " "; - } - file << endl; - } - -} - -void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) -{ - target_text *rule = new target_text; - //cerr << "line.target_phrase=" << line.target_phrase << endl; - - // target_phrase - vector<bool> nonTerms; - util::TokenIter<util::SingleCharacter> it; - it = util::TokenIter<util::SingleCharacter>(line.target_phrase, - util::SingleCharacter(' ')); - while (it) { - StringPiece word = *it; - //cerr << "word=" << word << endl; - - bool nonTerm = false; - if (scfg) { - // not really sure how to handle factored SCFG and NT - if (scfg && word[0] == '[' && word[word.size() - 1] == ']') { - //cerr << "NON-TERM=" << tok << " " << nonTerms.size() << endl; - nonTerm = true; - } - nonTerms.push_back(nonTerm); - } - - util::TokenIter<util::SingleCharacter> itFactor; - itFactor = util::TokenIter<util::SingleCharacter>(word, - util::SingleCharacter('|')); - while (itFactor) { - StringPiece factor = *itFactor; - - string factorStr = factor.as_string(); - uint32_t vocabId = m_vocab.GetVocabId(factorStr); - - rule->target_phrase.push_back(vocabId); - - itFactor++; - } - - it++; - } - - // probs - it = util::TokenIter<util::SingleCharacter>(line.prob, - util::SingleCharacter(' ')); - while (it) { - string tok = it->as_string(); - float prob = Scan<float>(tok); - - if (log_prob) { - prob = FloorScore(log(prob)); - if (prob == 0.0f) prob = 0.0000000001; - } - - rule->prob.push_back(prob); - it++; - } - - /* - cerr << "nonTerms="; - for (size_t i = 0; i < nonTerms.size(); ++i) { - cerr << nonTerms[i] << " "; - } - cerr << endl; - */ - - // alignment - it = util::TokenIter<util::SingleCharacter>(line.word_align, - util::SingleCharacter(' ')); - while (it) { - string tokPair = Trim(it->as_string()); - if (tokPair.empty()) { - break; - } - - vector<size_t> alignPair = Tokenize<size_t>(tokPair, "-"); - assert(alignPair.size() == 2); - - bool nonTerm = false; - size_t sourcePos = alignPair[0]; - size_t targetPos = alignPair[1]; - if (scfg) { - nonTerm = nonTerms[targetPos]; - } - - //cerr << targetPos << "=" << nonTerm << endl; - - if (nonTerm) { - rule->word_align_non_term.push_back(sourcePos); - rule->word_align_non_term.push_back(targetPos); - //cerr << (int) rule->word_all1.back() << " "; - } else { - rule->word_align_term.push_back(sourcePos); - rule->word_align_term.push_back(targetPos); - } - - it++; - } - - // extra scores - string prop = line.property.as_string(); - AppendLexRO(prop, rule->prob, log_prob); - - //cerr << "line.property=" << line.property << endl; - //cerr << "prop=" << prop << endl; - - // properties - /* - for (size_t i = 0; i < prop.size(); ++i) { - rule->property.push_back(prop[i]); - } - */ - m_coll.push_back(rule); -} - -uint32_t StoreTarget::GetAlignId(const std::vector<size_t> &align) -{ - boost::unordered_map<std::vector<size_t>, uint32_t>::iterator iter = - m_aligns.find(align); - if (iter == m_aligns.end()) { - uint32_t ind = m_aligns.size(); - m_aligns[align] = ind; - return ind; - } else { - return iter->second; - } -} - -void StoreTarget::AppendLexRO(std::string &prop, std::vector<float> &retvector, - bool log_prob) const -{ - size_t startPos = prop.find("{{LexRO "); - - if (startPos != string::npos) { - size_t endPos = prop.find("}}", startPos + 8); - string lexProb = prop.substr(startPos + 8, endPos - startPos - 8); - //cerr << "lexProb=" << lexProb << endl; - - // append lex probs to pt probs - vector<float> scores = Tokenize<float>(lexProb); - - if (log_prob) { - for (size_t i = 0; i < scores.size(); ++i) { - scores[i] = FloorScore(log(scores[i])); - if (scores[i] == 0.0f) scores[i] = 0.0000000001; - } - } - - for (size_t i = 0; i < scores.size(); ++i) { - retvector.push_back(scores[i]); - } - - // exclude LexRO property from property column - prop = prop.substr(0, startPos) - + prop.substr(endPos + 2, prop.size() - endPos - 2); - //cerr << "line.property_to_be_binarized=" << line.property_to_be_binarized << "AAAA" << endl; - } -} - -} /* namespace Moses2 */ diff --git a/moses/TranslationModel/ProbingPT/StoreTarget.h b/moses/TranslationModel/ProbingPT/StoreTarget.h deleted file mode 100644 index 331c197b3..000000000 --- a/moses/TranslationModel/ProbingPT/StoreTarget.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * StoreTarget.h - * - * Created on: 19 Jan 2016 - * Author: hieu - */ -#pragma once -#include <string> -#include <fstream> -#include <vector> -#include <inttypes.h> -#include <boost/unordered_map.hpp> -#include <boost/unordered_set.hpp> -#include "StoreVocab.h" - -namespace Moses -{ - -class line_text; -class target_text; - -class StoreTarget -{ -public: - StoreTarget(const std::string &basepath); - virtual ~StoreTarget(); - - uint64_t Save(); - void SaveAlignment(); - - void Append(const line_text &line, bool log_prob, bool scfg); -protected: - std::string m_basePath; - std::fstream m_fileTargetColl; - StoreVocab<uint32_t> m_vocab; - - typedef boost::unordered_map<std::vector<size_t>, uint32_t> Alignments; - Alignments m_aligns; - - std::vector<target_text*> m_coll; - - uint32_t GetAlignId(const std::vector<size_t> &align); - void Save(const target_text &rule); - - void AppendLexRO(std::string &prop, std::vector<float> &retvector, - bool log_prob) const; - -}; - -} /* namespace Moses2 */ - diff --git a/moses/TranslationModel/ProbingPT/StoreVocab.cpp b/moses/TranslationModel/ProbingPT/StoreVocab.cpp deleted file mode 100644 index 6515bac63..000000000 --- a/moses/TranslationModel/ProbingPT/StoreVocab.cpp +++ /dev/null @@ -1,13 +0,0 @@ -/* - * StoreVocab.cpp - * - * Created on: 15 Jun 2016 - * Author: hieu - */ -#include <fstream> -#include "StoreVocab.h" - -namespace Moses -{ - -} /* namespace Moses2 */ diff --git a/moses/TranslationModel/ProbingPT/StoreVocab.h b/moses/TranslationModel/ProbingPT/StoreVocab.h deleted file mode 100644 index 806dcebf4..000000000 --- a/moses/TranslationModel/ProbingPT/StoreVocab.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * StoreVocab.h - * - * Created on: 15 Jun 2016 - * Author: hieu - */ -#pragma once -#include <string> -#include <boost/unordered_map.hpp> -#include "moses/OutputFileStream.h" -#include "moses/Util.h" - -namespace Moses -{ - -template<typename VOCABID> -class StoreVocab -{ -protected: - std::string m_path; - - typedef boost::unordered_map<std::string, VOCABID> Coll; - Coll m_vocab; - -public: - StoreVocab(const std::string &path) - :m_path(path) - {} - - virtual ~StoreVocab() {} - - VOCABID GetVocabId(const std::string &word) { - typename Coll::iterator iter = m_vocab.find(word); - if (iter == m_vocab.end()) { - VOCABID ind = m_vocab.size() + 1; - m_vocab[word] = ind; - return ind; - } else { - return iter->second; - } - } - - void Insert(VOCABID id, const std::string &word) { - m_vocab[word] = id; - } - - void Save() { - OutputFileStream strme(m_path); - - typename Coll::const_iterator iter; - for (iter = m_vocab.begin(); iter != m_vocab.end(); ++iter) { - strme << iter->first << "\t" << iter->second << std::endl; - } - - strme.Close(); - } -}; - -} /* namespace Moses2 */ - diff --git a/moses/TranslationModel/ProbingPT/hash.cpp b/moses/TranslationModel/ProbingPT/hash.cpp deleted file mode 100644 index 47242e25d..000000000 --- a/moses/TranslationModel/ProbingPT/hash.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include <iostream> -#include "hash.hh" - -using namespace std; - -namespace Moses -{ - -uint64_t getHash(StringPiece text) -{ - std::size_t len = text.size(); - uint64_t key = util::MurmurHashNative(text.data(), len); - return key; -} - -std::vector<uint64_t> getVocabIDs(const StringPiece &textin) -{ - //Tokenize - std::vector<uint64_t> output; - - util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); - - while (itWord) { - StringPiece word = *itWord; - uint64_t id = 0; - - util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); - while (itFactor) { - StringPiece factor = *itFactor; - //cerr << "factor=" << factor << endl; - - id += getHash(factor); - itFactor++; - } - - output.push_back(id); - itWord++; - } - - return output; -} - -} - diff --git a/moses/TranslationModel/ProbingPT/hash.hh b/moses/TranslationModel/ProbingPT/hash.hh deleted file mode 100644 index f218ad9da..000000000 --- a/moses/TranslationModel/ProbingPT/hash.hh +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "util/string_piece.hh" -#include "util/murmur_hash.hh" -#include "util/string_piece.hh" //Tokenization and work with StringPiece -#include "util/tokenize_piece.hh" -#include <vector> - -namespace Moses -{ - -//Gets the MurmurmurHash for give string -uint64_t getHash(StringPiece text); - -std::vector<uint64_t> getVocabIDs(const StringPiece &textin); - -} diff --git a/moses/TranslationModel/ProbingPT/line_splitter.cpp b/moses/TranslationModel/ProbingPT/line_splitter.cpp deleted file mode 100644 index cb9e47fec..000000000 --- a/moses/TranslationModel/ProbingPT/line_splitter.cpp +++ /dev/null @@ -1,103 +0,0 @@ -#include "line_splitter.hh" - -namespace Moses -{ - -line_text splitLine(const StringPiece &textin, bool scfg) -{ - const char delim[] = "|||"; - line_text output; - - //Tokenize - util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim)); - //Get source phrase - output.source_phrase = Trim(*it); - //std::cerr << "output.source_phrase=" << output.source_phrase << "AAAA" << std::endl; - - //Get target_phrase - it++; - output.target_phrase = Trim(*it); - //std::cerr << "output.target_phrase=" << output.target_phrase << "AAAA" << std::endl; - - if (scfg) { - /* - std::cerr << "output.source_phrase=" << output.source_phrase << std::endl; - std::cerr << "output.target_phrase=" << output.target_phrase << std::endl; - reformatSCFG(output); - std::cerr << "output.source_phrase=" << output.source_phrase << std::endl; - std::cerr << "output.target_phrase=" << output.target_phrase << std::endl; - */ - } - - //Get probabilities - it++; - output.prob = Trim(*it); - //std::cerr << "output.prob=" << output.prob << "AAAA" << std::endl; - - //Get WordAllignment - it++; - if (it == util::TokenIter<util::MultiCharacter>::end()) return output; - output.word_align = Trim(*it); - //std::cerr << "output.word_align=" << output.word_align << "AAAA" << std::endl; - - //Get count - it++; - if (it == util::TokenIter<util::MultiCharacter>::end()) return output; - output.counts = Trim(*it); - //std::cerr << "output.counts=" << output.counts << "AAAA" << std::endl; - - //Get sparse_score - it++; - if (it == util::TokenIter<util::MultiCharacter>::end()) return output; - output.sparse_score = Trim(*it); - //std::cerr << "output.sparse_score=" << output.sparse_score << "AAAA" << std::endl; - - //Get property - it++; - if (it == util::TokenIter<util::MultiCharacter>::end()) return output; - output.property = Trim(*it); - //std::cerr << "output.property=" << output.property << "AAAA" << std::endl; - - return output; -} - -std::vector<unsigned char> splitWordAll1(const StringPiece &textin) -{ - const char delim[] = " "; - const char delim2[] = "-"; - std::vector<unsigned char> output; - - //Case with no word alignments. - if (textin.size() == 0) { - return output; - } - - //Split on space - util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim)); - - //For each int - while (it) { - //Split on dash (-) - util::TokenIter<util::MultiCharacter> itInner(*it, - util::MultiCharacter(delim2)); - - //Insert the two entries in the vector. User will read entry 0 and 1 to get the first, - //2 and 3 for second etc. Use unsigned char instead of int to save space, as - //word allignments are all very small numbers that fit in a single byte - output.push_back((unsigned char) (atoi(itInner->data()))); - itInner++; - output.push_back((unsigned char) (atoi(itInner->data()))); - it++; - } - - return output; - -} - -void reformatSCFG(line_text &output) -{ - -} - -} - diff --git a/moses/TranslationModel/ProbingPT/line_splitter.hh b/moses/TranslationModel/ProbingPT/line_splitter.hh deleted file mode 100644 index 01b86fc9b..000000000 --- a/moses/TranslationModel/ProbingPT/line_splitter.hh +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include "util/string_piece.hh" -#include "util/tokenize_piece.hh" -#include "util/file_piece.hh" -#include <vector> -#include <cstdlib> //atof -#include "util/string_piece.hh" //Tokenization and work with StringPiece -#include "util/tokenize_piece.hh" -#include <vector> - -namespace Moses -{ - -//Struct for holding processed line -struct line_text { - StringPiece source_phrase; - StringPiece target_phrase; - StringPiece prob; - StringPiece word_align; - StringPiece counts; - StringPiece sparse_score; - StringPiece property; - std::string property_to_be_binarized; -}; - -//Struct for holding processed line -struct target_text { - std::vector<unsigned int> target_phrase; - std::vector<float> prob; - std::vector<size_t> word_align_term; - std::vector<size_t> word_align_non_term; - std::vector<char> counts; - std::vector<char> sparse_score; - std::vector<char> property; - - /* - void Reset() - { - target_phrase.clear(); - prob.clear(); - word_all1.clear(); - counts.clear(); - sparse_score.clear(); - property.clear(); - } - */ -}; - -//Ask if it's better to have it receive a pointer to a line_text struct -line_text splitLine(const StringPiece &textin, bool scfg); -void reformatSCFG(line_text &output); - -std::vector<unsigned char> splitWordAll1(const StringPiece &textin); - -} - diff --git a/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp b/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp deleted file mode 100644 index f23f57d66..000000000 --- a/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "probing_hash_utils.hh" - -namespace Moses -{ - -//Read table from disk, return memory map location -char * readTable(const char * filename, size_t size) -{ - //Initial position of the file is the end of the file, thus we know the size - int fd; - char * map; - - fd = open(filename, O_RDONLY); - if (fd == -1) { - perror("Error opening file for reading"); - exit(EXIT_FAILURE); - } - - map = (char *) mmap(0, size, PROT_READ, MAP_SHARED, fd, 0); - - if (map == MAP_FAILED) { - close(fd); - perror("Error mmapping the file"); - exit(EXIT_FAILURE); - } - - return map; -} - -void serialize_table(char *mem, size_t size, const std::string &filename) -{ - std::ofstream os(filename.c_str(), std::ios::binary); - os.write((const char*) &mem[0], size); - os.close(); - -} - -uint64_t getKey(const uint64_t source_phrase[], size_t size) -{ - //TOO SLOW - //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size()); - uint64_t key = 0; - for (size_t i = 0; i < size; i++) { - key += (source_phrase[i] << i); - } - return key; -} - -} - diff --git a/moses/TranslationModel/ProbingPT/probing_hash_utils.hh b/moses/TranslationModel/ProbingPT/probing_hash_utils.hh deleted file mode 100644 index 998686b2e..000000000 --- a/moses/TranslationModel/ProbingPT/probing_hash_utils.hh +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include "util/probing_hash_table.hh" - -#include <sys/mman.h> -#include <boost/functional/hash.hpp> -#include <fcntl.h> -#include <fstream> - -namespace Moses -{ - -#define API_VERSION 15 - -//Hash table entry -struct Entry { - typedef uint64_t Key; - Key key; - - Key GetKey() const { - return key; - } - - void SetKey(Key to) { - key = to; - } - - uint64_t value; -}; - -#define NONE std::numeric_limits<uint64_t>::max() - -//Define table -typedef util::ProbingHashTable<Entry, boost::hash<uint64_t> > Table; - -void serialize_table(char *mem, size_t size, const std::string &filename); - -char * readTable(const char * filename, size_t size); - -uint64_t getKey(const uint64_t source_phrase[], size_t size); - -struct TargetPhraseInfo { - uint32_t alignTerm; - uint32_t alignNonTerm; - uint16_t numWords; - uint16_t propLength; - uint16_t filler; -}; - -} - diff --git a/moses/TranslationModel/ProbingPT/querying.cpp b/moses/TranslationModel/ProbingPT/querying.cpp deleted file mode 100644 index 10c35e361..000000000 --- a/moses/TranslationModel/ProbingPT/querying.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include "querying.hh" -#include "util/exception.hh" - -using namespace std; - -namespace Moses -{ - -QueryEngine::QueryEngine(const char * filepath) -{ - - //Create filepaths - std::string basepath(filepath); - std::string path_to_config = basepath + "/config"; - std::string path_to_hashtable = basepath + "/probing_hash.dat"; - std::string path_to_source_vocabid = basepath + "/source_vocabids"; - std::string alignPath = basepath + "/Alignments.dat"; - - if (!FileExists(path_to_config)) { - UTIL_THROW2("Binary table doesn't exist is didn't finish binarizing: " << path_to_config); - } - - ///Source phrase vocabids - read_map(source_vocabids, path_to_source_vocabid.c_str()); - - // alignments - read_alignments(alignPath); - - //Read config file - boost::unordered_map<std::string, std::string> keyValue; - - std::ifstream config(path_to_config.c_str()); - std::string line; - while (getline(config, line)) { - std::vector<std::string> toks = Tokenize(line, "\t"); - UTIL_THROW_IF2(toks.size() != 2, "Wrong config format:" << line); - keyValue[ toks[0] ] = toks[1]; - } - - bool found; - //Check API version: - int version; - found = Get(keyValue, "API_VERSION", version); - if (!found) { - std::cerr << "Old or corrupted version of ProbingPT. Please rebinarize your phrase tables." << std::endl; - } else if (version != API_VERSION) { - std::cerr << "The ProbingPT API has changed. " << version << "!=" - << API_VERSION << " Please rebinarize your phrase tables." << std::endl; - exit(EXIT_FAILURE); - } - - //Get tablesize. - int tablesize; - found = Get(keyValue, "uniq_entries", tablesize); - if (!found) { - std::cerr << "uniq_entries not found" << std::endl; - exit(EXIT_FAILURE); - } - - //Number of scores - found = Get(keyValue, "num_scores", num_scores); - if (!found) { - std::cerr << "num_scores not found" << std::endl; - exit(EXIT_FAILURE); - } - - //How may scores from lex reordering models - found = Get(keyValue, "num_lex_scores", num_lex_scores); - if (!found) { - std::cerr << "num_lex_scores not found" << std::endl; - exit(EXIT_FAILURE); - } - - // have the scores been log() and FloorScore()? - found = Get(keyValue, "log_prob", logProb); - if (!found) { - std::cerr << "logProb not found" << std::endl; - exit(EXIT_FAILURE); - } - - config.close(); - - //Read hashtable - table_filesize = Table::Size(tablesize, 1.2); - mem = readTable(path_to_hashtable.c_str(), table_filesize); - Table table_init(mem, table_filesize); - table = table_init; - - std::cerr << "Initialized successfully! " << std::endl; -} - -QueryEngine::~QueryEngine() -{ - //Clear mmap content from memory. - munmap(mem, table_filesize); - -} - -uint64_t QueryEngine::getKey(uint64_t source_phrase[], size_t size) const -{ - //TOO SLOW - //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size()); - return Moses::getKey(source_phrase, size); -} - -std::pair<bool, uint64_t> QueryEngine::query(uint64_t key) -{ - std::pair<bool, uint64_t> ret; - - const Entry * entry; - ret.first = table.Find(key, entry); - if (ret.first) { - ret.second = entry->value; - } - return ret; -} - -void QueryEngine::read_alignments(const std::string &alignPath) -{ - std::ifstream strm(alignPath.c_str()); - - string line; - while (getline(strm, line)) { - vector<string> toks = Tokenize(line, "\t "); - UTIL_THROW_IF2(toks.size() == 0, "Corrupt alignment file"); - - uint32_t alignInd = Scan<uint32_t>(toks[0]); - if (alignInd >= alignColl.size()) { - alignColl.resize(alignInd + 1); - } - - Alignments &aligns = alignColl[alignInd]; - for (size_t i = 1; i < toks.size(); ++i) { - size_t pos = Scan<size_t>(toks[i]); - aligns.push_back(pos); - } - } -} - -} - diff --git a/moses/TranslationModel/ProbingPT/querying.hh b/moses/TranslationModel/ProbingPT/querying.hh deleted file mode 100644 index 915bc4806..000000000 --- a/moses/TranslationModel/ProbingPT/querying.hh +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include <boost/unordered_map.hpp> -#include <sys/stat.h> //For finding size of file -#include "vocabid.hh" -#include <algorithm> //toLower -#include <deque> -#include "probing_hash_utils.hh" -#include "hash.hh" //Includes line splitter -#include "line_splitter.hh" -#include "moses//Util.h" - -namespace Moses -{ - -class QueryEngine -{ - std::map<uint64_t, std::string> source_vocabids; - - typedef std::vector<unsigned char> Alignments; - std::vector<Alignments> alignColl; - - Table table; - char *mem; //Memory for the table, necessary so that we can correctly destroy the object - - size_t table_filesize; - bool is_reordering; - - void read_alignments(const std::string &alignPath); - -public: - int num_scores; - int num_lex_scores; - bool logProb; - - QueryEngine(const char *); - ~QueryEngine(); - - std::pair<bool, uint64_t> query(uint64_t key); - - const std::map<uint64_t, std::string> &getSourceVocab() const { - return source_vocabids; - } - - const std::vector<Alignments> &getAlignments() const { - return alignColl; - } - - uint64_t getKey(uint64_t source_phrase[], size_t size) const; - - template<typename T> - inline bool Get(const boost::unordered_map<std::string, std::string> &keyValue, const std::string &sought, T &found) const { - boost::unordered_map<std::string, std::string>::const_iterator iter = keyValue.find(sought); - if (iter == keyValue.end()) { - return false; - } - - const std::string &foundStr = iter->second; - found = Scan<T>(foundStr); - return true; - } - -}; - -} - diff --git a/moses/TranslationModel/ProbingPT/storing.cpp b/moses/TranslationModel/ProbingPT/storing.cpp deleted file mode 100644 index baf6ae91e..000000000 --- a/moses/TranslationModel/ProbingPT/storing.cpp +++ /dev/null @@ -1,298 +0,0 @@ -#include <sys/stat.h> -#include <boost/foreach.hpp> -#include "line_splitter.hh" -#include "storing.hh" -#include "StoreTarget.h" -#include "StoreVocab.h" -#include "moses/Util.h" -#include "moses/InputFileStream.h" - -using namespace std; - -namespace Moses -{ - -/////////////////////////////////////////////////////////////////////// -void Node::Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos) -{ - if (pos < sourcePhrase.size()) { - uint64_t vocabId = sourcePhrase[pos]; - - Node *child; - Children::iterator iter = m_children.find(vocabId); - if (iter == m_children.end()) { - // New node. Write other children then discard them - BOOST_FOREACH(Children::value_type &valPair, m_children) { - Node &otherChild = valPair.second; - otherChild.Write(table); - } - m_children.clear(); - - // create new node - child = &m_children[vocabId]; - assert(!child->done); - child->key = key + (vocabId << pos); - } else { - child = &iter->second; - } - - child->Add(table, sourcePhrase, pos + 1); - } else { - // this node was written previously 'cos it has rules - done = true; - } -} - -void Node::Write(Table &table) -{ - //cerr << "START write " << done << " " << key << endl; - BOOST_FOREACH(Children::value_type &valPair, m_children) { - Node &child = valPair.second; - child.Write(table); - } - - if (!done) { - // save - Entry sourceEntry; - sourceEntry.value = NONE; - sourceEntry.key = key; - - //Put into table - table.Insert(sourceEntry); - } -} - -/////////////////////////////////////////////////////////////////////// -void createProbingPT(const std::string &phrasetable_path, - const std::string &basepath, int num_scores, int num_lex_scores, - bool log_prob, int max_cache_size, bool scfg) -{ - std::cerr << "Starting..." << std::endl; - - //Get basepath and create directory if missing - mkdir(basepath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); - - StoreTarget storeTarget(basepath); - - //Get uniq lines: - unsigned long uniq_entries = countUniqueSource(phrasetable_path); - - //Source phrase vocabids - StoreVocab<uint64_t> sourceVocab(basepath + "/source_vocabids"); - - //Read the file - util::FilePiece filein(phrasetable_path.c_str()); - - //Init the probing hash table - size_t size = Table::Size(uniq_entries, 1.2); - char * mem = new char[size]; - memset(mem, 0, size); - Table sourceEntries(mem, size); - - std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> cache; - float totalSourceCount = 0; - - //Keep track of the size of each group of target phrases - size_t line_num = 0; - - //Read everything and processs - std::string prevSource; - - Node sourcePhrases; - sourcePhrases.done = true; - sourcePhrases.key = 0; - - while (true) { - try { - //Process line read - line_text line; - line = splitLine(filein.ReadLine(), scfg); - //cerr << "line=" << line.source_phrase << endl; - - ++line_num; - if (line_num % 1000000 == 0) { - std::cerr << line_num << " " << std::flush; - } - - //Add source phrases to vocabularyIDs - add_to_map(sourceVocab, line.source_phrase); - - if (prevSource.empty()) { - // 1st line - prevSource = line.source_phrase.as_string(); - storeTarget.Append(line, log_prob, scfg); - } else if (prevSource == line.source_phrase) { - //If we still have the same line, just append to it: - storeTarget.Append(line, log_prob, scfg); - } else { - assert(prevSource != line.source_phrase); - - //Create a new entry even - - // save - uint64_t targetInd = storeTarget.Save(); - - // next line - storeTarget.Append(line, log_prob, scfg); - - //Create an entry for the previous source phrase: - Entry sourceEntry; - sourceEntry.value = targetInd; - //The key is the sum of hashes of individual words bitshifted by their position in the phrase. - //Probably not entirerly correct, but fast and seems to work fine in practise. - std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource); - if (scfg) { - // storing prefixes? - sourcePhrases.Add(sourceEntries, vocabid_source); - } - sourceEntry.key = getKey(vocabid_source); - - /* - cerr << "prevSource=" << prevSource << flush - << " vocabids=" << Debug(vocabid_source) << flush - << " key=" << sourceEntry.key << endl; - */ - //Put into table - sourceEntries.Insert(sourceEntry); - - // update cache - CURRENT source phrase, not prev - if (max_cache_size) { - std::string countStr = line.counts.as_string(); - countStr = Trim(countStr); - if (!countStr.empty()) { - std::vector<float> toks = Tokenize<float>(countStr); - //cerr << "CACHE:" << line.source_phrase << " " << countStr << " " << toks[1] << endl; - - if (toks.size() >= 2) { - totalSourceCount += toks[1]; - - // compute key for CURRENT source - std::vector<uint64_t> currVocabidSource = getVocabIDs(line.source_phrase.as_string()); - uint64_t currKey = getKey(currVocabidSource); - - CacheItem *item = new CacheItem( - Trim(line.source_phrase.as_string()), - currKey, - toks[1]); - cache.push(item); - - if (max_cache_size > 0 && cache.size() > max_cache_size) { - cache.pop(); - } - } - } - } - - //Set prevLine - prevSource = line.source_phrase.as_string(); - } - - } catch (util::EndOfFileException e) { - std::cerr - << "Reading phrase table finished, writing remaining files to disk." - << std::endl; - - //After the final entry is constructed we need to add it to the phrase_table - //Create an entry for the previous source phrase: - uint64_t targetInd = storeTarget.Save(); - - Entry sourceEntry; - sourceEntry.value = targetInd; - - //The key is the sum of hashes of individual words. Probably not entirerly correct, but fast - std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource); - sourceEntry.key = getKey(vocabid_source); - - //Put into table - sourceEntries.Insert(sourceEntry); - - break; - } - } - - sourcePhrases.Write(sourceEntries); - - storeTarget.SaveAlignment(); - - serialize_table(mem, size, (basepath + "/probing_hash.dat")); - - sourceVocab.Save(); - - serialize_cache(cache, (basepath + "/cache"), totalSourceCount); - - delete[] mem; - - //Write configfile - std::ofstream configfile; - configfile.open((basepath + "/config").c_str()); - configfile << "API_VERSION\t" << API_VERSION << '\n'; - configfile << "uniq_entries\t" << uniq_entries << '\n'; - configfile << "num_scores\t" << num_scores << '\n'; - configfile << "num_lex_scores\t" << num_lex_scores << '\n'; - configfile << "log_prob\t" << log_prob << '\n'; - configfile.close(); -} - -size_t countUniqueSource(const std::string &path) -{ - size_t ret = 0; - InputFileStream strme(path); - - std::string line, prevSource; - while (std::getline(strme, line)) { - std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||"); - assert(toks.size() != 0); - - if (prevSource != toks[0]) { - prevSource = toks[0]; - ++ret; - } - } - - return ret; -} - -void serialize_cache( - std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, - const std::string &path, float totalSourceCount) -{ - std::vector<const CacheItem*> vec(cache.size()); - - size_t ind = cache.size() - 1; - while (!cache.empty()) { - const CacheItem *item = cache.top(); - vec[ind] = item; - cache.pop(); - --ind; - } - - std::ofstream os(path.c_str()); - - os << totalSourceCount << std::endl; - for (size_t i = 0; i < vec.size(); ++i) { - const CacheItem *item = vec[i]; - os << item->count << "\t" << item->sourceKey << "\t" << item->source << std::endl; - delete item; - } - - os.close(); -} - -uint64_t getKey(const std::vector<uint64_t> &vocabid_source) -{ - return getKey(vocabid_source.data(), vocabid_source.size()); -} - -std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos) -{ - assert(endPos < vocabid_source.size()); - - std::vector<uint64_t> ret(endPos + 1); - for (size_t i = 0; i <= endPos; ++i) { - ret[i] = vocabid_source[i]; - } - return ret; -} - -} - diff --git a/moses/TranslationModel/ProbingPT/storing.hh b/moses/TranslationModel/ProbingPT/storing.hh deleted file mode 100644 index 994067515..000000000 --- a/moses/TranslationModel/ProbingPT/storing.hh +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include <boost/unordered_set.hpp> -#include <boost/unordered_map.hpp> -#include <cstdio> -#include <sstream> -#include <fstream> -#include <iostream> -#include <string> -#include <queue> -#include <sys/stat.h> //mkdir - -#include "hash.hh" //Includes line_splitter -#include "probing_hash_utils.hh" - -#include "util/file_piece.hh" -#include "util/file.hh" -#include "vocabid.hh" - -namespace Moses -{ -typedef std::vector<uint64_t> SourcePhrase; - - -class Node -{ - typedef boost::unordered_map<uint64_t, Node> Children; - Children m_children; - -public: - uint64_t key; - bool done; - - Node() - :done(false) - {} - - void Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos = 0); - void Write(Table &table); -}; - - -void createProbingPT(const std::string &phrasetable_path, - const std::string &basepath, int num_scores, int num_lex_scores, - bool log_prob, int max_cache_size, bool scfg); -uint64_t getKey(const std::vector<uint64_t> &source_phrase); - -std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos); - -template<typename T> -std::string Debug(const std::vector<T> &vec) -{ - std::stringstream strm; - for (size_t i = 0; i < vec.size(); ++i) { - strm << vec[i] << " "; - } - return strm.str(); -} - -size_t countUniqueSource(const std::string &path); - -class CacheItem -{ -public: - std::string source; - uint64_t sourceKey; - float count; - CacheItem(const std::string &vSource, uint64_t vSourceKey, float vCount) - :source(vSource) - ,sourceKey(vSourceKey) - ,count(vCount) { - } - - bool operator<(const CacheItem &other) const { - return count > other.count; - } -}; - -class CacheItemOrderer -{ -public: - bool operator()(const CacheItem* a, const CacheItem* b) const { - return (*a) < (*b); - } -}; - -void serialize_cache( - std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, - const std::string &path, float totalSourceCount); - -} - diff --git a/moses/TranslationModel/ProbingPT/vocabid.cpp b/moses/TranslationModel/ProbingPT/vocabid.cpp deleted file mode 100644 index d6f442323..000000000 --- a/moses/TranslationModel/ProbingPT/vocabid.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include <boost/foreach.hpp> -#include "vocabid.hh" -#include "StoreVocab.h" -#include "moses/Util.h" - -namespace Moses -{ - -void add_to_map(StoreVocab<uint64_t> &sourceVocab, - const StringPiece &textin) -{ - //Tokenize - util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); - - while (itWord) { - StringPiece word = *itWord; - - util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); - while (itFactor) { - StringPiece factor = *itFactor; - - sourceVocab.Insert(getHash(factor), factor.as_string()); - itFactor++; - } - itWord++; - } -} - -void serialize_map(const std::map<uint64_t, std::string> &karta, - const std::string &filename) -{ - std::ofstream os(filename.c_str()); - - std::map<uint64_t, std::string>::const_iterator iter; - for (iter = karta.begin(); iter != karta.end(); ++iter) { - os << iter->first << '\t' << iter->second << std::endl; - } - - os.close(); -} - -void read_map(std::map<uint64_t, std::string> &karta, const char* filename) -{ - std::ifstream is(filename); - - std::string line; - while (getline(is, line)) { - std::vector<std::string> toks = Tokenize(line, "\t"); - assert(toks.size() == 2); - uint64_t ind = Scan<uint64_t>(toks[1]); - karta[ind] = toks[0]; - } - - //Close the stream after we are done. - is.close(); -} - -} - diff --git a/moses/TranslationModel/ProbingPT/vocabid.hh b/moses/TranslationModel/ProbingPT/vocabid.hh deleted file mode 100644 index 7e1390874..000000000 --- a/moses/TranslationModel/ProbingPT/vocabid.hh +++ /dev/null @@ -1,29 +0,0 @@ -//Serialization -#include <boost/serialization/serialization.hpp> -#include <boost/serialization/map.hpp> -#include <boost/archive/text_iarchive.hpp> -#include <boost/archive/text_oarchive.hpp> -#include <fstream> -#include <iostream> -#include <vector> - -#include <map> //Container -#include "hash.hh" //Hash of elements - -#include "util/string_piece.hh" //Tokenization and work with StringPiece -#include "util/tokenize_piece.hh" - -namespace Moses -{ -template<typename VOCABID> -class StoreVocab; - -void add_to_map(StoreVocab<uint64_t> &sourceVocab, - const StringPiece &textin); - -void serialize_map(const std::map<uint64_t, std::string> &karta, - const std::string &filename); - -void read_map(std::map<uint64_t, std::string> &karta, const char* filename); - -} diff --git a/moses/TranslationModel/UG/Makefile b/moses/TranslationModel/UG/Makefile index 213f87e0c..0006af638 100644 --- a/moses/TranslationModel/UG/Makefile +++ b/moses/TranslationModel/UG/Makefile @@ -35,7 +35,7 @@ CXXFLAGS += -DMAX_NUM_FACTORS=4 CXXFLAGS += -DKENLM_MAX_ORDER=5 CXXFLAGS += -DWITH_THREADS CXXFLAGS += -DNO_MOSES -CXXFLAGS += -DMMT +# CXXFLAGS += -DMMT CXXFLAGS += -I$(dir ${MOSES_ROOT})mmt-only CXXFLAGS += -I${MOSES_ROOT} -I. -I${MOSES_ROOT}/opt/include @@ -47,7 +47,7 @@ else ifeq ($(variant),syntax) CXXFLAGS += -fsyntax-only endif -LDFLAGS = -L${MOSES_ROOT}/opt/lib64 -L./lib/ +LDFLAGS = -L${MOSES_ROOT}/opt/lib -L./lib/ # WDIR = build/$(variant)/${HOSTTYPE}/${KERNEL} WDIR = build/$(variant) @@ -73,7 +73,7 @@ DYNAMIC_LIBS = pthread #DYNAMIC_LIBS += tcmalloc LIBS = -Wl,-B$(link) -LIBS += -L${MOSES_ROOT}/opt/lib64 ${BOOSTLIBS} +LIBS += -L${MOSES_ROOT}/opt/lib ${BOOSTLIBS} LIBS += $(addprefix -l,${STATIC_LIBS}) LIBS += -Wl,-Bdynamic LIBS += $(addprefix -l,${DYNAMIC_LIBS}) diff --git a/moses/TranslationModel/UG/check-coverage5.cc b/moses/TranslationModel/UG/check-coverage5.cc new file mode 100644 index 000000000..fe9479136 --- /dev/null +++ b/moses/TranslationModel/UG/check-coverage5.cc @@ -0,0 +1,129 @@ +// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*- + +// read a text from stdin, report percentage of n-grams covered + +#include <boost/foreach.hpp> +#include <boost/format.hpp> +#include <boost/tokenizer.hpp> +#include <boost/shared_ptr.hpp> +#include <algorithm> +#include <iostream> +#include "mm/ug_bitext.h" +#include "generic/file_io/ug_stream.h" +#include <string> +#include <sstream> +#include "mm/ug_bitext_sampler.h" + +#include <boost/program_options.hpp> +#include <boost/math/distributions/binomial.hpp> + +// #include "LSA.h" + +namespace po=boost::program_options; +using namespace Moses; +using namespace sapt; +using namespace std; +using namespace boost; + +typedef sapt::L2R_Token<sapt::SimpleWordId> Token; +typedef mmTtrack<Token> ttrack_t; + +size_t ngram_size; +size_t verbosity; +string bname; +vector<string> ifiles; + +void interpret_args(int ac, char* av[]); + + +void +dump(mmTSA<Token>::tree_iterator& m, TokenIndex& V) +{ + if (m.size()) cout << m.str(NULL) << endl; + if (m.size()) cout << m.str(&V) << endl; + if (m.down()) + { + do { dump(m, V); } while (m.over()); + m.up(); + } +} + +int +main(int argc, char* argv[]) +{ + interpret_args(argc,argv); + TokenIndex V; + V.open(bname+".tdx"); V.setDynamic(true); V.iniReverseIndex(); + boost::shared_ptr<mmTtrack<Token> > T(new mmTtrack<Token>); + T->open(bname+".mct"); + mmTSA<Token> I; I.open(bname+".sfa", T); + + string line; + BOOST_FOREACH(string const& file, ifiles) + { + size_t total_ngrams=0; + float matched_ngrams=0; + ifstream in(file.c_str()); + while(getline(in,line)) + { + // cout << line << endl; + vector<id_type> snt; + V.fillIdSeq(line,snt); + if (snt.size() < ngram_size) continue; + total_ngrams += snt.size() - ngram_size + 1; + for (size_t i = 0; i + ngram_size <= snt.size(); ++i) + // for (size_t i = 0; i < snt.size(); ++i) + { + mmTSA<Token>::tree_iterator m(&I); + size_t stop = min(snt.size(), i+ngram_size); + size_t k = i; + while (k < stop && m.extend(snt[k])) ++k; + if (verbosity) cout << i << " " << k-i << " " << m.str(&V) << endl; + if (k - i == ngram_size) + ++matched_ngrams; + } + } + printf ("%5.1f%% matched %zu-grams (%.0f/%zu): %s\n", + (100 * matched_ngrams / total_ngrams), ngram_size, + matched_ngrams, total_ngrams, file.c_str()); + } +} + +void +interpret_args(int ac, char* av[]) +{ + po::variables_map vm; + po::options_description o("Options"); + o.add_options() + + ("help,h", "print this message") + ("ngram-size,n", po::value<size_t>(&ngram_size)->default_value(5), + "sample size") + ("verbose,v", po::value<size_t>(&verbosity)->default_value(0), + "verbosity") + ; + + po::options_description h("Hidden Options"); + h.add_options() + ("bname", po::value<string>(&bname), "base name of corpus") + ("ifiles", po::value<vector<string> >(&ifiles), "input files") + ; + + h.add(o); + po::positional_options_description a; + a.add("bname",1); + a.add("ifiles",-1); + + po::store(po::command_line_parser(ac,av) + .options(h) + .positional(a) + .run(),vm); + po::notify(vm); + if (vm.count("help")) + { + std::cout << "\nusage:\n\t" << av[0] + << " [options] <model file stem>" << std::endl; + std::cout << o << std::endl; + exit(0); + } +} diff --git a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc index 08e123782..4e8095f61 100644 --- a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc +++ b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc @@ -59,8 +59,8 @@ namespace sapt comp.base = reinterpret_cast<char const*>(endIdx); if (!unkToken.empty()) { - Entry const* bla = lower_bound(startIdx,endIdx,unkToken.c_str(),comp); - unkId = ((bla < endIdx && unkToken == comp.base+bla->offset) + Entry const* bla = lower_bound(startIdx,endIdx,unkToken.c_str(),comp); + unkId = ((bla < endIdx && unkToken == comp.base+bla->offset) ? bla->id : numTokens); } @@ -98,10 +98,10 @@ namespace sapt { if (startIdx != endIdx) { - Entry const* bla = lower_bound(startIdx,endIdx,p,comp); - if (bla != endIdx && !strcmp(comp.base+bla->offset,p)) - return bla->id; - if (!dynamic) return unkId; + Entry const* bla = lower_bound(startIdx,endIdx,p,comp); + if (bla != endIdx && !strcmp(comp.base+bla->offset,p)) + return bla->id; + if (!dynamic) return unkId; } else if (!dynamic) return strcmp(p,"NULL") && unkId; @@ -155,10 +155,10 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - // Someone else (multi-threading!) may have created the - // reverse index in the meantime, so let's check again - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + // Someone else (multi-threading!) may have created the + // reverse index in the meantime, so let's check again + if (!ridx.size()) ridx = reverseIndex(); } if (id < ridx.size()) return ridx[id]; @@ -175,8 +175,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } } @@ -187,8 +187,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } if (id < ridx.size()) return ridx[id]; @@ -204,8 +204,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } ostringstream buf; for (size_t i = 0; i < v.size(); i++) @@ -219,8 +219,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } ostringstream buf; for (size_t i = 0; i < v.size(); i++) @@ -234,8 +234,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } ostringstream buf; if (start < stop) @@ -251,8 +251,8 @@ namespace sapt { if (!ridx.size()) { - boost::lock_guard<boost::mutex> lk(*this->lock); - if (!ridx.size()) ridx = reverseIndex(); + boost::lock_guard<boost::mutex> lk(*this->lock); + if (!ridx.size()) ridx = reverseIndex(); } ostringstream buf; if (start < stop) diff --git a/moses/TranslationModel/UG/mm/ug_typedefs.h b/moses/TranslationModel/UG/mm/ug_typedefs.h index fc9d7faef..4815846a2 100644 --- a/moses/TranslationModel/UG/mm/ug_typedefs.h +++ b/moses/TranslationModel/UG/mm/ug_typedefs.h @@ -36,7 +36,7 @@ namespace sapt #ifndef SPTR #define SPTR boost::shared_ptr #endif -#define iptr boost::intrusive_ptr +#define boost_iptr boost::intrusive_ptr #define scoptr boost::scoped_ptr #define rcast reinterpret_cast #endif diff --git a/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc b/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc index 613e46360..ff9f1c722 100644 --- a/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc +++ b/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc @@ -76,7 +76,7 @@ int main(int argc, char* argv[]) { typedef vector<PhrasePair<Token> > pplist_t; interpret_args(argc, argv); - iptr<mmbitext> Bptr(new mmbitext); + boost_iptr<mmbitext> Bptr(new mmbitext); mmbitext& B = *Bptr;// static_cast<mmbitext*>(Bptr.get()); B.open(bname, L1, L2); B.V1->setDynamic(true); diff --git a/moses/parameters/AllOptions.cpp b/moses/parameters/AllOptions.cpp index 868b8e4fb..019e1b0c7 100644 --- a/moses/parameters/AllOptions.cpp +++ b/moses/parameters/AllOptions.cpp @@ -101,6 +101,11 @@ namespace Moses if (!syntax.update(param)) return false; return sanity_check(); } +#else + bool + AllOptions:: + update(std::map<std::string,xmlrpc_c::value>const& param) + {} #endif bool diff --git a/moses/parameters/CubePruningOptions.cpp b/moses/parameters/CubePruningOptions.cpp index 793ae2db6..847a421f8 100644 --- a/moses/parameters/CubePruningOptions.cpp +++ b/moses/parameters/CubePruningOptions.cpp @@ -72,6 +72,11 @@ namespace Moses return true; } +#else + bool + CubePruningOptions:: + update(std::map<std::string,xmlrpc_c::value>const& params) + {} #endif diff --git a/moses/parameters/InputOptions.cpp b/moses/parameters/InputOptions.cpp index dbdb2f60e..55bc49885 100644 --- a/moses/parameters/InputOptions.cpp +++ b/moses/parameters/InputOptions.cpp @@ -95,6 +95,11 @@ namespace Moses { xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second)); return true; } +#else + bool + InputOptions:: + update(std::map<std::string,xmlrpc_c::value>const& param) + {} #endif } diff --git a/moses/parameters/NBestOptions.cpp b/moses/parameters/NBestOptions.cpp index 3000f49df..afae0ed6c 100644 --- a/moses/parameters/NBestOptions.cpp +++ b/moses/parameters/NBestOptions.cpp @@ -62,6 +62,11 @@ update(std::map<std::string,xmlrpc_c::value>const& param) enabled = (nbest_size > 0); return true; } +#else +bool +NBestOptions:: +update(std::map<std::string,xmlrpc_c::value>const& param) +{} #endif diff --git a/moses/parameters/OOVHandlingOptions.cpp b/moses/parameters/OOVHandlingOptions.cpp index 154074664..a55026b98 100644 --- a/moses/parameters/OOVHandlingOptions.cpp +++ b/moses/parameters/OOVHandlingOptions.cpp @@ -43,6 +43,11 @@ namespace Moses { // xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second)); return true; } +#else + bool + OOVHandlingOptions:: + update(std::map<std::string,xmlrpc_c::value>const& param) + {} #endif } diff --git a/moses/parameters/OptionsBaseClass.cpp b/moses/parameters/OptionsBaseClass.cpp index a19aaf7a6..324359e93 100644 --- a/moses/parameters/OptionsBaseClass.cpp +++ b/moses/parameters/OptionsBaseClass.cpp @@ -24,5 +24,10 @@ namespace Moses { if (m == param.end()) return dfltval; return Scan<bool>(xmlrpc_c::value_string(m->second)); } +#else + bool + check(std::map<std::string, xmlrpc_c::value> const& param, + std::string const key, bool dfltval) + {} #endif } diff --git a/moses/parameters/ReportingOptions.cpp b/moses/parameters/ReportingOptions.cpp index 210950a3c..fe56a7356 100644 --- a/moses/parameters/ReportingOptions.cpp +++ b/moses/parameters/ReportingOptions.cpp @@ -128,11 +128,17 @@ namespace Moses { for (size_t i = 0; i < MAX_NUM_FACTORS; ++i) factor_order.push_back(i); } + + m = param.find("no-ReportSegmentation"); + if (m == param.end() || !Scan<bool>(xmlrpc_c::value_string(m->second))) { + + // If we are reporting alignment info, turn on ReportSegmentation, unless XML request explicitly says not to + m = param.find("align"); + if (m != param.end() && Scan<bool>(xmlrpc_c::value_string(m->second))) + ReportSegmentation = 1; - m = param.find("align"); - if (m != param.end() && Scan<bool>(xmlrpc_c::value_string(m->second))) - ReportSegmentation = 1; - + } + PrintAlignmentInfo = check(param,"word-align",PrintAlignmentInfo); m = param.find("factor-delimiter"); diff --git a/moses/parameters/SearchOptions.cpp b/moses/parameters/SearchOptions.cpp index 958569e94..35028b8fb 100644 --- a/moses/parameters/SearchOptions.cpp +++ b/moses/parameters/SearchOptions.cpp @@ -102,6 +102,11 @@ namespace Moses return true; } +#else + bool + SearchOptions:: + update(std::map<std::string,xmlrpc_c::value>const& params) + {} #endif } diff --git a/moses/parameters/SyntaxOptions.cpp b/moses/parameters/SyntaxOptions.cpp index f76c187ec..e977dd4c0 100644 --- a/moses/parameters/SyntaxOptions.cpp +++ b/moses/parameters/SyntaxOptions.cpp @@ -77,6 +77,11 @@ namespace Moses { // xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second)); return true; } +#else + bool + SyntaxOptions:: + update(std::map<std::string,xmlrpc_c::value>const& param) + {} #endif } diff --git a/moses/server/TranslationRequest.cpp b/moses/server/TranslationRequest.cpp index 7b3aa8c06..d2118ad8f 100644 --- a/moses/server/TranslationRequest.cpp +++ b/moses/server/TranslationRequest.cpp @@ -24,6 +24,9 @@ using Moses::FValue; using Moses::PhraseDictionaryMultiModel; using Moses::FindPhraseDictionary; using Moses::Sentence; +using Moses::TokenizeMultiCharSeparator; +using Moses::FeatureFunction; +using Moses::Scan; boost::shared_ptr<TranslationRequest> TranslationRequest:: @@ -82,13 +85,14 @@ void TranslationRequest:: add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const { - // if (!m_withAlignInfo) return; - if (!options()->output.ReportSegmentation) return; + if (!m_withAlignInfo) return; + // if (!options()->output.ReportSegmentation) return; Range const& trg = h.GetCurrTargetWordsRange(); Range const& src = h.GetCurrSourceWordsRange(); std::map<std::string, xmlrpc_c::value> pAlnInfo; pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos()); + pAlnInfo["tgt-end"] = xmlrpc_c::value_int(trg.GetEndPos()); pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos()); pAlnInfo["src-end"] = xmlrpc_c::value_int(src.GetEndPos()); aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo)); @@ -211,6 +215,9 @@ insertTranslationOptions(Moses::Manager& manager, for (size_t j = 0; j < scores.size(); ++j) scoresXml.push_back(xmlrpc_c::value_double(scores[j])); toptXml["scores"] = xmlrpc_c::value_array(scoresXml); + ostringstream buf; + topt->GetScoreBreakdown().OutputAllFeatureScores(buf, true); + toptXml["labelledScores"] = PackScores(topt->GetScoreBreakdown()); toptsXml.push_back(xmlrpc_c::value_struct(toptXml)); } } @@ -312,6 +319,106 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params) m_context.reset(new std::vector<std::string>(1,context)); } + si = params.find("context-scope"); + if (si != params.end()) + { + + string context = xmlrpc_c::value_string(si->second); + + string groupSeparator("Moses::ContextScope::GroupSeparator"); + string recordSeparator("Moses::ContextScope::RecordSeparator"); + + // Here, we assume that any XML-RPC value + // associated with the key "context-scope" + // has the following format: + // + // FeatureFunctionName followed by recordSeparator + // followed by the value of interest + // followed by groupSeparator + // + // In the following code, the value of interest will be stored + // in contextScope under the key FeatureFunctionName, + // where FeatureFunctionName is the actual name of the feature function + + boost::shared_ptr<Moses::ContextScope> contextScope = GetScope(); + + BOOST_FOREACH(string group, TokenizeMultiCharSeparator(context, groupSeparator)) { + + vector<string> record = TokenizeMultiCharSeparator(group, recordSeparator); + + // Use the feature function whose name is record[0] as a key + FeatureFunction& ff = Moses::FeatureFunction::FindFeatureFunction(record[0]); + void const* key = static_cast<void const*>(&ff); + + // Store (in the context scope) record[1] as the value associated with that key + boost::shared_ptr<string> value = contextScope->get<string>(key,true); + value->replace(value->begin(), value->end(), record[1]); + + } + } + + // Report alignment info if Moses config says to or if XML request says to + m_withAlignInfo = options()->output.ReportSegmentation || check(params, "align"); + + // Report word alignment info if Moses config says to or if XML request says to + m_withWordAlignInfo = options()->output.PrintAlignmentInfo || check(params, "word-align"); + + si = params.find("weights"); + if (si != params.end()) + { + + boost::unordered_map<string, FeatureFunction*> map; + { + const vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions(); + BOOST_FOREACH(FeatureFunction* const& ff, ffs) { + map[ff->GetScoreProducerDescription()] = ff; + } + } + + string allValues = xmlrpc_c::value_string(si->second); + + BOOST_FOREACH(string values, TokenizeMultiCharSeparator(allValues, "\t")) { + + vector<string> record = TokenizeMultiCharSeparator(values, "="); + + if (record.size() == 2) { + string featureName = record[0]; + string featureWeights = record[1]; + + boost::unordered_map<string, FeatureFunction*>::iterator ffi = map.find(featureName); + + if (ffi != map.end()) { + FeatureFunction* ff = ffi->second; + + size_t prevNumWeights = ff->GetNumScoreComponents(); + + vector<float> ffWeights; + BOOST_FOREACH(string weight, TokenizeMultiCharSeparator(featureWeights, " ")) { + ffWeights.push_back(Scan<float>(weight)); + } + + if (ffWeights.size() == ff->GetNumScoreComponents()) { + + // XXX: This is NOT thread-safe + Moses::StaticData::InstanceNonConst().SetWeights(ff, ffWeights); + VERBOSE(1, "WARNING: THIS IS NOT THREAD-SAFE!\tUpdating weights for " << featureName << " to " << featureWeights << "\n"); + + } else { + TRACE_ERR("ERROR: Unable to update weights for " << featureName << " because " << ff->GetNumScoreComponents() << " weights are required but only " << ffWeights.size() << " were provided\n"); + } + + } else { + TRACE_ERR("ERROR: No FeatureFunction with name " << featureName << ", no weight update\n"); + } + + } else { + TRACE_ERR("WARNING: XML-RPC weights update was improperly formatted:\t" << values << "\n"); + } + + } + + } + // // biased sampling for suffix-array-based sampling phrase table? // if ((si = params.find("bias")) != params.end()) @@ -366,8 +473,8 @@ pack_hypothesis(const Moses::Manager& manager, << std::endl); dest[key] = xmlrpc_c::value_string(target.str()); - // if (m_withAlignInfo) { - if (options()->output.ReportSegmentation) { + if (m_withAlignInfo) { + // if (options()->output.ReportSegmentation) { // phrase alignment, if requested vector<xmlrpc_c::value> p_aln; @@ -376,8 +483,8 @@ pack_hypothesis(const Moses::Manager& manager, dest["align"] = xmlrpc_c::value_array(p_aln); } - // if (m_withWordAlignInfo) { - if (options()->output.PrintAlignmentInfo) { + if (m_withWordAlignInfo) { + //if (options()->output.PrintAlignmentInfo) { // word alignment, if requested vector<xmlrpc_c::value> w_aln; BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) diff --git a/moses/server/TranslationRequest.h b/moses/server/TranslationRequest.h index 2554e5544..3463c72c8 100644 --- a/moses/server/TranslationRequest.h +++ b/moses/server/TranslationRequest.h @@ -38,8 +38,8 @@ TranslationRequest : public virtual Moses::TranslationTask Translator* m_translator; std::string m_source_string, m_target_string; - // bool m_withAlignInfo; - // bool m_withWordAlignInfo; + bool m_withAlignInfo; + bool m_withWordAlignInfo; bool m_withGraphInfo; bool m_withTopts; bool m_withScoreBreakdown; |