diff options
author | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 19:34:54 +0300 |
---|---|---|
committer | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 19:34:54 +0300 |
commit | 064b01c0545763f67fbb266fa038c869dc905c94 (patch) | |
tree | 9e4a1227902be1f9130c7b1460f0c32b48ede191 /moses | |
parent | 17ddbc7d38f9ca9bbfa2cab467402b3e8374afa6 (diff) |
Code reorganization and cleanup around Lexical Reordering Round 2.
Hope it compiles this time.
Diffstat (limited to 'moses')
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReordering.cpp | 166 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReordering.h | 167 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReorderingState.cpp | 571 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReorderingState.h | 475 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/SparseReordering.cpp | 17 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/SparseReordering.h | 10 | ||||
-rw-r--r-- | moses/WordsBitmap.h | 12 |
7 files changed, 715 insertions, 703 deletions
diff --git a/moses/FF/LexicalReordering/LexicalReordering.cpp b/moses/FF/LexicalReordering/LexicalReordering.cpp index 32693984e..533f2c485 100644 --- a/moses/FF/LexicalReordering/LexicalReordering.cpp +++ b/moses/FF/LexicalReordering/LexicalReordering.cpp @@ -1,135 +1,133 @@ #include <sstream> #include <boost/algorithm/string/predicate.hpp> +#include <boost/foreach.hpp> #include "moses/FF/FFState.h" #include "LexicalReordering.h" #include "LexicalReorderingState.h" #include "moses/StaticData.h" +#include "moses/Util.h" using namespace std; using namespace boost::algorithm; namespace Moses { - LexicalReordering:: - LexicalReordering(const std::string &line) + LexicalReordering(const std::string &line) : StatefulFeatureFunction(line) { - VERBOSE(1, "Initializing LexicalReordering.." << std::endl); - + std::cerr << "Initializing Lexical Reordering Feature.." << std::endl; + map<string,string> sparseArgs; m_haveDefaultScores = false; - for (size_t i = 0; i < m_args.size(); ++i) { - const vector<string> &args = m_args[i]; - - if (args[0] == "type") { - m_configuration.reset(new LexicalReorderingConfiguration(args[1])); - m_configuration->SetScoreProducer(this); - m_modelTypeString = m_configuration->GetModelString(); - } else if (args[0] == "input-factor") { - m_factorsF =Tokenize<FactorType>(args[1]); - } else if (args[0] == "output-factor") { - m_factorsE =Tokenize<FactorType>(args[1]); - } else if (args[0] == "path") { - m_filePath = args[1]; - } else if (starts_with(args[0], "sparse-")) { - sparseArgs[args[0].substr(7)] = args[1]; - } else if (args[0] == "default-scores") { - vector<string> tokens = Tokenize(args[1],","); - for(size_t i=0; i<tokens.size(); i++) { - m_defaultScores.push_back( TransformScore( Scan<float>(tokens[i]) ) ); - } - m_haveDefaultScores = true; - } else { - UTIL_THROW(util::Exception,"Unknown argument " + args[0]); - } - } - - switch(m_configuration->GetCondition()) { - case LexicalReorderingConfiguration::FE: - case LexicalReorderingConfiguration::E: - if(m_factorsE.empty()) { - UTIL_THROW(util::Exception, - "TL factor mask for lexical reordering is unexpectedly empty"); - } - if(m_configuration->GetCondition() == LexicalReorderingConfiguration::E) - break; // else fall through - case LexicalReorderingConfiguration::F: - if(m_factorsF.empty()) { - UTIL_THROW(util::Exception, - "SL factor mask for lexical reordering is unexpectedly empty"); + for (size_t i = 0; i < m_args.size(); ++i) + { + const vector<string> &args = m_args[i]; + + if (args[0] == "type") + { + m_configuration.reset(new LRModel(args[1])); + m_configuration->SetScoreProducer(this); + m_modelTypeString = m_configuration->GetModelString(); + } + else if (args[0] == "input-factor") + m_factorsF =Tokenize<FactorType>(args[1]); + else if (args[0] == "output-factor") + m_factorsE =Tokenize<FactorType>(args[1]); + else if (args[0] == "path") + m_filePath = args[1]; + else if (starts_with(args[0], "sparse-")) + sparseArgs[args[0].substr(7)] = args[1]; + else if (args[0] == "default-scores") + { + vector<string> tokens = Tokenize(args[1],","); + for(size_t i=0; i<tokens.size(); i++) + m_defaultScores.push_back( TransformScore( Scan<float>(tokens[i]))); + m_haveDefaultScores = true; + } + else UTIL_THROW2("Unknown argument " + args[0]); } - break; - default: - UTIL_THROW(util::Exception,"Unknown conditioning option!"); - } - // sanity check: number of default scores - if (m_haveDefaultScores) + switch(m_configuration->GetCondition()) { - if(m_defaultScores.size() != m_configuration->GetNumScoreComponents()) - { - UTIL_THROW(util::Exception,"wrong number of default scores (" - << m_defaultScores.size() - << ") for lexicalized reordering model (expected " - << m_configuration->GetNumScoreComponents() << ")"); - } + case LRModel::FE: + case LRModel::E: + UTIL_THROW_IF2(m_factorsE.empty(), + "TL factor mask for lexical reordering is " + << "unexpectedly empty"); + + if(m_configuration->GetCondition() == LRModel::E) + break; // else fall through + case LRModel::F: + UTIL_THROW_IF2(m_factorsF.empty(), + "SL factor mask for lexical reordering is " + << "unexpectedly empty"); + break; + default: + UTIL_THROW2("Unknown conditioning option!"); } + // sanity check: number of default scores + size_t numScores = m_configuration->GetNumScoreComponents(); + UTIL_THROW_IF2(m_haveDefaultScores && m_defaultScores.size() != numScores, + "wrong number of default scores (" << m_defaultScores.size() + << ") for lexicalized reordering model (expected " + << m_configuration->GetNumScoreComponents() << ")"); + m_configuration->ConfigureSparse(sparseArgs, this); } - + LexicalReordering:: ~LexicalReordering() + { } + + void + LexicalReordering:: + Load() { + typedef LexicalReorderingTable LRTable; + m_table.reset(LRTable::LoadAvailable(m_filePath, m_factorsF, + m_factorsE, std::vector<FactorType>())); } - - void LexicalReordering::Load() - { - typedef LexicalReorderingTable LRT; - m_table.reset(LRT::LoadAvailable(m_filePath, m_factorsF, m_factorsE, - std::vector<FactorType>())); - } - - Scores LexicalReordering::GetProb(const Phrase& f, const Phrase& e) const + + Scores + LexicalReordering:: + GetProb(const Phrase& f, const Phrase& e) const { return m_table->GetScore(f, e, Phrase(ARRAY_SIZE_INCR)); } - + FFState* LexicalReordering:: EvaluateWhenApplied(const Hypothesis& hypo, - const FFState* prev_state, - ScoreComponentCollection* out) const + const FFState* prev_state, + ScoreComponentCollection* out) const { - VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) START" - << std::endl); + VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) START" << std::endl); Scores score(GetNumScoreComponents(), 0); - const LexicalReorderingState *prev = dynamic_cast<const LexicalReorderingState *>(prev_state); - LexicalReorderingState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out); - + const LRState *prev = dynamic_cast<const LRState *>(prev_state); + LRState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out); + out->PlusEquals(this, score); VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) END" << std::endl); return next_state; } - const FFState* LexicalReordering::EmptyHypothesisState(const InputType &input) const + FFState const* + LexicalReordering::EmptyHypothesisState(const InputType &input) const { - return m_configuration->CreateLexicalReorderingState(input); + return m_configuration->CreateLRState(input); } - bool LexicalReordering::IsUseable(const FactorMask &mask) const + bool + LexicalReordering:: + IsUseable(const FactorMask &mask) const { - for (size_t i = 0; i < m_factorsE.size(); ++i) { - const FactorType &factor = m_factorsE[i]; - if (!mask[factor]) { - return false; - } - } + BOOST_FOREACH(FactorType const& f, m_factorsE) + { if (!mask[f]) return false; } return true; - } } diff --git a/moses/FF/LexicalReordering/LexicalReordering.h b/moses/FF/LexicalReordering/LexicalReordering.h index 444a5a68c..071f90751 100644 --- a/moses/FF/LexicalReordering/LexicalReordering.h +++ b/moses/FF/LexicalReordering/LexicalReordering.h @@ -1,5 +1,5 @@ -#ifndef moses_LexicalReordering_h -#define moses_LexicalReordering_h +// -*- c++ -*- +#pragma once #include <string> #include <vector> @@ -20,83 +20,92 @@ namespace Moses { + class Factor; + class Phrase; + class Hypothesis; + class InputType; + + // implementation of lexical reordering (Tilman ...) for phrase-based + // decoding + class LexicalReordering : public StatefulFeatureFunction + { + public: + LexicalReordering(const std::string &line); + virtual ~LexicalReordering(); + void Load(); + + virtual + bool + IsUseable(const FactorMask &mask) const; + + virtual + FFState const* + EmptyHypothesisState(const InputType &input) const; + + void + InitializeForInput(const InputType& i) + { m_table->InitializeForInput(i); } + + Scores + GetProb(const Phrase& f, const Phrase& e) const; + + virtual + FFState* + EvaluateWhenApplied(const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual + FFState* + EvaluateWhenApplied(const ChartHypothesis&, int featureID, + ScoreComponentCollection*) const + { UTIL_THROW2("LexicalReordering is not valid for chart decoder"); } + + void + EvaluateWithSourceContext + (const InputType &input, + const InputPath &inputPath, + const TargetPhrase &targetPhrase, + const StackVec *stackVec, + ScoreComponentCollection& scoreBreakdown, + ScoreComponentCollection* estimatedFutureScore = NULL) const + { } + + void + EvaluateTranslationOptionListWithSourceContext + (const InputType &input, const TranslationOptionList &transOptList) const + { } + + void + EvaluateInIsolation(const Phrase &source, + const TargetPhrase &targetPhrase, + ScoreComponentCollection &scoreBreakdown, + ScoreComponentCollection &estimatedFutureScore) const + { } + + bool + GetHaveDefaultScores() { return m_haveDefaultScores; } + + float + GetDefaultScore( size_t i ) { return m_defaultScores[i]; } + + private: + bool DecodeCondition(std::string s); + bool DecodeDirection(std::string s); + bool DecodeNumFeatureFunctions(std::string s); + + boost::scoped_ptr<LRModel> m_configuration; + std::string m_modelTypeString; + std::vector<std::string> m_modelType; + boost::scoped_ptr<LexicalReorderingTable> m_table; + std::vector<LRModel::Condition> m_condition; + std::vector<FactorType> m_factorsE, m_factorsF; + std::string m_filePath; + bool m_haveDefaultScores; + Scores m_defaultScores; + }; + +} -class Factor; -class Phrase; -class Hypothesis; -class InputType; - -/** implementation of lexical reordering (Tilman ...) for phrase-based decoding - */ -class LexicalReordering : public StatefulFeatureFunction -{ -public: - LexicalReordering(const std::string &line); - virtual ~LexicalReordering(); - void Load(); - - virtual bool IsUseable(const FactorMask &mask) const; - - virtual const FFState* EmptyHypothesisState(const InputType &input) const; - - void InitializeForInput(const InputType& i) { - m_table->InitializeForInput(i); - } - - Scores GetProb(const Phrase& f, const Phrase& e) const; - - virtual FFState* EvaluateWhenApplied(const Hypothesis& cur_hypo, - const FFState* prev_state, - ScoreComponentCollection* accumulator) const; - - virtual FFState* EvaluateWhenApplied(const ChartHypothesis&, - int /* featureID */, - ScoreComponentCollection*) const { - UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder"); - } - void EvaluateWithSourceContext(const InputType &input - , const InputPath &inputPath - , const TargetPhrase &targetPhrase - , const StackVec *stackVec - , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection *estimatedFutureScore = NULL) const { - } - - void EvaluateTranslationOptionListWithSourceContext(const InputType &input - , const TranslationOptionList &translationOptionList) const { - } - - void EvaluateInIsolation(const Phrase &source - , const TargetPhrase &targetPhrase - , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection &estimatedFutureScore) const { - } - bool GetHaveDefaultScores() { - return m_haveDefaultScores; - } - float GetDefaultScore( size_t i ) { - return m_defaultScores[i]; - } - -private: - bool DecodeCondition(std::string s); - bool DecodeDirection(std::string s); - bool DecodeNumFeatureFunctions(std::string s); - - boost::scoped_ptr<LexicalReorderingConfiguration> m_configuration; - std::string m_modelTypeString; - std::vector<std::string> m_modelType; - boost::scoped_ptr<LexicalReorderingTable> m_table; - //std::vector<Direction> m_direction; - std::vector<LexicalReorderingConfiguration::Condition> m_condition; - //std::vector<size_t> m_scoreOffset; - //bool m_oneScorePerDirection; - std::vector<FactorType> m_factorsE, m_factorsF; - std::string m_filePath; - bool m_haveDefaultScores; - Scores m_defaultScores; -}; -} -#endif diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.cpp b/moses/FF/LexicalReordering/LexicalReorderingState.cpp index 0bb9344bf..c35f0cd65 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingState.cpp @@ -1,4 +1,4 @@ -// -*- c++ -*- +// -*- c++ -*- #include <vector> #include <string> @@ -6,6 +6,7 @@ #include "moses/Hypothesis.h" #include "moses/WordsRange.h" #include "moses/TranslationOption.h" +#include "moses/Util.h" #include "LexicalReordering.h" #include "LexicalReorderingState.h" @@ -13,13 +14,10 @@ namespace Moses { - - typedef LexicalReorderingConfiguration LexReoConf; - bool IsMonotonicStep(WordsRange const& prev, // words range of last source phrase - WordsRange const& cur, // words range of current source phrase - WordsBitmap const& cov) // coverage bitmap + WordsRange const& cur, // words range of current source phrase + WordsBitmap const& cov) // coverage bitmap { size_t e = prev.GetEndPos() + 1; size_t s = cur.GetStartPos(); @@ -35,39 +33,95 @@ namespace Moses } size_t - LexicalReorderingConfiguration:: + LRModel:: GetNumberOfTypes() const { - return ((m_modelType == LexReoConf::MSD) ? 3 : - (m_modelType == LexReoConf::MSLR) ? 4 : 2); + return ((m_modelType == MSD) ? 3 : + (m_modelType == MSLR) ? 4 : 2); } size_t - LexicalReorderingConfiguration:: + LRModel:: GetNumScoreComponents() const { size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); return ((m_direction == Bidirectional) - ? 2 * score_per_dir + m_additionalScoreComponents - : score_per_dir + m_additionalScoreComponents); + ? 2 * score_per_dir + m_additionalScoreComponents + : score_per_dir + m_additionalScoreComponents); } - + void - LexicalReorderingConfiguration:: - ConfigureSparse(std::map<std::string,std::string> const& sparseArgs, - const LexicalReordering* producer) + LRModel:: + ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, + const LexicalReordering* producer) { - if (sparseArgs.size()) + if (sparseArgs.size()) { m_sparse.reset(new SparseReordering(sparseArgs, producer)); + } } - void LexicalReorderingConfiguration::SetAdditionalScoreComponents(size_t number) + void + LRModel:: + SetAdditionalScoreComponents(size_t number) { m_additionalScoreComponents = number; } - LexicalReorderingConfiguration:: - LexicalReorderingConfiguration(const std::string &modelType) + /// return orientation for the first phrase + LRModel::ReorderingType + LRModel:: + GetOrientation(WordsRange const& cur) const + { + UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None"); + return ((m_modelType == LeftRight) ? R : + (cur.GetStartPos() == 0) ? M : + (m_modelType == MSD) ? D : + (m_modelType == MSLR) ? DR : NM); + } + + LRModel::ReorderingType + LRModel:: + GetOrientation(WordsRange const& prev, WordsRange const& cur) const + { + UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified"); + return ((m_modelType == LeftRight) + ? prev.GetEndPos() <= cur.GetStartPos() ? R : L + : (cur.GetStartPos() == prev.GetEndPos() + 1) ? M + : (m_modelType == Monotonic) ? NM + : (prev.GetStartPos() == cur.GetEndPos() + 1) ? S + : (m_modelType == MSD) ? D + : (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL); + } + + LRModel::ReorderingType + LRModel:: + GetOrientation(int const reoDistance) const + { // this one is for HierarchicalReorderingBackwardState + return ((m_modelType == LeftRight) + ? (reoDistance >= 1) ? R : L + : (reoDistance == 1) ? M + : (m_modelType == Monotonic) ? NM + : (reoDistance == -1) ? S + : (m_modelType == MSD) ? D + : (reoDistance > 1) ? DR : DL); + } + + LRModel::ReorderingType + LRModel:: + GetOrientation(WordsRange const& prev, WordsRange const& cur, + WordsBitmap const& cov) const + { + return ((m_modelType == LeftRight) + ? cur.GetStartPos() > prev.GetEndPos() ? R : L + : IsMonotonicStep(prev,cur,cov) ? M + : (m_modelType == Monotonic) ? NM + : IsSwap(prev,cur,cov) ? S + : (m_modelType == MSD) ? D + : cur.GetStartPos() > prev.GetEndPos() ? DR : DL); + } + + LRModel:: + LRModel(const std::string &modelType) : m_modelString(modelType) , m_scoreProducer(NULL) , m_modelType(None) @@ -91,10 +145,9 @@ namespace Moses else if (config[i] == "monotonicity") { m_modelType = Monotonic; } else if (config[i] == "leftright") { m_modelType = LeftRight; } - else if (config[i] == "backward") { m_direction = Backward; } - - // note: unidirectional is deprecated, use backward instead + // unidirectional is deprecated, use backward instead else if (config[i] == "unidirectional") { m_direction = Backward; } + else if (config[i] == "backward") { m_direction = Backward; } else if (config[i] == "forward") { m_direction = Forward; } else if (config[i] == "bidirectional") { m_direction = Bidirectional; } @@ -121,66 +174,74 @@ namespace Moses } } - LexicalReorderingState * - LexicalReorderingConfiguration:: - CreateLexicalReorderingState(const InputType &input) const + LRState * + LRModel:: + CreateLRState(const InputType &input) const { - LexicalReorderingState *bwd = NULL, *fwd = NULL; + LRState *bwd = NULL, *fwd = NULL; size_t offset = 0; switch(m_direction) { case Backward: case Bidirectional: - bwd = (m_phraseBased - ? new PhraseBasedReorderingState(*this, Backward, offset); - : new HierarchicalReorderingBackwardState(*this, offset)); + if (m_phraseBased) + bwd = new PhraseBasedReorderingState(*this, Backward, offset); + else + bwd = new HReorderingBackwardState(*this, offset); offset += m_collapseScores ? 1 : GetNumberOfTypes(); if (m_direction == Backward) return bwd; // else fall through case Forward: - fwd = (m_phraseBased - ? new PhraseBasedReorderingState(*this, Forward, offset) - : new HierarchicalReorderingForwardState(*this, input.GetSize(), - offset)); + if (m_phraseBased) + fwd = new PhraseBasedReorderingState(*this, Forward, offset); + else + fwd = new HReorderingForwardState(*this, input.GetSize(), offset); offset += m_collapseScores ? 1 : GetNumberOfTypes(); if (m_direction == Forward) return fwd; } return new BidirectionalReorderingState(*this, bwd, fwd, 0); } + void - LexicalReorderingState:: + LRState:: CopyScores(ScoreComponentCollection* accum, const TranslationOption &topt, const InputType& input, ReorderingType reoType) const { // don't call this on a bidirectional object - UTIL_THROW_IF2(m_direction != Backward && m_direction != Forward, - "Unknown direction: " << m_direction); - - TranslationOption const* - relevantOpt = (m_direction == Backward) ? &topt : m_prevOption; + UTIL_THROW_IF2(m_direction != LRModel::Backward && + m_direction != LRModel::Forward, + "Unknown direction: " << m_direction); + + TranslationOption const* relevantOpt = ((m_direction == LRModel::Backward) + ? &topt : m_prevOption); - LexicalReordering* reotable = m_configuration.GetScoreProducer(); - Scores const* cachedScores = relevantOpt->GetLexReorderingScores(reotable); + LexicalReordering* producer = m_configuration.GetScoreProducer(); + Scores const* cached = relevantOpt->GetLexReorderingScores(producer); + // The approach here is bizarre! Why create a whole vector and do + // vector addition (acumm->PlusEquals) to update a single value? - UG size_t off_remote = m_offset + reoType; - size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote; + size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote; + UTIL_THROW_IF2(off_remote >= producer->GetNumScoreComponents(), + "offset out of vector bounds!"); + // look up applicable score from vectore of scores - if(cachedScores) + if(cached) { - Scores scores(reotable->GetNumScoreComponents(),0); - socres[off_local ] (*cachedScores)[off_remote]; - accum->PlusEquals(reotable, scores); + Scores scores(producer->GetNumScoreComponents(),0); + scores[off_local ] = (*cached)[off_remote]; + accum->PlusEquals(producer, scores); } - + // else: use default scores (if specified) - else if (reotable->GetHaveDefaultScores()) + else if (producer->GetHaveDefaultScores()) { - Scores scores(reotable->GetNumScoreComponents(),0); - scores[off_local] = reotable->GetDefaultScore(off_remote); + Scores scores(producer->GetNumScoreComponents(),0); + scores[off_local] = producer->GetDefaultScore(off_remote); accum->PlusEquals(m_configuration.GetScoreProducer(), scores); } // note: if no default score, no cost @@ -190,55 +251,65 @@ namespace Moses m_direction, accum); } - + int - LexicalReorderingState:: + LRState:: ComparePrevScores(const TranslationOption *other) const { - LexicalReordering* reotable = m_configuration.GetScoreProducer(); - const Scores* myPrevScores = m_prevOption->GetLexReorderingScores(reotable); - const Scores* otherPrevScores = other->GetLexReorderingScores(reotable); + LexicalReordering* producer = m_configuration.GetScoreProducer(); + const Scores* myScores = m_prevOption->GetLexReorderingScores(producer); + const Scores* yrScores = other->GetLexReorderingScores(producer); - if(myPrevScores == otherPrevScores) - return 0; + if(myScores == yrScores) return 0; // The pointers are NULL if a phrase pair isn't found in the reordering table. - if(otherPrevScores == NULL) - return -1; - if(myPrevScores == NULL) - return 1; - - for(size_t i = m_offset; i < m_offset + m_configuration.GetNumberOfTypes(); i++) - if((*myPrevScores)[i] < (*otherPrevScores)[i]) - return -1; - else if((*myPrevScores)[i] > (*otherPrevScores)[i]) - return 1; - + if(yrScores == NULL) return -1; + if(myScores == NULL) return 1; + + size_t stop = m_offset + m_configuration.GetNumberOfTypes(); + for(size_t i = m_offset; i < stop; i++) + { + if((*myScores)[i] < (*yrScores)[i]) return -1; + if((*myScores)[i] > (*yrScores)[i]) return 1; + } return 0; } + // =========================================================================== + // PHRASE BASED REORDERING STATE + // =========================================================================== bool PhraseBasedReorderingState::m_useFirstBackwardScore = true; - PhraseBasedReorderingState::PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt) - : LexicalReorderingState(prev, topt), m_prevRange(topt.GetSourceWordsRange()), m_first(false) {} + PhraseBasedReorderingState:: + PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, + const TranslationOption &topt) + : LRState(prev, topt) + , m_prevRange(topt.GetSourceWordsRange()) + , m_first(false) + { } - PhraseBasedReorderingState::PhraseBasedReorderingState(const LexReoConf &config, - LexReoConf::Direction dir, size_t offset) - : LexicalReorderingState(config, dir, offset), m_prevRange(NOT_FOUND,NOT_FOUND), m_first(true) {} + PhraseBasedReorderingState:: + PhraseBasedReorderingState(const LRModel &config, + LRModel::Direction dir, size_t offset) + : LRState(config, dir, offset) + , m_prevRange(NOT_FOUND,NOT_FOUND) + , m_first(true) + { } - int PhraseBasedReorderingState::Compare(const FFState& o) const + int + PhraseBasedReorderingState:: + Compare(const FFState& o) const { - if (&o == this) - return 0; + if (&o == this) return 0; const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o); if (m_prevRange == other->m_prevRange) { - if (m_direction == LexReoConf::Forward) { - return ComparePrevScores(other->m_prevOption); + if (m_direction == LRModel::Forward) { + return ComparePrevScores(other->m_prevOption); } else { - return 0; + return 0; } } else if (m_prevRange < other->m_prevRange) { return -1; @@ -246,218 +317,129 @@ namespace Moses return 1; } - LexicalReorderingState* PhraseBasedReorderingState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const + LRState* + PhraseBasedReorderingState:: + Expand(const TranslationOption& topt, const InputType& input, + ScoreComponentCollection* scores) const { - ReorderingType reoType; - const WordsRange currWordsRange = topt.GetSourceWordsRange(); - const LexReoConf::ModelType modelType = m_configuration.GetModelType(); - - if ((m_direction != LexReoConf::Forward && m_useFirstBackwardScore) || !m_first) { - if (modelType == LexReoConf::MSD) { - reoType = GetOrientationTypeMSD(currWordsRange); - } else if (modelType == LexReoConf::MSLR) { - reoType = GetOrientationTypeMSLR(currWordsRange); - } else if (modelType == LexReoConf::Monotonic) { - reoType = GetOrientationTypeMonotonic(currWordsRange); - } else { - reoType = GetOrientationTypeLeftRight(currWordsRange); + // const LRModel::ModelType modelType = m_configuration.GetModelType(); + + if ((m_direction != LRModel::Forward && m_useFirstBackwardScore) || !m_first) + { + LRModel const& lrmodel = m_configuration; + WordsRange const cur = topt.GetSourceWordsRange(); + ReorderingType reoType = (m_first ? lrmodel.GetOrientation(cur) + : lrmodel.GetOrientation(m_prevRange,cur)); + CopyScores(scores, topt, input, reoType); } - CopyScores(scores, topt, input, reoType); - } - return new PhraseBasedReorderingState(this, topt); } - LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSD(WordsRange currRange) const - { - if (m_first) { - if (currRange.GetStartPos() == 0) { - return M; - } else { - return D; - } - } - if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { - return M; - } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { - return S; - } - return D; - } - - LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSLR(WordsRange currRange) const - { - if (m_first) { - if (currRange.GetStartPos() == 0) { - return M; - } else { - return DR; - } - } - if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { - return M; - } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { - return S; - } else if (m_prevRange.GetEndPos() < currRange.GetStartPos()) { - return DR; - } - return DL; - } - - - LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMonotonic(WordsRange currRange) const - { - if ((m_first && currRange.GetStartPos() == 0) || - (m_prevRange.GetEndPos() == currRange.GetStartPos()-1)) { - return M; - } - return NM; - } - - LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeLeftRight(WordsRange currRange) const - { - if (m_first || - (m_prevRange.GetEndPos() <= currRange.GetStartPos())) { - return R; - } - return L; - } /////////////////////////// //BidirectionalReorderingState - int BidirectionalReorderingState::Compare(const FFState& o) const + int + BidirectionalReorderingState:: + Compare(FFState const& o) const { - if (&o == this) - return 0; + if (&o == this) return 0; + + BidirectionalReorderingState const &other + = static_cast<BidirectionalReorderingState const&>(o); - const BidirectionalReorderingState &other = static_cast<const BidirectionalReorderingState &>(o); - if(m_backward->Compare(*other.m_backward) < 0) - return -1; - else if(m_backward->Compare(*other.m_backward) > 0) - return 1; - else - return m_forward->Compare(*other.m_forward); + int cmp = m_backward->Compare(*other.m_backward); + return (cmp < 0) ? -1 : cmp ? 1 : m_forward->Compare(*other.m_forward); } - LexicalReorderingState* BidirectionalReorderingState::Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const + LRState* + BidirectionalReorderingState:: + Expand(const TranslationOption& topt, const InputType& input, + ScoreComponentCollection* scores) const { - LexicalReorderingState *newbwd = m_backward->Expand(topt,input, scores); - LexicalReorderingState *newfwd = m_forward->Expand(topt, input, scores); + LRState *newbwd = m_backward->Expand(topt,input, scores); + LRState *newfwd = m_forward->Expand(topt, input, scores); return new BidirectionalReorderingState(m_configuration, newbwd, newfwd, m_offset); } /////////////////////////// //HierarchicalReorderingBackwardState - HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const HierarchicalReorderingBackwardState *prev, - const TranslationOption &topt, ReorderingStack reoStack) - : LexicalReorderingState(prev, topt), m_reoStack(reoStack) {} - - HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const LexReoConf &config, size_t offset) - : LexicalReorderingState(config, LexReoConf::Backward, offset) {} + HReorderingBackwardState:: + HReorderingBackwardState(const HReorderingBackwardState *prev, + const TranslationOption &topt, + ReorderingStack reoStack) + : LRState(prev, topt), m_reoStack(reoStack) + { } + + HReorderingBackwardState:: + HReorderingBackwardState(const LRModel &config, size_t offset) + : LRState(config, LRModel::Backward, offset) + { } - int HierarchicalReorderingBackwardState::Compare(const FFState& o) const + int + HReorderingBackwardState:: + Compare(const FFState& o) const { - const HierarchicalReorderingBackwardState& other = static_cast<const HierarchicalReorderingBackwardState&>(o); + const HReorderingBackwardState& other + = static_cast<const HReorderingBackwardState&>(o); return m_reoStack.Compare(other.m_reoStack); } - - LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const + + LRState* + HReorderingBackwardState:: + Expand(const TranslationOption& topt, const InputType& input, + ScoreComponentCollection* scores) const { - - HierarchicalReorderingBackwardState* nextState = new HierarchicalReorderingBackwardState(this, topt, m_reoStack); - ReorderingType reoType; - const LexReoConf::ModelType modelType = m_configuration.GetModelType(); - - int reoDistance = nextState->m_reoStack.ShiftReduce(topt.GetSourceWordsRange()); - - if (modelType == LexReoConf::MSD) { - reoType = GetOrientationTypeMSD(reoDistance); - } else if (modelType == LexReoConf::MSLR) { - reoType = GetOrientationTypeMSLR(reoDistance); - } else if (modelType == LexReoConf::LeftRight) { - reoType = GetOrientationTypeLeftRight(reoDistance); - } else { - reoType = GetOrientationTypeMonotonic(reoDistance); - } - + HReorderingBackwardState* nextState; + nextState = new HReorderingBackwardState(this, topt, m_reoStack); + WordsRange swrange = topt.GetSourceWordsRange(); + int reoDistance = nextState->m_reoStack.ShiftReduce(swrange); + ReorderingType reoType = m_configuration.GetOrientation(reoDistance); CopyScores(scores, topt, input, reoType); return nextState; } - LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSD(int reoDistance) const - { - if (reoDistance == 1) { - return M; - } else if (reoDistance == -1) { - return S; - } - return D; - } - - LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSLR(int reoDistance) const - { - if (reoDistance == 1) { - return M; - } else if (reoDistance == -1) { - return S; - } else if (reoDistance > 1) { - return DR; - } - return DL; - } - - LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMonotonic(int reoDistance) const - { - if (reoDistance == 1) { - return M; - } - return NM; - } - - LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeLeftRight(int reoDistance) const - { - if (reoDistance >= 1) { - return R; - } - return L; - } - - - - /////////////////////////// - //HierarchicalReorderingForwardState - - HierarchicalReorderingForwardState:: - HierarchicalReorderingForwardState(const LexReoConf &config, size_t size, size_t offset) - : LexicalReorderingState(config, LexReoConf::Forward, offset), m_first(true), m_prevRange(NOT_FOUND,NOT_FOUND), m_coverage(size) {} - - HierarchicalReorderingForwardState::HierarchicalReorderingForwardState(const HierarchicalReorderingForwardState *prev, const TranslationOption &topt) - : LexicalReorderingState(prev, topt), m_first(false), m_prevRange(topt.GetSourceWordsRange()), m_coverage(prev->m_coverage) + //HReorderingForwardState + + HReorderingForwardState:: + HReorderingForwardState(const LRModel &config, + size_t size, size_t offset) + : LRState(config, LRModel::Forward, offset) + , m_first(true) + , m_prevRange(NOT_FOUND,NOT_FOUND) + , m_coverage(size) + { } + + HReorderingForwardState:: + HReorderingForwardState(const HReorderingForwardState *prev, + const TranslationOption &topt) + : LRState(prev, topt) + , m_first(false) + , m_prevRange(topt.GetSourceWordsRange()) + , m_coverage(prev->m_coverage) { - const WordsRange currWordsRange = topt.GetSourceWordsRange(); - m_coverage.SetValue(currWordsRange.GetStartPos(), currWordsRange.GetEndPos(), true); + m_coverage.SetValue(topt.GetSourceWordsRange(), true); } - int HierarchicalReorderingForwardState::Compare(const FFState& o) const + int + HReorderingForwardState:: + Compare(const FFState& o) const { - if (&o == this) - return 0; + if (&o == this) return 0; - const HierarchicalReorderingForwardState* other = static_cast<const HierarchicalReorderingForwardState*>(&o); - - if (m_prevRange == other->m_prevRange) { - return ComparePrevScores(other->m_prevOption); - } else if (m_prevRange < other->m_prevRange) { - return -1; - } - return 1; + HReorderingForwardState const& other + = static_cast<HReorderingForwardState const&>(o); + + return ((m_prevRange == other.m_prevRange) + ? ComparePrevScores(other.m_prevOption) + : (m_prevRange < other.m_prevRange) ? -1 : 1); } - // For compatibility with the phrase-based reordering model, scoring is one step delayed. + // For compatibility with the phrase-based reordering model, scoring is one + // step delayed. // The forward model takes determines orientations heuristically as follows: // mono: if the next phrase comes after the conditioning phrase and // - there is a gap to the right of the conditioning phrase, or @@ -465,70 +447,27 @@ namespace Moses // swap: if the next phrase goes before the conditioning phrase and // - there is a gap to the left of the conditioning phrase, or // - the next phrase immediately precedes it - // dright: if the next phrase follows the cond. phr. - // and other stuff comes in between - // dleft: if the next phrase precedes the conditioning phrase - // and other stuff comes in between - - LexicalReorderingState* - HierarchicalReorderingForwardState:: - Expand(const TranslationOption& topt, const InputType& input, - ScoreComponentCollection* scores) const + // dright: if the next phrase follows the conditioning phrase and other + // stuff comes in between + // dleft: if the next phrase precedes the conditioning phrase and other + // stuff comes in between + + LRState* + HReorderingForwardState:: + Expand(TranslationOption const& topt, InputType const& input, + ScoreComponentCollection* scores) const { - LexReoConf::ModelType const modelType = m_configuration.GetModelType(); - WordsRange const& currRange = topt.GetSourceWordsRange(); - - // keep track of the current cov. ourselves so we don't need the hypothesis - WordsBitmap cov = m_coverage; - cov.SetValue(currRange.GetStartPos(), currRange.GetEndPos(), true); - - - if (!m_first) + const WordsRange cur = topt.GetSourceWordsRange(); + // keep track of the current coverage ourselves so we don't need the hypothesis + WordsBitmap cov = m_coverage; + cov.SetValue(cur, true); + if (!m_first) { - ReorderingType reoType - = ((modelType == LexReoConf::MSD) - ? GetOrientationTypeMSD(currWordsRange, coverage) - : (modelType == LexReoConf::MSLR) - ? GetOrientationTypeMSLR(currWordsRange, coverage) - : (modelType == LexReoConf::Monotonic) - ? GetOrientationTypeMonotonic(currWordsRange, coverage); - : GetOrientationTypeLeftRight(currWordsRange, coverage)); - CopyScores(scores, topt, input, reoType); + LRModel::ReorderingType reoType; + reoType = m_configuration.GetOrientation(m_prevRange,cur,cov); + CopyScores(scores, topt, input, reoType); } - - return new HierarchicalReorderingForwardState(this, topt); + return new HReorderingForwardState(this, topt); } - - LexicalReorderingState::ReorderingType - HierarchicalReorderingForwardState:: - GetOrientationTypeMSD(WordsRange currRange, WordsBitmap coverage) const - { - return (IsMonotonicStep(m_prevRange,currRange,coverage) ? M - : IsSwap(m_prevRange, currRange, coverage) ? S : D); - } - - LexicalReorderingState::ReorderingType - HierarchicalReorderingForwardState:: - GetOrientationTypeMSLR(WordsRange currRange, WordsBitmap coverage) const - { - return (IsMonotonicStep(m_prevRange,currRange,coverage) ? M - : IsSwap(m_prevRange, currRange, coverage) ? S - : (currRange.GetStartPos() > m_prevRange.GetEndPos()) ? DR : DL); - } - - LexicalReorderingState::ReorderingType - HierarchicalReorderingForwardState:: - GetOrientationTypeMonotonic(WordsRange currRange, WordsBitmap coverage) const - { - return IsMonotonicStep(m_prevRange, currRange, coverage) ? M : NM; - } - - LexicalReorderingState::ReorderingType - HierarchicalReorderingForwardState:: - GetOrientationTypeLeftRight(WordsRange currRange, WordsBitmap coverage) const - { - return currRange.GetStartPos() > m_prevRange.GetEndPos() ? R : L; - } - - } + diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.h b/moses/FF/LexicalReordering/LexicalReorderingState.h index 6599dae5b..83607a66a 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.h +++ b/moses/FF/LexicalReordering/LexicalReorderingState.h @@ -1,4 +1,4 @@ - +// -*- c++ -*- #pragma once #include <vector> @@ -7,7 +7,6 @@ #include <boost/scoped_ptr.hpp> #include "moses/Hypothesis.h" -//#include "LexicalReordering.h" #include "moses/ScoreComponentCollection.h" #include "moses/WordsRange.h" #include "moses/WordsBitmap.h" @@ -18,210 +17,272 @@ namespace Moses { -class LexicalReorderingState; -class LexicalReordering; -class SparseReordering; - -/** Factory class for lexical reordering states - */ -class LexicalReorderingConfiguration -{ -public: - friend class LexicalReordering; - enum ModelType {Monotonic, MSD, MSLR, LeftRight, None}; - enum Direction {Forward, Backward, Bidirectional}; - enum Condition {F, E, FE}; - - LexicalReorderingConfiguration(const std::string &modelType); - - void ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer); - - LexicalReorderingState *CreateLexicalReorderingState(const InputType &input) const; - - size_t GetNumScoreComponents() const; - void SetAdditionalScoreComponents(size_t number); - size_t GetNumberOfTypes() const; - - LexicalReordering *GetScoreProducer() const { - return m_scoreProducer; - } - - - ModelType GetModelType() const { - return m_modelType; - } - - Direction GetDirection() const { - return m_direction; - } - - Condition GetCondition() const { - return m_condition; - } - - bool IsPhraseBased() const { - return m_phraseBased; - } - - bool CollapseScores() const { - return m_collapseScores; - } - - const SparseReordering* GetSparseReordering() const { - return m_sparse.get(); - } - -private: - void SetScoreProducer(LexicalReordering* scoreProducer) { - m_scoreProducer = scoreProducer; - } - - const std::string& GetModelString() const { - return m_modelString; - } - - std::string m_modelString; - LexicalReordering *m_scoreProducer; - ModelType m_modelType; - bool m_phraseBased; - bool m_collapseScores; - Direction m_direction; - Condition m_condition; - size_t m_additionalScoreComponents; - boost::scoped_ptr<SparseReordering> m_sparse; -}; - -//! Abstract class for lexical reordering model states -class LexicalReorderingState : public FFState -{ - typedef LexicalReorderingConfiguration::Forward Forward; - typedef LexicalReorderingConfiguration::Backward Backward; + class LRState; + class LexicalReordering; + class SparseReordering; -public: - virtual int Compare(const FFState& o) const = 0; - virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const = 0; - - static LexicalReorderingState* CreateLexicalReorderingState(const std::vector<std::string>& config, - LexicalReorderingConfiguration::Direction dir, const InputType &input); - typedef int ReorderingType; - -protected: - - - const LexicalReorderingConfiguration &m_configuration; - // The following is the true direction of the object, which can be Backward or Forward even if the Configuration has Bidirectional. - LexicalReorderingConfiguration::Direction m_direction; - size_t m_offset; - //forward scores are conditioned on prev option, so need to remember it - const TranslationOption *m_prevOption; - - inline LexicalReorderingState(const LexicalReorderingState *prev, const TranslationOption &topt) : - m_configuration(prev->m_configuration), m_direction(prev->m_direction), m_offset(prev->m_offset), - m_prevOption(&topt) {} - - inline LexicalReorderingState(const LexicalReorderingConfiguration &config, LexicalReorderingConfiguration::Direction dir, size_t offset) - : m_configuration(config), m_direction(dir), m_offset(offset), m_prevOption(NULL) {} - - // copy the right scores in the right places, taking into account forward/backward, offset, collapse - void CopyScores(ScoreComponentCollection* scores, const TranslationOption& topt, const InputType& input, ReorderingType reoType) const; - int ComparePrevScores(const TranslationOption *other) const; - - //constants for the different type of reorderings (corresponding to indexes in the table file) -public: - static const ReorderingType M = 0; // monotonic - static const ReorderingType NM = 1; // non-monotonic - static const ReorderingType S = 1; // swap - static const ReorderingType D = 2; // discontinuous - static const ReorderingType DL = 2; // discontinuous, left - static const ReorderingType DR = 3; // discontinuous, right - static const ReorderingType R = 0; // right - static const ReorderingType L = 1; // left - static const ReorderingType MAX = 3; //largest possible -}; - -//! @todo what is this? -class BidirectionalReorderingState : public LexicalReorderingState -{ -private: - const LexicalReorderingState *m_backward; - const LexicalReorderingState *m_forward; -public: - BidirectionalReorderingState(const LexicalReorderingConfiguration &config, const LexicalReorderingState *bw, const LexicalReorderingState *fw, size_t offset) : - LexicalReorderingState(config, LexicalReorderingConfiguration::Bidirectional, offset), m_backward(bw), m_forward(fw) {} - - ~BidirectionalReorderingState() { - delete m_backward; - delete m_forward; - } - - virtual int Compare(const FFState& o) const; - virtual LexicalReorderingState* Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const; -}; - -//! State for the standard Moses implementation of lexical reordering models -//! (see Koehn et al, Edinburgh System Description for the 2005 NIST MT Evaluation) -class PhraseBasedReorderingState : public LexicalReorderingState -{ -private: - WordsRange m_prevRange; - bool m_first; -public: - static bool m_useFirstBackwardScore; - PhraseBasedReorderingState(const LexicalReorderingConfiguration &config, LexicalReorderingConfiguration::Direction dir, size_t offset); - PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt); - - virtual int Compare(const FFState& o) const; - virtual LexicalReorderingState* Expand(const TranslationOption& topt,const InputType& input, ScoreComponentCollection* scores) const; - - ReorderingType GetOrientationTypeMSD(WordsRange currRange) const; - ReorderingType GetOrientationTypeMSLR(WordsRange currRange) const; - ReorderingType GetOrientationTypeMonotonic(WordsRange currRange) const; - ReorderingType GetOrientationTypeLeftRight(WordsRange currRange) const; -}; - -//! State for a hierarchical reordering model -//! (see Galley and Manning, A Simple and Effective Hierarchical Phrase Reordering Model, EMNLP 2008) -//!backward state (conditioned on the previous phrase) -class HierarchicalReorderingBackwardState : public LexicalReorderingState -{ -private: - ReorderingStack m_reoStack; -public: - HierarchicalReorderingBackwardState(const LexicalReorderingConfiguration &config, size_t offset); - HierarchicalReorderingBackwardState(const HierarchicalReorderingBackwardState *prev, - const TranslationOption &topt, ReorderingStack reoStack); - - virtual int Compare(const FFState& o) const; - virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const; - -private: - ReorderingType GetOrientationTypeMSD(int reoDistance) const; - ReorderingType GetOrientationTypeMSLR(int reoDistance) const; - ReorderingType GetOrientationTypeMonotonic(int reoDistance) const; - ReorderingType GetOrientationTypeLeftRight(int reoDistance) const; -}; - - -//!forward state (conditioned on the next phrase) -class HierarchicalReorderingForwardState : public LexicalReorderingState -{ -private: - bool m_first; - WordsRange m_prevRange; - WordsBitmap m_coverage; - -public: - HierarchicalReorderingForwardState(const LexicalReorderingConfiguration &config, size_t sentenceLength, size_t offset); - HierarchicalReorderingForwardState(const HierarchicalReorderingForwardState *prev, const TranslationOption &topt); - - virtual int Compare(const FFState& o) const; - virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const; - -private: - ReorderingType GetOrientationTypeMSD(WordsRange currRange, WordsBitmap coverage) const; - ReorderingType GetOrientationTypeMSLR(WordsRange currRange, WordsBitmap coverage) const; - ReorderingType GetOrientationTypeMonotonic(WordsRange currRange, WordsBitmap coverage) const; - ReorderingType GetOrientationTypeLeftRight(WordsRange currRange, WordsBitmap coverage) const; -}; - + //! Factory class for lexical reordering states + class LRModel + { + public: + typedef int ReorderingType; + friend class LexicalReordering; + enum ModelType { Monotonic, MSD, MSLR, LeftRight, None }; + enum Direction { Forward, Backward, Bidirectional }; + enum Condition { F, E, FE }; + + // constants for the different types of reordering + // (correspond to indices in the respective table) + static const ReorderingType M = 0; // monotonic + static const ReorderingType NM = 1; // non-monotonic + static const ReorderingType S = 1; // swap + static const ReorderingType D = 2; // discontinuous + static const ReorderingType DL = 2; // discontinuous, left + static const ReorderingType DR = 3; // discontinuous, right + static const ReorderingType R = 0; // right + static const ReorderingType L = 1; // left + static const ReorderingType MAX = 3; // largest possible + + // determine orientation, depending on model: + + + ReorderingType // for first phrase in phrase-based + GetOrientation(WordsRange const& cur) const; + + ReorderingType // for non-first phrases in phrase-based + GetOrientation(WordsRange const& prev, WordsRange const& cur) const; + + ReorderingType // for HReorderingForwardState + GetOrientation(WordsRange const& prev, WordsRange const& cur, + WordsBitmap const& cov) const; + + ReorderingType // for HReorderingBackwarddState + GetOrientation(int const reoDistance) const; + + LRModel(const std::string &modelType); + + void + ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, + const LexicalReordering* producer); + + LRState* + CreateLRState(const InputType &input) const; + + size_t GetNumberOfTypes() const; + size_t GetNumScoreComponents() const; + void SetAdditionalScoreComponents(size_t number); + + LexicalReordering* + GetScoreProducer() const { return m_scoreProducer; } + + ModelType GetModelType() const { return m_modelType; } + Direction GetDirection() const { return m_direction; } + Condition GetCondition() const { return m_condition; } + + bool + IsPhraseBased() const + { return m_phraseBased; } + + bool + CollapseScores() const + { return m_collapseScores; } + + SparseReordering const* + GetSparseReordering() const + { return m_sparse.get(); } + + private: + void + SetScoreProducer(LexicalReordering* scoreProducer) + { m_scoreProducer = scoreProducer; } + + std::string const& + GetModelString() const + { return m_modelString; } + + std::string m_modelString; + LexicalReordering *m_scoreProducer; + ModelType m_modelType; + bool m_phraseBased; + bool m_collapseScores; + Direction m_direction; + Condition m_condition; + size_t m_additionalScoreComponents; + boost::scoped_ptr<SparseReordering> m_sparse; + }; + + //! Abstract class for lexical reordering model states + class LRState : public FFState + { + public: + + typedef int ReorderingType; + + virtual + int + Compare(const FFState& o) const = 0; + + virtual + LRState* + Expand(const TranslationOption& hypo, const InputType& input, + ScoreComponentCollection* scores) const = 0; + + static + LRState* + CreateLRState(const std::vector<std::string>& config, + LRModel::Direction dir, + const InputType &input); + + protected: + + const LRModel& m_configuration; + + // The following is the true direction of the object, which can be + // Backward or Forward even if the Configuration has Bidirectional. + LRModel::Direction m_direction; + size_t m_offset; + //forward scores are conditioned on prev option, so need to remember it + const TranslationOption *m_prevOption; + + inline + LRState(const LRState *prev, + const TranslationOption &topt) + : m_configuration(prev->m_configuration) + , m_direction(prev->m_direction) + , m_offset(prev->m_offset) + , m_prevOption(&topt) + { } + + inline + LRState(const LRModel &config, + LRModel::Direction dir, + size_t offset) + : m_configuration(config) + , m_direction(dir) + , m_offset(offset) + , m_prevOption(NULL) + { } + + // copy the right scores in the right places, taking into account + // forward/backward, offset, collapse + void + CopyScores(ScoreComponentCollection* scores, + const TranslationOption& topt, + const InputType& input, ReorderingType reoType) const; + + int + ComparePrevScores(const TranslationOption *other) const; + }; + + //! @todo what is this? + class BidirectionalReorderingState + : public LRState + { + private: + const LRState *m_backward; + const LRState *m_forward; + public: + BidirectionalReorderingState(const LRModel &config, + const LRState *bw, + const LRState *fw, size_t offset) + : LRState(config, + LRModel::Bidirectional, + offset) + , m_backward(bw) + , m_forward(fw) + { } + + ~BidirectionalReorderingState() + { + delete m_backward; + delete m_forward; + } + + virtual + int + Compare(const FFState& o) const; + + virtual + LRState* + Expand(const TranslationOption& topt, const InputType& input, + ScoreComponentCollection* scores) const; + }; + + //! State for the standard Moses implementation of lexical reordering models + //! (see Koehn et al, Edinburgh System Description for the 2005 NIST MT + //! Evaluation) + class PhraseBasedReorderingState + : public LRState + { + private: + WordsRange m_prevRange; + bool m_first; + public: + static bool m_useFirstBackwardScore; + PhraseBasedReorderingState(const LRModel &config, + LRModel::Direction dir, + size_t offset); + PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, + const TranslationOption &topt); + + virtual + int + Compare(const FFState& o) const; + + virtual + LRState* + Expand(const TranslationOption& topt,const InputType& input, + ScoreComponentCollection* scores) const; + + ReorderingType GetOrientationTypeMSD(WordsRange currRange) const; + ReorderingType GetOrientationTypeMSLR(WordsRange currRange) const; + ReorderingType GetOrientationTypeMonotonic(WordsRange currRange) const; + ReorderingType GetOrientationTypeLeftRight(WordsRange currRange) const; + }; + + //! State for a hierarchical reordering model (see Galley and Manning, A + //! Simple and Effective Hierarchical Phrase Reordering Model, EMNLP 2008) + //! backward state (conditioned on the previous phrase) + class HReorderingBackwardState : public LRState + { + private: + ReorderingStack m_reoStack; + public: + HReorderingBackwardState(const LRModel &config, size_t offset); + HReorderingBackwardState(const HReorderingBackwardState *prev, + const TranslationOption &topt, + ReorderingStack reoStack); + + virtual int Compare(const FFState& o) const; + virtual LRState* Expand(const TranslationOption& hypo, const InputType& input, + ScoreComponentCollection* scores) const; + + private: + ReorderingType GetOrientationTypeMSD(int reoDistance) const; + ReorderingType GetOrientationTypeMSLR(int reoDistance) const; + ReorderingType GetOrientationTypeMonotonic(int reoDistance) const; + ReorderingType GetOrientationTypeLeftRight(int reoDistance) const; + }; + + + //!forward state (conditioned on the next phrase) + class HReorderingForwardState : public LRState + { + private: + bool m_first; + WordsRange m_prevRange; + WordsBitmap m_coverage; + + public: + HReorderingForwardState(const LRModel &config, size_t sentenceLength, + size_t offset); + HReorderingForwardState(const HReorderingForwardState *prev, + const TranslationOption &topt); + + virtual int Compare(const FFState& o) const; + virtual LRState* Expand(const TranslationOption& hypo, + const InputType& input, + ScoreComponentCollection* scores) const; + }; } + diff --git a/moses/FF/LexicalReordering/SparseReordering.cpp b/moses/FF/LexicalReordering/SparseReordering.cpp index 1561ef0af..22eca8520 100644 --- a/moses/FF/LexicalReordering/SparseReordering.cpp +++ b/moses/FF/LexicalReordering/SparseReordering.cpp @@ -100,7 +100,7 @@ void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, type <= SparseReorderingFeatureKey::Between; ++type) { for (size_t position = SparseReorderingFeatureKey::First; position <= SparseReorderingFeatureKey::Last; ++position) { - for (int reoType = 0; reoType <= LexicalReorderingState::MAX; ++reoType) { + for (int reoType = 0; reoType <= LRModel::MAX; ++reoType) { SparseReorderingFeatureKey key( index, static_cast<SparseReorderingFeatureKey::Type>(type), factor, isCluster, static_cast<SparseReorderingFeatureKey::Position>(position), side, reoType); @@ -152,7 +152,7 @@ void SparseReordering::ReadClusterMap(const string& filename, const string& id, void SparseReordering::AddFeatures( SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side, const Word& word, SparseReorderingFeatureKey::Position position, - LexicalReorderingState::ReorderingType reoType, + LRModel::ReorderingType reoType, ScoreComponentCollection* scores) const { @@ -194,13 +194,12 @@ void SparseReordering::CopyScores( const TranslationOption& currentOpt, const TranslationOption* previousOpt, const InputType& input, - LexicalReorderingState::ReorderingType reoType, - LexicalReorderingConfiguration::Direction direction, + LRModel::ReorderingType reoType, + LRModel::Direction direction, ScoreComponentCollection* scores) const { - if (m_useBetween && direction == LexicalReorderingConfiguration::Backward && - (reoType == LexicalReorderingState::D || reoType == LexicalReorderingState::DL || - reoType == LexicalReorderingState::DR)) { + if (m_useBetween && direction == LRModel::Backward && + (reoType == LRModel::D || reoType == LRModel::DL || reoType == LRModel::DR)){ size_t gapStart, gapEnd; //NB: Using a static cast for speed, but could be nasty if //using non-sentence input @@ -231,10 +230,10 @@ void SparseReordering::CopyScores( //phrase (backward) //stack (forward) SparseReorderingFeatureKey::Type type; - if (direction == LexicalReorderingConfiguration::Forward) { + if (direction == LRModel::Forward) { if (!m_useStack) return; type = SparseReorderingFeatureKey::Stack; - } else if (direction == LexicalReorderingConfiguration::Backward) { + } else if (direction == LRModel::Backward) { if (!m_usePhrase) return; type = SparseReorderingFeatureKey::Phrase; } else { diff --git a/moses/FF/LexicalReordering/SparseReordering.h b/moses/FF/LexicalReordering/SparseReordering.h index 50ec96a0d..de0106799 100644 --- a/moses/FF/LexicalReordering/SparseReordering.h +++ b/moses/FF/LexicalReordering/SparseReordering.h @@ -48,10 +48,10 @@ struct SparseReorderingFeatureKey { bool isCluster; enum Position {First, Last} position; enum Side {Source, Target} side; - LexicalReorderingState::ReorderingType reoType; + LRState::ReorderingType reoType; SparseReorderingFeatureKey(size_t id_, Type type_, const Factor* word_, bool isCluster_, - Position position_, Side side_, LexicalReorderingState::ReorderingType reoType_) + Position position_, Side side_, LRState::ReorderingType reoType_) : id(id_), type(type_), word(word_), isCluster(isCluster_), position(position_), side(side_), reoType(reoType_) { } @@ -94,8 +94,8 @@ public: void CopyScores(const TranslationOption& currentOpt, const TranslationOption* previousOpt, const InputType& input, - LexicalReorderingState::ReorderingType reoType, - LexicalReorderingConfiguration::Direction direction, + LRState::ReorderingType reoType, + LRModel::Direction direction, ScoreComponentCollection* scores) const ; private: @@ -120,7 +120,7 @@ private: void AddFeatures( SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side, const Word& word, SparseReorderingFeatureKey::Position position, - LexicalReorderingState::ReorderingType reoType, + LRState::ReorderingType reoType, ScoreComponentCollection* scores) const; }; diff --git a/moses/WordsBitmap.h b/moses/WordsBitmap.h index ab81471b9..08567330d 100644 --- a/moses/WordsBitmap.h +++ b/moses/WordsBitmap.h @@ -143,11 +143,17 @@ public: m_bitmap[pos] = value; } //! set value between 2 positions, inclusive - void SetValue( size_t startPos, size_t endPos, bool value ) { - for(size_t pos = startPos ; pos <= endPos ; pos++) { + void + SetValue( size_t startPos, size_t endPos, bool value ) + { + for(size_t pos = startPos ; pos <= endPos ; pos++) m_bitmap[pos] = value; - } } + + void + SetValue(WordsRange const& range, bool val) + { SetValue(range.GetStartPos(), range.GetEndPos(), val); } + //! whether every word has been translated bool IsComplete() const { return GetSize() == GetNumWordsCovered(); |