diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2016-03-23 17:28:49 +0300 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2016-03-23 17:29:32 +0300 |
commit | fa02a99f13afe1ec7a7834d34160781d07ba2444 (patch) | |
tree | ca52665c5314583c9284468615272f722819ee2e /moses/FF | |
parent | 105faa5673567f8ed4ffee5857b59715511a6a2d (diff) |
separate LRModel into its own file
Diffstat (limited to 'moses/FF')
-rw-r--r-- | moses/FF/LexicalReordering/LRModel.cpp | 219 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LRModel.h | 133 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReorderingState.cpp | 210 | ||||
-rw-r--r-- | moses/FF/LexicalReordering/LexicalReorderingState.h | 124 |
4 files changed, 353 insertions, 333 deletions
diff --git a/moses/FF/LexicalReordering/LRModel.cpp b/moses/FF/LexicalReordering/LRModel.cpp new file mode 100644 index 000000000..6da48c7db --- /dev/null +++ b/moses/FF/LexicalReordering/LRModel.cpp @@ -0,0 +1,219 @@ +#include "LRModel.h" +#include "moses/Range.h" +#include "moses/Bitmap.h" +#include "moses/InputType.h" +#include "HReorderingForwardState.h" +#include "HReorderingBackwardState.h" +#include "PhraseBasedReorderingState.h" +#include "BidirectionalReorderingState.h" +#include "SparseReordering.h" + +namespace Moses +{ + +bool +IsMonotonicStep(Range const& prev, // words range of last source phrase + Range const& cur, // words range of current source phrase + Bitmap const& cov) // coverage bitmap +{ + size_t e = prev.GetEndPos() + 1; + size_t s = cur.GetStartPos(); + return (s == e || (s >= e && !cov.GetValue(e))); +} + +bool +IsSwap(Range const& prev, Range const& cur, Bitmap const& cov) +{ + size_t s = prev.GetStartPos(); + size_t e = cur.GetEndPos(); + return (e+1 == s || (e < s && !cov.GetValue(s-1))); +} + +size_t +LRModel:: +GetNumberOfTypes() const +{ + return ((m_modelType == MSD) ? 3 : + (m_modelType == MSLR) ? 4 : 2); +} + +size_t +LRModel:: +GetNumScoreComponents() const +{ + size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); + return ((m_direction == Bidirectional) + ? 2 * score_per_dir + m_additionalScoreComponents + : score_per_dir + m_additionalScoreComponents); +} + +void +LRModel:: +ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, + const LexicalReordering* producer) +{ + if (sparseArgs.size()) { + m_sparse.reset(new SparseReordering(sparseArgs, producer)); + } +} + +void +LRModel:: +SetAdditionalScoreComponents(size_t number) +{ + m_additionalScoreComponents = number; +} + +/// return orientation for the first phrase +LRModel::ReorderingType +LRModel:: +GetOrientation(Range const& cur) const +{ + UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None"); + return ((m_modelType == LeftRight) ? R : + (cur.GetStartPos() == 0) ? M : + (m_modelType == MSD) ? D : + (m_modelType == MSLR) ? DR : NM); +} + +LRModel::ReorderingType +LRModel:: +GetOrientation(Range const& prev, Range const& cur) const +{ + UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified"); + return ((m_modelType == LeftRight) + ? prev.GetEndPos() <= cur.GetStartPos() ? R : L + : (cur.GetStartPos() == prev.GetEndPos() + 1) ? M + : (m_modelType == Monotonic) ? NM + : (prev.GetStartPos() == cur.GetEndPos() + 1) ? S + : (m_modelType == MSD) ? D + : (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL); +} + +LRModel::ReorderingType +LRModel:: +GetOrientation(int const reoDistance) const +{ + // this one is for HierarchicalReorderingBackwardState + return ((m_modelType == LeftRight) + ? (reoDistance >= 1) ? R : L + : (reoDistance == 1) ? M + : (m_modelType == Monotonic) ? NM + : (reoDistance == -1) ? S + : (m_modelType == MSD) ? D + : (reoDistance > 1) ? DR : DL); +} + +LRModel::ReorderingType +LRModel:: +GetOrientation(Range const& prev, Range const& cur, + Bitmap const& cov) const +{ + return ((m_modelType == LeftRight) + ? cur.GetStartPos() > prev.GetEndPos() ? R : L + : IsMonotonicStep(prev,cur,cov) ? M + : (m_modelType == Monotonic) ? NM + : IsSwap(prev,cur,cov) ? S + : (m_modelType == MSD) ? D + : cur.GetStartPos() > prev.GetEndPos() ? DR : DL); +} + +LRModel:: +LRModel(const std::string &modelType) + : m_modelString(modelType) + , m_scoreProducer(NULL) + , m_modelType(None) + , m_phraseBased(true) + , m_collapseScores(false) + , m_direction(Backward) + , m_additionalScoreComponents(0) +{ + std::vector<std::string> config = Tokenize<std::string>(modelType, "-"); + + for (size_t i=0; i<config.size(); ++i) { + if (config[i] == "hier") { + m_phraseBased = false; + } else if (config[i] == "phrase") { + m_phraseBased = true; + } else if (config[i] == "wbe") { + m_phraseBased = true; + } + // no word-based decoding available, fall-back to phrase-based + // This is the old lexical reordering model combination of moses + + else if (config[i] == "msd") { + m_modelType = MSD; + } else if (config[i] == "mslr") { + m_modelType = MSLR; + } else if (config[i] == "monotonicity") { + m_modelType = Monotonic; + } else if (config[i] == "leftright") { + m_modelType = LeftRight; + } + + // unidirectional is deprecated, use backward instead + else if (config[i] == "unidirectional") { + m_direction = Backward; + } else if (config[i] == "backward") { + m_direction = Backward; + } else if (config[i] == "forward") { + m_direction = Forward; + } else if (config[i] == "bidirectional") { + m_direction = Bidirectional; + } + + else if (config[i] == "f") { + m_condition = F; + } else if (config[i] == "fe") { + m_condition = FE; + } + + else if (config[i] == "collapseff") { + m_collapseScores = true; + } else if (config[i] == "allff") { + m_collapseScores = false; + } else { + std::cerr + << "Illegal part in the lexical reordering configuration string: " + << config[i] << std::endl; + exit(1); + } + } + + if (m_modelType == None) { + std::cerr + << "You need to specify the type of the reordering model " + << "(msd, monotonicity,...)" << std::endl; + exit(1); + } +} + +LRState * +LRModel:: +CreateLRState(const InputType &input) const +{ + LRState *bwd = NULL, *fwd = NULL; + size_t offset = 0; + + switch(m_direction) { + case Backward: + case Bidirectional: + if (m_phraseBased) + bwd = new PhraseBasedReorderingState(*this, Backward, offset); + else + bwd = new HReorderingBackwardState(*this, offset); + offset += m_collapseScores ? 1 : GetNumberOfTypes(); + if (m_direction == Backward) return bwd; // else fall through + case Forward: + if (m_phraseBased) + fwd = new PhraseBasedReorderingState(*this, Forward, offset); + else + fwd = new HReorderingForwardState(*this, input.GetSize(), offset); + offset += m_collapseScores ? 1 : GetNumberOfTypes(); + if (m_direction == Forward) return fwd; + } + return new BidirectionalReorderingState(*this, bwd, fwd, 0); +} + +} + diff --git a/moses/FF/LexicalReordering/LRModel.h b/moses/FF/LexicalReordering/LRModel.h new file mode 100644 index 000000000..bcfca1be8 --- /dev/null +++ b/moses/FF/LexicalReordering/LRModel.h @@ -0,0 +1,133 @@ +#pragma once +#include <string> +#include <map> +#include <boost/scoped_ptr.hpp> + +namespace Moses +{ +class Range; +class Bitmap; +class InputType; +class LRState; +class LexicalReordering; +class SparseReordering; + +//! Factory class for lexical reordering states +class LRModel +{ +public: + friend class LexicalReordering; + enum ModelType { Monotonic, MSD, MSLR, LeftRight, None }; + enum Direction { Forward, Backward, Bidirectional }; + enum Condition { F, E, FE }; + + // constants for the different types of reordering + // (correspond to indices in the respective table) +#if 0 + typedef int ReorderingType; + static const ReorderingType M = 0; // monotonic + static const ReorderingType NM = 1; // non-monotonic + static const ReorderingType S = 1; // swap + static const ReorderingType D = 2; // discontinuous + static const ReorderingType DL = 2; // discontinuous, left + static const ReorderingType DR = 3; // discontinuous, right + static const ReorderingType R = 0; // right + static const ReorderingType L = 1; // left + static const ReorderingType MAX = 3; // largest possible +#else + enum ReorderingType { + M = 0, // monotonic + NM = 1, // non-monotonic + S = 1, // swap + D = 2, // discontinuous + DL = 2, // discontinuous, left + DR = 3, // discontinuous, right + R = 0, // right + L = 1, // left + MAX = 3, // largest possible + NONE = 4 // largest possible + }; +#endif + // determine orientation, depending on model: + + + ReorderingType // for first phrase in phrase-based + GetOrientation(Range const& cur) const; + + ReorderingType // for non-first phrases in phrase-based + GetOrientation(Range const& prev, Range const& cur) const; + + ReorderingType // for HReorderingForwardState + GetOrientation(Range const& prev, Range const& cur, + Bitmap const& cov) const; + + ReorderingType // for HReorderingBackwarddState + GetOrientation(int const reoDistance) const; + + LRModel(const std::string &modelType); + + void + ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, + const LexicalReordering* producer); + + LRState* + CreateLRState(const InputType &input) const; + + size_t GetNumberOfTypes() const; + size_t GetNumScoreComponents() const; + void SetAdditionalScoreComponents(size_t number); + + LexicalReordering* + GetScoreProducer() const { + return m_scoreProducer; + } + + ModelType GetModelType() const { + return m_modelType; + } + Direction GetDirection() const { + return m_direction; + } + Condition GetCondition() const { + return m_condition; + } + + bool + IsPhraseBased() const { + return m_phraseBased; + } + + bool + CollapseScores() const { + return m_collapseScores; + } + + SparseReordering const* + GetSparseReordering() const { + return m_sparse.get(); + } + +private: + void + SetScoreProducer(LexicalReordering* scoreProducer) { + m_scoreProducer = scoreProducer; + } + + std::string const& + GetModelString() const { + return m_modelString; + } + + std::string m_modelString; + LexicalReordering *m_scoreProducer; + ModelType m_modelType; + bool m_phraseBased; + bool m_collapseScores; + Direction m_direction; + Condition m_condition; + size_t m_additionalScoreComponents; + boost::scoped_ptr<SparseReordering> m_sparse; +}; + +} + diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.cpp b/moses/FF/LexicalReordering/LexicalReorderingState.cpp index aeb4a572e..c62e33154 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingState.cpp @@ -10,220 +10,10 @@ #include "LexicalReordering.h" #include "LexicalReorderingState.h" -#include "ReorderingStack.h" -#include "HReorderingForwardState.h" -#include "HReorderingBackwardState.h" -#include "PhraseBasedReorderingState.h" -#include "BidirectionalReorderingState.h" namespace Moses { -bool -IsMonotonicStep(Range const& prev, // words range of last source phrase - Range const& cur, // words range of current source phrase - Bitmap const& cov) // coverage bitmap -{ - size_t e = prev.GetEndPos() + 1; - size_t s = cur.GetStartPos(); - return (s == e || (s >= e && !cov.GetValue(e))); -} - -bool -IsSwap(Range const& prev, Range const& cur, Bitmap const& cov) -{ - size_t s = prev.GetStartPos(); - size_t e = cur.GetEndPos(); - return (e+1 == s || (e < s && !cov.GetValue(s-1))); -} - -size_t -LRModel:: -GetNumberOfTypes() const -{ - return ((m_modelType == MSD) ? 3 : - (m_modelType == MSLR) ? 4 : 2); -} - -size_t -LRModel:: -GetNumScoreComponents() const -{ - size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); - return ((m_direction == Bidirectional) - ? 2 * score_per_dir + m_additionalScoreComponents - : score_per_dir + m_additionalScoreComponents); -} - -void -LRModel:: -ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, - const LexicalReordering* producer) -{ - if (sparseArgs.size()) { - m_sparse.reset(new SparseReordering(sparseArgs, producer)); - } -} - -void -LRModel:: -SetAdditionalScoreComponents(size_t number) -{ - m_additionalScoreComponents = number; -} - -/// return orientation for the first phrase -LRModel::ReorderingType -LRModel:: -GetOrientation(Range const& cur) const -{ - UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None"); - return ((m_modelType == LeftRight) ? R : - (cur.GetStartPos() == 0) ? M : - (m_modelType == MSD) ? D : - (m_modelType == MSLR) ? DR : NM); -} - -LRModel::ReorderingType -LRModel:: -GetOrientation(Range const& prev, Range const& cur) const -{ - UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified"); - return ((m_modelType == LeftRight) - ? prev.GetEndPos() <= cur.GetStartPos() ? R : L - : (cur.GetStartPos() == prev.GetEndPos() + 1) ? M - : (m_modelType == Monotonic) ? NM - : (prev.GetStartPos() == cur.GetEndPos() + 1) ? S - : (m_modelType == MSD) ? D - : (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL); -} - -LRModel::ReorderingType -LRModel:: -GetOrientation(int const reoDistance) const -{ - // this one is for HierarchicalReorderingBackwardState - return ((m_modelType == LeftRight) - ? (reoDistance >= 1) ? R : L - : (reoDistance == 1) ? M - : (m_modelType == Monotonic) ? NM - : (reoDistance == -1) ? S - : (m_modelType == MSD) ? D - : (reoDistance > 1) ? DR : DL); -} - -LRModel::ReorderingType -LRModel:: -GetOrientation(Range const& prev, Range const& cur, - Bitmap const& cov) const -{ - return ((m_modelType == LeftRight) - ? cur.GetStartPos() > prev.GetEndPos() ? R : L - : IsMonotonicStep(prev,cur,cov) ? M - : (m_modelType == Monotonic) ? NM - : IsSwap(prev,cur,cov) ? S - : (m_modelType == MSD) ? D - : cur.GetStartPos() > prev.GetEndPos() ? DR : DL); -} - -LRModel:: -LRModel(const std::string &modelType) - : m_modelString(modelType) - , m_scoreProducer(NULL) - , m_modelType(None) - , m_phraseBased(true) - , m_collapseScores(false) - , m_direction(Backward) - , m_additionalScoreComponents(0) -{ - std::vector<std::string> config = Tokenize<std::string>(modelType, "-"); - - for (size_t i=0; i<config.size(); ++i) { - if (config[i] == "hier") { - m_phraseBased = false; - } else if (config[i] == "phrase") { - m_phraseBased = true; - } else if (config[i] == "wbe") { - m_phraseBased = true; - } - // no word-based decoding available, fall-back to phrase-based - // This is the old lexical reordering model combination of moses - - else if (config[i] == "msd") { - m_modelType = MSD; - } else if (config[i] == "mslr") { - m_modelType = MSLR; - } else if (config[i] == "monotonicity") { - m_modelType = Monotonic; - } else if (config[i] == "leftright") { - m_modelType = LeftRight; - } - - // unidirectional is deprecated, use backward instead - else if (config[i] == "unidirectional") { - m_direction = Backward; - } else if (config[i] == "backward") { - m_direction = Backward; - } else if (config[i] == "forward") { - m_direction = Forward; - } else if (config[i] == "bidirectional") { - m_direction = Bidirectional; - } - - else if (config[i] == "f") { - m_condition = F; - } else if (config[i] == "fe") { - m_condition = FE; - } - - else if (config[i] == "collapseff") { - m_collapseScores = true; - } else if (config[i] == "allff") { - m_collapseScores = false; - } else { - std::cerr - << "Illegal part in the lexical reordering configuration string: " - << config[i] << std::endl; - exit(1); - } - } - - if (m_modelType == None) { - std::cerr - << "You need to specify the type of the reordering model " - << "(msd, monotonicity,...)" << std::endl; - exit(1); - } -} - -LRState * -LRModel:: -CreateLRState(const InputType &input) const -{ - LRState *bwd = NULL, *fwd = NULL; - size_t offset = 0; - - switch(m_direction) { - case Backward: - case Bidirectional: - if (m_phraseBased) - bwd = new PhraseBasedReorderingState(*this, Backward, offset); - else - bwd = new HReorderingBackwardState(*this, offset); - offset += m_collapseScores ? 1 : GetNumberOfTypes(); - if (m_direction == Backward) return bwd; // else fall through - case Forward: - if (m_phraseBased) - fwd = new PhraseBasedReorderingState(*this, Forward, offset); - else - fwd = new HReorderingForwardState(*this, input.GetSize(), offset); - offset += m_collapseScores ? 1 : GetNumberOfTypes(); - if (m_direction == Forward) return fwd; - } - return new BidirectionalReorderingState(*this, bwd, fwd, 0); -} - - void LRState:: CopyScores(ScoreComponentCollection* accum, diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.h b/moses/FF/LexicalReordering/LexicalReorderingState.h index 4a2ad2d7b..22987e04b 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.h +++ b/moses/FF/LexicalReordering/LexicalReorderingState.h @@ -3,138 +3,16 @@ #include <vector> #include <string> -#include <boost/scoped_ptr.hpp> - #include "moses/Hypothesis.h" #include "moses/ScoreComponentCollection.h" #include "moses/Range.h" #include "moses/Bitmap.h" #include "moses/TranslationOption.h" #include "moses/FF/FFState.h" -#include "ReorderingStack.h" +#include "LRModel.h" namespace Moses { -class LRState; -class LexicalReordering; -class SparseReordering; - -//! Factory class for lexical reordering states -class LRModel -{ -public: - friend class LexicalReordering; - enum ModelType { Monotonic, MSD, MSLR, LeftRight, None }; - enum Direction { Forward, Backward, Bidirectional }; - enum Condition { F, E, FE }; - - // constants for the different types of reordering - // (correspond to indices in the respective table) -#if 0 - typedef int ReorderingType; - static const ReorderingType M = 0; // monotonic - static const ReorderingType NM = 1; // non-monotonic - static const ReorderingType S = 1; // swap - static const ReorderingType D = 2; // discontinuous - static const ReorderingType DL = 2; // discontinuous, left - static const ReorderingType DR = 3; // discontinuous, right - static const ReorderingType R = 0; // right - static const ReorderingType L = 1; // left - static const ReorderingType MAX = 3; // largest possible -#else - enum ReorderingType { - M = 0, // monotonic - NM = 1, // non-monotonic - S = 1, // swap - D = 2, // discontinuous - DL = 2, // discontinuous, left - DR = 3, // discontinuous, right - R = 0, // right - L = 1, // left - MAX = 3, // largest possible - NONE = 4 // largest possible - }; -#endif - // determine orientation, depending on model: - - - ReorderingType // for first phrase in phrase-based - GetOrientation(Range const& cur) const; - - ReorderingType // for non-first phrases in phrase-based - GetOrientation(Range const& prev, Range const& cur) const; - - ReorderingType // for HReorderingForwardState - GetOrientation(Range const& prev, Range const& cur, - Bitmap const& cov) const; - - ReorderingType // for HReorderingBackwarddState - GetOrientation(int const reoDistance) const; - - LRModel(const std::string &modelType); - - void - ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, - const LexicalReordering* producer); - - LRState* - CreateLRState(const InputType &input) const; - - size_t GetNumberOfTypes() const; - size_t GetNumScoreComponents() const; - void SetAdditionalScoreComponents(size_t number); - - LexicalReordering* - GetScoreProducer() const { - return m_scoreProducer; - } - - ModelType GetModelType() const { - return m_modelType; - } - Direction GetDirection() const { - return m_direction; - } - Condition GetCondition() const { - return m_condition; - } - - bool - IsPhraseBased() const { - return m_phraseBased; - } - - bool - CollapseScores() const { - return m_collapseScores; - } - - SparseReordering const* - GetSparseReordering() const { - return m_sparse.get(); - } - -private: - void - SetScoreProducer(LexicalReordering* scoreProducer) { - m_scoreProducer = scoreProducer; - } - - std::string const& - GetModelString() const { - return m_modelString; - } - - std::string m_modelString; - LexicalReordering *m_scoreProducer; - ModelType m_modelType; - bool m_phraseBased; - bool m_collapseScores; - Direction m_direction; - Condition m_condition; - size_t m_additionalScoreComponents; - boost::scoped_ptr<SparseReordering> m_sparse; -}; //! Abstract class for lexical reordering model states class LRState : public FFState |