diff options
author | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 04:45:22 +0300 |
---|---|---|
committer | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 04:45:22 +0300 |
commit | ccf44f39fbef38eab0b818cb74c517b68a27ae0b (patch) | |
tree | 70ed8bb02cf049ffa6373572f3ded4fed0b43711 /moses | |
parent | e520a3d39e212110a6d390dace6f587b20e7e91a (diff) |
Code cleanup and reorganization. A few classes have been renamed to shorter names.
Diffstat (limited to 'moses')
6 files changed, 734 insertions, 689 deletions
diff --git a/moses/FF/LexicalReordering/LexicalReordering.cpp b/moses/FF/LexicalReordering/LexicalReordering.cpp index 0630d1077..32693984e 100644 --- a/moses/FF/LexicalReordering/LexicalReordering.cpp +++ b/moses/FF/LexicalReordering/LexicalReordering.cpp @@ -11,110 +11,125 @@ using namespace boost::algorithm; namespace Moses { -LexicalReordering::LexicalReordering(const std::string &line) - : StatefulFeatureFunction(line) -{ - std::cerr << "Initializing LexicalReordering.." << std::endl; - - map<string,string> sparseArgs; - m_haveDefaultScores = false; - for (size_t i = 0; i < m_args.size(); ++i) { - const vector<string> &args = m_args[i]; - - if (args[0] == "type") { - m_configuration.reset(new LexicalReorderingConfiguration(args[1])); - m_configuration->SetScoreProducer(this); - m_modelTypeString = m_configuration->GetModelString(); - } else if (args[0] == "input-factor") { - m_factorsF =Tokenize<FactorType>(args[1]); - } else if (args[0] == "output-factor") { - m_factorsE =Tokenize<FactorType>(args[1]); - } else if (args[0] == "path") { - m_filePath = args[1]; - } else if (starts_with(args[0], "sparse-")) { - sparseArgs[args[0].substr(7)] = args[1]; - } else if (args[0] == "default-scores") { - vector<string> tokens = Tokenize(args[1],","); - for(size_t i=0; i<tokens.size(); i++) { - m_defaultScores.push_back( TransformScore( Scan<float>(tokens[i]) ) ); + + LexicalReordering:: + LexicalReordering(const std::string &line) + : StatefulFeatureFunction(line) + { + VERBOSE(1, "Initializing LexicalReordering.." << std::endl); + + map<string,string> sparseArgs; + m_haveDefaultScores = false; + for (size_t i = 0; i < m_args.size(); ++i) { + const vector<string> &args = m_args[i]; + + if (args[0] == "type") { + m_configuration.reset(new LexicalReorderingConfiguration(args[1])); + m_configuration->SetScoreProducer(this); + m_modelTypeString = m_configuration->GetModelString(); + } else if (args[0] == "input-factor") { + m_factorsF =Tokenize<FactorType>(args[1]); + } else if (args[0] == "output-factor") { + m_factorsE =Tokenize<FactorType>(args[1]); + } else if (args[0] == "path") { + m_filePath = args[1]; + } else if (starts_with(args[0], "sparse-")) { + sparseArgs[args[0].substr(7)] = args[1]; + } else if (args[0] == "default-scores") { + vector<string> tokens = Tokenize(args[1],","); + for(size_t i=0; i<tokens.size(); i++) { + m_defaultScores.push_back( TransformScore( Scan<float>(tokens[i]) ) ); + } + m_haveDefaultScores = true; + } else { + UTIL_THROW(util::Exception,"Unknown argument " + args[0]); } - m_haveDefaultScores = true; - } else { - UTIL_THROW(util::Exception,"Unknown argument " + args[0]); } - } - switch(m_configuration->GetCondition()) { - case LexicalReorderingConfiguration::FE: - case LexicalReorderingConfiguration::E: - if(m_factorsE.empty()) { - UTIL_THROW(util::Exception,"TL factor mask for lexical reordering is unexpectedly empty"); - } - if(m_configuration->GetCondition() == LexicalReorderingConfiguration::E) - break; // else fall through - case LexicalReorderingConfiguration::F: - if(m_factorsF.empty()) { - UTIL_THROW(util::Exception,"SL factor mask for lexical reordering is unexpectedly empty"); + switch(m_configuration->GetCondition()) { + case LexicalReorderingConfiguration::FE: + case LexicalReorderingConfiguration::E: + if(m_factorsE.empty()) { + UTIL_THROW(util::Exception, + "TL factor mask for lexical reordering is unexpectedly empty"); + } + if(m_configuration->GetCondition() == LexicalReorderingConfiguration::E) + break; // else fall through + case LexicalReorderingConfiguration::F: + if(m_factorsF.empty()) { + UTIL_THROW(util::Exception, + "SL factor mask for lexical reordering is unexpectedly empty"); + } + break; + default: + UTIL_THROW(util::Exception,"Unknown conditioning option!"); } - break; - default: - UTIL_THROW(util::Exception,"Unknown conditioning option!"); - } - // sanity check: number of default scores - if (m_haveDefaultScores) { - if(m_defaultScores.size() != m_configuration->GetNumScoreComponents()) { - UTIL_THROW(util::Exception,"wrong number of default scores (" << m_defaultScores.size() << ") for lexicalized reordering model (expected " << m_configuration->GetNumScoreComponents() << ")"); - } + // sanity check: number of default scores + if (m_haveDefaultScores) + { + if(m_defaultScores.size() != m_configuration->GetNumScoreComponents()) + { + UTIL_THROW(util::Exception,"wrong number of default scores (" + << m_defaultScores.size() + << ") for lexicalized reordering model (expected " + << m_configuration->GetNumScoreComponents() << ")"); + } + } + + m_configuration->ConfigureSparse(sparseArgs, this); } - m_configuration->ConfigureSparse(sparseArgs, this); -} - -LexicalReordering::~LexicalReordering() -{ -} - -void LexicalReordering::Load() -{ - m_table.reset(LexicalReorderingTable::LoadAvailable(m_filePath, m_factorsF, m_factorsE, std::vector<FactorType>())); -} - -Scores LexicalReordering::GetProb(const Phrase& f, const Phrase& e) const -{ - return m_table->GetScore(f, e, Phrase(ARRAY_SIZE_INCR)); -} - -FFState* LexicalReordering::EvaluateWhenApplied(const Hypothesis& hypo, - const FFState* prev_state, - ScoreComponentCollection* out) const -{ - VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) START" << std::endl); - Scores score(GetNumScoreComponents(), 0); - const LexicalReorderingState *prev = dynamic_cast<const LexicalReorderingState *>(prev_state); - LexicalReorderingState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out); + LexicalReordering:: + ~LexicalReordering() + { + } + + void LexicalReordering::Load() + { + typedef LexicalReorderingTable LRT; + m_table.reset(LRT::LoadAvailable(m_filePath, m_factorsF, m_factorsE, + std::vector<FactorType>())); + } - out->PlusEquals(this, score); - VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) END" << std::endl); + Scores LexicalReordering::GetProb(const Phrase& f, const Phrase& e) const + { + return m_table->GetScore(f, e, Phrase(ARRAY_SIZE_INCR)); + } - return next_state; -} + FFState* + LexicalReordering:: + EvaluateWhenApplied(const Hypothesis& hypo, + const FFState* prev_state, + ScoreComponentCollection* out) const + { + VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) START" + << std::endl); + Scores score(GetNumScoreComponents(), 0); + const LexicalReorderingState *prev = dynamic_cast<const LexicalReorderingState *>(prev_state); + LexicalReorderingState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out); + + out->PlusEquals(this, score); + VERBOSE(3,"LexicalReordering::Evaluate(const Hypothesis& hypo,...) END" << std::endl); + + return next_state; + } -const FFState* LexicalReordering::EmptyHypothesisState(const InputType &input) const -{ - return m_configuration->CreateLexicalReorderingState(input); -} + const FFState* LexicalReordering::EmptyHypothesisState(const InputType &input) const + { + return m_configuration->CreateLexicalReorderingState(input); + } -bool LexicalReordering::IsUseable(const FactorMask &mask) const -{ - for (size_t i = 0; i < m_factorsE.size(); ++i) { - const FactorType &factor = m_factorsE[i]; - if (!mask[factor]) { - return false; + bool LexicalReordering::IsUseable(const FactorMask &mask) const + { + for (size_t i = 0; i < m_factorsE.size(); ++i) { + const FactorType &factor = m_factorsE[i]; + if (!mask[factor]) { + return false; + } } - } - return true; + return true; -} + } } diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.cpp b/moses/FF/LexicalReordering/LexicalReorderingState.cpp index 567d1b713..0bb9344bf 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingState.cpp @@ -1,4 +1,4 @@ - +// -*- c++ -*- #include <vector> #include <string> @@ -14,506 +14,521 @@ namespace Moses { -size_t LexicalReorderingConfiguration::GetNumberOfTypes() const -{ - switch (m_modelType) { - case LexicalReorderingConfiguration::MSD: - return 3; - break; - case LexicalReorderingConfiguration::MSLR: - return 4; - break; - default: - return 2; - } -} + typedef LexicalReorderingConfiguration LexReoConf; -size_t LexicalReorderingConfiguration::GetNumScoreComponents() const -{ - size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); - if (m_direction == Bidirectional) { - return 2 * score_per_dir + m_additionalScoreComponents; - } else { - return score_per_dir + m_additionalScoreComponents; + bool + IsMonotonicStep(WordsRange const& prev, // words range of last source phrase + WordsRange const& cur, // words range of current source phrase + WordsBitmap const& cov) // coverage bitmap + { + size_t e = prev.GetEndPos() + 1; + size_t s = cur.GetStartPos(); + return (s == e || (s >= e && !cov.GetValue(e))); } -} - -void LexicalReorderingConfiguration::ConfigureSparse -(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer) -{ - if (sparseArgs.size()) { - m_sparse.reset(new SparseReordering(sparseArgs, producer)); + + bool + IsSwap(WordsRange const& prev, WordsRange const& cur, WordsBitmap const& cov) + { + size_t s = prev.GetStartPos(); + size_t e = cur.GetEndPos(); + return (e+1 == s || (e < s && !cov.GetValue(s-1))); } -} -void LexicalReorderingConfiguration::SetAdditionalScoreComponents(size_t number) -{ - m_additionalScoreComponents = number; -} - -LexicalReorderingConfiguration::LexicalReorderingConfiguration(const std::string &modelType) - : m_modelString(modelType), m_scoreProducer(NULL), m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(Backward), m_additionalScoreComponents(0) -{ - std::vector<std::string> config = Tokenize<std::string>(modelType, "-"); - - for (size_t i=0; i<config.size(); ++i) { - if (config[i] == "hier") { - m_phraseBased = false; - } else if (config[i] == "phrase") { - m_phraseBased = true; - } else if (config[i] == "wbe") { - m_phraseBased = true; - // no word-based decoding available, fall-back to phrase-based - // This is the old lexical reordering model combination of moses - } else if (config[i] == "msd") { - m_modelType = MSD; - } else if (config[i] == "mslr") { - m_modelType = MSLR; - } else if (config[i] == "monotonicity") { - m_modelType = Monotonic; - } else if (config[i] == "leftright") { - m_modelType = LeftRight; - } else if (config[i] == "backward" || config[i] == "unidirectional") { - // note: unidirectional is deprecated, use backward instead - m_direction = Backward; - } else if (config[i] == "forward") { - m_direction = Forward; - } else if (config[i] == "bidirectional") { - m_direction = Bidirectional; - } else if (config[i] == "f") { - m_condition = F; - } else if (config[i] == "fe") { - m_condition = FE; - } else if (config[i] == "collapseff") { - m_collapseScores = true; - } else if (config[i] == "allff") { - m_collapseScores = false; - } else { - std::cerr << "Illegal part in the lexical reordering configuration string: " << config[i] << std::endl; - exit(1); - } + size_t + LexicalReorderingConfiguration:: + GetNumberOfTypes() const + { + return ((m_modelType == LexReoConf::MSD) ? 3 : + (m_modelType == LexReoConf::MSLR) ? 4 : 2); } - - if (m_modelType == None) { - std::cerr << "You need to specify the type of the reordering model (msd, monotonicity,...)" << std::endl; - exit(1); + + size_t + LexicalReorderingConfiguration:: + GetNumScoreComponents() const + { + size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); + return ((m_direction == Bidirectional) + ? 2 * score_per_dir + m_additionalScoreComponents + : score_per_dir + m_additionalScoreComponents); } -} - -LexicalReorderingState *LexicalReorderingConfiguration::CreateLexicalReorderingState(const InputType &input) const -{ - LexicalReorderingState *bwd = NULL, *fwd = NULL; - size_t offset = 0; - - switch(m_direction) { - case Backward: - case Bidirectional: - if (m_phraseBased) { //Same for forward and backward - bwd = new PhraseBasedReorderingState(*this, LexicalReorderingConfiguration::Backward, offset); - } else { - bwd = new HierarchicalReorderingBackwardState(*this, offset); - } - offset += m_collapseScores ? 1 : GetNumberOfTypes(); - if (m_direction == Backward) - return bwd; // else fall through - case Forward: - if (m_phraseBased) { //Same for forward and backward - fwd = new PhraseBasedReorderingState(*this, LexicalReorderingConfiguration::Forward, offset); - } else { - fwd = new HierarchicalReorderingForwardState(*this, input.GetSize(), offset); - } - offset += m_collapseScores ? 1 : GetNumberOfTypes(); - if (m_direction == Forward) - return fwd; + + void + LexicalReorderingConfiguration:: + ConfigureSparse(std::map<std::string,std::string> const& sparseArgs, + const LexicalReordering* producer) + { + if (sparseArgs.size()) + m_sparse.reset(new SparseReordering(sparseArgs, producer)); } - return new BidirectionalReorderingState(*this, bwd, fwd, 0); -} + void LexicalReorderingConfiguration::SetAdditionalScoreComponents(size_t number) + { + m_additionalScoreComponents = number; + } -void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const TranslationOption &topt, const InputType& input, ReorderingType reoType) const -{ - // don't call this on a bidirectional object - UTIL_THROW_IF2(m_direction != LexicalReorderingConfiguration::Backward && m_direction != LexicalReorderingConfiguration::Forward, - "Unknown direction: " << m_direction); - const TranslationOption* relevantOpt = &topt; - if (m_direction != LexicalReorderingConfiguration::Backward) relevantOpt = m_prevOption; - const Scores *cachedScores = relevantOpt->GetLexReorderingScores(m_configuration.GetScoreProducer()); - - // look up applicable score from vectore of scores - if(cachedScores) { - Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0); - - const Scores &scoreSet = *cachedScores; - if(m_configuration.CollapseScores()) { - scores[m_offset] = scoreSet[m_offset + reoType]; - } else { - std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0); - scores[m_offset + reoType] = scoreSet[m_offset + reoType]; - } - accum->PlusEquals(m_configuration.GetScoreProducer(), scores); + LexicalReorderingConfiguration:: + LexicalReorderingConfiguration(const std::string &modelType) + : m_modelString(modelType) + , m_scoreProducer(NULL) + , m_modelType(None) + , m_phraseBased(true) + , m_collapseScores(false) + , m_direction(Backward) + , m_additionalScoreComponents(0) + { + std::vector<std::string> config = Tokenize<std::string>(modelType, "-"); + + for (size_t i=0; i<config.size(); ++i) + { + if (config[i] == "hier") { m_phraseBased = false; } + else if (config[i] == "phrase") { m_phraseBased = true; } + else if (config[i] == "wbe") { m_phraseBased = true; } + // no word-based decoding available, fall-back to phrase-based + // This is the old lexical reordering model combination of moses + + else if (config[i] == "msd") { m_modelType = MSD; } + else if (config[i] == "mslr") { m_modelType = MSLR; } + else if (config[i] == "monotonicity") { m_modelType = Monotonic; } + else if (config[i] == "leftright") { m_modelType = LeftRight; } + + else if (config[i] == "backward") { m_direction = Backward; } + + // note: unidirectional is deprecated, use backward instead + else if (config[i] == "unidirectional") { m_direction = Backward; } + else if (config[i] == "forward") { m_direction = Forward; } + else if (config[i] == "bidirectional") { m_direction = Bidirectional; } + + else if (config[i] == "f") { m_condition = F; } + else if (config[i] == "fe") { m_condition = FE; } + + else if (config[i] == "collapseff") { m_collapseScores = true; } + else if (config[i] == "allff") { m_collapseScores = false; } + else + { + std::cerr + << "Illegal part in the lexical reordering configuration string: " + << config[i] << std::endl; + exit(1); + } + } + + if (m_modelType == None) + { + std::cerr + << "You need to specify the type of the reordering model " + << "(msd, monotonicity,...)" << std::endl; + exit(1); + } } - // else: use default scores (if specified) - else if (m_configuration.GetScoreProducer()->GetHaveDefaultScores()) { - Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0); - if(m_configuration.CollapseScores()) { - scores[m_offset] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType); - } else { - scores[m_offset + reoType] = m_configuration.GetScoreProducer()->GetDefaultScore(m_offset + reoType); - } - accum->PlusEquals(m_configuration.GetScoreProducer(), scores); + + LexicalReorderingState * + LexicalReorderingConfiguration:: + CreateLexicalReorderingState(const InputType &input) const + { + LexicalReorderingState *bwd = NULL, *fwd = NULL; + size_t offset = 0; + + switch(m_direction) + { + case Backward: + case Bidirectional: + bwd = (m_phraseBased + ? new PhraseBasedReorderingState(*this, Backward, offset); + : new HierarchicalReorderingBackwardState(*this, offset)); + offset += m_collapseScores ? 1 : GetNumberOfTypes(); + if (m_direction == Backward) return bwd; // else fall through + case Forward: + fwd = (m_phraseBased + ? new PhraseBasedReorderingState(*this, Forward, offset) + : new HierarchicalReorderingForwardState(*this, input.GetSize(), + offset)); + offset += m_collapseScores ? 1 : GetNumberOfTypes(); + if (m_direction == Forward) return fwd; + } + return new BidirectionalReorderingState(*this, bwd, fwd, 0); } - // note: if no default score, no cost - - const SparseReordering* sparse = m_configuration.GetSparseReordering(); - if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType, m_direction, accum); - -} - -int LexicalReorderingState::ComparePrevScores(const TranslationOption *other) const -{ - const Scores* myPrevScores = m_prevOption->GetLexReorderingScores(m_configuration.GetScoreProducer()); - const Scores* otherPrevScores = other->GetLexReorderingScores(m_configuration.GetScoreProducer()); + void + LexicalReorderingState:: + CopyScores(ScoreComponentCollection* accum, + const TranslationOption &topt, + const InputType& input, + ReorderingType reoType) const + { + // don't call this on a bidirectional object + UTIL_THROW_IF2(m_direction != Backward && m_direction != Forward, + "Unknown direction: " << m_direction); + + TranslationOption const* + relevantOpt = (m_direction == Backward) ? &topt : m_prevOption; + + LexicalReordering* reotable = m_configuration.GetScoreProducer(); + Scores const* cachedScores = relevantOpt->GetLexReorderingScores(reotable); + + size_t off_remote = m_offset + reoType; + size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote; + + // look up applicable score from vectore of scores + if(cachedScores) + { + Scores scores(reotable->GetNumScoreComponents(),0); + socres[off_local ] (*cachedScores)[off_remote]; + accum->PlusEquals(reotable, scores); + } + + // else: use default scores (if specified) + else if (reotable->GetHaveDefaultScores()) + { + Scores scores(reotable->GetNumScoreComponents(),0); + scores[off_local] = reotable->GetDefaultScore(off_remote); + accum->PlusEquals(m_configuration.GetScoreProducer(), scores); + } + // note: if no default score, no cost + + const SparseReordering* sparse = m_configuration.GetSparseReordering(); + if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType, + m_direction, accum); + } + - if(myPrevScores == otherPrevScores) - return 0; + int + LexicalReorderingState:: + ComparePrevScores(const TranslationOption *other) const + { + LexicalReordering* reotable = m_configuration.GetScoreProducer(); + const Scores* myPrevScores = m_prevOption->GetLexReorderingScores(reotable); + const Scores* otherPrevScores = other->GetLexReorderingScores(reotable); - // The pointers are NULL if a phrase pair isn't found in the reordering table. - if(otherPrevScores == NULL) - return -1; - if(myPrevScores == NULL) - return 1; + if(myPrevScores == otherPrevScores) + return 0; - for(size_t i = m_offset; i < m_offset + m_configuration.GetNumberOfTypes(); i++) - if((*myPrevScores)[i] < (*otherPrevScores)[i]) + // The pointers are NULL if a phrase pair isn't found in the reordering table. + if(otherPrevScores == NULL) return -1; - else if((*myPrevScores)[i] > (*otherPrevScores)[i]) + if(myPrevScores == NULL) return 1; - return 0; -} + for(size_t i = m_offset; i < m_offset + m_configuration.GetNumberOfTypes(); i++) + if((*myPrevScores)[i] < (*otherPrevScores)[i]) + return -1; + else if((*myPrevScores)[i] > (*otherPrevScores)[i]) + return 1; -bool PhraseBasedReorderingState::m_useFirstBackwardScore = true; + return 0; + } -PhraseBasedReorderingState::PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt) - : LexicalReorderingState(prev, topt), m_prevRange(topt.GetSourceWordsRange()), m_first(false) {} + bool PhraseBasedReorderingState::m_useFirstBackwardScore = true; + PhraseBasedReorderingState::PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt) + : LexicalReorderingState(prev, topt), m_prevRange(topt.GetSourceWordsRange()), m_first(false) {} -PhraseBasedReorderingState::PhraseBasedReorderingState(const LexicalReorderingConfiguration &config, - LexicalReorderingConfiguration::Direction dir, size_t offset) - : LexicalReorderingState(config, dir, offset), m_prevRange(NOT_FOUND,NOT_FOUND), m_first(true) {} + PhraseBasedReorderingState::PhraseBasedReorderingState(const LexReoConf &config, + LexReoConf::Direction dir, size_t offset) + : LexicalReorderingState(config, dir, offset), m_prevRange(NOT_FOUND,NOT_FOUND), m_first(true) {} -int PhraseBasedReorderingState::Compare(const FFState& o) const -{ - if (&o == this) - return 0; - const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o); - if (m_prevRange == other->m_prevRange) { - if (m_direction == LexicalReorderingConfiguration::Forward) { - return ComparePrevScores(other->m_prevOption); - } else { + int PhraseBasedReorderingState::Compare(const FFState& o) const + { + if (&o == this) return 0; + + const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o); + if (m_prevRange == other->m_prevRange) { + if (m_direction == LexReoConf::Forward) { + return ComparePrevScores(other->m_prevOption); + } else { + return 0; + } + } else if (m_prevRange < other->m_prevRange) { + return -1; } - } else if (m_prevRange < other->m_prevRange) { - return -1; + return 1; } - return 1; -} -LexicalReorderingState* PhraseBasedReorderingState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const -{ - ReorderingType reoType; - const WordsRange currWordsRange = topt.GetSourceWordsRange(); - const LexicalReorderingConfiguration::ModelType modelType = m_configuration.GetModelType(); - - if ((m_direction != LexicalReorderingConfiguration::Forward && m_useFirstBackwardScore) || !m_first) { - if (modelType == LexicalReorderingConfiguration::MSD) { - reoType = GetOrientationTypeMSD(currWordsRange); - } else if (modelType == LexicalReorderingConfiguration::MSLR) { - reoType = GetOrientationTypeMSLR(currWordsRange); - } else if (modelType == LexicalReorderingConfiguration::Monotonic) { - reoType = GetOrientationTypeMonotonic(currWordsRange); - } else { - reoType = GetOrientationTypeLeftRight(currWordsRange); + LexicalReorderingState* PhraseBasedReorderingState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const + { + ReorderingType reoType; + const WordsRange currWordsRange = topt.GetSourceWordsRange(); + const LexReoConf::ModelType modelType = m_configuration.GetModelType(); + + if ((m_direction != LexReoConf::Forward && m_useFirstBackwardScore) || !m_first) { + if (modelType == LexReoConf::MSD) { + reoType = GetOrientationTypeMSD(currWordsRange); + } else if (modelType == LexReoConf::MSLR) { + reoType = GetOrientationTypeMSLR(currWordsRange); + } else if (modelType == LexReoConf::Monotonic) { + reoType = GetOrientationTypeMonotonic(currWordsRange); + } else { + reoType = GetOrientationTypeLeftRight(currWordsRange); + } + CopyScores(scores, topt, input, reoType); } - CopyScores(scores, topt, input, reoType); - } - return new PhraseBasedReorderingState(this, topt); -} + return new PhraseBasedReorderingState(this, topt); + } -LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSD(WordsRange currRange) const -{ - if (m_first) { - if (currRange.GetStartPos() == 0) { + LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSD(WordsRange currRange) const + { + if (m_first) { + if (currRange.GetStartPos() == 0) { + return M; + } else { + return D; + } + } + if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { return M; - } else { - return D; + } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { + return S; } + return D; } - if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { - return M; - } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { - return S; - } - return D; -} -LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSLR(WordsRange currRange) const -{ - if (m_first) { - if (currRange.GetStartPos() == 0) { + LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMSLR(WordsRange currRange) const + { + if (m_first) { + if (currRange.GetStartPos() == 0) { + return M; + } else { + return DR; + } + } + if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { return M; - } else { + } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { + return S; + } else if (m_prevRange.GetEndPos() < currRange.GetStartPos()) { return DR; } + return DL; } - if (m_prevRange.GetEndPos() == currRange.GetStartPos()-1) { - return M; - } else if (m_prevRange.GetStartPos() == currRange.GetEndPos()+1) { - return S; - } else if (m_prevRange.GetEndPos() < currRange.GetStartPos()) { - return DR; - } - return DL; -} -LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMonotonic(WordsRange currRange) const -{ - if ((m_first && currRange.GetStartPos() == 0) || - (m_prevRange.GetEndPos() == currRange.GetStartPos()-1)) { - return M; + LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeMonotonic(WordsRange currRange) const + { + if ((m_first && currRange.GetStartPos() == 0) || + (m_prevRange.GetEndPos() == currRange.GetStartPos()-1)) { + return M; + } + return NM; } - return NM; -} -LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeLeftRight(WordsRange currRange) const -{ - if (m_first || - (m_prevRange.GetEndPos() <= currRange.GetStartPos())) { - return R; + LexicalReorderingState::ReorderingType PhraseBasedReorderingState::GetOrientationTypeLeftRight(WordsRange currRange) const + { + if (m_first || + (m_prevRange.GetEndPos() <= currRange.GetStartPos())) { + return R; + } + return L; } - return L; -} -/////////////////////////// -//BidirectionalReorderingState + /////////////////////////// + //BidirectionalReorderingState -int BidirectionalReorderingState::Compare(const FFState& o) const -{ - if (&o == this) - return 0; - - const BidirectionalReorderingState &other = static_cast<const BidirectionalReorderingState &>(o); - if(m_backward->Compare(*other.m_backward) < 0) - return -1; - else if(m_backward->Compare(*other.m_backward) > 0) - return 1; - else - return m_forward->Compare(*other.m_forward); -} + int BidirectionalReorderingState::Compare(const FFState& o) const + { + if (&o == this) + return 0; -LexicalReorderingState* BidirectionalReorderingState::Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const -{ - LexicalReorderingState *newbwd = m_backward->Expand(topt,input, scores); - LexicalReorderingState *newfwd = m_forward->Expand(topt, input, scores); - return new BidirectionalReorderingState(m_configuration, newbwd, newfwd, m_offset); -} + const BidirectionalReorderingState &other = static_cast<const BidirectionalReorderingState &>(o); + if(m_backward->Compare(*other.m_backward) < 0) + return -1; + else if(m_backward->Compare(*other.m_backward) > 0) + return 1; + else + return m_forward->Compare(*other.m_forward); + } -/////////////////////////// -//HierarchicalReorderingBackwardState + LexicalReorderingState* BidirectionalReorderingState::Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const + { + LexicalReorderingState *newbwd = m_backward->Expand(topt,input, scores); + LexicalReorderingState *newfwd = m_forward->Expand(topt, input, scores); + return new BidirectionalReorderingState(m_configuration, newbwd, newfwd, m_offset); + } -HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const HierarchicalReorderingBackwardState *prev, - const TranslationOption &topt, ReorderingStack reoStack) - : LexicalReorderingState(prev, topt), m_reoStack(reoStack) {} + /////////////////////////// + //HierarchicalReorderingBackwardState -HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const LexicalReorderingConfiguration &config, size_t offset) - : LexicalReorderingState(config, LexicalReorderingConfiguration::Backward, offset) {} + HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const HierarchicalReorderingBackwardState *prev, + const TranslationOption &topt, ReorderingStack reoStack) + : LexicalReorderingState(prev, topt), m_reoStack(reoStack) {} + HierarchicalReorderingBackwardState::HierarchicalReorderingBackwardState(const LexReoConf &config, size_t offset) + : LexicalReorderingState(config, LexReoConf::Backward, offset) {} -int HierarchicalReorderingBackwardState::Compare(const FFState& o) const -{ - const HierarchicalReorderingBackwardState& other = static_cast<const HierarchicalReorderingBackwardState&>(o); - return m_reoStack.Compare(other.m_reoStack); -} -LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const -{ + int HierarchicalReorderingBackwardState::Compare(const FFState& o) const + { + const HierarchicalReorderingBackwardState& other = static_cast<const HierarchicalReorderingBackwardState&>(o); + return m_reoStack.Compare(other.m_reoStack); + } - HierarchicalReorderingBackwardState* nextState = new HierarchicalReorderingBackwardState(this, topt, m_reoStack); - ReorderingType reoType; - const LexicalReorderingConfiguration::ModelType modelType = m_configuration.GetModelType(); + LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const + { - int reoDistance = nextState->m_reoStack.ShiftReduce(topt.GetSourceWordsRange()); + HierarchicalReorderingBackwardState* nextState = new HierarchicalReorderingBackwardState(this, topt, m_reoStack); + ReorderingType reoType; + const LexReoConf::ModelType modelType = m_configuration.GetModelType(); - if (modelType == LexicalReorderingConfiguration::MSD) { - reoType = GetOrientationTypeMSD(reoDistance); - } else if (modelType == LexicalReorderingConfiguration::MSLR) { - reoType = GetOrientationTypeMSLR(reoDistance); - } else if (modelType == LexicalReorderingConfiguration::LeftRight) { - reoType = GetOrientationTypeLeftRight(reoDistance); - } else { - reoType = GetOrientationTypeMonotonic(reoDistance); - } + int reoDistance = nextState->m_reoStack.ShiftReduce(topt.GetSourceWordsRange()); - CopyScores(scores, topt, input, reoType); - return nextState; -} + if (modelType == LexReoConf::MSD) { + reoType = GetOrientationTypeMSD(reoDistance); + } else if (modelType == LexReoConf::MSLR) { + reoType = GetOrientationTypeMSLR(reoDistance); + } else if (modelType == LexReoConf::LeftRight) { + reoType = GetOrientationTypeLeftRight(reoDistance); + } else { + reoType = GetOrientationTypeMonotonic(reoDistance); + } -LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSD(int reoDistance) const -{ - if (reoDistance == 1) { - return M; - } else if (reoDistance == -1) { - return S; + CopyScores(scores, topt, input, reoType); + return nextState; } - return D; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSLR(int reoDistance) const -{ - if (reoDistance == 1) { - return M; - } else if (reoDistance == -1) { - return S; - } else if (reoDistance > 1) { - return DR; + LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSD(int reoDistance) const + { + if (reoDistance == 1) { + return M; + } else if (reoDistance == -1) { + return S; + } + return D; } - return DL; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMonotonic(int reoDistance) const -{ - if (reoDistance == 1) { - return M; + LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMSLR(int reoDistance) const + { + if (reoDistance == 1) { + return M; + } else if (reoDistance == -1) { + return S; + } else if (reoDistance > 1) { + return DR; + } + return DL; } - return NM; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeLeftRight(int reoDistance) const -{ - if (reoDistance >= 1) { - return R; + LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeMonotonic(int reoDistance) const + { + if (reoDistance == 1) { + return M; + } + return NM; } - return L; -} + LexicalReorderingState::ReorderingType HierarchicalReorderingBackwardState::GetOrientationTypeLeftRight(int reoDistance) const + { + if (reoDistance >= 1) { + return R; + } + return L; + } -/////////////////////////// -//HierarchicalReorderingForwardState -HierarchicalReorderingForwardState::HierarchicalReorderingForwardState(const LexicalReorderingConfiguration &config, size_t size, size_t offset) - : LexicalReorderingState(config, LexicalReorderingConfiguration::Forward, offset), m_first(true), m_prevRange(NOT_FOUND,NOT_FOUND), m_coverage(size) {} + /////////////////////////// + //HierarchicalReorderingForwardState -HierarchicalReorderingForwardState::HierarchicalReorderingForwardState(const HierarchicalReorderingForwardState *prev, const TranslationOption &topt) - : LexicalReorderingState(prev, topt), m_first(false), m_prevRange(topt.GetSourceWordsRange()), m_coverage(prev->m_coverage) -{ - const WordsRange currWordsRange = topt.GetSourceWordsRange(); - m_coverage.SetValue(currWordsRange.GetStartPos(), currWordsRange.GetEndPos(), true); -} + HierarchicalReorderingForwardState:: + HierarchicalReorderingForwardState(const LexReoConf &config, size_t size, size_t offset) + : LexicalReorderingState(config, LexReoConf::Forward, offset), m_first(true), m_prevRange(NOT_FOUND,NOT_FOUND), m_coverage(size) {} -int HierarchicalReorderingForwardState::Compare(const FFState& o) const -{ - if (&o == this) - return 0; + HierarchicalReorderingForwardState::HierarchicalReorderingForwardState(const HierarchicalReorderingForwardState *prev, const TranslationOption &topt) + : LexicalReorderingState(prev, topt), m_first(false), m_prevRange(topt.GetSourceWordsRange()), m_coverage(prev->m_coverage) + { + const WordsRange currWordsRange = topt.GetSourceWordsRange(); + m_coverage.SetValue(currWordsRange.GetStartPos(), currWordsRange.GetEndPos(), true); + } - const HierarchicalReorderingForwardState* other = static_cast<const HierarchicalReorderingForwardState*>(&o); + int HierarchicalReorderingForwardState::Compare(const FFState& o) const + { + if (&o == this) + return 0; - if (m_prevRange == other->m_prevRange) { - return ComparePrevScores(other->m_prevOption); - } else if (m_prevRange < other->m_prevRange) { - return -1; - } - return 1; -} + const HierarchicalReorderingForwardState* other = static_cast<const HierarchicalReorderingForwardState*>(&o); -// For compatibility with the phrase-based reordering model, scoring is one step delayed. -// The forward model takes determines orientations heuristically as follows: -// mono: if the next phrase comes after the conditioning phrase and -// - there is a gap to the right of the conditioning phrase, or -// - the next phrase immediately follows it -// swap: if the next phrase goes before the conditioning phrase and -// - there is a gap to the left of the conditioning phrase, or -// - the next phrase immediately precedes it -// dright: if the next phrase follows the conditioning phrase and other stuff comes in between -// dleft: if the next phrase precedes the conditioning phrase and other stuff comes in between - -LexicalReorderingState* HierarchicalReorderingForwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const -{ - const LexicalReorderingConfiguration::ModelType modelType = m_configuration.GetModelType(); - const WordsRange currWordsRange = topt.GetSourceWordsRange(); - // keep track of the current coverage ourselves so we don't need the hypothesis - WordsBitmap coverage = m_coverage; - coverage.SetValue(currWordsRange.GetStartPos(), currWordsRange.GetEndPos(), true); - - ReorderingType reoType; - - if (m_first) { - - } else { - if (modelType == LexicalReorderingConfiguration::MSD) { - reoType = GetOrientationTypeMSD(currWordsRange, coverage); - } else if (modelType == LexicalReorderingConfiguration::MSLR) { - reoType = GetOrientationTypeMSLR(currWordsRange, coverage); - } else if (modelType == LexicalReorderingConfiguration::Monotonic) { - reoType = GetOrientationTypeMonotonic(currWordsRange, coverage); - } else { - reoType = GetOrientationTypeLeftRight(currWordsRange, coverage); + if (m_prevRange == other->m_prevRange) { + return ComparePrevScores(other->m_prevOption); + } else if (m_prevRange < other->m_prevRange) { + return -1; } - - CopyScores(scores, topt, input, reoType); + return 1; } - return new HierarchicalReorderingForwardState(this, topt); -} + // For compatibility with the phrase-based reordering model, scoring is one step delayed. + // The forward model takes determines orientations heuristically as follows: + // mono: if the next phrase comes after the conditioning phrase and + // - there is a gap to the right of the conditioning phrase, or + // - the next phrase immediately follows it + // swap: if the next phrase goes before the conditioning phrase and + // - there is a gap to the left of the conditioning phrase, or + // - the next phrase immediately precedes it + // dright: if the next phrase follows the cond. phr. + // and other stuff comes in between + // dleft: if the next phrase precedes the conditioning phrase + // and other stuff comes in between + + LexicalReorderingState* + HierarchicalReorderingForwardState:: + Expand(const TranslationOption& topt, const InputType& input, + ScoreComponentCollection* scores) const + { + LexReoConf::ModelType const modelType = m_configuration.GetModelType(); + WordsRange const& currRange = topt.GetSourceWordsRange(); + + // keep track of the current cov. ourselves so we don't need the hypothesis + WordsBitmap cov = m_coverage; + cov.SetValue(currRange.GetStartPos(), currRange.GetEndPos(), true); + + + if (!m_first) + { + ReorderingType reoType + = ((modelType == LexReoConf::MSD) + ? GetOrientationTypeMSD(currWordsRange, coverage) + : (modelType == LexReoConf::MSLR) + ? GetOrientationTypeMSLR(currWordsRange, coverage) + : (modelType == LexReoConf::Monotonic) + ? GetOrientationTypeMonotonic(currWordsRange, coverage); + : GetOrientationTypeLeftRight(currWordsRange, coverage)); + CopyScores(scores, topt, input, reoType); + } + + return new HierarchicalReorderingForwardState(this, topt); + } -LexicalReorderingState::ReorderingType HierarchicalReorderingForwardState::GetOrientationTypeMSD(WordsRange currRange, WordsBitmap coverage) const -{ - if (currRange.GetStartPos() > m_prevRange.GetEndPos() && - (!coverage.GetValue(m_prevRange.GetEndPos()+1) || currRange.GetStartPos() == m_prevRange.GetEndPos()+1)) { - return M; - } else if (currRange.GetEndPos() < m_prevRange.GetStartPos() && - (!coverage.GetValue(m_prevRange.GetStartPos()-1) || currRange.GetEndPos() == m_prevRange.GetStartPos()-1)) { - return S; + LexicalReorderingState::ReorderingType + HierarchicalReorderingForwardState:: + GetOrientationTypeMSD(WordsRange currRange, WordsBitmap coverage) const + { + return (IsMonotonicStep(m_prevRange,currRange,coverage) ? M + : IsSwap(m_prevRange, currRange, coverage) ? S : D); } - return D; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingForwardState::GetOrientationTypeMSLR(WordsRange currRange, WordsBitmap coverage) const -{ - if (currRange.GetStartPos() > m_prevRange.GetEndPos() && - (!coverage.GetValue(m_prevRange.GetEndPos()+1) || currRange.GetStartPos() == m_prevRange.GetEndPos()+1)) { - return M; - } else if (currRange.GetEndPos() < m_prevRange.GetStartPos() && - (!coverage.GetValue(m_prevRange.GetStartPos()-1) || currRange.GetEndPos() == m_prevRange.GetStartPos()-1)) { - return S; - } else if (currRange.GetStartPos() > m_prevRange.GetEndPos()) { - return DR; + LexicalReorderingState::ReorderingType + HierarchicalReorderingForwardState:: + GetOrientationTypeMSLR(WordsRange currRange, WordsBitmap coverage) const + { + return (IsMonotonicStep(m_prevRange,currRange,coverage) ? M + : IsSwap(m_prevRange, currRange, coverage) ? S + : (currRange.GetStartPos() > m_prevRange.GetEndPos()) ? DR : DL); } - return DL; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingForwardState::GetOrientationTypeMonotonic(WordsRange currRange, WordsBitmap coverage) const -{ - if (currRange.GetStartPos() > m_prevRange.GetEndPos() && - (!coverage.GetValue(m_prevRange.GetEndPos()+1) || currRange.GetStartPos() == m_prevRange.GetEndPos()+1)) { - return M; + LexicalReorderingState::ReorderingType + HierarchicalReorderingForwardState:: + GetOrientationTypeMonotonic(WordsRange currRange, WordsBitmap coverage) const + { + return IsMonotonicStep(m_prevRange, currRange, coverage) ? M : NM; } - return NM; -} -LexicalReorderingState::ReorderingType HierarchicalReorderingForwardState::GetOrientationTypeLeftRight(WordsRange currRange, WordsBitmap /* coverage */) const -{ - if (currRange.GetStartPos() > m_prevRange.GetEndPos()) { - return R; + LexicalReorderingState::ReorderingType + HierarchicalReorderingForwardState:: + GetOrientationTypeLeftRight(WordsRange currRange, WordsBitmap coverage) const + { + return currRange.GetStartPos() > m_prevRange.GetEndPos() ? R : L; } - return L; -} } diff --git a/moses/FF/LexicalReordering/LexicalReorderingState.h b/moses/FF/LexicalReordering/LexicalReorderingState.h index 79537f119..6599dae5b 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingState.h +++ b/moses/FF/LexicalReordering/LexicalReorderingState.h @@ -94,6 +94,9 @@ private: //! Abstract class for lexical reordering model states class LexicalReorderingState : public FFState { + typedef LexicalReorderingConfiguration::Forward Forward; + typedef LexicalReorderingConfiguration::Backward Backward; + public: virtual int Compare(const FFState& o) const = 0; virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const = 0; diff --git a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp index 23a3809f1..ec79163a7 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp @@ -418,7 +418,8 @@ namespace Moses if(d == InvalidOffT) d = fTell(ot); else { - TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '" << line << "\n"); + TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" + << lnc << "): '" << line << "\n"); return false; } } @@ -448,7 +449,8 @@ namespace Moses if(d == InvalidOffT) d = fTell(ot); else { - TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" << lnc << "): '" << line << "\n"); + TRACE_ERR("ERROR: source phrase already inserted (A)!\nline(" + << lnc << "): '" << line << "\n"); return false; } } @@ -602,7 +604,8 @@ namespace Moses auxCacheForSrcPhrase(f); } } - std::cerr << "Cached " << m_Cache.size() - prev_cache_size << " new primary reordering table keys\n"; + std::cerr << "Cached " << m_Cache.size() - prev_cache_size + << " new primary reordering table keys\n"; } } diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp index ad7591a7b..5f42bdeef 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp @@ -1,5 +1,6 @@ -// $Id$ +// -*- c++ -*- // vim:tabstop=2 +// $Id$ /*********************************************************************** Moses - factored phrase-based language decoder Copyright (C) 2006 University of Edinburgh @@ -24,151 +25,155 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) - : LexicalReorderingTable(f_factors, e_factors, c_factors), - m_inMemory(StaticData::Instance().UseMinlexrInMemory()), - m_numScoreComponent(6), m_multipleScoreTrees(true), - m_hash(10, 16), m_scoreTrees(1) -{ - Load(filePath); -} - -LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) - : LexicalReorderingTable(f_factors, e_factors, c_factors), - m_inMemory(StaticData::Instance().UseMinlexrInMemory()), - m_numScoreComponent(6), m_multipleScoreTrees(true), - m_hash(10, 16), m_scoreTrees(1) -{ } - -LexicalReorderingTableCompact::~LexicalReorderingTableCompact() -{ - for(size_t i = 0; i < m_scoreTrees.size(); i++) - delete m_scoreTrees[i]; -} - -std::vector<float> LexicalReorderingTableCompact::GetScore(const Phrase& f, - const Phrase& e, - const Phrase& c) -{ - std::string key; - Scores scores; - - if(0 == c.GetSize()) - key = MakeKey(f, e, c); - else - for(size_t i = 0; i <= c.GetSize(); ++i) { - Phrase sub_c(c.GetSubString(WordsRange(i,c.GetSize()-1))); - key = MakeKey(f,e,sub_c); - } - - size_t index = m_hash[key]; - if(m_hash.GetSize() != index) { - std::string scoresString; - if(m_inMemory) - scoresString = m_scoresMemory[index]; - else - scoresString = m_scoresMapped[index]; - - BitWrapper<> bitStream(scoresString); - for(size_t i = 0; i < m_numScoreComponent; i++) - scores.push_back(m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); - - return scores; + LexicalReorderingTableCompact:: + LexicalReorderingTableCompact(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + : LexicalReorderingTable(f_factors, e_factors, c_factors) + , m_inMemory(StaticData::Instance().UseMinlexrInMemory()) + , m_numScoreComponent(6) + , m_multipleScoreTrees(true) + , m_hash(10, 16) + , m_scoreTrees(1) + { + Load(filePath); } - return Scores(); -} - -std::string LexicalReorderingTableCompact::MakeKey(const Phrase& f, - const Phrase& e, - const Phrase& c) const -{ - return MakeKey(Trim(f.GetStringRep(m_FactorsF)), - Trim(e.GetStringRep(m_FactorsE)), - Trim(c.GetStringRep(m_FactorsC))); -} + LexicalReorderingTableCompact:: + LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + : LexicalReorderingTable(f_factors, e_factors, c_factors) + , m_inMemory(StaticData::Instance().UseMinlexrInMemory()) + , m_numScoreComponent(6) + , m_multipleScoreTrees(true) + , m_hash(10, 16) + , m_scoreTrees(1) + { } + + LexicalReorderingTableCompact:: + ~LexicalReorderingTableCompact() + { + for(size_t i = 0; i < m_scoreTrees.size(); i++) + delete m_scoreTrees[i]; + } -std::string LexicalReorderingTableCompact::MakeKey(const std::string& f, - const std::string& e, - const std::string& c) const -{ - std::string key; - if(!f.empty()) { - key += f; + std::vector<float> + LexicalReorderingTableCompact:: + GetScore(const Phrase& f, const Phrase& e, const Phrase& c) + { + std::string key; + Scores scores; + + if(0 == c.GetSize()) + key = MakeKey(f, e, c); + else + for(size_t i = 0; i <= c.GetSize(); ++i) + { + Phrase sub_c(c.GetSubString(WordsRange(i,c.GetSize()-1))); + key = MakeKey(f,e,sub_c); + } + + size_t index = m_hash[key]; + if(m_hash.GetSize() != index) + { + std::string scoresString; + if(m_inMemory) + scoresString = m_scoresMemory[index]; + else + scoresString = m_scoresMapped[index]; + + BitWrapper<> bitStream(scoresString); + for(size_t i = 0; i < m_numScoreComponent; i++) + scores.push_back(m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); + + return scores; + } + + return Scores(); } - if(!m_FactorsE.empty()) { - if(!key.empty()) { - key += " ||| "; - } - key += e; + + std::string + LexicalReorderingTableCompact:: + MakeKey(const Phrase& f, + const Phrase& e, + const Phrase& c) const + { + return MakeKey(Trim(f.GetStringRep(m_FactorsF)), + Trim(e.GetStringRep(m_FactorsE)), + Trim(c.GetStringRep(m_FactorsC))); } - if(!m_FactorsC.empty()) { - if(!key.empty()) { - key += " ||| "; - } - key += c; + + std::string + LexicalReorderingTableCompact:: + MakeKey(const std::string& f, + const std::string& e, + const std::string& c) const + { + std::string key; + if(!f.empty()) key += f; + if(!m_FactorsE.empty()) { if(!key.empty()) key += " ||| "; key += e; } + if(!m_FactorsC.empty()) { if(!key.empty()) key += " ||| "; key += c; } + key += " ||| "; + return key; } - key += " ||| "; - return key; -} -LexicalReorderingTable* LexicalReorderingTableCompact::CheckAndLoad( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) -{ + LexicalReorderingTable* + LexicalReorderingTableCompact:: + CheckAndLoad + (const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + { #ifdef HAVE_CMPH - std::string minlexr = ".minlexr"; - // file name is specified without suffix - if(FileExists(filePath + minlexr)) { - //there exists a compact binary version use that - VERBOSE(2,"Using compact lexical reordering table" << std::endl); - return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, e_factors, c_factors); - } - // file name is specified with suffix - if(filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) == minlexr - && FileExists(filePath)) { - //there exists a compact binary version use that - VERBOSE(2,"Using compact lexical reordering table" << std::endl); - return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, c_factors); - } + std::string minlexr = ".minlexr"; + // file name is specified without suffix + if(FileExists(filePath + minlexr)) { + //there exists a compact binary version use that + VERBOSE(2,"Using compact lexical reordering table" << std::endl); + return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, e_factors, c_factors); + } + // file name is specified with suffix + if(filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) == minlexr + && FileExists(filePath)) { + //there exists a compact binary version use that + VERBOSE(2,"Using compact lexical reordering table" << std::endl); + return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, c_factors); + } #endif - return 0; -} - -void LexicalReorderingTableCompact::Load(std::string filePath) -{ - std::FILE* pFile = std::fopen(filePath.c_str(), "r"); - if(m_inMemory) - m_hash.Load(pFile); - else - m_hash.LoadIndex(pFile); - - size_t read = 0; - read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile); - read += std::fread(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, pFile); - - if(m_multipleScoreTrees) { - m_scoreTrees.resize(m_numScoreComponent); - for(size_t i = 0; i < m_numScoreComponent; i++) - m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); - } else { - m_scoreTrees.resize(1); - m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); + return 0; } - if(m_inMemory) - m_scoresMemory.load(pFile, false); - else - m_scoresMapped.load(pFile, true); -} - + void + LexicalReorderingTableCompact:: + Load(std::string filePath) + { + std::FILE* pFile = std::fopen(filePath.c_str(), "r"); + if(m_inMemory) + m_hash.Load(pFile); + else + m_hash.LoadIndex(pFile); + + size_t read = 0; + read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile); + read += std::fread(&m_multipleScoreTrees, + sizeof(m_multipleScoreTrees), 1, pFile); + + if(m_multipleScoreTrees) { + m_scoreTrees.resize(m_numScoreComponent); + for(size_t i = 0; i < m_numScoreComponent; i++) + m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); + } else { + m_scoreTrees.resize(1); + m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); + } + + if(m_inMemory) + m_scoresMemory.load(pFile, false); + else + m_scoresMapped.load(pFile, true); + } + } diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h index 755398788..d54dd5273 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -36,49 +36,53 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -class LexicalReorderingTableCompact: public LexicalReorderingTable -{ -private: - bool m_inMemory; - - size_t m_numScoreComponent; - bool m_multipleScoreTrees; - - BlockHashIndex m_hash; - - typedef CanonicalHuffman<float> ScoreTree; - std::vector<ScoreTree*> m_scoreTrees; - - StringVector<unsigned char, unsigned long, MmapAllocator> m_scoresMapped; - StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory; - - std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const; - std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const; - -public: - LexicalReorderingTableCompact( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - LexicalReorderingTableCompact( - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - virtual ~LexicalReorderingTableCompact(); - - virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c); - - static LexicalReorderingTable* CheckAndLoad( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - void Load(std::string filePath); -}; + class LexicalReorderingTableCompact: + public LexicalReorderingTable + { + private: + bool m_inMemory; + + size_t m_numScoreComponent; + bool m_multipleScoreTrees; + + BlockHashIndex m_hash; + + typedef CanonicalHuffman<float> ScoreTree; + std::vector<ScoreTree*> m_scoreTrees; + + StringVector<unsigned char, unsigned long, MmapAllocator> m_scoresMapped; + StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory; + + std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const; + std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const; + + public: + LexicalReorderingTableCompact(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + virtual + ~LexicalReorderingTableCompact(); + + virtual + std::vector<float> + GetScore(const Phrase& f, const Phrase& e, const Phrase& c); + + static + LexicalReorderingTable* + CheckAndLoad(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + void + Load(std::string filePath); + }; } |