diff options
author | Eva <eva@deimos.(none)> | 2012-04-19 22:08:06 +0400 |
---|---|---|
committer | Eva <eva@deimos.(none)> | 2012-04-19 22:08:06 +0400 |
commit | 191a418aea695c44708fa7b3934cabbb4d7dded6 (patch) | |
tree | 4e619989dac3a6b22f4ff777f294d21b2bb17558 /moses | |
parent | 0986ca1b5d61acfe3dbf4db7cb304595477963ad (diff) |
handle terminal alignments for hierarchical models
Diffstat (limited to 'moses')
-rw-r--r-- | moses/src/AlignmentInfo.cpp | 6 | ||||
-rw-r--r-- | moses/src/AlignmentInfo.h | 43 | ||||
-rw-r--r-- | moses/src/AlignmentInfoCollection.cpp | 8 | ||||
-rw-r--r-- | moses/src/AlignmentInfoCollection.h | 1 | ||||
-rw-r--r-- | moses/src/RuleTableLoaderCompact.cpp | 9 | ||||
-rw-r--r-- | moses/src/RuleTableLoaderCompact.h | 3 | ||||
-rw-r--r-- | moses/src/RuleTableLoaderStandard.cpp | 3 | ||||
-rw-r--r-- | moses/src/SourceWordDeletionFeature.cpp | 33 | ||||
-rw-r--r-- | moses/src/SourceWordDeletionFeature.h | 6 | ||||
-rw-r--r-- | moses/src/TargetPhrase.cpp | 29 | ||||
-rw-r--r-- | moses/src/TargetPhrase.h | 4 | ||||
-rw-r--r-- | moses/src/TargetWordInsertionFeature.cpp | 31 | ||||
-rw-r--r-- | moses/src/TargetWordInsertionFeature.h | 8 | ||||
-rw-r--r-- | moses/src/WordTranslationFeature.cpp | 5 | ||||
-rw-r--r-- | moses/src/WordTranslationFeature.h | 2 |
15 files changed, 143 insertions, 48 deletions
diff --git a/moses/src/AlignmentInfo.cpp b/moses/src/AlignmentInfo.cpp index 3af17870e..959e7591f 100644 --- a/moses/src/AlignmentInfo.cpp +++ b/moses/src/AlignmentInfo.cpp @@ -26,7 +26,7 @@ namespace Moses { void AlignmentInfo::BuildNonTermIndexMap() -{ +{ if (m_collection.empty()) { return; } @@ -40,9 +40,9 @@ void AlignmentInfo::BuildNonTermIndexMap() m_nonTermIndexMap.resize(maxIndex+1, NOT_FOUND); size_t i = 0; for (p = begin(); p != end(); ++p) { - m_nonTermIndexMap[p->second] = i++; + //std::cerr << "nt point: " << p->second << " -> " << i << std::endl; + m_nonTermIndexMap[p->second] = i++; } - } bool compare_target(const std::pair<size_t,size_t> *a, const std::pair<size_t,size_t> *b) { diff --git a/moses/src/AlignmentInfo.h b/moses/src/AlignmentInfo.h index 9cb06f896..12c5ee2e9 100644 --- a/moses/src/AlignmentInfo.h +++ b/moses/src/AlignmentInfo.h @@ -19,26 +19,28 @@ #pragma once +#include <iostream> #include <ostream> #include <set> #include <vector> +#include <cstdlib> namespace Moses { class AlignmentInfoCollection; -// Collection of non-terminal alignment pairs, ordered by source index. +// Collection of non-terminal/terminal alignment pairs, ordered by source index. class AlignmentInfo { - typedef std::set<std::pair<size_t,size_t> > CollType; - friend std::ostream& operator<<(std::ostream &, const AlignmentInfo &); friend struct AlignmentInfoOrderer; friend class AlignmentInfoCollection; public: + typedef std::set<std::pair<size_t,size_t> > CollType; typedef std::vector<size_t> NonTermIndexMap; + typedef std::vector<size_t> TermIndexMap; typedef CollType::const_iterator const_iterator; const_iterator begin() const { return m_collection.begin(); } @@ -50,7 +52,17 @@ class AlignmentInfo const NonTermIndexMap &GetNonTermIndexMap() const { return m_nonTermIndexMap; } - + + // only used for hierarchical models, contains terminal alignments + const CollType &GetTerminalAlignments() const { + return m_terminalCollection; + } + + // for phrase-based models, this contains all alignments, for hierarchical models only the NT alignments + const CollType &GetAlignments() const { + return m_collection; + } + std::vector< const std::pair<size_t,size_t>* > GetSortedAlignments() const; private: @@ -60,10 +72,29 @@ class AlignmentInfo { BuildNonTermIndexMap(); } - + + // use this for hierarchical models + explicit AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs, int* indicator) + { + // split alignment set in terminals and non-terminals + std::set<std::pair<size_t,size_t> > terminalSet; + std::set<std::pair<size_t,size_t> > nonTerminalSet; + std::set<std::pair<size_t,size_t> >::iterator iter; + for (iter = pairs.begin(); iter != pairs.end(); ++iter) { + if (*indicator == 1) nonTerminalSet.insert(*iter); + else terminalSet.insert(*iter); + indicator++; + } + m_collection = nonTerminalSet; + m_terminalCollection = terminalSet; + + BuildNonTermIndexMap(); + } + void BuildNonTermIndexMap(); CollType m_collection; + CollType m_terminalCollection; NonTermIndexMap m_nonTermIndexMap; }; @@ -72,6 +103,8 @@ class AlignmentInfo struct AlignmentInfoOrderer { bool operator()(const AlignmentInfo &a, const AlignmentInfo &b) const { + if (a.m_collection == b.m_collection) + return a.m_terminalCollection < b.m_terminalCollection; return a.m_collection < b.m_collection; } }; diff --git a/moses/src/AlignmentInfoCollection.cpp b/moses/src/AlignmentInfoCollection.cpp index f871383c8..aa411fc63 100644 --- a/moses/src/AlignmentInfoCollection.cpp +++ b/moses/src/AlignmentInfoCollection.cpp @@ -43,4 +43,12 @@ const AlignmentInfo *AlignmentInfoCollection::Add( return &(*ret.first); } +const AlignmentInfo *AlignmentInfoCollection::Add( + const std::set<std::pair<size_t,size_t> > &pairs, int* indicator) +{ + std::pair<AlignmentInfoSet::iterator, bool> ret = + m_collection.insert(AlignmentInfo(pairs, indicator)); + return &(*ret.first); +} + } diff --git a/moses/src/AlignmentInfoCollection.h b/moses/src/AlignmentInfoCollection.h index 1a6f4549d..a6c90f135 100644 --- a/moses/src/AlignmentInfoCollection.h +++ b/moses/src/AlignmentInfoCollection.h @@ -37,6 +37,7 @@ class AlignmentInfoCollection // contains such an object then returns a pointer to it; otherwise a new // one is inserted. const AlignmentInfo *Add(const std::set<std::pair<size_t,size_t> > &); + const AlignmentInfo *Add(const std::set<std::pair<size_t,size_t> > &, int* indicator); // Returns a pointer to an empty AlignmentInfo object. const AlignmentInfo &GetEmptyAlignmentInfo() const; diff --git a/moses/src/RuleTableLoaderCompact.cpp b/moses/src/RuleTableLoaderCompact.cpp index dce3382e7..64297948f 100644 --- a/moses/src/RuleTableLoaderCompact.cpp +++ b/moses/src/RuleTableLoaderCompact.cpp @@ -72,7 +72,7 @@ bool RuleTableLoaderCompact::Load(const std::vector<FactorType> &input, // Load alignments. std::vector<const AlignmentInfo *> alignmentSets; - LoadAlignmentSection(reader, alignmentSets); + LoadAlignmentSection(reader, alignmentSets, sourcePhrases); // Load rules. if (!LoadRuleSection(reader, vocab, sourcePhrases, targetPhrases, @@ -136,7 +136,7 @@ void RuleTableLoaderCompact::LoadPhraseSection( } void RuleTableLoaderCompact::LoadAlignmentSection( - LineReader &reader, std::vector<const AlignmentInfo *> &alignmentSets) + LineReader &reader, std::vector<const AlignmentInfo *> &alignmentSets, std::vector<Phrase> &sourcePhrases) { // Read alignment set count. reader.ReadLine(); @@ -153,13 +153,16 @@ void RuleTableLoaderCompact::LoadAlignmentSection( reader.ReadLine(); Tokenize(tokens, reader.m_line); std::vector<std::string>::const_iterator p; + int indicator[tokens.size()]; + size_t index = 0; for (p = tokens.begin(); p != tokens.end(); ++p) { points.clear(); Tokenize<size_t>(points, *p, "-"); std::pair<size_t, size_t> alignmentPair(points[0], points[1]); alignmentInfo.insert(alignmentPair); + indicator[index++] = sourcePhrases[i].GetWord(points[0]).IsNonTerminal() ? 1: 0; } - alignmentSets[i] = AlignmentInfoCollection::Instance().Add(alignmentInfo); + alignmentSets[i] = AlignmentInfoCollection::Instance().Add(alignmentInfo, indicator); } } diff --git a/moses/src/RuleTableLoaderCompact.h b/moses/src/RuleTableLoaderCompact.h index add082291..56918782b 100644 --- a/moses/src/RuleTableLoaderCompact.h +++ b/moses/src/RuleTableLoaderCompact.h @@ -70,7 +70,8 @@ class RuleTableLoaderCompact : public RuleTableLoader std::vector<size_t> &); void LoadAlignmentSection(LineReader &, - std::vector<const AlignmentInfo *> &); + std::vector<const AlignmentInfo *> &, + std::vector<Phrase> &); bool LoadRuleSection(LineReader &, const std::vector<Word> &, diff --git a/moses/src/RuleTableLoaderStandard.cpp b/moses/src/RuleTableLoaderStandard.cpp index dc4fb7235..122e75c8c 100644 --- a/moses/src/RuleTableLoaderStandard.cpp +++ b/moses/src/RuleTableLoaderStandard.cpp @@ -220,7 +220,7 @@ bool RuleTableLoaderStandard::Load(FormatType format targetPhrase->SetSourcePhrase(sourcePhrase); // rest of target phrase - targetPhrase->SetAlignmentInfo(alignString); + targetPhrase->SetAlignmentInfo(alignString, sourcePhrase); targetPhrase->SetTargetLHS(targetLHS); targetPhrase->SetRuleCount(ruleCountString, scoreVector); //targetPhrase->SetDebugOutput(string("New Format pt ") + line); @@ -242,7 +242,6 @@ bool RuleTableLoaderStandard::Load(FormatType format else { // do nothing } - } // sort and prune each target phrase collection diff --git a/moses/src/SourceWordDeletionFeature.cpp b/moses/src/SourceWordDeletionFeature.cpp index 7c63fc722..4f247c7e7 100644 --- a/moses/src/SourceWordDeletionFeature.cpp +++ b/moses/src/SourceWordDeletionFeature.cpp @@ -34,18 +34,23 @@ void SourceWordDeletionFeature::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase(); - ComputeFeatures(targetPhrase, accumulator); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); } void SourceWordDeletionFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureId, ScoreComponentCollection* accumulator) const { TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase(); - ComputeFeatures(targetPhrase, accumulator); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); } void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, - ScoreComponentCollection* accumulator) const + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const { // handle special case: unknown words (they have no word alignment) size_t targetLength = targetPhrase.GetSize(); @@ -58,24 +63,26 @@ void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase } // flag aligned words - const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); bool aligned[16]; CHECK(sourceLength < 16); for(size_t i=0; i<sourceLength; i++) aligned[i] = false; - for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) aligned[ alignmentPoint->first ] = true; - + // process unaligned source words for(size_t i=0; i<sourceLength; i++) { if (!aligned[i]) { - const string &word = targetPhrase.GetSourcePhrase().GetWord(i).GetFactor(m_factorType)->GetString(); - if (word != "<s>" && word != "</s>") { - if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { - accumulator->PlusEquals(this,"OTHER",1); - } - else { - accumulator->PlusEquals(this,word,1); + Word w = targetPhrase.GetSourcePhrase().GetWord(i); + if (!w.IsNonTerminal()) { + const string &word = w.GetFactor(m_factorType)->GetString(); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + accumulator->PlusEquals(this,"OTHER",1); + } + else { + accumulator->PlusEquals(this,word,1); + } } } } diff --git a/moses/src/SourceWordDeletionFeature.h b/moses/src/SourceWordDeletionFeature.h index 62194e95f..6d8d21a42 100644 --- a/moses/src/SourceWordDeletionFeature.h +++ b/moses/src/SourceWordDeletionFeature.h @@ -6,6 +6,7 @@ #include "FeatureFunction.h" #include "FactorCollection.h" +#include "AlignmentInfo.h" namespace Moses { @@ -35,8 +36,9 @@ public: int featureId, ScoreComponentCollection* accumulator) const; - void ComputeFeatures(const TargetPhrase& targetPhrase, - ScoreComponentCollection* accumulator) const; + void ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const; // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "swd"; } diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index 26c8887e2..069284eb5 100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -318,11 +318,38 @@ void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString) SetAlignmentInfo(alignmentInfo); } +void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString, Phrase &sourcePhrase) +{ + std::vector<std::string> alignPoints; + boost::split(alignPoints, alignString, boost::is_any_of("\t ")); + int indicator[alignPoints.size()]; + int index = 0; + + set<pair<size_t,size_t> > alignmentInfo; + for (util::TokenIter<util::AnyCharacter, true> token(alignString, util::AnyCharacter(" \t")); token; ++token) { + util::TokenIter<util::AnyCharacter, false> dash(*token, util::AnyCharacter("-")); + MosesShouldUseExceptions(dash); + size_t sourcePos = boost::lexical_cast<size_t>(*dash++); + MosesShouldUseExceptions(dash); + size_t targetPos = boost::lexical_cast<size_t>(*dash++); + MosesShouldUseExceptions(!dash); + + alignmentInfo.insert(pair<size_t,size_t>(sourcePos, targetPos)); + indicator[index++] = sourcePhrase.GetWord(sourcePos).IsNonTerminal() ? 1: 0; + } + + SetAlignmentInfo(alignmentInfo, indicator); +} + void TargetPhrase::SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo) { m_alignmentInfo = AlignmentInfoCollection::Instance().Add(alignmentInfo); } +void TargetPhrase::SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo, int* indicator) +{ + m_alignmentInfo = AlignmentInfoCollection::Instance().Add(alignmentInfo, indicator); +} TO_STRING_BODY(TargetPhrase); @@ -350,7 +377,7 @@ void TargetPhrase::SetRuleCount(const StringPiece &ruleCountString, std::vector< } else { if (scoreVector.size() >= 1 ) p_f_given_e = scoreVector[0]; - std::cerr << "Warning: possibly wrong format of phrase translation scores, number of scores: " << scoreVector.size() << endl; +// std::cerr << "Warning: possibly wrong format of phrase translation scores, number of scores: " << scoreVector.size() << endl; } targetCount = Scan<float>(tokens[0]); diff --git a/moses/src/TargetPhrase.h b/moses/src/TargetPhrase.h index 9226b2636..377985c5e 100644 --- a/moses/src/TargetPhrase.h +++ b/moses/src/TargetPhrase.h @@ -152,7 +152,9 @@ public: { return m_lhsTarget; } void SetAlignmentInfo(const StringPiece &alignString); - void SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo); + void SetAlignmentInfo(const StringPiece &alignString, Phrase &sourcePhrase); + void SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo); + void SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo, int* indicator); void SetAlignmentInfo(const AlignmentInfo *alignmentInfo) { m_alignmentInfo = alignmentInfo; } diff --git a/moses/src/TargetWordInsertionFeature.cpp b/moses/src/TargetWordInsertionFeature.cpp index 620204464..f7d5a5c7e 100644 --- a/moses/src/TargetWordInsertionFeature.cpp +++ b/moses/src/TargetWordInsertionFeature.cpp @@ -34,19 +34,24 @@ void TargetWordInsertionFeature::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); - ComputeFeatures(targetPhrase, accumulator); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); } void TargetWordInsertionFeature::EvaluateChart(const ChartHypothesis& cur_hypo, - int featureID, - ScoreComponentCollection* accumulator) const + int featureID, + ScoreComponentCollection* accumulator) const { const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); - ComputeFeatures(targetPhrase, accumulator); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); } void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, - ScoreComponentCollection* accumulator) const + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const { // handle special case: unknown words (they have no word alignment) size_t targetLength = targetPhrase.GetSize(); @@ -59,7 +64,6 @@ void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhras } // flag aligned words - const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); bool aligned[16]; CHECK(targetLength < 16); for(size_t i=0; i<targetLength; i++) { @@ -72,14 +76,17 @@ void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhras // process unaligned target words for(size_t i=0; i<targetLength; i++) { if (!aligned[i]) { - const string &word = targetPhrase.GetWord(i).GetFactor(m_factorType)->GetString(); - if (word != "<s>" && word != "</s>") { - if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + Word w = targetPhrase.GetWord(i); + if (!w.IsNonTerminal()) { + const string &word = w.GetFactor(m_factorType)->GetString(); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { accumulator->PlusEquals(this,"OTHER",1); - } - else { + } + else { accumulator->PlusEquals(this,word,1); - } + } + } } } } diff --git a/moses/src/TargetWordInsertionFeature.h b/moses/src/TargetWordInsertionFeature.h index fd5bd30a8..0d53582d5 100644 --- a/moses/src/TargetWordInsertionFeature.h +++ b/moses/src/TargetWordInsertionFeature.h @@ -6,6 +6,7 @@ #include "FeatureFunction.h" #include "FactorCollection.h" +#include "AlignmentInfo.h" namespace Moses { @@ -23,7 +24,9 @@ public: StatelessFeatureFunction("twi", ScoreProducer::unlimited), m_factorType(factorType), m_unrestricted(true) - {} + { + std::cerr << "Initializing target word insertion feature.." << std::endl; + } bool Load(const std::string &filePath); void Evaluate(const Hypothesis& cur_hypo, @@ -35,7 +38,8 @@ public: ScoreComponentCollection* accumulator) const; void ComputeFeatures(const TargetPhrase& targetPhrase, - ScoreComponentCollection* accumulator) const; + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const; // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "twi"; } diff --git a/moses/src/WordTranslationFeature.cpp b/moses/src/WordTranslationFeature.cpp index 32c8f9291..c0b3e0025 100644 --- a/moses/src/WordTranslationFeature.cpp +++ b/moses/src/WordTranslationFeature.cpp @@ -192,9 +192,10 @@ void WordTranslationFeature::Evaluate(const Hypothesis& cur_hypo, ScoreComponent void WordTranslationFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection* accumulator) const { - const Sentence& input = *(m_local->input); + //const Sentence& input = *(m_local->input); const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); - const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); // process aligned words for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { diff --git a/moses/src/WordTranslationFeature.h b/moses/src/WordTranslationFeature.h index 2e3e8b7cf..39be24d41 100644 --- a/moses/src/WordTranslationFeature.h +++ b/moses/src/WordTranslationFeature.h @@ -60,7 +60,7 @@ public: m_sparseProducerWeight(1), m_ignorePunctuation(ignorePunctuation) { - std::cerr << "Creating word translation feature.. "; + std::cerr << "Initializing word translation feature.. "; if (m_simple == 1) std::cerr << "using simple word translations.. "; if (m_sourceContext == 1) std::cerr << "using source context.. "; if (m_targetContext == 1) std::cerr << "using target context.. "; |