diff options
author | Eva Hasler <evahasler@gmail.com> | 2012-04-09 23:47:51 +0400 |
---|---|---|
committer | Eva Hasler <evahasler@gmail.com> | 2012-04-09 23:47:51 +0400 |
commit | fdca25051659ccc01e27910d2f4a08570c7301a5 (patch) | |
tree | 01d0691c54a8a8002a67bc0bf709ed264fede332 /moses | |
parent | 96642447aa8cda377dc93435ec78a654af200c60 (diff) |
fix setting sourcePhrase, enable stateless ff for chart, add EvaluateChart to stateless ff, revert glmu to stateless ff, Evaluate gets hypothesis instead of target phrase, swd/twi/wt features implemented for chart (w/o context features)
Diffstat (limited to 'moses')
25 files changed, 439 insertions, 184 deletions
diff --git a/moses/src/BilingualDynSuffixArray.cpp b/moses/src/BilingualDynSuffixArray.cpp index 4ae3ab932..a2d5e3e90 100644 --- a/moses/src/BilingualDynSuffixArray.cpp +++ b/moses/src/BilingualDynSuffixArray.cpp @@ -351,7 +351,7 @@ SAPhrase BilingualDynSuffixArray::TrgPhraseFromSntIdx(const PhrasePair& phrasepa return phraseIds; } -TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase) const +TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase, const Phrase& sourcePhrase) const { TargetPhrase* targetPhrase = new TargetPhrase(Output); for(size_t i=0; i < phrase.words.size(); ++i) { // look up trg words @@ -359,6 +359,7 @@ TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase) CHECK(word != m_trgVocab->GetkOOVWord()); targetPhrase->AddWord(word); } + targetPhrase->SetSourcePhrase(sourcePhrase); // scoring return targetPhrase; } @@ -420,7 +421,7 @@ void BilingualDynSuffixArray::GetTargetPhrasesByLexicalWeight(const Phrase& src, std::multimap<Scores, const SAPhrase*, ScoresComp>::reverse_iterator ritr; for(ritr = phraseScores.rbegin(); ritr != phraseScores.rend(); ++ritr) { Scores scoreVector = ritr->first; - TargetPhrase *targetPhrase = GetMosesFactorIDs(*ritr->second); + TargetPhrase *targetPhrase = GetMosesFactorIDs(*ritr->second, src); target.push_back(make_pair( scoreVector, targetPhrase)); if(target.size() == m_maxSampleSize) break; } diff --git a/moses/src/BilingualDynSuffixArray.h b/moses/src/BilingualDynSuffixArray.h index 137978c14..1543c3709 100644 --- a/moses/src/BilingualDynSuffixArray.h +++ b/moses/src/BilingualDynSuffixArray.h @@ -120,7 +120,7 @@ private: int SampleSelection(std::vector<unsigned>&, int = 300) const; std::vector<int> GetSntIndexes(std::vector<unsigned>&, int, const std::vector<unsigned>&) const; - TargetPhrase* GetMosesFactorIDs(const SAPhrase&) const; + TargetPhrase* GetMosesFactorIDs(const SAPhrase&, const Phrase& sourcePhrase) const; SAPhrase TrgPhraseFromSntIdx(const PhrasePair&) const; bool GetLocalVocabIDs(const Phrase&, SAPhrase &) const; void CacheWordProbs(wordID_t) const; diff --git a/moses/src/ChartHypothesis.cpp b/moses/src/ChartHypothesis.cpp index 9c618d73d..88b930243 100644 --- a/moses/src/ChartHypothesis.cpp +++ b/moses/src/ChartHypothesis.cpp @@ -33,6 +33,8 @@ #include "ChartTranslationOption.h" #include "FFState.h" +using namespace std; + namespace Moses { @@ -156,19 +158,16 @@ void ChartHypothesis::CalcScore() // compute values of stateless feature functions that were not // cached in the translation option-- there is no principled distinction - - //const vector<const StatelessFeatureFunction*>& sfs = - // m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); - // TODO! - //for (unsigned i = 0; i < sfs.size(); ++i) { - // sfs[i]->ChartEvaluate(m_targetPhrase, &m_scoreBreakdown); - //} + const std::vector<const StatelessFeatureFunction*>& sfs = + m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); + for (unsigned i = 0; i < sfs.size(); ++i) + if (sfs[i]->ComputeValueInTranslationOption() == false) + sfs[i]->EvaluateChart(*this,i,&m_scoreBreakdown); const std::vector<const StatefulFeatureFunction*>& ffs = m_manager.GetTranslationSystem()->GetStatefulFeatureFunctions(); - for (unsigned i = 0; i < ffs.size(); ++i) { + for (unsigned i = 0; i < ffs.size(); ++i) m_ffStates[i] = ffs[i]->EvaluateChart(*this,i,&m_scoreBreakdown); - } m_totalScore = m_scoreBreakdown.GetWeightedScore(); } diff --git a/moses/src/DummyScoreProducers.cpp b/moses/src/DummyScoreProducers.cpp index 5191fd75b..03c635b03 100644 --- a/moses/src/DummyScoreProducers.cpp +++ b/moses/src/DummyScoreProducers.cpp @@ -96,8 +96,9 @@ std::string WordPenaltyProducer::GetScoreProducerWeightShortName(unsigned) const return "w"; } -void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollection* out) const +void WordPenaltyProducer::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* out) const { + const TargetPhrase& tp = cur_hypo.GetCurrTargetPhrase(); out->PlusEquals(this, -static_cast<float>(tp.GetSize())); } diff --git a/moses/src/DummyScoreProducers.h b/moses/src/DummyScoreProducers.h index 058e8f22d..d8d9393d2 100644 --- a/moses/src/DummyScoreProducers.h +++ b/moses/src/DummyScoreProducers.h @@ -48,8 +48,15 @@ public: std::string GetScoreProducerWeightShortName(unsigned) const; virtual void Evaluate( - const TargetPhrase& phrase, - ScoreComponentCollection* out) const; + const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; + + virtual void EvaluateChart( + const ChartHypothesis&, + int /* featureID */, + ScoreComponentCollection*) const { + // needs to be implemented but does nothing + } }; /** unknown word penalty */ diff --git a/moses/src/FeatureFunction.cpp b/moses/src/FeatureFunction.cpp index ad9db5e1c..d435df5af 100644 --- a/moses/src/FeatureFunction.cpp +++ b/moses/src/FeatureFunction.cpp @@ -11,17 +11,25 @@ bool StatelessFeatureFunction::IsStateless() const { return true; } + bool StatelessFeatureFunction::ComputeValueInTranslationOption() const { return false; } -void StatelessFeatureFunction::Evaluate( - const TargetPhrase& /* cur_hypo */, - ScoreComponentCollection* /* accumulator */) const + +void StatelessFeatureFunction::Evaluate(const Hypothesis& /* cur_hypo */, + ScoreComponentCollection* /* accumulator */) const { CHECK(!"Please implement Evaluate or set ComputeValueInTranslationOption to true"); } +void StatelessFeatureFunction::EvaluateChart(const ChartHypothesis& /*cur_hypo*/, + int /*featureID*/, + ScoreComponentCollection* /*accumulator*/) const +{ + CHECK(!"Please implement EvaluateChart or set ComputeValueInTranslationOption to true"); +} + bool StatefulFeatureFunction::IsStateless() const { return false; diff --git a/moses/src/FeatureFunction.h b/moses/src/FeatureFunction.h index 00307b293..4ef85ecd9 100644 --- a/moses/src/FeatureFunction.h +++ b/moses/src/FeatureFunction.h @@ -33,14 +33,17 @@ public: StatelessFeatureFunction(const std::string& description, size_t numScoreComponents) : FeatureFunction(description, numScoreComponents) {} //! Evaluate for stateless feature functions. Implement this. - virtual void Evaluate( - const TargetPhrase& cur_hypo, - ScoreComponentCollection* accumulator) const; + virtual void Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; + + virtual void EvaluateChart(const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const; // If true, this value is expected to be included in the // ScoreBreakdown in the TranslationOption once it has been // constructed. - // Default: true + // Default: false virtual bool ComputeValueInTranslationOption() const; bool IsStateless() const; diff --git a/moses/src/FeatureVector.cpp b/moses/src/FeatureVector.cpp index e78da68b7..f5c7a2fad 100644 --- a/moses/src/FeatureVector.cpp +++ b/moses/src/FeatureVector.cpp @@ -360,6 +360,16 @@ namespace Moses { out << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl; } + void FVector::printSparseHopeFeatureCounts() { + for (const_iterator i = cbegin(); i != cend(); ++i) + std::cerr << (i->first).name() << ": " << FName::getHopeIdCount((i->first).name()) << std::endl; + } + + void FVector::printSparseFearFeatureCounts() { + for (const_iterator i = cbegin(); i != cend(); ++i) + std::cerr << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl; + } + size_t FVector::pruneSparseFeatures(size_t threshold) { size_t count = 0; vector<FName> toErase; diff --git a/moses/src/FeatureVector.h b/moses/src/FeatureVector.h index b6fd19e08..7fd765fec 100644 --- a/moses/src/FeatureVector.h +++ b/moses/src/FeatureVector.h @@ -208,6 +208,8 @@ namespace Moses { void incrementSparseFearFeatures(); void printSparseHopeFeatureCounts(std::ofstream& out); void printSparseFearFeatureCounts(std::ofstream& out); + void printSparseHopeFeatureCounts(); + void printSparseFearFeatureCounts(); size_t pruneSparseFeatures(size_t threshold); size_t pruneZeroWeightFeatures(); diff --git a/moses/src/GlobalLexicalModel.h b/moses/src/GlobalLexicalModel.h index 1ae9b5a7a..3105cb466 100644 --- a/moses/src/GlobalLexicalModel.h +++ b/moses/src/GlobalLexicalModel.h @@ -77,6 +77,15 @@ public: void InitializeForInput( Sentence const& in ); void Evaluate(const TargetPhrase&, ScoreComponentCollection* ) const; + + void EvaluateChart( + const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const + { + std::cerr << "EvaluateChart not implemented." << std::endl; + exit(1); + } }; } diff --git a/moses/src/GlobalLexicalModelUnlimited.cpp b/moses/src/GlobalLexicalModelUnlimited.cpp index 1032cad1e..f1de65bd0 100644 --- a/moses/src/GlobalLexicalModelUnlimited.cpp +++ b/moses/src/GlobalLexicalModelUnlimited.cpp @@ -51,9 +51,7 @@ void GlobalLexicalModelUnlimited::InitializeForInput( Sentence const& in ) m_local->input = ∈ } -//void GlobalLexicalModelUnlimited::Evaluate(const TargetPhrase& targetPhrase, ScoreComponentCollection* accumulator) const -FFState* GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, - ScoreComponentCollection* accumulator) const +void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { const Sentence& input = *(m_local->input); const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); @@ -261,8 +259,6 @@ FFState* GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, const } } } - - return new DummyState(); } void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulator, diff --git a/moses/src/GlobalLexicalModelUnlimited.h b/moses/src/GlobalLexicalModelUnlimited.h index 1fec8d371..c05c11be8 100644 --- a/moses/src/GlobalLexicalModelUnlimited.h +++ b/moses/src/GlobalLexicalModelUnlimited.h @@ -33,8 +33,7 @@ class InputType; * feature is that it uses context words for disambiguation */ -//class GlobalLexicalModelUnlimited : public StatelessFeatureFunction -class GlobalLexicalModelUnlimited : public StatefulFeatureFunction +class GlobalLexicalModelUnlimited : public StatelessFeatureFunction { typedef std::map< char, short > CharHash; typedef std::map< std::string, short > StringHash; @@ -71,8 +70,7 @@ private: public: GlobalLexicalModelUnlimited(const std::vector< FactorType >& inFactors, const std::vector< FactorType >& outFactors, bool biasFeature, bool ignorePunctuation, size_t context): -// StatelessFeatureFunction("glm",ScoreProducer::unlimited), - StatefulFeatureFunction("glm",ScoreProducer::unlimited), + StatelessFeatureFunction("glm",ScoreProducer::unlimited), m_inputFactors(inFactors), m_outputFactors(outFactors), m_unrestricted(true), @@ -119,22 +117,21 @@ public: void InitializeForInput( Sentence const& in ); - //void Evaluate(const TargetPhrase&, ScoreComponentCollection* ) const; - const FFState* EmptyHypothesisState(const InputType &) const { return new DummyState(); } - FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, - ScoreComponentCollection* accumulator) const; + void Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; - FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */, int /* featureID */, - ScoreComponentCollection* ) const { + void EvaluateChart(const ChartHypothesis& /* cur_hypo */, + int /* featureID */, + ScoreComponentCollection* ) const { /* Not implemented */ assert(0); - return NULL; } + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } float GetSparseProducerWeight() const { return m_sparseProducerWeight; } diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 86e2e791b..4d6c6ce8d 100644 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp @@ -276,9 +276,8 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore) // cached in the translation option-- there is no principled distinction const vector<const StatelessFeatureFunction*>& sfs = m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); - for (unsigned i = 0; i < sfs.size(); ++i) { - sfs[i]->Evaluate(m_targetPhrase, &m_currScoreBreakdown); - } + for (unsigned i = 0; i < sfs.size(); ++i) + sfs[i]->Evaluate(*this, &m_currScoreBreakdown); const vector<const StatefulFeatureFunction*>& ffs = m_manager.GetTranslationSystem()->GetStatefulFeatureFunctions(); diff --git a/moses/src/PhraseLengthFeature.h b/moses/src/PhraseLengthFeature.h index ab5c878f6..f25142230 100644 --- a/moses/src/PhraseLengthFeature.h +++ b/moses/src/PhraseLengthFeature.h @@ -23,6 +23,13 @@ public: void Evaluate(const TargetPhrase& cur_phrase, ScoreComponentCollection* accumulator) const; + void EvaluateChart( + const ChartHypothesis&, + int /* featureID */, + ScoreComponentCollection*) const { + CHECK(0); // feature function not valid in chart decoder + } + // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "pl"; } size_t GetNumInputScores() const { return 0; } diff --git a/moses/src/PhrasePairFeature.cpp b/moses/src/PhrasePairFeature.cpp index faf9e0179..1f85f871c 100644 --- a/moses/src/PhrasePairFeature.cpp +++ b/moses/src/PhrasePairFeature.cpp @@ -1,6 +1,7 @@ #include "AlignmentInfo.h" #include "PhrasePairFeature.h" #include "TargetPhrase.h" +#include "Hypothesis.h" using namespace std; @@ -22,8 +23,9 @@ void PhrasePairFeature::InitializeForInput( Sentence const& in ) m_local->input = ∈ } -void PhrasePairFeature::Evaluate(const TargetPhrase& target, ScoreComponentCollection* accumulator) const { - const Phrase& source = target.GetSourcePhrase(); +void PhrasePairFeature::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { + const TargetPhrase& target = cur_hypo.GetCurrTargetPhrase(); + const Phrase& source = target.GetSourcePhrase(); /* const AlignmentInfo& align = cur_hypo.GetAlignmentInfo(); for (AlignmentInfo::const_iterator i = align.begin(); i != align.end(); ++i) { const Factor* sourceFactor = source.GetWord(i->first).GetFactor(m_sourceFactorId); diff --git a/moses/src/PhrasePairFeature.h b/moses/src/PhrasePairFeature.h index b2b9f41f2..6dab1a998 100644 --- a/moses/src/PhrasePairFeature.h +++ b/moses/src/PhrasePairFeature.h @@ -85,13 +85,19 @@ class PhrasePairFeature: public StatelessFeatureFunction { } } - virtual void Evaluate( - const TargetPhrase& cur_hypo, - ScoreComponentCollection* accumulator) const; + void Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart( + const ChartHypothesis&, + int /* featureID */, + ScoreComponentCollection*) const { + CHECK(0); // feature function not valid in chart decoder + } //NB: Should really precompute this feature, but don't have //good hooks to do this. - virtual bool ComputeValueInTranslationOption() const; + bool ComputeValueInTranslationOption() const; std::string GetScoreProducerWeightShortName(unsigned) const; size_t GetNumInputScores() const; diff --git a/moses/src/ScoreComponentCollection.h b/moses/src/ScoreComponentCollection.h index b88593713..dd6a5c855 100644 --- a/moses/src/ScoreComponentCollection.h +++ b/moses/src/ScoreComponentCollection.h @@ -368,6 +368,8 @@ public: void IncrementSparseFearFeatures() { m_scores.incrementSparseFearFeatures(); } void PrintSparseHopeFeatureCounts(std::ofstream& out) { m_scores.printSparseHopeFeatureCounts(out); } void PrintSparseFearFeatureCounts(std::ofstream& out) { m_scores.printSparseFearFeatureCounts(out); } + void PrintSparseHopeFeatureCounts() { m_scores.printSparseHopeFeatureCounts(); } + void PrintSparseFearFeatureCounts() { m_scores.printSparseFearFeatureCounts(); } size_t PruneSparseFeatures(size_t threshold) { return m_scores.pruneSparseFeatures(threshold); } size_t PruneZeroWeightFeatures() { return m_scores.pruneZeroWeightFeatures(); } diff --git a/moses/src/SourceWordDeletionFeature.cpp b/moses/src/SourceWordDeletionFeature.cpp index 3b851cdc7..7c63fc722 100644 --- a/moses/src/SourceWordDeletionFeature.cpp +++ b/moses/src/SourceWordDeletionFeature.cpp @@ -3,6 +3,7 @@ #include "Phrase.h" #include "TargetPhrase.h" #include "Hypothesis.h" +#include "ChartHypothesis.h" #include "ScoreComponentCollection.h" namespace Moses { @@ -29,36 +30,54 @@ bool SourceWordDeletionFeature::Load(const std::string &filePath) return true; } -void SourceWordDeletionFeature::Evaluate(const TargetPhrase& targetPhrase, - ScoreComponentCollection* accumulator) const +void SourceWordDeletionFeature::Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const +{ + TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase(); + ComputeFeatures(targetPhrase, accumulator); +} + +void SourceWordDeletionFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureId, + ScoreComponentCollection* accumulator) const +{ + TargetPhrase targetPhrase = cur_hypo.GetCurrTargetPhrase(); + ComputeFeatures(targetPhrase, accumulator); +} + +void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const { // handle special case: unknown words (they have no word alignment) - size_t targetLength = targetPhrase.GetSize(); - size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize(); - if (targetLength == 1 && sourceLength == 1) - return; + size_t targetLength = targetPhrase.GetSize(); + size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize(); + if (targetLength == 1 && sourceLength == 1) { + const Factor* f1 = targetPhrase.GetWord(0).GetFactor(1); + if (f1 && f1->GetString().compare(UNKNOWN_FACTOR) == 0) { + return; + } + } // flag aligned words const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); bool aligned[16]; CHECK(sourceLength < 16); - for(size_t i=0; i<sourceLength; i++) { + for(size_t i=0; i<sourceLength; i++) aligned[i] = false; - } - for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) aligned[ alignmentPoint->first ] = true; - } - + // process unaligned source words for(size_t i=0; i<sourceLength; i++) { if (!aligned[i]) { - const string &word = targetPhrase.GetSourcePhrase().GetWord(i).GetFactor(m_factorType)->GetString(); - if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { - accumulator->PlusEquals(this,"OTHER",1); - } - else { - accumulator->PlusEquals(this,word,1); - } + const string &word = targetPhrase.GetSourcePhrase().GetWord(i).GetFactor(m_factorType)->GetString(); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + accumulator->PlusEquals(this,"OTHER",1); + } + else { + accumulator->PlusEquals(this,word,1); + } + } } } } diff --git a/moses/src/SourceWordDeletionFeature.h b/moses/src/SourceWordDeletionFeature.h index 4f28ffc69..62194e95f 100644 --- a/moses/src/SourceWordDeletionFeature.h +++ b/moses/src/SourceWordDeletionFeature.h @@ -23,12 +23,21 @@ public: StatelessFeatureFunction("swd", ScoreProducer::unlimited), m_factorType(factorType), m_unrestricted(true) - {} + { + std::cerr << "Initializing source word deletion feature.." << std::endl; + } bool Load(const std::string &filePath); - void Evaluate(const TargetPhrase& cur_phrase, + void Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const; + void EvaluateChart(const ChartHypothesis& cur_hypo, + int featureId, + ScoreComponentCollection* accumulator) const; + + void ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const; + // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "swd"; } size_t GetNumInputScores() const { return 0; } diff --git a/moses/src/SparsePhraseDictionaryFeature.h b/moses/src/SparsePhraseDictionaryFeature.h index a7fff5de0..ad38e7078 100644 --- a/moses/src/SparsePhraseDictionaryFeature.h +++ b/moses/src/SparsePhraseDictionaryFeature.h @@ -19,6 +19,13 @@ public: void Evaluate(const TargetPhrase& cur_phrase, ScoreComponentCollection* accumulator) const; + void EvaluateChart( + const ChartHypothesis&, + int /* featureID */, + ScoreComponentCollection*) const { + CHECK(0); // feature function not valid in chart decoder + } + // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "stm"; } size_t GetNumInputScores() const { return 0; } diff --git a/moses/src/TargetWordInsertionFeature.cpp b/moses/src/TargetWordInsertionFeature.cpp index 88baf8779..0d39cc42f 100644 --- a/moses/src/TargetWordInsertionFeature.cpp +++ b/moses/src/TargetWordInsertionFeature.cpp @@ -3,6 +3,7 @@ #include "Phrase.h" #include "TargetPhrase.h" #include "Hypothesis.h" +#include "ChartHypothesis.h" #include "ScoreComponentCollection.h" namespace Moses { @@ -29,14 +30,33 @@ bool TargetWordInsertionFeature::Load(const std::string &filePath) return true; } -void TargetWordInsertionFeature::Evaluate(const TargetPhrase& targetPhrase, +void TargetWordInsertionFeature::Evaluate(const ChartHypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { + const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + ComputeFeatures(targetPhrase, accumulator); +} + +void TargetWordInsertionFeature::EvaluateChart(const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + ComputeFeatures(targetPhrase, accumulator); +} + +void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const +{ // handle special case: unknown words (they have no word alignment) size_t targetLength = targetPhrase.GetSize(); size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize(); - if (targetLength == 1 && sourceLength == 1) - return; + if (targetLength == 1 && sourceLength == 1) { + const Factor* f1 = targetPhrase.GetWord(0).GetFactor(1); + if (f1 && f1->GetString().compare(UNKNOWN_FACTOR) == 0) { + return; + } + } // flag aligned words const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); @@ -48,16 +68,18 @@ void TargetWordInsertionFeature::Evaluate(const TargetPhrase& targetPhrase, for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { aligned[ alignmentPoint->second ] = true; } - + // process unaligned target words for(size_t i=0; i<targetLength; i++) { if (!aligned[i]) { const string &word = targetPhrase.GetWord(i).GetFactor(m_factorType)->GetString(); - if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { - accumulator->PlusEquals(this,"OTHER",1); - } - else { - accumulator->PlusEquals(this,word,1); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + accumulator->PlusEquals(this,"OTHER",1); + } + else { + accumulator->PlusEquals(this,word,1); + } } } } diff --git a/moses/src/TargetWordInsertionFeature.h b/moses/src/TargetWordInsertionFeature.h index d130631ba..427ed1cad 100644 --- a/moses/src/TargetWordInsertionFeature.h +++ b/moses/src/TargetWordInsertionFeature.h @@ -26,9 +26,17 @@ public: {} bool Load(const std::string &filePath); - void Evaluate(const TargetPhrase& cur_phrase, + void Evaluate(const ChartHypothesis& cur_hypo, ScoreComponentCollection* accumulator) const; + void EvaluateChart( + const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const; + + void ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const; + // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "twi"; } size_t GetNumInputScores() const { return 0; } diff --git a/moses/src/WordTranslationFeature.cpp b/moses/src/WordTranslationFeature.cpp index 99deadc24..3e62c1724 100644 --- a/moses/src/WordTranslationFeature.cpp +++ b/moses/src/WordTranslationFeature.cpp @@ -3,6 +3,7 @@ #include "Phrase.h" #include "TargetPhrase.h" #include "Hypothesis.h" +#include "ChartHypothesis.h" #include "ScoreComponentCollection.h" namespace Moses { @@ -50,8 +51,7 @@ void WordTranslationFeature::InitializeForInput( Sentence const& in ) m_local->input = ∈ } -//void WordTranslationFeature::Evaluate(const TargetPhrase& targetPhrase, ScoreComponentCollection* accumulator) const -FFState* WordTranslationFeature::Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, ScoreComponentCollection* accumulator) const +void WordTranslationFeature::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const { const Sentence& input = *(m_local->input); const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); @@ -59,135 +59,280 @@ FFState* WordTranslationFeature::Evaluate(const Hypothesis& cur_hypo, const FFSt // process aligned words for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { - // look up words const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase(); int sourceIndex = alignmentPoint->first; int targetIndex = alignmentPoint->second; - const string &sourceWord = sourcePhrase.GetWord(sourceIndex).GetFactor(m_factorTypeSource)->GetString(); - const string &targetWord = targetPhrase.GetWord(targetIndex).GetFactor(m_factorTypeTarget)->GetString(); + Word ws = sourcePhrase.GetWord(sourceIndex); + if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue; + Word wt = targetPhrase.GetWord(targetIndex); + if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue; + string sourceWord = ws.GetFactor(m_factorTypeSource)->GetString(); + string targetWord = wt.GetFactor(m_factorTypeTarget)->GetString(); if (m_ignorePunctuation) { - // check if source or target are punctuation + // check if source or target are punctuation char firstChar = sourceWord.at(0); CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); if(charIterator != m_punctuationHash.end()) - continue; + continue; firstChar = targetWord.at(0); charIterator = m_punctuationHash.find( firstChar ); if(charIterator != m_punctuationHash.end()) continue; } - bool sourceExists = false; - bool targetExists = false; if (!m_unrestricted) { - sourceExists = m_vocabSource.find( sourceWord ) != m_vocabSource.end(); - targetExists = m_vocabTarget.find( targetWord ) != m_vocabTarget.end(); + if (m_vocabSource.find(sourceWord) == m_vocabSource.end()) + sourceWord = "OTHER"; + if (m_vocabTarget.find(targetWord) == m_vocabTarget.end()) + targetWord = "OTHER"; } - // no feature if both words are not in restricted vocabularies - if (m_unrestricted || (sourceExists && targetExists)) { - if (m_simple) { - // construct feature name - stringstream featureName; - featureName << "wt_"; - //featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER"); - featureName << sourceWord; - featureName << "~"; - //featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER"); - featureName << targetWord; - accumulator->SparsePlusEquals(featureName.str(), 1); + + if (m_simple) { + // construct feature name + stringstream featureName; + featureName << "wt_"; + //featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER"); + featureName << sourceWord; + featureName << "~"; + //featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER"); + featureName << targetWord; + accumulator->SparsePlusEquals(featureName.str(), 1); + } + if (m_sourceContext) { + size_t globalSourceIndex = cur_hypo.GetCurrSourceWordsRange().GetStartPos() + sourceIndex; + if (globalSourceIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << "<s>,"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); } - if (m_sourceContext) { - size_t globalSourceIndex = cur_hypo.GetCurrSourceWordsRange().GetStartPos() + sourceIndex; - if (globalSourceIndex == 0) { - // add <s> trigger feature for source + + // range over source words to get context + for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { + if (contextIndex == globalSourceIndex) continue; + string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = sourceTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_unrestricted || sourceTriggerExists) { stringstream feature; feature << "wt_"; - feature << "<s>,"; - feature << sourceWord; + if (contextIndex < globalSourceIndex) { + feature << sourceTrigger; + feature << ","; + feature << sourceWord; + } + else { + feature << sourceWord; + feature << ","; + feature << sourceTrigger; + } feature << "~"; feature << targetWord; accumulator->SparsePlusEquals(feature.str(), 1); } + } + } + if (m_targetContext) { + size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex; + if (globalTargetIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << "<s>,"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } - // range over source words to get context - for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { - if (contextIndex == globalSourceIndex) continue; - string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString(); - if (m_ignorePunctuation) { - // check if trigger is punctuation - char firstChar = sourceTrigger.at(0); - CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); - if(charIterator != m_punctuationHash.end()) - continue; - } - - bool sourceTriggerExists = false; - if (!m_unrestricted) - sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); - - if (m_unrestricted || sourceTriggerExists) { - stringstream feature; - feature << "wt_"; - if (contextIndex < globalSourceIndex) { - feature << sourceTrigger; - feature << ","; - feature << sourceWord; - } - else { - feature << sourceWord; - feature << ","; - feature << sourceTrigger; - } - feature << "~"; - feature << targetWord; - accumulator->SparsePlusEquals(feature.str(), 1); - } + // range over target words (up to current position) to get context + for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) { + string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = targetTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; } - } - if (m_targetContext) { - size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex; - if (globalTargetIndex == 0) { - // add <s> trigger feature for source + + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || targetTriggerExists) { stringstream feature; feature << "wt_"; feature << sourceWord; feature << "~"; - feature << "<s>,"; + feature << targetTrigger; + feature << ","; feature << targetWord; - accumulator->SparsePlusEquals(feature.str(), 1); + accumulator->SparsePlusEquals(feature.str(), 1); } + } + } + } +} - // range over target words (up to current position) to get context - for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) { - string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); - if (m_ignorePunctuation) { - // check if trigger is punctuation - char firstChar = targetTrigger.at(0); - CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); - if(charIterator != m_punctuationHash.end()) - continue; - } - - bool targetTriggerExists = false; - if (!m_unrestricted) - targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); - - if (m_unrestricted || targetTriggerExists) { - stringstream feature; - feature << "wt_"; +void WordTranslationFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, + ScoreComponentCollection* accumulator) const +{ + const Sentence& input = *(m_local->input); + const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); + + // process aligned words + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { + const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase(); + int sourceIndex = alignmentPoint->first; + int targetIndex = alignmentPoint->second; + Word ws = sourcePhrase.GetWord(sourceIndex); + if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue; + Word wt = targetPhrase.GetWord(targetIndex); + if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue; + string sourceWord = ws.GetFactor(m_factorTypeSource)->GetString(); + string targetWord = wt.GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if source or target are punctuation + char firstChar = sourceWord.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + firstChar = targetWord.at(0); + charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + if (!m_unrestricted) { + if (m_vocabSource.find(sourceWord) == m_vocabSource.end()) + sourceWord = "OTHER"; + if (m_vocabTarget.find(targetWord) == m_vocabTarget.end()) + targetWord = "OTHER"; + } + + if (m_simple) { + // construct feature name + stringstream featureName; + featureName << "wt_"; + //featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER"); + featureName << sourceWord; + featureName << "~"; + //featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER"); + featureName << targetWord; + accumulator->SparsePlusEquals(featureName.str(), 1); + } + /* if (m_sourceContext) { + size_t globalSourceIndex = cur_hypo.GetCurrSourceRange().GetStartPos() + sourceIndex; + if (globalSourceIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << "<s>,"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + + // range over source words to get context + for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { + if (contextIndex == globalSourceIndex) continue; + string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = sourceTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_unrestricted || sourceTriggerExists) { + stringstream feature; + feature << "wt_"; + if (contextIndex < globalSourceIndex) { + feature << sourceTrigger; + feature << ","; + feature << sourceWord; + } + else { feature << sourceWord; - feature << "~"; - feature << targetTrigger; - feature << ","; - feature << targetWord; - accumulator->SparsePlusEquals(feature.str(), 1); - } + feature << ","; + feature << sourceTrigger; + } + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; } } } - } +/* if (m_targetContext) { + size_t globalTargetIndex = 0; // TODO +// size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex; + if (globalTargetIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << "<s>,"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } - return new DummyState(); + // range over target words (up to current position) to get context + for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) { + Phrase outputPhrase = cur_hypo.GetOutputPhrase(); + string targetTrigger = outputPhrase.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + //string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = targetTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || targetTriggerExists) { + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << targetTrigger; + feature << ","; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + } + }*/ + } } } diff --git a/moses/src/WordTranslationFeature.h b/moses/src/WordTranslationFeature.h index fbe88e93c..2e3e8b7cf 100644 --- a/moses/src/WordTranslationFeature.h +++ b/moses/src/WordTranslationFeature.h @@ -19,8 +19,7 @@ namespace Moses /** Sets the features for word translation */ -//class WordTranslationFeature : public StatelessFeatureFunction { -class WordTranslationFeature : public StatefulFeatureFunction { +class WordTranslationFeature : public StatelessFeatureFunction { typedef std::map< char, short > CharHash; @@ -51,8 +50,7 @@ private: public: WordTranslationFeature(FactorType factorTypeSource, FactorType factorTypeTarget, bool simple, bool sourceContext, bool targetContext, bool ignorePunctuation): -// StatelessFeatureFunction("wt", ScoreProducer::unlimited), - StatefulFeatureFunction("wt", ScoreProducer::unlimited), + StatelessFeatureFunction("wt", ScoreProducer::unlimited), m_factorTypeSource(factorTypeSource), m_factorTypeTarget(factorTypeTarget), m_unrestricted(true), @@ -88,15 +86,12 @@ public: return new DummyState(); } - FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, - ScoreComponentCollection* accumulator) const; + void Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; - FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */, int /* featureID */, - ScoreComponentCollection* ) const { - /* Not implemented */ - assert(0); - return NULL; - } + void EvaluateChart(const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const; // basic properties std::string GetScoreProducerWeightShortName(unsigned) const { return "wt"; } diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp index 3e5c20689..f5eee83f9 100644 --- a/moses/src/XmlOption.cpp +++ b/moses/src/XmlOption.cpp @@ -342,6 +342,7 @@ bool ProcessAndStripXMLTags(string &line, vector<XmlOption*> &res, ReorderingCon TargetPhrase targetPhrase(Output); targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],factorDelimiter); targetPhrase.SetScore(scoreValue); + // TODO: targetPhrase.SetSourcePhrase() ? XmlOption *option = new XmlOption(range,targetPhrase); CHECK(option); |