diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2013-09-17 17:06:17 +0400 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2013-09-17 17:06:17 +0400 |
commit | 5ebb81a17af4008768f8fd93d7ad31442325f47c (patch) | |
tree | ce039914c5be207f06f663d944767e0b0eaf222d /moses | |
parent | 764684bb6f7b00bad9c79f0dc6d88bdc8093d20f (diff) |
constrained decoding FF works for both pb and hiero
Diffstat (limited to 'moses')
-rw-r--r-- | moses/BitmapContainer.cpp | 2 | ||||
-rw-r--r-- | moses/FF/ConstrainedDecoding.cpp | 33 | ||||
-rw-r--r-- | moses/FF/ConstrainedDecoding.h | 1 | ||||
-rw-r--r-- | moses/FF/Factory.cpp | 4 | ||||
-rw-r--r-- | moses/Hypothesis.cpp | 47 | ||||
-rw-r--r-- | moses/Hypothesis.h | 4 | ||||
-rw-r--r-- | moses/HypothesisStackCubePruning.cpp | 7 | ||||
-rw-r--r-- | moses/HypothesisStackNormal.cpp | 7 | ||||
-rw-r--r-- | moses/Manager.cpp | 10 | ||||
-rw-r--r-- | moses/Manager.h | 4 | ||||
-rw-r--r-- | moses/MockHypothesis.cpp | 2 | ||||
-rw-r--r-- | moses/Search.h | 1 | ||||
-rw-r--r-- | moses/SearchCubePruning.cpp | 5 | ||||
-rw-r--r-- | moses/SearchNormal.cpp | 8 | ||||
-rw-r--r-- | moses/SearchNormalBatch.cpp | 2 |
15 files changed, 71 insertions, 66 deletions
diff --git a/moses/BitmapContainer.cpp b/moses/BitmapContainer.cpp index 422bd3ecc..5a90b8a74 100644 --- a/moses/BitmapContainer.cpp +++ b/moses/BitmapContainer.cpp @@ -197,7 +197,7 @@ BackwardsEdge::Initialize() Hypothesis *BackwardsEdge::CreateHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt) { // create hypothesis and calculate all its scores - Hypothesis *newHypo = hypothesis.CreateNext(transOpt, NULL); // TODO FIXME This is absolutely broken - don't pass null here + Hypothesis *newHypo = hypothesis.CreateNext(transOpt); // TODO FIXME This is absolutely broken - don't pass null here newHypo->Evaluate(m_futurescore); return newHypo; diff --git a/moses/FF/ConstrainedDecoding.cpp b/moses/FF/ConstrainedDecoding.cpp index 5af6d772b..f8438e2cc 100644 --- a/moses/FF/ConstrainedDecoding.cpp +++ b/moses/FF/ConstrainedDecoding.cpp @@ -1,4 +1,6 @@ #include "ConstrainedDecoding.h" +#include "moses/Hypothesis.h" +#include "moses/Manager.h" #include "moses/ChartHypothesis.h" #include "moses/ChartManager.h" #include "util/exception.hh" @@ -7,6 +9,11 @@ using namespace std; namespace Moses { +ConstrainedDecodingState::ConstrainedDecodingState(const Hypothesis &hypo) +{ + hypo.GetOutputPhrase(m_outputPhrase); +} + ConstrainedDecodingState::ConstrainedDecodingState(const ChartHypothesis &hypo) { hypo.CreateOutputPhrase(m_outputPhrase); @@ -20,11 +27,33 @@ int ConstrainedDecodingState::Compare(const FFState& other) const } FFState* ConstrainedDecoding::Evaluate( - const Hypothesis& cur_hypo, + const Hypothesis& hypo, const FFState* prev_state, ScoreComponentCollection* accumulator) const { - UTIL_THROW(util::Exception, "Not implemented"); + const Manager &mgr = hypo.GetManager(); + + const Phrase *ref = mgr.GetConstraint(); + CHECK(ref); + + ConstrainedDecodingState *ret = new ConstrainedDecodingState(hypo); + const Phrase &outputPhrase = ret->GetPhrase(); + + size_t searchPos = ref->Find(outputPhrase); + + float score; + if (hypo.IsSourceCompleted()) { + // translated entire sentence. + score = (searchPos == 0) && (ref->GetSize() == outputPhrase.GetSize()) + ? 0 : - std::numeric_limits<float>::infinity(); + } + else { + score = (searchPos != NOT_FOUND) ? 0 : - std::numeric_limits<float>::infinity(); + } + + accumulator->PlusEquals(this, score); + + return ret; } FFState* ConstrainedDecoding::EvaluateChart( diff --git a/moses/FF/ConstrainedDecoding.h b/moses/FF/ConstrainedDecoding.h index 7f2213bcf..bdd9d1683 100644 --- a/moses/FF/ConstrainedDecoding.h +++ b/moses/FF/ConstrainedDecoding.h @@ -14,6 +14,7 @@ public: ConstrainedDecodingState() {} + ConstrainedDecodingState(const Hypothesis &hypo); ConstrainedDecodingState(const ChartHypothesis &hypo); int Compare(const FFState& other) const; diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index acfefba1f..c09a3e38b 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -31,7 +31,7 @@ #include "moses/FF/OSM-Feature/OpSequenceModel.h" #include "moses/FF/ControlRecombination.h" #include "moses/FF/ExternalFeature.h" -//#include "moses/FF/ConstrainedDecoding.h" +#include "moses/FF/ConstrainedDecoding.h" #include "moses/FF/SkeletonStatelessFF.h" #include "moses/FF/SkeletonStatefulFF.h" @@ -143,7 +143,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(PhrasePenalty); MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer); MOSES_FNAME(ControlRecombination); -// MOSES_FNAME(ConstrainedDecoding); + MOSES_FNAME(ConstrainedDecoding); MOSES_FNAME(ExternalFeature); MOSES_FNAME(SkeletonStatelessFF); diff --git a/moses/Hypothesis.cpp b/moses/Hypothesis.cpp index 740bc607f..6335a285b 100644 --- a/moses/Hypothesis.cpp +++ b/moses/Hypothesis.cpp @@ -147,64 +147,23 @@ void Hypothesis::AddArc(Hypothesis *loserHypo) /*** * return the subclass of Hypothesis most appropriate to the given translation option */ -Hypothesis* Hypothesis::CreateNext(const TranslationOption &transOpt, const Phrase* constraint) const +Hypothesis* Hypothesis::CreateNext(const TranslationOption &transOpt) const { - return Create(*this, transOpt, constraint); + return Create(*this, transOpt); } /*** * return the subclass of Hypothesis most appropriate to the given translation option */ -Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOption &transOpt, const Phrase* constrainingPhrase) +Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOption &transOpt) { - // This method includes code for constraint decoding - - bool createHypothesis = true; - - if (constrainingPhrase != NULL) { - - size_t constraintSize = constrainingPhrase->GetSize(); - - size_t start = 1 + prevHypo.GetCurrTargetWordsRange().GetEndPos(); - - const Phrase &transOptPhrase = transOpt.GetTargetPhrase(); - size_t transOptSize = transOptPhrase.GetSize(); - - size_t endpoint = start + transOptSize - 1; - - - if (endpoint < constraintSize) { - WordsRange range(start, endpoint); - Phrase relevantConstraint = constrainingPhrase->GetSubString(range); - - if ( ! relevantConstraint.IsCompatible(transOptPhrase) ) { - createHypothesis = false; - - } - } else { - createHypothesis = false; - } - - } - - - if (createHypothesis) { - #ifdef USE_HYPO_POOL Hypothesis *ptr = s_objectPool.getPtr(); return new(ptr) Hypothesis(prevHypo, transOpt); #else return new Hypothesis(prevHypo, transOpt); #endif - - } else { - // If the previous hypothesis plus the proposed translation option - // fail to match the provided constraint, - // return a null hypothesis. - return NULL; - } - } /*** * return the subclass of Hypothesis most appropriate to the given target phrase diff --git a/moses/Hypothesis.h b/moses/Hypothesis.h index 2200afb31..7d833073c 100644 --- a/moses/Hypothesis.h +++ b/moses/Hypothesis.h @@ -99,7 +99,7 @@ public: ~Hypothesis(); /** return the subclass of Hypothesis most appropriate to the given translation option */ - static Hypothesis* Create(const Hypothesis &prevHypo, const TranslationOption &transOpt, const Phrase* constraint); + static Hypothesis* Create(const Hypothesis &prevHypo, const TranslationOption &transOpt); static Hypothesis* Create(Manager& manager, const WordsBitmap &initialCoverage); @@ -107,7 +107,7 @@ public: static Hypothesis* Create(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt); /** return the subclass of Hypothesis most appropriate to the given translation option */ - Hypothesis* CreateNext(const TranslationOption &transOpt, const Phrase* constraint) const; + Hypothesis* CreateNext(const TranslationOption &transOpt) const; void PrintHypothesis() const; diff --git a/moses/HypothesisStackCubePruning.cpp b/moses/HypothesisStackCubePruning.cpp index 15ac88da6..7de62b0c1 100644 --- a/moses/HypothesisStackCubePruning.cpp +++ b/moses/HypothesisStackCubePruning.cpp @@ -81,6 +81,13 @@ pair<HypothesisStackCubePruning::iterator, bool> HypothesisStackCubePruning::Add bool HypothesisStackCubePruning::AddPrune(Hypothesis *hypo) { + if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) { + m_manager.GetSentenceStats().AddDiscarded(); + VERBOSE(3,"discarded, constraint" << std::endl); + FREEHYPO(hypo); + return false; + } + if (hypo->GetTotalScore() < m_worstScore) { // too bad for stack. don't bother adding hypo into collection m_manager.GetSentenceStats().AddDiscarded(); diff --git a/moses/HypothesisStackNormal.cpp b/moses/HypothesisStackNormal.cpp index 7d72f76aa..ae632a8de 100644 --- a/moses/HypothesisStackNormal.cpp +++ b/moses/HypothesisStackNormal.cpp @@ -88,6 +88,13 @@ pair<HypothesisStackNormal::iterator, bool> HypothesisStackNormal::Add(Hypothesi bool HypothesisStackNormal::AddPrune(Hypothesis *hypo) { + if (hypo->GetTotalScore() == - std::numeric_limits<float>::infinity()) { + m_manager.GetSentenceStats().AddDiscarded(); + VERBOSE(3,"discarded, constraint" << std::endl); + FREEHYPO(hypo); + return false; + } + // too bad for stack. don't bother adding hypo into collection if (!StaticData::Instance().GetDisableDiscarding() && hypo->GetTotalScore() < m_worstScore diff --git a/moses/Manager.cpp b/moses/Manager.cpp index 74d79fe60..1618f6416 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -60,7 +60,15 @@ Manager::Manager(size_t lineNumber, InputType const& source, SearchAlgorithm sea ,m_lineNumber(lineNumber) ,m_source(source) { - StaticData::Instance().InitializeForInput(source); + const StaticData &staticData = StaticData::Instance(); + staticData.InitializeForInput(source); + + long sentenceID = source.GetTranslationId(); + m_constraint = staticData.GetConstrainingPhrase(sentenceID); + if (m_constraint) { + VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl); + } + } Manager::~Manager() diff --git a/moses/Manager.h b/moses/Manager.h index fd329c309..d9f9aee09 100644 --- a/moses/Manager.h +++ b/moses/Manager.h @@ -120,6 +120,7 @@ protected: std::auto_ptr<SentenceStats> m_sentenceStats; int m_hypoId; //used to number the hypos as they are created. size_t m_lineNumber; + const Phrase *m_constraint; void GetConnectedGraph( std::map< int, bool >* pConnected, @@ -157,6 +158,9 @@ public: return m_source; } + const Phrase *GetConstraint() const + { return m_constraint; } + /*** * to be called after processing a sentence (which may consist of more than just calling ProcessSentence() ) */ diff --git a/moses/MockHypothesis.cpp b/moses/MockHypothesis.cpp index 6add6dd2d..8042fd931 100644 --- a/moses/MockHypothesis.cpp +++ b/moses/MockHypothesis.cpp @@ -66,7 +66,7 @@ MockHypothesisGuard::MockHypothesisGuard( m_targetPhrases.back().CreateFromString(Input, factors, *ti, "|", NULL); m_toptions.push_back(new TranslationOption (wordsRange,m_targetPhrases.back())); - m_hypothesis = Hypothesis::Create(*prevHypo,*m_toptions.back(),NULL); + m_hypothesis = Hypothesis::Create(*prevHypo,*m_toptions.back()); } diff --git a/moses/Search.h b/moses/Search.h index dec50f9ee..6b3421248 100644 --- a/moses/Search.h +++ b/moses/Search.h @@ -39,7 +39,6 @@ public: const TranslationOptionCollection &transOptColl); protected: - const Phrase *m_constraint; Manager& m_manager; InputPath m_inputPath; // for initial hypo TranslationOption m_initialTransOpt; /**< used to seed 1st hypo */ diff --git a/moses/SearchCubePruning.cpp b/moses/SearchCubePruning.cpp index e3f47ee43..0ad2878af 100644 --- a/moses/SearchCubePruning.cpp +++ b/moses/SearchCubePruning.cpp @@ -46,11 +46,6 @@ SearchCubePruning::SearchCubePruning(Manager& manager, const InputType &source, { const StaticData &staticData = StaticData::Instance(); - /* constraint search not implemented in cube pruning - long sentenceID = source.GetTranslationId(); - m_constraint = staticData.GetConstrainingPhrase(sentenceID); - */ - std::vector < HypothesisStackCubePruning >::iterator iterStack; for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind) { HypothesisStackCubePruning *sourceHypoColl = new HypothesisStackCubePruning(m_manager); diff --git a/moses/SearchNormal.cpp b/moses/SearchNormal.cpp index eb7d8710a..dc7c495f4 100644 --- a/moses/SearchNormal.cpp +++ b/moses/SearchNormal.cpp @@ -25,10 +25,6 @@ SearchNormal::SearchNormal(Manager& manager, const InputType &source, const Tran // only if constraint decoding (having to match a specified output) long sentenceID = source.GetTranslationId(); - m_constraint = staticData.GetConstrainingPhrase(sentenceID); - if (m_constraint) { - VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl); - } // initialize the stacks: create data structure and set limits std::vector < HypothesisStackNormal >::iterator iterStack; @@ -292,7 +288,7 @@ void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const Translat IFVERBOSE(2) { t = clock(); } - newHypo = hypothesis.CreateNext(transOpt, m_constraint); + newHypo = hypothesis.CreateNext(transOpt); IFVERBOSE(2) { stats.AddTimeBuildHyp( clock()-t ); } @@ -327,7 +323,7 @@ void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const Translat IFVERBOSE(2) { t = clock(); } - newHypo = hypothesis.CreateNext(transOpt, m_constraint); + newHypo = hypothesis.CreateNext(transOpt); if (newHypo==NULL) return; IFVERBOSE(2) { stats.AddTimeBuildHyp( clock()-t ); diff --git a/moses/SearchNormalBatch.cpp b/moses/SearchNormalBatch.cpp index b517f1454..4f648f652 100644 --- a/moses/SearchNormalBatch.cpp +++ b/moses/SearchNormalBatch.cpp @@ -125,7 +125,7 @@ void SearchNormalBatch::ExpandHypothesis(const Hypothesis &hypothesis, const Tra IFVERBOSE(2) { t = clock(); } - newHypo = hypothesis.CreateNext(transOpt, m_constraint); + newHypo = hypothesis.CreateNext(transOpt); IFVERBOSE(2) { stats.AddTimeBuildHyp( clock()-t ); } |