diff options
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_bitext_jstats.h | 3 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_lexical_reordering.cc | 24 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mm/ug_phrasepair.h | 119 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mmsapt.cpp | 39 | ||||
-rw-r--r-- | moses/TranslationModel/UG/mmsapt.h | 4 |
5 files changed, 168 insertions, 21 deletions
diff --git a/moses/TranslationModel/UG/mm/ug_bitext_jstats.h b/moses/TranslationModel/UG/mm/ug_bitext_jstats.h index 13c86e34d..ce2e89438 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_jstats.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_jstats.h @@ -43,6 +43,9 @@ namespace Moses bool valid(); uint32_t dcnt_fwd(PhraseOrientation const idx) const; uint32_t dcnt_bwd(PhraseOrientation const idx) const; + void fill_lr_vec(Moses::LRModel::Direction const& dir, + Moses::LRModel::ModelType const& mdl, + vector<float>& v); }; } } diff --git a/moses/TranslationModel/UG/mm/ug_lexical_reordering.cc b/moses/TranslationModel/UG/mm/ug_lexical_reordering.cc index 706c042c0..00f499f88 100644 --- a/moses/TranslationModel/UG/mm/ug_lexical_reordering.cc +++ b/moses/TranslationModel/UG/mm/ug_lexical_reordering.cc @@ -126,11 +126,25 @@ namespace Moses T = x = e1; B = a1.size()-1; if (expand_block(a1,a2,x,y,T,L,B,R) >= 0) return Moses::LRModel::S; - while (s2 && a2[s2].size() == 0) --s2; - if (a2[s2].size() == 0) return po_other; - if (a2[s2].back() < s1) return Moses::LRModel::DR; - if (a2[s2].front() >= e1) return Moses::LRModel::DL; - return po_other; + size_t s2x = s2; + while (s2-- && a2[s2].size() == 0); + + Moses::LRModel::ReorderingType ret; + ret = (a2[s2].size() == 0 ? po_other : + a2[s2].back() < s1 ? Moses::LRModel::DR : + a2[s2].front() >= e1 ? Moses::LRModel::DL : + po_other); +#if 0 + cout << "s1=" << s1 << endl; + cout << "s2=" << s2x << "=>" << s2 << endl; + cout << "e1=" << e1 << endl; + cout << "e2=" << e2 << endl; + cout << "a2[s2].size()=" << a2[s2].size() << endl; + cout << "a2[s2].back()=" << a2[s2].back() << endl; + cout << "a2[s2].front()=" << a2[s2].front() << endl; + cout << "RETURNING " << ret << endl; +#endif + return ret; } } // namespace bitext diff --git a/moses/TranslationModel/UG/mm/ug_phrasepair.h b/moses/TranslationModel/UG/mm/ug_phrasepair.h index 28a926587..70d4b0d82 100644 --- a/moses/TranslationModel/UG/mm/ug_phrasepair.h +++ b/moses/TranslationModel/UG/mm/ug_phrasepair.h @@ -3,7 +3,9 @@ #include <vector> #include "ug_typedefs.h" #include "ug_bitext_pstats.h" - +#include "moses/FF/LexicalReordering/LexicalReorderingState.h" +#include "boost/format.hpp" +#include "tpt_tokenindex.h" namespace Moses { namespace bitext @@ -45,7 +47,15 @@ namespace Moses PhrasePair const& update(uint64_t const pid2, Token const* x, uint32_t const len, jstats const& js); - + + void + fill_lr_vec(LRModel::Direction const& dir, + LRModel::ModelType const& mdl, + vector<float>& v) const; + void + print(ostream& out, TokenIndex const& V1, TokenIndex const& V2, + LRModel const& LR) const; + class SortByTargetIdSeq { public: @@ -98,20 +108,20 @@ namespace Moses assert(js.aln().size()); if (js.aln().size()) aln = js.aln()[0].second; - float total_fwd = 0, total_bwd = 0; - for (int i = 0; i <= Moses::LRModel::NONE; i++) - { - PhraseOrientation po = static_cast<PhraseOrientation>(i); - total_fwd += js.dcnt_fwd(po)+1; - total_bwd += js.dcnt_bwd(po)+1; - } + // float total_fwd = 0, total_bwd = 0; + // for (int i = 0; i <= Moses::LRModel::NONE; i++) + // { + // PhraseOrientation po = static_cast<PhraseOrientation>(i); + // total_fwd += js.dcnt_fwd(po)+1; + // total_bwd += js.dcnt_bwd(po)+1; + // } // should we do that here or leave the raw counts? for (int i = 0; i <= Moses::LRModel::NONE; i++) { PhraseOrientation po = static_cast<PhraseOrientation>(i); - dfwd[i] = float(js.dcnt_fwd(po)+1)/total_fwd; - dbwd[i] = float(js.dcnt_bwd(po)+1)/total_bwd; + dfwd[i] = js.dcnt_fwd(po); + dbwd[i] = js.dcnt_bwd(po); } indoc = js.indoc; @@ -162,6 +172,7 @@ namespace Moses joint += o.joint; sample1 += o.sample1; sample2 += o.sample2; + // todo: add distortion counts return *this; } @@ -226,7 +237,8 @@ namespace Moses } template<typename Token> - bool PhrasePair<Token> + bool + PhrasePair<Token> ::SortDescendingByJointCount ::operator()(PhrasePair const& a, PhrasePair const& b) const { @@ -234,7 +246,8 @@ namespace Moses } template<typename Token> - void PhrasePair<Token> + void + PhrasePair<Token> ::init() { inverse = false; @@ -242,5 +255,81 @@ namespace Moses start1 = start2 = NULL; p1 = p2 = 0; } - } -} + + + void + fill_lr_vec2(LRModel::ModelType mdl, float const* const cnt, + float const total, float* v); + + template<typename Token> + void + PhrasePair<Token> + ::fill_lr_vec(LRModel::Direction const& dir, + LRModel::ModelType const& mdl, + vector<float>& v) const + { + // how many distinct scores do we have? + size_t num_scores = (mdl == LRModel::MSLR ? 4 : mdl == LRModel::MSD ? 3 : 2); + size_t offset; + if (dir == LRModel::Bidirectional) + { + offset = num_scores; + num_scores *= 2; + } + else offset = 0; + + v.resize(num_scores); + + // determine the denominator + float total = 0; + for (size_t i = 0; i <= LRModel::NONE; ++i) + total += dfwd[i]; + + if (dir != LRModel::Forward) // i.e., Backward or Bidirectional + fill_lr_vec2(mdl, dbwd, total, &v[0]); + if (dir != LRModel::Backward) // i.e., Forward or Bidirectional + fill_lr_vec2(mdl, dfwd, total, &v[offset]); + } + + + template<typename Token> + void + PhrasePair<Token> + ::print(ostream& out, TokenIndex const& V1, TokenIndex const& V2, + LRModel const& LR) const + { + out << toString (V1, this->start1, this->len1) << " ::: " + << toString (V2, this->start2, this->len2) << " " + << this->joint << " ["; + for (size_t i = 0; i < this->indoc.size(); ++i) + { + if (i) out << " "; + out << this->indoc[i]; + } + out << "] ["; + vector<float> lrscores; + this->fill_lr_vec(LR.GetDirection(), LR.GetModelType(), lrscores); + for (size_t i = 0; i < lrscores.size(); ++i) + { + if (i) out << " "; + out << boost::format("%.2f") % exp(lrscores[i]); + } + out << "]" << endl; +#if 0 + for (int i = 0; i <= Moses::LRModel::NONE; i++) + { + // PhraseOrientation po = static_cast<PhraseOrientation>(i); + if (i) *log << " "; + *log << p.dfwd[i]; + } + *log << "] ["; + for (int i = 0; i <= Moses::LRModel::NONE; i++) + { + // PhraseOrientation po = static_cast<PhraseOrientation>(i); + if (i) *log << " "; + *log << p.dbwd[i]; + } +#endif + } + } // namespace bitext +} // namespace Moses diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index 82aa7c227..af1053438 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -73,6 +73,7 @@ namespace Moses : PhraseDictionary(line) , m_bias_log(NULL) , m_bias_loglevel(0) + , m_lr_func(NULL) , cache_key(((char*)this)+2) , context_key(((char*)this)+1) // , m_tpc_ctr(0) @@ -212,6 +213,9 @@ namespace Moses } } + if ((m = param.find("lr-func")) != param.end()) + m_lr_func_name = m->second; + if ((m = param.find("extra")) != param.end()) m_extra_data = m->second; @@ -244,6 +248,7 @@ namespace Moses known_parameters.push_back("lexalpha"); // known_parameters.push_back("limit"); // replaced by "table-limit" known_parameters.push_back("logcnt"); + known_parameters.push_back("lr-func"); // associated lexical reordering function known_parameters.push_back("name"); known_parameters.push_back("num-features"); known_parameters.push_back("output-factor"); @@ -533,6 +538,16 @@ namespace Moses tp->SetAlignTerm(pool.aln); tp->GetScoreBreakdown().Assign(this, fvals); tp->EvaluateInIsolation(src); + + if (m_lr_func) + { + LRModel::ModelType mdl = m_lr_func->GetModel().GetModelType(); + LRModel::Direction dir = m_lr_func->GetModel().GetDirection(); + sptr<Scores> scores(new Scores()); + pool.fill_lr_vec(dir, mdl, *scores); + tp->SetExtraScores(m_lr_func, scores); + } + return tp; } @@ -643,7 +658,6 @@ namespace Moses expand(mdyn, *dyn, *sdyn, ppdyn, m_bias_log); sort(ppdyn.begin(), ppdyn.end(),sort_by_tgt_id); } - // now we have two lists of Phrase Pairs, let's merge them ret = new TPCollWrapper(dyn->revision(), phrasekey); PhrasePair<Token>::SortByTargetIdSeq sorter; @@ -660,6 +674,20 @@ namespace Moses if (m_tableLimit) ret->Prune(true, m_tableLimit); else ret->Prune(true,ret->GetSize()); +#if 1 + if (m_bias_log && m_lr_func) + { + typename PhrasePair<Token>::SortDescendingByJointCount sorter; + sort(ppfix.begin(), ppfix.end(),sorter); + BOOST_FOREACH(PhrasePair<Token> const& pp, ppfix) + { + if (&pp != &ppfix.front() && pp.joint <= 1) break; + pp.print(*m_bias_log,*btfix.V1, *btfix.V2, m_lr_func->GetModel()); + } + } +#endif + + #if 0 if (combine_pstats(src, mfix.getPid(), sfix.get(), btfix, @@ -753,6 +781,15 @@ namespace Moses else localcache = m_cache; scope->set<TPCollCache>(cache_key, localcache); } + + if (m_lr_func_name.size() && m_lr_func == NULL) + { + FeatureFunction* lr = &FeatureFunction::FindFeatureFunction(m_lr_func_name); + m_lr_func = dynamic_cast<LexicalReordering*>(lr); + UTIL_THROW_IF2(lr == NULL, "FF " << m_lr_func_name + << " does not seem to be a lexical reordering function!"); + // todo: verify that lr_func implements a hierarchical reordering model + } } // bool diff --git a/moses/TranslationModel/UG/mmsapt.h b/moses/TranslationModel/UG/mmsapt.h index b4555a240..3ccdeb539 100644 --- a/moses/TranslationModel/UG/mmsapt.h +++ b/moses/TranslationModel/UG/mmsapt.h @@ -22,6 +22,8 @@ #include "moses/TranslationModel/UG/TargetPhraseCollectionCache.h" +#include "moses/FF/LexicalReordering/LexicalReordering.h" + #include "moses/InputFileStream.h" #include "moses/FactorTypeSet.h" #include "moses/TargetPhrase.h" @@ -77,6 +79,8 @@ namespace Moses boost::scoped_ptr<ofstream> m_bias_logger; // for logging to a file ostream* m_bias_log; int m_bias_loglevel; + LexicalReordering* m_lr_func; // associated lexical reordering function + string m_lr_func_name; // name of associated lexical reordering function public: void* const cache_key; // for getting cache from ttask void* const context_key; // for context scope from ttask |