diff options
author | Nicola Bertoldi <bertoldi@fbk.eu> | 2014-12-13 14:52:47 +0300 |
---|---|---|
committer | Nicola Bertoldi <bertoldi@fbk.eu> | 2014-12-13 14:52:47 +0300 |
commit | e4eb201c52be74fee74399a6f35fcbe8eb85d834 (patch) | |
tree | 7792ef96d63262f6e28f1857741e1162c7dccbc4 /moses/Incremental.cpp | |
parent | cea2d9d8bb34a81660974cae20d66aefec4e0468 (diff) | |
parent | a0b6b6a341e74b47bbef4652ad7fd928cf91e17c (diff) |
merged master into dynamic-models and solved conflicts
Diffstat (limited to 'moses/Incremental.cpp')
-rw-r--r-- | moses/Incremental.cpp | 197 |
1 files changed, 192 insertions, 5 deletions
diff --git a/moses/Incremental.cpp b/moses/Incremental.cpp index a01b49ae7..d5de7b4d7 100644 --- a/moses/Incremental.cpp +++ b/moses/Incremental.cpp @@ -8,6 +8,7 @@ #include "moses/StaticData.h" #include "moses/Util.h" #include "moses/LM/Base.h" +#include "moses/OutputCollector.h" #include "lm/model.hh" #include "search/applied.hh" @@ -102,7 +103,7 @@ public: return vertex.BestChild(); } - void Evaluate(const InputType &input, const InputPath &inputPath) { + void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath) { // TODO for input lattice } private: @@ -203,7 +204,7 @@ struct ChartCellBaseFactory { } // namespace Manager::Manager(const InputType &source) : - source_(source), + BaseManager(source), cells_(source, ChartCellBaseFactory()), parser_(source, cells_), n_best_(search::NBestConfig(StaticData::Instance().GetNBestSize())) {} @@ -220,7 +221,7 @@ template <class Model, class Best> search::History Manager::PopulateBest(const M search::Config config(abstract.GetWeight() * M_LN10, data.GetCubePruningPopLimit(), search::NBestConfig(data.GetNBestSize())); search::Context<Model> context(config, model); - size_t size = source_.GetSize(); + size_t size = m_source.GetSize(); boost::object_pool<search::Vertex> vertex_pool(std::max<size_t>(size * size / 2, 32)); for (int startPos = size-1; startPos >= 0; --startPos) { @@ -272,12 +273,198 @@ template void Manager::LMCallback<lm::ngram::QuantTrieModel>(const lm::ngram::Qu template void Manager::LMCallback<lm::ngram::ArrayTrieModel>(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words); template void Manager::LMCallback<lm::ngram::QuantArrayTrieModel>(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words); -const std::vector<search::Applied> &Manager::ProcessSentence() +void Manager::Decode() { LanguageModel::GetFirstLM().IncrementalCallback(*this); +} + +const std::vector<search::Applied> &Manager::GetNBest() const +{ return *completed_nbest_; } +void Manager::OutputNBest(OutputCollector *collector) const +{ + if (collector == NULL) { + return; + } + + OutputNBestList(collector, *completed_nbest_, m_source.GetTranslationId()); +} + +void Manager::OutputNBestList(OutputCollector *collector, const std::vector<search::Applied> &nbest, long translationId) const +{ + const StaticData &staticData = StaticData::Instance(); + const std::vector<Moses::FactorType> &outputFactorOrder = staticData.GetOutputFactorOrder(); + + std::ostringstream out; + // wtf? copied from the original OutputNBestList + if (collector->OutputIsCout()) { + FixPrecision(out); + } + Phrase outputPhrase; + ScoreComponentCollection features; + for (std::vector<search::Applied>::const_iterator i = nbest.begin(); i != nbest.end(); ++i) { + Incremental::PhraseAndFeatures(*i, outputPhrase, features); + // <s> and </s> + UTIL_THROW_IF2(outputPhrase.GetSize() < 2, + "Output phrase should have contained at least 2 words (beginning and end-of-sentence)"); + + outputPhrase.RemoveWord(0); + outputPhrase.RemoveWord(outputPhrase.GetSize() - 1); + out << translationId << " ||| "; + OutputSurface(out, outputPhrase, outputFactorOrder, false); + out << " ||| "; + OutputAllFeatureScores(features, out); + out << " ||| " << i->GetScore() << '\n'; + } + out << std::flush; + assert(collector); + collector->Write(translationId, out.str()); +} + +void Manager::OutputDetailedTranslationReport(OutputCollector *collector) const +{ + if (collector && !completed_nbest_->empty()) { + const search::Applied &applied = completed_nbest_->at(0); + OutputDetailedTranslationReport(collector, + &applied, + static_cast<const Sentence&>(m_source), + m_source.GetTranslationId()); + } + +} + +void Manager::OutputDetailedTranslationReport( + OutputCollector *collector, + const search::Applied *applied, + const Sentence &sentence, + long translationId) const +{ + if (applied == NULL) { + return; + } + std::ostringstream out; + ApplicationContext applicationContext; + + OutputTranslationOptions(out, applicationContext, applied, sentence, translationId); + collector->Write(translationId, out.str()); +} + +void Manager::OutputTranslationOptions(std::ostream &out, + ApplicationContext &applicationContext, + const search::Applied *applied, + const Sentence &sentence, long translationId) const +{ + if (applied != NULL) { + OutputTranslationOption(out, applicationContext, applied, sentence, translationId); + out << std::endl; + } + + // recursive + const search::Applied *child = applied->Children(); + for (size_t i = 0; i < applied->GetArity(); i++) { + OutputTranslationOptions(out, applicationContext, child++, sentence, translationId); + } +} + +void Manager::OutputTranslationOption(std::ostream &out, + ApplicationContext &applicationContext, + const search::Applied *applied, + const Sentence &sentence, + long translationId) const +{ + ReconstructApplicationContext(applied, sentence, applicationContext); + const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(applied->GetNote().vp); + out << "Trans Opt " << translationId + << " " << applied->GetRange() + << ": "; + WriteApplicationContext(out, applicationContext); + out << ": " << phrase.GetTargetLHS() + << "->" << phrase + << " " << applied->GetScore(); // << hypo->GetScoreBreakdown() TODO: missing in incremental search hypothesis +} + +// Given a hypothesis and sentence, reconstructs the 'application context' -- +// the source RHS symbols of the SCFG rule that was applied, plus their spans. +void Manager::ReconstructApplicationContext(const search::Applied *applied, + const Sentence &sentence, + ApplicationContext &context) const +{ + context.clear(); + const WordsRange &span = applied->GetRange(); + const search::Applied *child = applied->Children(); + size_t i = span.GetStartPos(); + size_t j = 0; + + while (i <= span.GetEndPos()) { + if (j == applied->GetArity() || i < child->GetRange().GetStartPos()) { + // Symbol is a terminal. + const Word &symbol = sentence.GetWord(i); + context.push_back(std::make_pair(symbol, WordsRange(i, i))); + ++i; + } else { + // Symbol is a non-terminal. + const Word &symbol = static_cast<const TargetPhrase*>(child->GetNote().vp)->GetTargetLHS(); + const WordsRange &range = child->GetRange(); + context.push_back(std::make_pair(symbol, range)); + i = range.GetEndPos()+1; + ++child; + ++j; + } + } +} + +void Manager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const +{ + if (collector == NULL || Completed().empty()) { + return; + } + + const search::Applied *applied = &Completed()[0]; + const Sentence &sentence = dynamic_cast<const Sentence &>(m_source); + const size_t translationId = m_source.GetTranslationId(); + + std::ostringstream out; + ApplicationContext applicationContext; + + OutputTreeFragmentsTranslationOptions(out, applicationContext, applied, sentence, translationId); + + //Tree of full sentence + //TODO: incremental search doesn't support stateful features + + collector->Write(translationId, out.str()); + +} + +void Manager::OutputTreeFragmentsTranslationOptions(std::ostream &out, + ApplicationContext &applicationContext, + const search::Applied *applied, + const Sentence &sentence, + long translationId) const +{ + + if (applied != NULL) { + OutputTranslationOption(out, applicationContext, applied, sentence, translationId); + + const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp); + + out << " ||| "; + if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) { + out << " " << *property->GetValueString(); + } else { + out << " " << "noTreeInfo"; + } + out << std::endl; + } + + // recursive + const search::Applied *child = applied->Children(); + for (size_t i = 0; i < applied->GetArity(); i++) { + OutputTreeFragmentsTranslationOptions(out, applicationContext, child++, sentence, translationId); + } +} + namespace { @@ -327,7 +514,7 @@ void PhraseAndFeatures(const search::Applied final, Phrase &phrase, ScoreCompone const LanguageModel &model = LanguageModel::GetFirstLM(); model.CalcScore(phrase, full, ignored_ngram, ignored_oov); - // CalcScore transforms, but EvaluateChart doesn't. + // CalcScore transforms, but EvaluateWhenApplied doesn't. features.Assign(&model, full); } |