merged master into dynamic-models and solved conflicts

1 files changed, 192 insertions, 5 deletions

diff --git a/moses/Incremental.cpp b/moses/Incremental.cpp
index a01b49ae7..d5de7b4d7 100644
--- a/moses/Incremental.cpp
+++ b/moses/Incremental.cpp
@@ -8,6 +8,7 @@
 #include "moses/StaticData.h"
 #include "moses/Util.h"
 #include "moses/LM/Base.h"
+#include "moses/OutputCollector.h"
 
 #include "lm/model.hh"
 #include "search/applied.hh"
@@ -102,7 +103,7 @@ public:
     return vertex.BestChild();
   }
 
-  void Evaluate(const InputType &input, const InputPath &inputPath) {
+  void EvaluateWithSourceContext(const InputType &input, const InputPath &inputPath) {
     // TODO for input lattice
   }
 private:
@@ -203,7 +204,7 @@ struct ChartCellBaseFactory {
 } // namespace
 
 Manager::Manager(const InputType &source) :
-  source_(source),
+  BaseManager(source),
   cells_(source, ChartCellBaseFactory()),
   parser_(source, cells_),
   n_best_(search::NBestConfig(StaticData::Instance().GetNBestSize())) {}
@@ -220,7 +221,7 @@ template <class Model, class Best> search::History Manager::PopulateBest(const M
   search::Config config(abstract.GetWeight() * M_LN10, data.GetCubePruningPopLimit(), search::NBestConfig(data.GetNBestSize()));
   search::Context<Model> context(config, model);
 
-  size_t size = source_.GetSize();
+  size_t size = m_source.GetSize();
   boost::object_pool<search::Vertex> vertex_pool(std::max<size_t>(size * size / 2, 32));
 
   for (int startPos = size-1; startPos >= 0; --startPos) {
@@ -272,12 +273,198 @@ template void Manager::LMCallback<lm::ngram::QuantTrieModel>(const lm::ngram::Qu
 template void Manager::LMCallback<lm::ngram::ArrayTrieModel>(const lm::ngram::ArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
 template void Manager::LMCallback<lm::ngram::QuantArrayTrieModel>(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
 
-const std::vector<search::Applied> &Manager::ProcessSentence()
+void Manager::Decode()
 {
   LanguageModel::GetFirstLM().IncrementalCallback(*this);
+}
+
+const std::vector<search::Applied> &Manager::GetNBest() const
+{
   return *completed_nbest_;
 }
 
+void Manager::OutputNBest(OutputCollector *collector)  const
+{
+  if (collector == NULL) {
+	  return;
+  }
+
+  OutputNBestList(collector, *completed_nbest_, m_source.GetTranslationId());
+}
+
+void Manager::OutputNBestList(OutputCollector *collector, const std::vector<search::Applied> &nbest, long translationId) const
+{
+  const StaticData &staticData = StaticData::Instance();
+  const std::vector<Moses::FactorType> &outputFactorOrder = staticData.GetOutputFactorOrder();
+
+  std::ostringstream out;
+  // wtf? copied from the original OutputNBestList
+  if (collector->OutputIsCout()) {
+    FixPrecision(out);
+  }
+  Phrase outputPhrase;
+  ScoreComponentCollection features;
+  for (std::vector<search::Applied>::const_iterator i = nbest.begin(); i != nbest.end(); ++i) {
+    Incremental::PhraseAndFeatures(*i, outputPhrase, features);
+    // <s> and </s>
+    UTIL_THROW_IF2(outputPhrase.GetSize() < 2,
+  		  "Output phrase should have contained at least 2 words (beginning and end-of-sentence)");
+
+    outputPhrase.RemoveWord(0);
+    outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
+    out << translationId << " ||| ";
+    OutputSurface(out, outputPhrase, outputFactorOrder, false);
+    out << " ||| ";
+    OutputAllFeatureScores(features, out);
+    out << " ||| " << i->GetScore() << '\n';
+  }
+  out << std::flush;
+  assert(collector);
+  collector->Write(translationId, out.str());
+}
+
+void Manager::OutputDetailedTranslationReport(OutputCollector *collector) const
+{
+	if (collector && !completed_nbest_->empty()) {
+		const search::Applied &applied = completed_nbest_->at(0);
+		OutputDetailedTranslationReport(collector,
+										&applied,
+										static_cast<const Sentence&>(m_source),
+										m_source.GetTranslationId());
+	}
+
+}
+
+void Manager::OutputDetailedTranslationReport(
+  OutputCollector *collector,
+  const search::Applied *applied,
+  const Sentence &sentence,
+  long translationId) const
+{
+  if (applied == NULL) {
+    return;
+  }
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTranslationOptions(out, applicationContext, applied, sentence, translationId);
+  collector->Write(translationId, out.str());
+}
+
+void Manager::OutputTranslationOptions(std::ostream &out,
+		ApplicationContext &applicationContext,
+		const search::Applied *applied,
+		const Sentence &sentence, long translationId) const
+{
+  if (applied != NULL) {
+    OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
+    out << std::endl;
+  }
+
+  // recursive
+  const search::Applied *child = applied->Children();
+  for (size_t i = 0; i < applied->GetArity(); i++) {
+      OutputTranslationOptions(out, applicationContext, child++, sentence, translationId);
+  }
+}
+
+void Manager::OutputTranslationOption(std::ostream &out,
+		ApplicationContext &applicationContext,
+		const search::Applied *applied,
+		const Sentence &sentence,
+		long translationId) const
+{
+  ReconstructApplicationContext(applied, sentence, applicationContext);
+  const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
+  out << "Trans Opt " << translationId
+      << " " << applied->GetRange()
+      << ": ";
+  WriteApplicationContext(out, applicationContext);
+  out << ": " << phrase.GetTargetLHS()
+      << "->" << phrase
+      << " " << applied->GetScore(); // << hypo->GetScoreBreakdown() TODO: missing in incremental search hypothesis
+}
+
+// Given a hypothesis and sentence, reconstructs the 'application context' --
+// the source RHS symbols of the SCFG rule that was applied, plus their spans.
+void Manager::ReconstructApplicationContext(const search::Applied *applied,
+    const Sentence &sentence,
+    ApplicationContext &context) const
+{
+  context.clear();
+  const WordsRange &span = applied->GetRange();
+  const search::Applied *child = applied->Children();
+  size_t i = span.GetStartPos();
+  size_t j = 0;
+
+  while (i <= span.GetEndPos()) {
+    if (j == applied->GetArity() || i < child->GetRange().GetStartPos()) {
+      // Symbol is a terminal.
+      const Word &symbol = sentence.GetWord(i);
+      context.push_back(std::make_pair(symbol, WordsRange(i, i)));
+      ++i;
+    } else {
+      // Symbol is a non-terminal.
+      const Word &symbol = static_cast<const TargetPhrase*>(child->GetNote().vp)->GetTargetLHS();
+      const WordsRange &range = child->GetRange();
+      context.push_back(std::make_pair(symbol, range));
+      i = range.GetEndPos()+1;
+      ++child;
+      ++j;
+    }
+  }
+}
+
+void Manager::OutputDetailedTreeFragmentsTranslationReport(OutputCollector *collector) const
+{
+  if (collector == NULL || Completed().empty()) {
+	  return;
+  }
+
+  const search::Applied *applied = &Completed()[0];
+  const Sentence &sentence = dynamic_cast<const Sentence &>(m_source);
+  const size_t translationId = m_source.GetTranslationId();
+
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTreeFragmentsTranslationOptions(out, applicationContext, applied, sentence, translationId);
+
+  //Tree of full sentence
+  //TODO: incremental search doesn't support stateful features
+
+  collector->Write(translationId, out.str());
+
+}
+
+void Manager::OutputTreeFragmentsTranslationOptions(std::ostream &out,
+		ApplicationContext &applicationContext,
+		const search::Applied *applied,
+		const Sentence &sentence,
+		long translationId) const
+{
+
+  if (applied != NULL) {
+    OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
+
+    const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
+
+    out << " ||| ";
+    if (const PhraseProperty *property = currTarPhr.GetProperty("Tree")) {
+      out << " " << *property->GetValueString();
+    } else {
+      out << " " << "noTreeInfo";
+    }
+    out << std::endl;
+  }
+
+  // recursive
+  const search::Applied *child = applied->Children();
+  for (size_t i = 0; i < applied->GetArity(); i++) {
+      OutputTreeFragmentsTranslationOptions(out, applicationContext, child++, sentence, translationId);
+  }
+}
+
 namespace
 {
 
@@ -327,7 +514,7 @@ void PhraseAndFeatures(const search::Applied final, Phrase &phrase, ScoreCompone
 
   const LanguageModel &model = LanguageModel::GetFirstLM();
   model.CalcScore(phrase, full, ignored_ngram, ignored_oov);
-  // CalcScore transforms, but EvaluateChart doesn't.
+  // CalcScore transforms, but EvaluateWhenApplied doesn't.
   features.Assign(&model, full);
 }