Implementation of Lattice sampling (Chatterjee and Cancedda, emnlp 2010)

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4295 1f5c12ca-751b-0410-a591-d2e778427230

3 files changed, 45 insertions, 15 deletions

diff --git a/moses-cmd/src/LatticeMBR.cpp b/moses-cmd/src/LatticeMBR.cpp
index 24b9fd6d6..ef7a5f71a 100644
--- a/moses-cmd/src/LatticeMBR.cpp
+++ b/moses-cmd/src/LatticeMBR.cpp
@@ -237,6 +237,7 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
         const ArcList *arcList = succHyp->GetArcList();
         if (arcList != NULL) {
           ArcList::const_iterator iterArcList;
+          //QUESTION: What happens if there's more than one loserPrevHypo?
           for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
             const Hypothesis *loserHypo = *iterArcList;
             const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
diff --git a/moses-cmd/src/LatticeMBR.h b/moses-cmd/src/LatticeMBR.h
index 5a5c15dff..8b54e6c51 100644
--- a/moses-cmd/src/LatticeMBR.h
+++ b/moses-cmd/src/LatticeMBR.h
@@ -19,17 +19,7 @@
 
 using namespace Moses;
 
-template<class T>
-T log_sum (T log_a, T log_b)
-{
-  T v;
-  if (log_a < log_b) {
-    v = log_b+log ( 1 + exp ( log_a-log_b ));
-  } else {
-    v = log_a+log ( 1 + exp ( log_b-log_a ));
-  }
-  return ( v );
-}
+
 
 class Edge;
 
diff --git a/moses-cmd/src/Main.cpp b/moses-cmd/src/Main.cpp
index 0a0da3341..76bbbee20 100644
--- a/moses-cmd/src/Main.cpp
+++ b/moses-cmd/src/Main.cpp
@@ -72,11 +72,13 @@ public:
 
   TranslationTask(size_t lineNumber,
                   InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
+                  OutputCollector* latticeSamplesCollector,
                   OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
                   OutputCollector* detailedTranslationCollector,
                   OutputCollector* alignmentInfoCollector ) :
     m_source(source), m_lineNumber(lineNumber),
     m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
+    m_latticeSamplesCollector(latticeSamplesCollector),
     m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
     m_detailedTranslationCollector(detailedTranslationCollector),
     m_alignmentInfoCollector(alignmentInfoCollector) {}
@@ -240,6 +242,15 @@ public:
       m_nbestCollector->Write(m_lineNumber, out.str());
     }
 
+    //lattice samples
+    if (m_latticeSamplesCollector) {
+      TrellisPathList latticeSamples;
+      ostringstream out;
+      manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
+      OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), manager.GetTranslationSystem(), m_lineNumber);
+      m_latticeSamplesCollector->Write(m_lineNumber, out.str());
+    }
+
     // detailed translation reporting
     if (m_detailedTranslationCollector) {
       ostringstream out;
@@ -264,6 +275,7 @@ private:
   size_t m_lineNumber;
   OutputCollector* m_outputCollector;
   OutputCollector* m_nbestCollector;
+  OutputCollector* m_latticeSamplesCollector;
   OutputCollector* m_wordGraphCollector;
   OutputCollector* m_searchGraphCollector;
   OutputCollector* m_detailedTranslationCollector;
@@ -352,6 +364,9 @@ int main(int argc, char** argv)
   const StaticData& staticData = StaticData::Instance();
 
 
+  //initialise random numbers
+  srand(time(NULL));
+
   // set up read/writing class
   IOWrapper* ioWrapper = GetIODevice(staticData);
   if (!ioWrapper) {
@@ -380,21 +395,43 @@ int main(int argc, char** argv)
   // because multithreading may return sentences in shuffled order
   auto_ptr<OutputCollector> outputCollector; // for translations
   auto_ptr<OutputCollector> nbestCollector;  // for n-best lists
+  auto_ptr<OutputCollector> latticeSamplesCollector; //for lattice samples
   auto_ptr<ofstream> nbestOut;
+  auto_ptr<ofstream> latticeSamplesOut;
   size_t nbestSize = staticData.GetNBestSize();
   string nbestFile = staticData.GetNBestFilePath();
+  bool output1best = true;
   if (nbestSize) {
     if (nbestFile == "-" || nbestFile == "/dev/stdout") {
       // nbest to stdout, no 1-best
       nbestCollector.reset(new OutputCollector());
+      output1best = false;
     } else {
       // nbest to file, 1-best to stdout
       nbestOut.reset(new ofstream(nbestFile.c_str()));
-      assert(nbestOut->good());
+      if (!nbestOut->good()) {
+        TRACE_ERR("ERROR: Failed to open " << nbestFile << " for nbest lists" << endl);
+        exit(1);
+      }
       nbestCollector.reset(new OutputCollector(nbestOut.get()));
-      outputCollector.reset(new OutputCollector());
     }
-  } else {
+  }
+  size_t latticeSamplesSize = staticData.GetLatticeSamplesSize();
+  string latticeSamplesFile = staticData.GetLatticeSamplesFilePath();
+  if (latticeSamplesSize) {
+    if (latticeSamplesFile == "-" || latticeSamplesFile == "/dev/stdout") {
+      latticeSamplesCollector.reset(new OutputCollector());
+      output1best = false;
+    } else {
+      latticeSamplesOut.reset(new ofstream(latticeSamplesFile.c_str()));
+      if (!latticeSamplesOut->good()) {
+        TRACE_ERR("ERROR: Failed to open " << latticeSamplesFile << " for lattice samples" << endl);
+        exit(1);
+      }
+      latticeSamplesCollector.reset(new OutputCollector(latticeSamplesOut.get()));
+    }
+  }
+  if (output1best) {
     outputCollector.reset(new OutputCollector());
   }
 
@@ -437,7 +474,9 @@ int main(int argc, char** argv)
     // set up task of translating one sentence
     TranslationTask* task =
       new TranslationTask(lineCount,source, outputCollector.get(),
-                          nbestCollector.get(), wordGraphCollector.get(),
+                          nbestCollector.get(),
+                          latticeSamplesCollector.get(),
+                          wordGraphCollector.get(),
                           searchGraphCollector.get(),
                           detailedTranslationCollector.get(),
                           alignmentInfoCollector.get() );