sentence-bleu less greedy regarding memory

Don't load all references, read them line by line.
Corpora with millions of sentences can now be evaluated without consuming gigabytes of RAM.

1 files changed, 28 insertions, 4 deletions

diff --git a/mert/sentence-bleu-nbest.cpp b/mert/sentence-bleu-nbest.cpp
index 023e9faae..f869386e3 100644
--- a/mert/sentence-bleu-nbest.cpp
+++ b/mert/sentence-bleu-nbest.cpp
@@ -1,9 +1,14 @@
+#include <fstream>
 #include <iostream>
 #include <vector>
 #include <string>
 
+#include <boost/shared_ptr.hpp>
+
 #include "BleuScorer.h"
+#include "Reference.h"
 #include "moses/Util.h"
+#include "util/exception.hh"
 
 using namespace MosesTuning;
 
@@ -24,21 +29,40 @@ int main(int argc, char **argv)
   BleuScorer scorer(config);
   scorer.setFactors(factors);
   scorer.setFilter(filter);
-  scorer.setReferenceFiles(refFiles); // TODO: we don't need to load the whole reference corpus into memory (this can take gigabytes of RAM if done with millions of sentences)
 
-  // Loading sentences and preparing statistics
+  // initialize reference streams
+  std::vector<boost::shared_ptr<std::ifstream> > refStreams;
+  for (std::vector<std::string>::const_iterator refFile=refFiles.begin(); refFile!=refFiles.end(); ++refFile)
+  {
+    TRACE_ERR("Loading reference from " << *refFile << std::endl);
+    boost::shared_ptr<std::ifstream> ifs(new std::ifstream(refFile->c_str()));
+    UTIL_THROW_IF2(!ifs, "Cannot open " << *refFile);
+    refStreams.push_back(ifs);
+  }
+
+  // load sentences, preparing statistics, score
   std::string nbestLine;
+  int sid = -1;
+  Reference ref;
   while ( getline(std::cin, nbestLine) ) 
   {
     std::vector<std::string> items;
     Moses::TokenizeMultiCharSeparator(items, nbestLine, " ||| ");
-    size_t sid = Moses::Scan<size_t>(items[0]);
+    int sidCurrent = Moses::Scan<int>(items[0]);
 
+    if (sidCurrent != sid) {
+      ref.clear();
+      if (!scorer.GetNextReferenceFromStreams(refStreams, ref)) {
+        UTIL_THROW2("Missing references");
+      }
+      sid = sidCurrent;
+    }
     ScoreStats scoreStats;
-    scorer.prepareStats(sid, items[1], scoreStats);
+    scorer.CalcBleuStats(ref, items[1], scoreStats);
     std::vector<float> stats(scoreStats.getArray(), scoreStats.getArray() + scoreStats.size());
     std::cout << smoothedSentenceBleu(stats) << std::endl;
   }
 
   return 0;
 }
+