Work on compression and defaults for outputting hypergraphs.

4 files changed, 73 insertions, 35 deletions

diff --git a/moses-cmd/IOWrapper.cpp b/moses-cmd/IOWrapper.cpp
index 2da30f380..335a570a6 100644
--- a/moses-cmd/IOWrapper.cpp
+++ b/moses-cmd/IOWrapper.cpp
@@ -189,15 +189,6 @@ InputType*IOWrapper::GetInput(InputType* inputType)
   }
 }
 
-  ofstream* IOWrapper::GetOutputSearchGraphHypergraphWeightsStream() {
-    const StaticData &staticData = StaticData::Instance();
-    stringstream fileName;
-    fileName << staticData.GetParam("output-search-graph-hypergraph")[1];
-    std::ofstream *file = new std::ofstream;
-    file->open(fileName.str().c_str());
-    return file;
-  }
-
 /***
  * print surface factor only for the given phrase
  */
diff --git a/moses-cmd/IOWrapper.h b/moses-cmd/IOWrapper.h
index 267a3a0bc..8dbdeda9c 100644
--- a/moses-cmd/IOWrapper.h
+++ b/moses-cmd/IOWrapper.h
@@ -117,8 +117,6 @@ public:
     return *m_outputSearchGraphStream;
   }
 
-  std::ofstream *GetOutputSearchGraphHypergraphWeightsStream();
-
   std::ostream &GetDetailedTranslationReportingStream() {
     assert (m_detailedTranslationReportingStream);
     return *m_detailedTranslationReportingStream;
diff --git a/moses-cmd/Jamfile b/moses-cmd/Jamfile
index 42d54568f..bddc10911 100644
--- a/moses-cmd/Jamfile
+++ b/moses-cmd/Jamfile
@@ -1,4 +1,4 @@
-alias deps : IOWrapper.cpp mbr.cpp LatticeMBR.cpp TranslationAnalysis.cpp ..//z ..//boost_iostreams ../moses//moses ;
+alias deps : IOWrapper.cpp mbr.cpp LatticeMBR.cpp TranslationAnalysis.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ;
 
 exe moses : Main.cpp deps ;
 exe lmbrgrid : LatticeMBRGrid.cpp deps ;
diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp
index 90c924a08..8f6a6c069 100644
--- a/moses-cmd/Main.cpp
+++ b/moses-cmd/Main.cpp
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  * Moses main, for single-threaded and multi-threaded.
  **/
 
+#include <boost/filesystem.hpp>
 #include <boost/iostreams/device/file.hpp>
 #include <boost/iostreams/filter/bzip2.hpp>
 #include <boost/iostreams/filter/gzip.hpp>
@@ -171,30 +172,76 @@ public:
 
     // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
     if (m_outputSearchGraphHypergraph) {
-      stringstream fileName;
-      fileName << staticData.GetParam("output-search-graph-hypergraph")[1] << "/" << m_lineNumber;
-      boost::iostreams::filtering_ostream *file = new boost::iostreams::filtering_ostream;
-      //      file->open(fileName.str().c_str());
-      string compression = staticData.GetParam("output-search-graph-hypergraph")[0];
-      if ( compression == "gzip" || compression == "gz" ) {
-	file->push( boost::iostreams::gzip_compressor() );
-      } else if ( compression == "bzip2" || compression == "bz2" ) {
-	file->push( boost::iostreams::bzip2_compressor() );
+
+      vector<string> hypergraphParameters = staticData.GetParam("output-search-graph-hypergraph");
+
+      bool appendSuffix;
+      if (hypergraphParameters.size() > 0 && hypergraphParameters[0] == "true") {
+	appendSuffix = true;
+      } else {
+	appendSuffix = false;
+      }
+
+      string compression;
+      if (hypergraphParameters.size() > 1) {
+	compression = hypergraphParameters[1];
+      } else {
+	compression = "txt";
+      }
+
+      string hypergraphDir;
+      if ( hypergraphParameters.size() > 2 ) {
+	hypergraphDir = hypergraphParameters[2];
+      } else {
+	string nbestFile = staticData.GetNBestFilePath();
+	if ( ! nbestFile.empty()) {
+	  boost::filesystem::path nbestPath(nbestFile);
+	  hypergraphDir = nbestPath.parent_path().filename();
+	} else {
+	  stringstream hypergraphDirName;
+	  hypergraphDirName << boost::filesystem::current_path() << "/hypergraph";
+	  hypergraphDir = hypergraphDirName.str();
+	}
+      }
+
+      if ( ! boost::filesystem::exists(hypergraphDir) ) {
+	boost::filesystem::create_directory(hypergraphDir);
       } 
-      file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) );
-      //      if (file->is_open() && file->good()) {
-      if (file->is_complete() && file->good()) {
-	//	ostringstream out;
-	//	fix(out,PRECISION);
-	fix(*file,PRECISION);
-	manager.OutputSearchGraphAsHypergraph(m_lineNumber, *file);
-	//	*file << out.str();
-	file -> flush();
+
+      if ( ! boost::filesystem::exists(hypergraphDir) ) {
+	TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because the directory does not exist" << std::endl);
+      } else if ( ! boost::filesystem::is_directory(hypergraphDir) ) {
+	TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because that path exists, but is not a directory" << std::endl);
       } else {
-	TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl);
+	
+	stringstream fileName;
+	fileName << hypergraphDir << "/" << m_lineNumber;
+	if ( appendSuffix ) {
+	  fileName << "." << compression;
+	}
+	boost::iostreams::filtering_ostream *file = new boost::iostreams::filtering_ostream;
+
+	if ( compression == "gz" ) {
+	  file->push( boost::iostreams::gzip_compressor() );
+	} else if ( compression == "bz2" ) {
+	  file->push( boost::iostreams::bzip2_compressor() );
+	} else if ( compression != "txt" ) {
+	  TRACE_ERR("Unrecognized hypergraph compression format (" << compression << ") - using uncompressed plain txt" << std::endl);
+	  compression = "txt";
+	}
+
+	file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) );
+
+	if (file->is_complete() && file->good()) {
+	  fix(*file,PRECISION);
+	  manager.OutputSearchGraphAsHypergraph(m_lineNumber, *file);
+	  file -> flush();
+	} else {
+	  TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber << " because the output file " << fileName.str() << " is not open or not ready for writing" << std::endl);
+	}
+	file -> pop();
+	delete file;
       }
-      file -> pop();
-      delete file;
     }
 
     // apply decision rule and output best translation(s)
@@ -548,8 +595,10 @@ int main(int argc, char** argv)
       TRACE_ERR(weights);
       TRACE_ERR("\n");
     }
-    if (staticData.GetOutputSearchGraphHypergraph() && staticData.GetParam("output-search-graph-hypergraph").size() > 1) {
-      ofstream* weightsOut = ioWrapper->GetOutputSearchGraphHypergraphWeightsStream();
+    if (staticData.GetOutputSearchGraphHypergraph() && staticData.GetParam("output-search-graph-hypergraph").size() > 3) {
+      ofstream* weightsOut = new std::ofstream;
+      string weightsFilename = staticData.GetParam("output-search-graph-hypergraph")[3];
+      weightsOut->open(weightsFilename.c_str());
       OutputFeatureWeightsForHypergraph(*weightsOut);
       weightsOut->flush();
       weightsOut->close();