diff options
author | Lane Schwartz <dowobeha@gmail.com> | 2013-03-22 00:48:47 +0400 |
---|---|---|
committer | Lane Schwartz <dowobeha@gmail.com> | 2013-03-22 00:48:47 +0400 |
commit | b2bba0bae344cdace0ec3fe9dc4292bcae0a2a43 (patch) | |
tree | a751a83bd24e8a09003d9d8210de2e5e5212bd4c /moses-cmd | |
parent | 28c980d58e4d3b53c257d2979de99a746bb7b0e3 (diff) |
Work on compression and defaults for outputting hypergraphs.
Diffstat (limited to 'moses-cmd')
-rw-r--r-- | moses-cmd/IOWrapper.cpp | 9 | ||||
-rw-r--r-- | moses-cmd/IOWrapper.h | 2 | ||||
-rw-r--r-- | moses-cmd/Jamfile | 2 | ||||
-rw-r--r-- | moses-cmd/Main.cpp | 95 |
4 files changed, 73 insertions, 35 deletions
diff --git a/moses-cmd/IOWrapper.cpp b/moses-cmd/IOWrapper.cpp index 2da30f380..335a570a6 100644 --- a/moses-cmd/IOWrapper.cpp +++ b/moses-cmd/IOWrapper.cpp @@ -189,15 +189,6 @@ InputType*IOWrapper::GetInput(InputType* inputType) } } - ofstream* IOWrapper::GetOutputSearchGraphHypergraphWeightsStream() { - const StaticData &staticData = StaticData::Instance(); - stringstream fileName; - fileName << staticData.GetParam("output-search-graph-hypergraph")[1]; - std::ofstream *file = new std::ofstream; - file->open(fileName.str().c_str()); - return file; - } - /*** * print surface factor only for the given phrase */ diff --git a/moses-cmd/IOWrapper.h b/moses-cmd/IOWrapper.h index 267a3a0bc..8dbdeda9c 100644 --- a/moses-cmd/IOWrapper.h +++ b/moses-cmd/IOWrapper.h @@ -117,8 +117,6 @@ public: return *m_outputSearchGraphStream; } - std::ofstream *GetOutputSearchGraphHypergraphWeightsStream(); - std::ostream &GetDetailedTranslationReportingStream() { assert (m_detailedTranslationReportingStream); return *m_detailedTranslationReportingStream; diff --git a/moses-cmd/Jamfile b/moses-cmd/Jamfile index 42d54568f..bddc10911 100644 --- a/moses-cmd/Jamfile +++ b/moses-cmd/Jamfile @@ -1,4 +1,4 @@ -alias deps : IOWrapper.cpp mbr.cpp LatticeMBR.cpp TranslationAnalysis.cpp ..//z ..//boost_iostreams ../moses//moses ; +alias deps : IOWrapper.cpp mbr.cpp LatticeMBR.cpp TranslationAnalysis.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; exe moses : Main.cpp deps ; exe lmbrgrid : LatticeMBRGrid.cpp deps ; diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 90c924a08..8f6a6c069 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * Moses main, for single-threaded and multi-threaded. **/ +#include <boost/filesystem.hpp> #include <boost/iostreams/device/file.hpp> #include <boost/iostreams/filter/bzip2.hpp> #include <boost/iostreams/filter/gzip.hpp> @@ -171,30 +172,76 @@ public: // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder if (m_outputSearchGraphHypergraph) { - stringstream fileName; - fileName << staticData.GetParam("output-search-graph-hypergraph")[1] << "/" << m_lineNumber; - boost::iostreams::filtering_ostream *file = new boost::iostreams::filtering_ostream; - // file->open(fileName.str().c_str()); - string compression = staticData.GetParam("output-search-graph-hypergraph")[0]; - if ( compression == "gzip" || compression == "gz" ) { - file->push( boost::iostreams::gzip_compressor() ); - } else if ( compression == "bzip2" || compression == "bz2" ) { - file->push( boost::iostreams::bzip2_compressor() ); + + vector<string> hypergraphParameters = staticData.GetParam("output-search-graph-hypergraph"); + + bool appendSuffix; + if (hypergraphParameters.size() > 0 && hypergraphParameters[0] == "true") { + appendSuffix = true; + } else { + appendSuffix = false; + } + + string compression; + if (hypergraphParameters.size() > 1) { + compression = hypergraphParameters[1]; + } else { + compression = "txt"; + } + + string hypergraphDir; + if ( hypergraphParameters.size() > 2 ) { + hypergraphDir = hypergraphParameters[2]; + } else { + string nbestFile = staticData.GetNBestFilePath(); + if ( ! nbestFile.empty()) { + boost::filesystem::path nbestPath(nbestFile); + hypergraphDir = nbestPath.parent_path().filename(); + } else { + stringstream hypergraphDirName; + hypergraphDirName << boost::filesystem::current_path() << "/hypergraph"; + hypergraphDir = hypergraphDirName.str(); + } + } + + if ( ! boost::filesystem::exists(hypergraphDir) ) { + boost::filesystem::create_directory(hypergraphDir); } - file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) ); - // if (file->is_open() && file->good()) { - if (file->is_complete() && file->good()) { - // ostringstream out; - // fix(out,PRECISION); - fix(*file,PRECISION); - manager.OutputSearchGraphAsHypergraph(m_lineNumber, *file); - // *file << out.str(); - file -> flush(); + + if ( ! boost::filesystem::exists(hypergraphDir) ) { + TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because the directory does not exist" << std::endl); + } else if ( ! boost::filesystem::is_directory(hypergraphDir) ) { + TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because that path exists, but is not a directory" << std::endl); } else { - TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl); + + stringstream fileName; + fileName << hypergraphDir << "/" << m_lineNumber; + if ( appendSuffix ) { + fileName << "." << compression; + } + boost::iostreams::filtering_ostream *file = new boost::iostreams::filtering_ostream; + + if ( compression == "gz" ) { + file->push( boost::iostreams::gzip_compressor() ); + } else if ( compression == "bz2" ) { + file->push( boost::iostreams::bzip2_compressor() ); + } else if ( compression != "txt" ) { + TRACE_ERR("Unrecognized hypergraph compression format (" << compression << ") - using uncompressed plain txt" << std::endl); + compression = "txt"; + } + + file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) ); + + if (file->is_complete() && file->good()) { + fix(*file,PRECISION); + manager.OutputSearchGraphAsHypergraph(m_lineNumber, *file); + file -> flush(); + } else { + TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber << " because the output file " << fileName.str() << " is not open or not ready for writing" << std::endl); + } + file -> pop(); + delete file; } - file -> pop(); - delete file; } // apply decision rule and output best translation(s) @@ -548,8 +595,10 @@ int main(int argc, char** argv) TRACE_ERR(weights); TRACE_ERR("\n"); } - if (staticData.GetOutputSearchGraphHypergraph() && staticData.GetParam("output-search-graph-hypergraph").size() > 1) { - ofstream* weightsOut = ioWrapper->GetOutputSearchGraphHypergraphWeightsStream(); + if (staticData.GetOutputSearchGraphHypergraph() && staticData.GetParam("output-search-graph-hypergraph").size() > 3) { + ofstream* weightsOut = new std::ofstream; + string weightsFilename = staticData.GetParam("output-search-graph-hypergraph")[3]; + weightsOut->open(weightsFilename.c_str()); OutputFeatureWeightsForHypergraph(*weightsOut); weightsOut->flush(); weightsOut->close(); |