From 55db902f5f37c2b58eb77d1a98c57d93434b22d9 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 29 Sep 2014 17:58:20 +0100 Subject: move IOWrapper into moses/ --- moses-cmd/IOWrapper.cpp | 661 ------------------------------------------- moses-cmd/IOWrapper.h | 164 ----------- moses-cmd/Jamfile | 2 +- moses-cmd/LatticeMBRGrid.cpp | 2 +- moses-cmd/Main.cpp | 2 +- 5 files changed, 3 insertions(+), 828 deletions(-) delete mode 100644 moses-cmd/IOWrapper.cpp delete mode 100644 moses-cmd/IOWrapper.h (limited to 'moses-cmd') diff --git a/moses-cmd/IOWrapper.cpp b/moses-cmd/IOWrapper.cpp deleted file mode 100644 index 1483e0472..000000000 --- a/moses-cmd/IOWrapper.cpp +++ /dev/null @@ -1,661 +0,0 @@ -// $Id$ - -/*********************************************************************** -Moses - factored phrase-based language decoder -Copyright (c) 2006 University of Edinburgh -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the University of Edinburgh nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS -BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - ***********************************************************************/ - -// example file on how to use moses library - -#include -#include -#include - -#include "moses/TypeDef.h" -#include "moses/Util.h" -#include "moses/Hypothesis.h" -#include "moses/WordsRange.h" -#include "moses/TrellisPathList.h" -#include "moses/StaticData.h" -#include "moses/FeatureVector.h" -#include "moses/InputFileStream.h" -#include "moses/FF/StatefulFeatureFunction.h" -#include "moses/FF/StatelessFeatureFunction.h" -#include "util/exception.hh" - -#include "IOWrapper.h" - -using namespace std; -using namespace Moses; - -namespace MosesCmd -{ - -IOWrapper::IOWrapper( - const vector &inputFactorOrder - , const vector &outputFactorOrder - , const FactorMask &inputFactorUsed - , size_t nBestSize - , const string &nBestFilePath) - :m_inputFactorOrder(inputFactorOrder) - ,m_outputFactorOrder(outputFactorOrder) - ,m_inputFactorUsed(inputFactorUsed) - ,m_inputFile(NULL) - ,m_inputStream(&std::cin) - ,m_nBestStream(NULL) - ,m_outputWordGraphStream(NULL) - ,m_outputSearchGraphStream(NULL) - ,m_detailedTranslationReportingStream(NULL) - ,m_alignmentOutputStream(NULL) -{ - Initialization(inputFactorOrder, outputFactorOrder - , inputFactorUsed - , nBestSize, nBestFilePath); -} - -IOWrapper::IOWrapper(const std::vector &inputFactorOrder - , const std::vector &outputFactorOrder - , const FactorMask &inputFactorUsed - , size_t nBestSize - , const std::string &nBestFilePath - , const std::string &inputFilePath) - :m_inputFactorOrder(inputFactorOrder) - ,m_outputFactorOrder(outputFactorOrder) - ,m_inputFactorUsed(inputFactorUsed) - ,m_inputFilePath(inputFilePath) - ,m_inputFile(new InputFileStream(inputFilePath)) - ,m_nBestStream(NULL) - ,m_outputWordGraphStream(NULL) - ,m_outputSearchGraphStream(NULL) - ,m_detailedTranslationReportingStream(NULL) - ,m_alignmentOutputStream(NULL) -{ - Initialization(inputFactorOrder, outputFactorOrder - , inputFactorUsed - , nBestSize, nBestFilePath); - - m_inputStream = m_inputFile; -} - -IOWrapper::~IOWrapper() -{ - if (m_inputFile != NULL) - delete m_inputFile; - if (m_nBestStream != NULL && !m_surpressSingleBestOutput) { - // outputting n-best to file, rather than stdout. need to close file and delete obj - delete m_nBestStream; - } - if (m_outputWordGraphStream != NULL) { - delete m_outputWordGraphStream; - } - if (m_outputSearchGraphStream != NULL) { - delete m_outputSearchGraphStream; - } - delete m_detailedTranslationReportingStream; - delete m_alignmentOutputStream; -} - -void IOWrapper::Initialization(const std::vector &/*inputFactorOrder*/ - , const std::vector &/*outputFactorOrder*/ - , const FactorMask &/*inputFactorUsed*/ - , size_t nBestSize - , const std::string &nBestFilePath) -{ - const StaticData &staticData = StaticData::Instance(); - - // n-best - m_surpressSingleBestOutput = false; - - if (nBestSize > 0) { - if (nBestFilePath == "-" || nBestFilePath == "/dev/stdout") { - m_nBestStream = &std::cout; - m_surpressSingleBestOutput = true; - } else { - std::ofstream *file = new std::ofstream; - m_nBestStream = file; - file->open(nBestFilePath.c_str()); - } - } - - // wordgraph output - if (staticData.GetOutputWordGraph()) { - string fileName = staticData.GetParam("output-word-graph")[0]; - std::ofstream *file = new std::ofstream; - m_outputWordGraphStream = file; - file->open(fileName.c_str()); - } - - - // search graph output - if (staticData.GetOutputSearchGraph()) { - string fileName; - if (staticData.GetOutputSearchGraphExtended()) - fileName = staticData.GetParam("output-search-graph-extended")[0]; - else - fileName = staticData.GetParam("output-search-graph")[0]; - std::ofstream *file = new std::ofstream; - m_outputSearchGraphStream = file; - file->open(fileName.c_str()); - } - - // detailed translation reporting - if (staticData.IsDetailedTranslationReportingEnabled()) { - const std::string &path = staticData.GetDetailedTranslationReportingFilePath(); - m_detailedTranslationReportingStream = new std::ofstream(path.c_str()); - UTIL_THROW_IF(!m_detailedTranslationReportingStream->good(), - util::FileOpenException, - "File for output of detailed translation report could not be open"); - } - - // sentence alignment output - if (! staticData.GetAlignmentOutputFile().empty()) { - m_alignmentOutputStream = new ofstream(staticData.GetAlignmentOutputFile().c_str()); - UTIL_THROW_IF(!m_alignmentOutputStream->good(), - util::FileOpenException, - "File for output of word alignment could not be open"); - } - -} - -InputType* -IOWrapper:: -GetInput(InputType* inputType) -{ - if(inputType->Read(*m_inputStream, m_inputFactorOrder)) { - if (long x = inputType->GetTranslationId()) { - if (x>=m_translationId) m_translationId = x+1; - } else inputType->SetTranslationId(m_translationId++); - - return inputType; - } else { - delete inputType; - return NULL; - } -} - -std::map GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) -{ - const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath(); - const Phrase &inputPhrase = inputPath.GetPhrase(); - - std::map ret; - - for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) { - const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor); - if (factor) { - std::set targetPos = hypo.GetTranslationOption().GetTargetPhrase().GetAlignTerm().GetAlignmentsForSource(sourcePos); - UTIL_THROW_IF2(targetPos.size() != 1, - "Placeholder should be aligned to 1, and only 1, word"); - ret[*targetPos.begin()] = factor; - } - } - - return ret; -} - -/*** - * print surface factor only for the given phrase - */ -void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector &outputFactorOrder, - char reportSegmentation, bool reportAllFactors) -{ - UTIL_THROW_IF2(outputFactorOrder.size() == 0, - "Must specific at least 1 output factor"); - const TargetPhrase& phrase = edge.GetCurrTargetPhrase(); - bool markUnknown = StaticData::Instance().GetMarkUnknown(); - if (reportAllFactors == true) { - out << phrase; - } else { - FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor(); - - std::map placeholders; - if (placeholderFactor != NOT_FOUND) { - // creates map of target position -> factor for placeholders - placeholders = GetPlaceholders(edge, placeholderFactor); - } - - size_t size = phrase.GetSize(); - for (size_t pos = 0 ; pos < size ; pos++) { - const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); - - if (placeholders.size()) { - // do placeholders - std::map::const_iterator iter = placeholders.find(pos); - if (iter != placeholders.end()) { - factor = iter->second; - } - } - - UTIL_THROW_IF2(factor == NULL, - "No factor 0 at position " << pos); - - //preface surface form with UNK if marking unknowns - const Word &word = phrase.GetWord(pos); - if(markUnknown && word.IsOOV()) { - out << "UNK" << *factor; - } else { - out << *factor; - } - - for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) { - const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]); - UTIL_THROW_IF2(factor == NULL, - "No factor " << i << " at position " << pos); - - out << "|" << *factor; - } - out << " "; - } - } - - // trace ("report segmentation") option "-t" / "-tt" - if (reportSegmentation > 0 && phrase.GetSize() > 0) { - const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); - const int sourceStart = sourceRange.GetStartPos(); - const int sourceEnd = sourceRange.GetEndPos(); - out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt" - if (reportSegmentation == 2) { - out << ",wa="; - const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm(); - OutputAlignment(out, ai, 0, 0); - out << ",total="; - out << edge.GetScore() - edge.GetPrevHypo()->GetScore(); - out << ","; - ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown()); - scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown()); - OutputAllFeatureScores(scoreBreakdown, out); - } - out << "| "; - } -} - -void OutputBestSurface(std::ostream &out, const Hypothesis *hypo, const std::vector &outputFactorOrder, - char reportSegmentation, bool reportAllFactors) -{ - if (hypo != NULL) { - // recursively retrace this best path through the lattice, starting from the end of the hypothesis sentence - OutputBestSurface(out, hypo->GetPrevHypo(), outputFactorOrder, reportSegmentation, reportAllFactors); - OutputSurface(out, *hypo, outputFactorOrder, reportSegmentation, reportAllFactors); - } -} - -void OutputAlignment(ostream &out, const AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset) -{ - typedef std::vector< const std::pair* > AlignVec; - AlignVec alignments = ai.GetSortedAlignments(); - - AlignVec::const_iterator it; - for (it = alignments.begin(); it != alignments.end(); ++it) { - const std::pair &alignment = **it; - out << alignment.first + sourceOffset << "-" << alignment.second + targetOffset << " "; - } - -} - -void OutputAlignment(ostream &out, const vector &edges) -{ - size_t targetOffset = 0; - - for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - const TargetPhrase &tp = edge.GetCurrTargetPhrase(); - size_t sourceOffset = edge.GetCurrSourceWordsRange().GetStartPos(); - - OutputAlignment(out, tp.GetAlignTerm(), sourceOffset, targetOffset); - - targetOffset += tp.GetSize(); - } - out << std::endl; -} - -void OutputAlignment(std::ostream &out, const Moses::Hypothesis *hypo) -{ - std::vector edges; - const Hypothesis *currentHypo = hypo; - while (currentHypo) { - edges.push_back(currentHypo); - currentHypo = currentHypo->GetPrevHypo(); - } - - OutputAlignment(out, edges); - -} - -void OutputAlignment(OutputCollector* collector, size_t lineNo , const vector &edges) -{ - ostringstream out; - OutputAlignment(out, edges); - - collector->Write(lineNo,out.str()); -} - -void OutputAlignment(OutputCollector* collector, size_t lineNo , const Hypothesis *hypo) -{ - if (collector) { - std::vector edges; - const Hypothesis *currentHypo = hypo; - while (currentHypo) { - edges.push_back(currentHypo); - currentHypo = currentHypo->GetPrevHypo(); - } - - OutputAlignment(collector,lineNo, edges); - } -} - -void OutputAlignment(OutputCollector* collector, size_t lineNo , const TrellisPath &path) -{ - if (collector) { - OutputAlignment(collector,lineNo, path.GetEdges()); - } -} - -void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/, char reportSegmentation, bool reportAllFactors, std::ostream &out) -{ - const std::vector &edges = path.GetEdges(); - - for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - OutputSurface(out, edge, StaticData::Instance().GetOutputFactorOrder(), reportSegmentation, reportAllFactors); - } - out << endl; -} - -void IOWrapper::Backtrack(const Hypothesis *hypo) -{ - - if (hypo->GetPrevHypo() != NULL) { - VERBOSE(3,hypo->GetId() << " <= "); - Backtrack(hypo->GetPrevHypo()); - } -} - -void OutputBestHypo(const std::vector& mbrBestHypo, long /*translationId*/, char /*reportSegmentation*/, bool /*reportAllFactors*/, ostream& out) -{ - - for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) { - const Factor *factor = mbrBestHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]); - UTIL_THROW_IF2(factor == NULL, - "No factor 0 at position " << i); - if (i>0) out << " " << *factor; - else out << *factor; - } - out << endl; -} - - -void OutputInput(std::vector& map, const Hypothesis* hypo) -{ - if (hypo->GetPrevHypo()) { - OutputInput(map, hypo->GetPrevHypo()); - map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase(); - } -} - -void OutputInput(std::ostream& os, const Hypothesis* hypo) -{ - size_t len = hypo->GetInput().GetSize(); - std::vector inp_phrases(len, 0); - OutputInput(inp_phrases, hypo); - for (size_t i=0; iGetTotalScore() << " "; - } - - if (StaticData::Instance().IsPathRecoveryEnabled()) { - OutputInput(cout, hypo); - cout << "||| "; - } - OutputBestSurface(cout, hypo, m_outputFactorOrder, reportSegmentation, reportAllFactors); - cout << endl; - } - } else { - VERBOSE(1, "NO BEST TRANSLATION" << endl); - if (!m_surpressSingleBestOutput) { - cout << endl; - } - } -} - -void OutputNBest(std::ostream& out - , const Moses::TrellisPathList &nBestList - , const std::vector& outputFactorOrder - , long translationId - , char reportSegmentation) -{ - const StaticData &staticData = StaticData::Instance(); - bool reportAllFactors = staticData.GetReportAllFactorsNBest(); - bool includeSegmentation = staticData.NBestIncludesSegmentation(); - bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest(); - - TrellisPathList::const_iterator iter; - for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { - const TrellisPath &path = **iter; - const std::vector &edges = path.GetEdges(); - - // print the surface factor of the translation - out << translationId << " ||| "; - for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - OutputSurface(out, edge, outputFactorOrder, reportSegmentation, reportAllFactors); - } - out << " |||"; - - // print scores with feature names - OutputAllFeatureScores(path.GetScoreBreakdown(), out ); - - // total - out << " ||| " << path.GetTotalScore(); - - //phrase-to-phrase segmentation - if (includeSegmentation) { - out << " |||"; - for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); - WordsRange targetRange = path.GetTargetWordsRange(edge); - out << " " << sourceRange.GetStartPos(); - if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) { - out << "-" << sourceRange.GetEndPos(); - } - out<< "=" << targetRange.GetStartPos(); - if (targetRange.GetStartPos() < targetRange.GetEndPos()) { - out<< "-" << targetRange.GetEndPos(); - } - } - } - - if (includeWordAlignment) { - out << " ||| "; - for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); - WordsRange targetRange = path.GetTargetWordsRange(edge); - const int sourceOffset = sourceRange.GetStartPos(); - const int targetOffset = targetRange.GetStartPos(); - const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm(); - - OutputAlignment(out, ai, sourceOffset, targetOffset); - - } - } - - if (StaticData::Instance().IsPathRecoveryEnabled()) { - out << " ||| "; - OutputInput(out, edges[0]); - } - - out << endl; - } - - out << std::flush; -} - -void OutputAllFeatureScores(const Moses::ScoreComponentCollection &features - , std::ostream &out) -{ - std::string lastName = ""; - const vector& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions(); - for( size_t i=0; iGetScoreProducerDescription() != "BleuScoreFeature" - && ff->IsTuneable()) { - OutputFeatureScores( out, features, ff, lastName ); - } - } - const vector& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions(); - for( size_t i=0; iIsTuneable()) { - OutputFeatureScores( out, features, ff, lastName ); - } - } -} - -void OutputFeatureScores( std::ostream& out - , const ScoreComponentCollection &features - , const FeatureFunction *ff - , std::string &lastName ) -{ - const StaticData &staticData = StaticData::Instance(); - bool labeledOutput = staticData.IsLabeledNBestList(); - - // regular features (not sparse) - if (ff->GetNumScoreComponents() != 0) { - if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) { - lastName = ff->GetScoreProducerDescription(); - out << " " << lastName << "="; - } - vector scores = features.GetScoresForProducer( ff ); - for (size_t j = 0; jfirst << "= " << i->second; - } -} - -void OutputLatticeMBRNBest(std::ostream& out, const vector& solutions,long translationId) -{ - for (vector::const_iterator si = solutions.begin(); si != solutions.end(); ++si) { - out << translationId; - out << " |||"; - const vector mbrHypo = si->GetWords(); - for (size_t i = 0 ; i < mbrHypo.size() ; i++) { - const Factor *factor = mbrHypo[i].GetFactor(StaticData::Instance().GetOutputFactorOrder()[0]); - if (i>0) out << " " << *factor; - else out << *factor; - } - out << " |||"; - out << " map: " << si->GetMapScore(); - out << " w: " << mbrHypo.size(); - const vector& ngramScores = si->GetNgramScores(); - for (size_t i = 0; i < ngramScores.size(); ++i) { - out << " " << ngramScores[i]; - } - out << " ||| " << si->GetScore(); - - out << endl; - } -} - - -void IOWrapper::OutputLatticeMBRNBestList(const vector& solutions,long translationId) -{ - OutputLatticeMBRNBest(*m_nBestStream, solutions,translationId); -} - -bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source) -{ - if (source) delete source; - switch(inputType) { - case SentenceInput: - source = ioWrapper.GetInput(new Sentence); - break; - case ConfusionNetworkInput: - source = ioWrapper.GetInput(new ConfusionNet); - break; - case WordLatticeInput: - source = ioWrapper.GetInput(new WordLattice); - break; - default: - TRACE_ERR("Unknown input type: " << inputType << "\n"); - source = NULL; - } - return (source ? true : false); -} - - - -IOWrapper *GetIOWrapper(const StaticData &staticData) -{ - IOWrapper *ioWrapper; - const std::vector &inputFactorOrder = staticData.GetInputFactorOrder() - ,&outputFactorOrder = staticData.GetOutputFactorOrder(); - FactorMask inputFactorUsed(inputFactorOrder); - - // io - if (staticData.GetParam("input-file").size() == 1) { - VERBOSE(2,"IO from File" << endl); - string filePath = staticData.GetParam("input-file")[0]; - - ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed - , staticData.GetNBestSize() - , staticData.GetNBestFilePath() - , filePath); - } else { - VERBOSE(1,"IO from STDOUT/STDIN" << endl); - ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed - , staticData.GetNBestSize() - , staticData.GetNBestFilePath()); - } - ioWrapper->ResetTranslationId(); - - IFVERBOSE(1) - PrintUserTime("Created input-output object"); - - return ioWrapper; -} - -} - diff --git a/moses-cmd/IOWrapper.h b/moses-cmd/IOWrapper.h deleted file mode 100644 index ed2537986..000000000 --- a/moses-cmd/IOWrapper.h +++ /dev/null @@ -1,164 +0,0 @@ -// $Id$ - -/*********************************************************************** -Moses - factored phrase-based language decoder -Copyright (c) 2006 University of Edinburgh -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the University of Edinburgh nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS -BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -***********************************************************************/ - -// example file on how to use moses library - -#ifndef moses_cmd_IOWrapper_h -#define moses_cmd_IOWrapper_h - -#include -#include -#include -#include - -#include "moses/TypeDef.h" -#include "moses/Sentence.h" -#include "moses/FactorTypeSet.h" -#include "moses/FactorCollection.h" -#include "moses/Hypothesis.h" -#include "moses/OutputCollector.h" -#include "moses/TrellisPathList.h" -#include "moses/InputFileStream.h" -#include "moses/InputType.h" -#include "moses/WordLattice.h" -#include "moses/LatticeMBR.h" - -namespace Moses -{ -class ScoreComponentCollection; -class Hypothesis; -class Factor; -} - -namespace MosesCmd -{ - -/** Helper class that holds misc variables to write data out to command line. - */ -class IOWrapper -{ -protected: - long m_translationId; - - const std::vector &m_inputFactorOrder; - const std::vector &m_outputFactorOrder; - const Moses::FactorMask &m_inputFactorUsed; - std::string m_inputFilePath; - Moses::InputFileStream *m_inputFile; - std::istream *m_inputStream; - std::ostream *m_nBestStream - ,*m_outputWordGraphStream,*m_outputSearchGraphStream; - std::ostream *m_detailedTranslationReportingStream; - std::ofstream *m_alignmentOutputStream; - bool m_surpressSingleBestOutput; - - void Initialization(const std::vector &inputFactorOrder - , const std::vector &outputFactorOrder - , const Moses::FactorMask &inputFactorUsed - , size_t nBestSize - , const std::string &nBestFilePath); - - -public: - IOWrapper(const std::vector &inputFactorOrder - , const std::vector &outputFactorOrder - , const Moses::FactorMask &inputFactorUsed - , size_t nBestSize - , const std::string &nBestFilePath); - - IOWrapper(const std::vector &inputFactorOrder - , const std::vector &outputFactorOrder - , const Moses::FactorMask &inputFactorUsed - , size_t nBestSize - , const std::string &nBestFilePath - , const std::string &infilePath); - ~IOWrapper(); - - Moses::InputType* GetInput(Moses::InputType *inputType); - - void OutputBestHypo(const Moses::Hypothesis *hypo, long translationId, char reportSegmentation, bool reportAllFactors); - void OutputLatticeMBRNBestList(const std::vector& solutions,long translationId); - void Backtrack(const Moses::Hypothesis *hypo); - - void ResetTranslationId() { - m_translationId = 0; - } - - std::ofstream *GetAlignmentOutputStream() { - return m_alignmentOutputStream; - } - - std::ostream &GetOutputWordGraphStream() { - return *m_outputWordGraphStream; - } - std::ostream &GetOutputSearchGraphStream() { - return *m_outputSearchGraphStream; - } - - std::ostream &GetDetailedTranslationReportingStream() { - assert (m_detailedTranslationReportingStream); - return *m_detailedTranslationReportingStream; - } -}; - -IOWrapper *GetIOWrapper(const Moses::StaticData &staticData); -bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source); -void OutputLanguageModelOrder(std::ostream &out, const Moses::Hypothesis *hypo, Moses::Manager &manager); -void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector &outputFactorOrder, char reportSegmentation, bool reportAllFactors); -void OutputLatticeMBRNBest(std::ostream& out, const std::vector& solutions,long translationId); -void OutputBestHypo(const std::vector& mbrBestHypo, long /*translationId*/, - char reportSegmentation, bool reportAllFactors, std::ostream& out); -void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,char reportSegmentation, bool reportAllFactors, std::ostream &out); -void OutputInput(std::ostream& os, const Moses::Hypothesis* hypo); -void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::Hypothesis *hypo); -void OutputAlignment(Moses::OutputCollector* collector, size_t lineNo, const Moses::TrellisPath &path); -void OutputAlignment(std::ostream &out, const Moses::Hypothesis *hypo); -void OutputAlignment(std::ostream &out, const Moses::AlignmentInfo &ai, size_t sourceOffset, size_t targetOffset); - -void OutputNBest(std::ostream& out - , const Moses::TrellisPathList &nBestList - , const std::vector& outputFactorOrder - , long translationId - , char reportSegmentation); -void OutputAllFeatureScores(const Moses::ScoreComponentCollection &features - , std::ostream &out); -void OutputFeatureScores( std::ostream& out - , const Moses::ScoreComponentCollection &features - , const Moses::FeatureFunction *ff - , std::string &lastName ); - -// creates a map of TARGET positions which should be replaced by word using placeholder -std::map GetPlaceholders(const Moses::Hypothesis &hypo, Moses::FactorType placeholderFactor); - -} - -#endif diff --git a/moses-cmd/Jamfile b/moses-cmd/Jamfile index 8d54e0515..4869d729f 100644 --- a/moses-cmd/Jamfile +++ b/moses-cmd/Jamfile @@ -1,4 +1,4 @@ -alias deps : IOWrapper.cpp mbr.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; +alias deps : mbr.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; exe moses : Main.cpp deps ; exe lmbrgrid : LatticeMBRGrid.cpp deps ; diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index 904275339..3573d0697 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -46,7 +46,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include -#include "IOWrapper.h" +#include "moses/IOWrapper.h" #include "moses/LatticeMBR.h" #include "moses/Manager.h" #include "moses/StaticData.h" diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 713ded2e3..0af4603cb 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -35,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #endif #include "moses/TranslationAnalysis.h" -#include "IOWrapper.h" +#include "moses/IOWrapper.h" #include "mbr.h" #include "moses/Hypothesis.h" -- cgit v1.2.3 From c40faed0d6b22faa4acd22414d7a893ceca781e0 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 30 Sep 2014 12:25:36 +0100 Subject: separate out class TranslationTask into it's on file --- moses-cmd/Jamfile | 2 +- moses-cmd/Main.cpp | 327 +----------------------------------------- moses-cmd/TranslationTask.cpp | 296 ++++++++++++++++++++++++++++++++++++++ moses-cmd/TranslationTask.h | 65 +++++++++ 4 files changed, 365 insertions(+), 325 deletions(-) create mode 100644 moses-cmd/TranslationTask.cpp create mode 100644 moses-cmd/TranslationTask.h (limited to 'moses-cmd') diff --git a/moses-cmd/Jamfile b/moses-cmd/Jamfile index 4869d729f..993e0df02 100644 --- a/moses-cmd/Jamfile +++ b/moses-cmd/Jamfile @@ -1,4 +1,4 @@ -alias deps : mbr.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; +alias deps : mbr.cpp TranslationTask.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; exe moses : Main.cpp deps ; exe lmbrgrid : LatticeMBRGrid.cpp deps ; diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 0af4603cb..1aace5401 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -34,22 +34,20 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA //#include #endif -#include "moses/TranslationAnalysis.h" #include "moses/IOWrapper.h" -#include "mbr.h" - #include "moses/Hypothesis.h" #include "moses/HypergraphOutput.h" #include "moses/Manager.h" #include "moses/StaticData.h" +#include "moses/TypeDef.h" #include "moses/Util.h" #include "moses/Timer.h" -#include "moses/ThreadPool.h" -#include "moses/OutputCollector.h" #include "moses/TranslationModel/PhraseDictionary.h" #include "moses/FF/StatefulFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h" +#include "TranslationTask.h" + #ifdef HAVE_PROTOBUF #include "hypergraph.pb.h" #endif @@ -60,325 +58,6 @@ using namespace MosesCmd; namespace MosesCmd { -// output floats with five significant digits -static const size_t PRECISION = 3; - -/** Enforce rounding */ -void fix(std::ostream& stream, size_t size) -{ - stream.setf(std::ios::fixed); - stream.precision(size); -} - -/** Translates a sentence. - * - calls the search (Manager) - * - applies the decision rule - * - outputs best translation and additional reporting - **/ -class TranslationTask : public Task -{ - -public: - - TranslationTask(size_t lineNumber, - InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector, - OutputCollector* latticeSamplesCollector, - OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector, - OutputCollector* detailedTranslationCollector, - OutputCollector* alignmentInfoCollector, - OutputCollector* unknownsCollector, - bool outputSearchGraphSLF, - boost::shared_ptr > hypergraphOutput) : - m_source(source), m_lineNumber(lineNumber), - m_outputCollector(outputCollector), m_nbestCollector(nbestCollector), - m_latticeSamplesCollector(latticeSamplesCollector), - m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector), - m_detailedTranslationCollector(detailedTranslationCollector), - m_alignmentInfoCollector(alignmentInfoCollector), - m_unknownsCollector(unknownsCollector), - m_outputSearchGraphSLF(outputSearchGraphSLF), - m_hypergraphOutput(hypergraphOutput) {} - - /** Translate one sentence - * gets called by main function implemented at end of this source file */ - void Run() { - // shorthand for "global data" - const StaticData &staticData = StaticData::Instance(); - - // input sentence - Sentence sentence; - - // report wall time spent on translation - Timer translationTime; - translationTime.start(); - - // report thread number -#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) - TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl); -#endif - - - // execute the translation - // note: this executes the search, resulting in a search graph - // we still need to apply the decision rule (MAP, MBR, ...) - Timer initTime; - initTime.start(); - Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm()); - VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl); - manager.ProcessSentence(); - - // we are done with search, let's look what we got - Timer additionalReportingTime; - additionalReportingTime.start(); - - // output word graph - if (m_wordGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.GetWordGraph(m_lineNumber, out); - m_wordGraphCollector->Write(m_lineNumber, out.str()); - } - - // output search graph - if (m_searchGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraph(m_lineNumber, out); - m_searchGraphCollector->Write(m_lineNumber, out.str()); - -#ifdef HAVE_PROTOBUF - if (staticData.GetOutputSearchGraphPB()) { - ostringstream sfn; - sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends; - string fn = sfn.str(); - VERBOSE(2, "Writing search graph to " << fn << endl); - fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out); - manager.SerializeSearchGraphPB(m_lineNumber, output); - } -#endif - } - - // Output search graph in HTK standard lattice format (SLF) - if (m_outputSearchGraphSLF) { - stringstream fileName; - fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf"; - std::ofstream *file = new std::ofstream; - file->open(fileName.str().c_str()); - if (file->is_open() && file->good()) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraphAsSLF(m_lineNumber, out); - *file << out.str(); - file -> flush(); - } else { - TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl); - } - delete file; - } - - // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder - if (m_hypergraphOutput.get()) { - m_hypergraphOutput->Write(manager); - } - - additionalReportingTime.stop(); - - // apply decision rule and output best translation(s) - if (m_outputCollector) { - ostringstream out; - ostringstream debug; - fix(debug,PRECISION); - - // all derivations - send them to debug stream - if (staticData.PrintAllDerivations()) { - additionalReportingTime.start(); - manager.PrintAllDerivations(m_lineNumber, debug); - additionalReportingTime.stop(); - } - - Timer decisionRuleTime; - decisionRuleTime.start(); - - // MAP decoding: best hypothesis - const Hypothesis* bestHypo = NULL; - if (!staticData.UseMBR()) { - bestHypo = manager.GetBestHypothesis(); - if (bestHypo) { - if (StaticData::Instance().GetOutputHypoScore()) { - out << bestHypo->GetTotalScore() << ' '; - } - if (staticData.IsPathRecoveryEnabled()) { - OutputInput(out, bestHypo); - out << "||| "; - } - if (staticData.GetParam("print-id").size() && Scan(staticData.GetParam("print-id")[0]) ) { - out << m_source->GetTranslationId() << " "; - } - - if (staticData.GetReportSegmentation() == 2) { - manager.GetOutputLanguageModelOrder(out, bestHypo); - } - OutputBestSurface( - out, - bestHypo, - staticData.GetOutputFactorOrder(), - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors()); - if (staticData.PrintAlignmentInfo()) { - out << "||| "; - OutputAlignment(out, bestHypo); - } - - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo); - IFVERBOSE(1) { - debug << "BEST TRANSLATION: " << *bestHypo << endl; - } - } else { - VERBOSE(1, "NO BEST TRANSLATION" << endl); - } - - out << endl; - } - - // MBR decoding (n-best MBR, lattice MBR, consensus) - else { - // we first need the n-best translations - size_t nBestSize = staticData.GetMBRSize(); - if (nBestSize <= 0) { - cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl; - exit(1); - } - TrellisPathList nBestList; - manager.CalcNBest(nBestSize, nBestList,true); - VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl); - IFVERBOSE(2) { - PrintUserTime("calculated n-best list for (L)MBR decoding"); - } - - // lattice MBR - if (staticData.UseLatticeMBR()) { - if (m_nbestCollector) { - //lattice mbr nbest - vector solutions; - size_t n = min(nBestSize, staticData.GetNBestSize()); - getLatticeMBRNBest(manager,nBestList,solutions,n); - ostringstream out; - OutputLatticeMBRNBest(out, solutions,m_lineNumber); - m_nbestCollector->Write(m_lineNumber, out.str()); - } else { - //Lattice MBR decoding - vector mbrBestHypo = doLatticeMBR(manager,nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - IFVERBOSE(2) { - PrintUserTime("finished Lattice MBR decoding"); - } - } - } - - // consensus decoding - else if (staticData.UseConsensusDecoding()) { - const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList); - OutputBestHypo(conBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished Consensus decoding"); - } - } - - // n-best MBR decoding - else { - const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished MBR decoding"); - } - } - } - - // report best translation to output collector - m_outputCollector->Write(m_lineNumber,out.str(),debug.str()); - - decisionRuleTime.stop(); - VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); - } - - additionalReportingTime.start(); - - // output n-best list - if (m_nbestCollector && !staticData.UseLatticeMBR()) { - TrellisPathList nBestList; - ostringstream out; - manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest()); - OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_nbestCollector->Write(m_lineNumber, out.str()); - } - - //lattice samples - if (m_latticeSamplesCollector) { - TrellisPathList latticeSamples; - ostringstream out; - manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples); - OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_latticeSamplesCollector->Write(m_lineNumber, out.str()); - } - - // detailed translation reporting - if (m_detailedTranslationCollector) { - ostringstream out; - fix(out,PRECISION); - TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis()); - m_detailedTranslationCollector->Write(m_lineNumber,out.str()); - } - - //list of unknown words - if (m_unknownsCollector) { - const vector& unknowns = manager.getSntTranslationOptions()->GetUnknownSources(); - ostringstream out; - for (size_t i = 0; i < unknowns.size(); ++i) { - out << *(unknowns[i]); - } - out << endl; - m_unknownsCollector->Write(m_lineNumber, out.str()); - } - - // report additional statistics - manager.CalcDecoderStatistics(); - VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); - VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl); - IFVERBOSE(2) { - PrintUserTime("Sentence Decoding Time:"); - } - } - - ~TranslationTask() { - delete m_source; - } - -private: - InputType* m_source; - size_t m_lineNumber; - OutputCollector* m_outputCollector; - OutputCollector* m_nbestCollector; - OutputCollector* m_latticeSamplesCollector; - OutputCollector* m_wordGraphCollector; - OutputCollector* m_searchGraphCollector; - OutputCollector* m_detailedTranslationCollector; - OutputCollector* m_alignmentInfoCollector; - OutputCollector* m_unknownsCollector; - bool m_outputSearchGraphSLF; - boost::shared_ptr > m_hypergraphOutput; - std::ofstream *m_alignmentStream; - - -}; static void PrintFeatureWeight(const FeatureFunction* ff) { diff --git a/moses-cmd/TranslationTask.cpp b/moses-cmd/TranslationTask.cpp new file mode 100644 index 000000000..f5864f75e --- /dev/null +++ b/moses-cmd/TranslationTask.cpp @@ -0,0 +1,296 @@ +#include "TranslationTask.h" +#include "moses/StaticData.h" +#include "moses/Sentence.h" +#include "moses/IOWrapper.h" +#include "moses/TranslationAnalysis.h" +#include "moses/TypeDef.h" +#include "moses/Util.h" +#include "moses/InputType.h" +#include "moses/OutputCollector.h" +#include "mbr.h" + +using namespace std; +using namespace Moses; + +namespace MosesCmd +{ + +TranslationTask::TranslationTask(size_t lineNumber, + InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector, + OutputCollector* latticeSamplesCollector, + OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector, + OutputCollector* detailedTranslationCollector, + OutputCollector* alignmentInfoCollector, + OutputCollector* unknownsCollector, + bool outputSearchGraphSLF, + boost::shared_ptr > hypergraphOutput) : + m_source(source), m_lineNumber(lineNumber), + m_outputCollector(outputCollector), m_nbestCollector(nbestCollector), + m_latticeSamplesCollector(latticeSamplesCollector), + m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector), + m_detailedTranslationCollector(detailedTranslationCollector), + m_alignmentInfoCollector(alignmentInfoCollector), + m_unknownsCollector(unknownsCollector), + m_outputSearchGraphSLF(outputSearchGraphSLF), + m_hypergraphOutput(hypergraphOutput) +{} + +void TranslationTask::Run() { + // shorthand for "global data" + const StaticData &staticData = StaticData::Instance(); + + // input sentence + Sentence sentence; + + // report wall time spent on translation + Timer translationTime; + translationTime.start(); + + // report thread number +#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) + TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << endl); +#endif + + + // execute the translation + // note: this executes the search, resulting in a search graph + // we still need to apply the decision rule (MAP, MBR, ...) + Timer initTime; + initTime.start(); + Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm()); + VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl); + manager.ProcessSentence(); + + // we are done with search, let's look what we got + Timer additionalReportingTime; + additionalReportingTime.start(); + + // output word graph + if (m_wordGraphCollector) { + ostringstream out; + fix(out,PRECISION); + manager.GetWordGraph(m_lineNumber, out); + m_wordGraphCollector->Write(m_lineNumber, out.str()); + } + + // output search graph + if (m_searchGraphCollector) { + ostringstream out; + fix(out,PRECISION); + manager.OutputSearchGraph(m_lineNumber, out); + m_searchGraphCollector->Write(m_lineNumber, out.str()); + +#ifdef HAVE_PROTOBUF + if (staticData.GetOutputSearchGraphPB()) { + ostringstream sfn; + sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends; + string fn = sfn.str(); + VERBOSE(2, "Writing search graph to " << fn << endl); + fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out); + manager.SerializeSearchGraphPB(m_lineNumber, output); + } +#endif + } + + // Output search graph in HTK standard lattice format (SLF) + if (m_outputSearchGraphSLF) { + stringstream fileName; + fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf"; + ofstream *file = new ofstream; + file->open(fileName.str().c_str()); + if (file->is_open() && file->good()) { + ostringstream out; + fix(out,PRECISION); + manager.OutputSearchGraphAsSLF(m_lineNumber, out); + *file << out.str(); + file -> flush(); + } else { + TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << endl); + } + delete file; + } + + // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder + if (m_hypergraphOutput.get()) { + m_hypergraphOutput->Write(manager); + } + + additionalReportingTime.stop(); + + // apply decision rule and output best translation(s) + if (m_outputCollector) { + ostringstream out; + ostringstream debug; + fix(debug,PRECISION); + + // all derivations - send them to debug stream + if (staticData.PrintAllDerivations()) { + additionalReportingTime.start(); + manager.PrintAllDerivations(m_lineNumber, debug); + additionalReportingTime.stop(); + } + + Timer decisionRuleTime; + decisionRuleTime.start(); + + // MAP decoding: best hypothesis + const Hypothesis* bestHypo = NULL; + if (!staticData.UseMBR()) { + bestHypo = manager.GetBestHypothesis(); + if (bestHypo) { + if (StaticData::Instance().GetOutputHypoScore()) { + out << bestHypo->GetTotalScore() << ' '; + } + if (staticData.IsPathRecoveryEnabled()) { + OutputInput(out, bestHypo); + out << "||| "; + } + if (staticData.GetParam("print-id").size() && Scan(staticData.GetParam("print-id")[0]) ) { + out << m_source->GetTranslationId() << " "; + } + + if (staticData.GetReportSegmentation() == 2) { + manager.GetOutputLanguageModelOrder(out, bestHypo); + } + OutputBestSurface( + out, + bestHypo, + staticData.GetOutputFactorOrder(), + staticData.GetReportSegmentation(), + staticData.GetReportAllFactors()); + if (staticData.PrintAlignmentInfo()) { + out << "||| "; + OutputAlignment(out, bestHypo); + } + + OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo); + IFVERBOSE(1) { + debug << "BEST TRANSLATION: " << *bestHypo << endl; + } + } else { + VERBOSE(1, "NO BEST TRANSLATION" << endl); + } + + out << endl; + } + + // MBR decoding (n-best MBR, lattice MBR, consensus) + else { + // we first need the n-best translations + size_t nBestSize = staticData.GetMBRSize(); + if (nBestSize <= 0) { + cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl; + exit(1); + } + TrellisPathList nBestList; + manager.CalcNBest(nBestSize, nBestList,true); + VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl); + IFVERBOSE(2) { + PrintUserTime("calculated n-best list for (L)MBR decoding"); + } + + // lattice MBR + if (staticData.UseLatticeMBR()) { + if (m_nbestCollector) { + //lattice mbr nbest + vector solutions; + size_t n = min(nBestSize, staticData.GetNBestSize()); + getLatticeMBRNBest(manager,nBestList,solutions,n); + ostringstream out; + OutputLatticeMBRNBest(out, solutions,m_lineNumber); + m_nbestCollector->Write(m_lineNumber, out.str()); + } else { + //Lattice MBR decoding + vector mbrBestHypo = doLatticeMBR(manager,nBestList); + OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(), + staticData.GetReportAllFactors(),out); + IFVERBOSE(2) { + PrintUserTime("finished Lattice MBR decoding"); + } + } + } + + // consensus decoding + else if (staticData.UseConsensusDecoding()) { + const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList); + OutputBestHypo(conBestHypo, m_lineNumber, + staticData.GetReportSegmentation(), + staticData.GetReportAllFactors(),out); + OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo); + IFVERBOSE(2) { + PrintUserTime("finished Consensus decoding"); + } + } + + // n-best MBR decoding + else { + const TrellisPath &mbrBestHypo = doMBR(nBestList); + OutputBestHypo(mbrBestHypo, m_lineNumber, + staticData.GetReportSegmentation(), + staticData.GetReportAllFactors(),out); + OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo); + IFVERBOSE(2) { + PrintUserTime("finished MBR decoding"); + } + } + } + + // report best translation to output collector + m_outputCollector->Write(m_lineNumber,out.str(),debug.str()); + + decisionRuleTime.stop(); + VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); + } + + additionalReportingTime.start(); + + // output n-best list + if (m_nbestCollector && !staticData.UseLatticeMBR()) { + TrellisPathList nBestList; + ostringstream out; + manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest()); + OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber, + staticData.GetReportSegmentation()); + m_nbestCollector->Write(m_lineNumber, out.str()); + } + + //lattice samples + if (m_latticeSamplesCollector) { + TrellisPathList latticeSamples; + ostringstream out; + manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples); + OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber, + staticData.GetReportSegmentation()); + m_latticeSamplesCollector->Write(m_lineNumber, out.str()); + } + + // detailed translation reporting + if (m_detailedTranslationCollector) { + ostringstream out; + fix(out,PRECISION); + TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis()); + m_detailedTranslationCollector->Write(m_lineNumber,out.str()); + } + + //list of unknown words + if (m_unknownsCollector) { + const vector& unknowns = manager.getSntTranslationOptions()->GetUnknownSources(); + ostringstream out; + for (size_t i = 0; i < unknowns.size(); ++i) { + out << *(unknowns[i]); + } + out << endl; + m_unknownsCollector->Write(m_lineNumber, out.str()); + } + + // report additional statistics + manager.CalcDecoderStatistics(); + VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); + VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl); + IFVERBOSE(2) { + PrintUserTime("Sentence Decoding Time:"); + } +} + + +} diff --git a/moses-cmd/TranslationTask.h b/moses-cmd/TranslationTask.h new file mode 100644 index 000000000..84a6a8bf5 --- /dev/null +++ b/moses-cmd/TranslationTask.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include "moses/ThreadPool.h" +#include "moses/Manager.h" +#include "moses/HypergraphOutput.h" + +namespace Moses +{ + class InputType; + class OutputCollector; +} + +namespace MosesCmd +{ + +/** Translates a sentence. + * - calls the search (Manager) + * - applies the decision rule + * - outputs best translation and additional reporting + **/ +class TranslationTask : public Moses::Task +{ + +public: + + TranslationTask(size_t lineNumber, + Moses::InputType* source, Moses::OutputCollector* outputCollector, Moses::OutputCollector* nbestCollector, + Moses::OutputCollector* latticeSamplesCollector, + Moses::OutputCollector* wordGraphCollector, Moses::OutputCollector* searchGraphCollector, + Moses::OutputCollector* detailedTranslationCollector, + Moses::OutputCollector* alignmentInfoCollector, + Moses::OutputCollector* unknownsCollector, + bool outputSearchGraphSLF, + boost::shared_ptr > hypergraphOutput); + + ~TranslationTask() { + delete m_source; + } + + /** Translate one sentence + * gets called by main function implemented at end of this source file */ + void Run(); + + +private: + Moses::InputType* m_source; + size_t m_lineNumber; + Moses::OutputCollector* m_outputCollector; + Moses::OutputCollector* m_nbestCollector; + Moses::OutputCollector* m_latticeSamplesCollector; + Moses::OutputCollector* m_wordGraphCollector; + Moses::OutputCollector* m_searchGraphCollector; + Moses::OutputCollector* m_detailedTranslationCollector; + Moses::OutputCollector* m_alignmentInfoCollector; + Moses::OutputCollector* m_unknownsCollector; + bool m_outputSearchGraphSLF; + boost::shared_ptr > m_hypergraphOutput; + std::ofstream *m_alignmentStream; + + +}; + + +} //namespace -- cgit v1.2.3 From ebd2e724942cf2f96050e470e4d14a217ea3da73 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 30 Sep 2014 12:47:28 +0100 Subject: separate out class TranslationTask into it's on file --- moses-cmd/TranslationTask.cpp | 4 ++++ moses-cmd/TranslationTask.h | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/TranslationTask.cpp b/moses-cmd/TranslationTask.cpp index f5864f75e..65e7e08bc 100644 --- a/moses-cmd/TranslationTask.cpp +++ b/moses-cmd/TranslationTask.cpp @@ -35,6 +35,10 @@ TranslationTask::TranslationTask(size_t lineNumber, m_hypergraphOutput(hypergraphOutput) {} +TranslationTask::~TranslationTask() { + delete m_source; +} + void TranslationTask::Run() { // shorthand for "global data" const StaticData &staticData = StaticData::Instance(); diff --git a/moses-cmd/TranslationTask.h b/moses-cmd/TranslationTask.h index 84a6a8bf5..05b257a6a 100644 --- a/moses-cmd/TranslationTask.h +++ b/moses-cmd/TranslationTask.h @@ -34,9 +34,7 @@ public: bool outputSearchGraphSLF, boost::shared_ptr > hypergraphOutput); - ~TranslationTask() { - delete m_source; - } + ~TranslationTask(); /** Translate one sentence * gets called by main function implemented at end of this source file */ -- cgit v1.2.3 From c20af584e73873ce4d06b14a7ff793752fb07505 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 30 Sep 2014 12:59:31 +0100 Subject: move TranslationTask into moses/ --- moses-cmd/Jamfile | 2 +- moses-cmd/Main.cpp | 3 +- moses-cmd/TranslationTask.cpp | 300 ------------------------------------------ moses-cmd/TranslationTask.h | 63 --------- moses-cmd/mbr.cpp | 178 ------------------------- moses-cmd/mbr.h | 28 ---- 6 files changed, 2 insertions(+), 572 deletions(-) delete mode 100644 moses-cmd/TranslationTask.cpp delete mode 100644 moses-cmd/TranslationTask.h delete mode 100644 moses-cmd/mbr.cpp delete mode 100644 moses-cmd/mbr.h (limited to 'moses-cmd') diff --git a/moses-cmd/Jamfile b/moses-cmd/Jamfile index 993e0df02..7ee90850c 100644 --- a/moses-cmd/Jamfile +++ b/moses-cmd/Jamfile @@ -1,4 +1,4 @@ -alias deps : mbr.cpp TranslationTask.cpp ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; +alias deps : ..//z ..//boost_iostreams ..//boost_filesystem ../moses//moses ; exe moses : Main.cpp deps ; exe lmbrgrid : LatticeMBRGrid.cpp deps ; diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 1aace5401..8328ddc78 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -45,8 +45,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "moses/TranslationModel/PhraseDictionary.h" #include "moses/FF/StatefulFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h" - -#include "TranslationTask.h" +#include "moses/TranslationTask.h" #ifdef HAVE_PROTOBUF #include "hypergraph.pb.h" diff --git a/moses-cmd/TranslationTask.cpp b/moses-cmd/TranslationTask.cpp deleted file mode 100644 index 65e7e08bc..000000000 --- a/moses-cmd/TranslationTask.cpp +++ /dev/null @@ -1,300 +0,0 @@ -#include "TranslationTask.h" -#include "moses/StaticData.h" -#include "moses/Sentence.h" -#include "moses/IOWrapper.h" -#include "moses/TranslationAnalysis.h" -#include "moses/TypeDef.h" -#include "moses/Util.h" -#include "moses/InputType.h" -#include "moses/OutputCollector.h" -#include "mbr.h" - -using namespace std; -using namespace Moses; - -namespace MosesCmd -{ - -TranslationTask::TranslationTask(size_t lineNumber, - InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector, - OutputCollector* latticeSamplesCollector, - OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector, - OutputCollector* detailedTranslationCollector, - OutputCollector* alignmentInfoCollector, - OutputCollector* unknownsCollector, - bool outputSearchGraphSLF, - boost::shared_ptr > hypergraphOutput) : - m_source(source), m_lineNumber(lineNumber), - m_outputCollector(outputCollector), m_nbestCollector(nbestCollector), - m_latticeSamplesCollector(latticeSamplesCollector), - m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector), - m_detailedTranslationCollector(detailedTranslationCollector), - m_alignmentInfoCollector(alignmentInfoCollector), - m_unknownsCollector(unknownsCollector), - m_outputSearchGraphSLF(outputSearchGraphSLF), - m_hypergraphOutput(hypergraphOutput) -{} - -TranslationTask::~TranslationTask() { - delete m_source; -} - -void TranslationTask::Run() { - // shorthand for "global data" - const StaticData &staticData = StaticData::Instance(); - - // input sentence - Sentence sentence; - - // report wall time spent on translation - Timer translationTime; - translationTime.start(); - - // report thread number -#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) - TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << endl); -#endif - - - // execute the translation - // note: this executes the search, resulting in a search graph - // we still need to apply the decision rule (MAP, MBR, ...) - Timer initTime; - initTime.start(); - Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm()); - VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl); - manager.ProcessSentence(); - - // we are done with search, let's look what we got - Timer additionalReportingTime; - additionalReportingTime.start(); - - // output word graph - if (m_wordGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.GetWordGraph(m_lineNumber, out); - m_wordGraphCollector->Write(m_lineNumber, out.str()); - } - - // output search graph - if (m_searchGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraph(m_lineNumber, out); - m_searchGraphCollector->Write(m_lineNumber, out.str()); - -#ifdef HAVE_PROTOBUF - if (staticData.GetOutputSearchGraphPB()) { - ostringstream sfn; - sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends; - string fn = sfn.str(); - VERBOSE(2, "Writing search graph to " << fn << endl); - fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out); - manager.SerializeSearchGraphPB(m_lineNumber, output); - } -#endif - } - - // Output search graph in HTK standard lattice format (SLF) - if (m_outputSearchGraphSLF) { - stringstream fileName; - fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf"; - ofstream *file = new ofstream; - file->open(fileName.str().c_str()); - if (file->is_open() && file->good()) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraphAsSLF(m_lineNumber, out); - *file << out.str(); - file -> flush(); - } else { - TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << endl); - } - delete file; - } - - // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder - if (m_hypergraphOutput.get()) { - m_hypergraphOutput->Write(manager); - } - - additionalReportingTime.stop(); - - // apply decision rule and output best translation(s) - if (m_outputCollector) { - ostringstream out; - ostringstream debug; - fix(debug,PRECISION); - - // all derivations - send them to debug stream - if (staticData.PrintAllDerivations()) { - additionalReportingTime.start(); - manager.PrintAllDerivations(m_lineNumber, debug); - additionalReportingTime.stop(); - } - - Timer decisionRuleTime; - decisionRuleTime.start(); - - // MAP decoding: best hypothesis - const Hypothesis* bestHypo = NULL; - if (!staticData.UseMBR()) { - bestHypo = manager.GetBestHypothesis(); - if (bestHypo) { - if (StaticData::Instance().GetOutputHypoScore()) { - out << bestHypo->GetTotalScore() << ' '; - } - if (staticData.IsPathRecoveryEnabled()) { - OutputInput(out, bestHypo); - out << "||| "; - } - if (staticData.GetParam("print-id").size() && Scan(staticData.GetParam("print-id")[0]) ) { - out << m_source->GetTranslationId() << " "; - } - - if (staticData.GetReportSegmentation() == 2) { - manager.GetOutputLanguageModelOrder(out, bestHypo); - } - OutputBestSurface( - out, - bestHypo, - staticData.GetOutputFactorOrder(), - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors()); - if (staticData.PrintAlignmentInfo()) { - out << "||| "; - OutputAlignment(out, bestHypo); - } - - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo); - IFVERBOSE(1) { - debug << "BEST TRANSLATION: " << *bestHypo << endl; - } - } else { - VERBOSE(1, "NO BEST TRANSLATION" << endl); - } - - out << endl; - } - - // MBR decoding (n-best MBR, lattice MBR, consensus) - else { - // we first need the n-best translations - size_t nBestSize = staticData.GetMBRSize(); - if (nBestSize <= 0) { - cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl; - exit(1); - } - TrellisPathList nBestList; - manager.CalcNBest(nBestSize, nBestList,true); - VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl); - IFVERBOSE(2) { - PrintUserTime("calculated n-best list for (L)MBR decoding"); - } - - // lattice MBR - if (staticData.UseLatticeMBR()) { - if (m_nbestCollector) { - //lattice mbr nbest - vector solutions; - size_t n = min(nBestSize, staticData.GetNBestSize()); - getLatticeMBRNBest(manager,nBestList,solutions,n); - ostringstream out; - OutputLatticeMBRNBest(out, solutions,m_lineNumber); - m_nbestCollector->Write(m_lineNumber, out.str()); - } else { - //Lattice MBR decoding - vector mbrBestHypo = doLatticeMBR(manager,nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - IFVERBOSE(2) { - PrintUserTime("finished Lattice MBR decoding"); - } - } - } - - // consensus decoding - else if (staticData.UseConsensusDecoding()) { - const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList); - OutputBestHypo(conBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished Consensus decoding"); - } - } - - // n-best MBR decoding - else { - const TrellisPath &mbrBestHypo = doMBR(nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished MBR decoding"); - } - } - } - - // report best translation to output collector - m_outputCollector->Write(m_lineNumber,out.str(),debug.str()); - - decisionRuleTime.stop(); - VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); - } - - additionalReportingTime.start(); - - // output n-best list - if (m_nbestCollector && !staticData.UseLatticeMBR()) { - TrellisPathList nBestList; - ostringstream out; - manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest()); - OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_nbestCollector->Write(m_lineNumber, out.str()); - } - - //lattice samples - if (m_latticeSamplesCollector) { - TrellisPathList latticeSamples; - ostringstream out; - manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples); - OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_latticeSamplesCollector->Write(m_lineNumber, out.str()); - } - - // detailed translation reporting - if (m_detailedTranslationCollector) { - ostringstream out; - fix(out,PRECISION); - TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis()); - m_detailedTranslationCollector->Write(m_lineNumber,out.str()); - } - - //list of unknown words - if (m_unknownsCollector) { - const vector& unknowns = manager.getSntTranslationOptions()->GetUnknownSources(); - ostringstream out; - for (size_t i = 0; i < unknowns.size(); ++i) { - out << *(unknowns[i]); - } - out << endl; - m_unknownsCollector->Write(m_lineNumber, out.str()); - } - - // report additional statistics - manager.CalcDecoderStatistics(); - VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); - VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl); - IFVERBOSE(2) { - PrintUserTime("Sentence Decoding Time:"); - } -} - - -} diff --git a/moses-cmd/TranslationTask.h b/moses-cmd/TranslationTask.h deleted file mode 100644 index 05b257a6a..000000000 --- a/moses-cmd/TranslationTask.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include -#include "moses/ThreadPool.h" -#include "moses/Manager.h" -#include "moses/HypergraphOutput.h" - -namespace Moses -{ - class InputType; - class OutputCollector; -} - -namespace MosesCmd -{ - -/** Translates a sentence. - * - calls the search (Manager) - * - applies the decision rule - * - outputs best translation and additional reporting - **/ -class TranslationTask : public Moses::Task -{ - -public: - - TranslationTask(size_t lineNumber, - Moses::InputType* source, Moses::OutputCollector* outputCollector, Moses::OutputCollector* nbestCollector, - Moses::OutputCollector* latticeSamplesCollector, - Moses::OutputCollector* wordGraphCollector, Moses::OutputCollector* searchGraphCollector, - Moses::OutputCollector* detailedTranslationCollector, - Moses::OutputCollector* alignmentInfoCollector, - Moses::OutputCollector* unknownsCollector, - bool outputSearchGraphSLF, - boost::shared_ptr > hypergraphOutput); - - ~TranslationTask(); - - /** Translate one sentence - * gets called by main function implemented at end of this source file */ - void Run(); - - -private: - Moses::InputType* m_source; - size_t m_lineNumber; - Moses::OutputCollector* m_outputCollector; - Moses::OutputCollector* m_nbestCollector; - Moses::OutputCollector* m_latticeSamplesCollector; - Moses::OutputCollector* m_wordGraphCollector; - Moses::OutputCollector* m_searchGraphCollector; - Moses::OutputCollector* m_detailedTranslationCollector; - Moses::OutputCollector* m_alignmentInfoCollector; - Moses::OutputCollector* m_unknownsCollector; - bool m_outputSearchGraphSLF; - boost::shared_ptr > m_hypergraphOutput; - std::ofstream *m_alignmentStream; - - -}; - - -} //namespace diff --git a/moses-cmd/mbr.cpp b/moses-cmd/mbr.cpp deleted file mode 100644 index 6a8dfa823..000000000 --- a/moses-cmd/mbr.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "moses/TrellisPathList.h" -#include "moses/TrellisPath.h" -#include "moses/StaticData.h" -#include "moses/Util.h" -#include "mbr.h" - -using namespace std ; -using namespace Moses; - - -/* Input : - 1. a sorted n-best list, with duplicates filtered out in the following format - 0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432 - - 2. a weight vector - 3. bleu order ( default = 4) - 4. scaling factor to weigh the weight vector (default = 1.0) - - Output : - translations that minimise the Bayes Risk of the n-best list - - -*/ - -int BLEU_ORDER = 4; -int SMOOTH = 1; -float min_interval = 1e-4; -void extract_ngrams(const vector& sentence, map < vector < const Factor* >, int > & allngrams) -{ - vector< const Factor* > ngram; - for (int k = 0; k < BLEU_ORDER; k++) { - for(int i =0; i < max((int)sentence.size()-k,0); i++) { - for ( int j = i; j<= i+k; j++) { - ngram.push_back(sentence[j]); - } - ++allngrams[ngram]; - ngram.clear(); - } - } -} - -float calculate_score(const vector< vector > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats ) -{ - int comps_n = 2*BLEU_ORDER+1; - vector comps(comps_n); - float logbleu = 0.0, brevity; - - int hyp_length = sents[hyp].size(); - - for (int i =0; i ,int > & hyp_ngrams = ngram_stats[hyp] ; - map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ; - - for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin(); - it != hyp_ngrams.end(); it++) { - map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first); - if(ref_it != ref_ngrams.end()) { - comps[2* (it->first.size()-1)] += min(ref_it->second,it->second); - } - } - comps[comps_n-1] = sents[ref].size(); - - for (int i=0; i 0 ) - logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH); - else - logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]); - } - logbleu /= BLEU_ORDER; - brevity = 1.0-(float)comps[comps_n-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length - if (brevity < 0.0) - logbleu += brevity; - return exp(logbleu); -} - -const TrellisPath doMBR(const TrellisPathList& nBestList) -{ - float marginal = 0; - - vector joint_prob_vec; - vector< vector > translations; - float joint_prob; - vector< map < vector , int > > ngram_stats; - - TrellisPathList::const_iterator iter; - - // get max score to prevent underflow - float maxScore = -1e20; - for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { - const TrellisPath &path = **iter; - float score = StaticData::Instance().GetMBRScale() - * path.GetScoreBreakdown().GetWeightedScore(); - if (maxScore < score) maxScore = score; - } - - for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { - const TrellisPath &path = **iter; - joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().GetWeightedScore() - maxScore); - marginal += joint_prob; - joint_prob_vec.push_back(joint_prob); - - // get words in translation - vector translation; - GetOutputFactors(path, translation); - - // collect n-gram counts - map < vector < const Factor *>, int > counts; - extract_ngrams(translation,counts); - - ngram_stats.push_back(counts); - translations.push_back(translation); - } - - vector mbr_loss; - float bleu, weightedLoss; - float weightedLossCumul = 0; - float minMBRLoss = 1000000; - int minMBRLossIdx = -1; - - /* Main MBR computation done here */ - iter = nBestList.begin(); - for (unsigned int i = 0; i < nBestList.GetSize(); i++) { - weightedLossCumul = 0; - for (unsigned int j = 0; j < nBestList.GetSize(); j++) { - if ( i != j) { - bleu = calculate_score(translations, j, i,ngram_stats ); - weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal); - weightedLossCumul += weightedLoss; - if (weightedLossCumul > minMBRLoss) - break; - } - } - if (weightedLossCumul < minMBRLoss) { - minMBRLoss = weightedLossCumul; - minMBRLossIdx = i; - } - iter++; - } - /* Find sentence that minimises Bayes Risk under 1- BLEU loss */ - return nBestList.at(minMBRLossIdx); - //return translations[minMBRLossIdx]; -} - -void GetOutputFactors(const TrellisPath &path, vector &translation) -{ - const std::vector &edges = path.GetEdges(); - const std::vector& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); - assert (outputFactorOrder.size() == 1); - - // print the surface factor of the translation - for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) { - const Hypothesis &edge = *edges[currEdge]; - const Phrase &phrase = edge.GetCurrTargetPhrase(); - size_t size = phrase.GetSize(); - for (size_t pos = 0 ; pos < size ; pos++) { - - const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); - translation.push_back(factor); - } - } -} - diff --git a/moses-cmd/mbr.h b/moses-cmd/mbr.h deleted file mode 100644 index d08b11a98..000000000 --- a/moses-cmd/mbr.h +++ /dev/null @@ -1,28 +0,0 @@ -// $Id$ - -/*********************************************************************** -Moses - factored phrase-based language decoder -Copyright (C) 2006 University of Edinburgh - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -***********************************************************************/ - -#ifndef moses_cmd_mbr_h -#define moses_cmd_mbr_h - -const Moses::TrellisPath doMBR(const Moses::TrellisPathList& nBestList); -void GetOutputFactors(const Moses::TrellisPath &path, std::vector &translation); -float calculate_score(const std::vector< std::vector > & sents, int ref, int hyp, std::vector < std::map < std::vector < const Moses::Factor *>, int > > & ngram_stats ); -#endif -- cgit v1.2.3 From 33ed15ef199a11c0690ff6db2bc2c8f446af3395 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 30 Sep 2014 14:22:38 +0100 Subject: move misc common functions into moses/ --- moses-cmd/Main.cpp | 38 -------------------------------------- 1 file changed, 38 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 8328ddc78..5b3e61515 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -58,44 +58,6 @@ using namespace MosesCmd; namespace MosesCmd { -static void PrintFeatureWeight(const FeatureFunction* ff) -{ - cout << ff->GetScoreProducerDescription() << "="; - size_t numScoreComps = ff->GetNumScoreComponents(); - vector values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff); - for (size_t i = 0; i < numScoreComps; ++i) { - cout << " " << values[i]; - } - cout << endl; -} - -static void ShowWeights() -{ - //TODO: Find a way of ensuring this order is synced with the nbest - fix(cout,6); - const vector& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions(); - const vector& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions(); - - for (size_t i = 0; i < sff.size(); ++i) { - const StatefulFeatureFunction *ff = sff[i]; - if (ff->IsTuneable()) { - PrintFeatureWeight(ff); - } - else { - cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl; - } - } - for (size_t i = 0; i < slf.size(); ++i) { - const StatelessFeatureFunction *ff = slf[i]; - if (ff->IsTuneable()) { - PrintFeatureWeight(ff); - } - else { - cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl; - } - } -} - void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream) { outputSearchGraphStream.setf(std::ios::fixed); -- cgit v1.2.3 From 8d0f74c6e4f67fb4e7f45bc130562ca2b7f3528b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 1 Oct 2014 17:43:51 +0100 Subject: merge TranslationTask and IOWrapper. Move m_singleBestOutputCollector into IOWrapper --- moses-cmd/Main.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 5b3e61515..c9a75b41c 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -138,7 +138,6 @@ int main(int argc, char** argv) // initialize output streams // note: we can't just write to STDOUT or files // because multithreading may return sentences in shuffled order - auto_ptr outputCollector; // for translations auto_ptr nbestCollector; // for n-best lists auto_ptr latticeSamplesCollector; //for lattice samples auto_ptr nbestOut; @@ -176,9 +175,6 @@ int main(int argc, char** argv) latticeSamplesCollector.reset(new OutputCollector(latticeSamplesOut.get())); } } - if (output1best) { - outputCollector.reset(new OutputCollector()); - } // initialize stream for word graph (aka: output lattice) auto_ptr wordGraphCollector; @@ -233,7 +229,7 @@ int main(int argc, char** argv) // set up task of translating one sentence TranslationTask* task = - new TranslationTask(lineCount,source, outputCollector.get(), + new TranslationTask(lineCount,source, *ioWrapper, nbestCollector.get(), latticeSamplesCollector.get(), wordGraphCollector.get(), -- cgit v1.2.3 From 8611dc8cdf2a0a0395c09dc511f86a0267f46554 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 1 Oct 2014 18:21:57 +0100 Subject: merge TranslationTask and IOWrapper. Move m_nBestOutputCollector into IOWrapper --- moses-cmd/Main.cpp | 4 ---- 1 file changed, 4 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index c9a75b41c..249b69704 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -138,7 +138,6 @@ int main(int argc, char** argv) // initialize output streams // note: we can't just write to STDOUT or files // because multithreading may return sentences in shuffled order - auto_ptr nbestCollector; // for n-best lists auto_ptr latticeSamplesCollector; //for lattice samples auto_ptr nbestOut; auto_ptr latticeSamplesOut; @@ -148,7 +147,6 @@ int main(int argc, char** argv) if (nbestSize) { if (nbestFile == "-" || nbestFile == "/dev/stdout") { // nbest to stdout, no 1-best - nbestCollector.reset(new OutputCollector()); output1best = false; } else { // nbest to file, 1-best to stdout @@ -157,7 +155,6 @@ int main(int argc, char** argv) TRACE_ERR("ERROR: Failed to open " << nbestFile << " for nbest lists" << endl); exit(1); } - nbestCollector.reset(new OutputCollector(nbestOut.get())); } } size_t latticeSamplesSize = staticData.GetLatticeSamplesSize(); @@ -230,7 +227,6 @@ int main(int argc, char** argv) // set up task of translating one sentence TranslationTask* task = new TranslationTask(lineCount,source, *ioWrapper, - nbestCollector.get(), latticeSamplesCollector.get(), wordGraphCollector.get(), searchGraphCollector.get(), -- cgit v1.2.3 From 00365257faadb9573fb97f113a83615fc7f99e70 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 1 Oct 2014 19:24:58 +0100 Subject: merge TranslationTask and IOWrapper. Move m_unknownsStream into IOWrapper --- moses-cmd/Main.cpp | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 249b69704..f5382bb4d 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -198,18 +198,6 @@ int main(int argc, char** argv) alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream())); } - //initialise stream for unknown (oov) words - auto_ptr unknownsCollector; - auto_ptr unknownsStream; - if (!staticData.GetOutputUnknownsFile().empty()) { - unknownsStream.reset(new ofstream(staticData.GetOutputUnknownsFile().c_str())); - if (!unknownsStream->good()) { - TRACE_ERR("Unable to open " << staticData.GetOutputUnknownsFile() << " for unknowns"); - exit(1); - } - unknownsCollector.reset(new OutputCollector(unknownsStream.get())); - } - #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif @@ -232,7 +220,6 @@ int main(int argc, char** argv) searchGraphCollector.get(), detailedTranslationCollector.get(), alignmentInfoCollector.get(), - unknownsCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From 0843a2a901d0954e1886d1a95b1b36f34e5ccf7a Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 2 Oct 2014 11:22:52 +0100 Subject: merge TranslationTask and IOWrapper. Move m_alignmentInfoCollector into IOWrapper --- moses-cmd/Main.cpp | 7 ------- 1 file changed, 7 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index f5382bb4d..e00e1ee29 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -192,12 +192,6 @@ int main(int argc, char** argv) detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream()))); } - // initialize stram for word alignment between input and output - auto_ptr alignmentInfoCollector; - if (!staticData.GetAlignmentOutputFile().empty()) { - alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream())); - } - #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif @@ -219,7 +213,6 @@ int main(int argc, char** argv) wordGraphCollector.get(), searchGraphCollector.get(), detailedTranslationCollector.get(), - alignmentInfoCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From c45967c9b4f10ab36157c33e048152eb9c74984b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 2 Oct 2014 12:20:49 +0100 Subject: merge TranslationTask and IOWrapper. Move m_searchGraphOutputCollector into IOWrapper --- moses-cmd/Main.cpp | 8 -------- 1 file changed, 8 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index e00e1ee29..4cb507610 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -179,13 +179,6 @@ int main(int argc, char** argv) wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream()))); } - // initialize stream for search graph - // note: this is essentially the same as above, but in a different format - auto_ptr searchGraphCollector; - if (staticData.GetOutputSearchGraph()) { - searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream()))); - } - // initialize stram for details about the decoder run auto_ptr detailedTranslationCollector; if (staticData.IsDetailedTranslationReportingEnabled()) { @@ -211,7 +204,6 @@ int main(int argc, char** argv) new TranslationTask(lineCount,source, *ioWrapper, latticeSamplesCollector.get(), wordGraphCollector.get(), - searchGraphCollector.get(), detailedTranslationCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); -- cgit v1.2.3 From f8b762a6ec0030f79b4fee44c00692407dd9bfef Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 2 Oct 2014 14:18:12 +0100 Subject: merge TranslationTask and IOWrapper. Move m_detailedTranslationCollector into IOWrapper --- moses-cmd/Main.cpp | 7 ------- 1 file changed, 7 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 4cb507610..db94c1de5 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -179,12 +179,6 @@ int main(int argc, char** argv) wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream()))); } - // initialize stram for details about the decoder run - auto_ptr detailedTranslationCollector; - if (staticData.IsDetailedTranslationReportingEnabled()) { - detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream()))); - } - #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif @@ -204,7 +198,6 @@ int main(int argc, char** argv) new TranslationTask(lineCount,source, *ioWrapper, latticeSamplesCollector.get(), wordGraphCollector.get(), - detailedTranslationCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From b907fb7492f8cc22937ef65aa5ac5e4de8a18800 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 2 Oct 2014 14:57:04 +0100 Subject: merge TranslationTask and IOWrapper. Move m_wordGraphCollector into IOWrapper --- moses-cmd/Main.cpp | 7 ------- 1 file changed, 7 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index db94c1de5..f78c71acd 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -173,12 +173,6 @@ int main(int argc, char** argv) } } - // initialize stream for word graph (aka: output lattice) - auto_ptr wordGraphCollector; - if (staticData.GetOutputWordGraph()) { - wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream()))); - } - #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif @@ -197,7 +191,6 @@ int main(int argc, char** argv) TranslationTask* task = new TranslationTask(lineCount,source, *ioWrapper, latticeSamplesCollector.get(), - wordGraphCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From 26666986f33985b13e205ae9bac115385ce0b407 Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Thu, 2 Oct 2014 18:34:34 -0400 Subject: Update TranslationAnalysis location for simulate-pe --- moses-cmd/simulate-pe.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'moses-cmd') diff --git a/moses-cmd/simulate-pe.cc b/moses-cmd/simulate-pe.cc index 5384d9886..9678e26c7 100644 --- a/moses-cmd/simulate-pe.cc +++ b/moses-cmd/simulate-pe.cc @@ -44,7 +44,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA //#include #endif -#include "TranslationAnalysis.h" +#include "moses/TranslationAnalysis.h" #include "IOWrapper.h" #include "mbr.h" -- cgit v1.2.3 From 056913df5780b1094dffdf2e0961a89513a1c6fa Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 3 Oct 2014 12:30:18 +0100 Subject: delete nbestOut --- moses-cmd/Main.cpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index f78c71acd..3751a597e 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -139,24 +139,9 @@ int main(int argc, char** argv) // note: we can't just write to STDOUT or files // because multithreading may return sentences in shuffled order auto_ptr latticeSamplesCollector; //for lattice samples - auto_ptr nbestOut; auto_ptr latticeSamplesOut; - size_t nbestSize = staticData.GetNBestSize(); - string nbestFile = staticData.GetNBestFilePath(); bool output1best = true; - if (nbestSize) { - if (nbestFile == "-" || nbestFile == "/dev/stdout") { - // nbest to stdout, no 1-best - output1best = false; - } else { - // nbest to file, 1-best to stdout - nbestOut.reset(new ofstream(nbestFile.c_str())); - if (!nbestOut->good()) { - TRACE_ERR("ERROR: Failed to open " << nbestFile << " for nbest lists" << endl); - exit(1); - } - } - } + size_t latticeSamplesSize = staticData.GetLatticeSamplesSize(); string latticeSamplesFile = staticData.GetLatticeSamplesFilePath(); if (latticeSamplesSize) { -- cgit v1.2.3 From 62fe00b1a5775dbba1f961e41511dd99b4f946cb Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 3 Oct 2014 15:04:55 +0100 Subject: delete m_latticeSamplesCollector --- moses-cmd/Main.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 3751a597e..ca251882f 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -175,7 +175,6 @@ int main(int argc, char** argv) // set up task of translating one sentence TranslationTask* task = new TranslationTask(lineCount,source, *ioWrapper, - latticeSamplesCollector.get(), staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From 65bd64ca6ab7b3e5ad54e3fc981173fc9fd0744c Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 3 Oct 2014 15:30:06 +0100 Subject: move lattice sample collector from Main to IOWrapper --- moses-cmd/Main.cpp | 24 ------------------------ 1 file changed, 24 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index ca251882f..398a3a824 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -134,30 +134,6 @@ int main(int argc, char** argv) hypergraphOutput.reset(new HypergraphOutput(PRECISION)); } - - // initialize output streams - // note: we can't just write to STDOUT or files - // because multithreading may return sentences in shuffled order - auto_ptr latticeSamplesCollector; //for lattice samples - auto_ptr latticeSamplesOut; - bool output1best = true; - - size_t latticeSamplesSize = staticData.GetLatticeSamplesSize(); - string latticeSamplesFile = staticData.GetLatticeSamplesFilePath(); - if (latticeSamplesSize) { - if (latticeSamplesFile == "-" || latticeSamplesFile == "/dev/stdout") { - latticeSamplesCollector.reset(new OutputCollector()); - output1best = false; - } else { - latticeSamplesOut.reset(new ofstream(latticeSamplesFile.c_str())); - if (!latticeSamplesOut->good()) { - TRACE_ERR("ERROR: Failed to open " << latticeSamplesFile << " for lattice samples" << endl); - exit(1); - } - latticeSamplesCollector.reset(new OutputCollector(latticeSamplesOut.get())); - } - } - #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif -- cgit v1.2.3 From 2f290f1d6d84be68540c5b1d3ada7fa6c133c2ff Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 3 Oct 2014 16:08:15 +0100 Subject: standardize GetIOWrapper(). --- moses-cmd/Main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 398a3a824..994837d4a 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -116,7 +116,7 @@ int main(int argc, char** argv) srand(time(NULL)); // set up read/writing class - IOWrapper* ioWrapper = GetIOWrapper(staticData); + IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData); if (!ioWrapper) { cerr << "Error; Failed to create IO object" << endl; exit(1); -- cgit v1.2.3 From 433186ae255ec2e446e067530bee26a8ec11afcd Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 7 Oct 2014 19:16:30 +0100 Subject: delete lineNumber variable from Manager and TranslationTask. It should always be carried by the input sentence itself --- moses-cmd/LatticeMBRGrid.cpp | 7 ++++--- moses-cmd/Main.cpp | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index 3573d0697..677adb791 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -159,7 +159,7 @@ int main(int argc, char* argv[]) StaticData& staticData = const_cast(StaticData::Instance()); staticData.SetUseLatticeMBR(true); - IOWrapper* ioWrapper = GetIOWrapper(staticData); + IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData); if (!ioWrapper) { throw runtime_error("Failed to initialise IOWrapper"); @@ -180,8 +180,9 @@ int main(int argc, char* argv[]) while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { ++lineCount; - Sentence sentence; - Manager manager(lineCount, *source, staticData.GetSearchAlgorithm()); + source->SetTranslationId(lineCount); + + Manager manager(*source, staticData.GetSearchAlgorithm()); manager.ProcessSentence(); TrellisPathList nBestList; manager.CalcNBest(nBestSize, nBestList,true); diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 994837d4a..513c3a187 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -142,6 +142,7 @@ int main(int argc, char** argv) InputType* source = NULL; size_t lineCount = staticData.GetStartTranslationId(); while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + source->SetTranslationId(lineCount); IFVERBOSE(1) { ResetUserTime(); } @@ -150,7 +151,7 @@ int main(int argc, char** argv) // set up task of translating one sentence TranslationTask* task = - new TranslationTask(lineCount,source, *ioWrapper, + new TranslationTask(source, *ioWrapper, staticData.GetOutputSearchGraphSLF(), hypergraphOutput); // execute task -- cgit v1.2.3 From bf089b56ac975d4aedb116dc55c78009b6e19a48 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 9 Oct 2014 12:52:06 +0100 Subject: consistent namespace --- moses-cmd/LatticeMBRGrid.cpp | 3 +-- moses-cmd/Main.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index 677adb791..9a3ccdff0 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -55,12 +55,11 @@ POSSIBILITY OF SUCH DAMAGE. using namespace std; using namespace Moses; -using namespace MosesCmd; //keys enum gridkey {lmbr_p,lmbr_r,lmbr_prune,lmbr_scale}; -namespace MosesCmd +namespace Moses { class Grid diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 513c3a187..2e4a4ffdc 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -53,9 +53,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using namespace std; using namespace Moses; -using namespace MosesCmd; -namespace MosesCmd +namespace Moses { void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream) -- cgit v1.2.3 From 89f8c7d284a3e405c5d8259cb12ba67548ea648b Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 9 Oct 2014 17:28:57 +0100 Subject: get ready to merge IOWrapper --- moses-cmd/LatticeMBRGrid.cpp | 2 +- moses-cmd/Main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index 9a3ccdff0..bbf5f2a8b 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -177,7 +177,7 @@ int main(int argc, char* argv[]) const vector& prune_grid = grid.getGrid(lmbr_prune); const vector& scale_grid = grid.getGrid(lmbr_scale); - while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + while(ioWrapper->ReadInput(*ioWrapper,staticData.GetInputType(),source)) { ++lineCount; source->SetTranslationId(lineCount); diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 2e4a4ffdc..f7d9a44da 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -140,7 +140,7 @@ int main(int argc, char** argv) // main loop over set of input sentences InputType* source = NULL; size_t lineCount = staticData.GetStartTranslationId(); - while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + while(ioWrapper->ReadInput(*ioWrapper,staticData.GetInputType(),source)) { source->SetTranslationId(lineCount); IFVERBOSE(1) { ResetUserTime(); -- cgit v1.2.3 From 93d11e36262f75b8b9efb52672aa607332b3a27f Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Fri, 10 Oct 2014 15:09:56 +0100 Subject: merge moses_chart and moses --- moses-cmd/Main.cpp | 30 +++++++++++++++++++++++------- moses-cmd/Main.h | 5 ++--- 2 files changed, 25 insertions(+), 10 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index f7d9a44da..5758103f3 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -84,8 +84,8 @@ int main(int argc, char** argv) } // set number of significant decimals in output - fix(cout,PRECISION); - fix(cerr,PRECISION); + IOWrapper::FixPrecision(cout); + IOWrapper::FixPrecision(cerr); // load all the settings into the Parameter class // (stores them as strings, or array of strings) @@ -128,9 +128,17 @@ int main(int argc, char** argv) TRACE_ERR(weights); TRACE_ERR("\n"); } + boost::shared_ptr > hypergraphOutput; + boost::shared_ptr > hypergraphOutputChart; + if (staticData.GetOutputSearchGraphHypergraph()) { - hypergraphOutput.reset(new HypergraphOutput(PRECISION)); + if (staticData.IsChart()) { + hypergraphOutputChart.reset(new HypergraphOutput(PRECISION)); + } + else { + hypergraphOutput.reset(new HypergraphOutput(PRECISION)); + } } #ifdef WITH_THREADS @@ -149,10 +157,18 @@ int main(int argc, char** argv) FeatureFunction::CallChangeSource(source); // set up task of translating one sentence - TranslationTask* task = - new TranslationTask(source, *ioWrapper, - staticData.GetOutputSearchGraphSLF(), - hypergraphOutput); + TranslationTask* task; + if (staticData.IsChart()) { + // scfg + task = new TranslationTask(source, *ioWrapper, hypergraphOutputChart); + } + else { + // pb + task = new TranslationTask(source, *ioWrapper, + staticData.GetOutputSearchGraphSLF(), + hypergraphOutput); + } + // execute task #ifdef WITH_THREADS pool.Submit(task); diff --git a/moses-cmd/Main.h b/moses-cmd/Main.h index 362c1f245..49fee0219 100644 --- a/moses-cmd/Main.h +++ b/moses-cmd/Main.h @@ -1,3 +1,4 @@ +#pragma once // $Id$ /*********************************************************************** @@ -32,12 +33,10 @@ POSSIBILITY OF SUCH DAMAGE. // example file on how to use moses library -#ifndef moses_cmd_Main_h -#define moses_cmd_Main_h #include "moses/StaticData.h" class IOWrapper; int main(int argc, char* argv[]); -#endif + -- cgit v1.2.3 From 90e4eca0a340d05b31b45c32255a0842d1bf7492 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Mon, 13 Oct 2014 12:46:58 +0100 Subject: cleaning up IOWrapper. Make all functions member of the class --- moses-cmd/LatticeMBRGrid.cpp | 4 ++-- moses-cmd/Main.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/LatticeMBRGrid.cpp b/moses-cmd/LatticeMBRGrid.cpp index bbf5f2a8b..f00f40fd0 100644 --- a/moses-cmd/LatticeMBRGrid.cpp +++ b/moses-cmd/LatticeMBRGrid.cpp @@ -177,7 +177,7 @@ int main(int argc, char* argv[]) const vector& prune_grid = grid.getGrid(lmbr_prune); const vector& scale_grid = grid.getGrid(lmbr_scale); - while(ioWrapper->ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + while(ioWrapper->ReadInput(staticData.GetInputType(),source)) { ++lineCount; source->SetTranslationId(lineCount); @@ -200,7 +200,7 @@ int main(int argc, char* argv[]) staticData.SetMBRScale(scale); cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| "; vector mbrBestHypo = doLatticeMBR(manager,nBestList); - OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(), + ioWrapper->OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(), staticData.GetReportAllFactors(),cout); } } diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 5758103f3..7e8a4a0e3 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -148,7 +148,7 @@ int main(int argc, char** argv) // main loop over set of input sentences InputType* source = NULL; size_t lineCount = staticData.GetStartTranslationId(); - while(ioWrapper->ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + while(ioWrapper->ReadInput(staticData.GetInputType(),source)) { source->SetTranslationId(lineCount); IFVERBOSE(1) { ResetUserTime(); -- cgit v1.2.3 From e9f192e65f1f0ae290ff7636389e9fedfa1a4d0a Mon Sep 17 00:00:00 2001 From: Ulrich Germann Date: Sun, 16 Nov 2014 16:32:41 +0000 Subject: Fixed after changes to the Moses code structure. --- moses-cmd/simulate-pe.cc | 777 +++++++---------------------------------------- 1 file changed, 102 insertions(+), 675 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/simulate-pe.cc b/moses-cmd/simulate-pe.cc index 9678e26c7..daba1a715 100644 --- a/moses-cmd/simulate-pe.cc +++ b/moses-cmd/simulate-pe.cc @@ -1,5 +1,4 @@ -// Fork of Main.cpp, to simulate post-editing sessions. -// Written by Ulrich Germann. +// $Id: MainMT.cpp 3045 2010-04-05 13:07:29Z hieuhoang1972 $ /*********************************************************************** Moses - factored phrase-based language decoder @@ -20,44 +19,41 @@ License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ -#include -#include -#include -#include -#include -#include -#include - +/** + * Moses main, for single-threaded and multi-threaded. + **/ #include #include #include #include +#include #include "util/usage.hh" #include "util/exception.hh" #include "moses/Util.h" + +#ifdef PT_UG #include "moses/TranslationModel/UG/mmsapt.h" #include "moses/TranslationModel/UG/generic/program_options/ug_splice_arglist.h" +#endif #ifdef WIN32 // Include Visual Leak Detector //#include #endif -#include "moses/TranslationAnalysis.h" -#include "IOWrapper.h" -#include "mbr.h" - +#include "moses/IOWrapper.h" #include "moses/Hypothesis.h" +#include "moses/HypergraphOutput.h" #include "moses/Manager.h" #include "moses/StaticData.h" +#include "moses/TypeDef.h" #include "moses/Util.h" #include "moses/Timer.h" -#include "moses/ThreadPool.h" -#include "moses/OutputCollector.h" #include "moses/TranslationModel/PhraseDictionary.h" #include "moses/FF/StatefulFeatureFunction.h" #include "moses/FF/StatelessFeatureFunction.h" +#include "moses/TranslationTask.h" #ifdef HAVE_PROTOBUF #include "hypergraph.pb.h" @@ -65,522 +61,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using namespace std; using namespace Moses; -using namespace MosesCmd; - -namespace MosesCmd -{ -// output floats with five significant digits -static const size_t PRECISION = 3; - -/** Enforce rounding */ -void fix(std::ostream& stream, size_t size) -{ - stream.setf(std::ios::fixed); - stream.precision(size); -} -/** Translates a sentence. - * - calls the search (Manager) - * - applies the decision rule - * - outputs best translation and additional reporting - **/ -class TranslationTask : public Task +namespace Moses { -public: - - TranslationTask(size_t lineNumber, - InputType* source, - OutputCollector* outputCollector, - OutputCollector* nbestCollector, - OutputCollector* latticeSamplesCollector, - OutputCollector* wordGraphCollector, - OutputCollector* searchGraphCollector, - OutputCollector* detailedTranslationCollector, - OutputCollector* alignmentInfoCollector, - OutputCollector* unknownsCollector, - bool outputSearchGraphSLF, - bool outputSearchGraphHypergraph) - : m_source(source) - , m_lineNumber(lineNumber) - , m_outputCollector(outputCollector) - , m_nbestCollector(nbestCollector) - , m_latticeSamplesCollector(latticeSamplesCollector) - , m_wordGraphCollector(wordGraphCollector) - , m_searchGraphCollector(searchGraphCollector) - , m_detailedTranslationCollector(detailedTranslationCollector) - , m_alignmentInfoCollector(alignmentInfoCollector) - , m_unknownsCollector(unknownsCollector) - , m_outputSearchGraphSLF(outputSearchGraphSLF) - , m_outputSearchGraphHypergraph(outputSearchGraphHypergraph) - { } - - /** Translate one sentence - * gets called by main function implemented at end of this source file */ - void Run() { - // shorthand for "global data" - const StaticData &staticData = StaticData::Instance(); - - // input sentence - Sentence sentence; - - // report wall time spent on translation - Timer translationTime; - translationTime.start(); - - // report thread number -#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS) - TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl); -#endif - - - // execute the translation - // note: this executes the search, resulting in a search graph - // we still need to apply the decision rule (MAP, MBR, ...) - Timer initTime; - initTime.start(); - Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm()); - VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl); - manager.ProcessSentence(); - - // we are done with search, let's look what we got - Timer additionalReportingTime; - additionalReportingTime.start(); - - // output word graph - if (m_wordGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.GetWordGraph(m_lineNumber, out); - m_wordGraphCollector->Write(m_lineNumber, out.str()); - } - - // output search graph - if (m_searchGraphCollector) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraph(m_lineNumber, out); - m_searchGraphCollector->Write(m_lineNumber, out.str()); - -#ifdef HAVE_PROTOBUF - if (staticData.GetOutputSearchGraphPB()) { - ostringstream sfn; - sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends; - string fn = sfn.str(); - VERBOSE(2, "Writing search graph to " << fn << endl); - fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out); - manager.SerializeSearchGraphPB(m_lineNumber, output); - } -#endif - } - - // Output search graph in HTK standard lattice format (SLF) - if (m_outputSearchGraphSLF) { - stringstream fileName; - fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf"; - std::ofstream *file = new std::ofstream; - file->open(fileName.str().c_str()); - if (file->is_open() && file->good()) { - ostringstream out; - fix(out,PRECISION); - manager.OutputSearchGraphAsSLF(m_lineNumber, out); - *file << out.str(); - file -> flush(); - } else { - TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl); - } - delete file; - } - - // Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder - if (m_outputSearchGraphHypergraph) { - - vector hypergraphParameters = staticData.GetParam("output-search-graph-hypergraph"); - - bool appendSuffix; - if (hypergraphParameters.size() > 0 && hypergraphParameters[0] == "true") { - appendSuffix = true; - } else { - appendSuffix = false; - } - - string compression; - if (hypergraphParameters.size() > 1) { - compression = hypergraphParameters[1]; - } else { - compression = "txt"; - } - - string hypergraphDir; - if ( hypergraphParameters.size() > 2 ) { - hypergraphDir = hypergraphParameters[2]; - } else { - string nbestFile = staticData.GetNBestFilePath(); - if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) { - boost::filesystem::path nbestPath(nbestFile); - - // In the Boost filesystem API version 2, - // which was the default prior to Boost 1.46, - // the filename() method returned a string. - // - // In the Boost filesystem API version 3, - // which is the default starting with Boost 1.46, - // the filename() method returns a path object. - // - // To get a string from the path object, - // the native() method must be called. - // hypergraphDir = nbestPath.parent_path().filename() - //#if BOOST_VERSION >= 104600 - // .native() - //#endif - //; - - // Hopefully the following compiles under all versions of Boost. - // - // If this line gives you compile errors, - // contact Lane Schwartz on the Moses mailing list - hypergraphDir = nbestPath.parent_path().string(); - - } else { - stringstream hypergraphDirName; - hypergraphDirName << boost::filesystem::current_path().string() << "/hypergraph"; - hypergraphDir = hypergraphDirName.str(); - } - } - - if ( ! boost::filesystem::exists(hypergraphDir) ) { - boost::filesystem::create_directory(hypergraphDir); - } - - if ( ! boost::filesystem::exists(hypergraphDir) ) { - TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because the directory does not exist" << std::endl); - } else if ( ! boost::filesystem::is_directory(hypergraphDir) ) { - TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because that path exists, but is not a directory" << std::endl); - } else { - stringstream fileName; - fileName << hypergraphDir << "/" << m_lineNumber; - if ( appendSuffix ) { - fileName << "." << compression; - } - boost::iostreams::filtering_ostream *file - = new boost::iostreams::filtering_ostream; - - if ( compression == "gz" ) { - file->push( boost::iostreams::gzip_compressor() ); - } else if ( compression == "bz2" ) { - file->push( boost::iostreams::bzip2_compressor() ); - } else if ( compression != "txt" ) { - TRACE_ERR("Unrecognized hypergraph compression format (" - << compression - << ") - using uncompressed plain txt" << std::endl); - compression = "txt"; - } - - file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) ); - - if (file->is_complete() && file->good()) { - fix(*file,PRECISION); - manager.OutputSearchGraphAsHypergraph(*file); - file -> flush(); - } else { - TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber - << " because the output file " << fileName.str() - << " is not open or not ready for writing" - << std::endl); - } - file -> pop(); - delete file; - } - } - additionalReportingTime.stop(); - - // apply decision rule and output best translation(s) - if (m_outputCollector) { - ostringstream out; - ostringstream debug; - fix(debug,PRECISION); - - // all derivations - send them to debug stream - if (staticData.PrintAllDerivations()) { - additionalReportingTime.start(); - manager.PrintAllDerivations(m_lineNumber, debug); - additionalReportingTime.stop(); - } - - Timer decisionRuleTime; - decisionRuleTime.start(); - - // MAP decoding: best hypothesis - const Hypothesis* bestHypo = NULL; - if (!staticData.UseMBR()) { - bestHypo = manager.GetBestHypothesis(); - if (bestHypo) { - if (StaticData::Instance().GetOutputHypoScore()) { - out << bestHypo->GetTotalScore() << ' '; - } - if (staticData.IsPathRecoveryEnabled()) { - OutputInput(out, bestHypo); - out << "||| "; - } - if (staticData.GetParam("print-id").size() && Scan(staticData.GetParam("print-id")[0]) ) { - out << m_source->GetTranslationId() << " "; - } - - if (staticData.GetReportSegmentation() == 2) { - manager.GetOutputLanguageModelOrder(out, bestHypo); - } - OutputBestSurface( - out, - bestHypo, - staticData.GetOutputFactorOrder(), - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors()); - if (staticData.PrintAlignmentInfo()) { - out << "||| "; - OutputAlignment(out, bestHypo); - } - - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo); - IFVERBOSE(1) { - debug << "BEST TRANSLATION: " << *bestHypo << endl; - } - } else { - VERBOSE(1, "NO BEST TRANSLATION" << endl); - } - - out << endl; - } - - // MBR decoding (n-best MBR, lattice MBR, consensus) - else { - // we first need the n-best translations - size_t nBestSize = staticData.GetMBRSize(); - if (nBestSize <= 0) { - cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl; - exit(1); - } - TrellisPathList nBestList; - manager.CalcNBest(nBestSize, nBestList,true); - VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl); - IFVERBOSE(2) { - PrintUserTime("calculated n-best list for (L)MBR decoding"); - } - - // lattice MBR - if (staticData.UseLatticeMBR()) { - if (m_nbestCollector) { - //lattice mbr nbest - vector solutions; - size_t n = min(nBestSize, staticData.GetNBestSize()); - getLatticeMBRNBest(manager,nBestList,solutions,n); - ostringstream out; - OutputLatticeMBRNBest(out, solutions,m_lineNumber); - m_nbestCollector->Write(m_lineNumber, out.str()); - } else { - //Lattice MBR decoding - vector mbrBestHypo = doLatticeMBR(manager,nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - IFVERBOSE(2) { - PrintUserTime("finished Lattice MBR decoding"); - } - } - } - - // consensus decoding - else if (staticData.UseConsensusDecoding()) { - const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList); - OutputBestHypo(conBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished Consensus decoding"); - } - } - - // n-best MBR decoding - else { - const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList); - OutputBestHypo(mbrBestHypo, m_lineNumber, - staticData.GetReportSegmentation(), - staticData.GetReportAllFactors(),out); - OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo); - IFVERBOSE(2) { - PrintUserTime("finished MBR decoding"); - } - } - } - - // report best translation to output collector - m_outputCollector->Write(m_lineNumber,out.str(),debug.str()); - - decisionRuleTime.stop(); - VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl); - } - - additionalReportingTime.start(); - - // output n-best list - if (m_nbestCollector && !staticData.UseLatticeMBR()) { - TrellisPathList nBestList; - ostringstream out; - manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest()); - OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_nbestCollector->Write(m_lineNumber, out.str()); - } - - //lattice samples - if (m_latticeSamplesCollector) { - TrellisPathList latticeSamples; - ostringstream out; - manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples); - OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber, - staticData.GetReportSegmentation()); - m_latticeSamplesCollector->Write(m_lineNumber, out.str()); - } - - // detailed translation reporting - if (m_detailedTranslationCollector) { - ostringstream out; - fix(out,PRECISION); - TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis()); - m_detailedTranslationCollector->Write(m_lineNumber,out.str()); - } - - //list of unknown words - if (m_unknownsCollector) { - const vector& unknowns = manager.getSntTranslationOptions()->GetUnknownSources(); - ostringstream out; - for (size_t i = 0; i < unknowns.size(); ++i) { - out << *(unknowns[i]); - } - out << endl; - m_unknownsCollector->Write(m_lineNumber, out.str()); - } - - // report additional statistics - manager.CalcDecoderStatistics(); - VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl); - VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl); - IFVERBOSE(2) { - PrintUserTime("Sentence Decoding Time:"); - } - } - - ~TranslationTask() { - delete m_source; - } - -private: - InputType* m_source; - size_t m_lineNumber; - OutputCollector* m_outputCollector; - OutputCollector* m_nbestCollector; - OutputCollector* m_latticeSamplesCollector; - OutputCollector* m_wordGraphCollector; - OutputCollector* m_searchGraphCollector; - OutputCollector* m_detailedTranslationCollector; - OutputCollector* m_alignmentInfoCollector; - OutputCollector* m_unknownsCollector; - bool m_outputSearchGraphSLF; - bool m_outputSearchGraphHypergraph; - std::ofstream *m_alignmentStream; - - -}; - -static void PrintFeatureWeight(const FeatureFunction* ff) -{ - cout << ff->GetScoreProducerDescription() << "="; - size_t numScoreComps = ff->GetNumScoreComponents(); - vector values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff); - for (size_t i = 0; i < numScoreComps; ++i) { - cout << " " << values[i]; - } - cout << endl; -} - -static void ShowWeights() -{ - //TODO: Find a way of ensuring this order is synced with the nbest - fix(cout,6); - const vector& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions(); - const vector& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions(); - - for (size_t i = 0; i < sff.size(); ++i) { - const StatefulFeatureFunction *ff = sff[i]; - if (ff->IsTuneable()) { - PrintFeatureWeight(ff); - } - else { - cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl; - } - } - for (size_t i = 0; i < slf.size(); ++i) { - const StatelessFeatureFunction *ff = slf[i]; - if (ff->IsTuneable()) { - PrintFeatureWeight(ff); - } - else { - cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl; - } - } -} - -size_t OutputFeatureWeightsForHypergraph(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream) -{ - size_t numScoreComps = ff->GetNumScoreComponents(); - if (numScoreComps != 0) { - vector values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff); - if (numScoreComps > 1) { - for (size_t i = 0; i < numScoreComps; ++i) { - outputSearchGraphStream << ff->GetScoreProducerDescription() - << i - << "=" << values[i] << endl; - } - } else { - outputSearchGraphStream << ff->GetScoreProducerDescription() - << "=" << values[0] << endl; - } - return index+numScoreComps; - } else { - UTIL_THROW2("Sparse features are not yet supported when outputting hypergraph format"); - } -} - void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream) { outputSearchGraphStream.setf(std::ios::fixed); outputSearchGraphStream.precision(6); - - const vector& slf =StatelessFeatureFunction::GetStatelessFeatureFunctions(); - const vector& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions(); - size_t featureIndex = 1; - for (size_t i = 0; i < sff.size(); ++i) { - featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, sff[i], outputSearchGraphStream); - } - for (size_t i = 0; i < slf.size(); ++i) { - /* - if (slf[i]->GetScoreProducerWeightShortName() != "u" && - slf[i]->GetScoreProducerWeightShortName() != "tm" && - slf[i]->GetScoreProducerWeightShortName() != "I" && - slf[i]->GetScoreProducerWeightShortName() != "g") - */ - { - featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, slf[i], outputSearchGraphStream); - } - } - const vector& pds = PhraseDictionary::GetColl(); - for( size_t i=0; i& gds = GenerationDictionary::GetColl(); - for( size_t i=0; i > argfilter(4); argfilter[0] = std::make_pair(string("--spe-src"),1); argfilter[1] = std::make_pair(string("--spe-trg"),1); @@ -616,16 +132,16 @@ int main(int argc, char** argv) filter_arguments(argc, argv, mo_acnt, &mo_args, my_acnt, &my_args, argfilter); ifstream spe_src,spe_trg,spe_aln; - // instead of translating show coverage by phrase tables for (int i = 0; i < my_acnt; i += 2) { - if (!strcmp(my_args[i],"--spe-src")) - spe_src.open(my_args[i+1]); - else if (!strcmp(my_args[i],"--spe-trg")) - spe_trg.open(my_args[i+1]); - else if (!strcmp(my_args[i],"--spe-aln")) - spe_aln.open(my_args[i+1]); + if (!strcmp(my_args[i],"--spe-src")) + spe_src.open(my_args[i+1]); + else if (!strcmp(my_args[i],"--spe-trg")) + spe_trg.open(my_args[i+1]); + else if (!strcmp(my_args[i],"--spe-aln")) + spe_aln.open(my_args[i+1]); } +#endif // load all the settings into the Parameter class // (stores them as strings, or array of strings) @@ -634,7 +150,6 @@ int main(int argc, char** argv) exit(1); } - // initialize all "global" variables, which are stored in StaticData // note: this also loads models such as the language model, etc. if (!StaticData::LoadDataStatic(¶ms, argv[0])) { @@ -650,12 +165,11 @@ int main(int argc, char** argv) // shorthand for accessing information in StaticData const StaticData& staticData = StaticData::Instance(); - //initialise random numbers srand(time(NULL)); // set up read/writing class - IOWrapper* ioWrapper = GetIOWrapper(staticData); + IOWrapper* ioWrapper = IOWrapper::GetIOWrapper(staticData); if (!ioWrapper) { cerr << "Error; Failed to create IO object" << endl; exit(1); @@ -668,113 +182,17 @@ int main(int argc, char** argv) TRACE_ERR(weights); TRACE_ERR("\n"); } - if (staticData.GetOutputSearchGraphHypergraph()) { - ofstream* weightsOut = new std::ofstream; - stringstream weightsFilename; - if (staticData.GetParam("output-search-graph-hypergraph").size() > 3) { - weightsFilename << staticData.GetParam("output-search-graph-hypergraph")[3]; - } else { - string nbestFile = staticData.GetNBestFilePath(); - if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) { - boost::filesystem::path nbestPath(nbestFile); - weightsFilename << nbestPath.parent_path().filename() << "/weights"; - } else { - weightsFilename << boost::filesystem::current_path().string() << "/hypergraph/weights"; - } - } - boost::filesystem::path weightsFilePath(weightsFilename.str()); - if ( ! boost::filesystem::exists(weightsFilePath.parent_path()) ) { - boost::filesystem::create_directory(weightsFilePath.parent_path()); - } - TRACE_ERR("The weights file is " << weightsFilename.str() << "\n"); - weightsOut->open(weightsFilename.str().c_str()); - OutputFeatureWeightsForHypergraph(*weightsOut); - weightsOut->flush(); - weightsOut->close(); - delete weightsOut; - } - - - // initialize output streams - // note: we can't just write to STDOUT or files - // because multithreading may return sentences in shuffled order - auto_ptr outputCollector; // for translations - auto_ptr nbestCollector; // for n-best lists - auto_ptr latticeSamplesCollector; //for lattice samples - auto_ptr nbestOut; - auto_ptr latticeSamplesOut; - size_t nbestSize = staticData.GetNBestSize(); - string nbestFile = staticData.GetNBestFilePath(); - bool output1best = true; - if (nbestSize) { - if (nbestFile == "-" || nbestFile == "/dev/stdout") { - // nbest to stdout, no 1-best - nbestCollector.reset(new OutputCollector()); - output1best = false; - } else { - // nbest to file, 1-best to stdout - nbestOut.reset(new ofstream(nbestFile.c_str())); - if (!nbestOut->good()) { - TRACE_ERR("ERROR: Failed to open " << nbestFile << " for nbest lists" << endl); - exit(1); - } - nbestCollector.reset(new OutputCollector(nbestOut.get())); - } - } - size_t latticeSamplesSize = staticData.GetLatticeSamplesSize(); - string latticeSamplesFile = staticData.GetLatticeSamplesFilePath(); - if (latticeSamplesSize) { - if (latticeSamplesFile == "-" || latticeSamplesFile == "/dev/stdout") { - latticeSamplesCollector.reset(new OutputCollector()); - output1best = false; - } else { - latticeSamplesOut.reset(new ofstream(latticeSamplesFile.c_str())); - if (!latticeSamplesOut->good()) { - TRACE_ERR("ERROR: Failed to open " << latticeSamplesFile << " for lattice samples" << endl); - exit(1); - } - latticeSamplesCollector.reset(new OutputCollector(latticeSamplesOut.get())); - } - } - if (output1best) { - outputCollector.reset(new OutputCollector()); - } - - // initialize stream for word graph (aka: output lattice) - auto_ptr wordGraphCollector; - if (staticData.GetOutputWordGraph()) { - wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream()))); - } - - // initialize stream for search graph - // note: this is essentially the same as above, but in a different format - auto_ptr searchGraphCollector; - if (staticData.GetOutputSearchGraph()) { - searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream()))); - } - - // initialize stram for details about the decoder run - auto_ptr detailedTranslationCollector; - if (staticData.IsDetailedTranslationReportingEnabled()) { - detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream()))); - } - // initialize stram for word alignment between input and output - auto_ptr alignmentInfoCollector; - if (!staticData.GetAlignmentOutputFile().empty()) { - alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream())); - } + boost::shared_ptr > hypergraphOutput; + boost::shared_ptr > hypergraphOutputChart; - //initialise stream for unknown (oov) words - auto_ptr unknownsCollector; - auto_ptr unknownsStream; - if (!staticData.GetOutputUnknownsFile().empty()) { - unknownsStream.reset(new ofstream(staticData.GetOutputUnknownsFile().c_str())); - if (!unknownsStream->good()) { - TRACE_ERR("Unable to open " << staticData.GetOutputUnknownsFile() << " for unknowns"); - exit(1); - } - unknownsCollector.reset(new OutputCollector(unknownsStream.get())); + if (staticData.GetOutputSearchGraphHypergraph()) { + if (staticData.IsChart()) { + hypergraphOutputChart.reset(new HypergraphOutput(PRECISION)); + } + else { + hypergraphOutput.reset(new HypergraphOutput(PRECISION)); + } } #ifdef WITH_THREADS @@ -784,45 +202,54 @@ int main(int argc, char** argv) // main loop over set of input sentences InputType* source = NULL; size_t lineCount = staticData.GetStartTranslationId(); - while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) { + while(ioWrapper->ReadInput(staticData.GetInputType(),source)) { + source->SetTranslationId(lineCount); IFVERBOSE(1) { ResetUserTime(); } + + FeatureFunction::CallChangeSource(source); + // set up task of translating one sentence - TranslationTask* task = - new TranslationTask(lineCount,source, outputCollector.get(), - nbestCollector.get(), - latticeSamplesCollector.get(), - wordGraphCollector.get(), - searchGraphCollector.get(), - detailedTranslationCollector.get(), - alignmentInfoCollector.get(), - unknownsCollector.get(), - staticData.GetOutputSearchGraphSLF(), - staticData.GetOutputSearchGraphHypergraph()); + TranslationTask* task; + if (staticData.IsChart()) { + // scfg + task = new TranslationTask(source, *ioWrapper, hypergraphOutputChart); + } + else { + // pb + task = new TranslationTask(source, *ioWrapper, + staticData.GetOutputSearchGraphSLF(), + hypergraphOutput); + } + // execute task #ifdef WITH_THREADS +#ifdef PT_UG if (my_acnt) - { - task->Run(); - delete task; - string src,trg,aln; - UTIL_THROW_IF2(!getline(spe_src,src), "[" << HERE << "] " - << "missing update data for simulated post-editing."); - UTIL_THROW_IF2(!getline(spe_trg,trg), "[" << HERE << "] " - << "missing update data for simulated post-editing."); - UTIL_THROW_IF2(!getline(spe_aln,aln), "[" << HERE << "] " - << "missing update data for simulated post-editing."); - BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) - { - Mmsapt* sapt = dynamic_cast(pd); - if (sapt) sapt->add(src,trg,aln); - VERBOSE(1,"[" << HERE << " added src] " << src << endl); - VERBOSE(1,"[" << HERE << " added trg] " << trg << endl); - VERBOSE(1,"[" << HERE << " added aln] " << aln << endl); - } - } - else pool.Submit(task); + { + // simulated post-editing: always run single-threaded! + task->Run(); + delete task; + string src,trg,aln; + UTIL_THROW_IF2(!getline(spe_src,src), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + UTIL_THROW_IF2(!getline(spe_trg,trg), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + UTIL_THROW_IF2(!getline(spe_aln,aln), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) + { + Mmsapt* sapt = dynamic_cast(pd); + if (sapt) sapt->add(src,trg,aln); + VERBOSE(1,"[" << HERE << " added src] " << src << endl); + VERBOSE(1,"[" << HERE << " added trg] " << trg << endl); + VERBOSE(1,"[" << HERE << " added aln] " << aln << endl); + } + } + else +#endif + pool.Submit(task); #else task->Run(); delete task; -- cgit v1.2.3 From 8137aeae43d187e7a324e8c1a490f1d8365a8876 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 20 Nov 2014 11:21:50 +0000 Subject: Small changes to Parameters and StaticData --- moses-cmd/Main.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 7e8a4a0e3..cef8379e1 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -171,6 +171,32 @@ int main(int argc, char** argv) // execute task #ifdef WITH_THREADS +#ifdef PT_UG + /* + bool spe = params.isParamSpecified("spe-src"); + if (spe) { + // simulated post-editing: always run single-threaded! + task->Run(); + delete task; + string src,trg,aln; + UTIL_THROW_IF2(!getline(ioWrapper->spe_src,src), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + UTIL_THROW_IF2(!getline(ioWrapper->spe_trg,trg), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + UTIL_THROW_IF2(!getline(ioWrapper->spe_aln,aln), "[" << HERE << "] " + << "missing update data for simulated post-editing."); + BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) + { + Mmsapt* sapt = dynamic_cast(pd); + if (sapt) sapt->add(src,trg,aln); + VERBOSE(1,"[" << HERE << " added src] " << src << endl); + VERBOSE(1,"[" << HERE << " added trg] " << trg << endl); + VERBOSE(1,"[" << HERE << " added aln] " << aln << endl); + } + } + else + */ +#endif pool.Submit(task); #else task->Run(); -- cgit v1.2.3 From 9e5be78a7da79c59ca3be429f586f88b4f01d06e Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 20 Nov 2014 17:28:50 +0000 Subject: simulated post editing --- moses-cmd/Main.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index cef8379e1..e3008c88f 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -172,7 +172,6 @@ int main(int argc, char** argv) // execute task #ifdef WITH_THREADS #ifdef PT_UG - /* bool spe = params.isParamSpecified("spe-src"); if (spe) { // simulated post-editing: always run single-threaded! @@ -195,7 +194,6 @@ int main(int argc, char** argv) } } else - */ #endif pool.Submit(task); #else -- cgit v1.2.3 From 646874d77b5750f3e749c1805847c12d9df67d43 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 20 Nov 2014 17:38:02 +0000 Subject: simulated post editing --- moses-cmd/Main.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'moses-cmd') diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index e3008c88f..319aede20 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -51,6 +51,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "hypergraph.pb.h" #endif +#ifdef PT_UG +#include +#include "moses/TranslationModel/UG/mmsapt.h" +#include "moses/TranslationModel/UG/generic/program_options/ug_splice_arglist.h" +#endif + using namespace std; using namespace Moses; @@ -178,11 +184,11 @@ int main(int argc, char** argv) task->Run(); delete task; string src,trg,aln; - UTIL_THROW_IF2(!getline(ioWrapper->spe_src,src), "[" << HERE << "] " + UTIL_THROW_IF2(!getline(*ioWrapper->spe_src,src), "[" << HERE << "] " << "missing update data for simulated post-editing."); - UTIL_THROW_IF2(!getline(ioWrapper->spe_trg,trg), "[" << HERE << "] " + UTIL_THROW_IF2(!getline(*ioWrapper->spe_trg,trg), "[" << HERE << "] " << "missing update data for simulated post-editing."); - UTIL_THROW_IF2(!getline(ioWrapper->spe_aln,aln), "[" << HERE << "] " + UTIL_THROW_IF2(!getline(*ioWrapper->spe_aln,aln), "[" << HERE << "] " << "missing update data for simulated post-editing."); BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl()) { -- cgit v1.2.3