diff options
author | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2010-04-08 21:16:10 +0400 |
---|---|---|
committer | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2010-04-08 21:16:10 +0400 |
commit | c117ef7c17925d6efe8490236c1234beb3a20571 (patch) | |
tree | 32050178973cecceea4d494ded0e34375a0c8644 /moses-chart-cmd/src | |
parent | 5f1fd96111be9cdb2922dab44d2d051ac0504c17 (diff) |
Copy in changes from the chart_merge branch (doing it manually because the
server doesn't seem to support subversion's --reintegrate option).
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3078 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'moses-chart-cmd/src')
-rw-r--r-- | moses-chart-cmd/src/IOWrapper.cpp | 470 | ||||
-rw-r--r-- | moses-chart-cmd/src/IOWrapper.h | 96 | ||||
-rw-r--r-- | moses-chart-cmd/src/Main.cpp | 251 | ||||
-rw-r--r-- | moses-chart-cmd/src/Main.h | 42 | ||||
-rw-r--r-- | moses-chart-cmd/src/Makefile.am | 10 | ||||
-rw-r--r-- | moses-chart-cmd/src/TranslationAnalysis.cpp | 46 | ||||
-rw-r--r-- | moses-chart-cmd/src/TranslationAnalysis.h | 24 | ||||
-rw-r--r-- | moses-chart-cmd/src/mbr.cpp | 179 | ||||
-rw-r--r-- | moses-chart-cmd/src/mbr.h | 27 |
9 files changed, 1145 insertions, 0 deletions
diff --git a/moses-chart-cmd/src/IOWrapper.cpp b/moses-chart-cmd/src/IOWrapper.cpp new file mode 100644 index 000000000..c58239f66 --- /dev/null +++ b/moses-chart-cmd/src/IOWrapper.cpp @@ -0,0 +1,470 @@ +// $Id: IOWrapper.cpp 2675 2010-01-26 18:37:42Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2006 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +// example file on how to use moses library + +#include <iostream> +#include "TypeDef.h" +#include "Util.h" +#include "IOWrapper.h" +#include "WordsRange.h" +#include "StaticData.h" +#include "DummyScoreProducers.h" +#include "InputFileStream.h" +#include "PhraseDictionary.h" +#include "../../moses-chart/src/ChartTrellisPathList.h" +#include "../../moses-chart/src/ChartTrellisPath.h" + +using namespace std; +using namespace Moses; +using namespace MosesChart; + +IOWrapper::IOWrapper( + const vector<FactorType> &inputFactorOrder + , const vector<FactorType> &outputFactorOrder + , const FactorMask &inputFactorUsed + , size_t nBestSize + , const string &nBestFilePath) +:m_inputFactorOrder(inputFactorOrder) +,m_outputFactorOrder(outputFactorOrder) +,m_inputFactorUsed(inputFactorUsed) +,m_inputFile(NULL) +,m_inputStream(&std::cin) +,m_nBestStream(NULL) +,m_outputSearchGraphStream(NULL) +{ + const StaticData &staticData = StaticData::Instance(); + + m_surpressSingleBestOutput = false; + if (nBestSize > 0) + { + if (nBestFilePath == "-") + { + m_nBestStream = &std::cout; + m_surpressSingleBestOutput = true; + } + else + { + std::ofstream *nBestFile = new std::ofstream; + m_nBestStream = nBestFile; + nBestFile->open(nBestFilePath.c_str()); + } + } + + // search graph output + if (staticData.GetOutputSearchGraph()) + { + string fileName = staticData.GetParam("output-search-graph")[0]; + std::ofstream *file = new std::ofstream; + m_outputSearchGraphStream = file; + file->open(fileName.c_str()); + } + +} + +IOWrapper::IOWrapper(const std::vector<FactorType> &inputFactorOrder + , const std::vector<FactorType> &outputFactorOrder + , const FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath + , const std::string &inputFilePath) +:m_inputFactorOrder(inputFactorOrder) +,m_outputFactorOrder(outputFactorOrder) +,m_inputFactorUsed(inputFactorUsed) +,m_inputFilePath(inputFilePath) +,m_inputFile(new InputFileStream(inputFilePath)) +,m_nBestStream(NULL) +,m_outputSearchGraphStream(NULL) +{ + const StaticData &staticData = StaticData::Instance(); + + m_surpressSingleBestOutput = false; + m_inputStream = m_inputFile; + + if (nBestSize > 0) + { + if (nBestFilePath == "-") + { + m_nBestStream = &std::cout; + m_surpressSingleBestOutput = true; + } + else + { + std::ofstream *nBestFile = new std::ofstream; + m_nBestStream = nBestFile; + nBestFile->open(nBestFilePath.c_str()); + } + } + + // search graph output + if (staticData.GetOutputSearchGraph()) + { + string fileName = staticData.GetParam("output-search-graph")[0]; + std::ofstream *file = new std::ofstream; + m_outputSearchGraphStream = file; + file->open(fileName.c_str()); + } + +} + +IOWrapper::~IOWrapper() +{ + if (m_inputFile != NULL) + delete m_inputFile; + if (m_nBestStream != NULL && !m_surpressSingleBestOutput) + { // outputting n-best to file, rather than stdout. need to close file and delete obj + delete m_nBestStream; + } + + if (m_outputSearchGraphStream != NULL) + { + delete m_outputSearchGraphStream; + } +} + +InputType*IOWrapper::GetInput(InputType* inputType) +{ + if(inputType->Read(*m_inputStream, m_inputFactorOrder)) + { + if (long x = inputType->GetTranslationId()) { if (x>=m_translationId) m_translationId = x+1; } + else inputType->SetTranslationId(m_translationId++); + + return inputType; + } + else + { + delete inputType; + return NULL; + } +} + +/*** + * print surface factor only for the given phrase + */ +void OutputSurface(std::ostream &out, const Phrase &phrase, const std::vector<FactorType> &outputFactorOrder, bool reportAllFactors) +{ + assert(outputFactorOrder.size() > 0); + if (reportAllFactors == true) + { + out << phrase; + } + else + { + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) + { + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); + out << *factor; + + for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) + { + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]); + out << "|" << *factor; + } + out << " "; + } + } +} + +void OutputSurface(std::ostream &out, const MosesChart::Hypothesis *hypo, const std::vector<FactorType> &outputFactorOrder + ,bool reportSegmentation, bool reportAllFactors) +{ + if ( hypo != NULL) + { + //OutputSurface(out, hypo->GetCurrTargetPhrase(), outputFactorOrder, reportAllFactors); + + const vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos(); + + vector<const MosesChart::Hypothesis*>::const_iterator iter; + for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) + { + const MosesChart::Hypothesis *prevHypo = *iter; + + OutputSurface(out, prevHypo, outputFactorOrder, reportSegmentation, reportAllFactors); + } + } +} + +void IOWrapper::Backtrack(const MosesChart::Hypothesis *hypo) +{ + const vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos(); + + vector<const MosesChart::Hypothesis*>::const_iterator iter; + for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) + { + const MosesChart::Hypothesis *prevHypo = *iter; + + VERBOSE(3,prevHypo->GetId() << " <= "); + Backtrack(prevHypo); + } +} + +void IOWrapper::OutputBestHypo(const std::vector<const Factor*>& mbrBestHypo, long /*translationId*/, bool reportSegmentation, bool reportAllFactors) +{ + for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) + { + const Factor *factor = mbrBestHypo[i]; + cout << *factor << " "; + } +} +/* +void OutputInput(std::vector<const Phrase*>& map, const MosesChart::Hypothesis* hypo) +{ + if (hypo->GetPrevHypos()) + { + OutputInput(map, hypo->GetPrevHypos()); + map[hypo->GetCurrSourceWordsRange().GetStartPos()] = hypo->GetSourcePhrase(); + } +} + +void OutputInput(std::ostream& os, const MosesChart::Hypothesis* hypo) +{ + size_t len = StaticData::Instance().GetInput()->GetSize(); + std::vector<const Phrase*> inp_phrases(len, 0); + OutputInput(inp_phrases, hypo); + for (size_t i=0; i<len; ++i) + if (inp_phrases[i]) os << *inp_phrases[i]; +} +*/ + +void OutputTranslationOptions(const MosesChart::Hypothesis *hypo, long translationId) +{ // recursive + if (hypo != NULL) + { + //cerr << "Trans Opt " << translationId << " " << hypo->GetCurrSourceRange() << " = " << hypo->GetCurrTargetPhrase() << endl; + cerr << *hypo << endl; + } + + const std::vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos(); + std::vector<const MosesChart::Hypothesis*>::const_iterator iter; + for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) + { + const MosesChart::Hypothesis *prevHypo = *iter; + OutputTranslationOptions(prevHypo, translationId); + } +} + +void IOWrapper::OutputBestHypo(const MosesChart::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors) +{ + if (hypo != NULL) + { + //cerr << *hypo << endl; + + VERBOSE(3,"Best path: "); + Backtrack(hypo); + VERBOSE(3,"0" << std::endl); + + if (StaticData::Instance().GetOutputHypoScore()) + { + cout << hypo->GetTotalScore() << " " + << MosesChart::Hypothesis::GetHypoCount() << " "; + } + + if (StaticData::Instance().IsDetailedTranslationReportingEnabled()) + { + OutputTranslationOptions(hypo, translationId); + } + + if (!m_surpressSingleBestOutput) + { + if (StaticData::Instance().IsPathRecoveryEnabled()) { + //OutputInput(cout, hypo); + cout << "||| "; + } + Phrase outPhrase(Output); + hypo->CreateOutputPhrase(outPhrase); + + // delete 1st & last + assert(outPhrase.GetSize() >= 2); + outPhrase.RemoveWord(0); + outPhrase.RemoveWord(outPhrase.GetSize() - 1); + + const std::vector<FactorType> outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); + string output = outPhrase.GetStringRep(outputFactorOrder); + cout << output << endl; + } + } + else + { + VERBOSE(1, "NO BEST TRANSLATION" << endl); + + if (StaticData::Instance().GetOutputHypoScore()) + { + cout << "0 "; + } + + if (!m_surpressSingleBestOutput) + { + cout << endl; + } + } +} + +void IOWrapper::OutputNBestList(const MosesChart::TrellisPathList &nBestList, long translationId) +{ + bool labeledOutput = StaticData::Instance().IsLabeledNBestList(); + //bool includeAlignment = StaticData::Instance().NBestIncludesAlignment(); + + MosesChart::TrellisPathList::const_iterator iter; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) + { + const MosesChart::TrellisPath &path = **iter; + //cerr << path << endl << endl; + + Moses::Phrase outputPhrase = path.GetOutputPhrase(); + + // delete 1st & last + assert(outputPhrase.GetSize() >= 2); + outputPhrase.RemoveWord(0); + outputPhrase.RemoveWord(outputPhrase.GetSize() - 1); + + // print the surface factor of the translation + *m_nBestStream << translationId << " ||| "; + OutputSurface(*m_nBestStream, outputPhrase, m_outputFactorOrder, false); + *m_nBestStream << " ||| "; + + // print the scores in a hardwired order + // before each model type, the corresponding command-line-like name must be emitted + // MERT script relies on this + + // lm + const LMList& lml = StaticData::Instance().GetAllLM(); + if (lml.size() > 0) { + if (labeledOutput) + *m_nBestStream << "lm: "; + LMList::const_iterator lmi = lml.begin(); + for (; lmi != lml.end(); ++lmi) { + *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " "; + } + } + + // translation components + if (StaticData::Instance().GetInputType()==SentenceInput){ + // translation components for text input + vector<PhraseDictionaryFeature*> pds = StaticData::Instance().GetPhraseDictionaries(); + if (pds.size() > 0) { + if (labeledOutput) + *m_nBestStream << "tm: "; + vector<PhraseDictionaryFeature*>::iterator iter; + for (iter = pds.begin(); iter != pds.end(); ++iter) { + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); + for (size_t j = 0; j<scores.size(); ++j) + *m_nBestStream << scores[j] << " "; + } + } + } + else{ + // translation components for Confusion Network input + // first translation component has GetNumInputScores() scores from the input Confusion Network + // at the beginning of the vector + vector<PhraseDictionaryFeature*> pds = StaticData::Instance().GetPhraseDictionaries(); + if (pds.size() > 0) { + vector<PhraseDictionaryFeature*>::iterator iter; + + iter = pds.begin(); + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); + + size_t pd_numinputscore = (*iter)->GetNumInputScores(); + + if (pd_numinputscore){ + + if (labeledOutput) + *m_nBestStream << "I: "; + + for (size_t j = 0; j < pd_numinputscore; ++j) + *m_nBestStream << scores[j] << " "; + } + + + for (iter = pds.begin() ; iter != pds.end(); ++iter) { + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); + + size_t pd_numinputscore = (*iter)->GetNumInputScores(); + + if (iter == pds.begin() && labeledOutput) + *m_nBestStream << "tm: "; + for (size_t j = pd_numinputscore; j < scores.size() ; ++j) + *m_nBestStream << scores[j] << " "; + } + } + } + + + + // word penalty + if (labeledOutput) + *m_nBestStream << "w: "; + *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetWordPenaltyProducer()) << " "; + + // generation + vector<GenerationDictionary*> gds = StaticData::Instance().GetGenerationDictionaries(); + if (gds.size() > 0) { + if (labeledOutput) + *m_nBestStream << "g: "; + vector<GenerationDictionary*>::iterator iter; + for (iter = gds.begin(); iter != gds.end(); ++iter) { + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); + for (size_t j = 0; j<scores.size(); j++) { + *m_nBestStream << scores[j] << " "; + } + } + } + + // total + *m_nBestStream << "||| " << path.GetTotalScore(); + + /* + if (includeAlignment) { + *m_nBestStream << " |||"; + for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) + { + const MosesChart::Hypothesis &edge = *edges[currEdge]; + WordsRange sourceRange = edge.GetCurrSourceWordsRange(); + WordsRange targetRange = edge.GetCurrTargetWordsRange(); + *m_nBestStream << " " << sourceRange.GetStartPos(); + if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) { + *m_nBestStream << "-" << sourceRange.GetEndPos(); + } + *m_nBestStream << "=" << targetRange.GetStartPos(); + if (targetRange.GetStartPos() < targetRange.GetEndPos()) { + *m_nBestStream << "-" << targetRange.GetEndPos(); + } + } + } + */ + + *m_nBestStream << endl; + } + + *m_nBestStream<<std::flush; +} + diff --git a/moses-chart-cmd/src/IOWrapper.h b/moses-chart-cmd/src/IOWrapper.h new file mode 100644 index 000000000..493365389 --- /dev/null +++ b/moses-chart-cmd/src/IOWrapper.h @@ -0,0 +1,96 @@ +// $Id: IOWrapper.h 2615 2009-11-12 13:18:20Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2006 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +// example file on how to use moses library + +#pragma once + +#include <fstream> +#include <vector> +#include "TypeDef.h" +#include "Sentence.h" +#include "FactorTypeSet.h" +#include "InputFileStream.h" +#include "TrellisPathList.h" +#include "../../moses-chart/src/ChartHypothesis.h" + +namespace Moses +{ +class FactorCollection; +} + +namespace MosesChart +{ +class TrellisPathList; +} + +class IOWrapper +{ +protected: + long m_translationId; + + const std::vector<Moses::FactorType> &m_inputFactorOrder; + const std::vector<Moses::FactorType> &m_outputFactorOrder; + const Moses::FactorMask &m_inputFactorUsed; + std::ostream *m_nBestStream, *m_outputSearchGraphStream; + std::string m_inputFilePath; + std::istream *m_inputStream; + Moses::InputFileStream *m_inputFile; + bool m_surpressSingleBestOutput; + +public: + IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder + , const std::vector<Moses::FactorType> &outputFactorOrder + , const Moses::FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath); + + IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder + , const std::vector<Moses::FactorType> &outputFactorOrder + , const Moses::FactorMask &inputFactorUsed + , size_t nBestSize + , const std::string &nBestFilePath + , const std::string &inputFilePath); + ~IOWrapper(); + + Moses::InputType* GetInput(Moses::InputType *inputType); + void OutputBestHypo(const MosesChart::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors); + void OutputBestHypo(const std::vector<const Moses::Factor*>& mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors); + void OutputNBestList(const MosesChart::TrellisPathList &nBestList, long translationId); + void Backtrack(const MosesChart::Hypothesis *hypo); + + void ResetTranslationId() { m_translationId = 0; } + + std::ostream &GetOutputSearchGraphStream() + { return *m_outputSearchGraphStream; } + +}; diff --git a/moses-chart-cmd/src/Main.cpp b/moses-chart-cmd/src/Main.cpp new file mode 100644 index 000000000..18c62631d --- /dev/null +++ b/moses-chart-cmd/src/Main.cpp @@ -0,0 +1,251 @@ +// $Id: Main.cpp 2615 2009-11-12 13:18:20Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2006 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +// example file on how to use moses library + +#ifdef WIN32 +// Include Visual Leak Detector +#include <vld.h> +#endif + +#include <fstream> +#include "Main.h" +#include "TrellisPath.h" +#include "FactorCollection.h" +#include "Manager.h" +#include "Phrase.h" +#include "Util.h" +#include "TrellisPathList.h" +#include "Timer.h" +#include "IOWrapper.h" +#include "Sentence.h" +#include "ConfusionNet.h" +#include "WordLattice.h" +#include "TreeInput.h" +#include "TranslationAnalysis.h" +#include "mbr.h" +#include "../../moses-chart/src/ChartManager.h" +#include "../../moses-chart/src/ChartHypothesis.h" +#include "../../moses-chart/src/ChartTrellisPathList.h" + +#if HAVE_CONFIG_H +#include "config.h" +#else +// those not using autoconf have to build MySQL support for now +# define USE_MYSQL 1 +#endif + +using namespace std; +using namespace Moses; +using namespace MosesChart; + +bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source) +{ + delete source; + switch(inputType) + { + case SentenceInput: source = ioWrapper.GetInput(new Sentence(Input)); break; + case ConfusionNetworkInput: source = ioWrapper.GetInput(new ConfusionNet); break; + case WordLatticeInput: source = ioWrapper.GetInput(new WordLattice); break; + case TreeInputType: source = ioWrapper.GetInput(new TreeInput(Input));break; + default: TRACE_ERR("Unknown input type: " << inputType << "\n"); + } + return (source ? true : false); +} + + +int main(int argc, char* argv[]) +{ + IFVERBOSE(1) + { + TRACE_ERR("command: "); + for(int i=0;i<argc;++i) TRACE_ERR(argv[i]<<" "); + TRACE_ERR(endl); + } + + cout.setf(std::ios::fixed); + cout.precision(3); + cerr.setf(std::ios::fixed); + cerr.precision(3); + + // load data structures + Parameter *parameter = new Parameter(); + if (!parameter->LoadParam(argc, argv)) + { + parameter->Explain(); + delete parameter; + return EXIT_FAILURE; + } + + const StaticData &staticData = StaticData::Instance(); + if (!StaticData::LoadDataStatic(parameter)) + return EXIT_FAILURE; + + assert(staticData.GetSearchAlgorithm() == ChartDecoding); + + // set up read/writing class + IOWrapper *ioWrapper = GetIODevice(staticData); + + // check on weights + vector<float> weights = staticData.GetAllWeights(); + IFVERBOSE(2) { + TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager()); + TRACE_ERR("The global weight vector looks like this:"); + for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); } + TRACE_ERR("\n"); + } + // every score must have a weight! check that here: + if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) { + TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl); + return EXIT_FAILURE; + } + + if (ioWrapper == NULL) + return EXIT_FAILURE; + + // read each sentence & decode + InputType *source=0; + size_t lineCount = 0; + while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) + { + // note: source is only valid within this while loop! + IFVERBOSE(1) + ResetUserTime(); + + VERBOSE(2,"\nTRANSLATING(" << ++lineCount << "): " << *source); + //cerr << *source << endl; + + MosesChart::Manager manager(*source); + manager.ProcessSentence(); + + assert(!staticData.UseMBR()); + + if (!staticData.UseMBR()){ + ioWrapper->OutputBestHypo(manager.GetBestHypothesis(), source->GetTranslationId(), + staticData.GetReportSegmentation(), + staticData.GetReportAllFactors() + ); + IFVERBOSE(2) { PrintUserTime("Best Hypothesis Generation Time:"); } + + // n-best + size_t nBestSize = staticData.GetNBestSize(); + if (nBestSize > 0) + { + VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl); + MosesChart::TrellisPathList nBestList; + manager.CalcNBest(nBestSize, nBestList,staticData.GetDistinctNBest()); + ioWrapper->OutputNBestList(nBestList, source->GetTranslationId()); + + IFVERBOSE(2) { PrintUserTime("N-Best Hypotheses Generation Time:"); } + } + + if (staticData.GetOutputSearchGraph()) + manager.GetSearchGraph(source->GetTranslationId(), ioWrapper->GetOutputSearchGraphStream()); + + } + else { + /* + size_t nBestSize = staticData.GetNBestSize(); + cerr << "NBEST SIZE : " << nBestSize << endl; + assert(nBestSize > 0); + + if (nBestSize > 0) + { + VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl); + MosesChart::TrellisPathList nBestList; + manager.CalcNBest(nBestSize, nBestList,true); + std::vector<const Factor*> mbrBestHypo = doMBR(nBestList); + ioWrapper->OutputBestHypo(mbrBestHypo, source->GetTranslationId(), + staticData.GetReportSegmentation(), + staticData.GetReportAllFactors() + ); + IFVERBOSE(2) { PrintUserTime("N-Best Hypotheses Generation Time:"); } + } + */ + + } + /* + if (staticData.IsDetailedTranslationReportingEnabled()) { + TranslationAnalysis::PrintTranslationAnalysis(std::cerr, manager.GetBestHypothesis()); + } + */ + + IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); } + + manager.CalcDecoderStatistics(); + } // while(ReadInput + + delete ioWrapper; + + IFVERBOSE(1) + PrintUserTime("End."); + + #ifdef HACK_EXIT + //This avoids that detructors are called (it can take a long time) + exit(EXIT_SUCCESS); + #else + return EXIT_SUCCESS; + #endif +} + +IOWrapper *GetIODevice(const StaticData &staticData) +{ + IOWrapper *ioWrapper; + const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder() + ,&outputFactorOrder = staticData.GetOutputFactorOrder(); + FactorMask inputFactorUsed(inputFactorOrder); + + // io + if (staticData.GetParam("input-file").size() == 1) + { + VERBOSE(2,"IO from File" << endl); + string filePath = staticData.GetParam("input-file")[0]; + + ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed + , staticData.GetNBestSize() + , staticData.GetNBestFilePath() + , filePath); + } + else + { + VERBOSE(1,"IO from STDOUT/STDIN" << endl); + ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed + , staticData.GetNBestSize() + , staticData.GetNBestFilePath()); + } + ioWrapper->ResetTranslationId(); + + IFVERBOSE(1) + PrintUserTime("Created input-output object"); + + return ioWrapper; +} diff --git a/moses-chart-cmd/src/Main.h b/moses-chart-cmd/src/Main.h new file mode 100644 index 000000000..d203ea264 --- /dev/null +++ b/moses-chart-cmd/src/Main.h @@ -0,0 +1,42 @@ +// $Id: Main.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (c) 2006 University of Edinburgh +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the University of Edinburgh nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +***********************************************************************/ + +// example file on how to use moses library + +#pragma once + +#include "StaticData.h" + +class IOWrapper; + +int main(int argc, char* argv[]); +IOWrapper *GetIODevice(const Moses::StaticData &staticData); diff --git a/moses-chart-cmd/src/Makefile.am b/moses-chart-cmd/src/Makefile.am new file mode 100644 index 000000000..1cb86ca76 --- /dev/null +++ b/moses-chart-cmd/src/Makefile.am @@ -0,0 +1,10 @@ +bin_PROGRAMS = moses_chart +moses_chart_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp +AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src + +moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt +moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a + + + + diff --git a/moses-chart-cmd/src/TranslationAnalysis.cpp b/moses-chart-cmd/src/TranslationAnalysis.cpp new file mode 100644 index 000000000..e7a57d081 --- /dev/null +++ b/moses-chart-cmd/src/TranslationAnalysis.cpp @@ -0,0 +1,46 @@ +// $Id: TranslationAnalysis.cpp 2474 2009-08-06 17:32:28Z hieuhoang1972 $ + +#include <iostream> +#include <sstream> +#include <algorithm> +#include "StaticData.h" +#include "TranslationAnalysis.h" +#include "TranslationOption.h" +#include "DecodeStepTranslation.h" + +using namespace std; +using namespace Moses; + +namespace TranslationAnalysis { + +void PrintTranslationAnalysis(ostream &os, const Hypothesis* hypo) +{ + /* + os << endl << "TRANSLATION HYPOTHESIS DETAILS:" << endl; + queue<const Hypothesis*> translationPath; + while (hypo) + { + translationPath.push(hypo); + hypo = hypo->GetPrevHypo(); + } + + while (!translationPath.empty()) + { + hypo = translationPath.front(); + translationPath.pop(); + const TranslationOption *transOpt = hypo->GetTranslationOption(); + if (transOpt != NULL) + { + os << hypo->GetCurrSourceWordsRange() << " "; + for (size_t decodeStepId = 0; decodeStepId < DecodeStepTranslation::GetNumTransStep(); ++decodeStepId) + os << decodeStepId << "=" << transOpt->GetSubRangeCount(decodeStepId) << ","; + os << *transOpt << endl; + } + } + + os << "END TRANSLATION" << endl; + */ +} + +} + diff --git a/moses-chart-cmd/src/TranslationAnalysis.h b/moses-chart-cmd/src/TranslationAnalysis.h new file mode 100644 index 000000000..28901a076 --- /dev/null +++ b/moses-chart-cmd/src/TranslationAnalysis.h @@ -0,0 +1,24 @@ +// $Id: TranslationAnalysis.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $ + +/* + * also see moses/SentenceStats + */ + +#ifndef _TRANSLATION_ANALYSIS_H_ +#define _TRANSLATION_ANALYSIS_H_ + +#include <iostream> +#include "Hypothesis.h" + +namespace TranslationAnalysis +{ + +/*** + * print details about the translation represented in hypothesis to + * os. Included information: phrase alignment, words dropped, scores + */ + void PrintTranslationAnalysis(std::ostream &os, const Moses::Hypothesis* hypo); + +} + +#endif diff --git a/moses-chart-cmd/src/mbr.cpp b/moses-chart-cmd/src/mbr.cpp new file mode 100644 index 000000000..5b8902aae --- /dev/null +++ b/moses-chart-cmd/src/mbr.cpp @@ -0,0 +1,179 @@ +#include <iostream> +#include <fstream> +#include <sstream> +#include <iomanip> +#include <vector> +#include <map> +#include <stdlib.h> +#include <math.h> +#include <algorithm> +#include <stdio.h> +#include "TrellisPathList.h" +#include "TrellisPath.h" +#include "StaticData.h" +#include "Util.h" +#include "mbr.h" + +using namespace std ; +using namespace Moses; + + +/* Input : + 1. a sorted n-best list, with duplicates filtered out in the following format + 0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432 + + 2. a weight vector + 3. bleu order ( default = 4) + 4. scaling factor to weigh the weight vector (default = 1.0) + + Output : + translations that minimise the Bayes Risk of the n-best list + + +*/ + +int BLEU_ORDER = 4; +int SMOOTH = 1; +int DEBUG = 0; +float min_interval = 1e-4; +void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const Factor* >, int > & allngrams) +{ + vector< const Factor* > ngram; + for (int k = 0; k < BLEU_ORDER; k++) + { + for(int i =0; i < max((int)sentence.size()-k,0); i++) + { + for ( int j = i; j<= i+k; j++) + { + ngram.push_back(sentence[j]); + } + ++allngrams[ngram]; + ngram.clear(); + } + } +} + +float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats ) { + int comps_n = 2*BLEU_ORDER+1; + vector<int> comps(comps_n); + float logbleu = 0.0, brevity; + + int hyp_length = sents[hyp].size(); + + for (int i =0; i<BLEU_ORDER;i++) + { + comps[2*i] = 0; + comps[2*i+1] = max(hyp_length-i,0); + } + + map< vector < const Factor * > ,int > & hyp_ngrams = ngram_stats[hyp] ; + map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ; + + for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin(); + it != hyp_ngrams.end(); it++) + { + map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first); + if(ref_it != ref_ngrams.end()) + { + comps[2* (it->first.size()-1)] += min(ref_it->second,it->second); + } + } + comps[comps_n-1] = sents[ref].size(); + + if (DEBUG) + { + for ( int i = 0; i < comps_n; i++) + cerr << "Comp " << i << " : " << comps[i]; + } + + for (int i=0; i<BLEU_ORDER; i++) + { + if (comps[0] == 0) + return 0.0; + if ( i > 0 ) + logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH); + else + logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]); + } + logbleu /= BLEU_ORDER; + brevity = 1.0-(float)comps[comps_n-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length + if (brevity < 0.0) + logbleu += brevity; + return exp(logbleu); +} + +vector<const Factor*> doMBR(const TrellisPathList& nBestList){ +// cerr << "Sentence " << sent << " has " << sents.size() << " candidate translations" << endl; + float marginal = 0; + + vector<float> joint_prob_vec; + vector< vector<const Factor*> > translations; + float joint_prob; + vector< map < vector <const Factor *>, int > > ngram_stats; + + TrellisPathList::const_iterator iter; + //TrellisPath* hyp = NULL; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) + { + const TrellisPath &path = **iter; + joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights())); + marginal += joint_prob; + joint_prob_vec.push_back(joint_prob); + //Cache ngram counts + map < vector < const Factor *>, int > counts; + vector<const Factor*> translation; + GetOutputFactors(path, translation); + + //TO DO + extract_ngrams(translation,counts); + ngram_stats.push_back(counts); + translations.push_back(translation); + } + + vector<float> mbr_loss; + float bleu, weightedLoss; + float weightedLossCumul = 0; + float minMBRLoss = 1000000; + int minMBRLossIdx = -1; + + /* Main MBR computation done here */ + for (int i = 0; i < nBestList.GetSize(); i++){ + weightedLossCumul = 0; + for (int j = 0; j < nBestList.GetSize(); j++){ + if ( i != j) { + bleu = calculate_score(translations, j, i,ngram_stats ); + weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal); + weightedLossCumul += weightedLoss; + if (weightedLossCumul > minMBRLoss) + break; + } + } + if (weightedLossCumul < minMBRLoss){ + minMBRLoss = weightedLossCumul; + minMBRLossIdx = i; + } + } + /* Find sentence that minimises Bayes Risk under 1- BLEU loss */ + return translations[minMBRLossIdx]; +} + +void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation){ + const std::vector<const Hypothesis *> &edges = path.GetEdges(); + const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); + assert (outputFactorOrder.size() == 1); + + // print the surface factor of the translation + for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) + { + const Hypothesis &edge = *edges[currEdge]; + const Phrase &phrase = edge.GetCurrTargetPhrase(); + size_t size = phrase.GetSize(); + for (size_t pos = 0 ; pos < size ; pos++) + { + + const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]); + translation.push_back(factor); + } + } +} + diff --git a/moses-chart-cmd/src/mbr.h b/moses-chart-cmd/src/mbr.h new file mode 100644 index 000000000..7d63ac0f5 --- /dev/null +++ b/moses-chart-cmd/src/mbr.h @@ -0,0 +1,27 @@ +// $Id: mbr.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $ + +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#pragma once + +std::vector<const Moses::Factor*> doMBR(const Moses::TrellisPathList& nBestList); +void GetOutputFactors(const Moses::TrellisPath &path, std::vector <const Moses::Factor*> &translation); +float calculate_score(const std::vector< std::vector<const Moses::Factor*> > & sents, int ref, int hyp, std::vector < std::map < std::vector < const Moses::Factor *>, int > > & ngram_stats ); + |