Copy in changes from the chart_merge branch (doing it manually because the

server doesn't seem to support subversion's --reintegrate option).


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3078 1f5c12ca-751b-0410-a591-d2e778427230

9 files changed, 1145 insertions, 0 deletions

diff --git a/moses-chart-cmd/src/IOWrapper.cpp b/moses-chart-cmd/src/IOWrapper.cpp
new file mode 100644
index 000000000..c58239f66
--- /dev/null
+++ b/moses-chart-cmd/src/IOWrapper.cpp
@@ -0,0 +1,470 @@
+// $Id: IOWrapper.cpp 2675 2010-01-26 18:37:42Z hieuhoang1972 $
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (c) 2006 University of Edinburgh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+			this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+			this list of conditions and the following disclaimer in the documentation
+			and/or other materials provided with the distribution.
+    * Neither the name of the University of Edinburgh nor the names of its contributors
+			may be used to endorse or promote products derived from this software
+			without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+***********************************************************************/
+
+// example file on how to use moses library
+
+#include <iostream>
+#include "TypeDef.h"
+#include "Util.h"
+#include "IOWrapper.h"
+#include "WordsRange.h"
+#include "StaticData.h"
+#include "DummyScoreProducers.h"
+#include "InputFileStream.h"
+#include "PhraseDictionary.h"
+#include "../../moses-chart/src/ChartTrellisPathList.h"
+#include "../../moses-chart/src/ChartTrellisPath.h"
+
+using namespace std;
+using namespace Moses;
+using namespace MosesChart;
+
+IOWrapper::IOWrapper(
+				const vector<FactorType>				&inputFactorOrder
+				, const vector<FactorType>			&outputFactorOrder
+				, const FactorMask							&inputFactorUsed
+				, size_t												nBestSize
+				, const string									&nBestFilePath)
+:m_inputFactorOrder(inputFactorOrder)
+,m_outputFactorOrder(outputFactorOrder)
+,m_inputFactorUsed(inputFactorUsed)
+,m_inputFile(NULL)
+,m_inputStream(&std::cin)
+,m_nBestStream(NULL)
+,m_outputSearchGraphStream(NULL)
+{
+	const StaticData &staticData = StaticData::Instance();
+
+	m_surpressSingleBestOutput = false;
+	if (nBestSize > 0)
+	{
+		if (nBestFilePath == "-")
+		{
+			m_nBestStream = &std::cout;
+			m_surpressSingleBestOutput = true;
+		}
+		else
+		{
+			std::ofstream *nBestFile = new std::ofstream;
+			m_nBestStream = nBestFile;
+			nBestFile->open(nBestFilePath.c_str());
+		}
+	}
+	
+	// search graph output
+	if (staticData.GetOutputSearchGraph())
+	{
+	  string fileName = staticData.GetParam("output-search-graph")[0];
+	  std::ofstream *file = new std::ofstream;
+	  m_outputSearchGraphStream = file;
+	  file->open(fileName.c_str());
+	}
+	
+}
+
+IOWrapper::IOWrapper(const std::vector<FactorType>	&inputFactorOrder
+						 , const std::vector<FactorType>	&outputFactorOrder
+							, const FactorMask							&inputFactorUsed
+							, size_t												nBestSize
+							, const std::string							&nBestFilePath
+							, const std::string							&inputFilePath)
+:m_inputFactorOrder(inputFactorOrder)
+,m_outputFactorOrder(outputFactorOrder)
+,m_inputFactorUsed(inputFactorUsed)
+,m_inputFilePath(inputFilePath)
+,m_inputFile(new InputFileStream(inputFilePath))
+,m_nBestStream(NULL)
+,m_outputSearchGraphStream(NULL)
+{
+	const StaticData &staticData = StaticData::Instance();
+
+	m_surpressSingleBestOutput = false;
+	m_inputStream = m_inputFile;
+
+	if (nBestSize > 0)
+	{
+		if (nBestFilePath == "-")
+		{
+			m_nBestStream = &std::cout;
+			m_surpressSingleBestOutput = true;
+		}
+		else
+		{
+			std::ofstream *nBestFile = new std::ofstream;
+			m_nBestStream = nBestFile;
+			nBestFile->open(nBestFilePath.c_str());
+		}
+	}
+
+	// search graph output
+	if (staticData.GetOutputSearchGraph())
+	{
+	  string fileName = staticData.GetParam("output-search-graph")[0];
+	  std::ofstream *file = new std::ofstream;
+	  m_outputSearchGraphStream = file;
+	  file->open(fileName.c_str());
+	}
+	
+}
+
+IOWrapper::~IOWrapper()
+{
+	if (m_inputFile != NULL)
+		delete m_inputFile;
+	if (m_nBestStream != NULL && !m_surpressSingleBestOutput)
+	{ // outputting n-best to file, rather than stdout. need to close file and delete obj
+		delete m_nBestStream;
+	}
+
+	if (m_outputSearchGraphStream != NULL)
+	{
+	  delete m_outputSearchGraphStream;
+	}
+}
+
+InputType*IOWrapper::GetInput(InputType* inputType)
+{
+	if(inputType->Read(*m_inputStream, m_inputFactorOrder))
+	{
+		if (long x = inputType->GetTranslationId()) { if (x>=m_translationId) m_translationId = x+1; }
+		else inputType->SetTranslationId(m_translationId++);
+
+		return inputType;
+	}
+	else
+	{
+		delete inputType;
+		return NULL;
+	}
+}
+
+/***
+ * print surface factor only for the given phrase
+ */
+void OutputSurface(std::ostream &out, const Phrase &phrase, const std::vector<FactorType> &outputFactorOrder, bool reportAllFactors)
+{
+	assert(outputFactorOrder.size() > 0);
+	if (reportAllFactors == true)
+	{
+		out << phrase;
+	}
+	else
+	{
+		size_t size = phrase.GetSize();
+		for (size_t pos = 0 ; pos < size ; pos++)
+		{
+			const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
+			out << *factor;
+
+			for (size_t i = 1 ; i < outputFactorOrder.size() ; i++)
+			{
+				const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
+				out << "|" << *factor;
+			}
+			out << " ";
+		}
+	}
+}
+
+void OutputSurface(std::ostream &out, const MosesChart::Hypothesis *hypo, const std::vector<FactorType> &outputFactorOrder
+									 ,bool reportSegmentation, bool reportAllFactors)
+{
+	if ( hypo != NULL)
+	{
+		//OutputSurface(out, hypo->GetCurrTargetPhrase(), outputFactorOrder, reportAllFactors);
+
+		const vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+		vector<const MosesChart::Hypothesis*>::const_iterator iter;
+		for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter)
+		{
+			const MosesChart::Hypothesis *prevHypo = *iter;
+
+			OutputSurface(out, prevHypo, outputFactorOrder, reportSegmentation, reportAllFactors);
+		}
+	}
+}
+
+void IOWrapper::Backtrack(const MosesChart::Hypothesis *hypo)
+{
+	const vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos();
+
+	vector<const MosesChart::Hypothesis*>::const_iterator iter;
+	for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter)
+	{
+		const MosesChart::Hypothesis *prevHypo = *iter;
+
+		VERBOSE(3,prevHypo->GetId() << " <= ");
+		Backtrack(prevHypo);
+	}
+}
+
+void IOWrapper::OutputBestHypo(const std::vector<const Factor*>&  mbrBestHypo, long /*translationId*/, bool reportSegmentation, bool reportAllFactors)
+{
+	for (size_t i = 0 ; i < mbrBestHypo.size() ; i++)
+			{
+				const Factor *factor = mbrBestHypo[i];
+				cout << *factor << " ";
+			}
+}
+/*
+void OutputInput(std::vector<const Phrase*>& map, const MosesChart::Hypothesis* hypo)
+{
+	if (hypo->GetPrevHypos())
+	{
+	  	OutputInput(map, hypo->GetPrevHypos());
+		map[hypo->GetCurrSourceWordsRange().GetStartPos()] = hypo->GetSourcePhrase();
+	}
+}
+
+void OutputInput(std::ostream& os, const MosesChart::Hypothesis* hypo)
+{
+	size_t len = StaticData::Instance().GetInput()->GetSize();
+	std::vector<const Phrase*> inp_phrases(len, 0);
+	OutputInput(inp_phrases, hypo);
+	for (size_t i=0; i<len; ++i)
+		if (inp_phrases[i]) os << *inp_phrases[i];
+}
+*/
+
+void OutputTranslationOptions(const MosesChart::Hypothesis *hypo, long translationId)
+{ // recursive
+	if (hypo != NULL)
+	{
+	  //cerr << "Trans Opt " << translationId << " " << hypo->GetCurrSourceRange() << " = " <<  hypo->GetCurrTargetPhrase() << endl;
+		cerr << *hypo << endl;
+	}
+	
+	const std::vector<const MosesChart::Hypothesis*> &prevHypos = hypo->GetPrevHypos();
+	std::vector<const MosesChart::Hypothesis*>::const_iterator iter;
+	for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter)
+	{
+		const MosesChart::Hypothesis *prevHypo = *iter;
+		OutputTranslationOptions(prevHypo, translationId);
+	}
+}
+
+void IOWrapper::OutputBestHypo(const MosesChart::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors)
+{
+	if (hypo != NULL)
+	{
+		//cerr << *hypo << endl;
+		
+		VERBOSE(3,"Best path: ");
+		Backtrack(hypo);
+		VERBOSE(3,"0" << std::endl);
+
+		if (StaticData::Instance().GetOutputHypoScore())
+		{
+			cout << hypo->GetTotalScore() << " " 
+					<< MosesChart::Hypothesis::GetHypoCount() << " ";
+		}
+		
+		if (StaticData::Instance().IsDetailedTranslationReportingEnabled())
+		{
+			OutputTranslationOptions(hypo, translationId);
+		}
+		
+		if (!m_surpressSingleBestOutput)
+		{
+			if (StaticData::Instance().IsPathRecoveryEnabled()) {
+				//OutputInput(cout, hypo);
+				cout << "||| ";
+			}
+			Phrase outPhrase(Output);
+			hypo->CreateOutputPhrase(outPhrase);
+
+			// delete 1st & last
+			assert(outPhrase.GetSize() >= 2);
+			outPhrase.RemoveWord(0);
+			outPhrase.RemoveWord(outPhrase.GetSize() - 1);
+
+			const std::vector<FactorType> outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
+			string output = outPhrase.GetStringRep(outputFactorOrder);
+			cout << output << endl;
+		}
+	}
+	else
+	{
+		VERBOSE(1, "NO BEST TRANSLATION" << endl);
+		
+		if (StaticData::Instance().GetOutputHypoScore())
+		{
+			cout << "0 ";
+		}
+		
+		if (!m_surpressSingleBestOutput)
+		{
+			cout << endl;
+		}
+	}
+}
+
+void IOWrapper::OutputNBestList(const MosesChart::TrellisPathList &nBestList, long translationId)
+{
+	bool labeledOutput = StaticData::Instance().IsLabeledNBestList();
+	//bool includeAlignment = StaticData::Instance().NBestIncludesAlignment();
+
+	MosesChart::TrellisPathList::const_iterator iter;
+	for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
+	{
+		const MosesChart::TrellisPath &path = **iter;
+		//cerr << path << endl << endl;
+
+		Moses::Phrase outputPhrase = path.GetOutputPhrase();
+
+		// delete 1st & last
+		assert(outputPhrase.GetSize() >= 2);
+		outputPhrase.RemoveWord(0);
+		outputPhrase.RemoveWord(outputPhrase.GetSize() - 1);
+
+		// print the surface factor of the translation
+		*m_nBestStream << translationId << " ||| ";
+		OutputSurface(*m_nBestStream, outputPhrase, m_outputFactorOrder, false);
+		*m_nBestStream << " ||| ";
+
+		// print the scores in a hardwired order
+    // before each model type, the corresponding command-line-like name must be emitted
+    // MERT script relies on this
+
+		// lm
+		const LMList& lml = StaticData::Instance().GetAllLM();
+    if (lml.size() > 0) {
+			if (labeledOutput)
+	      *m_nBestStream << "lm: ";
+		  LMList::const_iterator lmi = lml.begin();
+		  for (; lmi != lml.end(); ++lmi) {
+			  *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " ";
+		  }
+    }
+
+		// translation components
+		if (StaticData::Instance().GetInputType()==SentenceInput){
+			// translation components	for text input
+			vector<PhraseDictionaryFeature*> pds = StaticData::Instance().GetPhraseDictionaries();
+			if (pds.size() > 0) {
+				if (labeledOutput)
+					*m_nBestStream << "tm: ";
+				vector<PhraseDictionaryFeature*>::iterator iter;
+				for (iter = pds.begin(); iter != pds.end(); ++iter) {
+					vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
+					for (size_t j = 0; j<scores.size(); ++j)
+						*m_nBestStream << scores[j] << " ";
+				}
+			}
+		}
+		else{
+			// translation components for Confusion Network input
+			// first translation component has GetNumInputScores() scores from the input Confusion Network
+			// at the beginning of the vector
+			vector<PhraseDictionaryFeature*> pds = StaticData::Instance().GetPhraseDictionaries();
+			if (pds.size() > 0) {
+				vector<PhraseDictionaryFeature*>::iterator iter;
+
+				iter = pds.begin();
+				vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
+
+				size_t pd_numinputscore = (*iter)->GetNumInputScores();
+
+				if (pd_numinputscore){
+
+					if (labeledOutput)
+						*m_nBestStream << "I: ";
+
+					for (size_t j = 0; j < pd_numinputscore; ++j)
+						*m_nBestStream << scores[j] << " ";
+				}
+
+
+				for (iter = pds.begin() ; iter != pds.end(); ++iter) {
+					vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
+
+					size_t pd_numinputscore = (*iter)->GetNumInputScores();
+
+					if (iter == pds.begin() && labeledOutput)
+						*m_nBestStream << "tm: ";
+					for (size_t j = pd_numinputscore; j < scores.size() ; ++j)
+						*m_nBestStream << scores[j] << " ";
+				}
+			}
+		}
+
+
+
+		// word penalty
+		if (labeledOutput)
+	    *m_nBestStream << "w: ";
+		*m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetWordPenaltyProducer()) << " ";
+
+		// generation
+		vector<GenerationDictionary*> gds = StaticData::Instance().GetGenerationDictionaries();
+    if (gds.size() > 0) {
+			if (labeledOutput)
+	      *m_nBestStream << "g: ";
+		  vector<GenerationDictionary*>::iterator iter;
+		  for (iter = gds.begin(); iter != gds.end(); ++iter) {
+			  vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer(*iter);
+			  for (size_t j = 0; j<scores.size(); j++) {
+				  *m_nBestStream << scores[j] << " ";
+			  }
+		  }
+    }
+
+		// total
+    *m_nBestStream << "||| " << path.GetTotalScore();
+
+		/*
+    if (includeAlignment) {
+			*m_nBestStream << " |||";
+			for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--)
+			{
+				const MosesChart::Hypothesis &edge = *edges[currEdge];
+				WordsRange sourceRange = edge.GetCurrSourceWordsRange();
+				WordsRange targetRange = edge.GetCurrTargetWordsRange();
+				*m_nBestStream << " " << sourceRange.GetStartPos();
+				if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) {
+					*m_nBestStream << "-" << sourceRange.GetEndPos();
+				}
+				*m_nBestStream << "=" << targetRange.GetStartPos();
+				if (targetRange.GetStartPos() < targetRange.GetEndPos()) {
+					*m_nBestStream << "-" << targetRange.GetEndPos();
+				}
+			}
+    }
+		*/
+
+    *m_nBestStream << endl;
+	}
+
+	*m_nBestStream<<std::flush;
+}
+
diff --git a/moses-chart-cmd/src/IOWrapper.h b/moses-chart-cmd/src/IOWrapper.h
new file mode 100644
index 000000000..493365389
--- /dev/null
+++ b/moses-chart-cmd/src/IOWrapper.h
@@ -0,0 +1,96 @@
+// $Id: IOWrapper.h 2615 2009-11-12 13:18:20Z hieuhoang1972 $
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (c) 2006 University of Edinburgh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+			this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+			this list of conditions and the following disclaimer in the documentation 
+			and/or other materials provided with the distribution.
+    * Neither the name of the University of Edinburgh nor the names of its contributors 
+			may be used to endorse or promote products derived from this software 
+			without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.
+***********************************************************************/
+
+// example file on how to use moses library
+
+#pragma once
+
+#include <fstream>
+#include <vector>
+#include "TypeDef.h"
+#include "Sentence.h"
+#include "FactorTypeSet.h"
+#include "InputFileStream.h"
+#include "TrellisPathList.h"
+#include "../../moses-chart/src/ChartHypothesis.h"
+
+namespace Moses
+{
+class FactorCollection;
+}
+
+namespace MosesChart
+{
+class TrellisPathList;
+}
+
+class IOWrapper
+{
+protected:
+	long m_translationId;
+
+	const std::vector<Moses::FactorType>	&m_inputFactorOrder;
+	const std::vector<Moses::FactorType>	&m_outputFactorOrder;
+	const Moses::FactorMask								&m_inputFactorUsed;
+	std::ostream 									*m_nBestStream, *m_outputSearchGraphStream;
+	std::string										m_inputFilePath;
+	std::istream									*m_inputStream;
+	Moses::InputFileStream				*m_inputFile;
+	bool													m_surpressSingleBestOutput;
+	
+public:
+	IOWrapper(const std::vector<Moses::FactorType>	&inputFactorOrder
+		, const std::vector<Moses::FactorType>			&outputFactorOrder
+				, const Moses::FactorMask							&inputFactorUsed
+				, size_t												nBestSize
+				, const std::string							&nBestFilePath);
+
+	IOWrapper(const std::vector<Moses::FactorType>	&inputFactorOrder
+				, const std::vector<Moses::FactorType>	&outputFactorOrder
+				, const Moses::FactorMask							&inputFactorUsed
+				, size_t												nBestSize
+				, const std::string							&nBestFilePath
+				, const std::string							&inputFilePath);
+	~IOWrapper();
+
+	Moses::InputType* GetInput(Moses::InputType *inputType);
+	void OutputBestHypo(const MosesChart::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
+	void OutputBestHypo(const std::vector<const Moses::Factor*>&  mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors);
+	void OutputNBestList(const MosesChart::TrellisPathList &nBestList, long translationId);
+	void Backtrack(const MosesChart::Hypothesis *hypo);
+
+	void ResetTranslationId() { m_translationId = 0; }
+
+	std::ostream &GetOutputSearchGraphStream()
+	{ return *m_outputSearchGraphStream; }
+	
+};
diff --git a/moses-chart-cmd/src/Main.cpp b/moses-chart-cmd/src/Main.cpp
new file mode 100644
index 000000000..18c62631d
--- /dev/null
+++ b/moses-chart-cmd/src/Main.cpp
@@ -0,0 +1,251 @@
+// $Id: Main.cpp 2615 2009-11-12 13:18:20Z hieuhoang1972 $
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (c) 2006 University of Edinburgh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+			this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+			this list of conditions and the following disclaimer in the documentation 
+			and/or other materials provided with the distribution.
+    * Neither the name of the University of Edinburgh nor the names of its contributors 
+			may be used to endorse or promote products derived from this software 
+			without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.
+***********************************************************************/
+
+// example file on how to use moses library
+
+#ifdef WIN32
+// Include Visual Leak Detector
+#include <vld.h>
+#endif
+
+#include <fstream>
+#include "Main.h"
+#include "TrellisPath.h"
+#include "FactorCollection.h"
+#include "Manager.h"
+#include "Phrase.h"
+#include "Util.h"
+#include "TrellisPathList.h"
+#include "Timer.h"
+#include "IOWrapper.h"
+#include "Sentence.h"
+#include "ConfusionNet.h"
+#include "WordLattice.h"
+#include "TreeInput.h"
+#include "TranslationAnalysis.h"
+#include "mbr.h"
+#include "../../moses-chart/src/ChartManager.h"
+#include "../../moses-chart/src/ChartHypothesis.h"
+#include "../../moses-chart/src/ChartTrellisPathList.h"
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#else
+// those not using autoconf have to build MySQL support for now
+#  define USE_MYSQL 1
+#endif
+
+using namespace std;
+using namespace Moses;
+using namespace MosesChart;
+
+bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source) 
+{
+	delete source;
+	switch(inputType)
+	{
+		case SentenceInput:         source = ioWrapper.GetInput(new Sentence(Input)); break;
+		case ConfusionNetworkInput: source = ioWrapper.GetInput(new ConfusionNet);    break;
+		case WordLatticeInput:      source = ioWrapper.GetInput(new WordLattice);     break;
+		case TreeInputType:					source = ioWrapper.GetInput(new TreeInput(Input));break;
+		default: TRACE_ERR("Unknown input type: " << inputType << "\n");
+	}
+	return (source ? true : false);
+}
+
+
+int main(int argc, char* argv[])
+{
+	IFVERBOSE(1)
+	{
+		TRACE_ERR("command: ");
+		for(int i=0;i<argc;++i) TRACE_ERR(argv[i]<<" ");
+		TRACE_ERR(endl);
+	}
+
+	cout.setf(std::ios::fixed); 
+	cout.precision(3);
+	cerr.setf(std::ios::fixed); 
+	cerr.precision(3);
+
+	// load data structures
+	Parameter *parameter = new Parameter();
+	if (!parameter->LoadParam(argc, argv))
+	{
+		parameter->Explain();
+		delete parameter;
+		return EXIT_FAILURE;		
+	}
+
+	const StaticData &staticData = StaticData::Instance();
+	if (!StaticData::LoadDataStatic(parameter))
+		return EXIT_FAILURE;
+
+	assert(staticData.GetSearchAlgorithm() == ChartDecoding);
+
+	// set up read/writing class
+	IOWrapper *ioWrapper = GetIODevice(staticData);
+
+	// check on weights
+	vector<float> weights = staticData.GetAllWeights();
+	IFVERBOSE(2) {
+	  TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
+	  TRACE_ERR("The global weight vector looks like this:");
+	  for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
+	  TRACE_ERR("\n");
+	}
+	// every score must have a weight!  check that here:
+	if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
+	  TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
+	  return EXIT_FAILURE;
+	}
+
+	if (ioWrapper == NULL)
+		return EXIT_FAILURE;
+
+	// read each sentence & decode
+	InputType *source=0;
+	size_t lineCount = 0;
+	while(ReadInput(*ioWrapper,staticData.GetInputType(),source))
+	{
+			// note: source is only valid within this while loop!
+		IFVERBOSE(1)
+			ResetUserTime();
+			
+    VERBOSE(2,"\nTRANSLATING(" << ++lineCount << "): " << *source);
+		//cerr << *source << endl;
+		
+		MosesChart::Manager manager(*source);
+		manager.ProcessSentence();
+
+		assert(!staticData.UseMBR());
+
+		if (!staticData.UseMBR()){
+			ioWrapper->OutputBestHypo(manager.GetBestHypothesis(), source->GetTranslationId(),
+													 staticData.GetReportSegmentation(),
+													 staticData.GetReportAllFactors()
+													 );
+			IFVERBOSE(2) { PrintUserTime("Best Hypothesis Generation Time:"); }
+			
+			// n-best
+			size_t nBestSize = staticData.GetNBestSize();
+			if (nBestSize > 0)
+			{
+			  	VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
+					MosesChart::TrellisPathList nBestList;
+					manager.CalcNBest(nBestSize, nBestList,staticData.GetDistinctNBest());
+					ioWrapper->OutputNBestList(nBestList, source->GetTranslationId());
+			
+					IFVERBOSE(2) { PrintUserTime("N-Best Hypotheses Generation Time:"); }
+			}
+			
+			if (staticData.GetOutputSearchGraph())
+				manager.GetSearchGraph(source->GetTranslationId(), ioWrapper->GetOutputSearchGraphStream());
+
+		}
+		else {
+			/*
+			size_t nBestSize = staticData.GetNBestSize();
+			cerr << "NBEST SIZE : " << nBestSize << endl;
+			assert(nBestSize > 0);
+			
+		  if (nBestSize > 0)
+			{
+			  VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
+				MosesChart::TrellisPathList nBestList;
+				manager.CalcNBest(nBestSize, nBestList,true);
+				std::vector<const Factor*> mbrBestHypo = doMBR(nBestList);
+				ioWrapper->OutputBestHypo(mbrBestHypo, source->GetTranslationId(),
+													 staticData.GetReportSegmentation(),
+													 staticData.GetReportAllFactors()
+													 );
+				IFVERBOSE(2) { PrintUserTime("N-Best Hypotheses Generation Time:"); }
+		  }
+			*/
+
+		}	
+		/*		 
+		if (staticData.IsDetailedTranslationReportingEnabled()) {
+			TranslationAnalysis::PrintTranslationAnalysis(std::cerr, manager.GetBestHypothesis());
+		}
+		*/
+
+		IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
+    
+		manager.CalcDecoderStatistics();    
+	} // while(ReadInput
+	
+	delete ioWrapper;
+
+	IFVERBOSE(1)
+		PrintUserTime("End.");
+
+	#ifdef HACK_EXIT
+	//This avoids that detructors are called (it can take a long time)
+		exit(EXIT_SUCCESS);
+	#else
+		return EXIT_SUCCESS;
+	#endif
+}
+
+IOWrapper *GetIODevice(const StaticData &staticData) 
+{
+	IOWrapper *ioWrapper;
+	const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder()
+																,&outputFactorOrder = staticData.GetOutputFactorOrder();
+	FactorMask inputFactorUsed(inputFactorOrder);
+
+	// io
+	if (staticData.GetParam("input-file").size() == 1)
+	{
+	  VERBOSE(2,"IO from File" << endl);
+		string filePath = staticData.GetParam("input-file")[0];
+
+		ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
+																	, staticData.GetNBestSize()
+																	, staticData.GetNBestFilePath()
+																	, filePath);
+	}
+	else
+	{
+	  VERBOSE(1,"IO from STDOUT/STDIN" << endl);
+		ioWrapper = new IOWrapper(inputFactorOrder, outputFactorOrder, inputFactorUsed
+																	, staticData.GetNBestSize()
+																	, staticData.GetNBestFilePath());
+	}
+	ioWrapper->ResetTranslationId();
+
+	IFVERBOSE(1)
+		PrintUserTime("Created input-output object");
+
+	return ioWrapper;
+}
diff --git a/moses-chart-cmd/src/Main.h b/moses-chart-cmd/src/Main.h
new file mode 100644
index 000000000..d203ea264
--- /dev/null
+++ b/moses-chart-cmd/src/Main.h
@@ -0,0 +1,42 @@
+// $Id: Main.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (c) 2006 University of Edinburgh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, 
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+			this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+			this list of conditions and the following disclaimer in the documentation 
+			and/or other materials provided with the distribution.
+    * Neither the name of the University of Edinburgh nor the names of its contributors 
+			may be used to endorse or promote products derived from this software 
+			without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.
+***********************************************************************/
+
+// example file on how to use moses library
+
+#pragma once
+
+#include "StaticData.h"
+
+class IOWrapper;
+
+int main(int argc, char* argv[]);
+IOWrapper *GetIODevice(const Moses::StaticData &staticData);
diff --git a/moses-chart-cmd/src/Makefile.am b/moses-chart-cmd/src/Makefile.am
new file mode 100644
index 000000000..1cb86ca76
--- /dev/null
+++ b/moses-chart-cmd/src/Makefile.am
@@ -0,0 +1,10 @@
+bin_PROGRAMS = moses_chart
+moses_chart_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp
+AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src
+
+moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt
+moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
+
+
+
+
diff --git a/moses-chart-cmd/src/TranslationAnalysis.cpp b/moses-chart-cmd/src/TranslationAnalysis.cpp
new file mode 100644
index 000000000..e7a57d081
--- /dev/null
+++ b/moses-chart-cmd/src/TranslationAnalysis.cpp
@@ -0,0 +1,46 @@
+// $Id: TranslationAnalysis.cpp 2474 2009-08-06 17:32:28Z hieuhoang1972 $
+
+#include <iostream>
+#include <sstream>
+#include <algorithm>
+#include "StaticData.h"
+#include "TranslationAnalysis.h"
+#include "TranslationOption.h"
+#include "DecodeStepTranslation.h"
+
+using namespace std;
+using namespace Moses;
+
+namespace TranslationAnalysis {
+
+void PrintTranslationAnalysis(ostream &os, const Hypothesis* hypo)
+{
+	/*
+	os << endl << "TRANSLATION HYPOTHESIS DETAILS:" << endl;
+  queue<const Hypothesis*> translationPath;
+  while (hypo) 
+	{
+		translationPath.push(hypo);
+    hypo = hypo->GetPrevHypo();
+  }
+
+	while (!translationPath.empty())
+	{
+		hypo = translationPath.front();
+		translationPath.pop();
+		const TranslationOption *transOpt = hypo->GetTranslationOption();
+		if (transOpt != NULL)
+		{
+			os	<< hypo->GetCurrSourceWordsRange() << "  ";
+			for (size_t decodeStepId = 0; decodeStepId < DecodeStepTranslation::GetNumTransStep(); ++decodeStepId)
+				os << decodeStepId << "=" << transOpt->GetSubRangeCount(decodeStepId) << ",";
+			os	<< *transOpt << endl;
+		}
+	}
+
+	os << "END TRANSLATION" << endl;
+	*/
+}
+
+}
+
diff --git a/moses-chart-cmd/src/TranslationAnalysis.h b/moses-chart-cmd/src/TranslationAnalysis.h
new file mode 100644
index 000000000..28901a076
--- /dev/null
+++ b/moses-chart-cmd/src/TranslationAnalysis.h
@@ -0,0 +1,24 @@
+// $Id: TranslationAnalysis.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $
+
+/*
+ * also see moses/SentenceStats
+ */
+
+#ifndef _TRANSLATION_ANALYSIS_H_
+#define _TRANSLATION_ANALYSIS_H_
+
+#include <iostream>
+#include "Hypothesis.h"
+
+namespace TranslationAnalysis
+{
+
+/***
+ * print details about the translation represented in hypothesis to
+ * os.  Included information: phrase alignment, words dropped, scores
+ */
+	void PrintTranslationAnalysis(std::ostream &os, const Moses::Hypothesis* hypo);
+
+}
+
+#endif
diff --git a/moses-chart-cmd/src/mbr.cpp b/moses-chart-cmd/src/mbr.cpp
new file mode 100644
index 000000000..5b8902aae
--- /dev/null
+++ b/moses-chart-cmd/src/mbr.cpp
@@ -0,0 +1,179 @@
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <iomanip>
+#include <vector>
+#include <map>
+#include <stdlib.h>
+#include <math.h>
+#include <algorithm>
+#include <stdio.h>
+#include "TrellisPathList.h"
+#include "TrellisPath.h"
+#include "StaticData.h"
+#include "Util.h"
+#include "mbr.h"
+
+using namespace std ;
+using namespace Moses;
+
+
+/* Input : 
+   1. a sorted  n-best list, with duplicates filtered out in the following  format 
+   0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432
+
+   2. a weight vector 
+   3. bleu order ( default = 4)
+   4. scaling factor to weigh the weight vector (default = 1.0)
+
+   Output :
+   translations that minimise the Bayes Risk of the n-best list
+
+
+*/
+
+int BLEU_ORDER = 4;
+int SMOOTH = 1;
+int DEBUG = 0;
+float min_interval = 1e-4;
+void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const Factor* >, int >  & allngrams)
+{
+  vector< const Factor* > ngram;
+  for (int k = 0; k < BLEU_ORDER; k++)
+  {
+    for(int i =0; i < max((int)sentence.size()-k,0); i++)
+    {
+      for ( int j = i; j<= i+k; j++)
+      {
+        ngram.push_back(sentence[j]);
+      }
+      ++allngrams[ngram];
+      ngram.clear();
+    }
+  }
+}
+
+float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp,  vector < map < vector < const Factor *>, int > > & ngram_stats ) {
+  int comps_n = 2*BLEU_ORDER+1;
+  vector<int> comps(comps_n);
+  float logbleu = 0.0, brevity;
+  
+  int hyp_length = sents[hyp].size();
+
+  for (int i =0; i<BLEU_ORDER;i++)
+  {
+    comps[2*i] = 0;
+    comps[2*i+1] = max(hyp_length-i,0);
+  }
+
+  map< vector < const Factor * > ,int > & hyp_ngrams = ngram_stats[hyp] ;
+  map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ;
+
+  for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin();
+       it != hyp_ngrams.end(); it++)
+  {
+    map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first);
+    if(ref_it != ref_ngrams.end())
+    {
+      comps[2* (it->first.size()-1)] += min(ref_it->second,it->second);
+    }
+  }
+  comps[comps_n-1] = sents[ref].size();
+
+  if (DEBUG)
+  {
+    for ( int i = 0; i < comps_n; i++)
+      cerr << "Comp " << i << " : " << comps[i];
+  }
+
+  for (int i=0; i<BLEU_ORDER; i++)
+  {
+    if (comps[0] == 0)
+      return 0.0;
+    if ( i > 0 )
+      logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
+    else
+      logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
+  }
+  logbleu /= BLEU_ORDER;
+  brevity = 1.0-(float)comps[comps_n-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
+  if (brevity < 0.0)
+    logbleu += brevity;
+  return exp(logbleu);
+}
+
+vector<const Factor*> doMBR(const TrellisPathList& nBestList){
+//   cerr << "Sentence " << sent << " has " << sents.size() << " candidate translations" << endl;
+  float marginal = 0;
+
+  vector<float> joint_prob_vec;
+  vector< vector<const Factor*> > translations;
+  float joint_prob;
+  vector< map < vector <const Factor *>, int > > ngram_stats;
+
+  TrellisPathList::const_iterator iter;
+  //TrellisPath* hyp = NULL;
+	for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
+	{
+		const TrellisPath &path = **iter;
+    joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights()));
+    marginal += joint_prob;
+    joint_prob_vec.push_back(joint_prob);
+    //Cache ngram counts
+    map < vector < const Factor *>, int > counts;
+    vector<const Factor*> translation;
+    GetOutputFactors(path, translation);
+    
+    //TO DO
+    extract_ngrams(translation,counts);
+    ngram_stats.push_back(counts);
+    translations.push_back(translation);
+   }
+   
+   vector<float> mbr_loss;
+   float bleu, weightedLoss;
+   float weightedLossCumul = 0;
+   float minMBRLoss = 1000000;
+   int minMBRLossIdx = -1;
+   
+   /* Main MBR computation done here */
+   for (int i = 0; i < nBestList.GetSize(); i++){
+       weightedLossCumul = 0;
+       for (int j = 0; j < nBestList.GetSize(); j++){
+            if ( i != j) {
+               bleu = calculate_score(translations, j, i,ngram_stats );
+               weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal);
+               weightedLossCumul += weightedLoss;
+               if (weightedLossCumul > minMBRLoss)
+                   break;
+             }
+       }
+       if (weightedLossCumul < minMBRLoss){
+           minMBRLoss = weightedLossCumul;
+           minMBRLossIdx = i;
+       }
+   }
+   /* Find sentence that minimises Bayes Risk under 1- BLEU loss */
+   return translations[minMBRLossIdx];
+}
+
+void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation){
+	const std::vector<const Hypothesis *> &edges = path.GetEdges();
+	const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
+	assert (outputFactorOrder.size() == 1);
+
+	// print the surface factor of the translation
+	for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
+	{
+		const Hypothesis &edge = *edges[currEdge];
+		const Phrase &phrase = edge.GetCurrTargetPhrase();
+		size_t size = phrase.GetSize();
+		for (size_t pos = 0 ; pos < size ; pos++)
+		{
+			
+			const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
+			translation.push_back(factor);
+		}
+	}
+}
+
diff --git a/moses-chart-cmd/src/mbr.h b/moses-chart-cmd/src/mbr.h
new file mode 100644
index 000000000..7d63ac0f5
--- /dev/null
+++ b/moses-chart-cmd/src/mbr.h
@@ -0,0 +1,27 @@
+// $Id: mbr.h 2474 2009-08-06 17:32:28Z hieuhoang1972 $
+
+/***********************************************************************
+Moses - factored phrase-based language decoder
+Copyright (C) 2006 University of Edinburgh
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+
+std::vector<const Moses::Factor*> doMBR(const Moses::TrellisPathList& nBestList);
+void GetOutputFactors(const Moses::TrellisPath &path, std::vector <const Moses::Factor*> &translation);
+float calculate_score(const std::vector< std::vector<const Moses::Factor*> > & sents, int ref, int hyp,  std::vector < std::map < std::vector < const Moses::Factor *>, int > > & ngram_stats );
+