- translation of confusion nets integrated (works only with binary phrase table)

 - removed some obsolete stuff


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@250 1f5c12ca-751b-0410-a591-d2e778427230

23 files changed, 284 insertions, 162 deletions

diff --git a/moses/src/ConfusionNet.cpp b/moses/src/ConfusionNet.cpp
index d5a153c25..235ae87a4 100644
--- a/moses/src/ConfusionNet.cpp
+++ b/moses/src/ConfusionNet.cpp
@@ -4,6 +4,8 @@
 
 #include "FactorCollection.h"
 #include "Util.h"
+#include "PhraseDictionaryTreeAdaptor.h"
+#include "TranslationOptionCollectionConfusionNet.h"
 
 ConfusionNet::ConfusionNet(FactorCollection* p) : InputType(),m_factorCollection(p) {}
 
@@ -55,9 +57,6 @@ bool ConfusionNet::ReadFormat0(std::istream& in,const std::vector<FactorType>& f
 		}
 		else break;
 	}
-	std::cerr<<"conf net read: "<<data.size()<<"\n";
-
-
 	return !data.empty();
 }
 bool ConfusionNet::ReadFormat1(std::istream& in,const std::vector<FactorType>& factorOrder) {
@@ -76,7 +75,7 @@ bool ConfusionNet::ReadFormat1(std::istream& in,const std::vector<FactorType>& f
 		data[i].resize(s);
 		for(size_t j=0;j<s;++j)
 			if(is>>word>>prob) {
-				data[i][j].second=-log(prob); 
+				data[i][j].second=log(prob); 
 				if(data[i][j].second<0) {
 					std::cerr<<"WARN: neg costs: "<<data[i][j].second<<" -> set to 0\n";
 					data[i][j].second=0.0;}
@@ -111,3 +110,17 @@ std::ostream& operator<<(std::ostream& out,const ConfusionNet& cn)
 	cn.Print(out);return out;
 }
 
+TargetPhraseCollection const* ConfusionNet::CreateTargetPhraseCollection(PhraseDictionaryBase const& d,const WordsRange& r) const 
+{
+	if(PhraseDictionaryTreeAdaptor const* pdict=dynamic_cast<PhraseDictionaryTreeAdaptor const*>(&d))
+		return pdict->GetTargetPhraseCollection(*this,r);
+	std::cerr<<"ERROR: wrong phrase dictionary type for confusion net decoding!\n"
+		"has to be PhraseDictionaryTreeAdaptor\n";
+	abort();
+}
+TranslationOptionCollection* ConfusionNet::CreateTranslationOptionCollection() const 
+{
+	return new TranslationOptionCollectionConfusionNet(*this);
+}
+
+
diff --git a/moses/src/ConfusionNet.h b/moses/src/ConfusionNet.h
index 3581dcc62..8f27f4ccb 100644
--- a/moses/src/ConfusionNet.h
+++ b/moses/src/ConfusionNet.h
@@ -37,6 +37,9 @@ class ConfusionNet : public InputType {
 	const FactorArray& GetFactorArray(size_t pos) const;
 
 
+	TargetPhraseCollection const* CreateTargetPhraseCollection(PhraseDictionaryBase const& d,const WordsRange& r) const;
+	TranslationOptionCollection* CreateTranslationOptionCollection() const;
+
 
  private:
 	bool ReadFormat0(std::istream&,const std::vector<FactorType>& factorOrder);
diff --git a/moses/src/CreateTranslationOptionCollection.cpp b/moses/src/CreateTranslationOptionCollection.cpp
deleted file mode 100644
index f76cecde3..000000000
--- a/moses/src/CreateTranslationOptionCollection.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-// $Id$
-#include "CreateTranslationOptionCollection.h"
-#include "Sentence.h"
-#include "ConfusionNet.h"
-#include "TranslationOptionCollectionText.h"
-#include "TranslationOptionCollectionConfusionNet.h"
-
-TranslationOptionCollection* CreateTranslationOptionCollection(InputType const* src) 
-{
-	if(Sentence const * s=dynamic_cast<Sentence const*>(src)) 
-		return new TranslationOptionCollectionText(*s);
-	else if(ConfusionNet const* cn=dynamic_cast<ConfusionNet const*>(src)) 
-		return new TranslationOptionCollectionConfusionNet(*cn);
-	else 
-		{
-			std::cerr<<"ERROR: unknown InputType in "<<__FILE__<<"\n";
-			abort();
-		}
-}
-
diff --git a/moses/src/CreateTranslationOptionCollection.h b/moses/src/CreateTranslationOptionCollection.h
deleted file mode 100644
index b191b00c8..000000000
--- a/moses/src/CreateTranslationOptionCollection.h
+++ /dev/null
@@ -1,11 +0,0 @@
-// $Id$
-#ifndef CREATETRANSLATIONOPTIONCOLLECTION_H_
-#define CREATETRANSLATIONOPTIONCOLLECTION_H_
-
-class TranslationOptionCollection;
-class InputType;
-
-TranslationOptionCollection* CreateTranslationOptionCollection(InputType const* src);
-
-#endif
-
diff --git a/moses/src/Input.h b/moses/src/Input.h
index 194487673..bfb6a7a8c 100644
--- a/moses/src/Input.h
+++ b/moses/src/Input.h
@@ -9,7 +9,7 @@
 class WordsRange;
 class Factor;
 class PhraseDictionaryBase;
-
+class TranslationOptionCollection;
 
 // base class for sentences and confusion networks
 class InputType 
@@ -35,7 +35,7 @@ protected:
 	virtual int Read(std::istream& in,const std::vector<FactorType>& factorOrder, FactorCollection &factorCollection) =0;
 
 	virtual TargetPhraseCollection const* CreateTargetPhraseCollection(PhraseDictionaryBase const& d,const WordsRange& r) const=0;
-
+	virtual TranslationOptionCollection* CreateTranslationOptionCollection() const=0;
 
 	// these functions are not (yet) well-defined for confusion networks
 	virtual Phrase GetSubString(const WordsRange&) const =0;
diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am
index d3bab277e..fc81592de 100644
--- a/moses/src/Makefile.am
+++ b/moses/src/Makefile.am
@@ -1,9 +1,8 @@
 lib_LIBRARIES = libmoses.a
-AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES
+AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES 
 libmoses_a_SOURCES = \
 	Arc.cpp \
 	ConfusionNet.cpp \
-	CreateTranslationOptionCollection.cpp \
 	Dictionary.cpp \
 	Factor.cpp \
 	FactorCollection.cpp \
diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp
index 3da7df431..409a38238 100755
--- a/moses/src/Manager.cpp
+++ b/moses/src/Manager.cpp
@@ -38,7 +38,7 @@ Manager::Manager(InputType const& source,
 :m_source(source)
 ,m_hypoStack(source.GetSize() + 1)
 ,m_staticData(staticData)
-,m_possibleTranslations(toc)  //dynamic_cast<Sentence const&>(source))
+,m_possibleTranslations(toc)  
 {
 	std::vector < HypothesisCollection >::iterator iterStack;
 	for (iterStack = m_hypoStack.begin() ; iterStack != m_hypoStack.end() ; ++iterStack)
diff --git a/moses/src/Manager.h b/moses/src/Manager.h
index 4b2fafa48..67f7e2883 100755
--- a/moses/src/Manager.h
+++ b/moses/src/Manager.h
@@ -29,7 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "TranslationOption.h"
 #include "HypothesisCollection.h"
 #include "HypothesisCollectionIntermediate.h"
-#include "TranslationOptionCollectionText.h"
+#include "TranslationOptionCollection.h"
 #include "LatticePathList.h"
 #include "SquareMatrix.h"
 #include "WordsBitmap.h"
diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp
index 24a72bdf9..b44f7a02f 100755
--- a/moses/src/PhraseDictionary.cpp
+++ b/moses/src/PhraseDictionary.cpp
@@ -30,6 +30,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "Util.h"
 #include "InputFileStream.h"
 #include "StaticData.h"
+#include "Input.h"
+#include "WordsRange.h"
 
 using namespace std;
 
@@ -39,6 +41,12 @@ PhraseDictionaryBase::PhraseDictionaryBase(size_t noScoreComponent)
 {
 }
 PhraseDictionaryBase::~PhraseDictionaryBase() {}
+	
+const TargetPhraseCollection *PhraseDictionaryBase::
+GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const 
+{
+	return GetTargetPhraseCollection(src.GetSubString(range));
+}
 
 
 void PhraseDictionary::Load(const std::vector<FactorType> &input
diff --git a/moses/src/PhraseDictionary.h b/moses/src/PhraseDictionary.h
index d24276a3a..9fea1cbfe 100755
--- a/moses/src/PhraseDictionary.h
+++ b/moses/src/PhraseDictionary.h
@@ -32,7 +32,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "TargetPhraseCollection.h"
 
 class StaticData;
-
+class InputType;
+class WordsRange;
 
 class PhraseDictionaryBase : public Dictionary {
  protected:
@@ -47,13 +48,14 @@ class PhraseDictionaryBase : public Dictionary {
 		return Translate;
 	}
 	
+	virtual void InitializeForInput(InputType const&) {}
+
 	virtual void SetWeightTransModel(const std::vector<float> &weightT)=0;
 
 	virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const=0;
+	virtual const TargetPhraseCollection *GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const;
 
 	virtual void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)=0;
-	//	virtual const TargetPhraseCollection *FindEquivPhrase(const Phrase &source) const=0;
-
 };
 
 
diff --git a/moses/src/PhraseDictionaryTree.cpp b/moses/src/PhraseDictionaryTree.cpp
index 1458a880e..c5b448867 100644
--- a/moses/src/PhraseDictionaryTree.cpp
+++ b/moses/src/PhraseDictionaryTree.cpp
@@ -530,6 +530,14 @@ PhraseDictionaryTree::GetTargetCandidates(PrefixPtr p,
 	imp->GetTargetCandidates(p,tcands);
 	imp->ConvertTgtCand(tcands,rv,m_outFactorType);
 }
+void PhraseDictionaryTree::
+GetTargetCandidates(PrefixPtr p,
+										std::vector<StringTgtCand>& rv) const 
+{
+	TgtCands tcands;
+	imp->GetTargetCandidates(p,tcands);
+	imp->ConvertTgtCand(tcands,rv);
+}
 
 ////////////////////////////////////////////////////////////
 //
diff --git a/moses/src/PhraseDictionaryTree.h b/moses/src/PhraseDictionaryTree.h
index 3f4831a26..8f2b0a43c 100644
--- a/moses/src/PhraseDictionaryTree.h
+++ b/moses/src/PhraseDictionaryTree.h
@@ -93,6 +93,8 @@ public:
 	// requirement: the pointer has to evaluate to true
 	void GetTargetCandidates(PrefixPtr p,
 													 std::vector<FactorTgtCand>& rv) const;
+	void GetTargetCandidates(PrefixPtr p,
+													 std::vector<StringTgtCand>& rv) const;
 
 	// print target candidates for a given prefix pointer to a stream, mainly 
 	// for debugging
diff --git a/moses/src/PhraseDictionaryTreeAdaptor.cpp b/moses/src/PhraseDictionaryTreeAdaptor.cpp
index efd6d6ce1..5ff1ebabe 100644
--- a/moses/src/PhraseDictionaryTreeAdaptor.cpp
+++ b/moses/src/PhraseDictionaryTreeAdaptor.cpp
@@ -5,6 +5,8 @@
 #include "Phrase.h"
 #include "FactorCollection.h"
 #include "InputFileStream.h"
+#include "Input.h"
+#include "ConfusionNet.h"
 
 inline bool existsFile(const char* filename) {
   struct stat mystat;
@@ -25,10 +27,23 @@ struct PDTAimp {
 	PhraseDictionaryTreeAdaptor *m_obj;
 	int useCache;
 
+	typedef std::vector<TargetPhraseCollection const*> vTPC;
+	std::vector<vTPC> m_rangeCache;
+
+
 	PDTAimp(PhraseDictionaryTreeAdaptor *p) 
 		: m_languageModels(0),m_weightWP(0.0),m_factorCollection(0),m_dict(0),
 			m_obj(p),useCache(1) {}
 
+	void Factors2String(FactorArray const& w,std::string& s) const 
+	{
+		for(size_t j=0;j<m_input.size();++j)
+			{
+				if(s.size()) s+="|";
+				s+=w[m_input[j]]->ToString();
+			}
+	}
+
 	void CleanUp() 
 	{
 		assert(m_dict);
@@ -36,6 +51,7 @@ struct PDTAimp {
 		for(size_t i=0;i<m_tgtColls.size();++i) delete m_tgtColls[i];
 		m_tgtColls.clear();
 		m_cache.clear();
+		m_rangeCache.clear();
 	}
 
 	void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase) 
@@ -75,11 +91,7 @@ struct PDTAimp {
 		std::vector<std::string> srcString(src.GetSize());
 		// convert source Phrase into vector of strings
 		for(size_t i=0;i<srcString.size();++i)
-			for(size_t j=0;j<m_input.size();++j)
-				{
-					if(srcString[i].size()) srcString[i]+="|";
-					srcString[i]+=src.GetFactor(i,m_input[j])->ToString();
-				}
+			Factors2String(src.GetFactorArray(i),srcString[i]);
 
 		// get target phrases in string representation
 		std::vector<StringTgtCand> cands;
@@ -97,16 +109,7 @@ struct PDTAimp {
 				StringTgtCand::first_type const& factorStrings=cands[i].first;
 				StringTgtCand::second_type const& scoreVector=cands[i].second;
 
-				for(size_t k=0;k<factorStrings.size();++k) 
-					{
-						std::vector<std::string> factors=Tokenize(*factorStrings[k],"|");
-						FactorArray& fa=targetPhrase.AddWord();
-						for(size_t l=0;l<m_output.size();++l)
-							fa[m_output[l]]=m_factorCollection->AddFactor(Output, m_output[l], factors[l]);
-					}
-			
-				targetPhrase.SetScore(scoreVector, m_weights, *m_languageModels, m_weightWP);
-	
+				CreateTargetPhrase(targetPhrase,factorStrings,scoreVector);
 				costs.push_back(std::make_pair(targetPhrase.GetFutureScore(),tCands.size()));
 				tCands.push_back(targetPhrase);
 			}
@@ -167,6 +170,145 @@ struct PDTAimp {
 		TRACE_ERR("reading bin ttable\n");
 		m_dict->Read(filePath); 	
 	}
+
+	typedef PhraseDictionaryTree::PrefixPtr PPtr;
+	typedef std::pair<size_t,size_t> Range;
+	struct State {
+		PPtr ptr;
+		Range range;
+		float score;
+
+		State() : range(0,0),score(0.0) {}
+		State(size_t b,size_t e,const PPtr& v,float sc=0.0) : ptr(v),range(b,e),score(sc) {}
+		State(Range const& r,const PPtr& v,float sc=0.0) : ptr(v),range(r),score(sc) {}
+
+		size_t begin() const {return range.first;}
+		size_t end() const {return range.second;}
+		float GetScore() const {return score;}
+
+	};
+
+	void CreateTargetPhrase(TargetPhrase& targetPhrase,StringTgtCand::first_type const& factorStrings,StringTgtCand::second_type const& scoreVector) const
+	{
+
+		for(size_t k=0;k<factorStrings.size();++k) 
+			{
+				std::vector<std::string> factors=Tokenize(*factorStrings[k],"|");
+				FactorArray& fa=targetPhrase.AddWord();
+				for(size_t l=0;l<m_output.size();++l)
+					fa[m_output[l]]=m_factorCollection->AddFactor(Output, m_output[l], factors[l]);
+			}
+			
+		targetPhrase.SetScore(scoreVector, m_weights, *m_languageModels, m_weightWP);
+	
+	}
+
+
+	void CacheSource(ConfusionNet const& src) 
+	{
+		assert(m_dict);
+		std::vector<State> stack;
+		for(size_t i=0;i<src.GetSize();++i) stack.push_back(State(i,i,m_dict->GetRoot()));
+
+		typedef StringTgtCand::first_type sPhrase;
+		typedef std::map<StringTgtCand::first_type,std::pair<float,StringTgtCand::second_type> > E2Costs;
+
+		std::map<Range,E2Costs> cov2cand;
+
+		while(!stack.empty()) 
+			{
+				State curr(stack.back());
+				stack.pop_back();
+		
+				//std::cerr<<"processing state "<<curr<<" stack size: "<<stack.size()<<"\n";
+
+				assert(curr.end()<src.GetSize());
+				const ConfusionNet::Column &currCol=src[curr.end()];
+				for(size_t colidx=0;colidx<currCol.size();++colidx) 
+					{
+						const Word& w=currCol[colidx].first;
+						std::string s;
+						Factors2String(w.GetFactorArray(),s);
+						PPtr nextP=m_dict->Extend(curr.ptr,s);
+
+						if(nextP) 
+							{
+								Range newRange(curr.begin(),curr.end()+1);
+								if(newRange.second<src.GetSize())
+									stack.push_back(State(newRange,nextP,
+																				curr.GetScore()+currCol[colidx].second));
+								
+								std::vector<StringTgtCand> tcands;
+								m_dict->GetTargetCandidates(nextP,tcands);
+
+								if(tcands.size()) 
+									{
+										E2Costs& e2costs=cov2cand[newRange];
+
+										for(size_t i=0;i<tcands.size();++i)
+											{
+												float costs=CalcTranslationScore(tcands[i].second,m_weights);
+												costs-=tcands[i].first.size() * m_weightWP;
+												std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].first,std::make_pair(costs,tcands[i].second)));
+												if(!p.second)
+													{
+														if(p.first->second.first>costs)
+															p.first->second=std::make_pair(costs,tcands[i].second);						
+													}
+											}
+
+
+									}
+							}
+					}
+			} // end while(!stack.empty()) 
+
+		m_rangeCache.resize(src.GetSize(),vTPC(src.GetSize(),0));
+
+		
+		for(std::map<Range,E2Costs>::const_iterator i=cov2cand.begin();i!=cov2cand.end();++i)
+			{
+				assert(i->first.first<m_rangeCache.size());
+				assert(i->first.second>0);
+				assert(i->first.second-1<m_rangeCache[i->first.first].size());
+				assert(m_rangeCache[i->first.first][i->first.second-1]==0);
+
+				std::vector<TargetPhrase> tCands;tCands.reserve(i->second.size());
+				std::vector<std::pair<float,size_t> > costs;costs.reserve(i->second.size());
+
+				for(E2Costs::const_iterator j=i->second.begin();j!=i->second.end();++j)
+					{
+						TargetPhrase targetPhrase(Output, m_obj);
+						CreateTargetPhrase(targetPhrase,j->first,j->second.second);
+						costs.push_back(std::make_pair(targetPhrase.GetFutureScore(),tCands.size()));
+						tCands.push_back(targetPhrase);
+					}
+
+				// prune target candidates and sort according to score
+				std::vector<std::pair<float,size_t> >::iterator nth=costs.end();
+				if(m_obj->m_maxTargetPhrase>0 && costs.size()>m_obj->m_maxTargetPhrase) {
+					nth=costs.begin()+m_obj->m_maxTargetPhrase;
+					std::nth_element(costs.begin(),nth,costs.end(),std::greater<std::pair<float,size_t> >());
+				}
+				std::sort(costs.begin(),nth,std::greater<std::pair<float,size_t> >());
+
+				// convert into TargerPhraseCollection
+				TargetPhraseCollection *rv=new TargetPhraseCollection;
+				for(std::vector<std::pair<float,size_t> >::iterator it=costs.begin();it!=nth;++it) 
+					rv->push_back(tCands[it->second]);
+				
+				if(rv->empty()) 
+					delete rv;
+				else
+					{
+						m_rangeCache[i->first.first][i->first.second-1]=rv;
+						m_tgtColls.push_back(rv);
+					}
+
+			}
+
+
+	}
 };
 
 
@@ -192,6 +334,13 @@ void PhraseDictionaryTreeAdaptor::CleanUp()
 	MyBase::CleanUp();
 }
 
+void PhraseDictionaryTreeAdaptor::InitializeForInput(InputType const& source)
+{
+	// only required for confusion net
+	if(ConfusionNet const* cn=dynamic_cast<ConfusionNet const*>(&source))
+		imp->CacheSource(*cn);
+}
+
 void PhraseDictionaryTreeAdaptor::Create(const std::vector<FactorType> &input
 																				 , const std::vector<FactorType> &output
 																				 , FactorCollection &factorCollection
@@ -224,6 +373,14 @@ PhraseDictionaryTreeAdaptor::GetTargetPhraseCollection(Phrase const &src) const
 {
 	return imp->GetTargetPhraseCollection(src);
 }
+TargetPhraseCollection const* 
+PhraseDictionaryTreeAdaptor::GetTargetPhraseCollection(InputType const& src,WordsRange const &range) const
+{
+	if(imp->m_rangeCache.empty())
+		return imp->GetTargetPhraseCollection(src.GetSubString(range));
+	else
+		return imp->m_rangeCache[range.GetStartPos()][range.GetEndPos()];
+}
 
 void PhraseDictionaryTreeAdaptor::
 SetWeightTransModel(const std::vector<float> &weightT)
diff --git a/moses/src/PhraseDictionaryTreeAdaptor.h b/moses/src/PhraseDictionaryTreeAdaptor.h
index 7e182a6fe..6ecfa6327 100644
--- a/moses/src/PhraseDictionaryTreeAdaptor.h
+++ b/moses/src/PhraseDictionaryTreeAdaptor.h
@@ -8,6 +8,8 @@
 
 class Phrase;
 class PDTAimp;
+class WordsRange;
+class InputType;
 
 class PhraseDictionaryTreeAdaptor : public PhraseDictionaryBase {
 	typedef PhraseDictionaryBase MyBase;
@@ -40,10 +42,14 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionaryBase {
 	// get translation candidates for a given source phrase
 	// returns null pointer if nothing found
 	TargetPhraseCollection const* GetTargetPhraseCollection(Phrase const &src) const;
+	TargetPhraseCollection const* GetTargetPhraseCollection(InputType const& src,WordsRange const & srcRange) const;
 
 	// clean up temporary memory etc.
 	void CleanUp();
-	
+
+
+	void InitializeForInput(InputType const& source);
+
 	// change model scaling factors
 	void SetWeightTransModel(const std::vector<float> &weightT);
 
diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp
index ce10bc957..4fe36abef 100755
--- a/moses/src/Sentence.cpp
+++ b/moses/src/Sentence.cpp
@@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "Sentence.h"
 #include <boost/algorithm/string.hpp>
 #include "PhraseDictionary.h"
+#include "TranslationOptionCollectionText.h"
 
 int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder, FactorCollection &factorCollection) 
 {
@@ -40,3 +41,7 @@ TargetPhraseCollection const* Sentence::CreateTargetPhraseCollection(PhraseDicti
 	Phrase src=GetSubString(r);
 	return d.GetTargetPhraseCollection(src);
 }
+TranslationOptionCollection* Sentence::CreateTranslationOptionCollection() const 
+{
+	return new TranslationOptionCollectionText(*this);
+}
diff --git a/moses/src/Sentence.h b/moses/src/Sentence.h
index f8bb54985..b0fad09c3 100755
--- a/moses/src/Sentence.h
+++ b/moses/src/Sentence.h
@@ -61,5 +61,6 @@ class Sentence : public Phrase, public InputType
 	int Read(std::istream& in,const std::vector<FactorType>& factorOrder, FactorCollection &factorCollection);
 
 	TargetPhraseCollection const* CreateTargetPhraseCollection(PhraseDictionaryBase const& d,const WordsRange& r) const;
+	TranslationOptionCollection* CreateTranslationOptionCollection() const;
 };
 
diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp
index 3e2adf4c5..8471efd7e 100755
--- a/moses/src/StaticData.cpp
+++ b/moses/src/StaticData.cpp
@@ -502,3 +502,8 @@ void StaticData::CleanUpAfterSentenceProcessing()
 	for(size_t i=0;i<m_generationDictionary.size();++i)
 		m_generationDictionary[i]->CleanUp();
 }
+void StaticData::InitializeBeforeSentenceProcessing(InputType const& in) 
+{
+	for(size_t i=0;i<m_phraseDictionary.size();++i)
+		m_phraseDictionary[i]->InitializeForInput(in);
+}
diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h
index a52e820a6..aabb239be 100755
--- a/moses/src/StaticData.h
+++ b/moses/src/StaticData.h
@@ -35,6 +35,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "DecodeStep.h"
 //#include "UnknownWordHandler.h"
 
+class InputType;
+
 class StaticData
 {
 protected:	
@@ -218,6 +220,7 @@ public:
 	void SetWeightGeneration(const std::vector<float> &weight);
 	int GetInputType() const {return m_inputType;}
 
+	void InitializeBeforeSentenceProcessing(InputType const&);
 	void CleanUpAfterSentenceProcessing();
 
 };
diff --git a/moses/src/TranslationOptionCollection.cpp b/moses/src/TranslationOptionCollection.cpp
index 3f7d7f4ca..a57037cf1 100644
--- a/moses/src/TranslationOptionCollection.cpp
+++ b/moses/src/TranslationOptionCollection.cpp
@@ -63,7 +63,10 @@ CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
 		for (size_t endPos = startPos ; endPos < m_source.GetSize() ; ++endPos)
 			{
 				WordsRange wordsRange(startPos, endPos);
-				const TargetPhraseCollection *phraseColl= m_source.CreateTargetPhraseCollection(dictionary,wordsRange);
+
+				const TargetPhraseCollection *phraseColl= dictionary.GetTargetPhraseCollection(m_source,wordsRange);
+				//				const TargetPhraseCollection *phraseColl= m_source.CreateTargetPhraseCollection(dictionary,wordsRange);
+
 				mTPC[startPos][endPos]=phraseColl;
 				if (phraseColl != NULL)
 					{
@@ -96,9 +99,12 @@ CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
 					}
 			}
 
+	ComputeFutureScores(verboseLevel);
+}
 	  
-	  
-	  
+void TranslationOptionCollection::ComputeFutureScores(size_t verboseLevel) 
+{
+
 #if 1
 
   // create future score matrix
diff --git a/moses/src/TranslationOptionCollection.h b/moses/src/TranslationOptionCollection.h
index f82c381a4..c27c18288 100755
--- a/moses/src/TranslationOptionCollection.h
+++ b/moses/src/TranslationOptionCollection.h
@@ -46,14 +46,14 @@ protected:
 public:
   virtual ~TranslationOptionCollection();
 
-	void CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
-																, const LMList &languageModels  														
-																, const LMList &allLM
-																, FactorCollection &factorCollection
-																, float weightWordPenalty
-																, bool dropUnknown
-																, size_t verboseLevel);
-
+	virtual void CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
+																				, const LMList &languageModels  														
+																				, const LMList &allLM
+																				, FactorCollection &factorCollection
+																				, float weightWordPenalty
+																				, bool dropUnknown
+																				, size_t verboseLevel);
+	
 	// get length/size of source input
 	size_t GetSize() const;
 
@@ -77,6 +77,8 @@ public:
 																bool dropUnknown,
 																float weightWordPenalty
 																) =0; 
+
+	void ComputeFutureScores(size_t verboseLevel);
 };
 
 inline std::ostream& operator<<(std::ostream& out, const TranslationOptionCollection& coll)
diff --git a/moses/src/TranslationOptionCollectionConfusionNet.cpp b/moses/src/TranslationOptionCollectionConfusionNet.cpp
index 0224b7435..8ce8f45f4 100644
--- a/moses/src/TranslationOptionCollectionConfusionNet.cpp
+++ b/moses/src/TranslationOptionCollectionConfusionNet.cpp
@@ -9,102 +9,29 @@
 TranslationOptionCollectionConfusionNet::
 TranslationOptionCollectionConfusionNet(const ConfusionNet &input) 
 	: TranslationOptionCollection(input) {}
-#if 0
-size_t TranslationOptionCollectionConfusionNet::GetSourceSize() const 
-{
-	return m_source.GetSize();
-}
 
 void TranslationOptionCollectionConfusionNet::
 CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
-												 , const LMList & //languageModels
-												 , const LMList & //allLM
-												 , FactorCollection & //factorCollection
-												 , float //weightWordPenalty
-												 , bool //dropUnknown
-												 , size_t verboseLevel)
+												 , const LMList &languageModels  														
+												 , const LMList &allLM
+												 , FactorCollection &factorCollection
+												 , float weightWordPenalty
+												 , bool dropUnknown
+												 , size_t verboseLevel) 
 {
-  // loop over all substrings of the source sentence, look them up
-  // in the phraseDictionary (which is the- possibly filtered-- phrase
-  // table loaded on initialization), generate TranslationOption objects
-  // for all phrases
-  //
-  // possible optimization- don't consider phrases longer than the longest
-  // phrase in the PhraseDictionary?
-  
-  Dictionary *dictionary = decodeStepList.front().GetDictionaryPtr();
-  
-  typedef std::vector<TargetPhraseCollection const*> vTPC;
-  std::vector<vTPC> mTPC(m_source.GetSize(),vTPC(m_source.GetSize(),0));
-  size_t maxLen=0;
+#if 0
+	ConfusionNet const& source(dynamic_cast<ConfusionNet&>(m_source));
+	assert(dynamic_cast<PhraseDictionaryTreeAdaptor const*>(decodeStepList.front().GetDictionaryPtr()));
+	PhraseDictionaryTreeAdaptor const& pdict
+		=dynamic_cast<PhraseDictionaryTreeAdaptor const&>(*decodeStepList.front().GetDictionaryPtr());
 
-  for (size_t startPos = 0 ; startPos < m_source.GetSize() ; ++startPos)
-		for (size_t endPos = startPos ; endPos < m_source.GetSize() ; ++endPos)
-			{
-				WordsRange wordsRange(startPos, endPos);
-				const TargetPhraseCollection *phraseColl= 
-					CreateTargetPhraseCollection(dictionary,&m_source,wordsRange);
-				mTPC[startPos][endPos]=phraseColl;
-				if (phraseColl != NULL)
-					{
-						maxLen=std::max(endPos-startPos+1,maxLen);
-						if (verboseLevel >= 3) 
-							std::cout << "[" << m_source.GetSubString(wordsRange) << "; " 
-												<< startPos << "-" << endPos << "]\n";
-		
-						for(TargetPhraseCollection::const_iterator iterTargetPhrase=phraseColl->begin(); 
-								iterTargetPhrase != phraseColl->end() ; ++iterTargetPhrase)
-							{
-								TranslationOption transOpt(wordsRange, *iterTargetPhrase);
-								push_back(transOpt);
-								if (verboseLevel >= 3) std::cout << "\t" << transOpt << "\n";
-							}
-						if (verboseLevel >= 3) std::cout << std::endl;
-					}
-				else if (wordsRange.GetWordsCount() == 1)
-					{
-						// drop unk
-						m_initialCoverage.SetValue(startPos, startPos,1); 
-					}
-			}
 
-	  
-	  
-	  
-	  
-	// create future score matrix
-	// for each span in the source phrase (denoted by start and end)
+	std::vector<State> stack;
+	for(size_t i=0;i<src.GetSize();++i) stack.push_back(State(i,i,data.GetRoot()));
 
-	  
-	// init future costs
-	for(size_t endPos=0; endPos<m_source.GetSize(); ++endPos) 
-		for(size_t startPos=0; startPos<=endPos; ++startPos)   
-			{
-				float currScore=-std::numeric_limits<float>().infinity();
-				if(TargetPhraseCollection const *p=mTPC[startPos][endPos]) 
-					for(TargetPhraseCollection::const_iterator i=p->begin();i!=p->end();++i)
-						currScore=std::max(currScore,i->GetFutureScore());
-				m_futureScore.SetScore(startPos,endPos,currScore);
-			}
-		  
-	// solve DP recursion, similar to CYK parsing
-	for(size_t len=1;len<maxLen;++len)
-		for(size_t startPos=0; startPos<m_source.GetSize()-len; ++startPos)
-			{
-			  size_t endPos=startPos+len;
-			  float currScore=m_futureScore.GetScore(startPos,endPos);
-			  for(size_t k=startPos;k<endPos;++k)
-					currScore=std::max(currScore,
-														 m_futureScore.GetScore(startPos,k) 
-														 + m_futureScore.GetScore(k+1,endPos));
-				m_futureScore.SetScore(startPos,endPos,currScore);
 
-				if(verboseLevel > 0) 
-					std::cout<<"future cost from "<<startPos<<" to "<<endPos<<" is "
-									 <<m_futureScore.GetScore(startPos,endPos)<<std::endl;
-			}
-			  
-			  
-}
 
+#else
+	return TranslationOptionCollection::CreateTranslationOptions(decodeStepList,languageModels,allLM,factorCollection,weightWordPenalty,dropUnknown,verboseLevel);
 #endif
+}
diff --git a/moses/src/TranslationOptionCollectionConfusionNet.h b/moses/src/TranslationOptionCollectionConfusionNet.h
index af512d953..88d0d761d 100644
--- a/moses/src/TranslationOptionCollectionConfusionNet.h
+++ b/moses/src/TranslationOptionCollectionConfusionNet.h
@@ -17,7 +17,13 @@ class TranslationOptionCollectionConfusionNet : public TranslationOptionCollecti
 												,float //weightWordPenalty
 											 ) {return 0;}
 
-
+	void CreateTranslationOptions(const std::list < DecodeStep > &decodeStepList
+																, const LMList &languageModels  														
+																, const LMList &allLM
+																, FactorCollection &factorCollection
+																, float weightWordPenalty
+																, bool dropUnknown
+																, size_t verboseLevel);
 
 };
 #endif
diff --git a/moses/src/TranslationOptionCollectionText.cpp b/moses/src/TranslationOptionCollectionText.cpp
index 5c09e7bc6..e2a61f3dd 100644
--- a/moses/src/TranslationOptionCollectionText.cpp
+++ b/moses/src/TranslationOptionCollectionText.cpp
@@ -65,7 +65,7 @@ int TranslationOptionCollectionText::HandleUnkownWord(PhraseDictionaryBase& phra
 			targetPhraseOrig.SetScore(allLM, weightWordPenalty);
 		  
 			phraseDictionary.AddEquivPhrase(sourcePhrase, targetPhraseOrig);
-			const TargetPhraseCollection *phraseColl = phraseDictionary.GetTargetPhraseCollection(sourcePhrase); //FindEquivPhrase(sourcePhrase);
+			const TargetPhraseCollection *phraseColl = phraseDictionary.GetTargetPhraseCollection(sourcePhrase); 
 			assert(phraseColl);
 			const TargetPhrase &targetPhrase = *phraseColl->begin();