Workingandre_lynum

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/andre_lynum@1851 1f5c12ca-751b-0410-a591-d2e778427230

14 files changed, 313 insertions, 36 deletions

diff --git a/misc/.project b/misc/.project
index bdbea2cc5..79490ad84 100644
--- a/misc/.project
+++ b/misc/.project
@@ -9,13 +9,76 @@
 	<buildSpec>
 		<buildCommand>
 			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>?name?</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
+					<value>false</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildLocation</key>
+					<value>${workspace_loc:/misc/Debug}</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableFullBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
+					<value>clean</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.append_environment</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.contents</key>
+					<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildArguments</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildCommand</key>
+					<value>make</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.stopOnError</key>
+					<value>true</value>
+				</dictionary>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
 			<arguments>
 			</arguments>
 		</buildCommand>
 	</buildSpec>
 	<natures>
 		<nature>org.eclipse.cdt.core.cnature</nature>
-		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
 		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
 	</natures>
 </projectDescription>
diff --git a/moses-cmd/.project b/moses-cmd/.project
index d53cf546e..e17a15de8 100644
--- a/moses-cmd/.project
+++ b/moses-cmd/.project
@@ -9,13 +9,76 @@
 	<buildSpec>
 		<buildCommand>
 			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>?name?</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
+					<value>false</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildLocation</key>
+					<value>${workspace_loc:/moses-cmd/Debug}</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableFullBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
+					<value>clean</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.append_environment</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.contents</key>
+					<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildArguments</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildCommand</key>
+					<value>make</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.stopOnError</key>
+					<value>true</value>
+				</dictionary>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
 			<arguments>
 			</arguments>
 		</buildCommand>
 	</buildSpec>
 	<natures>
 		<nature>org.eclipse.cdt.core.cnature</nature>
-		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
 		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
 	</natures>
 </projectDescription>
diff --git a/moses-cmd/.settings/org.eclipse.cdt.core.prefs b/moses-cmd/.settings/org.eclipse.cdt.core.prefs
index 3e75ea7ec..162468540 100644
--- a/moses-cmd/.settings/org.eclipse.cdt.core.prefs
+++ b/moses-cmd/.settings/org.eclipse.cdt.core.prefs
@@ -1,3 +1,4 @@
-#Tue Nov 21 19:58:08 GMT 2006
+#Tue May 20 14:06:50 CEST 2008
 eclipse.preferences.version=1
+indexer/indexerId=org.eclipse.cdt.core.nullindexer
 indexerId=org.eclipse.cdt.core.nullindexer
diff --git a/moses-cmd/src/IOStream.cpp b/moses-cmd/src/IOStream.cpp
index 2cbf45b5c..14dc7e401 100755
--- a/moses-cmd/src/IOStream.cpp
+++ b/moses-cmd/src/IOStream.cpp
@@ -246,15 +246,18 @@ void IOStream::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, bo
 // SCORER start
 void IOStream::OutputScore(const Hypothesis *hypo) {
 	Phrase *trans = StaticData::Instance().GetTranslatedPhrase();
+	size_t unkTransCount = trans->GetSize() - hypo->GetTotalTargetSize();
+	float penalizedScore = hypo->GetTotalScore() +
+		(StaticData::Instance().GetScorerWordPenalty() * (unkTransCount + hypo->SourceWordsNotCovered()));
 	VERBOSE(1, "Source: " << hypo->GetSourcePhrase() << endl);
 	VERBOSE(1, "Translation: " << *trans << endl);
 	VERBOSE(1, "Hypothesis: " << *hypo << endl);
 	VERBOSE(1, "Hypothesis score: " << hypo->GetTotalScore() << endl);
-	VERBOSE(1, "Penalized score:  " << hypo->GetPenalizedScore() << endl);
+	VERBOSE(1, "Penalized score:  " << penalizedScore << endl);
 	VERBOSE(1, "Source unknown word count: " << hypo->SourceWordsNotCovered() << endl);
-	VERBOSE(1, "Translation unknown word count: " << trans->GetSize() - hypo->GetSize() << endl);
-	cout << hypo->GetPenalizedScore() << " " << hypo->GetTotalScore() << " "
-			 << hypo->SourceWordsNotCovered() << " " << trans->GetSize() - hypo->GetSize() << endl ;
+	VERBOSE(1, "Translation unknown word count: " << unkTransCount << endl);
+	cout << penalizedScore << " " << hypo->GetTotalScore() << " "
+			 << hypo->SourceWordsNotCovered() << " " << unkTransCount << endl ;
 }
 // SCORER end
 
diff --git a/moses-cmd/src/Main.cpp b/moses-cmd/src/Main.cpp
index d45aa8b08..21a478253 100644
--- a/moses-cmd/src/Main.cpp
+++ b/moses-cmd/src/Main.cpp
@@ -143,6 +143,7 @@ int main(int argc, char* argv[])
 			Phrase *transPhrase = new Phrase(trans->GetSubString(WordsRange(0, transSize)));
 			delete staticData.GetTranslatedPhrase();
 			staticData.SetTranslatedPhrase(transPhrase);
+			staticData.SetTranslationCoverage(0);
 		}
 
 		delete trans;
diff --git a/moses/.project b/moses/.project
index bad551be6..f49910362 100755
--- a/moses/.project
+++ b/moses/.project
@@ -7,13 +7,76 @@
 	<buildSpec>
 		<buildCommand>
 			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>?name?</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
+					<value>false</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildLocation</key>
+					<value>${workspace_loc:/moses/Debug}</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableFullBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
+					<value>clean</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.append_environment</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.contents</key>
+					<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
+					<value>true</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildArguments</key>
+					<value></value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.buildCommand</key>
+					<value>make</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
+					<value>all</value>
+				</dictionary>
+				<dictionary>
+					<key>org.eclipse.cdt.make.core.stopOnError</key>
+					<value>true</value>
+				</dictionary>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
 			<arguments>
 			</arguments>
 		</buildCommand>
 	</buildSpec>
 	<natures>
 		<nature>org.eclipse.cdt.core.cnature</nature>
-		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
 		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
 	</natures>
 </projectDescription>
diff --git a/moses/.settings/org.eclipse.cdt.core.prefs b/moses/.settings/org.eclipse.cdt.core.prefs
index 98c95a1cd..162468540 100644
--- a/moses/.settings/org.eclipse.cdt.core.prefs
+++ b/moses/.settings/org.eclipse.cdt.core.prefs
@@ -1,3 +1,4 @@
-#Tue Nov 14 23:21:36 GMT 2006
+#Tue May 20 14:06:50 CEST 2008
 eclipse.preferences.version=1
+indexer/indexerId=org.eclipse.cdt.core.nullindexer
 indexerId=org.eclipse.cdt.core.nullindexer
diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp
index 9d3f648e6..1135c13ff 100755
--- a/moses/src/Hypothesis.cpp
+++ b/moses/src/Hypothesis.cpp
@@ -58,7 +58,10 @@ Hypothesis::Hypothesis(InputType const& source, const TargetPhrase &emptyTarget)
 	, m_arcList(NULL)
 	, m_id(0)
 	, m_lmstats(NULL)
+		// SCORER start
 	, m_penalizedScore(0)
+	, m_skippedTransWords(0)
+		// SCORER end
 {	// used for initial seeding of trans process	
 	// initialize scores
 	//_hash_computed = false;
@@ -87,7 +90,10 @@ Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &tran
 	, m_arcList(NULL)
 	, m_id(s_HypothesesCreated++)
 	, m_lmstats(NULL)
+		// SCORER start
 	, m_penalizedScore(0)
+	, m_skippedTransWords(0)
+		// SCORER end
 {
 	// assert that we are not extending our hypothesis by retranslating something
 	// that this hypothesis has already translated!
@@ -328,13 +334,19 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore)
 	CalcDistortionScore();
 	
 	// LANGUAGE MODEL COST
-	CalcLMScore(staticData.GetAllLM());
+	// SCORER start
+	if (!staticData.GetScoreFlag())
+		CalcLMScore(staticData.GetAllLM());
+	// SCORER end
 
 	// WORD PENALTY
 	m_scoreBreakdown.PlusEquals(staticData.GetWordPenaltyProducer(), - (float) m_currTargetWordsRange.GetNumWordsCovered()); 
 
 	// FUTURE COST
-	CalcFutureScore(futureScore);
+	// SCORER start
+	if (!staticData.GetScoreFlag())
+		CalcFutureScore(futureScore);
+	// SCORER end
 
 	
 	//LEXICAL REORDERING COST
@@ -528,23 +540,27 @@ const ScoreComponentCollection &Hypothesis::GetCachedReorderingScore() const
 // within the output positions covered by the hypothesis
 bool Hypothesis::CompareHypothesisToPhrase(const Phrase *inputPhrase, size_t wordsSoFar) const
 {
+	WordsRange targetRange = WordsRange(m_currTargetWordsRange.GetStartPos(),
+																		 m_currTargetWordsRange.GetEndPos() - m_skippedTransWords);
 	// Check that the hypothesis chain doesn't cover more words than is in the phrase
-	if ((wordsSoFar + m_targetPhrase.GetSize()) > inputPhrase->GetSize())
+	if ((wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords) > inputPhrase->GetSize())
 		return false;
   // check for compatibility or if we're in the root hypothesis (better test for this ?)
-  if ((m_targetPhrase.GetSize() == 0) || inputPhrase->GetSubString(m_currTargetWordsRange).IsCompatible(m_targetPhrase)) {
+  if ((m_targetPhrase.GetSize() == 0) ||
+			inputPhrase->GetSubString(targetRange).IsCompatible(m_targetPhrase)) {
     // should follow the arclist too ???
     if (m_prevHypo == NULL)
       return true;
 
-    else if ((m_prevHypo != NULL) && m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
+    else if ((m_prevHypo != NULL) &&
+						 m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
       return true;
     
     else if ((m_arcList != NULL) && m_arcList->size() > 0) {
       for (ArcList::const_iterator it = m_arcList->begin(); it != m_arcList->end(); it++) {
 				Hypothesis *hypo = *it;
 
-				if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
+				if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
 					return true;
       }
     }
@@ -577,4 +593,18 @@ size_t Hypothesis::SourceWordsNotCovered() const {
   return GetWordsBitmap().GetSize() - GetWordsBitmap().GetNumWordsCovered();
 }
 
+size_t Hypothesis::GetAllSkippedWords() const {
+	if (m_prevHypo == NULL)
+		return m_skippedTransWords;
+	else
+		return m_skippedTransWords + m_prevHypo->GetAllSkippedWords();
+}
+
+size_t Hypothesis::GetTotalTargetSize() const {
+	if (m_prevHypo == NULL)
+		return GetCurrTargetLength();
+	else
+		return GetCurrTargetLength() - m_skippedTransWords + m_prevHypo->GetTotalTargetSize();
+}
+
 // SCORER end
diff --git a/moses/src/Hypothesis.h b/moses/src/Hypothesis.h
index e205da650..09656ffa2 100755
--- a/moses/src/Hypothesis.h
+++ b/moses/src/Hypothesis.h
@@ -69,7 +69,7 @@ protected:
 	//it's a confusion network in the end???
 	InputType const&  m_sourceInput;
 	WordsRange				m_currSourceWordsRange; /**< source word positions of the last phrase that was used to create this hypothesis */
-	WordsRange        m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
+	mutable WordsRange        m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
   bool							m_wordDeleted;
 	float							m_totalScore;  /**< score so far */
 	float							m_futureScore; /**< estimated future cost to translate rest of sentence */
@@ -81,6 +81,7 @@ protected:
 
 	// SCORER start
 	float m_penalizedScore;
+	mutable size_t m_skippedTransWords;
 	// SCORER end
 
 	int m_id; /**< numeric ID of this hypothesis, used for logging */
@@ -270,6 +271,14 @@ public:
 	float GetPenalizedScore() const {
 	  return m_penalizedScore;
 	}
+	// TODO release old WordsRange instance ???
+	void SetCurrTargetWordsRange(WordsRange range) const { m_currTargetWordsRange = range; }
+
+	size_t GetSkippedTransWords() const { return m_skippedTransWords; }
+	void SetSkippedTransWords(size_t val) const { m_skippedTransWords = val; }
+	void IncrSkippedTransWords() const { m_skippedTransWords++; }
+	size_t GetAllSkippedWords() const;
+	size_t GetTotalTargetSize() const;
 	// SCORER end
 };
 
diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp
index d89fc16d3..859d83323 100755
--- a/moses/src/Manager.cpp
+++ b/moses/src/Manager.cpp
@@ -109,7 +109,7 @@ void Manager::ProcessSentence()
 
 		// the stack is pruned before processing (lazy pruning):
 		VERBOSE(3,"processing hypothesis from next stack");
-	        // VERBOSE("processing next stack at ");
+		// VERBOSE("processing next stack at ");
 		sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
 		VERBOSE(3,std::endl);
 		sourceHypoColl.CleanupArcList();
@@ -122,6 +122,7 @@ void Manager::ProcessSentence()
 			}
 		// some logging
 		IFVERBOSE(2) { OutputHypoStackSize(); }
+
 	}
 
 	// some more logging
@@ -138,6 +139,7 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
 	// since we check for reordering limits, its good to have that limit handy
 	int maxDistortion = StaticData::Instance().GetMaxDistortion();
 	bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;
+	bool expanded = false;
 
 	// no limit of reordering: only check for overlap
 	if (maxDistortion < 0)
@@ -148,20 +150,37 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
 
 		for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
 		{
-      size_t maxSize = sourceSize - startPos;
-      size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
-      maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
+			size_t maxSize = sourceSize - startPos;
+			size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
+			maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
 
 			for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
 			{
 				if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
 				{
-					ExpandAllHypotheses(hypothesis
-												, m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos)));
+					
+					if (ExpandAllHypotheses(hypothesis,
+																	m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos))))
+						expanded = true;
 				}
 			}
 		}
 
+		if (!expanded) {
+			WordsRange oldRange = hypothesis.GetCurrTargetWordsRange();
+			size_t newEnd = oldRange.GetEndPos() + 1;
+			const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
+
+			if (newEnd < transPhrase->GetSize() - 1) { // problem here
+				// cout << "Skipping word at " <<  oldRange.GetEndPos() << " continuing from " << newEnd << endl;
+
+				hypothesis.SetCurrTargetWordsRange(WordsRange(oldRange.GetStartPos(),	newEnd));
+				hypothesis.IncrSkippedTransWords();
+				
+				ProcessOneHypothesis(hypothesis);
+			}
+		}
+
 		return; // done with special case (no reordering limit)
 	}
 
@@ -274,13 +293,18 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
  * \param transOptList list of translation options to be applied
  */
 
-void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
+bool Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
 {
 	TranslationOptionList::const_iterator iter;
+	bool expanded = false;
+	
 	for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
 	{
-		ExpandHypothesis(hypothesis, **iter);
+		if (ExpandHypothesis(hypothesis, **iter))
+			expanded = true;
 	}
+
+	return expanded;
 }
 
 /**
@@ -290,7 +314,8 @@ void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const Translation
  * \param transOpt translation option (phrase translation) 
  *        that is applied to create the new hypothesis
  */
-void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt) 
+// SCORER start
+bool Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt) 
 {
 	// create hypothesis and calculate all its scores
 #ifdef DEBUGLATTICE
@@ -310,20 +335,24 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp
 	// add to hypothesis stack
 	size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
 
-	// SCORER
 	// Only add hypothesises that agree with translation
 	if (SCORER_FILTER_HYPOS && StaticData::Instance().GetScoreFlag()) {
 	  const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
 		
-	  if (newHypo->CompareHypothesisToPhrase(transPhrase, 0))
+	  if (newHypo->CompareHypothesisToPhrase(transPhrase, 0)) {
 	    m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
+			return true;
+		}
 		else {
 			VERBOSE(3, "Not Compatible with translation." << endl);
 			FREEHYPO(newHypo);
+			return false;
 		}
 	}
-	else
+	else {
 	  m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
+		return true;
+	}
 	// SCORER end
 }
 
@@ -336,12 +365,13 @@ const Hypothesis *Manager::GetBestHypothesis() const
 	// SCORER start
 	// search all hypothesis stacks
 	const Hypothesis* bestHypo = NULL;
-
+	int i = 0;
+	
 	if (StaticData::Instance().GetScoreFlag()) {
 		// having trouble getting reverse iterators to work
 		std::vector<HypothesisStack>::const_iterator it = m_hypoStackColl.end();
 		it--;
-		for (;; it--) {
+		for (;;) {
 			const HypothesisStack &hypoColl = *it;
 			
 			if ((hypoColl.size() > 0) && (hypoColl.size() < 327685)) {// what constant is this ? 
@@ -349,12 +379,17 @@ const Hypothesis *Manager::GetBestHypothesis() const
 				if (bestHypo == NULL)
 					bestHypo = hypo;
 				else
-					if ((hypo->GetTotalScore() < 0.0) && (hypo->GetTotalScore() > bestHypo->GetTotalScore()))
+					if ((hypo->GetTotalScore() < 0.0) && 
+					    (hypo->GetTotalScore() > bestHypo->GetTotalScore()))
 						bestHypo = hypo;
 			}
 
-			if (it == m_hypoStackColl.begin()) break;
+			if ((it == m_hypoStackColl.begin()) || (bestHypo != NULL)) break;
+			i++;
+			it--;
 		}
+		
+		// cout << "Best hypo " << i << " from end." << endl;
 
 		return bestHypo;
 	}
@@ -505,3 +540,4 @@ void Manager::CalcDecoderStatistics() const
     }
   }
 }
+
diff --git a/moses/src/Manager.h b/moses/src/Manager.h
index c4f95b919..92742912f 100755
--- a/moses/src/Manager.h
+++ b/moses/src/Manager.h
@@ -87,8 +87,8 @@ protected:
 	
 	// functions for creating hypotheses
 	void ProcessOneHypothesis(const Hypothesis &hypothesis);
-	void ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
-	void ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
+	bool ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
+	bool ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
 
 	// logging
 	void OutputHypoStack(int stack = -1);
@@ -105,4 +105,5 @@ public:
 	 * to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
 	 */
 	void CalcDecoderStatistics() const;
+
 };
diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp
index e672d7c70..6f1ae35ce 100755
--- a/moses/src/StaticData.cpp
+++ b/moses/src/StaticData.cpp
@@ -170,7 +170,8 @@ bool StaticData::LoadData(Parameter *parameter)
 	  // the translation options are filtered
 	if (m_score) {
 	  m_useTransOptCache = false;
-		// m_isAlwaysCreateDirectTranslationOption = true;
+		m_isAlwaysCreateDirectTranslationOption = false;
+		m_maxDistortion = -1;
 	}
 	if (m_parameter->GetParam("scorePenalty").size() == 1) {
 		m_factorDelimiter = Scan<float>(m_parameter->GetParam("scorePenalty")[0]);
diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h
index 6efb676f0..e1de70557 100755
--- a/moses/src/StaticData.h
+++ b/moses/src/StaticData.h
@@ -57,6 +57,7 @@ protected:
 	mutable Phrase *m_translatedPhrase;
 	// unaccounted word penalty
 	float m_scorerWordPenalty;
+	mutable int m_translationCoverage;
 	// SCORER end
 	std::vector<PhraseDictionary*>	m_phraseDictionary;
 	std::vector<GenerationDictionary*>	m_generationDictionary;
@@ -387,5 +388,8 @@ public:
 	// Accessor for the scorer flag
 	const bool GetScoreFlag() const { return m_score; }
 	float GetScorerWordPenalty() const { return m_scorerWordPenalty; }
+	int GetTranslationCoverage() const { return m_translationCoverage; }
+	void SetTranslationCoverage(int x) const { m_translationCoverage = x; }
+	int AddTranslationCoverage(int x) const { return m_translationCoverage += x; }
 	// SCORER end
 };
diff --git a/moses/src/TranslationOptionCollection.cpp b/moses/src/TranslationOptionCollection.cpp
index 6799aca00..ebc3d1471 100644
--- a/moses/src/TranslationOptionCollection.cpp
+++ b/moses/src/TranslationOptionCollection.cpp
@@ -164,7 +164,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
 				}
 		}
 	}
-		
+	/* SCORER need to disable this
 	bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
 	// create unknown words for 1 word coverage where we don't have any trans options
 	for (size_t pos = 0 ; pos < size ; ++pos)
@@ -173,6 +173,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
 		if (fullList.size() == 0 || alwaysCreateDirectTranslationOption)
 			ProcessUnknownWord(pos);
 	}
+	*/
 }
 
 /** special handling of ONE unknown words. Either add temporarily add word to translation table,