roll back last change. seg fault in release build

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@1549 1f5c12ca-751b-0410-a591-d2e778427230

11 files changed, 39 insertions, 117 deletions

diff --git a/moses-cmd/moses-cmd.vcproj b/moses-cmd/moses-cmd.vcproj
index 82fb54950..ef560a5f4 100644
--- a/moses-cmd/moses-cmd.vcproj
+++ b/moses-cmd/moses-cmd.vcproj
@@ -41,7 +41,7 @@
 				Name="VCCLCompilerTool"

 				Optimization="0"

 				AdditionalIncludeDirectories=""$(SolutionDir)\src";"$(SolutionDir)..\mysqlpp\lib";"$(SolutionDir)..\irstlm\src";"$(SolutionDir)..\srilm\src""

-				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;LM_INTERNAL;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE"

+				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;LM_INTERNAL;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE;EXIT_RETURN"

 				MinimalRebuild="true"

 				BasicRuntimeChecks="3"

 				RuntimeLibrary="3"

diff --git a/moses-cmd/src/Main.cpp b/moses-cmd/src/Main.cpp
index d9f14ce32..41bc79c33 100644
--- a/moses-cmd/src/Main.cpp
+++ b/moses-cmd/src/Main.cpp
@@ -195,12 +195,12 @@ int main(int argc, char* argv[])
 	IFVERBOSE(1)
 		PrintUserTime("End.");
 
-	//#ifdef HACK_EXIT
+	#ifndef EXIT_RETURN
 	//This avoids that detructors are called (it can take a long time)
 		exit(EXIT_SUCCESS);
-	//#else
-	//	return EXIT_SUCCESS;
-	//#endif
+	#else
+		return EXIT_SUCCESS;
+	#endif
 }
 
 IOStream *GetIODevice(const StaticData &staticData)
diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp
index 476af95a8..c5416e939 100755
--- a/moses/src/Hypothesis.cpp
+++ b/moses/src/Hypothesis.cpp
@@ -185,8 +185,10 @@ int Hypothesis::NGramCompare(const Hypothesis &compare) const
 	// 0	= this ==compare
 	if (m_languageModelStates < compare.m_languageModelStates) return -1;
 	if (m_languageModelStates > compare.m_languageModelStates) return 1;
-	if (m_sourceCompleted.GetCompressedRepresentation() < compare.m_sourceCompleted.GetCompressedRepresentation()) return -1;
-	if (m_sourceCompleted.GetCompressedRepresentation() > compare.m_sourceCompleted.GetCompressedRepresentation()) return 1;
+
+	int compareBitmap = m_sourceCompleted.Compare(compare.m_sourceCompleted);
+	if (compareBitmap != 0)
+		return compareBitmap;
 	if (m_currSourceWordsRange.GetEndPos() < compare.m_currSourceWordsRange.GetEndPos()) return -1;
 	if (m_currSourceWordsRange.GetEndPos() > compare.m_currSourceWordsRange.GetEndPos()) return 1;
 	if (! StaticData::Instance().GetSourceStartPosMattersForRecombination()) return 0;
diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp
index f077f5173..8b9ef579e 100755
--- a/moses/src/Manager.cpp
+++ b/moses/src/Manager.cpp
@@ -147,10 +147,10 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
 
 		for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
 		{
-			size_t maxSize = sourceSize - startPos;
-			size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
-			maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
-			
+      size_t maxSize = sourceSize - startPos;
+      size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
+      maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
+
 			for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
 			{
 				if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
@@ -296,20 +296,6 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp
 	if (debug2) { std::cerr << "::EXT: " << transOpt << "\n"; }
 #endif
 	Hypothesis *newHypo = hypothesis.CreateNext(transOpt);
-	// expand hypothesis further if transOpt was linked
-	for (std::vector<TranslationOption*>::const_iterator iterLinked = transOpt.GetLinkedTransOpts().begin();
-	       iterLinked != transOpt.GetLinkedTransOpts().end(); iterLinked++) {
-		const WordsBitmap hypoBitmap = newHypo->GetWordsBitmap();
-		if (hypoBitmap.Overlap((**iterLinked).GetSourceWordsRange())) {
-			// don't want to add a hypothesis that has some but not all of a linked TO set, so return
-			return;
-		}
-		else
-		{
-			newHypo->CalcScore(m_transOptColl->GetFutureScore());
-			newHypo = newHypo->CreateNext(**iterLinked);
-		}
-	}
 	newHypo->CalcScore(m_transOptColl->GetFutureScore());
 	
 	// logging for the curious
diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp
index 5a2f7b2be..a8a720ebc 100755
--- a/moses/src/Sentence.cpp
+++ b/moses/src/Sentence.cpp
@@ -42,7 +42,7 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
 	//parse XML markup in translation line
 	const StaticData &staticData = StaticData::Instance();
 	if (staticData.GetXmlInputType() != XmlPassThrough)
-		m_xmlOptionsList = ProcessAndStripXMLTags(line, *this);
+		m_xmlOptionsList = ProcessAndStripXMLTags(line);
 	Phrase::CreateFromString(factorOrder, line, factorDelimiter);
 	
 	//only fill the vector if we are parsing XML
@@ -50,10 +50,9 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
 		for (size_t i=0; i<GetSize();i++) {
 			m_xmlCoverageMap.push_back(false);
 		}
-		for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin();
-		        iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) {
+		for (size_t i=0; i< m_xmlOptionsList.size();i++) {
 			//m_xmlOptionsList will be empty for XmlIgnore
-			for(size_t j=(**iterXMLOpts).GetSourceWordsRange().GetStartPos();j<=(**iterXMLOpts).GetSourceWordsRange().GetEndPos();j++) {
+			for(size_t j=m_xmlOptionsList[i].startPos;j<=m_xmlOptionsList[i].endPos;j++) {
 				m_xmlCoverageMap[j]=true;
 				
 			}
@@ -89,12 +88,24 @@ bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const {
 
 void Sentence::GetXmlTranslationOptions(std::vector <TranslationOption*> &list, size_t startPos, size_t endPos) const {
 	//iterate over XmlOptions list, find exact source/target matches
+	//we don't worry about creating the objects ahead of time because this should only be called once for each unique start/end when a given sentence is processed
 	const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
 	
-	for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin();
-	        iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) {
-		if (startPos == (**iterXMLOpts).GetSourceWordsRange().GetStartPos() && endPos == (**iterXMLOpts).GetSourceWordsRange().GetEndPos()) {
- 			list.push_back(*iterXMLOpts);
+	for(size_t i=0;i<m_xmlOptionsList.size();i++) {
+		if (startPos == m_xmlOptionsList[i].startPos && endPos == m_xmlOptionsList[i].endPos) {
+			//create TranslationOptions
+			
+			for (size_t j=0;j<m_xmlOptionsList[i].targetPhrases.size();j++) {
+				TargetPhrase targetPhrase(Output);
+				targetPhrase.CreateFromString(outputFactorOrder,m_xmlOptionsList[i].targetPhrases[j],StaticData::Instance().GetFactorDelimiter());
+				targetPhrase.SetScore(m_xmlOptionsList[i].targetScores[j]);
+				WordsRange range(m_xmlOptionsList[i].startPos,m_xmlOptionsList[i].endPos);
+				
+				TranslationOption *option = new TranslationOption(range,targetPhrase,*this);
+				assert(option);
+				list.push_back(option);
+
+			}
 		}
 	}
 }
diff --git a/moses/src/Sentence.h b/moses/src/Sentence.h
index 0f0b4aa7a..8a05787ee 100755
--- a/moses/src/Sentence.h
+++ b/moses/src/Sentence.h
@@ -48,7 +48,7 @@ class Sentence : public Phrase, public InputType
 	 * and returns the value of that tag if present, empty string otherwise
 	 */
 	static std::string ParseXmlTagAttribute(const std::string& tag,const std::string& attributeName);
-	std::vector <TranslationOption*> m_xmlOptionsList;
+	std::vector <XmlOption> m_xmlOptionsList;
 	std::vector <bool> m_xmlCoverageMap;
 
  public:
@@ -90,4 +90,3 @@ class Sentence : public Phrase, public InputType
 	TranslationOptionCollection* CreateTranslationOptionCollection() const;
 };
 
-
diff --git a/moses/src/TranslationOption.h b/moses/src/TranslationOption.h
index 440d4461d..441cd403c 100755
--- a/moses/src/TranslationOption.h
+++ b/moses/src/TranslationOption.h
@@ -60,7 +60,6 @@ protected:
 	Phrase				      *m_sourcePhrase; /*< input phrase translated by this */
 	const WordsRange		m_sourceWordsRange; /*< word position in the input that are covered by this translation option */
 	float               m_futureScore; /*< estimate of total cost when using this translation option, includes language model probabilities */
-	std::vector<TranslationOption*> m_linkedTransOpts; /* list of linked TOs which must be included with this in any hypothesis */
 	
 	//! in TranslationOption, m_scoreBreakdown is not complete.  It cannot,
 	//! for example, know the full n-gram score since the length of the
@@ -113,18 +112,6 @@ public:
 	{
 	  return m_sourcePhrase;
 	}
-	
-	/** returns linked TOs */
-	inline const std::vector<TranslationOption*> &GetLinkedTransOpts() const
-	{
-		return m_linkedTransOpts;
-	}
-	
-	/** add link to another TO */
-	inline void AddLinkedTransOpt(TranslationOption* to)
-	{
-		m_linkedTransOpts.push_back(to);
-	}
 
 	/** whether source span overlaps with those of a hypothesis */
 	bool Overlap(const Hypothesis &hypothesis) const;
@@ -180,4 +167,3 @@ public:
 };
 
 
-
diff --git a/moses/src/WordsBitmap.cpp b/moses/src/WordsBitmap.cpp
index 97c50cda0..316bd9e08 100755
--- a/moses/src/WordsBitmap.cpp
+++ b/moses/src/WordsBitmap.cpp
@@ -56,23 +56,3 @@ int WordsBitmap::GetFutureCosts(int lastPos) const
 	return sum;
 }
 
-
-std::vector<size_t> WordsBitmap::GetCompressedRepresentation() const
-{
-	int vectorSize = 1 + m_size / (sizeof(int) * CHAR_BIT);
-	std::vector<size_t> res(vectorSize, 0);
-	size_t x=0; size_t ci=0;
-	for(size_t i=0;i<m_size;++i) {
-		x <<= 1;
-		x |= (size_t)m_bitmap[i];
-		if ((i + 1) % (sizeof(int) * CHAR_BIT) == 0) {
-			res[ci++] = x; x = 0;
-		}
-	}
-	x <<= vectorSize * sizeof(int) * CHAR_BIT - m_size;
-	res[ci] = x;
-	return res;
-}
-
-
-
diff --git a/moses/src/WordsBitmap.h b/moses/src/WordsBitmap.h
index 7a874d649..77418808d 100755
--- a/moses/src/WordsBitmap.h
+++ b/moses/src/WordsBitmap.h
@@ -149,11 +149,7 @@ public:
 	{
 		return m_size;
 	}
-	/** represent this bitmap as 1 or more vector of integers.
-		* Used for exact matching of source words translated in hypothesis recombination
-		*/
-	std::vector<size_t> GetCompressedRepresentation() const;
-	
+
 	//! transitive comparison of WordsBitmap
 	inline int Compare (const WordsBitmap &compare) const
 	{
diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp
index 6c18aac12..e71dfc97f 100644
--- a/moses/src/XmlOption.cpp
+++ b/moses/src/XmlOption.cpp
@@ -26,7 +26,6 @@
 #include <iostream>
 #include "Util.h"
 #include "StaticData.h"
-#include "TranslationOption.h"
 
 namespace {
 
@@ -92,11 +91,10 @@ inline std::vector<std::string> TokenizeXml(const std::string& str)
 
 }
 
-std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source) {
+std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line) {
 	//parse XML markup in translation line
-	std::vector<TranslationOption*> res;
+	std::vector<XmlOption> res;
 	std::string rstr;
-	std::string linkedStr;
 	if (line.find_first_of('<') == std::string::npos) { return res; }
 	std::vector<std::string> xmlTokens = TokenizeXml(line);
 	std::string tagName = "";
@@ -108,7 +106,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
 	size_t curWord=0;
 	int numUnary = 0;
 	bool doClose = false;
-	bool isLinked = false;
 	for (size_t xmlTokenPos = 0 ; xmlTokenPos < xmlTokens.size() ; xmlTokenPos++)
 	{
 		if(!isXmlTag(xmlTokens[xmlTokenPos]))
@@ -130,30 +127,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
 				nextTagName = tag.substr(0,endOfName);
 				tagContents = tag.substr(endOfName+1);
 			}
-			if (nextTagName == "linked") {
-				isLinked = true;
-				linkedStr = "";
-			}
-			else if (nextTagName == "/linked") {
-				isLinked = false;
-				// recurse to process linked tags
-				std::vector<TranslationOption*> tOptions = ProcessAndStripXMLTags(linkedStr, source);
-				// link them together
-				std::vector<TranslationOption*>::const_iterator iterTransOpts1;
-				std::vector<TranslationOption*>::const_iterator iterTransOpts2;
-				for (iterTransOpts1 = tOptions.begin(); iterTransOpts1 != tOptions.end(); iterTransOpts1++) {
-					for (iterTransOpts2 = tOptions.begin(); iterTransOpts2 != tOptions.end(); iterTransOpts2++) {
-						if (iterTransOpts1 != iterTransOpts2) {
-							(**iterTransOpts1).AddLinkedTransOpt(*iterTransOpts2);
-						}
-					}
-					res.push_back(*iterTransOpts1);
-				}
-			}
-			else if (isLinked) {
-				linkedStr += xmlTokens[xmlTokenPos];
-			}
-			else if (isOpen)
+			if (isOpen)
 			{
 				//this is an open tag
 				tagName = nextTagName;
@@ -208,7 +182,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
 				//TODO: deal with multiple XML options here
 
 				if (StaticData::Instance().GetXmlInputType() != XmlIgnore) {
-					const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
 					for (size_t i=0; i<altTexts.size(); ++i) {
 						//only store options if we aren't ignoring them
 						//set default probability
@@ -216,15 +189,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
 						if (altProbs[i] != "") probValue = Scan<float>(altProbs[i]);
 						//Convert from prob to log-prob
 						float scoreValue = FloorScore(TransformScore(probValue));
-						
-						TargetPhrase targetPhrase(Output);
-						targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],StaticData::Instance().GetFactorDelimiter());
-						targetPhrase.SetScore(scoreValue);
-						WordsRange range(tagStart,tagEnd);
-						
-						TranslationOption *option = new TranslationOption(range,targetPhrase,source);
-						assert(option);
-						
+						XmlOption option(tagStart,tagEnd,altTexts[i],scoreValue);
 						res.push_back(option);
 					}
 				}
@@ -240,4 +205,3 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
 	return res;
 }
 
-
diff --git a/moses/src/XmlOption.h b/moses/src/XmlOption.h
index dc9efe650..e211a6398 100644
--- a/moses/src/XmlOption.h
+++ b/moses/src/XmlOption.h
@@ -2,7 +2,6 @@
 
 #include <vector>
 #include <string>
-#include "InputType.h"
 
 /** This struct is used for storing XML force translation data for a given range in the sentence
  */
@@ -19,6 +18,5 @@ struct XmlOption {
 
 };
 
-std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source);
-
+std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line);