multiple LM per factor

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@464 1f5c12ca-751b-0410-a591-d2e778427230

9 files changed, 16 insertions, 59 deletions

diff --git a/moses/src/Factor.cpp b/moses/src/Factor.cpp
index cf75d3081..ec60e0aa1 100755
--- a/moses/src/Factor.cpp
+++ b/moses/src/Factor.cpp
@@ -25,11 +25,10 @@ using namespace std;
 
 size_t Factor::s_id = 0;
 
-Factor::Factor(FactorDirection direction, FactorType factorType, const std::string *factorString, LmId lmId)
+Factor::Factor(FactorDirection direction, FactorType factorType, const std::string *factorString)
 :m_direction(direction)
 ,m_factorType(factorType)
 ,m_ptrString(factorString)
-,m_lmId(lmId)
 ,m_id(s_id++)
 {
 }
diff --git a/moses/src/Factor.h b/moses/src/Factor.h
index 6e184e022..a58bd12d2 100755
--- a/moses/src/Factor.h
+++ b/moses/src/Factor.h
@@ -59,15 +59,9 @@ protected:
 	FactorType				m_factorType;
 	const std::string	*m_ptrString;
 	const size_t			m_id;
-	LmId 							m_lmId;
 
-	Factor(FactorDirection direction, FactorType factorType, const std::string *factorString, LmId lmId);
+	Factor(FactorDirection direction, FactorType factorType, const std::string *factorString);
 	
-	inline void SetLmId(LmId lmId)
-	{
-		m_lmId = lmId;
-	}
-
 public:
 	inline size_t hash() const
 	{
@@ -93,11 +87,6 @@ public:
 	{
 		return m_id;
 	}
-	inline LmId GetLmId() const
-	{
-		return m_lmId;
-	}
-	
 	// do it properly. needed for insert & finding of words in dictionary
 	inline int Compare(const Factor &compare) const
 	{
diff --git a/moses/src/FactorCollection.cpp b/moses/src/FactorCollection.cpp
index 5cb022e3b..1a276c2ba 100755
--- a/moses/src/FactorCollection.cpp
+++ b/moses/src/FactorCollection.cpp
@@ -53,7 +53,7 @@ bool FactorCollection::Exists(FactorDirection direction, FactorType factorType,
 	const string *ptrString=&(*m_factorStringCollection.insert(factorString).first);
 
 	FactorSet::const_iterator iterFactor;
-	Factor search(direction, factorType, ptrString, LanguageModel::UNKNOWN_LM_ID);
+	Factor search(direction, factorType, ptrString);
 
 	iterFactor = m_collection.find(search);
 	return iterFactor != m_collection.end();
@@ -61,26 +61,11 @@ bool FactorCollection::Exists(FactorDirection direction, FactorType factorType,
 
 const Factor *FactorCollection::AddFactor(FactorDirection direction
 																				, FactorType 			factorType
-																				, const string 		&factorString
-																				, LmId						lmId)
+																				, const string 		&factorString)
 {
 	// find string id
 	const string *ptrString=&(*m_factorStringCollection.insert(factorString).first);
-//	Factor findFactor(direction, factorType, ptr, lmId);
-	return &(*m_collection.insert(Factor(direction, factorType, ptrString, lmId)).first);
-}
-
-const Factor *FactorCollection::AddFactor(FactorDirection direction
-																				, FactorType 			factorType
-																				, const string 		&factorString)
-{
-	return AddFactor(direction, factorType, factorString, LanguageModel::UNKNOWN_LM_ID);
-}
-
-void FactorCollection::SetFactorLmId(const Factor *factor, LmId lmId)
-{ // only used by non-srilm code
-	Factor *changeFactor = const_cast<Factor *>(factor);
-	changeFactor->SetLmId(lmId);
+	return &(*m_collection.insert(Factor(direction, factorType, ptrString)).first);
 }
 
 FactorCollection::~FactorCollection()
diff --git a/moses/src/FactorCollection.h b/moses/src/FactorCollection.h
index f3d20aff2..39122be46 100755
--- a/moses/src/FactorCollection.h
+++ b/moses/src/FactorCollection.h
@@ -65,8 +65,6 @@ public:
 
 	bool Exists(FactorDirection direction, FactorType factorType, const std::string &factorString);	
 	const Factor *AddFactor(FactorDirection direction, FactorType factorType, const std::string &factorString);	
-	const Factor *AddFactor(FactorDirection direction, FactorType factorType, const std::string &factorString, LmId lmId);	
-	void SetFactorLmId(const Factor *factor, LmId lmId);
 
 	void LoadVocab(FactorDirection direction, FactorType factorType, const std::string &fileName);
 	
diff --git a/moses/src/LanguageModel.cpp b/moses/src/LanguageModel.cpp
index 7e9f2f959..df3f92d97 100755
--- a/moses/src/LanguageModel.cpp
+++ b/moses/src/LanguageModel.cpp
@@ -36,7 +36,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 using namespace std;
 
 // static variable init
-const LmId LanguageModel::UNKNOWN_LM_ID(-1);
 LanguageModel::State LanguageModel::UnknownState=0;
 
 LanguageModel::LanguageModel() 
diff --git a/moses/src/LanguageModel.h b/moses/src/LanguageModel.h
index b6090307f..93a2ecbfc 100755
--- a/moses/src/LanguageModel.h
+++ b/moses/src/LanguageModel.h
@@ -44,8 +44,6 @@ public:
   typedef const void* State;
   static State UnknownState;
 
-	static const LmId UNKNOWN_LM_ID;
-
 	LanguageModel();
 	virtual ~LanguageModel();
 	virtual void Load(const std::string &fileName
diff --git a/moses/src/LanguageModel_IRST.cpp b/moses/src/LanguageModel_IRST.cpp
index f30ce2daa..a5cbd08c5 100755
--- a/moses/src/LanguageModel_IRST.cpp
+++ b/moses/src/LanguageModel_IRST.cpp
@@ -75,19 +75,21 @@ void LanguageModel_IRST::CreateFactors(FactorCollection &factorCollection)
 { // add factors which have srilm id
 	
 	dict_entry *entry;
-	LmId lmId;
 	dictionary_iter iter(m_lmtb->dict);
 	while ( (entry = iter.next()) != NULL)
 	{
-		LmId lmId;
-    lmId.irst = entry->code;
-		factorCollection.AddFactor(Output, m_factorType, entry->word, lmId);
+		const Factor *factor = factorCollection.AddFactor(Output, m_factorType, entry->word);
+		m_lmIdLookup[factor] = entry->code;
 	}
 	
+	int lmId;
 	lmId = GetLmID(BOS_);
-	m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, SENTENCE_START, lmId);
+	m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, SENTENCE_START);
+	m_lmIdLookup[m_sentenceStart] = lmId;
+		
 	lmId = GetLmID(EOS_);
-	m_sentenceEnd		= factorCollection.AddFactor(Output, m_factorType, SENTENCE_END, lmId);
+	m_sentenceEnd		= factorCollection.AddFactor(Output, m_factorType, SENTENCE_END);
+	m_lmIdLookup[m_sentenceEnd] = lmId;
 }
 
 int LanguageModel_IRST::GetLmID( const std::string &str ) const
@@ -112,7 +114,7 @@ float LanguageModel_IRST::GetValue(const vector<const Factor*> &contextFactor, S
 		std::cout << i <<"="<<contextFactor[i]->GetLmId().irst <<"," << contextFactor[i]->GetString()<<" ";
 #endif
     int lmId = GetLmID(contextFactor[i]);
-		ng.pushc( (lmId == UNKNOWN_LM_ID.irst) ? m_unknownId : lmId);
+		ng.pushc(lmId);
 	}
 #ifdef CDYER_DEBUG_LMSCORE
 	std::cout <<" (ng='" << ng << "')\n";
diff --git a/moses/src/LanguageModel_SRI.cpp b/moses/src/LanguageModel_SRI.cpp
index efea9392f..0055ce0e0 100755
--- a/moses/src/LanguageModel_SRI.cpp
+++ b/moses/src/LanguageModel_SRI.cpp
@@ -120,14 +120,12 @@ float LanguageModel_SRI::GetValue(const vector<const Factor*> &contextFactor, St
 	size_t count = contextFactor.size();
 	for (size_t i = 0 ; i < count - 1 ; i++)
 	{
-    VocabIndex lmId = GetLmID(contextFactor[count-2-i]);
-		context[i] = (lmId == UNKNOWN_LM_ID.sri) ? m_unknownId : lmId;
+		context[i] =  GetLmID(contextFactor[count-2-i]);
 	}
 	context[count-1] = Vocab_None;
 	
 	// call sri lm fn
-  VocabIndex lmId = GetLmID(contextFactor[count-1]);
-	lmId= (lmId == UNKNOWN_LM_ID.sri) ? m_unknownId : lmId;
+	VocabIndex lmId= GetLmID(contextFactor[count-1]);
 	float ret = GetValue(lmId, context);
 
 	if (finalState) {
diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h
index 67b901835..16f696e14 100755
--- a/moses/src/TypeDef.h
+++ b/moses/src/TypeDef.h
@@ -66,17 +66,6 @@ const size_t DEFAULT_VERBOSE_LEVEL = 1;
 #endif
 ///////////////////////////////////////////////// 
 
-class NGramNode;
-union LmId {
-  unsigned int			sri;
-  const NGramNode*	internal;
-  int								irst;
-  public:
-		LmId() {}
-    LmId(int i) { irst = i; };
-    LmId(const NGramNode *node) { internal = node; };
-};
-
 // enums. 
 // must be 0, 1, 2, ..., unless otherwise stated