added dese-schemeoption;

7 files changed, 170 insertions, 296 deletions

diff --git a/moses/FF/Dsg-Feature/Desegmenter.cpp b/moses/FF/Dsg-Feature/Desegmenter.cpp
index 93b6db528..a9713831b 100644
--- a/moses/FF/Dsg-Feature/Desegmenter.cpp
+++ b/moses/FF/Dsg-Feature/Desegmenter.cpp
@@ -1,133 +1,86 @@
-#include <iostream>     // std::cout

-#include <fstream>      // std::ifstream

-#include<string>

-#include<sstream>

-#include<vector>

-#include<map>

-

-#include "Desegmenter.h"

-#include <boost/algorithm/string/replace.hpp>

-

-using namespace std;

-

+#include <fstream>
+#include <iostream>
+#include<string>
+#include<sstream>
+#include<vector>
+#include<map>
+#include "Desegmenter.h"
+#include <boost/algorithm/string/replace.hpp>
+
+using namespace std;
+
 namespace Moses
-{

-void Desegmenter::Load(const string filename){

-

-	std::ifstream myFile(filename.c_str() );//, std::ifstream::in);

-	if (myFile.is_open()){

-		cerr << "Desegmentation File open successful." << endl;

-		string line;

-		while (getline(myFile, line)){

-			stringstream ss(line);

-			string token;

-			vector<string> myline;

-			while (getline(ss, token, '\t')){

-				myline.push_back(token);

-			}

-			mmDesegTable.insert(pair<string, string>(myline[2], myline[1] ));

-		}

-		myFile.close();

-	}

-	else

-		cerr << "open() failed: check if Desegmentation file is in right folder" << endl;

-}

-

-

-

-

-vector<string> Desegmenter::Search(string myKey){

-	multimap<string, string>::const_iterator  mmiPairFound = mmDesegTable.find(myKey);

-	vector<string> result;

-	if (mmiPairFound != mmDesegTable.end()){

-		size_t nNumPairsInMap = mmDesegTable.count(myKey);

-		

-		for (size_t nValuesCounter = 0; nValuesCounter < nNumPairsInMap; ++nValuesCounter){

-			

-			if (mmiPairFound != mmDesegTable.end())	{

-				result.push_back(mmiPairFound->second);

-			}

-			++mmiPairFound;

-		}

-		return result;

-	}

-	else{

-		string rule_deseg ;

-		rule_deseg = ApplyRules(myKey);

-		result.push_back(rule_deseg);

-		return result;

-	}

-}

-

-string Desegmenter::ApplyRules(string & segToken){

-	string desegToken=segToken;

-	

-	boost::replace_all(desegToken, "l+ All", "ll");

-	boost::replace_all(desegToken, "l+ Al", "ll"); 

-	boost::replace_all(desegToken, "y+ y ", "y"); 

-	boost::replace_all(desegToken, "p+ ", "t"); 

-	boost::replace_all(desegToken, "' +", "}"); 

-	boost::replace_all(desegToken, "y +", "A"); 

-	boost::replace_all(desegToken, "n +n", "n"); 

-	boost::replace_all(desegToken, "mn +m", "mm"); 

-	boost::replace_all(desegToken, "En +m", "Em"); 

-	boost::replace_all(desegToken, "An +lA", "Em");

-	boost::replace_all(desegToken, "-LRB-", "(");

-	boost::replace_all(desegToken, "-RRB-", ")");

-	boost::replace_all(desegToken, "+ +", "");

-	

-	boost::replace_all(desegToken, "+ ", "");

-	boost::replace_all(desegToken, " +", "");

-	

-	return desegToken;

-}

-

-

-Desegmenter::~Desegmenter()

-{}

-

-

-

-/*

-void Completer::Load(const string filename){

-

-	std::ifstream myFile(filename.c_str() );

-	if (myFile.is_open()){

-		cerr << "Completer File open successful." << endl;

-		string line;

-		while (getline(myFile, line)){

-			stringstream ss(line);

-			string token;

-			vector<string> myline;

-			while (getline(ss, token, '\t')){

-				myline.push_back(token);

-			}

-			mmDetok.insert(pair<string, string>(myline[0], myline[1] ));

-		}

-		myFile.close();

-	}

-	else

-		cerr << "open() failed: check if Desegmentation file is in right folder" << endl;

-	//return mmDetok;

-}

-

-string Completer::Search(string myKey){

-	

-	//unordered_multimap<string, string>::const_iterator  mmiPairFound = mmDetok.find(myKey);

-	map<string, string>::const_iterator  mi = mmDetok.find(myKey);

-	//vector<string> result;

-	string result="";

-	if (mi != mmDetok.end()){

-		result=mi->second;		

-		return result;

-	}

-	else{

-		return result;

-	}

-}

-

-Completer::~Completer()

-{}

-*/

-

-}

+{
+void Desegmenter::Load(const string filename){
+
+	std::ifstream myFile(filename.c_str() );
+	if (myFile.is_open()){
+		cerr << "Desegmentation File open successful." << endl;
+		string line;
+		while (getline(myFile, line)){
+			stringstream ss(line);
+			string token;
+			vector<string> myline;
+			while (getline(ss, token, '\t')){
+				myline.push_back(token);
+			}
+			mmDesegTable.insert(pair<string, string>(myline[2], myline[1] ));
+		}
+		myFile.close();
+	}
+	else
+		cerr << "open() failed: check if Desegmentation file is in right folder" << endl;
+}
+
+
+vector<string> Desegmenter::Search(string myKey){
+	multimap<string, string>::const_iterator  mmiPairFound = mmDesegTable.find(myKey);
+	vector<string> result;
+	if (mmiPairFound != mmDesegTable.end()){
+		size_t nNumPairsInMap = mmDesegTable.count(myKey);		
+		for (size_t nValuesCounter = 0; nValuesCounter < nNumPairsInMap; ++nValuesCounter){			
+			if (mmiPairFound != mmDesegTable.end())	{
+				result.push_back(mmiPairFound->second);
+			}
+			++mmiPairFound;
+		}
+		return result;
+	}
+	else{
+		string rule_deseg ;
+		rule_deseg = ApplyRules(myKey);
+		result.push_back(rule_deseg);
+		return result;
+	}
+}
+
+
+string Desegmenter::ApplyRules(string & segToken){
+
+	string desegToken=segToken;
+  if (!simple){
+	boost::replace_all(desegToken, "l+ All", "ll");
+	boost::replace_all(desegToken, "l+ Al", "ll"); 
+	boost::replace_all(desegToken, "y+ y ", "y"); 
+	boost::replace_all(desegToken, "p+ ", "t"); 
+	boost::replace_all(desegToken, "' +", "}"); 
+	boost::replace_all(desegToken, "y +", "A"); 
+	boost::replace_all(desegToken, "n +n", "n"); 
+	boost::replace_all(desegToken, "mn +m", "mm"); 
+	boost::replace_all(desegToken, "En +m", "Em"); 
+	boost::replace_all(desegToken, "An +lA", "Em");
+	boost::replace_all(desegToken, "-LRB-", "(");
+	boost::replace_all(desegToken, "-RRB-", ")");
+	}
+	
+    boost::replace_all(desegToken, "+ +", "");	
+	boost::replace_all(desegToken, "+ ", "");
+	boost::replace_all(desegToken, " +", "");
+	
+	return desegToken;
+}
+
+Desegmenter::~Desegmenter()
+{}
+
+}
diff --git a/moses/FF/Dsg-Feature/Desegmenter.h b/moses/FF/Dsg-Feature/Desegmenter.h
index 133bcee4d..397140f91 100644
--- a/moses/FF/Dsg-Feature/Desegmenter.h
+++ b/moses/FF/Dsg-Feature/Desegmenter.h
@@ -1,52 +1,31 @@
-#pragma once

-

-#include<string>

-#include<map>

-

-

-using namespace std;

-

+#pragma once
+
+#include<string>
+#include<map>
+
+
+using namespace std;
+
 namespace Moses
-{

-class Desegmenter

-{

-private:

+{
+class Desegmenter
+{
+private:
 	std::multimap<string, string> mmDesegTable;
-	std::string filename;

-	void Load(const string filename);

-

-public:

-	Desegmenter(const std::string& file){
-		filename = file; 

-		Load(filename);//, mmDetok);

-	}

-	string getFileName(){ return filename; }

-	

-	vector<string> Search(string myKey);

-	string ApplyRules(string &);

-

-	~Desegmenter();

-};

-

-

-/*class Completer

-{

-private:

-	//std::multimap<string, string,std::less< std::string > > mmDetok;
-	std::map<string, string> mmDetok;
-	std::string filename;

-	void Load(const string filename);

-

-public:

-	Completer(const std::string& file){
-		filename = file; 

-		Load(filename);//, mmDetok);

-	}

-	string getFileName(){ return filename; }	

-	string Search(string myKey);

-

-	~Completer();

-};

-*/

-

-}

+	std::string filename;
+	bool simple;
+	void Load(const string filename);
+
+public:
+	Desegmenter(const std::string& file, const bool scheme){
+		filename = file;
+		simple=scheme;
+		Load(filename);
+	}
+	string getFileName(){ return filename; }
+	
+	vector<string> Search(string myKey);
+	string ApplyRules(string &);
+	~Desegmenter();
+};
+}
diff --git a/moses/FF/Dsg-Feature/DsgModel.cpp b/moses/FF/Dsg-Feature/DsgModel.cpp
index 635109774..090b5545a 100644
--- a/moses/FF/Dsg-Feature/DsgModel.cpp
+++ b/moses/FF/Dsg-Feature/DsgModel.cpp
@@ -28,23 +28,23 @@ namespace Moses
   void DesegModel :: readLanguageModel(const char *lmFile)
   {
     DSGM = ConstructDsgLM(m_lmPath.c_str());
-    State startState = DSGM->NullContextState(); // MSAL
-    desegT=new Desegmenter(m_desegPath);// Desegmentation Table
+    State startState = DSGM->NullContextState(); 
+    desegT=new Desegmenter(m_desegPath,m_simple);// Desegmentation Table
   }
 
 
   void DesegModel::Load(AllOptions::ptr const& opts)
   {
-    m_options = opts; //ADDED
+    m_options = opts;
     readLanguageModel(m_lmPath.c_str());
   }
 
 
 
   void DesegModel:: EvaluateInIsolation(const Phrase &source
-					, const TargetPhrase &targetPhrase
-					, ScoreComponentCollection &scoreBreakdown
-					, ScoreComponentCollection &estimatedScores) const
+          , const TargetPhrase &targetPhrase
+          , ScoreComponentCollection &scoreBreakdown
+          , ScoreComponentCollection &estimatedScores) const
   {
 
     dsgHypothesis obj;
@@ -62,16 +62,14 @@ namespace Moses
     obj.calculateDsgProbinIsol(*DSGM,*desegT,align);
     obj.populateScores(scores,numFeatures);
     estimatedScores.PlusEquals(this, scores);
-
   }
 
 
   FFState* DesegModel::EvaluateWhenApplied(
-					   const Hypothesis& cur_hypo,
-					   const FFState* prev_state,
-					   ScoreComponentCollection* accumulator) const
+             const Hypothesis& cur_hypo,
+             const FFState* prev_state,
+             ScoreComponentCollection* accumulator) const
   {
-
     const TargetPhrase &target = cur_hypo.GetCurrTargetPhrase();
     const Range &src_rng =cur_hypo.GetCurrSourceWordsRange();
     const AlignmentInfo &align = cur_hypo.GetCurrTargetPhrase().GetAlignTerm();
@@ -97,12 +95,11 @@ namespace Moses
   }
 
   FFState* DesegModel::EvaluateWhenApplied(
-					   const ChartHypothesis& /* cur_hypo */,
-					   int /* featureID - used to index the state in the previous hypotheses */,
-					   ScoreComponentCollection* accumulator) const
+             const ChartHypothesis& /* cur_hypo */,
+             int /* featureID - used to index the state in the previous hypotheses */,
+             ScoreComponentCollection* accumulator) const
   {
     UTIL_THROW2("Chart decoding not support by UTIL_THROW2");
-
   }
 
   const FFState* DesegModel::EmptyHypothesisState(const InputType &input) const
@@ -110,7 +107,6 @@ namespace Moses
     VERBOSE(3,"DesegModel::EmptyHypothesisState()" << endl);
     State startState = DSGM->BeginSentenceState();
     dsgState ss= dsgState(startState);
-    /////ss.setDelta(0.0);
     return new dsgState(ss);
   }
 
@@ -134,11 +130,16 @@ namespace Moses
       tFactor = Scan<int>(value);
     } else if (key == "optimistic") {
       if (value == "n")
-	optimistic = 0;
+      optimistic = 0;
       else
-	optimistic = 1;
+      optimistic = 1;
     } else if (key == "deseg-path") {
-      m_desegPath = value;
+      m_desegPath = Scan<int>(value);
+    } else if (key == "deseg-scheme") {
+      if(value == "s")
+        m_simple = 1;
+      else
+        m_simple = 0;
     } else if (key == "order") {
       order = Scan<int>(value);
     } else {
diff --git a/moses/FF/Dsg-Feature/DsgModel.h b/moses/FF/Dsg-Feature/DsgModel.h
index 8db0eea05..f456123d2 100644
--- a/moses/FF/Dsg-Feature/DsgModel.h
+++ b/moses/FF/Dsg-Feature/DsgModel.h
@@ -18,10 +18,10 @@ namespace Moses
   public:
 
     DsgLM * DSGM;
-    Desegmenter* desegT; //MSAL
+    Desegmenter* desegT; 
     int tFactor;// Target Factor ...
-    int order; //MSAL
-    int numFeatures;   // Number of features used an be 1 (unsegmented LM)or 4 (with 3 contiguity features)
+    int order; 
+    int numFeatures;   // Number of features used an be 1 (unsegmented LM)or 5 (with 3 contiguity features and 1 UnsegWP)
     bool optimistic;
 
     DesegModel(const std::string &line);
@@ -57,6 +57,7 @@ namespace Moses
     typedef std::vector<float> Scores;
     std::string m_lmPath;
     std::string m_desegPath;
+    bool m_simple; //desegmentation scheme; if 1 then use simple, else use rule and backoff to simple
   };
 
 
diff --git a/moses/FF/Dsg-Feature/KenDsg.h b/moses/FF/Dsg-Feature/KenDsg.h
index 44d7ea6eb..d32a2d98a 100644
--- a/moses/FF/Dsg-Feature/KenDsg.h
+++ b/moses/FF/Dsg-Feature/KenDsg.h
@@ -2,7 +2,6 @@
 
 #include <string>
 #include "lm/model.hh"
-//#include <boost/shared_ptr.hpp>
 
 namespace Moses
 {
@@ -50,7 +49,6 @@ template <class KenModel>
 
 
  private:
-  //  boost::shared_ptr<KenModel> m_kenlm;
   KenModel m_kenlm;
 };
 
diff --git a/moses/FF/Dsg-Feature/dsgHyp.cpp b/moses/FF/Dsg-Feature/dsgHyp.cpp
index 9712ccf67..7daba14c0 100644
--- a/moses/FF/Dsg-Feature/dsgHyp.cpp
+++ b/moses/FF/Dsg-Feature/dsgHyp.cpp
@@ -2,9 +2,9 @@
 #include <sstream>
 #include <boost/algorithm/string.hpp>
 #include <algorithm>
-#include <cstdlib> //NEW
-#include <math.h> //NEW
-#include <map>  //NEW
+#include <cstdlib> 
+#include <math.h> 
+#include <map>  
 
 
 using namespace std;
@@ -19,25 +19,22 @@ namespace Moses
 
   void dsgState::saveState( std::vector<std::string> danglingTok, std::vector<int> srcSpans,float deltaValue)
   {
-    //gap.clear();
     buffer = danglingTok;
     span=srcSpans;
-    delta=deltaValue;//NEW
+    delta=deltaValue;
   }
 
 
-  size_t dsgState::hash() const  //CHECKKKKKKKKKK
+  size_t dsgState::hash() const
   {
 
     size_t ret = 0;
     boost::hash_combine(ret, lmState);
 
     /*size_t ret = delta;
-
   boost::hash_combine(ret, buffer);
   boost::hash_combine(ret, span);
   boost::hash_combine(ret, lmState.length);
-
   return ret;*/
   }
 
@@ -48,23 +45,15 @@ namespace Moses
     if (lmState < other.lmState) return false;
     if (lmState == other.lmState) return true;
     return false;
-
-    /*if (buffer.size()!=other.buffer.size()){return false;}
-  if (span.size()!=other.span.size()){return false;};
-  if (delta!=other.delta){return false;}
-  if (lmState.length!=other.lmState.length){return false;}
-  //if (lmState == other.lmState) {return true;}
-  return true;*/
-
   }
 
+  // ----------------------------------------
+  
   std::string dsgState :: getName() const
   {
     return "done";
   }
 
-  //////////////////////////////////////////////////
-
   dsgHypothesis :: dsgHypothesis()
   {
     lmProb = 0;
@@ -73,7 +62,6 @@ namespace Moses
     discontig2 = 0;
     UnsegWP = 0;
     m_buffer.clear();//="";
-    //delta=0.0;
   }
 
   void dsgHypothesis :: setState(const FFState* prev_state)
@@ -90,14 +78,13 @@ namespace Moses
   {
     dsgState * statePtr = new dsgState(lmState);
     statePtr->saveState(m_buffer, m_span, delta);
-    //statePtr->saveState(gap,span,0.0);
     return statePtr;
   }
 
   void dsgHypothesis :: populateScores(vector <float> & scores , const int numFeatures)
   {
     scores.clear();
-    scores.push_back(lmProb); //TODAY
+    scores.push_back(lmProb);
 
     if (numFeatures == 1)
       return;
@@ -155,7 +142,7 @@ namespace Moses
     std::vector<std::string> chain;
     std::vector<int> chain_ids;
     std::vector<std::string> allchains;
-    chain_ids=m_span;//MSAL
+    chain_ids=m_span;
 
     if (!m_buffer.empty() && !isolation){// if evaluate in isolation is called, then do not add buffer content
       for (int i = 0; i < m_buffer.size(); i++){   // initialize chain with the content of the buffer
@@ -171,7 +158,7 @@ namespace Moses
 	if (sourcePosSet.empty()==false){
 	  for (std::set<size_t>::iterator it(sourcePosSet.begin());it != sourcePosSet.end(); it++) {
 	    int cur=*it;
-	    chain_ids.push_back(cur+sourceOffset); //MSAL
+	    chain_ids.push_back(cur+sourceOffset); 
 	  }
 	}
       }
@@ -194,16 +181,10 @@ namespace Moses
 	if (sourcePosSet.empty()==false){
 	  for (std::set<size_t>::iterator it(sourcePosSet.begin());it != sourcePosSet.end(); it++) {
 	    int cur=*it;
-	    chain_ids.push_back(cur+sourceOffset); //MSAL
+	    chain_ids.push_back(cur+sourceOffset); 
 	  }
 	}
-	/*else {
-	  //chain_ids.push_back(sourceOffset);
-	  //std::cout  << sourceOffset <<" $ ";
-	  //chain_ids.push_back({});
-	  std::cout  << "NONE $ ";
-	  }*/
-	//chain_ids.push_back(i+sourceOffset);//MSAL
+
       }
 
     }
@@ -232,21 +213,8 @@ namespace Moses
     UnsegWP=0;
 
     currFVec = m_buffer;
-
-    /*
-      std::cout << "GAP: ";
-      for (int j=0 ; j< m_buffer.size();j++){cout << "   " << m_buffer[j];}
-      std::cout << endl;
-      std::cout << "Phrase: ";
-      for (int j=0 ; j< m_curr_phr.size();j++){cout << "   " << m_curr_phr[j];}
-      std::cout << endl; */
-
     currFVec.insert( currFVec.end(), m_curr_phr.begin(), m_curr_phr.end() );
 
-    //std::cout << "First: ";
-    //for (int j=0 ; j< currFVec.size();j++){cout << "   " << currFVec[j];}
-    //std::cout << endl;
-
     int vecSize=currFVec.size();
 
     // phrases with suffix-starts and prefix-end
@@ -255,7 +223,6 @@ namespace Moses
     if (currFVec.size()>0  && isSuffix (currFVec.front())) {
       UnsegWP-=0.5;}
 
-
     /* //Dropping prefix-end and suffix-start
        while  (currFVec.size()>0 && isPrefix (currFVec.back())){
        currFVec.pop_back(); //drop prefix appearing at end of phrase
@@ -265,7 +232,6 @@ namespace Moses
        currFVec.erase (currFVec.begin()); //drop suffix appearning at start of a phrase
        } */
 
-
     vector<vector<int> > chain_ids;
     words = grouper(currFVec,chain_ids,0,align,1);
 
@@ -282,7 +248,6 @@ namespace Moses
 	lmProb += ptrDsgLM.Score(temp,words[i],currState);
       }
     }
-    //opProb=TransformLMScore(opProb);
     lmState = currState;
   }
 
@@ -310,60 +275,45 @@ namespace Moses
     words = grouper(currFVec,all_chain_ids,sourceOffset,align,0);
 
     for (int i = 0; i < words.size(); i++) {
-      temp = currState; //NEW ADDED
+      temp = currState; 
 
       if (i==words.size()-1){
 	if (completePhraseSuffixEnd){   //i.e if phrase ends with suffix, which marks an end of a word
 	  m_buffer.clear();// ="";
-	  m_span.clear();// ={};//MSAL
-	  //delta=0.0; //Dont enable this, wrong
+	  m_span.clear();// ={};
 	}
 	else if (!isCompleted) {  // not end of sentence( or final hypothesis), and probably the last token is not a complete word
 	  m_buffer.clear();
 	  if (optimistic == 1){
-	    //  (2)Comment the below if you want delayed scoring
 	    if ( isPrefix (currFVec.back())){  // this will delay scoring of prefix in prefix-ending phrases until the next hypothesis arrives
-
-	      //pscore = ptrDsgLM.Score(temp,desegmented,currState); NEW
-
-	      // enable the 3 lines below with (1) and disable lines below it
-	      //opProb = opProb + pscore - delta; //NEW
-	      //delta=pscore;
-	      //currState=temp;
-
+	      //pscore = ptrDsgLM.Score(temp,desegmented,currState); 
 	      lmProb -= delta;
 	      delta = 0.0;
-	    }//*/
+	    }
 
-	     //Comments these else statements below with (2) if you want to delay prefix-end scoring
-	    else if (words[i].find(" ")!=std::string::npos){ //NEW
-	      desegmented=desegT.Search(words[i])[0];     //NEW
+	    else if (words[i].find(" ")!=std::string::npos){ 
+	      desegmented=desegT.Search(words[i])[0];     
 	      pscore=ptrDsgLM.Score(temp,desegmented,currState);
-	      //opProb += pscore-delta;
-	      lmProb = lmProb + pscore - delta; //NEW
+	      lmProb = lmProb + pscore - delta; 
 	      delta=pscore;
 	      currState=temp;
 	      }
 	    else{
-	      boost::replace_all(words[i], "-LRB-", "("); //NEW CHECK
-	      boost::replace_all(words[i], "-RRB-", ")"); //NEW CHECK
+	      boost::replace_all(words[i], "-LRB-", "("); 
+	      boost::replace_all(words[i], "-RRB-", ")"); 
 	      pscore=ptrDsgLM.Score(temp,words[i],currState);
-	      //opProb += pscore-delta; //NEW
-	      lmProb = lmProb + pscore - delta; //NEW
-	      delta=pscore; //NEW
+	      lmProb = lmProb + pscore - delta; 
+	      delta=pscore; 
 	      currState=temp;
-	    }   }//*/
+	    }   }
 
 	  m_buffer.push_back(words.back());
-	  //gap=words.back();
-	  m_span=all_chain_ids.back();//MSAL
-	  //opProb=TransformLMScore(opProb);
-	  //lmState = currState;
+	  m_span=all_chain_ids.back();
 	  break;
 	}
       }
 
-      //temp = currState; NEW COMMENTED
+      //temp = currState;
       if (words[i].find(" ")!=std::string::npos){
 	UnsegWP+=1;
 	desegmented=desegT.Search(words[i])[0];
@@ -375,16 +325,11 @@ namespace Moses
 	    int mynext=*next;
 	    if (std::abs(cur - mynext)>= 3) {
 	      dsc.push_back(3);
-	      //discontig2+=1;
-	      //break;
 	    }
 	    else if (std::abs(cur - mynext)== 2){
-	      //discontig1+=1;
 	      dsc.push_back(2);
-	      //break;
 	    }
 	    else if (std::abs(cur - mynext)<= 1){
-	      //discontig0+=1;
 	      dsc.push_back(1);
 	    }
 	  }
@@ -397,7 +342,6 @@ namespace Moses
 	  discontig0 += 1;
 	}
 
-	//opProb += ptrDsgLM.Score(temp,ptrDsgLM.GetVocabulary().Index(desegmented),currState);
 	lmProb += ptrDsgLM.Score(temp,desegmented,currState);
       }
       else{
@@ -412,7 +356,6 @@ namespace Moses
       temp = currState;
       lmProb = lmProb + ptrDsgLM.ScoreEndSentence(temp,currState) - delta;
     }
-    //opProb=TransformLMScore(opProb);
     lmState = currState;
   }
 
diff --git a/moses/FF/Dsg-Feature/dsgHyp.h b/moses/FF/Dsg-Feature/dsgHyp.h
index a609b7fb6..0df4af11a 100644
--- a/moses/FF/Dsg-Feature/dsgHyp.h
+++ b/moses/FF/Dsg-Feature/dsgHyp.h
@@ -19,8 +19,7 @@ namespace Moses
   public:
 
     dsgState(const lm::ngram::State & val);
-    //int Compare(const FFState& other) const;
-    virtual bool operator==(const FFState& other) const; //CHECK
+    virtual bool operator==(const FFState& other) const;
     void saveState( std::vector<std::string>  bufferVal,std::vector<int> spanVal, float deltaValue);
 
     std::vector<std::string> getBuffer() const {
@@ -35,11 +34,11 @@ namespace Moses
       return lmState;
     }
 
-    float getDelta() const {  //NEW
+    float getDelta() const {  
       return delta;
     }
 
-    void setDelta(double val1 ) { //NEWWWW
+    void setDelta(double val1 ) { 
       delta = val1;
     }
 
@@ -72,7 +71,7 @@ class dsgHypothesis
   int discontig0;
   int discontig1;
   int discontig2;
-  double UnsegWP;
+  double UnsegWP; //Word Penalty score based on count of words
 
  public:
 
@@ -81,11 +80,11 @@ class dsgHypothesis
   void calculateDsgProb(DsgLM& ptrDsgLM, Desegmenter &, bool isCompleted, const AlignmentInfo &align, int  sourceOffset, bool optimistic);
   void calculateDsgProbinIsol(DsgLM& ptrDsgLM, Desegmenter &, const AlignmentInfo &align);
 
-  void setPhrases(std::vector<std::string> & val1 ) {//MSAL
+  void setPhrases(std::vector<std::string> & val1 ) {
     m_curr_phr = val1;
   }
 
-  void setDelta(double val1 ) { //NEW
+  void setDelta(double val1 ) { 
     delta = val1;
   }