2Merge branch 'master' of https://machinetranslation.visualstudio.com/DefaultCollection/moses-mstranslator/_git/moses-mstranslator

8 files changed, 41 insertions, 79 deletions

diff --git a/moses2/FF/FeatureFunction.h b/moses2/FF/FeatureFunction.h
index 828a712b9..4fa2ee7c3 100644
--- a/moses2/FF/FeatureFunction.h
+++ b/moses2/FF/FeatureFunction.h
@@ -96,7 +96,7 @@ public:
                                          const SCFG::TargetPhrases &tps, const Phrase<SCFG::Word> &sourcePhrase) const {
   }
 
-  virtual void InitializeForInput(const System &system, const InputType &input) { };
+  virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input) { };
 
   // clean up temporary memory, called after processing each sentence
   virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const {
diff --git a/moses2/FF/FeatureFunctions.cpp b/moses2/FF/FeatureFunctions.cpp
index 39e2436b6..c11a2aadf 100644
--- a/moses2/FF/FeatureFunctions.cpp
+++ b/moses2/FF/FeatureFunctions.cpp
@@ -229,10 +229,10 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const
   }
 }
 
-void FeatureFunctions::InitializeForInput(const InputType &input) 
+void FeatureFunctions::InitializeForInput(const ManagerBase &mgr, const InputType &input) 
 {
   BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) {
-    ff->InitializeForInput(m_system, input);
+    ff->InitializeForInput(mgr, input);
   }
 }
 
diff --git a/moses2/FF/FeatureFunctions.h b/moses2/FF/FeatureFunctions.h
index 4c15d674a..43a5793c4 100644
--- a/moses2/FF/FeatureFunctions.h
+++ b/moses2/FF/FeatureFunctions.h
@@ -87,7 +87,7 @@ public:
 
   void EvaluateWhenAppliedBatch(const Batch &batch) const;
 
-  void InitializeForInput(const InputType &input);
+  void InitializeForInput(const ManagerBase &mgr, const InputType &input);
   void CleanUpAfterSentenceProcessing(const InputType &input) const;
 
   void ShowWeights(const Weights &allWeights);
diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp
index cce30efb1..83cca53b1 100644
--- a/moses2/PhraseBased/Manager.cpp
+++ b/moses2/PhraseBased/Manager.cpp
@@ -63,7 +63,7 @@ void Manager::Init()
   //TODO: need option to choose Sentence vs SentenceWithCandidates
   m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr);
   //cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush;
-  system.featureFunctions.InitializeForInput(*m_input);
+  system.featureFunctions.InitializeForInput(*this, *m_input);
 
   m_bitmaps = new Bitmaps(GetPool());
 
diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp
index 6e4190a4e..b13a648ee 100644
--- a/moses2/PhraseBased/SentenceWithCandidates.cpp
+++ b/moses2/PhraseBased/SentenceWithCandidates.cpp
@@ -38,10 +38,10 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
       input_parts.push_back(copy_range<std::string>(*It));
   }
 
-  cerr << "Number of subparts: " << input_parts.size() << endl;
+  //cerr << "Number of subparts: " << input_parts.size() << endl;
 
   if (input_parts.size() ==2 ) {
-      cerr << "correct number of parts" << endl ;
+      //cerr << "correct number of parts" << endl ;
   } else {
       // TODO: how to handle wrong input format 
       cerr << "INCORRECT number of parts" << endl ;
@@ -50,8 +50,8 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
 
   trim(input_parts[0]);
   trim(input_parts[1]);
-  cerr << "Input String: " << input_parts[0] << endl ;
-  cerr << "Phrase Table: " << input_parts[1] << endl ;
+  //cerr << "Input String: " << input_parts[0] << endl ;
+  //cerr << "Phrase Table: " << input_parts[1] << endl ;
 
   ///// Process the text part of the input 
   const string partstr = input_parts[0];
@@ -72,7 +72,7 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
   ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n");
     // ret->m_phraseTableString="constant phrase table";
 //   cerr << "Extracted Phrase Table String: " << ret->m_phraseTableString << endl; 
-   cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl;
+   //cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl;
 
   return ret;
 }
@@ -80,17 +80,17 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
 SentenceWithCandidates::SentenceWithCandidates(MemPool &pool, size_t size)
 :Sentence(pool, size)
 {
-    cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl;
+    //cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl;
 }
 
 SentenceWithCandidates::~SentenceWithCandidates()
 {
-    cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl;
+    //cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl;
 }
 
 std::string SentenceWithCandidates::Debug(const System &system) const
 {
-  cerr << "SentenceWithCandidates::Debug" << endl;
+  return "SentenceWithCandidates::Debug";
 }
 
 } /* namespace Moses2 */
diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp
index d72d0b395..a30169f29 100644
--- a/moses2/TranslationModel/MSPT/MSPT.cpp
+++ b/moses2/TranslationModel/MSPT/MSPT.cpp
@@ -30,19 +30,18 @@
 #include "../../SCFG/Manager.h"
 
 #include "../../PhraseBased/SentenceWithCandidates.h"
+#include "../../PhraseBased/Manager.h"
 
 using namespace std;
 
 namespace Moses2
 {
-
+thread_local MSPT::PBNODE *MSPT::m_rootPb;
 
 ////////////////////////////////////////////////////////////////////////
 
 MSPT::MSPT(size_t startInd, const std::string &line)
   :PhraseTable(startInd, line)
-  ,m_rootPb(NULL)
-  ,m_rootSCFG(NULL)
 {
   ReadParameters();
 }
@@ -50,21 +49,20 @@ MSPT::MSPT(size_t startInd, const std::string &line)
 MSPT::~MSPT()
 {
   delete m_rootPb;
-  delete m_rootSCFG;
 }
 
-void MSPT::CreatePTForInput(const System &system, string phraseTableString)
+void MSPT::CreatePTForInput(const ManagerBase &mgr, string phraseTableString)
 {
-  cerr << "In CreatePTForInput" << endl << flush;
-
+  //cerr << "In CreatePTForInput" << endl << flush;
+  const System &system = mgr.system;
   FactorCollection &vocab = system.GetVocab();
-  MemPool &systemPool = system.GetSystemPool();
+  MemPool &pool = mgr.GetPool();
   MemPool tmpSourcePool;
 
   if (system.isPb) {
     m_rootPb = new PBNODE();
   } else {
-    m_rootSCFG = new SCFGNODE();
+    abort();
     //cerr << "m_rootSCFG=" << m_rootSCFG << endl;
   }
 
@@ -86,7 +84,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
       PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
                            toks[0]);
       //cerr << "created soure" << endl;
-      TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system,
+      TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, system,
                                  toks[1]);
       //cerr << "created target" << endl;
       target->GetScores().CreateFromString(toks[2], *this, system, true);
@@ -103,48 +101,22 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
         //strcpy(target->properties, toks[6].c_str());
       }
 
-      system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
+      system.featureFunctions.EvaluateInIsolation(pool, system, *source,
           *target);
-      //cerr << "EvaluateInIsolation:" << *target << endl;
+      //cerr << "EvaluateInIsolation:" << target->Debug(system) << endl;
       m_rootPb->AddRule(m_input, *source, target);
 
       //cerr << "target=" << target->Debug(system) << endl;
     } else {
-      SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
-                                 toks[0]);
-      //cerr << "created source:" << *source << endl;
-      SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this,
-                                       system, toks[1]);
-
-      //cerr << "created target " << *target << " source=" << *source << endl;
-
-      target->GetScores().CreateFromString(toks[2], *this, system, true);
-      //cerr << "created scores:" << *target << endl;
-
-      //vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
-      //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0;
-
-      target->SetAlignmentInfo(toks[3]);
-
-      // properties
-      if (toks.size() == 7) {
-        //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
-        //strcpy(target->properties, toks[6].c_str());
-      }
-
-      system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
-          *target);
-      //cerr << "EvaluateInIsolation:" << *target << endl;
-      m_rootSCFG->AddRule(m_input, *source, target);
+      abort();
     }
   }
 
   if (system.isPb) {
-    m_rootPb->SortAndPrune(m_tableLimit, systemPool, system);
+    m_rootPb->SortAndPrune(m_tableLimit, pool, system);
     //cerr << "root=" << &m_rootPb << endl;
   } else {
-    m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system);
-    //cerr << "root=" << &m_rootPb << endl;
+      abort();
   }
   /*
   BOOST_FOREACH(const PtMem::Node<Word>::Children::value_type &valPair, m_rootPb.GetChildren()) {
@@ -156,32 +128,28 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
 
 }
 
-void MSPT::InitializeForInput(const System &system, const InputType &input)
+void MSPT::InitializeForInput(const ManagerBase &mgr, const InputType &input)
 {
-  cerr << "InitializeForInput MSPT" << endl;
-  cerr << input.Debug(system) << endl << flush;
-  cerr << "HH1" << endl << flush;
-  
   // downcast to SentenceWithCandidates
-  //const SentenceWithCandidates &inputObj = static_cast<const SentenceWithCandidates&>(input);
-  const SentenceWithCandidates &inputObj = dynamic_cast<const SentenceWithCandidates&>(input);
-  cerr << "Casting done." << endl << flush;
-  cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl;
-
-  cerr << "Hardcoding sample PhraseTableString" << endl << flush; 
-  string phraseTableString="a ||| x ||| 0.4 $$$ a ||| y ||| 0.6 $$$ b ||| y ||| 0.1 $$$ b ||| z ||| 0.9";
-  CreatePTForInput(system,phraseTableString);
-
+  const SentenceWithCandidates &inputObj = static_cast<const SentenceWithCandidates&>(input);
+  CreatePTForInput(mgr, inputObj.getPhraseTableString());
 }
 
 TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool,
     InputPath &inputPath) const
 {
+  //cerr << "MSPT::Lookup inputPath:" << inputPath.Debug(mgr.system) << endl;
   const SubPhrase<Moses2::Word> &phrase = inputPath.subPhrase;
   TargetPhrases *tps = m_rootPb->Find(m_input, phrase);
+  //cerr << "MSPT::Lookup tps:" << tps->Debug(mgr.system) << endl;
+  //cerr << "MSPT::Lookup done" << endl;
   return tps;
 }
 
+void MSPT::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const {
+  delete m_rootPb;
+}
+
 void MSPT::InitActiveChart(
   MemPool &pool,
   const SCFG::Manager &mgr,
diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h
index 744158ea0..fe37e34b9 100644
--- a/moses2/TranslationModel/MSPT/MSPT.h
+++ b/moses2/TranslationModel/MSPT/MSPT.h
@@ -65,11 +65,11 @@ public:
               const SCFG::Stacks &stacks,
               SCFG::InputPath &path) const;
 
-  virtual void InitializeForInput(const System &system, const InputType &input);
+  virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input);
+  virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const;
 
 protected:
-  PBNODE    *m_rootPb;
-  SCFGNODE  *m_rootSCFG;
+  thread_local static PBNODE    *m_rootPb;
 
   void LookupGivenNode(
     MemPool &pool,
@@ -80,7 +80,7 @@ protected:
     const Moses2::Range &subPhraseRange,
     SCFG::InputPath &outPath) const;
 
-  void CreatePTForInput(const System &system, std::string phraseTableString);
+  void CreatePTForInput(const ManagerBase &mgr, std::string phraseTableString);
 
 };
 
diff --git a/moses2/TranslationModel/PhraseTable.cpp b/moses2/TranslationModel/PhraseTable.cpp
index fef6771d6..1f87b8040 100644
--- a/moses2/TranslationModel/PhraseTable.cpp
+++ b/moses2/TranslationModel/PhraseTable.cpp
@@ -80,13 +80,7 @@ void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const
 
     if (SatisfyBackoff(mgr, *path)) {
       TargetPhrases *tpsPtr = Lookup(mgr, mgr.GetPool(), *path);
-      /*
-      cerr << "tpsPtr=" << tpsPtr << " ";
-       if (tps.get()) {
-       cerr << tps.get()->GetSize();
-       }
-       cerr << endl;
-       */
+      //cerr << "tpsPtr=" << tpsPtr << endl;
 
       path->AddTargetPhrases(*this, tpsPtr);
     }