Merge branch 'master' of https://machinetranslation.visualstudio.com/DefaultCollection/moses-mstranslator/_git/moses-mstranslator

10 files changed, 675 insertions, 65 deletions

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 8eb153e35..fddd0faea 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -58,40 +58,40 @@ steps:
       doxygen 
   displayName: 'Install Ubuntu packages'
 
-# - script: |   
-#     wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download"
-#     mv download  cmph-2.0.2.tar.gz  
-#     tar xvzf cmph-2.0.2.tar.gz
-#     cd cmph-2.0.2
-#     ./configure --prefix=$PWD
-#     make
-#     make install
-#     cd ..
-#   displayName: 'Build and Install cmph'    
-
-# - script: |   
-#     wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download"
-#     mv download  xmlrpc-c-1.51.06.tgz
-#     tar xvzf xmlrpc-c-1.51.06.tgz
-#     cd xmlrpc-c-1.51.06
-#     ./configure --prefix=$PWD
-#     make
-#     make install
-#     sudo ldconfig
-#     cd ..
-#   displayName: 'Build and Install xmlrpc-c'    
+- script: |   
+    wget "https://sourceforge.net/projects/cmph/files/v2.0.2/cmph-2.0.2.tar.gz/download"
+    mv download  cmph-2.0.2.tar.gz  
+    tar xvzf cmph-2.0.2.tar.gz
+    cd cmph-2.0.2
+    ./configure --prefix=$PWD
+    make
+    make install
+    cd ..
+  displayName: 'Build and Install cmph'    
 
-# - script: |   
-#     ./bjam \
-#       --with-cmph=$PWD/cmph-2.0.2 \
-#       --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \
-#       -j2
-#   displayName: 'Build Moses'    
+- script: |   
+    wget "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz/download"
+    mv download  xmlrpc-c-1.51.06.tgz
+    tar xvzf xmlrpc-c-1.51.06.tgz
+    cd xmlrpc-c-1.51.06
+    ./configure --prefix=$PWD
+    make
+    make install
+    sudo ldconfig
+    cd ..
+  displayName: 'Build and Install xmlrpc-c'    
 
 - script: |   
     ./bjam \
+      --with-cmph=$PWD/cmph-2.0.2 \
+      --with-xmlrpc-c=$PWD/xmlrpc-c-1.51.06 \
       -j2
-  displayName: 'Build Moses'  
+  displayName: 'Build Moses'    
+
+# - script: |   
+#     ./bjam \
+#       -j2
+#   displayName: 'Build Moses'  
 
 # - task: ComponentGovernanceComponentDetection@0
 #   inputs:
diff --git a/cgmanifest.json b/cgmanifest.json
index 6e7a9eafb..36cc60cff 100644
--- a/cgmanifest.json
+++ b/cgmanifest.json
@@ -1,33 +1,33 @@
-{
-    "Registrations":[
-        {
-            "component": { 
-             "type": "git", 
-             "git": { 
-               "repositoryUrl": "https://github.com/moses-smt/mosesdecoder", 
-               "commitHash": "78ca5f3cc5aa671a8a5d36c56452e217e6f00828" 
-               }
-             }
-          },
-          {
-            "component": { 
-             "type": "git", 
-             "git": { 
-               "repositoryUrl": "https://git.code.sf.net/p/cmph/git", 
-               "commitHash": "a250982ade093f4eed0552bbdd22dd7b0432007f" 
-               }
-             }
-          },          
-        {
-            "Component": {
-                "Type": "other",
-                "Other": {
-                    "Name": "xml-rpc-c",
-                    "Version": "1.51.06",
-                    "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz"
-                }
-            }
-        },        
-    ]
-}
-
+{

+    "Registrations":[

+        {

+            "component": { 

+             "type": "git", 

+             "git": { 

+               "repositoryUrl": "https://github.com/moses-smt/mosesdecoder", 

+               "commitHash": "78ca5f3cc5aa671a8a5d36c56452e217e6f00828" 

+               }

+             }

+          },

+          {

+            "component": { 

+             "type": "git", 

+             "git": { 

+               "repositoryUrl": "https://git.code.sf.net/p/cmph/git", 

+               "commitHash": "a250982ade093f4eed0552bbdd22dd7b0432007f" 

+               }

+             }

+          },          

+        {

+            "Component": {

+                "Type": "other",

+                "Other": {

+                    "Name": "xml-rpc-c",

+                    "Version": "1.51.06",

+                    "DownloadUrl": "https://sourceforge.net/projects/xmlrpc-c/files/Xmlrpc-c%20Super%20Stable/1.51.06/xmlrpc-c-1.51.06.tgz"

+                }

+            }

+        },        

+    ]

+}

+

diff --git a/moses2/Jamfile b/moses2/Jamfile
index 42676c065..e8a6457fd 100644
--- a/moses2/Jamfile
+++ b/moses2/Jamfile
@@ -108,7 +108,8 @@ alias deps :  ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors
 		PhraseBased/ReorderingConstraint.cpp 
 		PhraseBased/TargetPhrases.cpp
  	 	PhraseBased/Search.cpp
-	    PhraseBased/Sentence.cpp
+    PhraseBased/Sentence.cpp
+    PhraseBased/SentenceWithCandidates.cpp
 		PhraseBased/TargetPhraseImpl.cpp 
  	 	PhraseBased/TrellisPath.cpp
 
diff --git a/moses2/PhraseBased/Manager.cpp b/moses2/PhraseBased/Manager.cpp
index 28073d4f6..bb3c130c5 100644
--- a/moses2/PhraseBased/Manager.cpp
+++ b/moses2/PhraseBased/Manager.cpp
@@ -13,6 +13,7 @@
 #include "TargetPhraseImpl.h"
 #include "InputPath.h"
 #include "Sentence.h"
+#include "SentenceWithCandidates.h"
 
 #include "Normal/Search.h"
 #include "CubePruningMiniStack/Search.h"
@@ -59,7 +60,8 @@ void Manager::Init()
   InitPools();
 
   FactorCollection &vocab = system.GetVocab();
-  m_input = Moses2::Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr);
+  //TODO: need option to choose Sentence vs SentenceWithCandidates
+  m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr);
 
   m_bitmaps = new Bitmaps(GetPool());
 
diff --git a/moses2/PhraseBased/SentenceWithCandidates.cpp b/moses2/PhraseBased/SentenceWithCandidates.cpp
new file mode 100644
index 000000000..c4115cbe8
--- /dev/null
+++ b/moses2/PhraseBased/SentenceWithCandidates.cpp
@@ -0,0 +1,80 @@
+/*
+ * SentenceWithCandidates.cpp
+ *
+ *  Created on: 14 Dec 2015
+ *      Author: hieu
+ */
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+#include <boost/algorithm/string.hpp> 
+
+#include "SentenceWithCandidates.h"
+#include "../System.h"
+#include "../parameters/AllOptions.h"
+#include "../legacy/Util2.h"
+#include <unordered_map>
+
+using namespace std;
+using namespace boost;
+
+namespace Moses2
+{
+
+const string SentenceWithCandidates::INPUT_PART_DELIM = "@@@";
+const string SentenceWithCandidates::PT_LINE_DELIM = "$$$";
+
+SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, FactorCollection &vocab,
+                                     const System &system, const std::string &str)
+{
+  SentenceWithCandidates *ret;
+  
+  // Break input into two parts: the parts are delimited by 
+  typedef split_iterator<string::const_iterator> string_split_iterator;
+  vector<string> input_parts;
+  for(string_split_iterator It= make_split_iterator(str, first_finder(SentenceWithCandidates::INPUT_PART_DELIM, is_iequal()));    
+                It!=string_split_iterator();    
+                ++It)
+  {
+      input_parts.push_back(copy_range<std::string>(*It));
+  }
+
+  cerr << "Number of subparts: " << input_parts.size() << endl;
+
+  if (input_parts.size() ==2 ) {
+      cerr << "correct number of parts" << endl ;
+  } else {
+      // TODO: how to handle wrong input format 
+      cerr << "INCORRECT number of parts" << endl ;
+      exit(1);
+  }
+
+  trim(input_parts[0]);
+  trim(input_parts[1]);
+  cerr << "Input String: " << input_parts[0] << endl ;
+  cerr << "Phrase Table: " << input_parts[1] << endl ;
+
+  ///// Process the text part of the input 
+  const string partstr = input_parts[0];
+ 
+  // no xml
+  //cerr << "PB SentenceWithCandidates" << endl;
+  std::vector<std::string> toks = Tokenize(partstr);
+
+  size_t size = toks.size();
+  ret = new (pool.Allocate<SentenceWithCandidates>()) SentenceWithCandidates(pool, size);
+  ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
+
+  //cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl;
+  //cerr << "ret=" << ret->Debug(system) << endl;
+
+
+  //// Parse the phrase table of the input 
+  ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n");
+  cerr << "Extracted Phrase Table String" << endl; 
+  cerr << ret->m_phraseTableString << endl; 
+
+  return ret;
+}
+
+} /* namespace Moses2 */
+
diff --git a/moses2/PhraseBased/SentenceWithCandidates.h b/moses2/PhraseBased/SentenceWithCandidates.h
new file mode 100644
index 000000000..114ff9e67
--- /dev/null
+++ b/moses2/PhraseBased/SentenceWithCandidates.h
@@ -0,0 +1,45 @@
+/*
+ * SentenceWithCandidates.h
+ *
+ *  Created on: 14 Dec 2015
+ *      Author: hieu
+ */
+#pragma once
+
+#include <boost/property_tree/ptree.hpp>
+#include <string>
+#include "PhraseImpl.h"
+#include "Sentence.h"
+#include "../MemPool.h"
+#include "../pugixml.hpp"
+#include "../legacy/Util2.h"
+
+namespace Moses2
+{
+class FactorCollection;
+class System;
+
+class SentenceWithCandidates: public Sentence
+{
+public:
+
+  static const std::string INPUT_PART_DELIM;
+  static const std::string PT_LINE_DELIM;
+
+  static SentenceWithCandidates *CreateFromString(MemPool &pool, FactorCollection &vocab,
+                                    const System &system, const std::string &str);
+
+  SentenceWithCandidates(MemPool &pool, size_t size)
+    :Sentence(pool, size)
+  {}
+
+  virtual ~SentenceWithCandidates()
+  {}
+
+private:
+  std::string m_phraseTableString;
+
+};
+
+} /* namespace Moses2 */
+
diff --git a/moses2/TranslationModel/MSPT/MSNode.h b/moses2/TranslationModel/MSPT/MSNode.h
new file mode 100644
index 000000000..ad6d0842d
--- /dev/null
+++ b/moses2/TranslationModel/MSPT/MSNode.h
@@ -0,0 +1,131 @@
+/*
+ * Node.h
+ *
+ *  Created on: 22 Apr 2016
+ *      Author: hieu
+ */
+#pragma once
+#include <boost/unordered_map.hpp>
+#include <boost/foreach.hpp>
+#include "../../PhraseBased/TargetPhrases.h"
+#include "../../System.h"
+#include "../../Phrase.h"
+
+namespace Moses2
+{
+class System;
+
+namespace MSPTNS
+{
+
+template<class WORD, class SP, class TP, class TPS>
+class Node
+{
+public:
+  typedef boost::unordered_map<size_t, Node> Children;
+
+  Node()
+    :m_targetPhrases(NULL)
+    ,m_unsortedTPS(NULL)
+  {}
+
+  ~Node()
+  {}
+
+  void AddRule(const std::vector<FactorType> &factors, SP &source, TP *target) {
+    AddRule(factors, source, target, 0);
+  }
+
+  TPS *Find(const std::vector<FactorType> &factors, const SP &source, size_t pos = 0) const {
+    assert(source.GetSize());
+    if (pos == source.GetSize()) {
+      return m_targetPhrases;
+    } else {
+      const WORD &word = source[pos];
+      //cerr << "word=" << word << endl;
+      typename Children::const_iterator iter = m_children.find(word.hash(factors));
+      if (iter == m_children.end()) {
+        return NULL;
+      } else {
+        const Node &child = iter->second;
+        return child.Find(factors, source, pos + 1);
+      }
+    }
+  }
+
+  const Node *Find(const std::vector<FactorType> &factors, const WORD &word) const {
+    typename Children::const_iterator iter = m_children.find(word.hash(factors));
+    if (iter == m_children.end()) {
+      return NULL;
+    } else {
+      const Node &child = iter->second;
+      return &child;
+    }
+  }
+
+  const TPS *GetTargetPhrases() const {
+    return m_targetPhrases;
+  }
+
+  void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) {
+    BOOST_FOREACH(typename Children::value_type &val, m_children) {
+      Node &child = val.second;
+      child.SortAndPrune(tableLimit, pool, system);
+    }
+
+    // prune target phrases in this node
+    if (m_unsortedTPS) {
+      m_targetPhrases = new (pool.Allocate<TPS>()) TPS(pool, m_unsortedTPS->size());
+
+      for (size_t i = 0; i < m_unsortedTPS->size(); ++i) {
+        TP *tp = (*m_unsortedTPS)[i];
+        m_targetPhrases->AddTargetPhrase(*tp);
+      }
+
+      m_targetPhrases->SortAndPrune(tableLimit);
+      system.featureFunctions.EvaluateAfterTablePruning(system.GetSystemPool(), *m_targetPhrases, *m_source);
+
+      delete m_unsortedTPS;
+    }
+  }
+
+  const Children &GetChildren() const {
+    return m_children;
+  }
+
+  void Debug(std::ostream &out, const System &system) const {
+    BOOST_FOREACH(const typename Children::value_type &valPair, m_children) {
+      const WORD &word = valPair.first;
+      //std::cerr << word << "(" << word.hash() << ") ";
+    }
+  }
+protected:
+  Children m_children;
+  TPS *m_targetPhrases;
+  Phrase<WORD> *m_source;
+  std::vector<TP*> *m_unsortedTPS;
+
+  Node &AddRule(const std::vector<FactorType> &factors, SP &source, TP *target, size_t pos) {
+    if (pos == source.GetSize()) {
+      if (m_unsortedTPS == NULL) {
+        m_unsortedTPS = new std::vector<TP*>();
+        m_source = &source;
+      }
+
+      m_unsortedTPS->push_back(target);
+      return *this;
+    } else {
+      const WORD &word = source[pos];
+      Node &child = m_children[word.hash(factors)];
+      //std::cerr << "added " << word << " " << &child << " from " << this << std::endl;
+
+      return child.AddRule(factors, source, target, pos + 1);
+    }
+  }
+
+};
+
+
+}
+} // namespace
+
diff --git a/moses2/TranslationModel/MSPT/MSPT.cpp b/moses2/TranslationModel/MSPT/MSPT.cpp
new file mode 100644
index 000000000..c905d5240
--- /dev/null
+++ b/moses2/TranslationModel/MSPT/MSPT.cpp
@@ -0,0 +1,265 @@
+/*
+ * MSPT.cpp
+ *
+ *  Created on: 28 Oct 2015
+ *      Author: hieu
+ */
+
+#include <cassert>
+#include <boost/foreach.hpp>
+#include "MSPT.h"
+#include "../../PhraseBased/PhraseImpl.h"
+#include "../../Phrase.h"
+#include "../../System.h"
+#include "../../Scores.h"
+#include "../../InputPathsBase.h"
+#include "../../legacy/InputFileStream.h"
+#include "util/exception.hh"
+
+#include "../../PhraseBased/InputPath.h"
+#include "../../PhraseBased/TargetPhraseImpl.h"
+#include "../../PhraseBased/TargetPhrases.h"
+
+#include "../../SCFG/PhraseImpl.h"
+#include "../../SCFG/TargetPhraseImpl.h"
+#include "../../SCFG/InputPath.h"
+#include "../../SCFG/Stack.h"
+#include "../../SCFG/Stacks.h"
+#include "../../SCFG/Manager.h"
+
+
+using namespace std;
+
+namespace Moses2
+{
+
+
+////////////////////////////////////////////////////////////////////////
+
+MSPT::MSPT(size_t startInd, const std::string &line)
+  :PhraseTable(startInd, line)
+  ,m_rootPb(NULL)
+  ,m_rootSCFG(NULL)
+{
+  ReadParameters();
+}
+
+MSPT::~MSPT()
+{
+  delete m_rootPb;
+  delete m_rootSCFG;
+}
+
+void MSPT::Load(System &system)
+{
+  FactorCollection &vocab = system.GetVocab();
+  MemPool &systemPool = system.GetSystemPool();
+  MemPool tmpSourcePool;
+
+  if (system.isPb) {
+    m_rootPb = new PBNODE();
+  } else {
+    m_rootSCFG = new SCFGNODE();
+    //cerr << "m_rootSCFG=" << m_rootSCFG << endl;
+  }
+
+  vector<string> toks;
+  size_t lineNum = 0;
+  InputFileStream strme(m_path);
+  string line;
+  while (getline(strme, line)) {
+    if (++lineNum % 1000000 == 0) {
+      cerr << lineNum << " ";
+    }
+    toks.clear();
+    TokenizeMultiCharSeparator(toks, line, "|||");
+    UTIL_THROW_IF2(toks.size() < 3, "Wrong format");
+    //cerr << "line=" << line << endl;
+    //cerr << "system.isPb=" << system.isPb << endl;
+
+    if (system.isPb) {
+      PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
+                           toks[0]);
+      //cerr << "created soure" << endl;
+      TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system,
+                                 toks[1]);
+      //cerr << "created target" << endl;
+      target->GetScores().CreateFromString(toks[2], *this, system, true);
+      //cerr << "created scores:" << *target << endl;
+
+      if (toks.size() >= 4) {
+        //cerr << "alignstr=" << toks[3] << endl;
+        target->SetAlignmentInfo(toks[3]);
+      }
+
+      // properties
+      if (toks.size() == 7) {
+        //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
+        //strcpy(target->properties, toks[6].c_str());
+      }
+
+      system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
+          *target);
+      //cerr << "EvaluateInIsolation:" << *target << endl;
+      m_rootPb->AddRule(m_input, *source, target);
+
+      //cerr << "target=" << target->Debug(system) << endl;
+    } else {
+      SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
+                                 toks[0]);
+      //cerr << "created source:" << *source << endl;
+      SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this,
+                                       system, toks[1]);
+
+      //cerr << "created target " << *target << " source=" << *source << endl;
+
+      target->GetScores().CreateFromString(toks[2], *this, system, true);
+      //cerr << "created scores:" << *target << endl;
+
+      //vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
+      //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0;
+
+      target->SetAlignmentInfo(toks[3]);
+
+      // properties
+      if (toks.size() == 7) {
+        //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
+        //strcpy(target->properties, toks[6].c_str());
+      }
+
+      system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
+          *target);
+      //cerr << "EvaluateInIsolation:" << *target << endl;
+      m_rootSCFG->AddRule(m_input, *source, target);
+    }
+  }
+
+  if (system.isPb) {
+    m_rootPb->SortAndPrune(m_tableLimit, systemPool, system);
+    //cerr << "root=" << &m_rootPb << endl;
+  } else {
+    m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system);
+    //cerr << "root=" << &m_rootPb << endl;
+  }
+  /*
+  BOOST_FOREACH(const PtMem::Node<Word>::Children::value_type &valPair, m_rootPb.GetChildren()) {
+    const Word &word = valPair.first;
+    cerr << word << " ";
+  }
+  cerr << endl;
+  */
+}
+
+TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool,
+    InputPath &inputPath) const
+{
+  const SubPhrase<Moses2::Word> &phrase = inputPath.subPhrase;
+  TargetPhrases *tps = m_rootPb->Find(m_input, phrase);
+  return tps;
+}
+
+void MSPT::InitActiveChart(
+  MemPool &pool,
+  const SCFG::Manager &mgr,
+  SCFG::InputPath &path) const
+{
+  size_t ptInd = GetPtInd();
+  ActiveChartEntryMem *chartEntry = new (pool.Allocate<ActiveChartEntryMem>()) ActiveChartEntryMem(pool, *m_rootSCFG);
+  path.AddActiveChartEntry(ptInd, chartEntry);
+  //cerr << "InitActiveChart=" << path << endl;
+}
+
+void MSPT::Lookup(MemPool &pool,
+                               const SCFG::Manager &mgr,
+                               size_t maxChartSpan,
+                               const SCFG::Stacks &stacks,
+                               SCFG::InputPath &path) const
+{
+  if (path.range.GetNumWordsCovered() > maxChartSpan) {
+    return;
+  }
+
+  size_t endPos = path.range.GetEndPos();
+
+  const SCFG::InputPath *prevPath = static_cast<const SCFG::InputPath*>(path.prefixPath);
+  UTIL_THROW_IF2(prevPath == NULL, "prefixPath == NULL");
+
+  // TERMINAL
+  const SCFG::Word &lastWord = path.subPhrase.Back();
+
+  const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1);
+
+  //cerr << "BEFORE LookupGivenWord=" << *prevPath << endl;
+  LookupGivenWord(pool, mgr, *prevPath, lastWord, NULL, subPhrasePath.range, path);
+  //cerr << "AFTER LookupGivenWord=" << *prevPath << endl;
+
+  // NON-TERMINAL
+  //const SCFG::InputPath *prefixPath = static_cast<const SCFG::InputPath*>(path.prefixPath);
+  while (prevPath) {
+    const Range &prevRange = prevPath->range;
+    //cerr << "prevRange=" << prevRange << endl;
+
+    size_t startPos = prevRange.GetEndPos() + 1;
+    size_t ntSize = endPos - startPos + 1;
+    const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(startPos, ntSize);
+
+    LookupNT(pool, mgr, subPhrasePath.range, *prevPath, stacks, path);
+
+    prevPath = static_cast<const SCFG::InputPath*>(prevPath->prefixPath);
+  }
+}
+
+void MSPT::LookupGivenNode(
+  MemPool &pool,
+  const SCFG::Manager &mgr,
+  const SCFG::ActiveChartEntry &prevEntry,
+  const SCFG::Word &wordSought,
+  const Moses2::Hypotheses *hypos,
+  const Moses2::Range &subPhraseRange,
+  SCFG::InputPath &outPath) const
+{
+  const ActiveChartEntryMem &prevEntryCast = static_cast<const ActiveChartEntryMem&>(prevEntry);
+
+  const SCFGNODE &prevNode = prevEntryCast.node;
+  UTIL_THROW_IF2(&prevNode == NULL, "node == NULL");
+
+  size_t ptInd = GetPtInd();
+  const SCFGNODE *nextNode = prevNode.Find(m_input, wordSought);
+
+  /*
+  if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) {
+    cerr  << "range=" << outPath.range
+          << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system)
+          << " wordSought=" << wordSought.Debug(mgr.system)
+          << " nextNode=" << nextNode
+          << endl;
+  }
+  */
+  if (nextNode) {
+    // new entries
+    ActiveChartEntryMem *chartEntry = new (pool.Allocate<ActiveChartEntryMem>()) ActiveChartEntryMem(pool, *nextNode, prevEntry);
+
+    chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this);
+    //cerr << "AFTER Add=" << symbolBind << endl;
+
+    outPath.AddActiveChartEntry(ptInd, chartEntry);
+
+    const SCFG::TargetPhrases *tps = nextNode->GetTargetPhrases();
+    if (tps) {
+      // there are some rules
+      /*
+      cerr << "outPath=" << outPath.range
+      	  << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system)
+      	  << " pt=" << GetPtInd()
+        << " tps=" << tps->Debug(mgr.system) << endl;
+      */
+      outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind());
+
+    }
+
+    //cerr << "AFTER outPath=" << outPath << endl;
+  }
+}
+
+}
+
diff --git a/moses2/TranslationModel/MSPT/MSPT.h b/moses2/TranslationModel/MSPT/MSPT.h
new file mode 100644
index 000000000..d3946d353
--- /dev/null
+++ b/moses2/TranslationModel/MSPT/MSPT.h
@@ -0,0 +1,85 @@
+/*
+ * MSPT.h
+ *
+ *  Created on: 28 Oct 2015
+ *      Author: hieu
+ */
+#pragma once
+
+#include "../PhraseTable.h"
+#include "../../legacy/Util2.h"
+#include "../../SCFG/InputPath.h"
+#include "MSNode.h"
+#include "../../PhraseBased/PhraseImpl.h"
+#include "../../PhraseBased/TargetPhraseImpl.h"
+#include "../../PhraseBased/TargetPhrases.h"
+#include "../../SCFG/PhraseImpl.h"
+#include "../../SCFG/TargetPhraseImpl.h"
+#include "../../SCFG/TargetPhrases.h"
+
+namespace Moses2
+{
+
+class MSPT: public PhraseTable
+{
+  typedef MSPTNS::Node<Word, Phrase<Word>, TargetPhraseImpl, TargetPhrases> PBNODE;
+  typedef MSPTNS::Node<SCFG::Word, Phrase<SCFG::Word>, SCFG::TargetPhraseImpl, SCFG::TargetPhrases> SCFGNODE;
+
+//////////////////////////////////////
+  class ActiveChartEntryMem : public SCFG::ActiveChartEntry
+  {
+    typedef SCFG::ActiveChartEntry Parent;
+  public:
+    const MSPT::SCFGNODE &node;
+
+    ActiveChartEntryMem(MemPool &pool, const MSPT::SCFGNODE &vnode)
+      :Parent(pool)
+      ,node(vnode)
+    {}
+
+    ActiveChartEntryMem(
+      MemPool &pool,
+      const MSPT::SCFGNODE &vnode,
+      const ActiveChartEntry &prevEntry)
+      :Parent(prevEntry)
+      ,node(vnode)
+    {}
+  };
+
+  //////////////////////////////////////
+public:
+  MSPT(size_t startInd, const std::string &line);
+  virtual ~MSPT();
+
+  virtual void Load(System &system);
+  virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool,
+                                InputPath &inputPath) const;
+
+  virtual void InitActiveChart(
+    MemPool &pool,
+    const SCFG::Manager &mgr,
+    SCFG::InputPath &path) const;
+
+  void Lookup(MemPool &pool,
+              const SCFG::Manager &mgr,
+              size_t maxChartSpan,
+              const SCFG::Stacks &stacks,
+              SCFG::InputPath &path) const;
+
+protected:
+  PBNODE    *m_rootPb;
+  SCFGNODE  *m_rootSCFG;
+
+  void LookupGivenNode(
+    MemPool &pool,
+    const SCFG::Manager &mgr,
+    const SCFG::ActiveChartEntry &prevEntry,
+    const SCFG::Word &wordSought,
+    const Moses2::Hypotheses *hypos,
+    const Moses2::Range &subPhraseRange,
+    SCFG::InputPath &outPath) const;
+
+};
+
+}
+
diff --git a/moses2/TypeDef.h b/moses2/TypeDef.h
index aed39ac11..d96257ac2 100644
--- a/moses2/TypeDef.h
+++ b/moses2/TypeDef.h
@@ -71,7 +71,8 @@ enum InputTypeEnum {
   TreeInputType         = 3,
   //,WordLatticeInput2 = 4,
   TabbedSentenceInput    = 5,
-  ForestInputType        = 6
+  ForestInputType        = 6,
+  SentenceInputWithCandidates = 7,
 };
 
 enum XmlInputType {