Merge remote-tracking branch 'upstream/master'

65 files changed, 984 insertions, 1628 deletions

diff --git a/moses/FF/SkeletonStatefulFF.cpp b/moses/FF/ExampleStatefulFF.cpp
index 2acaf2d2e..5a53c4f87 100644
--- a/moses/FF/SkeletonStatefulFF.cpp
+++ b/moses/FF/ExampleStatefulFF.cpp
@@ -1,5 +1,5 @@
 #include <vector>
-#include "SkeletonStatefulFF.h"
+#include "ExampleStatefulFF.h"
 #include "moses/ScoreComponentCollection.h"
 #include "moses/Hypothesis.h"
 
@@ -9,7 +9,7 @@ namespace Moses
 {
 
 ////////////////////////////////////////////////////////////////
-SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
+ExampleStatefulFF::ExampleStatefulFF(const std::string &line)
   :StatefulFeatureFunction(3, line)
 {
   ReadParameters();
@@ -19,7 +19,7 @@ SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
 // An empty implementation of this function is provided by StatefulFeatureFunction.
 // Unless you are actually implementing this, please remove it from your
 // implementation (and the declaration in the header file to reduce code clutter.
-void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source
+void ExampleStatefulFF::EvaluateInIsolation(const Phrase &source
     , const TargetPhrase &targetPhrase
     , ScoreComponentCollection &scoreBreakdown
     , ScoreComponentCollection &estimatedScores) const
@@ -28,7 +28,7 @@ void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source
 // An empty implementation of this function is provided by StatefulFeatureFunction.
 // Unless you are actually implementing this, please remove it from your
 // implementation (and the declaration in the header file to reduce code clutter.
-void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input
+void ExampleStatefulFF::EvaluateWithSourceContext(const InputType &input
     , const InputPath &inputPath
     , const TargetPhrase &targetPhrase
     , const StackVec *stackVec
@@ -39,11 +39,11 @@ void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input
 // An empty implementation of this function is provided by StatefulFeatureFunction.
 // Unless you are actually implementing this, please remove it from your
 // implementation (and the declaration in the header file to reduce code clutter.
-void SkeletonStatefulFF::EvaluateTranslationOptionListWithSourceContext
+void ExampleStatefulFF::EvaluateTranslationOptionListWithSourceContext
 (const InputType &input, const TranslationOptionList &translationOptionList) const
 {}
 
-FFState* SkeletonStatefulFF::EvaluateWhenApplied(
+FFState* ExampleStatefulFF::EvaluateWhenApplied(
   const Hypothesis& cur_hypo,
   const FFState* prev_state,
   ScoreComponentCollection* accumulator) const
@@ -59,18 +59,18 @@ FFState* SkeletonStatefulFF::EvaluateWhenApplied(
   accumulator->PlusEquals(this, "sparse-name", 2.4);
 
   // int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize(); // ??? [UG]
-  return new SkeletonState(0);
+  return new ExampleState(0);
 }
 
-FFState* SkeletonStatefulFF::EvaluateWhenApplied(
+FFState* ExampleStatefulFF::EvaluateWhenApplied(
   const ChartHypothesis& /* cur_hypo */,
   int /* featureID - used to index the state in the previous hypotheses */,
   ScoreComponentCollection* accumulator) const
 {
-  return new SkeletonState(0);
+  return new ExampleState(0);
 }
 
-void SkeletonStatefulFF::SetParameter(const std::string& key, const std::string& value)
+void ExampleStatefulFF::SetParameter(const std::string& key, const std::string& value)
 {
   if (key == "arg") {
     // set value here
diff --git a/moses/FF/SkeletonStatefulFF.h b/moses/FF/ExampleStatefulFF.h
index 7544ddd30..d66274295 100644
--- a/moses/FF/SkeletonStatefulFF.h
+++ b/moses/FF/ExampleStatefulFF.h
@@ -7,11 +7,11 @@
 namespace Moses
 {
 
-class SkeletonState : public FFState
+class ExampleState : public FFState
 {
   int m_targetLen;
 public:
-  SkeletonState(int targetLen)
+  ExampleState(int targetLen)
     :m_targetLen(targetLen) {
   }
 
@@ -19,22 +19,22 @@ public:
     return (size_t) m_targetLen;
   }
   virtual bool operator==(const FFState& o) const {
-    const SkeletonState& other = static_cast<const SkeletonState&>(o);
+    const ExampleState& other = static_cast<const ExampleState&>(o);
     return m_targetLen == other.m_targetLen;
   }
 
 };
 
-class SkeletonStatefulFF : public StatefulFeatureFunction
+class ExampleStatefulFF : public StatefulFeatureFunction
 {
 public:
-  SkeletonStatefulFF(const std::string &line);
+  ExampleStatefulFF(const std::string &line);
 
   bool IsUseable(const FactorMask &mask) const {
     return true;
   }
   virtual const FFState* EmptyHypothesisState(const InputType &input) const {
-    return new SkeletonState(0);
+    return new ExampleState(0);
   }
 
   // An empty implementation of this function is provided by StatefulFeatureFunction.
diff --git a/moses/FF/SkeletonStatelessFF.cpp b/moses/FF/ExampleStatelessFF.cpp
index 8474efe76..0e62ad0ad 100644
--- a/moses/FF/SkeletonStatelessFF.cpp
+++ b/moses/FF/ExampleStatelessFF.cpp
@@ -1,5 +1,5 @@
 #include <vector>
-#include "SkeletonStatelessFF.h"
+#include "ExampleStatelessFF.h"
 #include "moses/ScoreComponentCollection.h"
 #include "moses/TargetPhrase.h"
 
@@ -7,13 +7,13 @@ using namespace std;
 
 namespace Moses
 {
-SkeletonStatelessFF::SkeletonStatelessFF(const std::string &line)
+ExampleStatelessFF::ExampleStatelessFF(const std::string &line)
   :StatelessFeatureFunction(2, line)
 {
   ReadParameters();
 }
 
-void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source
+void ExampleStatelessFF::EvaluateInIsolation(const Phrase &source
     , const TargetPhrase &targetPhrase
     , ScoreComponentCollection &scoreBreakdown
     , ScoreComponentCollection &estimatedScores) const
@@ -29,7 +29,7 @@ void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source
 
 }
 
-void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
+void ExampleStatelessFF::EvaluateWithSourceContext(const InputType &input
     , const InputPath &inputPath
     , const TargetPhrase &targetPhrase
     , const StackVec *stackVec
@@ -43,20 +43,20 @@ void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
   }
 }
 
-void SkeletonStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input
+void ExampleStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input
 
     , const TranslationOptionList &translationOptionList) const
 {}
 
-void SkeletonStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
+void ExampleStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
     ScoreComponentCollection* accumulator) const
 {}
 
-void SkeletonStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
+void ExampleStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
     ScoreComponentCollection* accumulator) const
 {}
 
-void SkeletonStatelessFF::SetParameter(const std::string& key, const std::string& value)
+void ExampleStatelessFF::SetParameter(const std::string& key, const std::string& value)
 {
   if (key == "arg") {
     // set value here
diff --git a/moses/FF/SkeletonStatelessFF.h b/moses/FF/ExampleStatelessFF.h
index 0dc46e214..e1f007d21 100644
--- a/moses/FF/SkeletonStatelessFF.h
+++ b/moses/FF/ExampleStatelessFF.h
@@ -6,10 +6,10 @@
 namespace Moses
 {
 
-class SkeletonStatelessFF : public StatelessFeatureFunction
+class ExampleStatelessFF : public StatelessFeatureFunction
 {
 public:
-  SkeletonStatelessFF(const std::string &line);
+  ExampleStatelessFF(const std::string &line);
 
   bool IsUseable(const FactorMask &mask) const {
     return true;
diff --git a/moses/FF/SkeletonTranslationOptionListFeature.h b/moses/FF/ExampleTranslationOptionListFeature.h
index e47e691aa..7686eb3ff 100644
--- a/moses/FF/SkeletonTranslationOptionListFeature.h
+++ b/moses/FF/ExampleTranslationOptionListFeature.h
@@ -6,10 +6,10 @@
 namespace Moses
 {
 
-class SkeletonTranslationOptionListFeature : public StatelessFeatureFunction
+class ExampleTranslationOptionListFeature : public StatelessFeatureFunction
 {
 public:
-  SkeletonTranslationOptionListFeature(const std::string &line)
+  ExampleTranslationOptionListFeature(const std::string &line)
     :StatelessFeatureFunction(1, line) {
     ReadParameters();
   }
diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp
index 2c4668377..00dcac27d 100644
--- a/moses/FF/Factory.cpp
+++ b/moses/FF/Factory.cpp
@@ -15,8 +15,9 @@
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h"
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h"
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
-#include "moses/TranslationModel/ProbingPT/ProbingPT.h"
+#include "moses/TranslationModel/ProbingPT.h"
 #include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h"
+#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h"
 
 #include "moses/FF/LexicalReordering/LexicalReordering.h"
 
@@ -65,15 +66,16 @@
 #include "SyntaxRHS.h"
 #include "DeleteRules.h"
 
-#include "moses/FF/SkeletonStatelessFF.h"
-#include "moses/FF/SkeletonStatefulFF.h"
-#include "moses/LM/SkeletonLM.h"
-#include "moses/FF/SkeletonTranslationOptionListFeature.h"
+#include "moses/FF/ExampleStatelessFF.h"
+#include "moses/FF/ExampleStatefulFF.h"
+#include "moses/LM/ExampleLM.h"
+#include "moses/FF/ExampleTranslationOptionListFeature.h"
 #include "moses/LM/BilingualLM.h"
-#include "moses/TranslationModel/SkeletonPT.h"
+#include "moses/TranslationModel/ExamplePT.h"
 #include "moses/Syntax/InputWeightFF.h"
 #include "moses/Syntax/RuleTableFF.h"
 
+#include "moses/LM/InMemoryPerSentenceOnDemandLM.h"
 #include "moses/FF/EditOps.h"
 #include "moses/FF/CorrectionPattern.h"
 
@@ -246,6 +248,7 @@ FeatureRegistry::FeatureRegistry()
   MOSES_FNAME(PhraseDictionaryFuzzyMatch);
   MOSES_FNAME(ProbingPT);
   MOSES_FNAME(PhraseDictionaryMemoryPerSentence);
+  MOSES_FNAME(PhraseDictionaryMemoryPerSentenceOnDemand);
   MOSES_FNAME2("RuleTable", Syntax::RuleTableFF);
   MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF);
 
@@ -296,12 +299,13 @@ FeatureRegistry::FeatureRegistry()
   MOSES_FNAME(UnalignedWordCountFeature);
   MOSES_FNAME(DeleteRules);
 
-  MOSES_FNAME(SkeletonStatelessFF);
-  MOSES_FNAME(SkeletonStatefulFF);
-  MOSES_FNAME(SkeletonLM);
-  MOSES_FNAME(SkeletonTranslationOptionListFeature);
-  MOSES_FNAME(SkeletonPT);
+  MOSES_FNAME(ExampleStatelessFF);
+  MOSES_FNAME(ExampleStatefulFF);
+  MOSES_FNAME(ExampleLM);
+  MOSES_FNAME(ExampleTranslationOptionListFeature);
+  MOSES_FNAME(ExamplePT);
 
+  MOSES_FNAME(InMemoryPerSentenceOnDemandLM);
   MOSES_FNAME(EditOps);
   MOSES_FNAME(CorrectionPattern);
 
diff --git a/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp b/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp
index 22f550ba8..5d264e4c8 100644
--- a/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp
+++ b/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp
@@ -15,7 +15,7 @@ size_t BidirectionalReorderingState::hash() const
 
 bool BidirectionalReorderingState::operator==(const FFState& o) const
 {
-  if (&o == this) return 0;
+  if (&o == this) return true;
 
   BidirectionalReorderingState const &other
   = static_cast<BidirectionalReorderingState const&>(o);
diff --git a/moses/GenerationDictionary.cpp b/moses/GenerationDictionary.cpp
index 29a4fa2b3..35546e62c 100644
--- a/moses/GenerationDictionary.cpp
+++ b/moses/GenerationDictionary.cpp
@@ -120,7 +120,14 @@ const OutputWordCollection *GenerationDictionary::FindWord(const Word &word) con
 {
   const OutputWordCollection *ret;
 
-  Collection::const_iterator iter = m_collection.find(&word);
+  Word wordInput;
+  const std::vector<FactorType> &inputFactors = GetInput();
+  for (size_t i = 0; i < inputFactors.size(); ++i) {
+    FactorType factorType = inputFactors[i];
+    wordInput[factorType] = word[factorType];
+  }
+
+  Collection::const_iterator iter = m_collection.find(&wordInput);
   if (iter == m_collection.end()) {
     // can't find source phrase
     ret = NULL;
diff --git a/moses/Jamfile b/moses/Jamfile
index 49aab9025..5200029fb 100644
--- a/moses/Jamfile
+++ b/moses/Jamfile
@@ -122,10 +122,10 @@ vwfiles synlm mmlib mserver headers
 FF_Factory.o 
 LM//LM 
 TranslationModel/CompactPT//CompactPT 
-TranslationModel/ProbingPT//ProbingPT 
 ThreadPool
 ..//search 
 ../util/double-conversion//double-conversion 
+../probingpt//probingpt
 ..//z 
 ../OnDiskPt//OnDiskPt 
 $(TOP)//boost_filesystem 
@@ -139,5 +139,5 @@ alias headers-to-install : [ glob-tree *.h ] ;
 
 import testing ;
 
-unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;
+unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ../probingpt//probingpt ..//boost_unit_test_framework ;
 
diff --git a/moses/LM/SkeletonLM.cpp b/moses/LM/ExampleLM.cpp
index 23958e688..034afef2e 100644
--- a/moses/LM/SkeletonLM.cpp
+++ b/moses/LM/ExampleLM.cpp
@@ -1,16 +1,19 @@
 
-#include "SkeletonLM.h"
+#include "ExampleLM.h"
 #include "moses/FactorCollection.h"
 
 using namespace std;
 
 namespace Moses
 {
-SkeletonLM::SkeletonLM(const std::string &line)
+ExampleLM::ExampleLM(const std::string &line)
   :LanguageModelSingleFactor(line)
 {
   ReadParameters();
 
+  UTIL_THROW_IF2(m_nGramOrder == NOT_FOUND, "Must set order");
+  UTIL_THROW_IF2(m_nGramOrder <= 1, "Ngram order must be more than 1");
+
   FactorCollection &factorCollection = FactorCollection::Instance();
 
   // needed by parent language model classes. Why didn't they set these themselves?
@@ -21,11 +24,11 @@ SkeletonLM::SkeletonLM(const std::string &line)
   m_sentenceEndWord[m_factorType] = m_sentenceEnd;
 }
 
-SkeletonLM::~SkeletonLM()
+ExampleLM::~ExampleLM()
 {
 }
 
-LMResult SkeletonLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
+LMResult ExampleLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
 {
   LMResult ret;
   ret.score = contextFactor.size();
diff --git a/moses/LM/SkeletonLM.h b/moses/LM/ExampleLM.h
index 988c9def9..292462917 100644
--- a/moses/LM/SkeletonLM.h
+++ b/moses/LM/ExampleLM.h
@@ -7,13 +7,13 @@
 namespace Moses
 {
 
-class SkeletonLM : public LanguageModelSingleFactor
+class ExampleLM : public LanguageModelSingleFactor
 {
 protected:
 
 public:
-  SkeletonLM(const std::string &line);
-  ~SkeletonLM();
+  ExampleLM(const std::string &line);
+  ~ExampleLM();
 
   virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
 };
diff --git a/moses/LM/Implementation.cpp b/moses/LM/Implementation.cpp
index eb67100ca..399a270d7 100644
--- a/moses/LM/Implementation.cpp
+++ b/moses/LM/Implementation.cpp
@@ -42,6 +42,7 @@ namespace Moses
 {
 LanguageModelImplementation::LanguageModelImplementation(const std::string &line)
   :LanguageModel(line)
+  ,m_nGramOrder(NOT_FOUND)
 {
 }
 
@@ -61,7 +62,7 @@ void LanguageModelImplementation::ShiftOrPush(std::vector<const Word*> &contextF
 {
   if (contextFactor.size() < GetNGramOrder()) {
     contextFactor.push_back(&word);
-  } else {
+  } else if (GetNGramOrder() > 0) {
     // shift
     for (size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder() - 1 ; currNGramOrder++) {
       contextFactor[currNGramOrder] = contextFactor[currNGramOrder + 1];
diff --git a/moses/LM/InMemoryPerSentenceOnDemandLM.cpp b/moses/LM/InMemoryPerSentenceOnDemandLM.cpp
new file mode 100644
index 000000000..0c9a43539
--- /dev/null
+++ b/moses/LM/InMemoryPerSentenceOnDemandLM.cpp
@@ -0,0 +1,101 @@
+#include <boost/foreach.hpp>
+#include "InMemoryPerSentenceOnDemandLM.h"
+#include "moses/FactorCollection.h"
+#include "moses/Util.h"
+#include "moses/StaticData.h"
+#include "moses/TranslationTask.h"
+#include "moses/ContextScope.h"
+#include "moses/LM/Ken.h"
+#include "lm/model.hh"
+#include "util/mmap.hh"
+
+#include <cstdio>
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+namespace Moses
+{
+InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line)
+{
+  ReadParameters();
+}
+
+InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM()
+{
+}
+
+void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask)
+{
+
+  // The context scope object for this translation task
+  //     contains a map of translation task-specific data
+  boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
+
+  // The key to the map is this object
+  void const* key = static_cast<void const*>(this);
+
+  // The value stored in the map is a string representing a phrase table
+  boost::shared_ptr<string> value = contextScope->get<string>(key);
+
+  // Create a stream to read the phrase table data
+  stringstream strme(*(value.get()));
+
+  char * nullpointer = (char *) 0;
+  const char * filename = std::tmpnam(nullpointer);
+  ofstream tmp;
+  tmp.open(filename);
+
+  // Read the phrase table data, one line at a time
+  string line;
+  while (getline(strme, line)) {
+
+    tmp << line << "\n";
+
+  }
+
+  tmp.close();
+
+  //  m_tmpFilename.reset(new std::string("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm"));
+  m_tmpFilename.reset(new std::string(filename));
+
+  //LanguageModelKen<lm::ngram::ProbingModel> & lm =
+  GetPerThreadLM();
+
+  //  std::remove(filename);
+
+}
+
+LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPerThreadLM() const
+{
+
+  LanguageModelKen<lm::ngram::ProbingModel> *lm;
+  lm = m_perThreadLM.get();
+  if (lm == NULL) {
+    lm = new LanguageModelKen<lm::ngram::ProbingModel>();
+
+    string* filename = m_tmpFilename.get();
+    if (filename == NULL) {
+      UTIL_THROW(util::Exception, "Can't get a thread-specific LM because no temporary filename has been set for this thread\n");
+    } else {
+      lm->LoadModel(*filename, util::POPULATE_OR_READ);
+    }
+
+    VERBOSE(1, filename);
+    VERBOSE(1, "\tLM initialized\n");
+
+    m_perThreadLM.reset(lm);
+  }
+  assert(lm);
+
+  return *lm;
+
+}
+
+
+
+}
+
+
+
diff --git a/moses/LM/InMemoryPerSentenceOnDemandLM.h b/moses/LM/InMemoryPerSentenceOnDemandLM.h
new file mode 100644
index 000000000..7f4b60a5f
--- /dev/null
+++ b/moses/LM/InMemoryPerSentenceOnDemandLM.h
@@ -0,0 +1,166 @@
+// $Id$
+#pragma once
+
+#include <string>
+#include <vector>
+#include "SingleFactor.h"
+#include <boost/thread/tss.hpp>
+#include "lm/model.hh"
+#include "moses/LM/Ken.h"
+#include "moses/FF/FFState.h"
+
+namespace Moses
+{
+
+struct InMemoryPerSentenceOnDemandLMState : public FFState {
+  lm::ngram::State state;
+  virtual size_t hash() const {
+    size_t ret = hash_value(state);
+    return ret;
+  }
+  virtual bool operator==(const FFState& o) const {
+    const InMemoryPerSentenceOnDemandLMState &other = static_cast<const InMemoryPerSentenceOnDemandLMState &>(o);
+    bool ret = state == other.state;
+    return ret;
+  }
+
+};
+
+class InMemoryPerSentenceOnDemandLM : public LanguageModel
+{
+public:
+  InMemoryPerSentenceOnDemandLM(const std::string &line);
+  ~InMemoryPerSentenceOnDemandLM();
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  virtual void SetParameter(const std::string& key, const std::string& value) {
+    GetPerThreadLM().SetParameter(key, value);
+  }
+
+  virtual const FFState* EmptyHypothesisState(const InputType &input) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EmptyHypothesisState(input);
+    } else {
+      return new InMemoryPerSentenceOnDemandLMState();
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+
+  virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
+    if (isInitialized()) {
+      GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScore called prior to being initialized");
+    }
+  }
+
+  virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
+    if (isInitialized()) {
+      GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScoreFromCache called prior to being initialized");
+    }
+  }
+
+  virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) {
+    if (isInitialized()) {
+      GetPerThreadLM().IssueRequestsFor(hypo, input_state);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IssueRequestsFor called prior to being initialized");
+    }
+  }
+
+  virtual void sync() {
+    if (isInitialized()) {
+      GetPerThreadLM().sync();
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::sync called prior to being initialized");
+    }
+  }
+
+  virtual void SetFFStateIdx(int state_idx) {
+    if (isInitialized()) {
+      GetPerThreadLM().SetFFStateIdx(state_idx);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::SetFFStateIdx called prior to being initialized");
+    }
+  }
+
+  virtual void IncrementalCallback(Incremental::Manager &manager) const {
+    if (isInitialized()) {
+      GetPerThreadLM().IncrementalCallback(manager);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IncrementalCallback called prior to being initialized");
+    }
+  }
+
+  virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
+    if (isInitialized()) {
+      GetPerThreadLM().ReportHistoryOrder(out, phrase);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::ReportHistoryOrder called prior to being initialized");
+    }
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const {
+    if (isInitialized()) {
+      GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores);
+    } else {
+      //      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::EvaluateInIsolation called prior to being initialized");
+    }
+  }
+
+  bool IsUseable(const FactorMask &mask) const {
+    bool ret = mask[m_factorType];
+    return ret;
+  }
+
+
+protected:
+  LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const;
+
+  mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM;
+  mutable boost::thread_specific_ptr<std::string> m_tmpFilename;
+
+  FactorType m_factorType;
+
+  bool isInitialized() const {
+    if (m_tmpFilename.get() == NULL) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+};
+
+
+}
diff --git a/moses/LM/Jamfile b/moses/LM/Jamfile
index 75b66603c..0c152d555 100644
--- a/moses/LM/Jamfile
+++ b/moses/LM/Jamfile
@@ -138,7 +138,7 @@ if $(with-dalm) {
 
 #Top-level LM library.  If you've added a file that doesn't depend on external
 #libraries, put it here.  
-alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp 
+alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ExampleLM.cpp
   ../../lm//kenlm ..//headers $(dependencies) ;
 
 alias macros : : : : <define>$(lmmacros) ;
diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp
index c7ac663cc..e42e60274 100644
--- a/moses/LM/Ken.cpp
+++ b/moses/LM/Ken.cpp
@@ -105,6 +105,7 @@ template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string
   config.load_method = load_method;
 
   m_ngram.reset(new Model(file.c_str(), config));
+  VERBOSE(2, "LanguageModelKen " << m_description << " reset to " << file << "\n");
 }
 
 template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
@@ -116,6 +117,15 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::stri
   LoadModel(file, load_method);
 }
 
+template <class Model> LanguageModelKen<Model>::LanguageModelKen()
+  :LanguageModel("KENLM")
+  ,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_))
+  ,m_factorType(0)
+{
+  ReadParameters();
+}
+
+
 template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> &copy_from)
   :LanguageModel(copy_from.GetArgLine()),
    m_ngram(copy_from.m_ngram),
diff --git a/moses/LM/Ken.h b/moses/LM/Ken.h
index 4934228c2..33590d659 100644
--- a/moses/LM/Ken.h
+++ b/moses/LM/Ken.h
@@ -33,11 +33,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "moses/TypeDef.h"
 #include "moses/Word.h"
 
+
+
 namespace Moses
 {
 
 //class LanguageModel;
 class FFState;
+class InMemoryPerSentenceOnDemandLM;
 
 LanguageModel *ConstructKenLM(const std::string &line);
 
@@ -67,6 +70,8 @@ public:
 
   virtual bool IsUseable(const FactorMask &mask) const;
 
+  friend class InMemoryPerSentenceOnDemandLM;
+
 protected:
   boost::shared_ptr<Model> m_ngram;
 
@@ -84,6 +89,7 @@ protected:
   std::vector<lm::WordIndex> m_lmIdLookup;
 
 private:
+  LanguageModelKen();
   LanguageModelKen(const LanguageModelKen<Model> &copy_from);
 
   // Convert last words of hypothesis into vocab ids, returning an end pointer.
diff --git a/moses/LM/Reloading.h b/moses/LM/Reloading.h
index d5ae83d17..88f8e8869 100644
--- a/moses/LM/Reloading.h
+++ b/moses/LM/Reloading.h
@@ -35,113 +35,48 @@ namespace Moses
 {
 
 class FFState;
-
-//LanguageModel *ConstructReloadingLM(const std::string &line);
-//LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
-/*
- namespace {
-class MappingBuilder : public lm::EnumerateVocab
-{
-public:
-  MappingBuilder(FactorCollection &factorCollection, std::vector<lm::WordIndex> &mapping)
-    : m_factorCollection(factorCollection), m_mapping(mapping) {}
-
-  void Add(lm::WordIndex index, const StringPiece &str) {
-    std::size_t factorId = m_factorCollection.AddFactor(str)->GetId();
-    if (m_mapping.size() <= factorId) {
-      // 0 is <unk> :-)
-      m_mapping.resize(factorId + 1);
-    }
-    m_mapping[factorId] = index;
-  }
-
-private:
-  FactorCollection &m_factorCollection;
-  std::vector<lm::WordIndex> &m_mapping;
-};
- }
-*/
 template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model>
 {
 public:
-  // TODO(Lane) copy less code, update to load_method
+
   ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line, file, factorType, lazy ? util::LAZY : util::POPULATE_OR_READ), m_file(file), m_lazy(lazy) {
 
-    std::cerr << "ReloadingLM constructor: " << m_file << std::endl;
-    //    std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl;
+    VERBOSE(1, "ReloadingLM constructor: " << m_file << std::endl);
 
   }
 
   virtual void InitializeForInput(ttasksptr const& ttask) {
-    std::cerr << "ReloadingLM InitializeForInput" << std::endl;
-    // TODO(lane): load_method
-    LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ);
-    /*
-    lm::ngram::Config config;
-    if(this->m_verbosity >= 1) {
-      config.messages = &std::cerr;
-    } else {
-      config.messages = NULL;
-    }
-    FactorCollection &collection = FactorCollection::Instance();
-    MappingBuilder builder(collection, m_lmIdLookup);
-    config.enumerate_vocab = &builder;
-    config.load_method = m_lazy ? util::LAZY : util::POPULATE_OR_READ;
-
-    m_ngram.reset(new Model(m_file.c_str(), config));
-
-    m_beginSentenceFactor = collection.AddFactor(BOS_);
-    */
-  };
+    VERBOSE(1, "ReloadingLM InitializeForInput" << std::endl);
 
-  /*
-  ReloadingLanguageModel(const std::string &line) : LanguageModelKen<Model>(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) {
-    std::cerr << "ReloadingLM constructor" << std::endl;
-    std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl;
-  }
-  */
-  /*
-  ~ReloadingLanguageModel() {
-    delete m_lm;
-  }
+    // The context scope object for this translation task
+    //     contains a map of translation task-specific data
+    boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
 
-  virtual const FFState *EmptyHypothesisState(const InputType &input) const {
-    return m_lm->EmptyHypothesisState(input);
-  }
+    // The key to the map is this object
+    void const* key = static_cast<void const*>(this);
 
-  virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
-    m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount);
-  }
+    // The value stored in the map is a string representing a phrase table
+    boost::shared_ptr<string> value = contextScope->get<string>(key);
 
-  virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
-    return m_lm->EvaluateWhenApplied(hypo, ps, out);
-  }
-
-  virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
-    return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator);
-  }
+    // Create a stream to read the phrase table data
+    stringstream strme(*(value.get()));
 
-  virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
-    return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator);
-  }
+    ofstream tmp;
+    tmp.open(m_file.c_str());
 
-  virtual void IncrementalCallback(Incremental::Manager &manager) const {
-    m_lm->IncrementalCallback(manager);
-  }
+    // Read the phrase table data, one line at a time
+    string line;
+    while (getline(strme, line)) {
 
-  virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
-    m_lm->ReportHistoryOrder(out, phrase);
-  }
+      tmp << line << "\n";
 
-  virtual bool IsUseable(const FactorMask &mask) const {
-    return m_lm->IsUseable(mask);
-  }
+    }
 
+    tmp.close();
 
-  private:
+    LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ);
+  };
 
-  LanguageModel *m_lm;
-  */
 
 protected:
 
diff --git a/moses/Manager.cpp b/moses/Manager.cpp
index 3650baabb..0cd540991 100644
--- a/moses/Manager.cpp
+++ b/moses/Manager.cpp
@@ -1724,7 +1724,7 @@ OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) c
     for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
       const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
       if (factor) out << fd << *factor;
-      else        out << fd << UNKNOWN_FACTOR;
+      //else        out << fd << UNKNOWN_FACTOR;
     }
 
     if(markUnknown && word.IsOOV()) {
diff --git a/moses/OutputCollector.h b/moses/OutputCollector.h
index 0d6f37472..797cc85cf 100644
--- a/moses/OutputCollector.h
+++ b/moses/OutputCollector.h
@@ -32,6 +32,7 @@
 #include <iostream>
 #include <map>
 #include <ostream>
+#include <fstream>
 #include <string>
 #include "Util.h"
 #include "util/exception.hh"
diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp
index 4d3e96000..b65c22eb4 100644
--- a/moses/StaticData.cpp
+++ b/moses/StaticData.cpp
@@ -423,8 +423,13 @@ LoadDecodeGraphsOld(const vector<string> &mappingVector,
     if (m_decodeGraphs.size() < decodeGraphInd + 1) {
       DecodeGraph *decodeGraph;
       if (is_syntax(m_options->search.algo)) {
-        size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN;
-        VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
+        size_t maxChartSpan;
+        if (decodeGraphInd < maxChartSpans.size()) {
+          maxChartSpan = maxChartSpans[decodeGraphInd];
+          VERBOSE(1,"max-chart-span: " << maxChartSpans[decodeGraphInd] << endl);
+        } else {
+          maxChartSpan = DEFAULT_MAX_CHART_SPAN;
+        }
         decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan);
       } else {
         decodeGraph = new DecodeGraph(m_decodeGraphs.size());
diff --git a/moses/TrainingTask.h b/moses/TrainingTask.h
index 4d2152920..83933691d 100644
--- a/moses/TrainingTask.h
+++ b/moses/TrainingTask.h
@@ -39,6 +39,18 @@ public:
          boost::shared_ptr<IOWrapper> const& ioWrapper) {
     boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper));
     ret->m_self = ret;
+    ret->m_scope.reset(new ContextScope);
+    return ret;
+  }
+
+  // factory function
+  static boost::shared_ptr<TrainingTask>
+  create(boost::shared_ptr<InputType> const& source,
+         boost::shared_ptr<IOWrapper> const& ioWrapper,
+         boost::shared_ptr<ContextScope> const& scope) {
+    boost::shared_ptr<TrainingTask> ret(new TrainingTask(source, ioWrapper));
+    ret->m_self = ret;
+    ret->m_scope = scope;
     return ret;
   }
 
diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp
index ca219f249..6c80e30af 100644
--- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp
+++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp
@@ -18,7 +18,7 @@
  ***********************************************************************/
 
 #include <iostream>
-#include "ChartRuleLookupManagerSkeleton.h"
+#include "ChartRuleLookupManagerExample.h"
 #include "DotChartInMemory.h"
 
 #include "moses/Util.h"
@@ -29,29 +29,29 @@
 #include "moses/NonTerminal.h"
 #include "moses/ChartCellCollection.h"
 #include "moses/TranslationModel/PhraseDictionaryMemory.h"
-#include "moses/TranslationModel/SkeletonPT.h"
+#include "moses/TranslationModel/ExamplePT.h"
 
 using namespace std;
 
 namespace Moses
 {
 
-ChartRuleLookupManagerSkeleton::ChartRuleLookupManagerSkeleton(
+ChartRuleLookupManagerExample::ChartRuleLookupManagerExample(
   const ChartParser &parser,
   const ChartCellCollectionBase &cellColl,
-  const SkeletonPT &skeletonPt)
+  const ExamplePT &skeletonPt)
   : ChartRuleLookupManager(parser, cellColl)
   , m_skeletonPT(skeletonPt)
 {
-  cerr << "starting ChartRuleLookupManagerSkeleton" << endl;
+  cerr << "starting ChartRuleLookupManagerExample" << endl;
 }
 
-ChartRuleLookupManagerSkeleton::~ChartRuleLookupManagerSkeleton()
+ChartRuleLookupManagerExample::~ChartRuleLookupManagerExample()
 {
   // RemoveAllInColl(m_tpColl);
 }
 
-void ChartRuleLookupManagerSkeleton::GetChartRuleCollection(
+void ChartRuleLookupManagerExample::GetChartRuleCollection(
   const InputPath &inputPath,
   size_t last,
   ChartParserCallback &outColl)
@@ -74,12 +74,12 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection(
 }
 
 TargetPhrase *
-ChartRuleLookupManagerSkeleton::
+ChartRuleLookupManagerExample::
 CreateTargetPhrase(const Word &sourceWord) const
 {
-  // create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:'
+  // create a target phrase from the 1st word of the source, prefix with 'ChartManagerExample:'
   string str = sourceWord.GetFactor(0)->GetString().as_string();
-  str = "ChartManagerSkeleton:" + str;
+  str = "ChartManagerExample:" + str;
 
   TargetPhrase *tp = new TargetPhrase(&m_skeletonPT);
   Word &word = tp->AddWord();
diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h
index d01f3b9bd..3b3f59ace 100644
--- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h
+++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h
@@ -29,16 +29,16 @@ class TargetPhraseCollection;
 class ChartParserCallback;
 class DottedRuleColl;
 class Range;
-class SkeletonPT;
+class ExamplePT;
 
-class ChartRuleLookupManagerSkeleton : public ChartRuleLookupManager
+class ChartRuleLookupManagerExample  : public ChartRuleLookupManager
 {
 public:
-  ChartRuleLookupManagerSkeleton(const ChartParser &parser,
-                                 const ChartCellCollectionBase &cellColl,
-                                 const SkeletonPT &skeletonPt);
+  ChartRuleLookupManagerExample(const ChartParser &parser,
+                                const ChartCellCollectionBase &cellColl,
+                                const ExamplePT &skeletonPt);
 
-  ~ChartRuleLookupManagerSkeleton();
+  ~ChartRuleLookupManagerExample();
 
   virtual void GetChartRuleCollection(
     const InputPath &inputPath,
@@ -50,7 +50,7 @@ private:
 
   StackVec m_stackVec;
   std::vector<TargetPhraseCollection::shared_ptr > m_tpColl;
-  const SkeletonPT &m_skeletonPT;
+  const ExamplePT &m_skeletonPT;
 };
 
 }  // namespace Moses
diff --git a/moses/TranslationModel/SkeletonPT.cpp b/moses/TranslationModel/ExamplePT.cpp
index 6b42212f9..198ce2814 100644
--- a/moses/TranslationModel/SkeletonPT.cpp
+++ b/moses/TranslationModel/ExamplePT.cpp
@@ -1,29 +1,29 @@
 // vim:tabstop=2
-#include "SkeletonPT.h"
-#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
+#include "ExamplePT.h"
+#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h"
 
 using namespace std;
 
 namespace Moses
 {
-SkeletonPT::SkeletonPT(const std::string &line)
+ExamplePT::ExamplePT(const std::string &line)
   : PhraseDictionary(line, true)
 {
   ReadParameters();
 }
 
-void SkeletonPT::Load(AllOptions::ptr const& opts)
+void ExamplePT::Load(AllOptions::ptr const& opts)
 {
   m_options = opts;
   SetFeaturesToApply();
 }
 
-void SkeletonPT::InitializeForInput(ttasksptr const& ttask)
+void ExamplePT::InitializeForInput(ttasksptr const& ttask)
 {
   ReduceCache();
 }
 
-void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
+void ExamplePT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
 {
   CacheColl &cache = GetCache();
 
@@ -46,14 +46,14 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQu
   }
 }
 
-TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
+TargetPhrase *ExamplePT::CreateTargetPhrase(const Phrase &sourcePhrase) const
 {
-  // create a target phrase from the 1st word of the source, prefix with 'SkeletonPT:'
+  // create a target phrase from the 1st word of the source, prefix with 'ExamplePT:'
   assert(sourcePhrase.GetSize());
   assert(m_output.size() == 1);
 
   string str = sourcePhrase.GetWord(0).GetFactor(0)->GetString().as_string();
-  str = "SkeletonPT:" + str;
+  str = "ExamplePT:" + str;
 
   TargetPhrase *tp = new TargetPhrase(this);
   Word &word = tp->AddWord();
@@ -69,17 +69,17 @@ TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
   return tp;
 }
 
-ChartRuleLookupManager* SkeletonPT::CreateRuleLookupManager(const ChartParser &parser,
+ChartRuleLookupManager* ExamplePT::CreateRuleLookupManager(const ChartParser &parser,
     const ChartCellCollectionBase &cellCollection,
     std::size_t /*maxChartSpan*/)
 {
-  return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this);
+  return new ChartRuleLookupManagerExample(parser, cellCollection, *this);
 }
 
-TO_STRING_BODY(SkeletonPT);
+TO_STRING_BODY(ExamplePT);
 
 // friend
-ostream& operator<<(ostream& out, const SkeletonPT& phraseDict)
+ostream& operator<<(ostream& out, const ExamplePT& phraseDict)
 {
   return out;
 }
diff --git a/moses/TranslationModel/SkeletonPT.h b/moses/TranslationModel/ExamplePT.h
index 443f1cc8e..6ec7764c9 100644
--- a/moses/TranslationModel/SkeletonPT.h
+++ b/moses/TranslationModel/ExamplePT.h
@@ -9,12 +9,12 @@ class ChartParser;
 class ChartCellCollectionBase;
 class ChartRuleLookupManager;
 
-class SkeletonPT : public PhraseDictionary
+class ExamplePT : public PhraseDictionary
 {
-  friend std::ostream& operator<<(std::ostream&, const SkeletonPT&);
+  friend std::ostream& operator<<(std::ostream&, const ExamplePT&);
 
 public:
-  SkeletonPT(const std::string &line);
+  ExamplePT(const std::string &line);
 
   void Load(AllOptions::ptr const& opts);
 
diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp b/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
index 36a28089b..fc62f0679 100644
--- a/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
+++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
@@ -1,6 +1,6 @@
 // vim:tabstop=2
 #include "PhraseDictionaryMemoryPerSentence.h"
-#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
+#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h"
 
 using namespace std;
 
diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp
new file mode 100644
index 000000000..acf834cbd
--- /dev/null
+++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp
@@ -0,0 +1,149 @@
+// vim:tabstop=2
+#include "PhraseDictionaryMemoryPerSentenceOnDemand.h"
+#include <sstream>
+
+using namespace std;
+
+namespace Moses
+{
+PhraseDictionaryMemoryPerSentenceOnDemand::PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line)
+  : PhraseDictionary(line, true), m_valuesAreProbabilities(true)
+{
+  ReadParameters();
+}
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  SetFeaturesToApply();
+
+  // don't load anything. Load when we have the input
+}
+
+
+TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const
+{
+
+  Coll &coll = GetColl();
+
+  return coll[source];
+
+}
+
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(ttasksptr const& ttask)
+{
+  Coll &coll = GetColl();
+  coll.clear();
+
+  VERBOSE(2, "Initializing PhraseDictionaryMemoryPerSentenceOnDemand " << m_description << "\n");
+
+  // The context scope object for this translation task
+  //     contains a map of translation task-specific data
+  boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
+
+  // The key to the map is this object
+  void const* key = static_cast<void const*>(this);
+
+  // The value stored in the map is a string representing a phrase table
+  boost::shared_ptr<string> value = contextScope->get<string>(key);
+
+  // Create a stream to read the phrase table data
+  stringstream strme(*(value.get()));
+
+  // Read the phrase table data, one line at a time
+  string line;
+  while (getline(strme, line)) {
+
+    VERBOSE(3, "\t" << line);
+
+    vector<string> toks = TokenizeMultiCharSeparator(line, "|||");
+    Phrase source;
+    source.CreateFromString(Input, m_input, toks[0], NULL);
+
+    TargetPhrase *target = new TargetPhrase(this);
+    target->CreateFromString(Output, m_output, toks[1], NULL);
+
+    // score for this phrase table
+    vector<float> scores = Tokenize<float>(toks[2]);
+    if (m_valuesAreProbabilities) {
+      std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
+      std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore);
+    }
+    target->GetScoreBreakdown().PlusEquals(this, scores);
+
+    // score of all other ff when this rule is being loaded
+    target->EvaluateInIsolation(source, GetFeaturesToApply());
+
+    // add to coll
+    TargetPhraseCollection::shared_ptr &tpsPtr = coll[source];
+    TargetPhraseCollection *tps = tpsPtr.get();
+    if (tps == NULL) {
+      tps = new TargetPhraseCollection();
+      tpsPtr.reset(tps);
+    }
+    tps->Add(target);
+  }
+}
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
+{
+  InputPathList::const_iterator iter;
+  for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
+    InputPath &inputPath = **iter;
+    const Phrase &source = inputPath.GetPhrase();
+
+    Coll &coll = GetColl();
+    Coll::const_iterator iter = coll.find(source);
+    if (iter == coll.end()) {
+      TargetPhraseCollection::shared_ptr tprPtr;
+      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
+    } else {
+      const TargetPhraseCollection::shared_ptr &tprPtr = iter->second;
+      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
+    }
+  }
+}
+
+
+ChartRuleLookupManager* PhraseDictionaryMemoryPerSentenceOnDemand::CreateRuleLookupManager(const ChartParser &parser,
+    const ChartCellCollectionBase &cellCollection,
+    std::size_t /*maxChartSpan*/)
+{
+  abort();
+}
+
+PhraseDictionaryMemoryPerSentenceOnDemand::Coll &PhraseDictionaryMemoryPerSentenceOnDemand::GetColl() const
+{
+  Coll *coll;
+  coll = m_coll.get();
+  if (coll == NULL) {
+    coll = new Coll;
+    m_coll.reset(coll);
+  }
+  assert(coll);
+  return *coll;
+}
+
+void
+PhraseDictionaryMemoryPerSentenceOnDemand::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "path") {
+    UTIL_THROW(util::Exception, "PhraseDictionaryMemoryPerSentenceOnDemand does not support key \"path\".");
+  } else if (key == "valuesAreProbabilities") {
+    m_valuesAreProbabilities = Scan<bool>(value);
+  } else {
+    PhraseDictionary::SetParameter(key, value);
+  }
+}
+
+
+TO_STRING_BODY(PhraseDictionaryMemoryPerSentenceOnDemand);
+
+// friend
+ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentenceOnDemand& phraseDict)
+{
+  return out;
+}
+
+}
diff --git a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h
new file mode 100644
index 000000000..e3fe6514a
--- /dev/null
+++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h
@@ -0,0 +1,48 @@
+
+#pragma once
+
+#include "PhraseDictionary.h"
+#include "moses/TypeDef.h"
+#include "moses/TranslationTask.h"
+
+namespace Moses
+{
+class ChartParser;
+class ChartCellCollectionBase;
+class ChartRuleLookupManager;
+
+class PhraseDictionaryMemoryPerSentenceOnDemand : public PhraseDictionary
+{
+  friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryMemoryPerSentenceOnDemand&);
+
+public:
+  PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line);
+
+  void Load(AllOptions::ptr const& opts);
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  // for phrase-based model
+  void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
+
+  // for syntax/hiero model (CKY+ decoding)
+  ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, const ChartCellCollectionBase&, std::size_t);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  TargetPhraseCollection::shared_ptr  GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const;
+
+  TO_STRING();
+
+
+protected:
+  typedef boost::unordered_map<Phrase, TargetPhraseCollection::shared_ptr> Coll;
+  mutable boost::thread_specific_ptr<Coll> m_coll;
+
+  bool m_valuesAreProbabilities;
+
+  Coll &GetColl() const;
+
+};
+
+}  // namespace Moses
diff --git a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp
index 3d1664822..2ffe880c7 100644
--- a/moses/TranslationModel/PhraseDictionaryTransliteration.cpp
+++ b/moses/TranslationModel/PhraseDictionaryTransliteration.cpp
@@ -2,7 +2,6 @@
 #include <cstdlib>
 
 #include "PhraseDictionaryTransliteration.h"
-#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
 #include "moses/DecodeGraph.h"
 #include "moses/DecodeStep.h"
 #include "util/tempfile.hh"
diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.cpp b/moses/TranslationModel/ProbingPT.cpp
index 1ae0c67c3..dca7835f5 100644
--- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp
+++ b/moses/TranslationModel/ProbingPT.cpp
@@ -4,8 +4,8 @@
 #include "moses/FactorCollection.h"
 #include "moses/TargetPhraseCollection.h"
 #include "moses/InputFileStream.h"
-#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
-#include "querying.hh"
+#include "probingpt/querying.h"
+#include "probingpt/probing_hash_utils.h"
 
 using namespace std;
 
@@ -14,6 +14,7 @@ namespace Moses
 ProbingPT::ProbingPT(const std::string &line)
   : PhraseDictionary(line,true)
   ,m_engine(NULL)
+  ,load_method(util::POPULATE_OR_READ)
 {
   ReadParameters();
 
@@ -31,7 +32,7 @@ void ProbingPT::Load(AllOptions::ptr const& opts)
   m_options = opts;
   SetFeaturesToApply();
 
-  m_engine = new QueryEngine(m_filePath.c_str());
+  m_engine = new probingpt::QueryEngine(m_filePath.c_str(), load_method);
 
   m_unkId = 456456546456;
 
@@ -116,6 +117,28 @@ void ProbingPT::CreateAlignmentMap(const std::string path)
   }
 }
 
+void ProbingPT::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "load") {
+    if (value == "lazy") {
+      load_method = util::LAZY;
+    } else if (value == "populate_or_lazy") {
+      load_method = util::POPULATE_OR_LAZY;
+    } else if (value == "populate_or_read" || value == "populate") {
+      load_method = util::POPULATE_OR_READ;
+    } else if (value == "read") {
+      load_method = util::READ;
+    } else if (value == "parallel_read") {
+      load_method = util::PARALLEL_READ;
+    } else {
+      UTIL_THROW2("load method not supported" << value);
+    }
+  } else {
+    PhraseDictionary::SetParameter(key, value);
+  }
+
+}
+
 void ProbingPT::InitializeForInput(ttasksptr const& ttask)
 {
 
@@ -256,12 +279,12 @@ TargetPhraseCollection *ProbingPT::CreateTargetPhrases(
 TargetPhrase *ProbingPT::CreateTargetPhrase(
   const char *&offset) const
 {
-  TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset;
+  probingpt::TargetPhraseInfo *tpInfo = (probingpt::TargetPhraseInfo*) offset;
   size_t numRealWords = tpInfo->numWords / m_output.size();
 
   TargetPhrase *tp = new TargetPhrase(this);
 
-  offset += sizeof(TargetPhraseInfo);
+  offset += sizeof(probingpt::TargetPhraseInfo);
 
   // scores
   float *scores = (float*) offset;
diff --git a/moses/TranslationModel/ProbingPT/ProbingPT.h b/moses/TranslationModel/ProbingPT.h
index 953a2dc2f..1c996f5fa 100644
--- a/moses/TranslationModel/ProbingPT/ProbingPT.h
+++ b/moses/TranslationModel/ProbingPT.h
@@ -3,16 +3,20 @@
 #include <boost/iostreams/device/mapped_file.hpp>
 #include <boost/bimap.hpp>
 #include <boost/unordered_map.hpp>
-#include "../PhraseDictionary.h"
+#include "PhraseDictionary.h"
+#include "util/mmap.hh"
 
+namespace probingpt
+{
+class QueryEngine;
+class target_text;
+}
 
 namespace Moses
 {
 class ChartParser;
 class ChartCellCollectionBase;
 class ChartRuleLookupManager;
-class QueryEngine;
-class target_text;
 
 class ProbingPT : public PhraseDictionary
 {
@@ -26,6 +30,8 @@ public:
 
   void InitializeForInput(ttasksptr const& ttask);
 
+  void SetParameter(const std::string& key, const std::string& value);
+
   // for phrase-based model
   void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
 
@@ -39,12 +45,13 @@ public:
 
 
 protected:
-  QueryEngine *m_engine;
+  probingpt::QueryEngine *m_engine;
   uint64_t m_unkId;
 
   std::vector<uint64_t> m_sourceVocab; // factor id -> pt id
   std::vector<const Factor*> m_targetVocab; // pt id -> factor*
   std::vector<const AlignmentInfo*> m_aligns;
+  util::LoadMethod load_method;
 
   boost::iostreams::mapped_file_source file;
   const char *data;
diff --git a/moses/TranslationModel/ProbingPT/Jamfile b/moses/TranslationModel/ProbingPT/Jamfile
deleted file mode 100644
index 29c6ec41d..000000000
--- a/moses/TranslationModel/ProbingPT/Jamfile
+++ /dev/null
@@ -1,8 +0,0 @@
-local current = "" ;
-local includes = ;
-
-fakelib ProbingPT : [ glob *.cpp ] ../..//headers : $(includes) <dependency>$(PT-LOG) : : $(includes) ;
-
-path-constant PT-LOG : bin/pt.log ;
-update-if-changed $(PT-LOG) $(current) ;
-
diff --git a/moses/TranslationModel/ProbingPT/StoreTarget.cpp b/moses/TranslationModel/ProbingPT/StoreTarget.cpp
deleted file mode 100644
index f586a26b9..000000000
--- a/moses/TranslationModel/ProbingPT/StoreTarget.cpp
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * StoreTarget.cpp
- *
- *  Created on: 19 Jan 2016
- *      Author: hieu
- */
-#include <boost/foreach.hpp>
-#include "StoreTarget.h"
-#include "line_splitter.hh"
-#include "probing_hash_utils.hh"
-#include "moses/OutputFileStream.h"
-#include "moses/Util.h"
-
-using namespace std;
-
-namespace Moses
-{
-
-StoreTarget::StoreTarget(const std::string &basepath)
-  :m_basePath(basepath)
-  ,m_vocab(basepath + "/TargetVocab.dat")
-{
-  std::string path = basepath + "/TargetColl.dat";
-  m_fileTargetColl.open(path.c_str(),
-                        std::ios::out | std::ios::binary | std::ios::ate | std::ios::trunc);
-  if (!m_fileTargetColl.is_open()) {
-    throw "can't create file ";
-  }
-
-}
-
-StoreTarget::~StoreTarget()
-{
-  assert(m_coll.empty());
-  m_fileTargetColl.close();
-
-  // vocab
-  m_vocab.Save();
-}
-
-uint64_t StoreTarget::Save()
-{
-  uint64_t ret = m_fileTargetColl.tellp();
-
-  // save to disk
-  uint64_t numTP = m_coll.size();
-  m_fileTargetColl.write((char*) &numTP, sizeof(uint64_t));
-
-  for (size_t i = 0; i < m_coll.size(); ++i) {
-    Save(*m_coll[i]);
-  }
-
-  // clear coll
-  RemoveAllInColl(m_coll);
-  m_coll.clear();
-
-  // starting position of coll
-  return ret;
-}
-
-void StoreTarget::Save(const target_text &rule)
-{
-  // metadata for each tp
-  TargetPhraseInfo tpInfo;
-  tpInfo.alignTerm = GetAlignId(rule.word_align_term);
-  tpInfo.alignNonTerm = GetAlignId(rule.word_align_non_term);
-  tpInfo.numWords = rule.target_phrase.size();
-  tpInfo.propLength = rule.property.size();
-
-  //cerr << "TPInfo=" << sizeof(TPInfo);
-  m_fileTargetColl.write((char*) &tpInfo, sizeof(TargetPhraseInfo));
-
-  // scores
-  for (size_t i = 0; i < rule.prob.size(); ++i) {
-    float prob = rule.prob[i];
-    m_fileTargetColl.write((char*) &prob, sizeof(prob));
-  }
-
-  // tp
-  for (size_t i = 0; i < rule.target_phrase.size(); ++i) {
-    uint32_t vocabId = rule.target_phrase[i];
-    m_fileTargetColl.write((char*) &vocabId, sizeof(vocabId));
-  }
-
-  // prop TODO
-
-}
-
-void StoreTarget::SaveAlignment()
-{
-  std::string path = m_basePath + "/Alignments.dat";
-  OutputFileStream file(path);
-
-  BOOST_FOREACH(Alignments::value_type &valPair, m_aligns) {
-    file << valPair.second << "\t";
-
-    const std::vector<size_t> &aligns = valPair.first;
-    BOOST_FOREACH(size_t align, aligns) {
-      file << align << " ";
-    }
-    file << endl;
-  }
-
-}
-
-void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg)
-{
-  target_text *rule = new target_text;
-  //cerr << "line.target_phrase=" << line.target_phrase << endl;
-
-  // target_phrase
-  vector<bool> nonTerms;
-  util::TokenIter<util::SingleCharacter> it;
-  it = util::TokenIter<util::SingleCharacter>(line.target_phrase,
-       util::SingleCharacter(' '));
-  while (it) {
-    StringPiece word = *it;
-    //cerr << "word=" << word << endl;
-
-    bool nonTerm = false;
-    if (scfg) {
-      // not really sure how to handle factored SCFG and NT
-      if (scfg && word[0] == '[' && word[word.size() - 1] == ']') {
-        //cerr << "NON-TERM=" << tok << " " << nonTerms.size() << endl;
-        nonTerm = true;
-      }
-      nonTerms.push_back(nonTerm);
-    }
-
-    util::TokenIter<util::SingleCharacter> itFactor;
-    itFactor = util::TokenIter<util::SingleCharacter>(word,
-               util::SingleCharacter('|'));
-    while (itFactor) {
-      StringPiece factor = *itFactor;
-
-      string factorStr = factor.as_string();
-      uint32_t vocabId = m_vocab.GetVocabId(factorStr);
-
-      rule->target_phrase.push_back(vocabId);
-
-      itFactor++;
-    }
-
-    it++;
-  }
-
-  // probs
-  it = util::TokenIter<util::SingleCharacter>(line.prob,
-       util::SingleCharacter(' '));
-  while (it) {
-    string tok = it->as_string();
-    float prob = Scan<float>(tok);
-
-    if (log_prob) {
-      prob = FloorScore(log(prob));
-      if (prob == 0.0f) prob = 0.0000000001;
-    }
-
-    rule->prob.push_back(prob);
-    it++;
-  }
-
-  /*
-  cerr << "nonTerms=";
-  for (size_t i = 0; i < nonTerms.size(); ++i) {
-    cerr << nonTerms[i] << " ";
-  }
-  cerr << endl;
-  */
-
-  // alignment
-  it = util::TokenIter<util::SingleCharacter>(line.word_align,
-       util::SingleCharacter(' '));
-  while (it) {
-    string tokPair = Trim(it->as_string());
-    if (tokPair.empty()) {
-      break;
-    }
-
-    vector<size_t> alignPair = Tokenize<size_t>(tokPair, "-");
-    assert(alignPair.size() == 2);
-
-    bool nonTerm = false;
-    size_t sourcePos = alignPair[0];
-    size_t targetPos = alignPair[1];
-    if (scfg) {
-      nonTerm = nonTerms[targetPos];
-    }
-
-    //cerr << targetPos << "=" << nonTerm << endl;
-
-    if (nonTerm) {
-      rule->word_align_non_term.push_back(sourcePos);
-      rule->word_align_non_term.push_back(targetPos);
-      //cerr << (int) rule->word_all1.back() << " ";
-    } else {
-      rule->word_align_term.push_back(sourcePos);
-      rule->word_align_term.push_back(targetPos);
-    }
-
-    it++;
-  }
-
-  // extra scores
-  string prop = line.property.as_string();
-  AppendLexRO(prop, rule->prob, log_prob);
-
-  //cerr << "line.property=" << line.property << endl;
-  //cerr << "prop=" << prop << endl;
-
-  // properties
-  /*
-   for (size_t i = 0; i < prop.size(); ++i) {
-   rule->property.push_back(prop[i]);
-   }
-   */
-  m_coll.push_back(rule);
-}
-
-uint32_t StoreTarget::GetAlignId(const std::vector<size_t> &align)
-{
-  boost::unordered_map<std::vector<size_t>, uint32_t>::iterator iter =
-    m_aligns.find(align);
-  if (iter == m_aligns.end()) {
-    uint32_t ind = m_aligns.size();
-    m_aligns[align] = ind;
-    return ind;
-  } else {
-    return iter->second;
-  }
-}
-
-void StoreTarget::AppendLexRO(std::string &prop, std::vector<float> &retvector,
-                              bool log_prob) const
-{
-  size_t startPos = prop.find("{{LexRO ");
-
-  if (startPos != string::npos) {
-    size_t endPos = prop.find("}}", startPos + 8);
-    string lexProb = prop.substr(startPos + 8, endPos - startPos - 8);
-    //cerr << "lexProb=" << lexProb << endl;
-
-    // append lex probs to pt probs
-    vector<float> scores = Tokenize<float>(lexProb);
-
-    if (log_prob) {
-      for (size_t i = 0; i < scores.size(); ++i) {
-        scores[i] = FloorScore(log(scores[i]));
-        if (scores[i] == 0.0f) scores[i] = 0.0000000001;
-      }
-    }
-
-    for (size_t i = 0; i < scores.size(); ++i) {
-      retvector.push_back(scores[i]);
-    }
-
-    // exclude LexRO property from property column
-    prop = prop.substr(0, startPos)
-           + prop.substr(endPos + 2, prop.size() - endPos - 2);
-    //cerr << "line.property_to_be_binarized=" << line.property_to_be_binarized << "AAAA" << endl;
-  }
-}
-
-} /* namespace Moses2 */
diff --git a/moses/TranslationModel/ProbingPT/StoreTarget.h b/moses/TranslationModel/ProbingPT/StoreTarget.h
deleted file mode 100644
index 331c197b3..000000000
--- a/moses/TranslationModel/ProbingPT/StoreTarget.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * StoreTarget.h
- *
- *  Created on: 19 Jan 2016
- *      Author: hieu
- */
-#pragma once
-#include <string>
-#include <fstream>
-#include <vector>
-#include <inttypes.h>
-#include <boost/unordered_map.hpp>
-#include <boost/unordered_set.hpp>
-#include "StoreVocab.h"
-
-namespace Moses
-{
-
-class line_text;
-class target_text;
-
-class StoreTarget
-{
-public:
-  StoreTarget(const std::string &basepath);
-  virtual ~StoreTarget();
-
-  uint64_t Save();
-  void SaveAlignment();
-
-  void Append(const line_text &line, bool log_prob, bool scfg);
-protected:
-  std::string m_basePath;
-  std::fstream m_fileTargetColl;
-  StoreVocab<uint32_t> m_vocab;
-
-  typedef boost::unordered_map<std::vector<size_t>, uint32_t> Alignments;
-  Alignments m_aligns;
-
-  std::vector<target_text*> m_coll;
-
-  uint32_t GetAlignId(const std::vector<size_t> &align);
-  void Save(const target_text &rule);
-
-  void AppendLexRO(std::string &prop, std::vector<float> &retvector,
-                   bool log_prob) const;
-
-};
-
-} /* namespace Moses2 */
-
diff --git a/moses/TranslationModel/ProbingPT/StoreVocab.cpp b/moses/TranslationModel/ProbingPT/StoreVocab.cpp
deleted file mode 100644
index 6515bac63..000000000
--- a/moses/TranslationModel/ProbingPT/StoreVocab.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * StoreVocab.cpp
- *
- *  Created on: 15 Jun 2016
- *      Author: hieu
- */
-#include <fstream>
-#include "StoreVocab.h"
-
-namespace Moses
-{
-
-} /* namespace Moses2 */
diff --git a/moses/TranslationModel/ProbingPT/StoreVocab.h b/moses/TranslationModel/ProbingPT/StoreVocab.h
deleted file mode 100644
index 806dcebf4..000000000
--- a/moses/TranslationModel/ProbingPT/StoreVocab.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * StoreVocab.h
- *
- *  Created on: 15 Jun 2016
- *      Author: hieu
- */
-#pragma once
-#include <string>
-#include <boost/unordered_map.hpp>
-#include "moses/OutputFileStream.h"
-#include "moses/Util.h"
-
-namespace Moses
-{
-
-template<typename VOCABID>
-class StoreVocab
-{
-protected:
-  std::string m_path;
-
-  typedef boost::unordered_map<std::string, VOCABID> Coll;
-  Coll m_vocab;
-
-public:
-  StoreVocab(const std::string &path)
-    :m_path(path)
-  {}
-
-  virtual ~StoreVocab() {}
-
-  VOCABID GetVocabId(const std::string &word) {
-    typename Coll::iterator iter = m_vocab.find(word);
-    if (iter == m_vocab.end()) {
-      VOCABID ind = m_vocab.size() + 1;
-      m_vocab[word] = ind;
-      return ind;
-    } else {
-      return iter->second;
-    }
-  }
-
-  void Insert(VOCABID id, const std::string &word) {
-    m_vocab[word] = id;
-  }
-
-  void Save() {
-    OutputFileStream strme(m_path);
-
-    typename Coll::const_iterator iter;
-    for (iter = m_vocab.begin(); iter != m_vocab.end(); ++iter) {
-      strme << iter->first << "\t" << iter->second << std::endl;
-    }
-
-    strme.Close();
-  }
-};
-
-} /* namespace Moses2 */
-
diff --git a/moses/TranslationModel/ProbingPT/hash.cpp b/moses/TranslationModel/ProbingPT/hash.cpp
deleted file mode 100644
index 47242e25d..000000000
--- a/moses/TranslationModel/ProbingPT/hash.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-#include <iostream>
-#include "hash.hh"
-
-using namespace std;
-
-namespace Moses
-{
-
-uint64_t getHash(StringPiece text)
-{
-  std::size_t len = text.size();
-  uint64_t key = util::MurmurHashNative(text.data(), len);
-  return key;
-}
-
-std::vector<uint64_t> getVocabIDs(const StringPiece &textin)
-{
-  //Tokenize
-  std::vector<uint64_t> output;
-
-  util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' '));
-
-  while (itWord) {
-    StringPiece word = *itWord;
-    uint64_t id = 0;
-
-    util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|'));
-    while (itFactor) {
-      StringPiece factor = *itFactor;
-      //cerr << "factor=" << factor << endl;
-
-      id += getHash(factor);
-      itFactor++;
-    }
-
-    output.push_back(id);
-    itWord++;
-  }
-
-  return output;
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/hash.hh b/moses/TranslationModel/ProbingPT/hash.hh
deleted file mode 100644
index f218ad9da..000000000
--- a/moses/TranslationModel/ProbingPT/hash.hh
+++ /dev/null
@@ -1,17 +0,0 @@
-#pragma once
-
-#include "util/string_piece.hh"
-#include "util/murmur_hash.hh"
-#include "util/string_piece.hh"  //Tokenization and work with StringPiece
-#include "util/tokenize_piece.hh"
-#include <vector>
-
-namespace Moses
-{
-
-//Gets the MurmurmurHash for give string
-uint64_t getHash(StringPiece text);
-
-std::vector<uint64_t> getVocabIDs(const StringPiece &textin);
-
-}
diff --git a/moses/TranslationModel/ProbingPT/line_splitter.cpp b/moses/TranslationModel/ProbingPT/line_splitter.cpp
deleted file mode 100644
index cb9e47fec..000000000
--- a/moses/TranslationModel/ProbingPT/line_splitter.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-#include "line_splitter.hh"
-
-namespace Moses
-{
-
-line_text splitLine(const StringPiece &textin, bool scfg)
-{
-  const char delim[] = "|||";
-  line_text output;
-
-  //Tokenize
-  util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim));
-  //Get source phrase
-  output.source_phrase = Trim(*it);
-  //std::cerr << "output.source_phrase=" << output.source_phrase << "AAAA" << std::endl;
-
-  //Get target_phrase
-  it++;
-  output.target_phrase = Trim(*it);
-  //std::cerr << "output.target_phrase=" << output.target_phrase << "AAAA" << std::endl;
-
-  if (scfg) {
-    /*
-    std::cerr << "output.source_phrase=" << output.source_phrase << std::endl;
-    std::cerr << "output.target_phrase=" << output.target_phrase << std::endl;
-    reformatSCFG(output);
-    std::cerr << "output.source_phrase=" << output.source_phrase << std::endl;
-    std::cerr << "output.target_phrase=" << output.target_phrase << std::endl;
-    */
-  }
-
-  //Get probabilities
-  it++;
-  output.prob = Trim(*it);
-  //std::cerr << "output.prob=" << output.prob << "AAAA" << std::endl;
-
-  //Get WordAllignment
-  it++;
-  if (it == util::TokenIter<util::MultiCharacter>::end()) return output;
-  output.word_align = Trim(*it);
-  //std::cerr << "output.word_align=" << output.word_align << "AAAA" << std::endl;
-
-  //Get count
-  it++;
-  if (it == util::TokenIter<util::MultiCharacter>::end()) return output;
-  output.counts = Trim(*it);
-  //std::cerr << "output.counts=" << output.counts << "AAAA" << std::endl;
-
-  //Get sparse_score
-  it++;
-  if (it == util::TokenIter<util::MultiCharacter>::end()) return output;
-  output.sparse_score = Trim(*it);
-  //std::cerr << "output.sparse_score=" << output.sparse_score << "AAAA" << std::endl;
-
-  //Get property
-  it++;
-  if (it == util::TokenIter<util::MultiCharacter>::end()) return output;
-  output.property = Trim(*it);
-  //std::cerr << "output.property=" << output.property << "AAAA" << std::endl;
-
-  return output;
-}
-
-std::vector<unsigned char> splitWordAll1(const StringPiece &textin)
-{
-  const char delim[] = " ";
-  const char delim2[] = "-";
-  std::vector<unsigned char> output;
-
-  //Case with no word alignments.
-  if (textin.size() == 0) {
-    return output;
-  }
-
-  //Split on space
-  util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim));
-
-  //For each int
-  while (it) {
-    //Split on dash (-)
-    util::TokenIter<util::MultiCharacter> itInner(*it,
-        util::MultiCharacter(delim2));
-
-    //Insert the two entries in the vector. User will read entry 0 and 1 to get the first,
-    //2 and 3 for second etc. Use unsigned char instead of int to save space, as
-    //word allignments are all very small numbers that fit in a single byte
-    output.push_back((unsigned char) (atoi(itInner->data())));
-    itInner++;
-    output.push_back((unsigned char) (atoi(itInner->data())));
-    it++;
-  }
-
-  return output;
-
-}
-
-void reformatSCFG(line_text &output)
-{
-
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/line_splitter.hh b/moses/TranslationModel/ProbingPT/line_splitter.hh
deleted file mode 100644
index 01b86fc9b..000000000
--- a/moses/TranslationModel/ProbingPT/line_splitter.hh
+++ /dev/null
@@ -1,57 +0,0 @@
-#pragma once
-
-#include "util/string_piece.hh"
-#include "util/tokenize_piece.hh"
-#include "util/file_piece.hh"
-#include <vector>
-#include <cstdlib> //atof
-#include "util/string_piece.hh"  //Tokenization and work with StringPiece
-#include "util/tokenize_piece.hh"
-#include <vector>
-
-namespace Moses
-{
-
-//Struct for holding processed line
-struct line_text {
-  StringPiece source_phrase;
-  StringPiece target_phrase;
-  StringPiece prob;
-  StringPiece word_align;
-  StringPiece counts;
-  StringPiece sparse_score;
-  StringPiece property;
-  std::string property_to_be_binarized;
-};
-
-//Struct for holding processed line
-struct target_text {
-  std::vector<unsigned int> target_phrase;
-  std::vector<float> prob;
-  std::vector<size_t> word_align_term;
-  std::vector<size_t> word_align_non_term;
-  std::vector<char> counts;
-  std::vector<char> sparse_score;
-  std::vector<char> property;
-
-  /*
-  void Reset()
-  {
-    target_phrase.clear();
-    prob.clear();
-    word_all1.clear();
-    counts.clear();
-    sparse_score.clear();
-    property.clear();
-  }
-  */
-};
-
-//Ask if it's better to have it receive a pointer to a line_text struct
-line_text splitLine(const StringPiece &textin, bool scfg);
-void reformatSCFG(line_text &output);
-
-std::vector<unsigned char> splitWordAll1(const StringPiece &textin);
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp b/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp
deleted file mode 100644
index f23f57d66..000000000
--- a/moses/TranslationModel/ProbingPT/probing_hash_utils.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-#include "probing_hash_utils.hh"
-
-namespace Moses
-{
-
-//Read table from disk, return memory map location
-char * readTable(const char * filename, size_t size)
-{
-  //Initial position of the file is the end of the file, thus we know the size
-  int fd;
-  char * map;
-
-  fd = open(filename, O_RDONLY);
-  if (fd == -1) {
-    perror("Error opening file for reading");
-    exit(EXIT_FAILURE);
-  }
-
-  map = (char *) mmap(0, size, PROT_READ, MAP_SHARED, fd, 0);
-
-  if (map == MAP_FAILED) {
-    close(fd);
-    perror("Error mmapping the file");
-    exit(EXIT_FAILURE);
-  }
-
-  return map;
-}
-
-void serialize_table(char *mem, size_t size, const std::string &filename)
-{
-  std::ofstream os(filename.c_str(), std::ios::binary);
-  os.write((const char*) &mem[0], size);
-  os.close();
-
-}
-
-uint64_t getKey(const uint64_t source_phrase[], size_t size)
-{
-  //TOO SLOW
-  //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size());
-  uint64_t key = 0;
-  for (size_t i = 0; i < size; i++) {
-    key += (source_phrase[i] << i);
-  }
-  return key;
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/probing_hash_utils.hh b/moses/TranslationModel/ProbingPT/probing_hash_utils.hh
deleted file mode 100644
index 998686b2e..000000000
--- a/moses/TranslationModel/ProbingPT/probing_hash_utils.hh
+++ /dev/null
@@ -1,51 +0,0 @@
-#pragma once
-
-#include "util/probing_hash_table.hh"
-
-#include <sys/mman.h>
-#include <boost/functional/hash.hpp>
-#include <fcntl.h>
-#include <fstream>
-
-namespace Moses
-{
-
-#define API_VERSION 15
-
-//Hash table entry
-struct Entry {
-  typedef uint64_t Key;
-  Key key;
-
-  Key GetKey() const {
-    return key;
-  }
-
-  void SetKey(Key to) {
-    key = to;
-  }
-
-  uint64_t value;
-};
-
-#define NONE       std::numeric_limits<uint64_t>::max()
-
-//Define table
-typedef util::ProbingHashTable<Entry, boost::hash<uint64_t> > Table;
-
-void serialize_table(char *mem, size_t size, const std::string &filename);
-
-char * readTable(const char * filename, size_t size);
-
-uint64_t getKey(const uint64_t source_phrase[], size_t size);
-
-struct TargetPhraseInfo {
-  uint32_t alignTerm;
-  uint32_t alignNonTerm;
-  uint16_t numWords;
-  uint16_t propLength;
-  uint16_t filler;
-};
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/querying.cpp b/moses/TranslationModel/ProbingPT/querying.cpp
deleted file mode 100644
index 10c35e361..000000000
--- a/moses/TranslationModel/ProbingPT/querying.cpp
+++ /dev/null
@@ -1,141 +0,0 @@
-#include "querying.hh"
-#include "util/exception.hh"
-
-using namespace std;
-
-namespace Moses
-{
-
-QueryEngine::QueryEngine(const char * filepath)
-{
-
-  //Create filepaths
-  std::string basepath(filepath);
-  std::string path_to_config = basepath + "/config";
-  std::string path_to_hashtable = basepath + "/probing_hash.dat";
-  std::string path_to_source_vocabid = basepath + "/source_vocabids";
-  std::string alignPath = basepath + "/Alignments.dat";
-
-  if (!FileExists(path_to_config)) {
-    UTIL_THROW2("Binary table doesn't exist is didn't finish binarizing: " << path_to_config);
-  }
-
-  ///Source phrase vocabids
-  read_map(source_vocabids, path_to_source_vocabid.c_str());
-
-  // alignments
-  read_alignments(alignPath);
-
-  //Read config file
-  boost::unordered_map<std::string, std::string> keyValue;
-
-  std::ifstream config(path_to_config.c_str());
-  std::string line;
-  while (getline(config, line)) {
-    std::vector<std::string> toks = Tokenize(line, "\t");
-    UTIL_THROW_IF2(toks.size() != 2, "Wrong config format:" << line);
-    keyValue[ toks[0] ] = toks[1];
-  }
-
-  bool found;
-  //Check API version:
-  int version;
-  found = Get(keyValue, "API_VERSION", version);
-  if (!found) {
-    std::cerr << "Old or corrupted version of ProbingPT. Please rebinarize your phrase tables." << std::endl;
-  } else if (version != API_VERSION) {
-    std::cerr << "The ProbingPT API has changed. " << version << "!="
-              << API_VERSION << " Please rebinarize your phrase tables." << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  //Get tablesize.
-  int tablesize;
-  found = Get(keyValue, "uniq_entries", tablesize);
-  if (!found) {
-    std::cerr << "uniq_entries not found" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  //Number of scores
-  found = Get(keyValue, "num_scores", num_scores);
-  if (!found) {
-    std::cerr << "num_scores not found" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  //How may scores from lex reordering models
-  found = Get(keyValue, "num_lex_scores", num_lex_scores);
-  if (!found) {
-    std::cerr << "num_lex_scores not found" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  // have the scores been log() and FloorScore()?
-  found = Get(keyValue, "log_prob", logProb);
-  if (!found) {
-    std::cerr << "logProb not found" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  config.close();
-
-  //Read hashtable
-  table_filesize = Table::Size(tablesize, 1.2);
-  mem = readTable(path_to_hashtable.c_str(), table_filesize);
-  Table table_init(mem, table_filesize);
-  table = table_init;
-
-  std::cerr << "Initialized successfully! " << std::endl;
-}
-
-QueryEngine::~QueryEngine()
-{
-  //Clear mmap content from memory.
-  munmap(mem, table_filesize);
-
-}
-
-uint64_t QueryEngine::getKey(uint64_t source_phrase[], size_t size) const
-{
-  //TOO SLOW
-  //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size());
-  return Moses::getKey(source_phrase, size);
-}
-
-std::pair<bool, uint64_t> QueryEngine::query(uint64_t key)
-{
-  std::pair<bool, uint64_t> ret;
-
-  const Entry * entry;
-  ret.first = table.Find(key, entry);
-  if (ret.first) {
-    ret.second = entry->value;
-  }
-  return ret;
-}
-
-void QueryEngine::read_alignments(const std::string &alignPath)
-{
-  std::ifstream strm(alignPath.c_str());
-
-  string line;
-  while (getline(strm, line)) {
-    vector<string> toks = Tokenize(line, "\t ");
-    UTIL_THROW_IF2(toks.size() == 0, "Corrupt alignment file");
-
-    uint32_t alignInd = Scan<uint32_t>(toks[0]);
-    if (alignInd >= alignColl.size()) {
-      alignColl.resize(alignInd + 1);
-    }
-
-    Alignments &aligns = alignColl[alignInd];
-    for (size_t i = 1; i < toks.size(); ++i) {
-      size_t pos = Scan<size_t>(toks[i]);
-      aligns.push_back(pos);
-    }
-  }
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/querying.hh b/moses/TranslationModel/ProbingPT/querying.hh
deleted file mode 100644
index 915bc4806..000000000
--- a/moses/TranslationModel/ProbingPT/querying.hh
+++ /dev/null
@@ -1,66 +0,0 @@
-#pragma once
-
-#include <boost/unordered_map.hpp>
-#include <sys/stat.h> //For finding size of file
-#include "vocabid.hh"
-#include <algorithm> //toLower
-#include <deque>
-#include "probing_hash_utils.hh"
-#include "hash.hh" //Includes line splitter
-#include "line_splitter.hh"
-#include "moses//Util.h"
-
-namespace Moses
-{
-
-class QueryEngine
-{
-  std::map<uint64_t, std::string> source_vocabids;
-
-  typedef std::vector<unsigned char> Alignments;
-  std::vector<Alignments> alignColl;
-
-  Table table;
-  char *mem; //Memory for the table, necessary so that we can correctly destroy the object
-
-  size_t table_filesize;
-  bool is_reordering;
-
-  void read_alignments(const std::string &alignPath);
-
-public:
-  int num_scores;
-  int num_lex_scores;
-  bool logProb;
-
-  QueryEngine(const char *);
-  ~QueryEngine();
-
-  std::pair<bool, uint64_t> query(uint64_t key);
-
-  const std::map<uint64_t, std::string> &getSourceVocab() const {
-    return source_vocabids;
-  }
-
-  const std::vector<Alignments> &getAlignments() const {
-    return alignColl;
-  }
-
-  uint64_t getKey(uint64_t source_phrase[], size_t size) const;
-
-  template<typename T>
-  inline bool Get(const boost::unordered_map<std::string, std::string> &keyValue, const std::string &sought, T &found) const {
-    boost::unordered_map<std::string, std::string>::const_iterator iter = keyValue.find(sought);
-    if (iter == keyValue.end()) {
-      return false;
-    }
-
-    const std::string &foundStr = iter->second;
-    found = Scan<T>(foundStr);
-    return true;
-  }
-
-};
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/storing.cpp b/moses/TranslationModel/ProbingPT/storing.cpp
deleted file mode 100644
index baf6ae91e..000000000
--- a/moses/TranslationModel/ProbingPT/storing.cpp
+++ /dev/null
@@ -1,298 +0,0 @@
-#include <sys/stat.h>
-#include <boost/foreach.hpp>
-#include "line_splitter.hh"
-#include "storing.hh"
-#include "StoreTarget.h"
-#include "StoreVocab.h"
-#include "moses/Util.h"
-#include "moses/InputFileStream.h"
-
-using namespace std;
-
-namespace Moses
-{
-
-///////////////////////////////////////////////////////////////////////
-void Node::Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos)
-{
-  if (pos < sourcePhrase.size()) {
-    uint64_t vocabId = sourcePhrase[pos];
-
-    Node *child;
-    Children::iterator iter = m_children.find(vocabId);
-    if (iter == m_children.end()) {
-      // New node. Write other children then discard them
-      BOOST_FOREACH(Children::value_type &valPair, m_children) {
-        Node &otherChild = valPair.second;
-        otherChild.Write(table);
-      }
-      m_children.clear();
-
-      // create new node
-      child = &m_children[vocabId];
-      assert(!child->done);
-      child->key = key + (vocabId << pos);
-    } else {
-      child = &iter->second;
-    }
-
-    child->Add(table, sourcePhrase, pos + 1);
-  } else {
-    // this node was written previously 'cos it has rules
-    done = true;
-  }
-}
-
-void Node::Write(Table &table)
-{
-  //cerr << "START write " << done << " " << key << endl;
-  BOOST_FOREACH(Children::value_type &valPair, m_children) {
-    Node &child = valPair.second;
-    child.Write(table);
-  }
-
-  if (!done) {
-    // save
-    Entry sourceEntry;
-    sourceEntry.value = NONE;
-    sourceEntry.key = key;
-
-    //Put into table
-    table.Insert(sourceEntry);
-  }
-}
-
-///////////////////////////////////////////////////////////////////////
-void createProbingPT(const std::string &phrasetable_path,
-                     const std::string &basepath, int num_scores, int num_lex_scores,
-                     bool log_prob, int max_cache_size, bool scfg)
-{
-  std::cerr << "Starting..." << std::endl;
-
-  //Get basepath and create directory if missing
-  mkdir(basepath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-
-  StoreTarget storeTarget(basepath);
-
-  //Get uniq lines:
-  unsigned long uniq_entries = countUniqueSource(phrasetable_path);
-
-  //Source phrase vocabids
-  StoreVocab<uint64_t> sourceVocab(basepath + "/source_vocabids");
-
-  //Read the file
-  util::FilePiece filein(phrasetable_path.c_str());
-
-  //Init the probing hash table
-  size_t size = Table::Size(uniq_entries, 1.2);
-  char * mem = new char[size];
-  memset(mem, 0, size);
-  Table sourceEntries(mem, size);
-
-  std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> cache;
-  float totalSourceCount = 0;
-
-  //Keep track of the size of each group of target phrases
-  size_t line_num = 0;
-
-  //Read everything and processs
-  std::string prevSource;
-
-  Node sourcePhrases;
-  sourcePhrases.done = true;
-  sourcePhrases.key = 0;
-
-  while (true) {
-    try {
-      //Process line read
-      line_text line;
-      line = splitLine(filein.ReadLine(), scfg);
-      //cerr << "line=" << line.source_phrase << endl;
-
-      ++line_num;
-      if (line_num % 1000000 == 0) {
-        std::cerr << line_num << " " << std::flush;
-      }
-
-      //Add source phrases to vocabularyIDs
-      add_to_map(sourceVocab, line.source_phrase);
-
-      if (prevSource.empty()) {
-        // 1st line
-        prevSource = line.source_phrase.as_string();
-        storeTarget.Append(line, log_prob, scfg);
-      } else if (prevSource == line.source_phrase) {
-        //If we still have the same line, just append to it:
-        storeTarget.Append(line, log_prob, scfg);
-      } else {
-        assert(prevSource != line.source_phrase);
-
-        //Create a new entry even
-
-        // save
-        uint64_t targetInd = storeTarget.Save();
-
-        // next line
-        storeTarget.Append(line, log_prob, scfg);
-
-        //Create an entry for the previous source phrase:
-        Entry sourceEntry;
-        sourceEntry.value = targetInd;
-        //The key is the sum of hashes of individual words bitshifted by their position in the phrase.
-        //Probably not entirerly correct, but fast and seems to work fine in practise.
-        std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource);
-        if (scfg) {
-          // storing prefixes?
-          sourcePhrases.Add(sourceEntries, vocabid_source);
-        }
-        sourceEntry.key = getKey(vocabid_source);
-
-        /*
-        cerr << "prevSource=" << prevSource << flush
-            << " vocabids=" << Debug(vocabid_source) << flush
-            << " key=" << sourceEntry.key << endl;
-        */
-        //Put into table
-        sourceEntries.Insert(sourceEntry);
-
-        // update cache - CURRENT source phrase, not prev
-        if (max_cache_size) {
-          std::string countStr = line.counts.as_string();
-          countStr = Trim(countStr);
-          if (!countStr.empty()) {
-            std::vector<float> toks = Tokenize<float>(countStr);
-            //cerr << "CACHE:" << line.source_phrase << " " << countStr << " " << toks[1] << endl;
-
-            if (toks.size() >= 2) {
-              totalSourceCount += toks[1];
-
-              // compute key for CURRENT source
-              std::vector<uint64_t> currVocabidSource = getVocabIDs(line.source_phrase.as_string());
-              uint64_t currKey = getKey(currVocabidSource);
-
-              CacheItem *item = new CacheItem(
-                Trim(line.source_phrase.as_string()),
-                currKey,
-                toks[1]);
-              cache.push(item);
-
-              if (max_cache_size > 0 && cache.size() > max_cache_size) {
-                cache.pop();
-              }
-            }
-          }
-        }
-
-        //Set prevLine
-        prevSource = line.source_phrase.as_string();
-      }
-
-    } catch (util::EndOfFileException e) {
-      std::cerr
-          << "Reading phrase table finished, writing remaining files to disk."
-          << std::endl;
-
-      //After the final entry is constructed we need to add it to the phrase_table
-      //Create an entry for the previous source phrase:
-      uint64_t targetInd = storeTarget.Save();
-
-      Entry sourceEntry;
-      sourceEntry.value = targetInd;
-
-      //The key is the sum of hashes of individual words. Probably not entirerly correct, but fast
-      std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource);
-      sourceEntry.key = getKey(vocabid_source);
-
-      //Put into table
-      sourceEntries.Insert(sourceEntry);
-
-      break;
-    }
-  }
-
-  sourcePhrases.Write(sourceEntries);
-
-  storeTarget.SaveAlignment();
-
-  serialize_table(mem, size, (basepath + "/probing_hash.dat"));
-
-  sourceVocab.Save();
-
-  serialize_cache(cache, (basepath + "/cache"), totalSourceCount);
-
-  delete[] mem;
-
-  //Write configfile
-  std::ofstream configfile;
-  configfile.open((basepath + "/config").c_str());
-  configfile << "API_VERSION\t" << API_VERSION << '\n';
-  configfile << "uniq_entries\t" << uniq_entries << '\n';
-  configfile << "num_scores\t" << num_scores << '\n';
-  configfile << "num_lex_scores\t" << num_lex_scores << '\n';
-  configfile << "log_prob\t" << log_prob << '\n';
-  configfile.close();
-}
-
-size_t countUniqueSource(const std::string &path)
-{
-  size_t ret = 0;
-  InputFileStream strme(path);
-
-  std::string line, prevSource;
-  while (std::getline(strme, line)) {
-    std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
-    assert(toks.size() != 0);
-
-    if (prevSource != toks[0]) {
-      prevSource = toks[0];
-      ++ret;
-    }
-  }
-
-  return ret;
-}
-
-void serialize_cache(
-  std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache,
-  const std::string &path, float totalSourceCount)
-{
-  std::vector<const CacheItem*> vec(cache.size());
-
-  size_t ind = cache.size() - 1;
-  while (!cache.empty()) {
-    const CacheItem *item = cache.top();
-    vec[ind] = item;
-    cache.pop();
-    --ind;
-  }
-
-  std::ofstream os(path.c_str());
-
-  os << totalSourceCount << std::endl;
-  for (size_t i = 0; i < vec.size(); ++i) {
-    const CacheItem *item = vec[i];
-    os << item->count << "\t" << item->sourceKey << "\t" << item->source << std::endl;
-    delete item;
-  }
-
-  os.close();
-}
-
-uint64_t getKey(const std::vector<uint64_t> &vocabid_source)
-{
-  return getKey(vocabid_source.data(), vocabid_source.size());
-}
-
-std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos)
-{
-  assert(endPos < vocabid_source.size());
-
-  std::vector<uint64_t> ret(endPos + 1);
-  for (size_t i = 0; i <= endPos; ++i) {
-    ret[i] = vocabid_source[i];
-  }
-  return ret;
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/storing.hh b/moses/TranslationModel/ProbingPT/storing.hh
deleted file mode 100644
index 994067515..000000000
--- a/moses/TranslationModel/ProbingPT/storing.hh
+++ /dev/null
@@ -1,92 +0,0 @@
-#pragma once
-
-#include <boost/unordered_set.hpp>
-#include <boost/unordered_map.hpp>
-#include <cstdio>
-#include <sstream>
-#include <fstream>
-#include <iostream>
-#include <string>
-#include <queue>
-#include <sys/stat.h> //mkdir
-
-#include "hash.hh" //Includes line_splitter
-#include "probing_hash_utils.hh"
-
-#include "util/file_piece.hh"
-#include "util/file.hh"
-#include "vocabid.hh"
-
-namespace Moses
-{
-typedef std::vector<uint64_t> SourcePhrase;
-
-
-class Node
-{
-  typedef boost::unordered_map<uint64_t, Node> Children;
-  Children m_children;
-
-public:
-  uint64_t key;
-  bool done;
-
-  Node()
-    :done(false)
-  {}
-
-  void Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos = 0);
-  void Write(Table &table);
-};
-
-
-void createProbingPT(const std::string &phrasetable_path,
-                     const std::string &basepath, int num_scores, int num_lex_scores,
-                     bool log_prob, int max_cache_size, bool scfg);
-uint64_t getKey(const std::vector<uint64_t> &source_phrase);
-
-std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos);
-
-template<typename T>
-std::string Debug(const std::vector<T> &vec)
-{
-  std::stringstream strm;
-  for (size_t i = 0; i < vec.size(); ++i) {
-    strm << vec[i] << " ";
-  }
-  return strm.str();
-}
-
-size_t countUniqueSource(const std::string &path);
-
-class CacheItem
-{
-public:
-  std::string source;
-  uint64_t sourceKey;
-  float count;
-  CacheItem(const std::string &vSource, uint64_t vSourceKey, float vCount)
-    :source(vSource)
-    ,sourceKey(vSourceKey)
-    ,count(vCount) {
-  }
-
-  bool operator<(const CacheItem &other) const {
-    return count > other.count;
-  }
-};
-
-class CacheItemOrderer
-{
-public:
-  bool operator()(const CacheItem* a, const CacheItem* b) const {
-    return (*a) < (*b);
-  }
-};
-
-void serialize_cache(
-  std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache,
-  const std::string &path, float totalSourceCount);
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/vocabid.cpp b/moses/TranslationModel/ProbingPT/vocabid.cpp
deleted file mode 100644
index d6f442323..000000000
--- a/moses/TranslationModel/ProbingPT/vocabid.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-#include <boost/foreach.hpp>
-#include "vocabid.hh"
-#include "StoreVocab.h"
-#include "moses/Util.h"
-
-namespace Moses
-{
-
-void add_to_map(StoreVocab<uint64_t> &sourceVocab,
-                const StringPiece &textin)
-{
-  //Tokenize
-  util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' '));
-
-  while (itWord) {
-    StringPiece word = *itWord;
-
-    util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|'));
-    while (itFactor) {
-      StringPiece factor = *itFactor;
-
-      sourceVocab.Insert(getHash(factor), factor.as_string());
-      itFactor++;
-    }
-    itWord++;
-  }
-}
-
-void serialize_map(const std::map<uint64_t, std::string> &karta,
-                   const std::string &filename)
-{
-  std::ofstream os(filename.c_str());
-
-  std::map<uint64_t, std::string>::const_iterator iter;
-  for (iter = karta.begin(); iter != karta.end(); ++iter) {
-    os << iter->first << '\t' << iter->second << std::endl;
-  }
-
-  os.close();
-}
-
-void read_map(std::map<uint64_t, std::string> &karta, const char* filename)
-{
-  std::ifstream is(filename);
-
-  std::string line;
-  while (getline(is, line)) {
-    std::vector<std::string> toks = Tokenize(line, "\t");
-    assert(toks.size() == 2);
-    uint64_t ind = Scan<uint64_t>(toks[1]);
-    karta[ind] = toks[0];
-  }
-
-  //Close the stream after we are done.
-  is.close();
-}
-
-}
-
diff --git a/moses/TranslationModel/ProbingPT/vocabid.hh b/moses/TranslationModel/ProbingPT/vocabid.hh
deleted file mode 100644
index 7e1390874..000000000
--- a/moses/TranslationModel/ProbingPT/vocabid.hh
+++ /dev/null
@@ -1,29 +0,0 @@
-//Serialization
-#include <boost/serialization/serialization.hpp>
-#include <boost/serialization/map.hpp>
-#include <boost/archive/text_iarchive.hpp>
-#include <boost/archive/text_oarchive.hpp>
-#include <fstream>
-#include <iostream>
-#include <vector>
-
-#include <map> //Container
-#include "hash.hh" //Hash of elements
-
-#include "util/string_piece.hh"  //Tokenization and work with StringPiece
-#include "util/tokenize_piece.hh"
-
-namespace Moses
-{
-template<typename VOCABID>
-class StoreVocab;
-
-void add_to_map(StoreVocab<uint64_t> &sourceVocab,
-                const StringPiece &textin);
-
-void serialize_map(const std::map<uint64_t, std::string> &karta,
-                   const std::string &filename);
-
-void read_map(std::map<uint64_t, std::string> &karta, const char* filename);
-
-}
diff --git a/moses/TranslationModel/UG/Makefile b/moses/TranslationModel/UG/Makefile
index 213f87e0c..0006af638 100644
--- a/moses/TranslationModel/UG/Makefile
+++ b/moses/TranslationModel/UG/Makefile
@@ -35,7 +35,7 @@ CXXFLAGS += -DMAX_NUM_FACTORS=4
 CXXFLAGS += -DKENLM_MAX_ORDER=5
 CXXFLAGS += -DWITH_THREADS
 CXXFLAGS += -DNO_MOSES
-CXXFLAGS += -DMMT
+# CXXFLAGS += -DMMT
 CXXFLAGS += -I$(dir ${MOSES_ROOT})mmt-only
 CXXFLAGS += -I${MOSES_ROOT} -I. -I${MOSES_ROOT}/opt/include 
 
@@ -47,7 +47,7 @@ else ifeq ($(variant),syntax)
 CXXFLAGS += -fsyntax-only
 endif
 
-LDFLAGS  = -L${MOSES_ROOT}/opt/lib64 -L./lib/ 
+LDFLAGS  = -L${MOSES_ROOT}/opt/lib -L./lib/ 
 
 # WDIR = build/$(variant)/${HOSTTYPE}/${KERNEL}
 WDIR = build/$(variant)
@@ -73,7 +73,7 @@ DYNAMIC_LIBS  = pthread
 #DYNAMIC_LIBS  += tcmalloc
 
 LIBS  = -Wl,-B$(link) 
-LIBS += -L${MOSES_ROOT}/opt/lib64 ${BOOSTLIBS}
+LIBS += -L${MOSES_ROOT}/opt/lib ${BOOSTLIBS}
 LIBS += $(addprefix -l,${STATIC_LIBS})
 LIBS += -Wl,-Bdynamic
 LIBS += $(addprefix -l,${DYNAMIC_LIBS})
diff --git a/moses/TranslationModel/UG/check-coverage5.cc b/moses/TranslationModel/UG/check-coverage5.cc
new file mode 100644
index 000000000..fe9479136
--- /dev/null
+++ b/moses/TranslationModel/UG/check-coverage5.cc
@@ -0,0 +1,129 @@
+// -*- mode: c++; indent-tabs-mode: nil; tab-width:2  -*-
+
+// read a text from stdin, report percentage of n-grams covered
+
+#include <boost/foreach.hpp>
+#include <boost/format.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/shared_ptr.hpp>
+#include <algorithm>
+#include <iostream>
+#include "mm/ug_bitext.h"
+#include "generic/file_io/ug_stream.h"
+#include <string>
+#include <sstream>
+#include "mm/ug_bitext_sampler.h"
+
+#include <boost/program_options.hpp>
+#include <boost/math/distributions/binomial.hpp>
+
+// #include "LSA.h"
+
+namespace po=boost::program_options;
+using namespace Moses;
+using namespace sapt;
+using namespace std;
+using namespace boost;
+
+typedef sapt::L2R_Token<sapt::SimpleWordId> Token;
+typedef mmTtrack<Token> ttrack_t;
+
+size_t ngram_size;
+size_t verbosity;
+string bname;
+vector<string> ifiles;
+
+void interpret_args(int ac, char* av[]);
+
+
+void
+dump(mmTSA<Token>::tree_iterator& m, TokenIndex& V)
+{
+  if (m.size()) cout << m.str(NULL) << endl;
+  if (m.size()) cout << m.str(&V) << endl;
+  if (m.down())
+    {
+      do { dump(m, V); } while (m.over());
+      m.up();
+    }
+}
+
+int
+main(int argc, char* argv[])
+{
+  interpret_args(argc,argv);
+  TokenIndex V;
+  V.open(bname+".tdx"); V.setDynamic(true); V.iniReverseIndex();
+  boost::shared_ptr<mmTtrack<Token> > T(new mmTtrack<Token>);
+  T->open(bname+".mct");
+  mmTSA<Token> I; I.open(bname+".sfa", T);
+
+  string line;
+  BOOST_FOREACH(string const& file, ifiles)
+    {
+      size_t total_ngrams=0;
+      float matched_ngrams=0;
+      ifstream in(file.c_str());
+      while(getline(in,line))
+        {
+          // cout << line << endl;
+          vector<id_type> snt;
+          V.fillIdSeq(line,snt);
+          if (snt.size() < ngram_size) continue;
+          total_ngrams += snt.size() - ngram_size + 1;
+          for (size_t i = 0; i + ngram_size <= snt.size(); ++i)
+            // for (size_t i = 0; i < snt.size(); ++i)
+            {
+              mmTSA<Token>::tree_iterator m(&I);
+              size_t stop = min(snt.size(), i+ngram_size);
+              size_t k = i; 
+              while (k < stop && m.extend(snt[k])) ++k;
+              if (verbosity) cout << i << " " << k-i << " " << m.str(&V) << endl;
+              if (k - i == ngram_size)
+                ++matched_ngrams;
+            }
+        }
+      printf ("%5.1f%% matched %zu-grams (%.0f/%zu): %s\n",
+              (100 * matched_ngrams / total_ngrams), ngram_size,
+              matched_ngrams, total_ngrams, file.c_str());
+    }
+}
+
+void
+interpret_args(int ac, char* av[])
+{
+  po::variables_map vm;
+  po::options_description o("Options");
+  o.add_options()
+
+    ("help,h",  "print this message")
+    ("ngram-size,n", po::value<size_t>(&ngram_size)->default_value(5),
+     "sample size")
+    ("verbose,v", po::value<size_t>(&verbosity)->default_value(0),
+     "verbosity")
+    ;
+
+  po::options_description h("Hidden Options");
+  h.add_options()
+    ("bname", po::value<string>(&bname), "base name of corpus")
+    ("ifiles", po::value<vector<string> >(&ifiles), "input files")
+    ;
+
+  h.add(o);
+  po::positional_options_description a;
+  a.add("bname",1);
+  a.add("ifiles",-1);
+
+  po::store(po::command_line_parser(ac,av)
+            .options(h)
+            .positional(a)
+            .run(),vm);
+  po::notify(vm);
+  if (vm.count("help"))
+    {
+      std::cout << "\nusage:\n\t" << av[0]
+                << " [options] <model file stem>" << std::endl;
+      std::cout << o << std::endl;
+      exit(0);
+    }
+}
diff --git a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc
index 08e123782..4e8095f61 100644
--- a/moses/TranslationModel/UG/mm/tpt_tokenindex.cc
+++ b/moses/TranslationModel/UG/mm/tpt_tokenindex.cc
@@ -59,8 +59,8 @@ namespace sapt
     comp.base = reinterpret_cast<char const*>(endIdx);
     if (!unkToken.empty())
       {
-	Entry const* bla = lower_bound(startIdx,endIdx,unkToken.c_str(),comp);
-	unkId = ((bla < endIdx && unkToken == comp.base+bla->offset)
+        Entry const* bla = lower_bound(startIdx,endIdx,unkToken.c_str(),comp);
+        unkId = ((bla < endIdx && unkToken == comp.base+bla->offset)
                  ? bla->id
                  : numTokens);
       }
@@ -98,10 +98,10 @@ namespace sapt
   {
     if (startIdx != endIdx)
       {
-	Entry const* bla = lower_bound(startIdx,endIdx,p,comp);
-	if (bla != endIdx && !strcmp(comp.base+bla->offset,p))
-	  return bla->id;
-	if (!dynamic) return unkId;
+        Entry const* bla = lower_bound(startIdx,endIdx,p,comp);
+        if (bla != endIdx && !strcmp(comp.base+bla->offset,p))
+          return bla->id;
+        if (!dynamic) return unkId;
       }
     else if (!dynamic) return strcmp(p,"NULL") && unkId;
     
@@ -155,10 +155,10 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	// Someone else (multi-threading!) may have created the 
-	// reverse index in the meantime, so let's check again
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        // Someone else (multi-threading!) may have created the 
+        // reverse index in the meantime, so let's check again
+        if (!ridx.size()) ridx = reverseIndex();
       }
     if (id < ridx.size())
       return ridx[id];
@@ -175,8 +175,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
   }
 
@@ -187,8 +187,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
     if (id < ridx.size())
       return ridx[id];
@@ -204,8 +204,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
     ostringstream buf;
     for (size_t i = 0; i < v.size(); i++)
@@ -219,8 +219,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
     ostringstream buf;
     for (size_t i = 0; i < v.size(); i++)
@@ -234,8 +234,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
     ostringstream buf;
     if (start < stop)
@@ -251,8 +251,8 @@ namespace sapt
   {
     if (!ridx.size())
       {
-	boost::lock_guard<boost::mutex> lk(*this->lock);
-	if (!ridx.size()) ridx = reverseIndex();
+        boost::lock_guard<boost::mutex> lk(*this->lock);
+        if (!ridx.size()) ridx = reverseIndex();
       }
     ostringstream buf;
     if (start < stop)
diff --git a/moses/TranslationModel/UG/mm/ug_typedefs.h b/moses/TranslationModel/UG/mm/ug_typedefs.h
index fc9d7faef..4815846a2 100644
--- a/moses/TranslationModel/UG/mm/ug_typedefs.h
+++ b/moses/TranslationModel/UG/mm/ug_typedefs.h
@@ -36,7 +36,7 @@ namespace sapt
 #ifndef SPTR
 #define SPTR   boost::shared_ptr
 #endif
-#define iptr   boost::intrusive_ptr
+#define boost_iptr   boost::intrusive_ptr
 #define scoptr boost::scoped_ptr
 #define rcast  reinterpret_cast
 #endif
diff --git a/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc b/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc
index 613e46360..ff9f1c722 100644
--- a/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc
+++ b/moses/TranslationModel/UG/test-ranked-phrase-lookup.cc
@@ -76,7 +76,7 @@ int main(int argc, char* argv[])
 {
   typedef vector<PhrasePair<Token> > pplist_t;
   interpret_args(argc, argv);
-  iptr<mmbitext> Bptr(new mmbitext);
+  boost_iptr<mmbitext> Bptr(new mmbitext);
   mmbitext& B = *Bptr;// static_cast<mmbitext*>(Bptr.get());
   B.open(bname, L1, L2);
   B.V1->setDynamic(true);
diff --git a/moses/parameters/AllOptions.cpp b/moses/parameters/AllOptions.cpp
index 868b8e4fb..019e1b0c7 100644
--- a/moses/parameters/AllOptions.cpp
+++ b/moses/parameters/AllOptions.cpp
@@ -101,6 +101,11 @@ namespace Moses
     if (!syntax.update(param))     return false;
     return sanity_check();
   }
+#else
+  bool 
+  AllOptions::
+  update(std::map<std::string,xmlrpc_c::value>const& param)
+  {}
 #endif
 
   bool
diff --git a/moses/parameters/CubePruningOptions.cpp b/moses/parameters/CubePruningOptions.cpp
index 793ae2db6..847a421f8 100644
--- a/moses/parameters/CubePruningOptions.cpp
+++ b/moses/parameters/CubePruningOptions.cpp
@@ -72,6 +72,11 @@ namespace Moses
 
       return true;
     }
+#else
+    bool 
+    CubePruningOptions::
+    update(std::map<std::string,xmlrpc_c::value>const& params)
+    {}
 #endif
 
 
diff --git a/moses/parameters/InputOptions.cpp b/moses/parameters/InputOptions.cpp
index dbdb2f60e..55bc49885 100644
--- a/moses/parameters/InputOptions.cpp
+++ b/moses/parameters/InputOptions.cpp
@@ -95,6 +95,11 @@ namespace Moses {
       xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second));
     return true;
   }
+#else
+  bool 
+  InputOptions::
+  update(std::map<std::string,xmlrpc_c::value>const& param)
+  {}
 #endif
 
 }
diff --git a/moses/parameters/NBestOptions.cpp b/moses/parameters/NBestOptions.cpp
index 3000f49df..afae0ed6c 100644
--- a/moses/parameters/NBestOptions.cpp
+++ b/moses/parameters/NBestOptions.cpp
@@ -62,6 +62,11 @@ update(std::map<std::string,xmlrpc_c::value>const& param)
   enabled = (nbest_size > 0);
   return true;
 }
+#else
+bool 
+NBestOptions::
+update(std::map<std::string,xmlrpc_c::value>const& param)
+{}
 #endif
 
 
diff --git a/moses/parameters/OOVHandlingOptions.cpp b/moses/parameters/OOVHandlingOptions.cpp
index 154074664..a55026b98 100644
--- a/moses/parameters/OOVHandlingOptions.cpp
+++ b/moses/parameters/OOVHandlingOptions.cpp
@@ -43,6 +43,11 @@ namespace Moses {
     //   xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second));
     return true;
   }
+#else
+  bool 
+  OOVHandlingOptions::
+  update(std::map<std::string,xmlrpc_c::value>const& param)
+  {}
 #endif
 
 }
diff --git a/moses/parameters/OptionsBaseClass.cpp b/moses/parameters/OptionsBaseClass.cpp
index a19aaf7a6..324359e93 100644
--- a/moses/parameters/OptionsBaseClass.cpp
+++ b/moses/parameters/OptionsBaseClass.cpp
@@ -24,5 +24,10 @@ namespace Moses {
     if (m == param.end()) return dfltval;
     return Scan<bool>(xmlrpc_c::value_string(m->second));
   }
+#else
+  bool 
+  check(std::map<std::string, xmlrpc_c::value> const& param, 
+        std::string const key, bool dfltval)
+  {}
 #endif
 }
diff --git a/moses/parameters/ReportingOptions.cpp b/moses/parameters/ReportingOptions.cpp
index 210950a3c..fe56a7356 100644
--- a/moses/parameters/ReportingOptions.cpp
+++ b/moses/parameters/ReportingOptions.cpp
@@ -128,11 +128,17 @@ namespace Moses {
       for (size_t i = 0; i < MAX_NUM_FACTORS; ++i)
         factor_order.push_back(i);
     }
+
+    m = param.find("no-ReportSegmentation");
+    if (m == param.end() || !Scan<bool>(xmlrpc_c::value_string(m->second))) {
+
+      // If we are reporting alignment info, turn on ReportSegmentation, unless XML request explicitly says not to
+      m = param.find("align");
+      if (m != param.end() && Scan<bool>(xmlrpc_c::value_string(m->second)))
+        ReportSegmentation = 1;
     
-    m = param.find("align");
-    if (m != param.end() && Scan<bool>(xmlrpc_c::value_string(m->second)))
-      ReportSegmentation = 1;
-    
+    }
+
     PrintAlignmentInfo = check(param,"word-align",PrintAlignmentInfo);
 
     m = param.find("factor-delimiter");
diff --git a/moses/parameters/SearchOptions.cpp b/moses/parameters/SearchOptions.cpp
index 958569e94..35028b8fb 100644
--- a/moses/parameters/SearchOptions.cpp
+++ b/moses/parameters/SearchOptions.cpp
@@ -102,6 +102,11 @@ namespace Moses
       
       return true;
     }
+#else
+    bool 
+    SearchOptions::
+    update(std::map<std::string,xmlrpc_c::value>const& params)
+    {}
 #endif
 
 }
diff --git a/moses/parameters/SyntaxOptions.cpp b/moses/parameters/SyntaxOptions.cpp
index f76c187ec..e977dd4c0 100644
--- a/moses/parameters/SyntaxOptions.cpp
+++ b/moses/parameters/SyntaxOptions.cpp
@@ -77,6 +77,11 @@ namespace Moses {
     //   xml_policy = Scan<XmlInputType>(xmlrpc_c::value_string(si->second));
     return true;
   }
+#else
+  bool 
+  SyntaxOptions::
+  update(std::map<std::string,xmlrpc_c::value>const& param)
+  {}
 #endif
 
 }
diff --git a/moses/server/TranslationRequest.cpp b/moses/server/TranslationRequest.cpp
index 7b3aa8c06..d2118ad8f 100644
--- a/moses/server/TranslationRequest.cpp
+++ b/moses/server/TranslationRequest.cpp
@@ -24,6 +24,9 @@ using Moses::FValue;
 using Moses::PhraseDictionaryMultiModel;
 using Moses::FindPhraseDictionary;
 using Moses::Sentence;
+using Moses::TokenizeMultiCharSeparator;
+using Moses::FeatureFunction;
+using Moses::Scan;
 
 boost::shared_ptr<TranslationRequest>
 TranslationRequest::
@@ -82,13 +85,14 @@ void
 TranslationRequest::
 add_phrase_aln_info(Hypothesis const& h, vector<xmlrpc_c::value>& aInfo) const
 {
-  // if (!m_withAlignInfo) return;
-  if (!options()->output.ReportSegmentation) return;
+  if (!m_withAlignInfo) return;
+  //  if (!options()->output.ReportSegmentation) return;
   Range const& trg = h.GetCurrTargetWordsRange();
   Range const& src = h.GetCurrSourceWordsRange();
 
   std::map<std::string, xmlrpc_c::value> pAlnInfo;
   pAlnInfo["tgt-start"] = xmlrpc_c::value_int(trg.GetStartPos());
+  pAlnInfo["tgt-end"] = xmlrpc_c::value_int(trg.GetEndPos());
   pAlnInfo["src-start"] = xmlrpc_c::value_int(src.GetStartPos());
   pAlnInfo["src-end"]   = xmlrpc_c::value_int(src.GetEndPos());
   aInfo.push_back(xmlrpc_c::value_struct(pAlnInfo));
@@ -211,6 +215,9 @@ insertTranslationOptions(Moses::Manager& manager,
         for (size_t j = 0; j < scores.size(); ++j)
           scoresXml.push_back(xmlrpc_c::value_double(scores[j]));
         toptXml["scores"] = xmlrpc_c::value_array(scoresXml);
+	ostringstream buf;
+	topt->GetScoreBreakdown().OutputAllFeatureScores(buf, true);
+	toptXml["labelledScores"] = PackScores(topt->GetScoreBreakdown());
         toptsXml.push_back(xmlrpc_c::value_struct(toptXml));
       }
     }
@@ -312,6 +319,106 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params)
       m_context.reset(new std::vector<std::string>(1,context));
     }
 
+  si = params.find("context-scope");
+  if (si != params.end())
+    {
+
+      string context = xmlrpc_c::value_string(si->second);
+
+      string groupSeparator("Moses::ContextScope::GroupSeparator");
+      string recordSeparator("Moses::ContextScope::RecordSeparator");
+
+      // Here, we assume that any XML-RPC value
+      //       associated with the key "context-scope"
+      //       has the following format:
+      //
+      // FeatureFunctionName followed by recordSeparator
+      //                     followed by the value of interest
+      //                     followed by groupSeparator
+      //
+      // In the following code, the value of interest will be stored
+      //        in contextScope under the key FeatureFunctionName,
+      //        where FeatureFunctionName is the actual name of the feature function
+
+      boost::shared_ptr<Moses::ContextScope> contextScope = GetScope();
+
+      BOOST_FOREACH(string group, TokenizeMultiCharSeparator(context, groupSeparator)) {
+
+	vector<string> record = TokenizeMultiCharSeparator(group, recordSeparator);
+
+	// Use the feature function whose name is record[0] as a key
+	FeatureFunction& ff = Moses::FeatureFunction::FindFeatureFunction(record[0]);
+	void const* key = static_cast<void const*>(&ff);
+
+	// Store (in the context scope) record[1] as the value associated with that key
+	boost::shared_ptr<string> value = contextScope->get<string>(key,true);
+	value->replace(value->begin(), value->end(), record[1]);
+
+      }
+    }
+
+  // Report alignment info if Moses config says to or if XML request says to
+  m_withAlignInfo = options()->output.ReportSegmentation || check(params, "align");
+
+  // Report word alignment info if Moses config says to or if XML request says to
+  m_withWordAlignInfo = options()->output.PrintAlignmentInfo || check(params, "word-align");
+
+  si = params.find("weights");
+  if (si != params.end())
+    {
+
+      boost::unordered_map<string, FeatureFunction*> map;
+      {
+	const vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+	BOOST_FOREACH(FeatureFunction* const& ff, ffs) {
+	  map[ff->GetScoreProducerDescription()] = ff;
+	}
+      }
+
+      string allValues = xmlrpc_c::value_string(si->second);
+
+      BOOST_FOREACH(string values, TokenizeMultiCharSeparator(allValues, "\t")) {
+
+	vector<string> record = TokenizeMultiCharSeparator(values, "=");
+
+	if (record.size() == 2) {
+	  string featureName = record[0];
+	  string featureWeights = record[1];
+
+	  boost::unordered_map<string, FeatureFunction*>::iterator ffi = map.find(featureName);
+
+	  if (ffi != map.end()) {
+	    FeatureFunction* ff = ffi->second;
+
+	    size_t prevNumWeights = ff->GetNumScoreComponents();
+
+	    vector<float> ffWeights;
+	    BOOST_FOREACH(string weight, TokenizeMultiCharSeparator(featureWeights, " ")) {
+	      ffWeights.push_back(Scan<float>(weight));
+	    }
+
+	    if (ffWeights.size() == ff->GetNumScoreComponents()) {
+
+	      // XXX: This is NOT thread-safe
+	      Moses::StaticData::InstanceNonConst().SetWeights(ff, ffWeights);
+	      VERBOSE(1, "WARNING: THIS IS NOT THREAD-SAFE!\tUpdating weights for " << featureName << " to " << featureWeights << "\n");
+
+	    } else {
+	      TRACE_ERR("ERROR: Unable to update weights for " << featureName << " because " << ff->GetNumScoreComponents() << " weights are required but only " << ffWeights.size() << " were provided\n");
+	    }
+
+	  } else {
+	    TRACE_ERR("ERROR: No FeatureFunction with name " << featureName << ", no weight update\n");
+	  }
+
+	} else {
+	  TRACE_ERR("WARNING: XML-RPC weights update was improperly formatted:\t" << values << "\n");
+	}
+
+      }
+
+    }
+
 
   // // biased sampling for suffix-array-based sampling phrase table?
   // if ((si = params.find("bias")) != params.end())
@@ -366,8 +473,8 @@ pack_hypothesis(const Moses::Manager& manager,
 	   << std::endl);
   dest[key] = xmlrpc_c::value_string(target.str());
 
-  // if (m_withAlignInfo) {
-  if (options()->output.ReportSegmentation) {
+  if (m_withAlignInfo) {
+  //  if (options()->output.ReportSegmentation) {
     // phrase alignment, if requested
 
     vector<xmlrpc_c::value> p_aln;
@@ -376,8 +483,8 @@ pack_hypothesis(const Moses::Manager& manager,
     dest["align"] = xmlrpc_c::value_array(p_aln);
   }
 
-  // if (m_withWordAlignInfo) {
-  if (options()->output.PrintAlignmentInfo) { 
+  if (m_withWordAlignInfo) {
+    //if (options()->output.PrintAlignmentInfo) { 
     // word alignment, if requested
     vector<xmlrpc_c::value> w_aln;
     BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
diff --git a/moses/server/TranslationRequest.h b/moses/server/TranslationRequest.h
index 2554e5544..3463c72c8 100644
--- a/moses/server/TranslationRequest.h
+++ b/moses/server/TranslationRequest.h
@@ -38,8 +38,8 @@ TranslationRequest : public virtual Moses::TranslationTask
 
   Translator* m_translator;
   std::string m_source_string, m_target_string;
-  // bool m_withAlignInfo;
-  // bool m_withWordAlignInfo;
+  bool m_withAlignInfo;
+  bool m_withWordAlignInfo;
   bool m_withGraphInfo;
   bool m_withTopts;
   bool m_withScoreBreakdown;