allow configuration of operation sequence model loading, allow specification of KENLM/OSM loading in experiment.perl / train-model.perl

4 files changed, 20 insertions, 3 deletions

diff --git a/moses/FF/OSM-Feature/KenOSM.cpp b/moses/FF/OSM-Feature/KenOSM.cpp
index 25a1e6a93..d20e762f6 100644
--- a/moses/FF/OSM-Feature/KenOSM.cpp
+++ b/moses/FF/OSM-Feature/KenOSM.cpp
@@ -3,10 +3,11 @@
 namespace Moses
 {
 
-OSMLM* ConstructOSMLM(const char *file)
+OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method)
 {
   lm::ngram::ModelType model_type;
   lm::ngram::Config config;
+  config.load_method = load_method;
   if (lm::ngram::RecognizeBinary(file, model_type)) {
     switch(model_type) {
     case lm::ngram::PROBING:
diff --git a/moses/FF/OSM-Feature/KenOSM.h b/moses/FF/OSM-Feature/KenOSM.h
index 53268442b..ce3872a35 100644
--- a/moses/FF/OSM-Feature/KenOSM.h
+++ b/moses/FF/OSM-Feature/KenOSM.h
@@ -47,7 +47,7 @@ private:
 
 typedef KenOSMBase OSMLM;
 
-OSMLM* ConstructOSMLM(const char *file);
+OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method);
 
 
 } // namespace
diff --git a/moses/FF/OSM-Feature/OpSequenceModel.cpp b/moses/FF/OSM-Feature/OpSequenceModel.cpp
index 4118c8690..1c889e329 100644
--- a/moses/FF/OSM-Feature/OpSequenceModel.cpp
+++ b/moses/FF/OSM-Feature/OpSequenceModel.cpp
@@ -17,6 +17,7 @@ OpSequenceModel::OpSequenceModel(const std::string &line)
   tFactor = 0;
   numFeatures = 5;
   ReadParameters();
+  load_method = util::READ;
 }
 
 OpSequenceModel::~OpSequenceModel()
@@ -27,7 +28,7 @@ OpSequenceModel::~OpSequenceModel()
 void OpSequenceModel :: readLanguageModel(const char *lmFile)
 {
   string unkOp = "_TRANS_SLF_";
-  OSM = ConstructOSMLM(m_lmPath.c_str());
+  OSM = ConstructOSMLM(m_lmPath.c_str(), load_method);
 
   State startState = OSM->NullContextState();
   State endState;
@@ -248,6 +249,20 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
     sFactor = Scan<int>(value);
   } else if (key == "output-factor") {
     tFactor = Scan<int>(value);
+  } else if (key == "load") {
+    if (value == "lazy") {
+      load_method = util::LAZY;
+    } else if (value == "populate_or_lazy") {
+      load_method = util::POPULATE_OR_LAZY;
+    } else if (value == "populate_or_read" || value == "populate") {
+      load_method = util::POPULATE_OR_READ;
+    } else if (value == "read") {
+      load_method = util::READ;
+    } else if (value == "parallel_read") {
+      load_method = util::PARALLEL_READ;
+    } else {
+      UTIL_THROW2("Unknown KenLM load method " << value);
+    }
   } else {
     StatefulFeatureFunction::SetParameter(key, value);
   }
diff --git a/moses/FF/OSM-Feature/OpSequenceModel.h b/moses/FF/OSM-Feature/OpSequenceModel.h
index 925f9c83a..f13d93dcb 100644
--- a/moses/FF/OSM-Feature/OpSequenceModel.h
+++ b/moses/FF/OSM-Feature/OpSequenceModel.h
@@ -20,6 +20,7 @@ public:
   int sFactor;	// Source Factor ...
   int tFactor;	// Target Factor ...
   int numFeatures;   // Number of features used ...
+  util::LoadMethod load_method; // method to load model 
 
   OpSequenceModel(const std::string &line);
   ~OpSequenceModel();