diff options
author | Philipp Koehn <phi@jhu.edu> | 2016-05-29 18:46:42 +0300 |
---|---|---|
committer | Philipp Koehn <phi@jhu.edu> | 2016-05-29 18:46:42 +0300 |
commit | 942eb5a8b1183f321b6abb980ffecfdf39749710 (patch) | |
tree | ec8e3e7b923c626d05776299c8d34838699ff11e /moses/FF | |
parent | 227e54162a343bb9c8610a381be7480e7ec53476 (diff) |
allow configuration of operation sequence model loading, allow specification of KENLM/OSM loading in experiment.perl / train-model.perl
Diffstat (limited to 'moses/FF')
-rw-r--r-- | moses/FF/OSM-Feature/KenOSM.cpp | 3 | ||||
-rw-r--r-- | moses/FF/OSM-Feature/KenOSM.h | 2 | ||||
-rw-r--r-- | moses/FF/OSM-Feature/OpSequenceModel.cpp | 17 | ||||
-rw-r--r-- | moses/FF/OSM-Feature/OpSequenceModel.h | 1 |
4 files changed, 20 insertions, 3 deletions
diff --git a/moses/FF/OSM-Feature/KenOSM.cpp b/moses/FF/OSM-Feature/KenOSM.cpp index 25a1e6a93..d20e762f6 100644 --- a/moses/FF/OSM-Feature/KenOSM.cpp +++ b/moses/FF/OSM-Feature/KenOSM.cpp @@ -3,10 +3,11 @@ namespace Moses { -OSMLM* ConstructOSMLM(const char *file) +OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method) { lm::ngram::ModelType model_type; lm::ngram::Config config; + config.load_method = load_method; if (lm::ngram::RecognizeBinary(file, model_type)) { switch(model_type) { case lm::ngram::PROBING: diff --git a/moses/FF/OSM-Feature/KenOSM.h b/moses/FF/OSM-Feature/KenOSM.h index 53268442b..ce3872a35 100644 --- a/moses/FF/OSM-Feature/KenOSM.h +++ b/moses/FF/OSM-Feature/KenOSM.h @@ -47,7 +47,7 @@ private: typedef KenOSMBase OSMLM; -OSMLM* ConstructOSMLM(const char *file); +OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method); } // namespace diff --git a/moses/FF/OSM-Feature/OpSequenceModel.cpp b/moses/FF/OSM-Feature/OpSequenceModel.cpp index 4118c8690..1c889e329 100644 --- a/moses/FF/OSM-Feature/OpSequenceModel.cpp +++ b/moses/FF/OSM-Feature/OpSequenceModel.cpp @@ -17,6 +17,7 @@ OpSequenceModel::OpSequenceModel(const std::string &line) tFactor = 0; numFeatures = 5; ReadParameters(); + load_method = util::READ; } OpSequenceModel::~OpSequenceModel() @@ -27,7 +28,7 @@ OpSequenceModel::~OpSequenceModel() void OpSequenceModel :: readLanguageModel(const char *lmFile) { string unkOp = "_TRANS_SLF_"; - OSM = ConstructOSMLM(m_lmPath.c_str()); + OSM = ConstructOSMLM(m_lmPath.c_str(), load_method); State startState = OSM->NullContextState(); State endState; @@ -248,6 +249,20 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va sFactor = Scan<int>(value); } else if (key == "output-factor") { tFactor = Scan<int>(value); + } else if (key == "load") { + if (value == "lazy") { + load_method = util::LAZY; + } else if (value == "populate_or_lazy") { + load_method = util::POPULATE_OR_LAZY; + } else if (value == "populate_or_read" || value == "populate") { + load_method = util::POPULATE_OR_READ; + } else if (value == "read") { + load_method = util::READ; + } else if (value == "parallel_read") { + load_method = util::PARALLEL_READ; + } else { + UTIL_THROW2("Unknown KenLM load method " << value); + } } else { StatefulFeatureFunction::SetParameter(key, value); } diff --git a/moses/FF/OSM-Feature/OpSequenceModel.h b/moses/FF/OSM-Feature/OpSequenceModel.h index 925f9c83a..f13d93dcb 100644 --- a/moses/FF/OSM-Feature/OpSequenceModel.h +++ b/moses/FF/OSM-Feature/OpSequenceModel.h @@ -20,6 +20,7 @@ public: int sFactor; // Source Factor ... int tFactor; // Target Factor ... int numFeatures; // Number of features used ... + util::LoadMethod load_method; // method to load model OpSequenceModel(const std::string &line); ~OpSequenceModel(); |