Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/moses-smt/mosesdecoder.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/moses/FF
diff options
context:
space:
mode:
Diffstat (limited to 'moses/FF')
-rw-r--r--moses/FF/DistortionScoreProducer.h4
-rw-r--r--moses/FF/FeatureFunction.cpp53
-rw-r--r--moses/FF/FeatureFunction.h20
-rw-r--r--moses/FF/PhrasePairFeature.cpp5
-rw-r--r--moses/FF/PhrasePairFeature.h1
-rw-r--r--moses/FF/SourceWordDeletionFeature.cpp49
-rw-r--r--moses/FF/SourceWordDeletionFeature.h5
-rw-r--r--moses/FF/StatelessFeatureFunction.h8
-rw-r--r--moses/FF/TargetBigramFeature.cpp41
-rw-r--r--moses/FF/TargetBigramFeature.h4
-rw-r--r--moses/FF/TargetNgramFeature.cpp28
-rw-r--r--moses/FF/TargetNgramFeature.h1
-rw-r--r--moses/FF/TargetWordInsertionFeature.cpp28
-rw-r--r--moses/FF/TargetWordInsertionFeature.h1
-rw-r--r--moses/FF/WordTranslationFeature.cpp60
-rw-r--r--moses/FF/WordTranslationFeature.h1
16 files changed, 187 insertions, 122 deletions
diff --git a/moses/FF/DistortionScoreProducer.h b/moses/FF/DistortionScoreProducer.h
index 305326bd7..a51f3bdb8 100644
--- a/moses/FF/DistortionScoreProducer.h
+++ b/moses/FF/DistortionScoreProducer.h
@@ -18,8 +18,8 @@ class DistortionScoreProducer : public StatefulFeatureFunction
{
public:
DistortionScoreProducer(const std::string &line)
- : StatefulFeatureFunction("Distortion", 1, line)
- {}
+ : StatefulFeatureFunction("Distortion", 1, line) {
+ }
bool IsUseable(const FactorMask &mask) const {
return true;
diff --git a/moses/FF/FeatureFunction.cpp b/moses/FF/FeatureFunction.cpp
index 092caea32..ae5a38983 100644
--- a/moses/FF/FeatureFunction.cpp
+++ b/moses/FF/FeatureFunction.cpp
@@ -22,10 +22,10 @@ std::vector<const StatefulFeatureFunction*> StatefulFeatureFunction::m_stateful
FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name)
{
for (size_t i = 0; i < m_producers.size(); ++i) {
- FeatureFunction &ff = *m_producers[i];
- if (ff.GetScoreProducerDescription() == name) {
- return ff;
- }
+ FeatureFunction &ff = *m_producers[i];
+ if (ff.GetScoreProducerDescription() == name) {
+ return ff;
+ }
}
throw "Unknown feature " + name;
@@ -50,25 +50,24 @@ void FeatureFunction::Initialize(const std::string& description, const std::stri
size_t ind = 0;
while (ind < m_args.size()) {
- vector<string> &args = m_args[ind];
+ vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
- m_args.erase(m_args.begin() + ind);
- }
- else {
- ++ind;
+ m_args.erase(m_args.begin() + ind);
+ } else {
+ ++ind;
}
}
if (m_description == "") {
- size_t index = description_counts.count(description);
+ size_t index = description_counts.count(description);
- ostringstream dstream;
- dstream << description;
- dstream << index;
+ ostringstream dstream;
+ dstream << description;
+ dstream << index;
- description_counts.insert(description);
- m_description = dstream.str();
+ description_counts.insert(description);
+ m_description = dstream.str();
}
ScoreComponentCollection::RegisterScoreProducer(this);
@@ -93,18 +92,18 @@ void FeatureFunction::ParseLine(const std::string& description, const std::strin
bool FeatureFunction::OverrideParameter(const std::string& key, const std::string& value)
{
- if (key == "num-features") {
- m_numScoreComponents = Scan<size_t>(value);
- } else if (key == "name") {
- m_description = value;
- } else if (key == "tuneable") {
- m_tuneable = Scan<bool>(value);
- } else {
- //UTIL_THROW(util::Exception, "unknown key" << key);
- return false;
- }
-
- return true;
+ if (key == "num-features") {
+ m_numScoreComponents = Scan<size_t>(value);
+ } else if (key == "name") {
+ m_description = value;
+ } else if (key == "tuneable") {
+ m_tuneable = Scan<bool>(value);
+ } else {
+ //UTIL_THROW(util::Exception, "unknown key" << key);
+ return false;
+ }
+
+ return true;
}
}
diff --git a/moses/FF/FeatureFunction.h b/moses/FF/FeatureFunction.h
index 71688b436..8b95718e2 100644
--- a/moses/FF/FeatureFunction.h
+++ b/moses/FF/FeatureFunction.h
@@ -55,8 +55,8 @@ public:
virtual ~FeatureFunction();
//! override to load model files
- virtual void Load()
- {}
+ virtual void Load() {
+ }
static void ResetDescriptionCounts() {
description_counts.clear();
@@ -80,12 +80,12 @@ public:
}
//! Called before search and collecting of translation options
- virtual void InitializeForInput(InputType const& source)
- {}
+ virtual void InitializeForInput(InputType const& source) {
+ }
// clean up temporary memory, called after processing each sentence
- virtual void CleanUpAfterSentenceProcessing(const InputType& source)
- {}
+ virtual void CleanUpAfterSentenceProcessing(const InputType& source) {
+ }
const std::string &GetArgLine() const {
return m_argLine;
@@ -99,12 +99,12 @@ public:
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
- , ScoreComponentCollection &estimatedFutureScore) const
- {}
+ , ScoreComponentCollection &estimatedFutureScore) const {
+ }
virtual void Evaluate(const InputType &source
- , ScoreComponentCollection &scoreBreakdown) const
- {}
+ , ScoreComponentCollection &scoreBreakdown) const {
+ }
virtual bool OverrideParameter(const std::string& key, const std::string& value);
};
diff --git a/moses/FF/PhrasePairFeature.cpp b/moses/FF/PhrasePairFeature.cpp
index 9cd50c08b..e3f7e5d9a 100644
--- a/moses/FF/PhrasePairFeature.cpp
+++ b/moses/FF/PhrasePairFeature.cpp
@@ -45,6 +45,11 @@ PhrasePairFeature::PhrasePairFeature(const std::string &line)
Load(filePathSource);
}
+bool PhrasePairFeature::OverrideParameter(const std::string& key, const std::string& value)
+{
+
+}
+
bool PhrasePairFeature::Load(const std::string &filePathSource/*, const std::string &filePathTarget*/)
{
if (m_domainTrigger) {
diff --git a/moses/FF/PhrasePairFeature.h b/moses/FF/PhrasePairFeature.h
index 84d94f09b..bd3ab733a 100644
--- a/moses/FF/PhrasePairFeature.h
+++ b/moses/FF/PhrasePairFeature.h
@@ -45,6 +45,7 @@ public:
}
bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/);
+ bool OverrideParameter(const std::string& key, const std::string& value);
};
diff --git a/moses/FF/SourceWordDeletionFeature.cpp b/moses/FF/SourceWordDeletionFeature.cpp
index 693812105..c7055a79f 100644
--- a/moses/FF/SourceWordDeletionFeature.cpp
+++ b/moses/FF/SourceWordDeletionFeature.cpp
@@ -10,6 +10,7 @@
#include "moses/Util.h"
#include "util/string_piece_hash.hh"
+#include "util/exception.hh"
namespace Moses
{
@@ -22,37 +23,42 @@ SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;
- string filename;
- for (size_t i = 0; i < m_args.size(); ++i) {
- const vector<string> &args = m_args[i];
-
- if (args[0] == "factor") {
- m_factorType = Scan<FactorType>(args[1]);
- } else if (args[0] == "path") {
- filename = args[1];
+ size_t ind = 0;
+ while (ind < m_args.size()) {
+ vector<string> &args = m_args[ind];
+ bool consumed = OverrideParameter(args[0], args[1]);
+ if (consumed) {
+ m_args.erase(m_args.begin() + ind);
} else {
- throw "Unknown argument " + args[0];
+ ++ind;
}
}
- // load word list for restricted feature set
- if (filename != "") {
- cerr << "loading source word deletion word list from " << filename << endl;
- if (!Load(filename)) {
- UserMessage::Add("Unable to load word list for source word deletion feature from file " + filename);
- //return false;
- }
+}
+
+bool SourceWordDeletionFeature::OverrideParameter(const std::string& key, const std::string& value)
+{
+ if (key == "factor") {
+ m_factorType = Scan<FactorType>(value);
+ } else if (key == "path") {
+ m_filename = value;
+ } else {
+ StatelessFeatureFunction::OverrideParameter(key, value);
}
+
}
-bool SourceWordDeletionFeature::Load(const std::string &filePath)
+void SourceWordDeletionFeature::Load()
{
- ifstream inFile(filePath.c_str());
- if (!inFile) {
- cerr << "could not open file " << filePath << endl;
- return false;
+ if (m_filename == "") {
+ return;
}
+ cerr << "loading source word deletion word list from " << m_filename << endl;
+
+ ifstream inFile(m_filename.c_str());
+ UTIL_THROW_IF(!inFile, util::Exception, "Can't open file " << m_filename);
+
std::string line;
while (getline(inFile, line)) {
m_vocab.insert(line);
@@ -61,7 +67,6 @@ bool SourceWordDeletionFeature::Load(const std::string &filePath)
inFile.close();
m_unrestricted = false;
- return true;
}
void SourceWordDeletionFeature::Evaluate(const Phrase &source
diff --git a/moses/FF/SourceWordDeletionFeature.h b/moses/FF/SourceWordDeletionFeature.h
index 0c4faa5e1..3784ca518 100644
--- a/moses/FF/SourceWordDeletionFeature.h
+++ b/moses/FF/SourceWordDeletionFeature.h
@@ -19,11 +19,12 @@ private:
boost::unordered_set<std::string> m_vocab;
FactorType m_factorType;
bool m_unrestricted;
+ std::string m_filename;
public:
SourceWordDeletionFeature(const std::string &line);
- bool Load(const std::string &filePath);
+ void Load();
bool IsUseable(const FactorMask &mask) const {
return true;
@@ -38,6 +39,8 @@ public:
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
+ bool OverrideParameter(const std::string& key, const std::string& value);
+
};
}
diff --git a/moses/FF/StatelessFeatureFunction.h b/moses/FF/StatelessFeatureFunction.h
index 3f120a1de..1e2c6d450 100644
--- a/moses/FF/StatelessFeatureFunction.h
+++ b/moses/FF/StatelessFeatureFunction.h
@@ -24,15 +24,15 @@ public:
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
- ScoreComponentCollection* accumulator) const
- {}
+ ScoreComponentCollection* accumulator) const {
+ }
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
- ScoreComponentCollection* accumulator) const
- {}
+ ScoreComponentCollection* accumulator) const {
+ }
virtual bool IsStateless() const {
return true;
diff --git a/moses/FF/TargetBigramFeature.cpp b/moses/FF/TargetBigramFeature.cpp
index 629c65b19..d04b4af2b 100644
--- a/moses/FF/TargetBigramFeature.cpp
+++ b/moses/FF/TargetBigramFeature.cpp
@@ -4,6 +4,7 @@
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "util/string_piece_hash.hh"
+#include "util/exception.hh"
using namespace std;
@@ -21,30 +22,43 @@ TargetBigramFeature::TargetBigramFeature(const std::string &line)
{
std::cerr << "Initializing target bigram feature.." << std::endl;
- vector<string> tokens = Tokenize(line);
- //CHECK(tokens[0] == m_description);
-
- // set factor
- m_factorType = Scan<FactorType>(tokens[1]);
+ size_t ind = 0;
+ while (ind < m_args.size()) {
+ vector<string> &args = m_args[ind];
+ bool consumed = OverrideParameter(args[0], args[1]);
+ if (consumed) {
+ m_args.erase(m_args.begin() + ind);
+ } else {
+ ++ind;
+ }
+ }
FactorCollection& factorCollection = FactorCollection::Instance();
const Factor* bosFactor =
factorCollection.AddFactor(Output,m_factorType,BOS_);
m_bos.SetFactor(m_factorType,bosFactor);
- const string &filePath = tokens[2];
- Load(filePath);
-
}
-bool TargetBigramFeature::Load(const std::string &filePath)
+bool TargetBigramFeature::OverrideParameter(const std::string& key, const std::string& value)
{
- if (filePath == "*") return true; //allow all
- ifstream inFile(filePath.c_str());
- if (!inFile) {
- return false;
+ if (key == "factor") {
+ m_factorType = Scan<FactorType>(value);
+ } else if (key == "path") {
+ m_filePath = value;
+ } else {
+ StatefulFeatureFunction::OverrideParameter(key, value);
}
+}
+
+void TargetBigramFeature::Load()
+{
+ if (m_filePath == "*")
+ return ; //allow all
+ ifstream inFile(m_filePath.c_str());
+ UTIL_THROW_IF(!inFile, util::Exception, "Can't open file " << m_filePath);
+
std::string line;
m_vocab.insert(BOS_);
m_vocab.insert(BOS_);
@@ -53,7 +67,6 @@ bool TargetBigramFeature::Load(const std::string &filePath)
}
inFile.close();
- return true;
}
diff --git a/moses/FF/TargetBigramFeature.h b/moses/FF/TargetBigramFeature.h
index d90c6276a..5e36a542f 100644
--- a/moses/FF/TargetBigramFeature.h
+++ b/moses/FF/TargetBigramFeature.h
@@ -33,7 +33,7 @@ class TargetBigramFeature : public StatefulFeatureFunction
public:
TargetBigramFeature(const std::string &line);
- bool Load(const std::string &filePath);
+ void Load();
bool IsUseable(const FactorMask &mask) const;
@@ -47,10 +47,12 @@ public:
ScoreComponentCollection* ) const {
abort();
}
+ bool OverrideParameter(const std::string& key, const std::string& value);
private:
FactorType m_factorType;
Word m_bos;
+ std::string m_filePath;
boost::unordered_set<std::string> m_vocab;
};
diff --git a/moses/FF/TargetNgramFeature.cpp b/moses/FF/TargetNgramFeature.cpp
index f9970b025..d6ef6004a 100644
--- a/moses/FF/TargetNgramFeature.cpp
+++ b/moses/FF/TargetNgramFeature.cpp
@@ -42,13 +42,29 @@ TargetNgramFeature::TargetNgramFeature(const std::string &line)
{
std::cerr << "Initializing target ngram feature.." << std::endl;
- vector<string> tokens = Tokenize(line);
- //CHECK(tokens[0] == m_description);
+ size_t ind = 0;
+ while (ind < m_args.size()) {
+ vector<string> &args = m_args[ind];
+ bool consumed = OverrideParameter(args[0], args[1]);
+ if (consumed) {
+ m_args.erase(m_args.begin() + ind);
+ } else {
+ ++ind;
+ }
+ }
+}
- CHECK(tokens.size() == 4);
- m_factorType = Scan<FactorType>(tokens[1]);
- m_n = Scan<size_t>(tokens[2]);
- m_lower_ngrams = Scan<bool>(tokens[3]);
+bool TargetNgramFeature::OverrideParameter(const std::string& key, const std::string& value)
+{
+ if (key == "factor") {
+ m_factorType = Scan<FactorType>(value);
+ } else if (key == "n") {
+ m_n = Scan<size_t>(value);
+ } else if (key == "lower-ngrams") {
+ m_lower_ngrams = Scan<bool>(value);
+ } else {
+ StatefulFeatureFunction::OverrideParameter(key, value);
+ }
}
diff --git a/moses/FF/TargetNgramFeature.h b/moses/FF/TargetNgramFeature.h
index c15f54e49..e886c4520 100644
--- a/moses/FF/TargetNgramFeature.h
+++ b/moses/FF/TargetNgramFeature.h
@@ -191,6 +191,7 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
+ bool OverrideParameter(const std::string& key, const std::string& value);
private:
FactorType m_factorType;
diff --git a/moses/FF/TargetWordInsertionFeature.cpp b/moses/FF/TargetWordInsertionFeature.cpp
index fd3aa321a..40fd63afa 100644
--- a/moses/FF/TargetWordInsertionFeature.cpp
+++ b/moses/FF/TargetWordInsertionFeature.cpp
@@ -20,18 +20,28 @@ TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
m_unrestricted(true)
{
std::cerr << "Initializing target word insertion feature.." << std::endl;
-
- for (size_t i = 0; i < m_args.size(); ++i) {
- const vector<string> &args = m_args[i];
-
- if (args[0] == "factor") {
- m_factorType = Scan<FactorType>(args[1]);
- } else if (args[0] == "path") {
- m_filename = args[1];
+ size_t ind = 0;
+ while (ind < m_args.size()) {
+ vector<string> &args = m_args[ind];
+ bool consumed = OverrideParameter(args[0], args[1]);
+ if (consumed) {
+ m_args.erase(m_args.begin() + ind);
} else {
- throw "Unknown argument " + args[0];
+ ++ind;
}
}
+
+}
+
+bool TargetWordInsertionFeature::OverrideParameter(const std::string& key, const std::string& value)
+{
+ if (key == "factor") {
+ m_factorType = Scan<FactorType>(value);
+ } else if (key == "path") {
+ m_filename = value;
+ } else {
+ StatelessFeatureFunction::OverrideParameter(key, value);
+ }
}
void TargetWordInsertionFeature::Load()
diff --git a/moses/FF/TargetWordInsertionFeature.h b/moses/FF/TargetWordInsertionFeature.h
index 4cfb6037b..5479448a2 100644
--- a/moses/FF/TargetWordInsertionFeature.h
+++ b/moses/FF/TargetWordInsertionFeature.h
@@ -37,6 +37,7 @@ public:
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
+ bool OverrideParameter(const std::string& key, const std::string& value);
};
diff --git a/moses/FF/WordTranslationFeature.cpp b/moses/FF/WordTranslationFeature.cpp
index b353348c8..7a1a92f96 100644
--- a/moses/FF/WordTranslationFeature.cpp
+++ b/moses/FF/WordTranslationFeature.cpp
@@ -27,33 +27,14 @@ WordTranslationFeature::WordTranslationFeature(const std::string &line)
{
std::cerr << "Initializing word translation feature.. " << endl;
- string texttype;
-
- for (size_t i = 0; i < m_args.size(); ++i) {
- const vector<string> &args = m_args[i];
-
- if (args[0] == "input-factor") {
- m_factorTypeSource = Scan<FactorType>(args[1]);
- } else if (args[0] == "output-factor") {
- m_factorTypeTarget = Scan<FactorType>(args[1]);
- } else if (args[0] == "simple") {
- m_simple = Scan<bool>(args[1]);
- } else if (args[0] == "source-context") {
- m_sourceContext = Scan<bool>(args[1]);
- } else if (args[0] == "target-context") {
- m_targetContext = Scan<bool>(args[1]);
- } else if (args[0] == "ignore-punctuation") {
- m_ignorePunctuation = Scan<bool>(args[1]);
- } else if (args[0] == "domain-trigger") {
- m_domainTrigger = Scan<bool>(args[1]);
- } else if (args[0] == "texttype") {
- texttype = args[1];
- } else if (args[0] == "source-path") {
- m_filePathSource = args[1];
- } else if (args[0] == "target-path") {
- m_filePathTarget = args[1];
+ size_t ind = 0;
+ while (ind < m_args.size()) {
+ vector<string> &args = m_args[ind];
+ bool consumed = OverrideParameter(args[0], args[1]);
+ if (consumed) {
+ m_args.erase(m_args.begin() + ind);
} else {
- throw "Unknown argument " + args[0];
+ ++ind;
}
}
@@ -89,6 +70,33 @@ WordTranslationFeature::WordTranslationFeature(const std::string &line)
}
+bool WordTranslationFeature::OverrideParameter(const std::string& key, const std::string& value)
+{
+ if (key == "input-factor") {
+ m_factorTypeSource = Scan<FactorType>(value);
+ } else if (key == "output-factor") {
+ m_factorTypeTarget = Scan<FactorType>(value);
+ } else if (key == "simple") {
+ m_simple = Scan<bool>(value);
+ } else if (key == "source-context") {
+ m_sourceContext = Scan<bool>(value);
+ } else if (key == "target-context") {
+ m_targetContext = Scan<bool>(value);
+ } else if (key == "ignore-punctuation") {
+ m_ignorePunctuation = Scan<bool>(value);
+ } else if (key == "domain-trigger") {
+ m_domainTrigger = Scan<bool>(value);
+ } else if (key == "texttype") {
+ //texttype = value; TODO not used
+ } else if (key == "source-path") {
+ m_filePathSource = value;
+ } else if (key == "target-path") {
+ m_filePathTarget = value;
+ } else {
+ StatelessFeatureFunction::OverrideParameter(key, value);
+ }
+}
+
void WordTranslationFeature::Load()
{
// load word list for restricted feature set
diff --git a/moses/FF/WordTranslationFeature.h b/moses/FF/WordTranslationFeature.h
index 1905f7b9f..d99afe715 100644
--- a/moses/FF/WordTranslationFeature.h
+++ b/moses/FF/WordTranslationFeature.h
@@ -52,6 +52,7 @@ public:
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
+ bool OverrideParameter(const std::string& key, const std::string& value);
};
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-19 01:21:03 +0300