diff options
author | Kenneth Heafield <github@kheafield.com> | 2016-08-18 00:44:24 +0300 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2016-08-18 00:44:24 +0300 |
commit | c7de0dc624f4d08337a6a7fc1f525c09f4d889c3 (patch) | |
tree | 7c56490604d3acddc0b82f666e795c98bfb998ec /moses | |
parent | f260e432f55da5b1bed2c0b7036a1c43bed7bd72 (diff) | |
parent | 8bac020955a4083257104ebf2e4d61d483c29b36 (diff) |
Merge branch 'master' of github.com:moses-smt/mosesdecoder
Diffstat (limited to 'moses')
111 files changed, 3945 insertions, 637 deletions
diff --git a/moses/ChartCell.cpp b/moses/ChartCell.cpp index 39e2fff33..780c9d93f 100644 --- a/moses/ChartCell.cpp +++ b/moses/ChartCell.cpp @@ -36,7 +36,6 @@ using namespace std; namespace Moses { -extern bool g_mosesDebug; ChartCellBase::ChartCellBase(size_t startPos, size_t endPos) : m_coverage(startPos, endPos), diff --git a/moses/ChartManager.cpp b/moses/ChartManager.cpp index ad970d2e6..b7920b614 100644 --- a/moses/ChartManager.cpp +++ b/moses/ChartManager.cpp @@ -41,8 +41,6 @@ using namespace std; namespace Moses { -extern bool g_mosesDebug; - /* constructor. Initialize everything prior to decoding a particular sentence. * \param source the sentence to be decoded * \param system which particular set of models to use. diff --git a/moses/ExportInterface.cpp b/moses/ExportInterface.cpp index 0a15ee669..10814864b 100644 --- a/moses/ExportInterface.cpp +++ b/moses/ExportInterface.cpp @@ -71,6 +71,8 @@ using namespace Moses; namespace Moses { +//extern size_t g_numHypos; + void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream) { outputSearchGraphStream.setf(std::ios::fixed); @@ -294,9 +296,11 @@ batch_run() pool.Stop(true); //flush remaining jobs #endif +// cerr << "g_numHypos=" << Moses::g_numHypos << endl; + FeatureFunction::Destroy(); - IFVERBOSE(1) util::PrintUsage(std::cerr); + IFVERBOSE(0) util::PrintUsage(std::cerr); #ifndef EXIT_RETURN //This avoids that destructors are called (it can take a long time) diff --git a/moses/FF/CorrectionPattern.cpp b/moses/FF/CorrectionPattern.cpp new file mode 100644 index 000000000..9770f7d66 --- /dev/null +++ b/moses/FF/CorrectionPattern.cpp @@ -0,0 +1,354 @@ +#include <sstream> +#include "CorrectionPattern.h" +#include "moses/Phrase.h" +#include "moses/TargetPhrase.h" +#include "moses/InputPath.h" +#include "moses/Hypothesis.h" +#include "moses/ChartHypothesis.h" +#include "moses/ScoreComponentCollection.h" +#include "moses/TranslationOption.h" +#include "util/string_piece_hash.hh" +#include "util/exception.hh" + +#include <functional> +#include <algorithm> + +#include <boost/foreach.hpp> +#include <boost/algorithm/string.hpp> + +#include "Diffs.h" + +namespace Moses +{ + +using namespace std; + +std::string MakePair(const std::string &s1, const std::string &s2, bool general) +{ + std::vector<std::string> sourceList; + std::vector<std::string> targetList; + + if(general) { + Diffs diffs = CreateDiff(s1, s2); + + size_t i = 0, j = 0; + char lastType = 'm'; + + std::string source, target; + std::string match; + + int count = 1; + + BOOST_FOREACH(Diff type, diffs) { + if(type == 'm') { + if(lastType != 'm') { + sourceList.push_back(source); + targetList.push_back(target); + } + source.clear(); + target.clear(); + + if(s1[i] == '+') { + if(match.size() >= 3) { + sourceList.push_back("(\\w{3,})·"); + std::string temp = "1"; + sprintf((char*)temp.c_str(), "%d", count); + targetList.push_back("\\" + temp + "·"); + count++; + } else { + sourceList.push_back(match + "·"); + targetList.push_back(match + "·"); + } + match.clear(); + } else + match.push_back(s1[i]); + + i++; + j++; + } else if(type == 'd') { + if(s1[i] == '+') + source += "·"; + else + source.push_back(s1[i]); + i++; + } else if(type == 'i') { + if(s2[j] == '+') + target += "·"; + else + target.push_back(s2[j]); + j++; + } + if(type != 'm' && !match.empty()) { + if(match.size() >= 3) { + sourceList.push_back("(\\w{3,})"); + std::string temp = "1"; + sprintf((char*)temp.c_str(), "%d", count); + targetList.push_back("\\" + temp); + count++; + } else { + sourceList.push_back(match); + targetList.push_back(match); + } + + match.clear(); + } + + lastType = type; + } + if(lastType != 'm') { + sourceList.push_back(source); + targetList.push_back(target); + } + + if(!match.empty()) { + if(match.size() >= 3) { + sourceList.push_back("(\\w{3,})"); + std::string temp = "1"; + sprintf((char*)temp.c_str(), "%d", count); + targetList.push_back("\\"+ temp); + count++; + } else { + sourceList.push_back(match); + targetList.push_back(match); + } + } + match.clear(); + } else { + std::string cs1 = s1; + std::string cs2 = s2; + boost::replace_all(cs1, "+", "·"); + boost::replace_all(cs2, "+", "·"); + + sourceList.push_back(cs1); + targetList.push_back(cs2); + } + + std::stringstream out; + out << "sub(«"; + out << boost::join(sourceList, ""); + out << "»,«"; + out << boost::join(targetList, ""); + out << "»)"; + + return out.str(); +} + +std::string CorrectionPattern::CreateSinglePattern(const Tokens &s1, const Tokens &s2) const +{ + std::stringstream out; + if(s1.empty()) { + out << "ins(«" << boost::join(s2, "·") << "»)"; + return out.str(); + } else if(s2.empty()) { + out << "del(«" << boost::join(s1, "·") << "»)"; + return out.str(); + } else { + Tokens::value_type v1 = boost::join(s1, "+"); + Tokens::value_type v2 = boost::join(s2, "+"); + out << MakePair(v1, v2, m_general); + return out.str(); + } +} + +std::vector<std::string> GetContext(size_t pos, + size_t len, + size_t window, + const InputType &input, + const InputPath &inputPath, + const std::vector<FactorType>& factorTypes, + bool isRight) +{ + + const Sentence& sentence = static_cast<const Sentence&>(input); + const Range& range = inputPath.GetWordsRange(); + + int leftPos = range.GetStartPos() + pos - len - 1; + int rightPos = range.GetStartPos() + pos; + + std::vector<std::string> contexts; + + for(int length = 1; length <= (int)window; ++length) { + std::vector<std::string> current; + if(!isRight) { + for(int i = 0; i < length; i++) { + if(leftPos - i >= 0) { + current.push_back(sentence.GetWord(leftPos - i).GetString(factorTypes, false)); + } else { + current.push_back("<s>"); + } + } + + if(current.back() == "<s>" && current.size() >= 2 && current[current.size()-2] == "<s>") + continue; + + std::reverse(current.begin(), current.end()); + contexts.push_back("left(«" + boost::join(current, "·") + "»)_"); + } + if(isRight) { + for(int i = 0; i < length; i++) { + if(rightPos + i < (int)sentence.GetSize()) { + current.push_back(sentence.GetWord(rightPos + i).GetString(factorTypes, false)); + } else { + current.push_back("</s>"); + } + } + + if(current.back() == "</s>" && current.size() >= 2 && current[current.size()-2] == "</s>") + continue; + + contexts.push_back("_right(«" + boost::join(current, "·") + "»)"); + } + } + return contexts; +} + +std::vector<std::string> +CorrectionPattern::CreatePattern(const Tokens &s1, + const Tokens &s2, + const InputType &input, + const InputPath &inputPath) const +{ + + Diffs diffs = CreateDiff(s1, s2); + size_t i = 0, j = 0; + char lastType = 'm'; + std::vector<std::string> patternList; + Tokens source, target; + BOOST_FOREACH(Diff type, diffs) { + if(type == 'm') { + if(lastType != 'm') { + std::string pattern = CreateSinglePattern(source, target); + patternList.push_back(pattern); + + if(m_context > 0) { + std::vector<std::string> leftContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, false); + std::vector<std::string> rightContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, true); + + BOOST_FOREACH(std::string left, leftContexts) + patternList.push_back(left + pattern); + + BOOST_FOREACH(std::string right, rightContexts) + patternList.push_back(pattern + right); + + BOOST_FOREACH(std::string left, leftContexts) + BOOST_FOREACH(std::string right, rightContexts) + patternList.push_back(left + pattern + right); + } + } + source.clear(); + target.clear(); + if(s1[i] != s2[j]) { + source.push_back(s1[i]); + target.push_back(s2[j]); + } + i++; + j++; + } else if(type == 'd') { + source.push_back(s1[i]); + i++; + } else if(type == 'i') { + target.push_back(s2[j]); + j++; + } + lastType = type; + } + if(lastType != 'm') { + std::string pattern = CreateSinglePattern(source, target); + patternList.push_back(pattern); + + if(m_context > 0) { + std::vector<std::string> leftContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, false); + std::vector<std::string> rightContexts = GetContext(i, source.size(), m_context, input, inputPath, m_contextFactors, true); + + BOOST_FOREACH(std::string left, leftContexts) + patternList.push_back(left + pattern); + + BOOST_FOREACH(std::string right, rightContexts) + patternList.push_back(pattern + right); + + BOOST_FOREACH(std::string left, leftContexts) + BOOST_FOREACH(std::string right, rightContexts) + patternList.push_back(left + pattern + right); + } + } + + return patternList; +} + +CorrectionPattern::CorrectionPattern(const std::string &line) + : StatelessFeatureFunction(0, line), m_factors(1, 0), m_general(false), + m_context(0), m_contextFactors(1, 0) +{ + std::cerr << "Initializing correction pattern feature.." << std::endl; + ReadParameters(); +} + +void CorrectionPattern::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "factor") { + m_factors = std::vector<FactorType>(1, Scan<FactorType>(value)); + } else if (key == "context-factor") { + m_contextFactors = std::vector<FactorType>(1, Scan<FactorType>(value)); + } else if (key == "general") { + m_general = Scan<bool>(value); + } else if (key == "context") { + m_context = Scan<size_t>(value); + } else { + StatelessFeatureFunction::SetParameter(key, value); + } +} + +void CorrectionPattern::EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedFutureScore) const +{ + ComputeFeatures(input, inputPath, targetPhrase, &scoreBreakdown); +} + +void CorrectionPattern::ComputeFeatures( + const InputType &input, + const InputPath &inputPath, + const TargetPhrase& target, + ScoreComponentCollection* accumulator) const +{ + const Phrase &source = inputPath.GetPhrase(); + + std::vector<std::string> sourceTokens; + for(size_t i = 0; i < source.GetSize(); ++i) + sourceTokens.push_back(source.GetWord(i).GetString(m_factors, false)); + + std::vector<std::string> targetTokens; + for(size_t i = 0; i < target.GetSize(); ++i) + targetTokens.push_back(target.GetWord(i).GetString(m_factors, false)); + + std::vector<std::string> patternList = CreatePattern(sourceTokens, targetTokens, input, inputPath); + for(size_t i = 0; i < patternList.size(); ++i) + accumulator->PlusEquals(this, patternList[i], 1); + + /* + BOOST_FOREACH(std::string w, sourceTokens) + std::cerr << w << " "; + std::cerr << std::endl; + BOOST_FOREACH(std::string w, targetTokens) + std::cerr << w << " "; + std::cerr << std::endl; + BOOST_FOREACH(std::string w, patternList) + std::cerr << w << " "; + std::cerr << std::endl << std::endl; + */ +} + +bool CorrectionPattern::IsUseable(const FactorMask &mask) const +{ + bool ret = true; + for(size_t i = 0; i < m_factors.size(); ++i) + ret = ret && mask[m_factors[i]]; + for(size_t i = 0; i < m_contextFactors.size(); ++i) + ret = ret && mask[m_contextFactors[i]]; + return ret; +} + +} diff --git a/moses/FF/CorrectionPattern.h b/moses/FF/CorrectionPattern.h new file mode 100644 index 000000000..516a56ce2 --- /dev/null +++ b/moses/FF/CorrectionPattern.h @@ -0,0 +1,73 @@ +#ifndef moses_CorrectionPattern_h +#define moses_CorrectionPattern_h + +#include <string> +#include <boost/unordered_set.hpp> + +#include "StatelessFeatureFunction.h" +#include "moses/FactorCollection.h" +#include "moses/AlignmentInfo.h" + +namespace Moses +{ + +typedef std::vector<std::string> Tokens; + +/** Sets the features for length of source phrase, target phrase, both. + */ +class CorrectionPattern : public StatelessFeatureFunction +{ +private: + std::vector<FactorType> m_factors; + bool m_general; + size_t m_context; + std::vector<FactorType> m_contextFactors; + +public: + CorrectionPattern(const std::string &line); + + bool IsUseable(const FactorMask &mask) const; + + void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedFutureScore) const + {} + + virtual void EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedFutureScore = NULL) const; + + void EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const + {} + + void EvaluateWhenApplied(const Hypothesis& hypo, + ScoreComponentCollection* accumulator) const + {} + void EvaluateWhenApplied(const ChartHypothesis &hypo, + ScoreComponentCollection* accumulator) const + {} + + void ComputeFeatures(const InputType &input, + const InputPath &inputPath, + const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const; + + void SetParameter(const std::string& key, const std::string& value); + + std::vector<std::string> CreatePattern(const Tokens &s1, + const Tokens &s2, + const InputType &input, + const InputPath &inputPath) const; + + std::string CreateSinglePattern(const Tokens &s1, const Tokens &s2) const; + +}; + +} + +#endif // moses_CorrectionPattern_h diff --git a/moses/FF/Diffs.h b/moses/FF/Diffs.h new file mode 100644 index 000000000..8935d1fb9 --- /dev/null +++ b/moses/FF/Diffs.h @@ -0,0 +1,150 @@ +#ifndef moses_Diffs_h +#define moses_Diffs_h + +#include <cmath> + +namespace Moses +{ + +typedef char Diff; +typedef std::vector<Diff> Diffs; + +template <class Sequence, class Pred> +void CreateDiffRec(size_t** c, + const Sequence &s1, + const Sequence &s2, + size_t start, + size_t i, + size_t j, + Diffs& diffs, + Pred pred) +{ + if(i > 0 && j > 0 && pred(s1[i - 1 + start], s2[j - 1 + start])) { + CreateDiffRec(c, s1, s2, start, i - 1, j - 1, diffs, pred); + diffs.push_back(Diff('m')); + } else if(j > 0 && (i == 0 || c[i][j-1] >= c[i-1][j])) { + CreateDiffRec(c, s1, s2, start, i, j-1, diffs, pred); + diffs.push_back(Diff('i')); + } else if(i > 0 && (j == 0 || c[i][j-1] < c[i-1][j])) { + CreateDiffRec(c, s1, s2, start, i-1, j, diffs, pred); + diffs.push_back(Diff('d')); + } +} + +template <class Sequence, class Pred> +Diffs CreateDiff(const Sequence& s1, + const Sequence& s2, + Pred pred) +{ + + Diffs diffs; + + size_t n = s2.size(); + + int start = 0; + int m_end = s1.size() - 1; + int n_end = s2.size() - 1; + + while(start <= m_end && start <= n_end && pred(s1[start], s2[start])) { + diffs.push_back(Diff('m')); + start++; + } + while(start <= m_end && start <= n_end && pred(s1[m_end], s2[n_end])) { + m_end--; + n_end--; + } + + size_t m_new = m_end - start + 1; + size_t n_new = n_end - start + 1; + + size_t** c = new size_t*[m_new + 1]; + for(size_t i = 0; i <= m_new; ++i) { + c[i] = new size_t[n_new + 1]; + c[i][0] = 0; + } + for(size_t j = 0; j <= n_new; ++j) + c[0][j] = 0; + for(size_t i = 1; i <= m_new; ++i) + for(size_t j = 1; j <= n_new; ++j) + if(pred(s1[i - 1 + start], s2[j - 1 + start])) + c[i][j] = c[i-1][j-1] + 1; + else + c[i][j] = c[i][j-1] > c[i-1][j] ? c[i][j-1] : c[i-1][j]; + + CreateDiffRec(c, s1, s2, start, m_new, n_new, diffs, pred); + + for(size_t i = 0; i <= m_new; ++i) + delete[] c[i]; + delete[] c; + + for (size_t i = n_end + 1; i < n; ++i) + diffs.push_back(Diff('m')); + + return diffs; +} + +template <class Sequence> +Diffs CreateDiff(const Sequence& s1, const Sequence& s2) +{ + return CreateDiff(s1, s2, std::equal_to<typename Sequence::value_type>()); +} + +template <class Sequence, class Sig, class Stats> +void AddStats(const Sequence& s1, const Sequence& s2, const Sig& sig, Stats& stats) +{ + if(sig.size() != stats.size()) + throw "Signature size differs from score array size."; + + size_t m = 0, d = 0, i = 0, s = 0; + Diffs diff = CreateDiff(s1, s2); + + for(int j = 0; j < (int)diff.size(); ++j) { + if(diff[j] == 'm') + m++; + else if(diff[j] == 'd') { + d++; + int k = 0; + while(j - k >= 0 && j + 1 + k < (int)diff.size() && + diff[j - k] == 'd' && diff[j + 1 + k] == 'i') { + d--; + s++; + k++; + } + j += k; + } else if(diff[j] == 'i') + i++; + } + + for(size_t j = 0; j < sig.size(); ++j) { + switch (sig[j]) { + case 'l': + stats[j] += d + i + s; + break; + case 'm': + stats[j] += m; + break; + case 'd': + stats[j] += d; + break; + case 'i': + stats[j] += i; + break; + case 's': + stats[j] += s; + break; + case 'r': + float macc = 1; + if (d + i + s + m) + macc = 1.0 - (float)(d + i + s)/(float)(d + i + s + m); + if(macc > 0) + stats[j] += log(macc); + else + stats[j] += log(1.0/(float)(d + i + s + m + 1)); + break; + } + } +} + +} + +#endif diff --git a/moses/FF/Dsg-Feature/Desegmenter.cpp b/moses/FF/Dsg-Feature/Desegmenter.cpp new file mode 100644 index 000000000..677de6e6e --- /dev/null +++ b/moses/FF/Dsg-Feature/Desegmenter.cpp @@ -0,0 +1,87 @@ +#include <fstream> +#include <iostream> +#include<string> +#include<sstream> +#include<vector> +#include<map> +#include "Desegmenter.h" +#include <boost/algorithm/string/replace.hpp> + +using namespace std; + +namespace Moses +{ +void Desegmenter::Load(const string filename) +{ + + std::ifstream myFile(filename.c_str() ); + if (myFile.is_open()) { + cerr << "Desegmentation File open successful." << endl; + string line; + while (getline(myFile, line)) { + stringstream ss(line); + string token; + vector<string> myline; + while (getline(ss, token, '\t')) { + myline.push_back(token); + } + mmDesegTable.insert(pair<string, string>(myline[2], myline[1] )); + } + myFile.close(); + } else + cerr << "open() failed: check if Desegmentation file is in right folder" << endl; +} + + +vector<string> Desegmenter::Search(string myKey) +{ + multimap<string, string>::const_iterator mmiPairFound = mmDesegTable.find(myKey); + vector<string> result; + if (mmiPairFound != mmDesegTable.end()) { + size_t nNumPairsInMap = mmDesegTable.count(myKey); + for (size_t nValuesCounter = 0; nValuesCounter < nNumPairsInMap; ++nValuesCounter) { + if (mmiPairFound != mmDesegTable.end()) { + result.push_back(mmiPairFound->second); + } + ++mmiPairFound; + } + return result; + } else { + string rule_deseg ; + rule_deseg = ApplyRules(myKey); + result.push_back(rule_deseg); + return result; + } +} + + +string Desegmenter::ApplyRules(string & segToken) +{ + + string desegToken=segToken; + if (!simple) { + boost::replace_all(desegToken, "l+ All", "ll"); + boost::replace_all(desegToken, "l+ Al", "ll"); + boost::replace_all(desegToken, "y+ y ", "y"); + boost::replace_all(desegToken, "p+ ", "t"); + boost::replace_all(desegToken, "' +", "}"); + boost::replace_all(desegToken, "y +", "A"); + boost::replace_all(desegToken, "n +n", "n"); + boost::replace_all(desegToken, "mn +m", "mm"); + boost::replace_all(desegToken, "En +m", "Em"); + boost::replace_all(desegToken, "An +lA", "Em"); + boost::replace_all(desegToken, "-LRB-", "("); + boost::replace_all(desegToken, "-RRB-", ")"); + } + + boost::replace_all(desegToken, "+ +", ""); + boost::replace_all(desegToken, "+ ", ""); + boost::replace_all(desegToken, " +", ""); + + return desegToken; +} + +Desegmenter::~Desegmenter() +{} + +} diff --git a/moses/FF/Dsg-Feature/Desegmenter.h b/moses/FF/Dsg-Feature/Desegmenter.h new file mode 100644 index 000000000..21da78d2e --- /dev/null +++ b/moses/FF/Dsg-Feature/Desegmenter.h @@ -0,0 +1,33 @@ +#pragma once + +#include<string> +#include<map> + + +using namespace std; + +namespace Moses +{ +class Desegmenter +{ +private: + std::multimap<string, string> mmDesegTable; + std::string filename; + bool simple; + void Load(const string filename); + +public: + Desegmenter(const std::string& file, const bool scheme) { + filename = file; + simple=scheme; + Load(filename); + } + string getFileName() { + return filename; + } + + vector<string> Search(string myKey); + string ApplyRules(string &); + ~Desegmenter(); +}; +} diff --git a/moses/FF/Dsg-Feature/DsgModel.cpp b/moses/FF/Dsg-Feature/DsgModel.cpp new file mode 100644 index 000000000..0bd25a50b --- /dev/null +++ b/moses/FF/Dsg-Feature/DsgModel.cpp @@ -0,0 +1,156 @@ +#include <fstream> +#include "DsgModel.h" +#include "dsgHyp.h" +#include "moses/Util.h" +#include "util/exception.hh" + +using namespace std; +using namespace lm::ngram; + +namespace Moses +{ + +DesegModel::DesegModel(const std::string &line) + :StatefulFeatureFunction(5, line ) +{ + tFactor = 0; + order=5; + numFeatures = 5; + optimistic = 1; + ReadParameters(); +} + +DesegModel::~DesegModel() +{ + delete DSGM; +} + +void DesegModel :: readLanguageModel(const char *lmFile) +{ + DSGM = ConstructDsgLM(m_lmPath.c_str()); + State startState = DSGM->NullContextState(); + desegT=new Desegmenter(m_desegPath,m_simple);// Desegmentation Table +} + + +void DesegModel::Load(AllOptions::ptr const& opts) +{ + m_options = opts; + readLanguageModel(m_lmPath.c_str()); +} + + + +void DesegModel:: EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedScores) const +{ + + dsgHypothesis obj; + vector <string> myTargetPhrase; + vector<float> scores; + vector<string> targ_phrase; //stores the segmented tokens in the target phrase + const AlignmentInfo &align = targetPhrase.GetAlignTerm(); + + for (int i = 0; i < targetPhrase.GetSize(); i++) { + targ_phrase.push_back(targetPhrase.GetWord(i).GetFactor(tFactor)->GetString().as_string()); + } + + obj.setState(DSGM->NullContextState()); + obj.setPhrases(targ_phrase); + obj.calculateDsgProbinIsol(*DSGM,*desegT,align); + obj.populateScores(scores,numFeatures); + estimatedScores.PlusEquals(this, scores); +} + + +FFState* DesegModel::EvaluateWhenApplied( + const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase &target = cur_hypo.GetCurrTargetPhrase(); + const Range &src_rng =cur_hypo.GetCurrSourceWordsRange(); + const AlignmentInfo &align = cur_hypo.GetCurrTargetPhrase().GetAlignTerm(); + size_t sourceOffset = src_rng.GetStartPos(); + + dsgHypothesis obj; + vector<float> scores; + vector<string> targ_phrase; //stores the segmented tokens in the target phrase + bool isCompleted; + + isCompleted=cur_hypo.IsSourceCompleted(); + for (int i = 0; i < cur_hypo.GetCurrTargetLength(); i++) { + targ_phrase.push_back(target.GetWord(i).GetFactor(tFactor)->GetString().as_string()); + } + + obj.setState(prev_state); + obj.setPhrases( targ_phrase ); + obj.calculateDsgProb(*DSGM,*desegT,isCompleted,align, sourceOffset, optimistic); + obj.populateScores(scores,numFeatures); + accumulator->PlusEquals(this, scores); + return obj.saveState(); + +} + +FFState* DesegModel::EvaluateWhenApplied( + const ChartHypothesis& /* cur_hypo */, + int /* featureID - used to index the state in the previous hypotheses */, + ScoreComponentCollection* accumulator) const +{ + UTIL_THROW2("Chart decoding not support by UTIL_THROW2"); +} + +const FFState* DesegModel::EmptyHypothesisState(const InputType &input) const +{ + VERBOSE(3,"DesegModel::EmptyHypothesisState()" << endl); + State startState = DSGM->BeginSentenceState(); + dsgState ss= dsgState(startState); + return new dsgState(ss); +} + +std::string DesegModel::GetScoreProducerWeightShortName(unsigned idx) const +{ + return "dsg"; +} + + +void DesegModel::SetParameter(const std::string& key, const std::string& value) +{ + + if (key == "path") { + m_lmPath = value; + } else if (key == "contiguity-features") { + if(value == "no") + numFeatures = 1; + else + numFeatures = 5; + } else if (key == "output-factor") { + tFactor = Scan<int>(value); + } else if (key == "optimistic") { + if (value == "n") + optimistic = 0; + else + optimistic = 1; + } else if (key == "deseg-path") { + m_desegPath = Scan<int>(value); + } else if (key == "deseg-scheme") { + if(value == "s") + m_simple = 1; + else + m_simple = 0; + } else if (key == "order") { + order = Scan<int>(value); + } else { + StatefulFeatureFunction::SetParameter(key, value); + } +} + +bool DesegModel::IsUseable(const FactorMask &mask) const +{ + bool ret = mask[0]; + return ret; +} + +} // namespace diff --git a/moses/FF/Dsg-Feature/DsgModel.h b/moses/FF/Dsg-Feature/DsgModel.h new file mode 100644 index 000000000..7c19f0a63 --- /dev/null +++ b/moses/FF/Dsg-Feature/DsgModel.h @@ -0,0 +1,64 @@ +#pragma once + +#include <string> +#include <map> +#include <vector> +#include "moses/FF/StatefulFeatureFunction.h" +#include "moses/Manager.h" +#include "moses/FF/Dsg-Feature/dsgHyp.h" +#include "moses/FF/Dsg-Feature/Desegmenter.h" +#include "KenDsg.h" + + +namespace Moses +{ + +class DesegModel : public StatefulFeatureFunction +{ +public: + + DsgLM * DSGM; + Desegmenter* desegT; + int tFactor;// Target Factor ... + int order; + int numFeatures; // Number of features used an be 1 (unsegmented LM)or 5 (with 3 contiguity features and 1 UnsegWP) + bool optimistic; + + DesegModel(const std::string &line); + ~DesegModel(); + + void readLanguageModel(const char *); + void Load(AllOptions::ptr const& opts); + + FFState* EvaluateWhenApplied( + const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual FFState* EvaluateWhenApplied( + const ChartHypothesis& /* cur_hypo */, + int /* featureID - used to index the state in the previous hypotheses */, + ScoreComponentCollection* accumulator) const; + + void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedScores) const; + + virtual const FFState* EmptyHypothesisState(const InputType &input) const; + + virtual std::string GetScoreProducerWeightShortName(unsigned idx=0) const; + + void SetParameter(const std::string& key, const std::string& value); + + bool IsUseable(const FactorMask &mask) const; + +protected: + typedef std::vector<float> Scores; + std::string m_lmPath; + std::string m_desegPath; + bool m_simple; //desegmentation scheme; if 1 then use simple, else use rule and backoff to simple +}; + + +} diff --git a/moses/FF/Dsg-Feature/KenDsg.cpp b/moses/FF/Dsg-Feature/KenDsg.cpp new file mode 100644 index 000000000..b9fd92aef --- /dev/null +++ b/moses/FF/Dsg-Feature/KenDsg.cpp @@ -0,0 +1,34 @@ +#include "KenDsg.h" + +namespace Moses +{ + +DsgLM* ConstructDsgLM(const char *file) +{ + lm::ngram::ModelType model_type; + lm::ngram::Config config; + if (lm::ngram::RecognizeBinary(file, model_type)) { + switch(model_type) { + case lm::ngram::PROBING: + return new KenDsg<lm::ngram::ProbingModel>(file, config); + case lm::ngram::REST_PROBING: + return new KenDsg<lm::ngram::RestProbingModel>(file, config); + case lm::ngram::TRIE: + return new KenDsg<lm::ngram::TrieModel>(file, config); + case lm::ngram::QUANT_TRIE: + return new KenDsg<lm::ngram::QuantTrieModel>(file, config); + case lm::ngram::ARRAY_TRIE: + return new KenDsg<lm::ngram::ArrayTrieModel>(file, config); + case lm::ngram::QUANT_ARRAY_TRIE: + return new KenDsg<lm::ngram::QuantArrayTrieModel>(file, config); + default: + UTIL_THROW2("Unrecognized kenlm model type " << model_type); + } + } else { + return new KenDsg<lm::ngram::ProbingModel>(file, config); + } +} + +} // namespace + + diff --git a/moses/FF/Dsg-Feature/KenDsg.h b/moses/FF/Dsg-Feature/KenDsg.h new file mode 100644 index 000000000..3fc07003f --- /dev/null +++ b/moses/FF/Dsg-Feature/KenDsg.h @@ -0,0 +1,60 @@ +#pragma once + +#include <string> +#include "lm/model.hh" + +namespace Moses +{ + +class KenDsgBase +{ +public: + virtual ~KenDsgBase() {} + + virtual float Score(const lm::ngram::State&, StringPiece, + lm::ngram::State&) const = 0; + + virtual const lm::ngram::State &BeginSentenceState() const = 0; + + virtual const lm::ngram::State &NullContextState() const = 0; + + virtual float ScoreEndSentence(const lm::ngram::State &in_state, lm::ngram::State &out_state) const = 0; +}; + +template <class KenModel> +class KenDsg : public KenDsgBase +{ +public: + KenDsg(const char *file, const lm::ngram::Config &config) + : m_kenlm(file, config) {} + + float Score(const lm::ngram::State &in_state, + StringPiece word, + lm::ngram::State &out_state) const { + return m_kenlm.Score(in_state, m_kenlm.GetVocabulary().Index(word), + out_state); + } + + const lm::ngram::State &BeginSentenceState() const { + return m_kenlm.BeginSentenceState(); + } + + const lm::ngram::State &NullContextState() const { + return m_kenlm.NullContextState(); + } + + float ScoreEndSentence(const lm::ngram::State &in_state, lm::ngram::State &out_state) const { + return m_kenlm.Score(in_state, m_kenlm.GetVocabulary().EndSentence(), out_state); + } + + +private: + KenModel m_kenlm; +}; + +typedef KenDsgBase DsgLM; + +DsgLM* ConstructDsgLM(const char *file); + + +} // namespace diff --git a/moses/FF/Dsg-Feature/dsgHyp.cpp b/moses/FF/Dsg-Feature/dsgHyp.cpp new file mode 100644 index 000000000..d33262d11 --- /dev/null +++ b/moses/FF/Dsg-Feature/dsgHyp.cpp @@ -0,0 +1,391 @@ +#include "dsgHyp.h" +#include <sstream> +#include <boost/algorithm/string.hpp> +#include <algorithm> +#include <cstdlib> +#include <math.h> +#include <map> + + +using namespace std; +using namespace lm::ngram; + +namespace Moses +{ +dsgState::dsgState(const State & val) +{ + lmState = val; +} + +void dsgState::saveState( std::vector<std::string> danglingTok, std::vector<int> srcSpans,float deltaValue) +{ + buffer = danglingTok; + span=srcSpans; + delta=deltaValue; +} + + +size_t dsgState::hash() const +{ + + size_t ret = 0; + boost::hash_combine(ret, lmState); + + /*size_t ret = delta; + boost::hash_combine(ret, buffer); + boost::hash_combine(ret, span); + boost::hash_combine(ret, lmState.length); + return ret;*/ +} + +bool dsgState::operator==(const FFState& otherBase) const //CHECK +{ + const dsgState &other = static_cast<const dsgState&>(otherBase); + + if (lmState < other.lmState) return false; + if (lmState == other.lmState) return true; + return false; +} + +// ---------------------------------------- + +std::string dsgState :: getName() const +{ + return "done"; +} + +dsgHypothesis :: dsgHypothesis() +{ + lmProb = 0; + discontig0 = 0; + discontig1 = 0; + discontig2 = 0; + UnsegWP = 0; + m_buffer.clear();//=""; +} + +void dsgHypothesis :: setState(const FFState* prev_state) +{ + if(prev_state != NULL) { + m_buffer = static_cast <const dsgState *> (prev_state)->getBuffer(); + m_span = static_cast <const dsgState *> (prev_state)->getSpan(); + lmState = static_cast <const dsgState *> (prev_state)->getLMState(); + delta = static_cast <const dsgState *> (prev_state)->getDelta(); //NEW + } +} + +dsgState * dsgHypothesis :: saveState() +{ + dsgState * statePtr = new dsgState(lmState); + statePtr->saveState(m_buffer, m_span, delta); + return statePtr; +} + +void dsgHypothesis :: populateScores(vector <float> & scores , const int numFeatures) +{ + scores.clear(); + scores.push_back(lmProb); + + if (numFeatures == 1) + return; + scores.push_back(discontig0); + scores.push_back(discontig1); + scores.push_back(discontig2); + scores.push_back(UnsegWP); +} + + + +bool dsgHypothesis::isPrefix(const std::string &tok) +{ + if ((tok.at(tok.size() - 1) == '+' )&& (tok != "+")) { + return true; + } else { + return false; + }; +} + +bool dsgHypothesis::isSuffix(const std::string &tok) +{ + if ((tok.at(0) == '+' )&& (tok != "+")) { + return true; + } else { + return false; + }; +} + +bool dsgHypothesis::isStem(const std::string &tok) +{ + if ((tok.at(0) != '+') && (tok.at(tok.size() - 1) != '+')) { + return true; + } else { + return false; + }; +} + + + +/** + * chain stores segmented tokens that are in process of building a word + * The function checks if tok contributes to the word being formed in chain + * + */ +bool dsgHypothesis::isValidChain(const std::string &tok, std::vector<std::string> &chain) +{ + std::string last_tok; + if (chain.size() >= 1) { + last_tok = chain[chain.size() - 1]; + } else { + last_tok = "NULL"; + } + if(tok=="+") { + return false; + } + if (isPrefix(tok) && (chain.size() == 0 || isPrefix(last_tok))) { + return true; + } else if (isSuffix(tok) && (chain.size() != 0 && ( isStem(last_tok) || isPrefix(last_tok)))) { + return true; // allows one suffix ONLY + } + //else if (isSuffix(tok) && (chain.size() != 0 && ( isStem(last_tok) || isPrefix(last_tok) || isSuffix(last_tok) ))) { return true; } // allows multiple suffixes + else if (isStem(tok) && (chain.size() == 0 || isPrefix(last_tok))) { + return true; + } else { + return false; + } +} + +/** + * grouper function groups tokens that form a word together + */ +vector<string> dsgHypothesis::grouper(std::vector<std::string> &phr_vec,vector<vector<int> > &allchain_ids, int sourceOffset,const AlignmentInfo &align, bool isolation) +{ + + std::vector<std::string> chain; + std::vector<int> chain_ids; + std::vector<std::string> allchains; + chain_ids=m_span; + + if (!m_buffer.empty() && !isolation) { // if evaluate in isolation is called, then do not add buffer content + for (int i = 0; i < m_buffer.size(); i++) { // initialize chain with the content of the buffer + chain.push_back(m_buffer[i]); + } + } + + for (int i = 0; i < phr_vec.size(); i++) { + std::set<std::size_t> sourcePosSet = align.GetAlignmentsForTarget(i); + + if (isValidChain(phr_vec[i], chain)) { + chain.push_back(phr_vec[i]); + if (sourcePosSet.empty()==false) { + for (std::set<size_t>::iterator it(sourcePosSet.begin()); it != sourcePosSet.end(); it++) { + int cur=*it; + chain_ids.push_back(cur+sourceOffset); + } + } + } + + else if (chain.size() == 0) { // start of a suffix at hypothesis0 + allchains.push_back(phr_vec[i]); + allchain_ids.push_back(chain_ids); + chain_ids.clear();//={}; + } + + else { // tokens formed a complete word; add tokens segmented by space to allchains + std::string joined = boost::algorithm::join(chain, " "); + allchains.push_back(joined); + allchain_ids.push_back(chain_ids); + + chain.clear();// = {}; + chain_ids.clear();//={}; + + chain.push_back(phr_vec[i]); + if (sourcePosSet.empty()==false) { + for (std::set<size_t>::iterator it(sourcePosSet.begin()); it != sourcePosSet.end(); it++) { + int cur=*it; + chain_ids.push_back(cur+sourceOffset); + } + } + + } + + } + + if (!chain.empty()) { + std::string joined = boost::algorithm::join(chain, " "); + allchains.push_back(joined); + allchain_ids.push_back(chain_ids); + } + return allchains; +} + + + +void dsgHypothesis :: calculateDsgProbinIsol(DsgLM & ptrDsgLM, Desegmenter &desegT, const AlignmentInfo &align ) +{ + lmProb = 0; + State currState = lmState; + State temp; + string desegmented=""; + vector <string> words; + vector <string> currFVec; + + discontig0=0; + discontig1=0; + discontig2=0; + UnsegWP=0; + + currFVec = m_buffer; + currFVec.insert( currFVec.end(), m_curr_phr.begin(), m_curr_phr.end() ); + + int vecSize=currFVec.size(); + + // phrases with suffix-starts and prefix-end + if (currFVec.size()>0 && isPrefix (currFVec.back())) { + UnsegWP-=0.5; + } + if (currFVec.size()>0 && isSuffix (currFVec.front())) { + UnsegWP-=0.5; + } + + /* //Dropping prefix-end and suffix-start + while (currFVec.size()>0 && isPrefix (currFVec.back())){ + currFVec.pop_back(); //drop prefix appearing at end of phrase + } + + while (currFVec.size()>0 && isSuffix (currFVec.front())){ + currFVec.erase (currFVec.begin()); //drop suffix appearning at start of a phrase + } */ + + vector<vector<int> > chain_ids; + words = grouper(currFVec,chain_ids,0,align,1); + + for (int i = 0; i<words.size(); i++) { + UnsegWP+=1; + temp = currState; + if (words[i].find(" ")!=std::string::npos) { + desegmented=desegT.Search(words[i])[0]; + lmProb += ptrDsgLM.Score(temp,desegmented,currState); + } else { + boost::replace_all(words[i], "-LRB-", "("); + boost::replace_all(words[i], "-RRB-", ")"); + lmProb += ptrDsgLM.Score(temp,words[i],currState); + } + } + lmState = currState; +} + +void dsgHypothesis :: calculateDsgProb(DsgLM& ptrDsgLM, Desegmenter &desegT, bool isCompleted , const AlignmentInfo &align, int sourceOffset, bool optimistic) +{ + lmProb = 0; + discontig0=0; + discontig1=0; + discontig2=0; + UnsegWP=0; + + State currState = lmState; + State temp; + string desegmented=""; + vector <string> words; + vector <string> currFVec; + bool completePhraseSuffixEnd = false; + vector<vector<int> > all_chain_ids; + double pscore; + currFVec=m_curr_phr; + + // Check if the the phrase ends in a suffix, which means that it completes a full word;Make sure to change the isValidChain + if (isSuffix (currFVec.back()) && (currFVec.back()!="+")) { + completePhraseSuffixEnd=true; + } + + words = grouper(currFVec,all_chain_ids,sourceOffset,align,0); + + for (int i = 0; i < words.size(); i++) { + temp = currState; + + if (i==words.size()-1) { + if (completePhraseSuffixEnd) { //i.e if phrase ends with suffix, which marks an end of a word + m_buffer.clear();// =""; + m_span.clear();// ={}; + } else if (!isCompleted) { // not end of sentence( or final hypothesis), and probably the last token is not a complete word + m_buffer.clear(); + if (optimistic == 1) { + if ( isPrefix (currFVec.back())) { // this will delay scoring of prefix in prefix-ending phrases until the next hypothesis arrives + //pscore = ptrDsgLM.Score(temp,desegmented,currState); + lmProb -= delta; + delta = 0.0; + } + + else if (words[i].find(" ")!=std::string::npos) { + desegmented=desegT.Search(words[i])[0]; + pscore=ptrDsgLM.Score(temp,desegmented,currState); + lmProb = lmProb + pscore - delta; + delta=pscore; + currState=temp; + } else { + boost::replace_all(words[i], "-LRB-", "("); + boost::replace_all(words[i], "-RRB-", ")"); + pscore=ptrDsgLM.Score(temp,words[i],currState); + lmProb = lmProb + pscore - delta; + delta=pscore; + currState=temp; + } + } + + m_buffer.push_back(words.back()); + m_span=all_chain_ids.back(); + break; + } + } + + //temp = currState; + if (words[i].find(" ")!=std::string::npos) { + UnsegWP+=1; + desegmented=desegT.Search(words[i])[0]; + std::set<int> cur_chain_ids(all_chain_ids[i].begin(),all_chain_ids[i].end()); + if (cur_chain_ids.size()>1) { + vector<int> dsc; + for (std::set<int>::iterator it(cur_chain_ids.begin()), next(it); it != cur_chain_ids.end() && ++next != cur_chain_ids.end(); it = next) { + int cur=*it; + int mynext=*next; + if (std::abs(cur - mynext)>= 3) { + dsc.push_back(3); + } else if (std::abs(cur - mynext)== 2) { + dsc.push_back(2); + } else if (std::abs(cur - mynext)<= 1) { + dsc.push_back(1); + } + } + int mymax=*std::max_element(dsc.begin(),dsc.end()); + if (mymax==3) { + discontig2+=1; + } else if (mymax==2) { + discontig1+=1; + } else { + discontig0+=1; + } + } else { + discontig0 += 1; + } + + lmProb += ptrDsgLM.Score(temp,desegmented,currState); + } else { + UnsegWP+=1; + boost::replace_all(words[i], "-LRB-", "("); + boost::replace_all(words[i], "-RRB-", ")"); + lmProb += ptrDsgLM.Score(temp,words[i],currState); + } + } + + if (isCompleted) { + temp = currState; + lmProb = lmProb + ptrDsgLM.ScoreEndSentence(temp,currState) - delta; + } + lmState = currState; +} + + +void dsgHypothesis :: print() +{} + + +} // namespace diff --git a/moses/FF/Dsg-Feature/dsgHyp.h b/moses/FF/Dsg-Feature/dsgHyp.h new file mode 100644 index 000000000..d36ad0530 --- /dev/null +++ b/moses/FF/Dsg-Feature/dsgHyp.h @@ -0,0 +1,108 @@ +#pragma once + + +# include "moses/FF/FFState.h" +# include "moses/Manager.h" +# include <set> +# include <map> +# include <string> +# include <vector> +# include "moses/FF/Dsg-Feature/Desegmenter.h" +# include "KenDsg.h" + + +namespace Moses +{ + +class dsgState : public FFState +{ +public: + + dsgState(const lm::ngram::State & val); + virtual bool operator==(const FFState& other) const; + void saveState( std::vector<std::string> bufferVal,std::vector<int> spanVal, float deltaValue); + + std::vector<std::string> getBuffer() const { + return buffer; + } + + std::vector<int> getSpan() const { + return span; + } + + lm::ngram::State getLMState() const { + return lmState; + } + + float getDelta() const { + return delta; + } + + void setDelta(double val1 ) { + delta = val1; + } + + void print() const; + std::string getName() const; + + virtual size_t hash() const; + + +protected: + std::vector<std::string> buffer; + std::vector<int> span; + lm::ngram::State lmState; + double delta; //NEW +}; + + + +class dsgHypothesis +{ + +private: + std::vector<std::string> m_buffer;// maintains dangling affix from previous hypothesis + std::vector<int> m_span;// maintains source alignment for dangling affix from previous hypothesis + lm::ngram::State lmState; // KenLM's Model State ... + std::vector<std::string> m_curr_phr; //phrase from current hypothesis + double delta; //NEW + + double lmProb; + int discontig0; + int discontig1; + int discontig2; + double UnsegWP; //Word Penalty score based on count of words + +public: + + dsgHypothesis(); + ~dsgHypothesis() {}; + void calculateDsgProb(DsgLM& ptrDsgLM, Desegmenter &, bool isCompleted, const AlignmentInfo &align, int sourceOffset, bool optimistic); + void calculateDsgProbinIsol(DsgLM& ptrDsgLM, Desegmenter &, const AlignmentInfo &align); + + void setPhrases(std::vector<std::string> & val1 ) { + m_curr_phr = val1; + } + + void setDelta(double val1 ) { + delta = val1; + } + + void setState(const FFState* prev_state); + dsgState * saveState(); + void print(); + void populateScores(std::vector <float> & scores , const int numFeatures); + void setState(const lm::ngram::State & val) { + lmState = val; + } + + bool isPrefix(const std::string &); + bool isSuffix(const std::string &); + bool isStem(const std::string &); + bool isValidChain(const std::string &, std::vector<std::string> &chain); + vector<string> grouper(std::vector<std::string> &,std::vector<std::vector<int> > &,int,const AlignmentInfo &align,bool); + +}; +} // namespace + + diff --git a/moses/FF/EditOps.cpp b/moses/FF/EditOps.cpp new file mode 100644 index 000000000..fa66acf1c --- /dev/null +++ b/moses/FF/EditOps.cpp @@ -0,0 +1,119 @@ +#include <sstream> +#include "EditOps.h" +#include "moses/Phrase.h" +#include "moses/TargetPhrase.h" +#include "moses/Hypothesis.h" +#include "moses/ChartHypothesis.h" +#include "moses/ScoreComponentCollection.h" +#include "moses/TranslationOption.h" +#include "util/string_piece_hash.hh" +#include "util/exception.hh" + +#include <functional> + +#include <boost/foreach.hpp> +#include <boost/algorithm/string.hpp> + +#include "Diffs.h" + +namespace Moses +{ + +using namespace std; + +std::string ParseScores(const std::string &line, const std::string& defaultScores) +{ + std::vector<std::string> toks = Tokenize(line); + UTIL_THROW_IF2(toks.empty(), "Empty line"); + + for (size_t i = 1; i < toks.size(); ++i) { + std::vector<std::string> args = TokenizeFirstOnly(toks[i], "="); + UTIL_THROW_IF2(args.size() != 2, + "Incorrect format for feature function arg: " << toks[i]); + + if (args[0] == "scores") { + return args[1]; + } + } + return defaultScores; +} + +EditOps::EditOps(const std::string &line) + : StatelessFeatureFunction(ParseScores(line, "dis").size(), line) + , m_factorType(0), m_chars(false), m_scores(ParseScores(line, "dis")) +{ + std::cerr << "Initializing EditOps feature.." << std::endl; + ReadParameters(); +} + +void EditOps::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "factor") { + m_factorType = Scan<FactorType>(value); + } else if (key == "chars") { + m_chars = Scan<bool>(value); + } else if (key == "scores") { + m_scores = value; + } else { + StatelessFeatureFunction::SetParameter(key, value); + } +} + +void EditOps::Load() +{ } + +void EditOps::EvaluateInIsolation(const Phrase &source + , const TargetPhrase &target + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedFutureScore) const +{ + ComputeFeatures(source, target, &scoreBreakdown); +} + +void EditOps::ComputeFeatures( + const Phrase &source, + const TargetPhrase& target, + ScoreComponentCollection* accumulator) const +{ + std::vector<float> ops(GetNumScoreComponents(), 0); + + if(m_chars) { + std::vector<FactorType> factors; + factors.push_back(m_factorType); + + std::string sourceStr = source.GetStringRep(factors); + std::string targetStr = target.GetStringRep(factors); + + AddStats(sourceStr, targetStr, m_scores, ops); + } else { + std::vector<std::string> sourceTokens; + //std::cerr << "Ed src: "; + for(size_t i = 0; i < source.GetSize(); ++i) { + if(!source.GetWord(i).IsNonTerminal()) + sourceTokens.push_back(source.GetWord(i).GetFactor(m_factorType)->GetString().as_string()); + //std::cerr << sourceTokens.back() << " "; + } + //std::cerr << std::endl; + + std::vector<std::string> targetTokens; + //std::cerr << "Ed trg: "; + for(size_t i = 0; i < target.GetSize(); ++i) { + if(!target.GetWord(i).IsNonTerminal()) + targetTokens.push_back(target.GetWord(i).GetFactor(m_factorType)->GetString().as_string()); + //std::cerr << targetTokens.back() << " "; + } + //std::cerr << std::endl; + + AddStats(sourceTokens, targetTokens, m_scores, ops); + } + + accumulator->PlusEquals(this, ops); +} + +bool EditOps::IsUseable(const FactorMask &mask) const +{ + bool ret = mask[m_factorType]; + return ret; +} + +} diff --git a/moses/FF/EditOps.h b/moses/FF/EditOps.h new file mode 100644 index 000000000..e7e7dd315 --- /dev/null +++ b/moses/FF/EditOps.h @@ -0,0 +1,64 @@ +#ifndef moses_EditOps_h +#define moses_EditOps_h + +#include <string> +#include <boost/unordered_set.hpp> + +#include "StatelessFeatureFunction.h" +#include "moses/FactorCollection.h" +#include "moses/AlignmentInfo.h" + +namespace Moses +{ + +typedef std::vector<std::string> Tokens; + +/** Calculates string edit operations that transform source phrase into target + * phrase using the LCS algorithm. Potentially usefule for monolingual tasks + * like paraphrasing, summarization, correction. + */ +class EditOps : public StatelessFeatureFunction +{ +private: + FactorType m_factorType; + bool m_chars; + std::string m_scores; + +public: + EditOps(const std::string &line); + + bool IsUseable(const FactorMask &mask) const; + + void Load(); + + virtual void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedFutureScore) const; + + void EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedFutureScore = NULL) const + {} + void EvaluateWhenApplied(const Hypothesis& hypo, + ScoreComponentCollection* accumulator) const + {} + void EvaluateWhenApplied(const ChartHypothesis &hypo, + ScoreComponentCollection* accumulator) const + {} + void EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const + {} + + void ComputeFeatures(const Phrase &source, + const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator) const; + void SetParameter(const std::string& key, const std::string& value); +}; + +} + +#endif // moses_CorrectionPattern_h diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index 10db90d1b..a41b8cb2c 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -30,6 +30,7 @@ #include "moses/FF/TargetBigramFeature.h" #include "moses/FF/TargetNgramFeature.h" #include "moses/FF/PhraseBoundaryFeature.h" +#include "moses/FF/PhraseDistanceFeature.h" #include "moses/FF/PhrasePairFeature.h" #include "moses/FF/RulePairUnlexicalizedSource.h" #include "moses/FF/PhraseLengthFeature.h" @@ -39,6 +40,7 @@ #include "moses/FF/InputFeature.h" #include "moses/FF/PhrasePenalty.h" #include "moses/FF/OSM-Feature/OpSequenceModel.h" +#include "moses/FF/Dsg-Feature/DsgModel.h" #include "moses/FF/ControlRecombination.h" #include "moses/FF/ConstrainedDecoding.h" #include "moses/FF/SoftSourceSyntacticConstraintsFeature.h" @@ -71,8 +73,14 @@ #include "moses/Syntax/InputWeightFF.h" #include "moses/Syntax/RuleTableFF.h" +#include "moses/FF/EditOps.h" +#include "moses/FF/CorrectionPattern.h" + #ifdef HAVE_VW #include "moses/FF/VW/VW.h" +#include "moses/FF/VW/VWFeatureContextBigrams.h" +#include "moses/FF/VW/VWFeatureContextBilingual.h" +#include "moses/FF/VW/VWFeatureContextWindow.h" #include "moses/FF/VW/VWFeatureSourceBagOfWords.h" #include "moses/FF/VW/VWFeatureSourceBigrams.h" #include "moses/FF/VW/VWFeatureSourceIndicator.h" @@ -245,6 +253,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(SourceWordDeletionFeature); MOSES_FNAME(TargetWordInsertionFeature); MOSES_FNAME(PhraseBoundaryFeature); + MOSES_FNAME(PhraseDistanceFeature); MOSES_FNAME(PhraseLengthFeature); MOSES_FNAME(WordTranslationFeature); MOSES_FNAME(TargetBigramFeature); @@ -258,6 +267,7 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME2("WordPenalty", WordPenaltyProducer); MOSES_FNAME(InputFeature); MOSES_FNAME(OpSequenceModel); + MOSES_FNAME(DesegModel); MOSES_FNAME(PhrasePenalty); MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer); MOSES_FNAME(ControlRecombination); @@ -290,8 +300,14 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(SkeletonTranslationOptionListFeature); MOSES_FNAME(SkeletonPT); + MOSES_FNAME(EditOps); + MOSES_FNAME(CorrectionPattern); + #ifdef HAVE_VW MOSES_FNAME(VW); + MOSES_FNAME(VWFeatureContextBigrams); + MOSES_FNAME(VWFeatureContextBilingual); + MOSES_FNAME(VWFeatureContextWindow); MOSES_FNAME(VWFeatureSourceBagOfWords); MOSES_FNAME(VWFeatureSourceBigrams); MOSES_FNAME(VWFeatureSourceIndicator); diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index 986b427b0..fa4b49685 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -173,10 +173,12 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP return score; } -void GlobalLexicalModel::EvaluateInIsolation(const Phrase &source +void GlobalLexicalModel::EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath , const TargetPhrase &targetPhrase + , const StackVec *stackVec , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection &estimatedScores) const + , ScoreComponentCollection *estimatedScores) const { scoreBreakdown.PlusEquals( this, GetFromCacheOrScorePhrase(targetPhrase) ); } diff --git a/moses/FF/GlobalLexicalModel.h b/moses/FF/GlobalLexicalModel.h index 1701195f7..8391609a2 100644 --- a/moses/FF/GlobalLexicalModel.h +++ b/moses/FF/GlobalLexicalModel.h @@ -75,7 +75,8 @@ public: void EvaluateInIsolation(const Phrase &source , const TargetPhrase &targetPhrase , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection &estimatedScores) const; + , ScoreComponentCollection &estimatedScores) const { + } void EvaluateWhenApplied(const Hypothesis& hypo, ScoreComponentCollection* accumulator) const { @@ -89,8 +90,7 @@ public: , const TargetPhrase &targetPhrase , const StackVec *stackVec , ScoreComponentCollection &scoreBreakdown - , ScoreComponentCollection *estimatedScores = NULL) const { - } + , ScoreComponentCollection *estimatedScores = NULL) const; void EvaluateTranslationOptionListWithSourceContext(const InputType &input , const TranslationOptionList &translationOptionList) const { diff --git a/moses/FF/LexicalReordering/PhraseBasedReorderingState.h b/moses/FF/LexicalReordering/PhraseBasedReorderingState.h index cb8bb68e2..1ab5bc9b6 100644 --- a/moses/FF/LexicalReordering/PhraseBasedReorderingState.h +++ b/moses/FF/LexicalReordering/PhraseBasedReorderingState.h @@ -27,11 +27,6 @@ public: LRState* Expand(const TranslationOption& topt,const InputType& input, ScoreComponentCollection* scores) const; - - ReorderingType GetOrientationTypeMSD(Range currRange) const; - ReorderingType GetOrientationTypeMSLR(Range currRange) const; - ReorderingType GetOrientationTypeMonotonic(Range currRange) const; - ReorderingType GetOrientationTypeLeftRight(Range currRange) const; }; } diff --git a/moses/FF/OSM-Feature/KenOSM.cpp b/moses/FF/OSM-Feature/KenOSM.cpp index 25a1e6a93..d20e762f6 100644 --- a/moses/FF/OSM-Feature/KenOSM.cpp +++ b/moses/FF/OSM-Feature/KenOSM.cpp @@ -3,10 +3,11 @@ namespace Moses { -OSMLM* ConstructOSMLM(const char *file) +OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method) { lm::ngram::ModelType model_type; lm::ngram::Config config; + config.load_method = load_method; if (lm::ngram::RecognizeBinary(file, model_type)) { switch(model_type) { case lm::ngram::PROBING: diff --git a/moses/FF/OSM-Feature/KenOSM.h b/moses/FF/OSM-Feature/KenOSM.h index b5fa66ab9..ce3872a35 100644 --- a/moses/FF/OSM-Feature/KenOSM.h +++ b/moses/FF/OSM-Feature/KenOSM.h @@ -27,10 +27,10 @@ public: : m_kenlm(file, config) {} float Score(const lm::ngram::State &in_state, - StringPiece word, - lm::ngram::State &out_state) const { + StringPiece word, + lm::ngram::State &out_state) const { return m_kenlm.Score(in_state, m_kenlm.GetVocabulary().Index(word), - out_state); + out_state); } const lm::ngram::State &BeginSentenceState() const { @@ -47,7 +47,7 @@ private: typedef KenOSMBase OSMLM; -OSMLM* ConstructOSMLM(const char *file); +OSMLM* ConstructOSMLM(const char *file, util::LoadMethod load_method); } // namespace diff --git a/moses/FF/OSM-Feature/OpSequenceModel.cpp b/moses/FF/OSM-Feature/OpSequenceModel.cpp index 4118c8690..1c889e329 100644 --- a/moses/FF/OSM-Feature/OpSequenceModel.cpp +++ b/moses/FF/OSM-Feature/OpSequenceModel.cpp @@ -17,6 +17,7 @@ OpSequenceModel::OpSequenceModel(const std::string &line) tFactor = 0; numFeatures = 5; ReadParameters(); + load_method = util::READ; } OpSequenceModel::~OpSequenceModel() @@ -27,7 +28,7 @@ OpSequenceModel::~OpSequenceModel() void OpSequenceModel :: readLanguageModel(const char *lmFile) { string unkOp = "_TRANS_SLF_"; - OSM = ConstructOSMLM(m_lmPath.c_str()); + OSM = ConstructOSMLM(m_lmPath.c_str(), load_method); State startState = OSM->NullContextState(); State endState; @@ -248,6 +249,20 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va sFactor = Scan<int>(value); } else if (key == "output-factor") { tFactor = Scan<int>(value); + } else if (key == "load") { + if (value == "lazy") { + load_method = util::LAZY; + } else if (value == "populate_or_lazy") { + load_method = util::POPULATE_OR_LAZY; + } else if (value == "populate_or_read" || value == "populate") { + load_method = util::POPULATE_OR_READ; + } else if (value == "read") { + load_method = util::READ; + } else if (value == "parallel_read") { + load_method = util::PARALLEL_READ; + } else { + UTIL_THROW2("Unknown KenLM load method " << value); + } } else { StatefulFeatureFunction::SetParameter(key, value); } diff --git a/moses/FF/OSM-Feature/OpSequenceModel.h b/moses/FF/OSM-Feature/OpSequenceModel.h index 925f9c83a..94beac5aa 100644 --- a/moses/FF/OSM-Feature/OpSequenceModel.h +++ b/moses/FF/OSM-Feature/OpSequenceModel.h @@ -20,6 +20,7 @@ public: int sFactor; // Source Factor ... int tFactor; // Target Factor ... int numFeatures; // Number of features used ... + util::LoadMethod load_method; // method to load model OpSequenceModel(const std::string &line); ~OpSequenceModel(); diff --git a/moses/FF/PhraseDistanceFeature.cpp b/moses/FF/PhraseDistanceFeature.cpp new file mode 100644 index 000000000..0544ec9fb --- /dev/null +++ b/moses/FF/PhraseDistanceFeature.cpp @@ -0,0 +1,123 @@ +#include "PhraseDistanceFeature.h" + +#include <vector> +#include <boost/foreach.hpp> +#include "moses/InputType.h" +#include "moses/ScoreComponentCollection.h" +#include "moses/StaticData.h" +#include "util/exception.hh" + +using namespace std; + +namespace Moses +{ +PhraseDistanceFeature::PhraseDistanceFeature(const string &line) + : StatelessFeatureFunction(2, line) + , m_space("") + , m_spaceID(0) + , m_measure(EuclideanDistance) +{ + ReadParameters(); +} + +void PhraseDistanceFeature::EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedScores) const +{ + vector<float> scores(m_numScoreComponents, 0); + bool broken = false; + // Input coord + map<size_t const, vector<float> >::const_iterator ii; + if (input.m_coordMap) { + ii = input.m_coordMap->find(m_spaceID); + } else { + TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl); + TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl); + broken = true; + } + if (ii == input.m_coordMap->end()) { + TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl); + TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl); + broken = true; + } + // Target phrase coord + vector<SPTR<vector<float> > > const* tpp = targetPhrase.GetCoordList(m_spaceID); + if (tpp == NULL) { + TRACE_ERR("No coordinates for space " << m_space << " on target phrase (PhraseDictionary implementation needs to set)" << endl); + TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl); + broken = true; + } + // Compute scores + if (!broken) { + vector<float> const& inputCoord = ii->second; + vector<SPTR<vector<float> > > const& tpCoord = *tpp; + // Centroid of target phrase instances (from phrase extraction) + vector<float> centroid = vector<float>(inputCoord.size(), 0); + BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) { + for (size_t i = 0; i < inputCoord.size(); ++i) { + centroid[i] += (*coord)[i]; + } + } + for (size_t i = 0; i < inputCoord.size(); ++i) { + centroid[i] /= tpCoord.size(); + } + // Average distance from the target phrase instances to (1) the input and + // (2) the target phrase centroid + float inputDistance = 0; + float centroidDistance = 0; + if (m_measure == EuclideanDistance) { + BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) { + float pointInputDistance = 0; + float pointCentroidDistance = 0; + for (size_t i = 0; i < inputCoord.size(); ++i) { + pointInputDistance += pow(inputCoord[i] - (*coord)[i], 2); + pointCentroidDistance += pow(centroid[i] - (*coord)[i], 2); + } + inputDistance += sqrt(pointInputDistance); + centroidDistance += sqrt(pointCentroidDistance); + } + } else if (m_measure == TotalVariationDistance) { + BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) { + float pointInputDistance = 0; + float pointCentroidDistance = 0; + for (size_t i = 0; i < inputCoord.size(); ++i) { + pointInputDistance += abs(inputCoord[i] - (*coord)[i]); + pointCentroidDistance += abs(centroid[i] - (*coord)[i]); + } + inputDistance += pointInputDistance / 2; + centroidDistance += pointCentroidDistance / 2; + } + } + inputDistance /= tpCoord.size(); + centroidDistance /= tpCoord.size(); + // Log transform scores, max with float epsilon to avoid domain error + scores[0] = log(max(inputDistance, Moses::FLOAT_EPSILON)); + scores[1] = log(max(centroidDistance, Moses::FLOAT_EPSILON)); + } + // Set scores + scoreBreakdown.Assign(this, scores); + return; +} + +void PhraseDistanceFeature::SetParameter(const string& key, const string& value) +{ + if (key == "space") { + m_space = value; + m_spaceID = StaticData::InstanceNonConst().MapCoordSpace(m_space); + } else if (key == "measure") { + if (value == "euc") { + m_measure = EuclideanDistance; + } else if (value == "var") { + m_measure = TotalVariationDistance; + } else { + UTIL_THROW2("Unknown measure " << value << ", choices: euc var"); + } + } else { + StatelessFeatureFunction::SetParameter(key, value); + } +} + +} // namespace diff --git a/moses/FF/PhraseDistanceFeature.h b/moses/FF/PhraseDistanceFeature.h new file mode 100644 index 000000000..8c9e1a361 --- /dev/null +++ b/moses/FF/PhraseDistanceFeature.h @@ -0,0 +1,56 @@ +#pragma once + +#include "StatelessFeatureFunction.h" + +namespace Moses +{ + +class PhraseDistanceFeature : public StatelessFeatureFunction +{ + enum Measure { + EuclideanDistance, + TotalVariationDistance, + }; + +public: + PhraseDistanceFeature(const std::string &line); + + bool IsUseable(const FactorMask &mask) const { + return true; + } + + virtual void EvaluateInIsolation(const Phrase &source + , const TargetPhrase &targetPhrase + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection &estimatedScores) const { + } + + void EvaluateWhenApplied(const Hypothesis& hypo, + ScoreComponentCollection* accumulator) const { + } + void EvaluateWhenApplied(const ChartHypothesis &hypo, + ScoreComponentCollection* accumulator) const { + } + void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge, + ScoreComponentCollection* accumulator) const { + } + + void EvaluateWithSourceContext(const InputType &input + , const InputPath &inputPath + , const TargetPhrase &targetPhrase + , const StackVec *stackVec + , ScoreComponentCollection &scoreBreakdown + , ScoreComponentCollection *estimatedScores = NULL) const; + + void EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const { + } + void SetParameter(const std::string& key, const std::string& value); + +protected: + Measure m_measure; + std::string m_space; + size_t m_spaceID; +}; + +} //namespace diff --git a/moses/FF/VW/AlignmentConstraint.h b/moses/FF/VW/AlignmentConstraint.h new file mode 100644 index 000000000..28ba7d4f3 --- /dev/null +++ b/moses/FF/VW/AlignmentConstraint.h @@ -0,0 +1,40 @@ +#pragma once + +namespace Moses +{ + +/** + * Helper class for storing alignment constraints. + */ +class AlignmentConstraint +{ +public: + AlignmentConstraint() : m_min(std::numeric_limits<int>::max()), m_max(-1) {} + + AlignmentConstraint(int min, int max) : m_min(min), m_max(max) {} + + /** + * We are aligned to point => our min cannot be larger, our max cannot be smaller. + */ + void Update(int point) { + if (m_min > point) m_min = point; + if (m_max < point) m_max = point; + } + + bool IsSet() const { + return m_max != -1; + } + + int GetMin() const { + return m_min; + } + + int GetMax() const { + return m_max; + } + +private: + int m_min, m_max; +}; + +} diff --git a/moses/FF/VW/VW.cpp b/moses/FF/VW/VW.cpp new file mode 100644 index 000000000..e5e5316b6 --- /dev/null +++ b/moses/FF/VW/VW.cpp @@ -0,0 +1,637 @@ +#include <string> +#include <map> +#include <limits> +#include <vector> + +#include <boost/unordered_map.hpp> +#include <boost/functional/hash.hpp> + +#include "moses/FF/StatefulFeatureFunction.h" +#include "moses/PP/CountsPhraseProperty.h" +#include "moses/TranslationOptionList.h" +#include "moses/TranslationOption.h" +#include "moses/Util.h" +#include "moses/TypeDef.h" +#include "moses/StaticData.h" +#include "moses/Phrase.h" +#include "moses/AlignmentInfo.h" +#include "moses/AlignmentInfoCollection.h" +#include "moses/Word.h" +#include "moses/FactorCollection.h" + +#include "Normalizer.h" +#include "Classifier.h" +#include "VWFeatureBase.h" +#include "TabbedSentence.h" +#include "ThreadLocalByFeatureStorage.h" +#include "TrainingLoss.h" +#include "VWTargetSentence.h" +#include "VWState.h" +#include "VW.h" + +namespace Moses +{ + +VW::VW(const std::string &line) + : StatefulFeatureFunction(1, line) + , TLSTargetSentence(this) + , m_train(false) + , m_sentenceStartWord(Word()) +{ + ReadParameters(); + Discriminative::ClassifierFactory *classifierFactory = m_train + ? new Discriminative::ClassifierFactory(m_modelPath) + : new Discriminative::ClassifierFactory(m_modelPath, m_vwOptions); + + m_tlsClassifier = new TLSClassifier(this, *classifierFactory); + + m_tlsFutureScores = new TLSFloatHashMap(this); + m_tlsComputedStateExtensions = new TLSStateExtensions(this); + m_tlsTranslationOptionFeatures = new TLSFeatureVectorMap(this); + m_tlsTargetContextFeatures = new TLSFeatureVectorMap(this); + + if (! m_normalizer) { + VERBOSE(1, "VW :: No loss function specified, assuming logistic loss.\n"); + m_normalizer = (Discriminative::Normalizer *) new Discriminative::LogisticLossNormalizer(); + } + + if (! m_trainingLoss) { + VERBOSE(1, "VW :: Using basic 1/0 loss calculation in training.\n"); + m_trainingLoss = (TrainingLoss *) new TrainingLossBasic(); + } + + // create a virtual beginning-of-sentence word with all factors replaced by <S> + const Factor *bosFactor = FactorCollection::Instance().AddFactor(BOS_); + for (size_t i = 0; i < MAX_NUM_FACTORS; i++) + m_sentenceStartWord.SetFactor(i, bosFactor); +} + +VW::~VW() +{ + delete m_tlsClassifier; + delete m_normalizer; + // TODO delete more stuff +} + +FFState* VW::EvaluateWhenApplied( + const Hypothesis& curHypo, + const FFState* prevState, + ScoreComponentCollection* accumulator) const +{ + VERBOSE(3, "VW :: Evaluating translation options\n"); + + const VWState& prevVWState = *static_cast<const VWState *>(prevState); + + const std::vector<VWFeatureBase*>& contextFeatures = + VWFeatureBase::GetTargetContextFeatures(GetScoreProducerDescription()); + + if (contextFeatures.empty()) { + // no target context features => we already evaluated everything in + // EvaluateTranslationOptionListWithSourceContext(). Nothing to do now, + // no state information to track. + return new VWState(); + } + + size_t spanStart = curHypo.GetTranslationOption().GetStartPos(); + size_t spanEnd = curHypo.GetTranslationOption().GetEndPos(); + + // compute our current key + size_t cacheKey = MakeCacheKey(prevState, spanStart, spanEnd); + + boost::unordered_map<size_t, FloatHashMap> &computedStateExtensions + = *m_tlsComputedStateExtensions->GetStored(); + + if (computedStateExtensions.find(cacheKey) == computedStateExtensions.end()) { + // we have not computed this set of translation options yet + const TranslationOptionList *topts = + curHypo.GetManager().getSntTranslationOptions()->GetTranslationOptionList(spanStart, spanEnd); + + const InputType& input = curHypo.GetManager().GetSource(); + + Discriminative::Classifier &classifier = *m_tlsClassifier->GetStored(); + + // extract target context features + size_t contextHash = prevVWState.hash(); + + FeatureVectorMap &contextFeaturesCache = *m_tlsTargetContextFeatures->GetStored(); + + FeatureVectorMap::const_iterator contextIt = contextFeaturesCache.find(contextHash); + if (contextIt == contextFeaturesCache.end()) { + // we have not extracted features for this context yet + + const Phrase &targetContext = prevVWState.GetPhrase(); + Discriminative::FeatureVector contextVector; + const AlignmentInfo *alignInfo = TransformAlignmentInfo(curHypo, targetContext.GetSize()); + for(size_t i = 0; i < contextFeatures.size(); ++i) + (*contextFeatures[i])(input, targetContext, *alignInfo, classifier, contextVector); + + contextFeaturesCache[contextHash] = contextVector; + VERBOSE(3, "VW :: context cache miss\n"); + } else { + // context already in cache, simply put feature IDs in the classifier object + classifier.AddLabelIndependentFeatureVector(contextIt->second); + VERBOSE(3, "VW :: context cache hit\n"); + } + + std::vector<float> losses(topts->size()); + + for (size_t toptIdx = 0; toptIdx < topts->size(); toptIdx++) { + const TranslationOption *topt = topts->Get(toptIdx); + const TargetPhrase &targetPhrase = topt->GetTargetPhrase(); + size_t toptHash = hash_value(*topt); + + // start with pre-computed source-context-only VW scores + losses[toptIdx] = m_tlsFutureScores->GetStored()->find(toptHash)->second; + + // add all features associated with this translation option + // (pre-computed when evaluated with source context) + const Discriminative::FeatureVector &targetFeatureVector = + m_tlsTranslationOptionFeatures->GetStored()->find(toptHash)->second; + + classifier.AddLabelDependentFeatureVector(targetFeatureVector); + + // add classifier score with context+target features only to the total loss + losses[toptIdx] += classifier.Predict(MakeTargetLabel(targetPhrase)); + } + + // normalize classifier scores to get a probability distribution + (*m_normalizer)(losses); + + // fill our cache with the results + FloatHashMap &toptScores = computedStateExtensions[cacheKey]; + for (size_t toptIdx = 0; toptIdx < topts->size(); toptIdx++) { + const TranslationOption *topt = topts->Get(toptIdx); + size_t toptHash = hash_value(*topt); + toptScores[toptHash] = FloorScore(TransformScore(losses[toptIdx])); + } + + VERBOSE(3, "VW :: cache miss\n"); + } else { + VERBOSE(3, "VW :: cache hit\n"); + } + + // now our cache is guaranteed to contain the required score, simply look it up + std::vector<float> newScores(m_numScoreComponents); + size_t toptHash = hash_value(curHypo.GetTranslationOption()); + newScores[0] = computedStateExtensions[cacheKey][toptHash]; + VERBOSE(3, "VW :: adding score: " << newScores[0] << "\n"); + accumulator->PlusEquals(this, newScores); + + return new VWState(prevVWState, curHypo); +} + +const FFState* VW::EmptyHypothesisState(const InputType &input) const +{ + size_t maxContextSize = VWFeatureBase::GetMaximumContextSize(GetScoreProducerDescription()); + Phrase initialPhrase; + for (size_t i = 0; i < maxContextSize; i++) + initialPhrase.AddWord(m_sentenceStartWord); + + return new VWState(initialPhrase); +} + +void VW::EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const +{ + Discriminative::Classifier &classifier = *m_tlsClassifier->GetStored(); + + if (translationOptionList.size() == 0) + return; // nothing to do + + VERBOSE(3, "VW :: Evaluating translation options\n"); + + // which feature functions do we use (on the source and target side) + const std::vector<VWFeatureBase*>& sourceFeatures = + VWFeatureBase::GetSourceFeatures(GetScoreProducerDescription()); + + const std::vector<VWFeatureBase*>& contextFeatures = + VWFeatureBase::GetTargetContextFeatures(GetScoreProducerDescription()); + + const std::vector<VWFeatureBase*>& targetFeatures = + VWFeatureBase::GetTargetFeatures(GetScoreProducerDescription()); + + size_t maxContextSize = VWFeatureBase::GetMaximumContextSize(GetScoreProducerDescription()); + + // only use stateful score computation when needed + bool haveTargetContextFeatures = ! contextFeatures.empty(); + + const Range &sourceRange = translationOptionList.Get(0)->GetSourceWordsRange(); + + if (m_train) { + // + // extract features for training the classifier (only call this when using vwtrainer, not in Moses!) + // + + // find which topts are correct + std::vector<bool> correct(translationOptionList.size()); + std::vector<int> startsAt(translationOptionList.size()); + std::set<int> uncoveredStartingPositions; + + for (size_t i = 0; i < translationOptionList.size(); i++) { + std::pair<bool, int> isCorrect = IsCorrectTranslationOption(* translationOptionList.Get(i)); + correct[i] = isCorrect.first; + startsAt[i] = isCorrect.second; + if (isCorrect.first) { + uncoveredStartingPositions.insert(isCorrect.second); + } + } + + // optionally update translation options using leave-one-out + std::vector<bool> keep = (m_leaveOneOut.size() > 0) + ? LeaveOneOut(translationOptionList, correct) + : std::vector<bool>(translationOptionList.size(), true); + + while (! uncoveredStartingPositions.empty()) { + int currentStart = *uncoveredStartingPositions.begin(); + uncoveredStartingPositions.erase(uncoveredStartingPositions.begin()); + + // check whether we (still) have some correct translation + int firstCorrect = -1; + for (size_t i = 0; i < translationOptionList.size(); i++) { + if (keep[i] && correct[i] && startsAt[i] == currentStart) { + firstCorrect = i; + break; + } + } + + // do not train if there are no positive examples + if (firstCorrect == -1) { + VERBOSE(3, "VW :: skipping topt collection, no correct translation for span at current tgt start position\n"); + continue; + } + + // the first correct topt can be used by some loss functions + const TargetPhrase &correctPhrase = translationOptionList.Get(firstCorrect)->GetTargetPhrase(); + + // feature extraction *at prediction time* outputs feature hashes which can be cached; + // this is training time, simply store everything in this dummyVector + Discriminative::FeatureVector dummyVector; + + // extract source side features + for(size_t i = 0; i < sourceFeatures.size(); ++i) + (*sourceFeatures[i])(input, sourceRange, classifier, dummyVector); + + // build target-side context + Phrase targetContext; + for (size_t i = 0; i < maxContextSize; i++) + targetContext.AddWord(m_sentenceStartWord); + + const Phrase *targetSent = GetStored()->m_sentence; + + // word alignment info shifted by context size + AlignmentInfo contextAlignment = TransformAlignmentInfo(*GetStored()->m_alignment, maxContextSize, currentStart); + + if (currentStart > 0) + targetContext.Append(targetSent->GetSubString(Range(0, currentStart - 1))); + + // extract target-context features + for(size_t i = 0; i < contextFeatures.size(); ++i) + (*contextFeatures[i])(input, targetContext, contextAlignment, classifier, dummyVector); + + // go over topts, extract target side features and train the classifier + for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { + + // this topt was discarded by leaving one out + if (! keep[toptIdx]) + continue; + + // extract target-side features for each topt + const TargetPhrase &targetPhrase = translationOptionList.Get(toptIdx)->GetTargetPhrase(); + for(size_t i = 0; i < targetFeatures.size(); ++i) + (*targetFeatures[i])(input, targetPhrase, classifier, dummyVector); + + bool isCorrect = correct[toptIdx] && startsAt[toptIdx] == currentStart; + float loss = (*m_trainingLoss)(targetPhrase, correctPhrase, isCorrect); + + // train classifier on current example + classifier.Train(MakeTargetLabel(targetPhrase), loss); + } + } + } else { + // + // predict using a trained classifier, use this in decoding (=at test time) + // + + std::vector<float> losses(translationOptionList.size()); + + Discriminative::FeatureVector outFeaturesSourceNamespace; + + // extract source side features + for(size_t i = 0; i < sourceFeatures.size(); ++i) + (*sourceFeatures[i])(input, sourceRange, classifier, outFeaturesSourceNamespace); + + for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { + const TranslationOption *topt = translationOptionList.Get(toptIdx); + const TargetPhrase &targetPhrase = topt->GetTargetPhrase(); + Discriminative::FeatureVector outFeaturesTargetNamespace; + + // extract target-side features for each topt + for(size_t i = 0; i < targetFeatures.size(); ++i) + (*targetFeatures[i])(input, targetPhrase, classifier, outFeaturesTargetNamespace); + + // cache the extracted target features (i.e. features associated with given topt) + // for future use at decoding time + size_t toptHash = hash_value(*topt); + m_tlsTranslationOptionFeatures->GetStored()->insert( + std::make_pair(toptHash, outFeaturesTargetNamespace)); + + // get classifier score + losses[toptIdx] = classifier.Predict(MakeTargetLabel(targetPhrase)); + } + + // normalize classifier scores to get a probability distribution + std::vector<float> rawLosses = losses; + (*m_normalizer)(losses); + + // update scores of topts + for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { + TranslationOption *topt = *(translationOptionList.begin() + toptIdx); + if (! haveTargetContextFeatures) { + // no target context features; evaluate the FF now + std::vector<float> newScores(m_numScoreComponents); + newScores[0] = FloorScore(TransformScore(losses[toptIdx])); + + ScoreComponentCollection &scoreBreakDown = topt->GetScoreBreakdown(); + scoreBreakDown.PlusEquals(this, newScores); + + topt->UpdateScore(); + } else { + // We have target context features => this is just a partial score, + // do not add it to the score component collection. + size_t toptHash = hash_value(*topt); + + // Subtract the score contribution of target-only features, otherwise it would + // be included twice. + Discriminative::FeatureVector emptySource; + const Discriminative::FeatureVector &targetFeatureVector = + m_tlsTranslationOptionFeatures->GetStored()->find(toptHash)->second; + classifier.AddLabelIndependentFeatureVector(emptySource); + classifier.AddLabelDependentFeatureVector(targetFeatureVector); + float targetOnlyLoss = classifier.Predict(VW_DUMMY_LABEL); + + float futureScore = rawLosses[toptIdx] - targetOnlyLoss; + m_tlsFutureScores->GetStored()->insert(std::make_pair(toptHash, futureScore)); + } + } + } +} + +void VW::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "train") { + m_train = Scan<bool>(value); + } else if (key == "path") { + m_modelPath = value; + } else if (key == "vw-options") { + m_vwOptions = value; + } else if (key == "leave-one-out-from") { + m_leaveOneOut = value; + } else if (key == "training-loss") { + // which type of loss to use for training + if (value == "basic") { + m_trainingLoss = (TrainingLoss *) new TrainingLossBasic(); + } else if (value == "bleu") { + m_trainingLoss = (TrainingLoss *) new TrainingLossBLEU(); + } else { + UTIL_THROW2("Unknown training loss type:" << value); + } + } else if (key == "loss") { + // which normalizer to use (theoretically depends on the loss function used for training the + // classifier (squared/logistic/hinge/...), hence the name "loss" + if (value == "logistic") { + m_normalizer = (Discriminative::Normalizer *) new Discriminative::LogisticLossNormalizer(); + } else if (value == "squared") { + m_normalizer = (Discriminative::Normalizer *) new Discriminative::SquaredLossNormalizer(); + } else { + UTIL_THROW2("Unknown loss type:" << value); + } + } else { + StatefulFeatureFunction::SetParameter(key, value); + } +} + +void VW::InitializeForInput(ttasksptr const& ttask) +{ + // do not keep future cost estimates across sentences! + m_tlsFutureScores->GetStored()->clear(); + + // invalidate our caches after each sentence + m_tlsComputedStateExtensions->GetStored()->clear(); + + // it's not certain that we should clear these caches; we do it + // because they shouldn't be allowed to grow indefinitely large but + // target contexts and translation options will have identical features + // the next time we extract them... + m_tlsTargetContextFeatures->GetStored()->clear(); + m_tlsTranslationOptionFeatures->GetStored()->clear(); + + InputType const& source = *(ttask->GetSource().get()); + // tabbed sentence is assumed only in training + if (! m_train) + return; + + UTIL_THROW_IF2(source.GetType() != TabbedSentenceInput, + "This feature function requires the TabbedSentence input type"); + + const TabbedSentence& tabbedSentence = static_cast<const TabbedSentence&>(source); + UTIL_THROW_IF2(tabbedSentence.GetColumns().size() < 2, + "TabbedSentence must contain target<tab>alignment"); + + // target sentence represented as a phrase + Phrase *target = new Phrase(); + target->CreateFromString( + Output + , StaticData::Instance().options()->output.factor_order + , tabbedSentence.GetColumns()[0] + , NULL); + + // word alignment between source and target sentence + // we don't store alignment info in AlignmentInfoCollection because we keep alignments of whole + // sentences, not phrases + AlignmentInfo *alignment = new AlignmentInfo(tabbedSentence.GetColumns()[1]); + + VWTargetSentence &targetSent = *GetStored(); + targetSent.Clear(); + targetSent.m_sentence = target; + targetSent.m_alignment = alignment; + + // pre-compute max- and min- aligned points for faster translation option checking + targetSent.SetConstraints(source.GetSize()); +} + +/************************************************************************************* + * private methods + ************************************************************************************/ + +const AlignmentInfo *VW::TransformAlignmentInfo(const Hypothesis &curHypo, size_t contextSize) const +{ + std::set<std::pair<size_t, size_t> > alignmentPoints; + const Hypothesis *contextHypo = curHypo.GetPrevHypo(); + int idxInContext = contextSize - 1; + int processedWordsInHypo = 0; + while (idxInContext >= 0 && contextHypo) { + int idxInHypo = contextHypo->GetCurrTargetLength() - 1 - processedWordsInHypo; + if (idxInHypo >= 0) { + const AlignmentInfo &hypoAlign = contextHypo->GetCurrTargetPhrase().GetAlignTerm(); + std::set<size_t> alignedToTgt = hypoAlign.GetAlignmentsForTarget(idxInHypo); + size_t srcOffset = contextHypo->GetCurrSourceWordsRange().GetStartPos(); + BOOST_FOREACH(size_t srcIdx, alignedToTgt) { + alignmentPoints.insert(std::make_pair(srcOffset + srcIdx, idxInContext)); + } + processedWordsInHypo++; + idxInContext--; + } else { + processedWordsInHypo = 0; + contextHypo = contextHypo->GetPrevHypo(); + } + } + + return AlignmentInfoCollection::Instance().Add(alignmentPoints); +} + +AlignmentInfo VW::TransformAlignmentInfo(const AlignmentInfo &alignInfo, size_t contextSize, int currentStart) const +{ + std::set<std::pair<size_t, size_t> > alignmentPoints; + for (int i = std::max(0, currentStart - (int)contextSize); i < currentStart; i++) { + std::set<size_t> alignedToTgt = alignInfo.GetAlignmentsForTarget(i); + BOOST_FOREACH(size_t srcIdx, alignedToTgt) { + alignmentPoints.insert(std::make_pair(srcIdx, i + contextSize)); + } + } + return AlignmentInfo(alignmentPoints); +} + +std::pair<bool, int> VW::IsCorrectTranslationOption(const TranslationOption &topt) const +{ + + //std::cerr << topt.GetSourceWordsRange() << std::endl; + + int sourceStart = topt.GetSourceWordsRange().GetStartPos(); + int sourceEnd = topt.GetSourceWordsRange().GetEndPos(); + + const VWTargetSentence &targetSentence = *GetStored(); + + // [targetStart, targetEnd] spans aligned target words + int targetStart = targetSentence.m_sentence->GetSize(); + int targetEnd = -1; + + // get the left-most and right-most alignment point within source span + for(int i = sourceStart; i <= sourceEnd; ++i) { + if(targetSentence.m_sourceConstraints[i].IsSet()) { + if(targetStart > targetSentence.m_sourceConstraints[i].GetMin()) + targetStart = targetSentence.m_sourceConstraints[i].GetMin(); + if(targetEnd < targetSentence.m_sourceConstraints[i].GetMax()) + targetEnd = targetSentence.m_sourceConstraints[i].GetMax(); + } + } + // there was no alignment + if(targetEnd == -1) + return std::make_pair(false, -1); + + //std::cerr << "Shorter: " << targetStart << " " << targetEnd << std::endl; + + // [targetStart2, targetEnd2] spans unaligned words left and right of [targetStart, targetEnd] + int targetStart2 = targetStart; + for(int i = targetStart2; i >= 0 && !targetSentence.m_targetConstraints[i].IsSet(); --i) + targetStart2 = i; + + int targetEnd2 = targetEnd; + for(int i = targetEnd2; + i < targetSentence.m_sentence->GetSize() && !targetSentence.m_targetConstraints[i].IsSet(); + ++i) + targetEnd2 = i; + + //std::cerr << "Longer: " << targetStart2 << " " << targetEnd2 << std::endl; + + const TargetPhrase &tphrase = topt.GetTargetPhrase(); + //std::cerr << tphrase << std::endl; + + // if target phrase is shorter than inner span return false + if(tphrase.GetSize() < targetEnd - targetStart + 1) + return std::make_pair(false, -1); + + // if target phrase is longer than outer span return false + if(tphrase.GetSize() > targetEnd2 - targetStart2 + 1) + return std::make_pair(false, -1); + + // for each possible starting point + for(int tempStart = targetStart2; tempStart <= targetStart; tempStart++) { + bool found = true; + // check if the target phrase is within longer span + for(int i = tempStart; i <= targetEnd2 && i < tphrase.GetSize() + tempStart; ++i) { + if(tphrase.GetWord(i - tempStart) != targetSentence.m_sentence->GetWord(i)) { + found = false; + break; + } + } + // return true if there was a match + if(found) { + //std::cerr << "Found" << std::endl; + return std::make_pair(true, tempStart); + } + } + + return std::make_pair(false, -1); +} + +std::vector<bool> VW::LeaveOneOut(const TranslationOptionList &topts, const std::vector<bool> &correct) const +{ + UTIL_THROW_IF2(m_leaveOneOut.size() == 0 || ! m_train, "LeaveOneOut called in wrong setting!"); + + float sourceRawCount = 0.0; + const float ONE = 1.0001; // I don't understand floating point numbers + + std::vector<bool> keepOpt; + + for (size_t i = 0; i < topts.size(); i++) { + TranslationOption *topt = *(topts.begin() + i); + const TargetPhrase &targetPhrase = topt->GetTargetPhrase(); + + // extract raw counts from phrase-table property + const CountsPhraseProperty *property = + static_cast<const CountsPhraseProperty *>(targetPhrase.GetProperty("Counts")); + + if (! property) { + VERBOSE(2, "VW :: Counts not found for topt! Is this an OOV?\n"); + // keep all translation opts without updating, this is either OOV or bad usage... + keepOpt.assign(topts.size(), true); + return keepOpt; + } + + if (sourceRawCount == 0.0) { + sourceRawCount = property->GetSourceMarginal() - ONE; // discount one occurrence of the source phrase + if (sourceRawCount <= 0) { + // no translation options survived, source phrase was a singleton + keepOpt.assign(topts.size(), false); + return keepOpt; + } + } + + float discount = correct[i] ? ONE : 0.0; + float target = property->GetTargetMarginal() - discount; + float joint = property->GetJointCount() - discount; + if (discount != 0.0) VERBOSE(3, "VW :: leaving one out!\n"); + + if (joint > 0) { + // topt survived leaving one out, update its scores + const FeatureFunction *feature = &FindFeatureFunction(m_leaveOneOut); + std::vector<float> scores = targetPhrase.GetScoreBreakdown().GetScoresForProducer(feature); + UTIL_THROW_IF2(scores.size() != 4, "Unexpected number of scores in feature " << m_leaveOneOut); + scores[0] = TransformScore(joint / target); // P(f|e) + scores[2] = TransformScore(joint / sourceRawCount); // P(e|f) + + ScoreComponentCollection &scoreBreakDown = topt->GetScoreBreakdown(); + scoreBreakDown.Assign(feature, scores); + topt->UpdateScore(); + keepOpt.push_back(true); + } else { + // they only occurred together once, discard topt + VERBOSE(2, "VW :: discarded topt when leaving one out\n"); + keepOpt.push_back(false); + } + } + + return keepOpt; +} + +} // namespace Moses diff --git a/moses/FF/VW/VW.h b/moses/FF/VW/VW.h index da8a5cfb8..d94cce502 100644 --- a/moses/FF/VW/VW.h +++ b/moses/FF/VW/VW.h @@ -3,8 +3,12 @@ #include <string> #include <map> #include <limits> +#include <vector> -#include "moses/FF/StatelessFeatureFunction.h" +#include <boost/unordered_map.hpp> +#include <boost/functional/hash.hpp> + +#include "moses/FF/StatefulFeatureFunction.h" #include "moses/PP/CountsPhraseProperty.h" #include "moses/TranslationOptionList.h" #include "moses/TranslationOption.h" @@ -13,6 +17,8 @@ #include "moses/StaticData.h" #include "moses/Phrase.h" #include "moses/AlignmentInfo.h" +#include "moses/Word.h" +#include "moses/FactorCollection.h" #include "Normalizer.h" #include "Classifier.h" @@ -20,119 +26,50 @@ #include "TabbedSentence.h" #include "ThreadLocalByFeatureStorage.h" #include "TrainingLoss.h" +#include "VWTargetSentence.h" + +/* + * VW classifier feature. See vw/README.md for further information. + * + * TODO: say which paper to cite. + */ namespace Moses { -const std::string VW_DUMMY_LABEL = "1111"; // VW does not use the actual label, other classifiers might +// dummy class label; VW does not use the actual label, other classifiers might +const std::string VW_DUMMY_LABEL = "1111"; -/** - * Helper class for storing alignment constraints. - */ -class Constraint -{ -public: - Constraint() : m_min(std::numeric_limits<int>::max()), m_max(-1) {} +// thread-specific classifier instance +typedef ThreadLocalByFeatureStorage<Discriminative::Classifier, Discriminative::ClassifierFactory &> TLSClassifier; - Constraint(int min, int max) : m_min(min), m_max(max) {} +// current target sentence, used in VW training (vwtrainer), not in decoding (prediction time) +typedef ThreadLocalByFeatureStorage<VWTargetSentence> TLSTargetSentence; - /** - * We are aligned to point => our min cannot be larger, our max cannot be smaller. - */ - void Update(int point) { - if (m_min > point) m_min = point; - if (m_max < point) m_max = point; - } +// hash table of feature vectors +typedef boost::unordered_map<size_t, Discriminative::FeatureVector> FeatureVectorMap; - bool IsSet() const { - return m_max != -1; - } +// thread-specific feature vector hash +typedef ThreadLocalByFeatureStorage<FeatureVectorMap> TLSFeatureVectorMap; - int GetMin() const { - return m_min; - } +// hash table of partial scores +typedef boost::unordered_map<size_t, float> FloatHashMap; - int GetMax() const { - return m_max; - } +// thread-specific score hash table, used for caching +typedef ThreadLocalByFeatureStorage<FloatHashMap> TLSFloatHashMap; -private: - int m_min, m_max; -}; +// thread-specific hash tablei for caching full classifier outputs +typedef ThreadLocalByFeatureStorage<boost::unordered_map<size_t, FloatHashMap> > TLSStateExtensions; -/** - * VW thread-specific data about target sentence. +/* + * VW feature function. A discriminative classifier with source and target context features. */ -struct VWTargetSentence { - VWTargetSentence() : m_sentence(NULL), m_alignment(NULL) {} - - void Clear() { - if (m_sentence) delete m_sentence; - if (m_alignment) delete m_alignment; - } - - ~VWTargetSentence() { - Clear(); - } - - void SetConstraints(size_t sourceSize) { - // initialize to unconstrained - m_sourceConstraints.assign(sourceSize, Constraint()); - m_targetConstraints.assign(m_sentence->GetSize(), Constraint()); - - // set constraints according to alignment points - AlignmentInfo::const_iterator it; - for (it = m_alignment->begin(); it != m_alignment->end(); it++) { - int src = it->first; - int tgt = it->second; - - if (src >= m_sourceConstraints.size() || tgt >= m_targetConstraints.size()) { - UTIL_THROW2("VW :: alignment point out of bounds: " << src << "-" << tgt); - } - - m_sourceConstraints[src].Update(tgt); - m_targetConstraints[tgt].Update(src); - } - } - - Phrase *m_sentence; - AlignmentInfo *m_alignment; - std::vector<Constraint> m_sourceConstraints, m_targetConstraints; -}; - -typedef ThreadLocalByFeatureStorage<Discriminative::Classifier, Discriminative::ClassifierFactory &> TLSClassifier; - -typedef ThreadLocalByFeatureStorage<VWTargetSentence> TLSTargetSentence; - -class VW : public StatelessFeatureFunction, public TLSTargetSentence +class VW : public StatefulFeatureFunction, public TLSTargetSentence { public: - VW(const std::string &line) - : StatelessFeatureFunction(1, line) - , TLSTargetSentence(this) - , m_train(false) { - ReadParameters(); - Discriminative::ClassifierFactory *classifierFactory = m_train - ? new Discriminative::ClassifierFactory(m_modelPath) - : new Discriminative::ClassifierFactory(m_modelPath, m_vwOptions); - - m_tlsClassifier = new TLSClassifier(this, *classifierFactory); - - if (! m_normalizer) { - VERBOSE(1, "VW :: No loss function specified, assuming logistic loss.\n"); - m_normalizer = (Discriminative::Normalizer *) new Discriminative::LogisticLossNormalizer(); - } - - if (! m_trainingLoss) { - VERBOSE(1, "VW :: Using basic 1/0 loss calculation in training.\n"); - m_trainingLoss = (TrainingLoss *) new TrainingLossBasic(); - } - } + VW(const std::string &line); - virtual ~VW() { - delete m_tlsClassifier; - delete m_normalizer; - } + virtual ~VW(); bool IsUseable(const FactorMask &mask) const { return true; @@ -152,335 +89,89 @@ public: , ScoreComponentCollection *estimatedFutureScore = NULL) const { } - void EvaluateTranslationOptionListWithSourceContext(const InputType &input - , const TranslationOptionList &translationOptionList) const { - Discriminative::Classifier &classifier = *m_tlsClassifier->GetStored(); - - if (translationOptionList.size() == 0) - return; // nothing to do - - VERBOSE(2, "VW :: Evaluating translation options\n"); - - // which feature functions do we use (on the source and target side) - const std::vector<VWFeatureBase*>& sourceFeatures = - VWFeatureBase::GetSourceFeatures(GetScoreProducerDescription()); - - const std::vector<VWFeatureBase*>& targetFeatures = - VWFeatureBase::GetTargetFeatures(GetScoreProducerDescription()); - - const Range &sourceRange = translationOptionList.Get(0)->GetSourceWordsRange(); - const InputPath &inputPath = translationOptionList.Get(0)->GetInputPath(); - - if (m_train) { - // - // extract features for training the classifier (only call this when using vwtrainer, not in Moses!) - // - - // find which topts are correct - std::vector<bool> correct(translationOptionList.size()); - for (size_t i = 0; i < translationOptionList.size(); i++) - correct[i] = IsCorrectTranslationOption(* translationOptionList.Get(i)); - - // optionally update translation options using leave-one-out - std::vector<bool> keep = (m_leaveOneOut.size() > 0) - ? LeaveOneOut(translationOptionList, correct) - : std::vector<bool>(translationOptionList.size(), true); - - // check whether we (still) have some correct translation - int firstCorrect = -1; - for (size_t i = 0; i < translationOptionList.size(); i++) { - if (keep[i] && correct[i]) { - firstCorrect = i; - break; - } - } - - // do not train if there are no positive examples - if (firstCorrect == -1) { - VERBOSE(2, "VW :: skipping topt collection, no correct translation for span\n"); - return; - } - - // the first correct topt can be used by some loss functions - const TargetPhrase &correctPhrase = translationOptionList.Get(firstCorrect)->GetTargetPhrase(); - - // extract source side features - for(size_t i = 0; i < sourceFeatures.size(); ++i) - (*sourceFeatures[i])(input, inputPath, sourceRange, classifier); - - // go over topts, extract target side features and train the classifier - for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { - - // this topt was discarded by leaving one out - if (! keep[toptIdx]) - continue; - - // extract target-side features for each topt - const TargetPhrase &targetPhrase = translationOptionList.Get(toptIdx)->GetTargetPhrase(); - for(size_t i = 0; i < targetFeatures.size(); ++i) - (*targetFeatures[i])(input, inputPath, targetPhrase, classifier); - - float loss = (*m_trainingLoss)(targetPhrase, correctPhrase, correct[toptIdx]); - - // train classifier on current example - classifier.Train(MakeTargetLabel(targetPhrase), loss); - } - } else { - // - // predict using a trained classifier, use this in decoding (=at test time) - // - - std::vector<float> losses(translationOptionList.size()); - - // extract source side features - for(size_t i = 0; i < sourceFeatures.size(); ++i) - (*sourceFeatures[i])(input, inputPath, sourceRange, classifier); - - for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { - const TranslationOption *topt = translationOptionList.Get(toptIdx); - const TargetPhrase &targetPhrase = topt->GetTargetPhrase(); - - // extract target-side features for each topt - for(size_t i = 0; i < targetFeatures.size(); ++i) - (*targetFeatures[i])(input, inputPath, targetPhrase, classifier); - - // get classifier score - losses[toptIdx] = classifier.Predict(MakeTargetLabel(targetPhrase)); - } - - // normalize classifier scores to get a probability distribution - (*m_normalizer)(losses); - - // update scores of topts - for (size_t toptIdx = 0; toptIdx < translationOptionList.size(); toptIdx++) { - TranslationOption *topt = *(translationOptionList.begin() + toptIdx); - std::vector<float> newScores(m_numScoreComponents); - newScores[0] = FloorScore(TransformScore(losses[toptIdx])); - - ScoreComponentCollection &scoreBreakDown = topt->GetScoreBreakdown(); - scoreBreakDown.PlusEquals(this, newScores); - - topt->UpdateScore(); - } - } - } - - void EvaluateWhenApplied(const Hypothesis& hypo, - ScoreComponentCollection* accumulator) const { - } + // This behavior of this method depends on whether it's called during VW + // training (feature extraction) by vwtrainer or during decoding (prediction + // time) by Moses. + // + // When predicting, it evaluates all translation options with the VW model; + // if no target-context features are defined, this is the final score and it + // is added directly to the TranslationOption score. If there are target + // context features, the score is a partial score and it is only stored in + // cache; the final score is computed based on target context in + // EvaluateWhenApplied(). + // + // This method is also used in training by vwtrainer in which case features + // are written to a file, no classifier predictions take place. Target-side + // context is constant at training time (we know the true target sentence), + // so target-context features are extracted here as well. + virtual void EvaluateTranslationOptionListWithSourceContext(const InputType &input + , const TranslationOptionList &translationOptionList) const; + + // Evaluate VW during decoding. This is only used at prediction time (not in training). + // When no target-context features are defined, VW predictions were already fully calculated + // in EvaluateTranslationOptionListWithSourceContext() and the scores were added to the model. + // If there are target-context features, we compute the context-dependent part of the + // classifier score and combine it with the source-context only partial score which was computed + // in EvaluateTranslationOptionListWithSourceContext(). Various caches are used to make this + // method more efficient. + virtual FFState* EvaluateWhenApplied( + const Hypothesis& curHypo, + const FFState* prevState, + ScoreComponentCollection* accumulator) const; + + virtual FFState* EvaluateWhenApplied( + const ChartHypothesis&, + int, + ScoreComponentCollection* accumulator) const { + throw new std::logic_error("hiearchical/syntax not supported"); + } + + // Initial VW state; contains unaligned BOS symbols. + const FFState* EmptyHypothesisState(const InputType &input) const; + + void SetParameter(const std::string& key, const std::string& value); + + // At prediction time, this clears our caches. At training time, we load the next sentence, its + // translation and word alignment. + virtual void InitializeForInput(ttasksptr const& ttask); - void EvaluateWhenApplied(const ChartHypothesis &hypo, - ScoreComponentCollection* accumulator) const { +private: + inline std::string MakeTargetLabel(const TargetPhrase &targetPhrase) const { + return VW_DUMMY_LABEL; // VW does not care about class labels in our setting (--csoaa_ldf mc). } - void SetParameter(const std::string& key, const std::string& value) { - if (key == "train") { - m_train = Scan<bool>(value); - } else if (key == "path") { - m_modelPath = value; - } else if (key == "vw-options") { - m_vwOptions = value; - } else if (key == "leave-one-out-from") { - m_leaveOneOut = value; - } else if (key == "training-loss") { - // which type of loss to use for training - if (value == "basic") { - m_trainingLoss = (TrainingLoss *) new TrainingLossBasic(); - } else if (value == "bleu") { - m_trainingLoss = (TrainingLoss *) new TrainingLossBLEU(); - } else { - UTIL_THROW2("Unknown training loss type:" << value); - } - } else if (key == "loss") { - // which normalizer to use (theoretically depends on the loss function used for training the - // classifier (squared/logistic/hinge/...), hence the name "loss" - if (value == "logistic") { - m_normalizer = (Discriminative::Normalizer *) new Discriminative::LogisticLossNormalizer(); - } else if (value == "squared") { - m_normalizer = (Discriminative::Normalizer *) new Discriminative::SquaredLossNormalizer(); - } else { - UTIL_THROW2("Unknown loss type:" << value); - } - } else { - StatelessFeatureFunction::SetParameter(key, value); - } + inline size_t MakeCacheKey(const FFState *prevState, size_t spanStart, size_t spanEnd) const { + size_t key = 0; + boost::hash_combine(key, prevState); + boost::hash_combine(key, spanStart); + boost::hash_combine(key, spanEnd); + return key; } - virtual void InitializeForInput(ttasksptr const& ttask) { - InputType const& source = *(ttask->GetSource().get()); - // tabbed sentence is assumed only in training - if (! m_train) - return; - - UTIL_THROW_IF2(source.GetType() != TabbedSentenceInput, - "This feature function requires the TabbedSentence input type"); - - const TabbedSentence& tabbedSentence = static_cast<const TabbedSentence&>(source); - UTIL_THROW_IF2(tabbedSentence.GetColumns().size() < 2, - "TabbedSentence must contain target<tab>alignment"); - - // target sentence represented as a phrase - Phrase *target = new Phrase(); - target->CreateFromString( - Output - , StaticData::Instance().options()->output.factor_order - , tabbedSentence.GetColumns()[0] - , NULL); - - // word alignment between source and target sentence - // we don't store alignment info in AlignmentInfoCollection because we keep alignments of whole - // sentences, not phrases - AlignmentInfo *alignment = new AlignmentInfo(tabbedSentence.GetColumns()[1]); - - VWTargetSentence &targetSent = *GetStored(); - targetSent.Clear(); - targetSent.m_sentence = target; - targetSent.m_alignment = alignment; - - // pre-compute max- and min- aligned points for faster translation option checking - targetSent.SetConstraints(source.GetSize()); - } + // used in decoding to transform the global word alignment information into + // context-phrase internal alignment information (i.e., with target indices correspoding + // to positions in contextPhrase) + const AlignmentInfo *TransformAlignmentInfo(const Hypothesis &curHypo, size_t contextSize) const; + // used during training to extract relevant alignment points from the full sentence alignment + // and shift them by target context size + AlignmentInfo TransformAlignmentInfo(const AlignmentInfo &alignInfo, size_t contextSize, int currentStart) const; -private: - std::string MakeTargetLabel(const TargetPhrase &targetPhrase) const { - return VW_DUMMY_LABEL; - } + // At training time, determine whether a translation option is correct for the current target sentence + // based on word alignment. This is a bit complicated because we need to handle various corner-cases + // where some word(s) on phrase borders are unaligned. + std::pair<bool, int> IsCorrectTranslationOption(const TranslationOption &topt) const; - bool IsCorrectTranslationOption(const TranslationOption &topt) const { - - //std::cerr << topt.GetSourceWordsRange() << std::endl; - - int sourceStart = topt.GetSourceWordsRange().GetStartPos(); - int sourceEnd = topt.GetSourceWordsRange().GetEndPos(); - - const VWTargetSentence &targetSentence = *GetStored(); - - // [targetStart, targetEnd] spans aligned target words - int targetStart = targetSentence.m_sentence->GetSize(); - int targetEnd = -1; - - // get the left-most and right-most alignment point within source span - for(int i = sourceStart; i <= sourceEnd; ++i) { - if(targetSentence.m_sourceConstraints[i].IsSet()) { - if(targetStart > targetSentence.m_sourceConstraints[i].GetMin()) - targetStart = targetSentence.m_sourceConstraints[i].GetMin(); - if(targetEnd < targetSentence.m_sourceConstraints[i].GetMax()) - targetEnd = targetSentence.m_sourceConstraints[i].GetMax(); - } - } - // there was no alignment - if(targetEnd == -1) - return false; - - //std::cerr << "Shorter: " << targetStart << " " << targetEnd << std::endl; - - // [targetStart2, targetEnd2] spans unaligned words left and right of [targetStart, targetEnd] - int targetStart2 = targetStart; - for(int i = targetStart2; i >= 0 && !targetSentence.m_targetConstraints[i].IsSet(); --i) - targetStart2 = i; - - int targetEnd2 = targetEnd; - for(int i = targetEnd2; - i < targetSentence.m_sentence->GetSize() && !targetSentence.m_targetConstraints[i].IsSet(); - ++i) - targetEnd2 = i; - - //std::cerr << "Longer: " << targetStart2 << " " << targetEnd2 << std::endl; - - const TargetPhrase &tphrase = topt.GetTargetPhrase(); - //std::cerr << tphrase << std::endl; - - // if target phrase is shorter than inner span return false - if(tphrase.GetSize() < targetEnd - targetStart + 1) - return false; - - // if target phrase is longer than outer span return false - if(tphrase.GetSize() > targetEnd2 - targetStart2 + 1) - return false; - - // for each possible starting point - for(int tempStart = targetStart2; tempStart <= targetStart; tempStart++) { - bool found = true; - // check if the target phrase is within longer span - for(int i = tempStart; i <= targetEnd2 && i < tphrase.GetSize() + tempStart; ++i) { - if(tphrase.GetWord(i - tempStart) != targetSentence.m_sentence->GetWord(i)) { - found = false; - break; - } - } - // return true if there was a match - if(found) { - //std::cerr << "Found" << std::endl; - return true; - } - } - - return false; - } - - std::vector<bool> LeaveOneOut(const TranslationOptionList &topts, const std::vector<bool> &correct) const { - UTIL_THROW_IF2(m_leaveOneOut.size() == 0 || ! m_train, "LeaveOneOut called in wrong setting!"); - - float sourceRawCount = 0.0; - const float ONE = 1.0001; // I don't understand floating point numbers - - std::vector<bool> keepOpt; - - for (size_t i = 0; i < topts.size(); i++) { - TranslationOption *topt = *(topts.begin() + i); - const TargetPhrase &targetPhrase = topt->GetTargetPhrase(); - - // extract raw counts from phrase-table property - const CountsPhraseProperty *property = - static_cast<const CountsPhraseProperty *>(targetPhrase.GetProperty("Counts")); - - if (! property) { - VERBOSE(1, "VW :: Counts not found for topt! Is this an OOV?\n"); - // keep all translation opts without updating, this is either OOV or bad usage... - keepOpt.assign(topts.size(), true); - return keepOpt; - } - - if (sourceRawCount == 0.0) { - sourceRawCount = property->GetSourceMarginal() - ONE; // discount one occurrence of the source phrase - if (sourceRawCount <= 0) { - // no translation options survived, source phrase was a singleton - keepOpt.assign(topts.size(), false); - return keepOpt; - } - } - - float discount = correct[i] ? ONE : 0.0; - float target = property->GetTargetMarginal() - discount; - float joint = property->GetJointCount() - discount; - if (discount != 0.0) VERBOSE(2, "VW :: leaving one out!\n"); - - if (joint > 0) { - // topt survived leaving one out, update its scores - const FeatureFunction *feature = &FindFeatureFunction(m_leaveOneOut); - std::vector<float> scores = targetPhrase.GetScoreBreakdown().GetScoresForProducer(feature); - UTIL_THROW_IF2(scores.size() != 4, "Unexpected number of scores in feature " << m_leaveOneOut); - scores[0] = TransformScore(joint / target); // P(f|e) - scores[2] = TransformScore(joint / sourceRawCount); // P(e|f) - - ScoreComponentCollection &scoreBreakDown = topt->GetScoreBreakdown(); - scoreBreakDown.Assign(feature, scores); - topt->UpdateScore(); - keepOpt.push_back(true); - } else { - // they only occurred together once, discard topt - VERBOSE(2, "VW :: discarded topt when leaving one out\n"); - keepOpt.push_back(false); - } - } - - return keepOpt; - } + // At training time, optionally discount occurrences of phrase pairs from the current sentence, helps prevent + // over-fitting. + std::vector<bool> LeaveOneOut(const TranslationOptionList &topts, const std::vector<bool> &correct) const; bool m_train; // false means predict - std::string m_modelPath; - std::string m_vwOptions; + std::string m_modelPath; // path to the VW model file; at training time, this is where extracted features are stored + std::string m_vwOptions; // options for Vowpal Wabbit + + // BOS token, all factors + Word m_sentenceStartWord; // calculator of training loss TrainingLoss *m_trainingLoss = NULL; @@ -488,9 +179,16 @@ private: // optionally contains feature name of a phrase table where we recompute scores with leaving one out std::string m_leaveOneOut; + // normalizer, typically this means softmax Discriminative::Normalizer *m_normalizer = NULL; + + // thread-specific classifier instance TLSClassifier *m_tlsClassifier; + + // caches for partial scores and feature vectors + TLSFloatHashMap *m_tlsFutureScores; + TLSStateExtensions *m_tlsComputedStateExtensions; + TLSFeatureVectorMap *m_tlsTranslationOptionFeatures, *m_tlsTargetContextFeatures; }; } - diff --git a/moses/FF/VW/VWFeatureBase.cpp b/moses/FF/VW/VWFeatureBase.cpp index 874544203..e51396b3f 100644 --- a/moses/FF/VW/VWFeatureBase.cpp +++ b/moses/FF/VW/VWFeatureBase.cpp @@ -2,11 +2,26 @@ #include <string> #include "VWFeatureBase.h" +#include "VWFeatureContext.h" namespace Moses { std::map<std::string, std::vector<VWFeatureBase*> > VWFeatureBase::s_features; std::map<std::string, std::vector<VWFeatureBase*> > VWFeatureBase::s_sourceFeatures; +std::map<std::string, std::vector<VWFeatureBase*> > VWFeatureBase::s_targetContextFeatures; std::map<std::string, std::vector<VWFeatureBase*> > VWFeatureBase::s_targetFeatures; + +std::map<std::string, size_t> VWFeatureBase::s_targetContextLength; + + +void VWFeatureBase::UpdateContextSize(const std::string &usedBy) +{ + // using the standard map behavior here: if the entry does not + // exist, it will be added and initialized to zero + size_t currentSize = s_targetContextLength[usedBy]; + size_t newSize = static_cast<VWFeatureContext *const>(this)->GetContextSize(); + s_targetContextLength[usedBy] = std::max(currentSize, newSize); +} + } diff --git a/moses/FF/VW/VWFeatureBase.h b/moses/FF/VW/VWFeatureBase.h index c8bd60a81..ca3317d31 100644 --- a/moses/FF/VW/VWFeatureBase.h +++ b/moses/FF/VW/VWFeatureBase.h @@ -12,11 +12,17 @@ namespace Moses { +enum VWFeatureType { + vwft_source, + vwft_target, + vwft_targetContext +}; + class VWFeatureBase : public StatelessFeatureFunction { public: - VWFeatureBase(const std::string &line, bool isSource = true) - : StatelessFeatureFunction(0, line), m_usedBy(1, "VW0"), m_isSource(isSource) { + VWFeatureBase(const std::string &line, VWFeatureType featureType = vwft_source) + : StatelessFeatureFunction(0, line), m_usedBy(1, "VW0"), m_featureType(featureType) { // defaults m_sourceFactors.push_back(0); m_targetFactors.push_back(0); @@ -71,26 +77,47 @@ public: return s_sourceFeatures[name]; } + // Return only target-context classifier features + static const std::vector<VWFeatureBase*>& GetTargetContextFeatures(std::string name = "VW0") { + // don't throw an exception when there are no target-context features, this feature type is not mandatory + return s_targetContextFeatures[name]; + } + // Return only target-dependent classifier features static const std::vector<VWFeatureBase*>& GetTargetFeatures(std::string name = "VW0") { UTIL_THROW_IF2(s_targetFeatures.count(name) == 0, "No target features registered for parent classifier: " + name); return s_targetFeatures[name]; } + // Required length context (maximum context size of defined target-context features) + static size_t GetMaximumContextSize(std::string name = "VW0") { + return s_targetContextLength[name]; // 0 by default + } + // Overload to process source-dependent data, create features once for every // source sentence word range. virtual void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const = 0; + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const = 0; // Overload to process target-dependent features, create features once for - // every target phrase. One source word range will have at leat one target + // every target phrase. One source word range will have at least one target // phrase, but may have more. virtual void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const = 0; + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const = 0; + + // Overload to process target-context dependent features, these features are + // evaluated during decoding. For efficiency, features are not fed directly into + // the classifier object but instead output in the vector "features" and managed + // separately in VW.h. + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const = 0; protected: std::vector<FactorType> m_sourceFactors, m_targetFactors; @@ -99,10 +126,15 @@ protected: for(std::vector<std::string>::const_iterator it = m_usedBy.begin(); it != m_usedBy.end(); it++) { s_features[*it].push_back(this); - if(m_isSource) + + if(m_featureType == vwft_source) { s_sourceFeatures[*it].push_back(this); - else + } else if (m_featureType == vwft_targetContext) { + s_targetContextFeatures[*it].push_back(this); + UpdateContextSize(*it); + } else { s_targetFeatures[*it].push_back(this); + } } } @@ -112,11 +144,16 @@ private: Tokenize(m_usedBy, usedBy, ","); } + void UpdateContextSize(const std::string &usedBy); + std::vector<std::string> m_usedBy; - bool m_isSource; + VWFeatureType m_featureType; static std::map<std::string, std::vector<VWFeatureBase*> > s_features; static std::map<std::string, std::vector<VWFeatureBase*> > s_sourceFeatures; + static std::map<std::string, std::vector<VWFeatureBase*> > s_targetContextFeatures; static std::map<std::string, std::vector<VWFeatureBase*> > s_targetFeatures; + + static std::map<std::string, size_t> s_targetContextLength; }; } diff --git a/moses/FF/VW/VWFeatureContext.h b/moses/FF/VW/VWFeatureContext.h new file mode 100644 index 000000000..18632d91b --- /dev/null +++ b/moses/FF/VW/VWFeatureContext.h @@ -0,0 +1,116 @@ +#pragma once + +#include <string> +#include <boost/foreach.hpp> +#include "VWFeatureBase.h" +#include "moses/InputType.h" +#include "moses/TypeDef.h" +#include "moses/Word.h" + +namespace Moses +{ + +// Inherit from this for source-dependent classifier features. They will +// automatically register with the classifier class named VW0 or one or more +// names specified by the used-by=name1,name2,... parameter. +// +// The classifier gets a full list by calling +// VWFeatureBase::GetTargetContextFeatures(GetScoreProducerDescription()) + + +class VWFeatureContext : public VWFeatureBase +{ +public: + VWFeatureContext(const std::string &line, size_t contextSize) + : VWFeatureBase(line, vwft_targetContext), m_contextSize(contextSize) { + } + + // Gets its pure virtual functions from VWFeatureBase + + virtual void operator()(const InputType &input + , const TargetPhrase &targetPhrase + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + } + + virtual void operator()(const InputType &input + , const Range &sourceRange + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + } + + virtual void SetParameter(const std::string& key, const std::string& value) { + if (key == "size") { + m_contextSize = Scan<size_t>(value); + } else if (key == "factor-positions") { + // factor positions: assuming a factor such as positional morphological tag, use this + // option to select only certain positions; this assumes that only a single + // target-side factor is defined + Tokenize<size_t>(m_factorPositions, value, ","); + } else { + VWFeatureBase::SetParameter(key, value); + } + } + + size_t GetContextSize() { + return m_contextSize; + } + +protected: + // Get word with the correct subset of factors as string. Because we're target + // context features, we look at a limited number of words to the left of the + // current translation. posFromEnd is interpreted like this: + // 0 = last word of the hypothesis + // 1 = next to last word + // ...etc. + inline std::string GetWord(const Phrase &phrase, size_t posFromEnd) const { + const Word &word = phrase.GetWord(phrase.GetSize() - posFromEnd - 1); + if (m_factorPositions.empty()) { + return word.GetString(m_targetFactors, false); + } else { + if (m_targetFactors.size() != 1) + UTIL_THROW2("You can only use factor-positions when a single target-side factor is defined."); + const std::string &fullFactor = word.GetFactor(m_targetFactors[0])->GetString().as_string(); + + // corner cases: at sentence beginning/end, we don't have the correct factors set up + // similarly for UNK + if (fullFactor == BOS_ || fullFactor == EOS_ || fullFactor == UNKNOWN_FACTOR) + return fullFactor; + + std::string subFactor(m_factorPositions.size(), 'x'); // initialize string with correct size and placeholder chars + for (size_t i = 0; i < m_factorPositions.size(); i++) + subFactor[i] = fullFactor[m_factorPositions[i]]; + + return subFactor; + } + } + + // some target-context feature functions also look at the source + inline std::string GetSourceWord(const InputType &input, size_t pos) const { + return input.GetWord(pos).GetString(m_sourceFactors, false); + } + + // get source words aligned to a particular context word + std::vector<std::string> GetAlignedSourceWords(const Phrase &contextPhrase + , const InputType &input + , const AlignmentInfo &alignInfo + , size_t posFromEnd) const { + size_t idx = contextPhrase.GetSize() - posFromEnd - 1; + std::set<size_t> alignedToTarget = alignInfo.GetAlignmentsForTarget(idx); + std::vector<std::string> out; + out.reserve(alignedToTarget.size()); + BOOST_FOREACH(size_t srcIdx, alignedToTarget) { + out.push_back(GetSourceWord(input, srcIdx)); + } + return out; + } + + // required context size + size_t m_contextSize; + + // factor positions: assuming a factor such as positional morphological tag, use this + // option to select only certain positions + std::vector<size_t> m_factorPositions; +}; + +} diff --git a/moses/FF/VW/VWFeatureContextBigrams.h b/moses/FF/VW/VWFeatureContextBigrams.h new file mode 100644 index 000000000..92b652123 --- /dev/null +++ b/moses/FF/VW/VWFeatureContextBigrams.h @@ -0,0 +1,40 @@ +#pragma once + +#include <string> +#include <algorithm> +#include "VWFeatureContext.h" +#include "moses/Util.h" + +namespace Moses +{ + +class VWFeatureContextBigrams : public VWFeatureContext +{ +public: + VWFeatureContextBigrams(const std::string &line) + : VWFeatureContext(line, DEFAULT_WINDOW_SIZE) { + ReadParameters(); + + // Call this last + VWFeatureBase::UpdateRegister(); + } + + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + for (size_t i = 1; i < m_contextSize; i++) + outFeatures.push_back(classifier.AddLabelIndependentFeature("tcbigram^-" + SPrint(i + 1) + + "^" + GetWord(contextPhrase, i - 1) + "^" + GetWord(contextPhrase, i))); + } + + virtual void SetParameter(const std::string& key, const std::string& value) { + VWFeatureContext::SetParameter(key, value); + } + +private: + static const int DEFAULT_WINDOW_SIZE = 1; +}; + +} diff --git a/moses/FF/VW/VWFeatureContextBilingual.h b/moses/FF/VW/VWFeatureContextBilingual.h new file mode 100644 index 000000000..f681fcb78 --- /dev/null +++ b/moses/FF/VW/VWFeatureContextBilingual.h @@ -0,0 +1,45 @@ +#pragma once + +#include <string> +#include <boost/foreach.hpp> +#include <algorithm> +#include "VWFeatureContext.h" +#include "moses/Util.h" + +namespace Moses +{ + +class VWFeatureContextBilingual : public VWFeatureContext +{ +public: + VWFeatureContextBilingual(const std::string &line) + : VWFeatureContext(line, DEFAULT_WINDOW_SIZE) { + ReadParameters(); + + // Call this last + VWFeatureBase::UpdateRegister(); + } + + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + for (size_t i = 0; i < m_contextSize; i++) { + std::string tgtWord = GetWord(contextPhrase, i); + std::vector<std::string> alignedTo = GetAlignedSourceWords(contextPhrase, input, alignmentInfo, i); + BOOST_FOREACH(const std::string &srcWord, alignedTo) { + outFeatures.push_back(classifier.AddLabelIndependentFeature("tcblng^-" + SPrint(i + 1) + "^" + tgtWord + "^" + srcWord)); + } + } + } + + virtual void SetParameter(const std::string& key, const std::string& value) { + VWFeatureContext::SetParameter(key, value); + } + +private: + static const int DEFAULT_WINDOW_SIZE = 1; +}; + +} diff --git a/moses/FF/VW/VWFeatureContextWindow.h b/moses/FF/VW/VWFeatureContextWindow.h new file mode 100644 index 000000000..66c9c3ec5 --- /dev/null +++ b/moses/FF/VW/VWFeatureContextWindow.h @@ -0,0 +1,39 @@ +#pragma once + +#include <string> +#include <algorithm> +#include "VWFeatureContext.h" +#include "moses/Util.h" + +namespace Moses +{ + +class VWFeatureContextWindow : public VWFeatureContext +{ +public: + VWFeatureContextWindow(const std::string &line) + : VWFeatureContext(line, DEFAULT_WINDOW_SIZE) { + ReadParameters(); + + // Call this last + VWFeatureBase::UpdateRegister(); + } + + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + for (size_t i = 0; i < m_contextSize; i++) + outFeatures.push_back(classifier.AddLabelIndependentFeature("tcwin^-" + SPrint(i + 1) + "^" + GetWord(contextPhrase, i))); + } + + virtual void SetParameter(const std::string& key, const std::string& value) { + VWFeatureContext::SetParameter(key, value); + } + +private: + static const int DEFAULT_WINDOW_SIZE = 1; +}; + +} diff --git a/moses/FF/VW/VWFeatureSource.h b/moses/FF/VW/VWFeatureSource.h index 564f4a3b6..7a306b59c 100644 --- a/moses/FF/VW/VWFeatureSource.h +++ b/moses/FF/VW/VWFeatureSource.h @@ -19,15 +19,22 @@ class VWFeatureSource : public VWFeatureBase { public: VWFeatureSource(const std::string &line) - : VWFeatureBase(line, true) { + : VWFeatureBase(line, vwft_source) { } // Gets its pure virtual functions from VWFeatureBase virtual void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + } + + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { } virtual void SetParameter(const std::string& key, const std::string& value) { diff --git a/moses/FF/VW/VWFeatureSourceBagOfWords.h b/moses/FF/VW/VWFeatureSourceBagOfWords.h index 97a1cc6c3..b815b4d0e 100644 --- a/moses/FF/VW/VWFeatureSourceBagOfWords.h +++ b/moses/FF/VW/VWFeatureSourceBagOfWords.h @@ -18,11 +18,11 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { for (size_t i = 0; i < input.GetSize(); i++) { - classifier.AddLabelIndependentFeature("bow^" + GetWord(input, i)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("bow^" + GetWord(input, i))); } } diff --git a/moses/FF/VW/VWFeatureSourceBigrams.h b/moses/FF/VW/VWFeatureSourceBigrams.h index ce5430ab8..5de3ab2c3 100644 --- a/moses/FF/VW/VWFeatureSourceBigrams.h +++ b/moses/FF/VW/VWFeatureSourceBigrams.h @@ -18,11 +18,11 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { for (size_t i = 1; i < input.GetSize(); i++) { - classifier.AddLabelIndependentFeature("bigram^" + GetWord(input, i - 1) + "^" + GetWord(input, i)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("bigram^" + GetWord(input, i - 1) + "^" + GetWord(input, i))); } } diff --git a/moses/FF/VW/VWFeatureSourceExternalFeatures.h b/moses/FF/VW/VWFeatureSourceExternalFeatures.h index bacc5d231..9995ad1b2 100644 --- a/moses/FF/VW/VWFeatureSourceExternalFeatures.h +++ b/moses/FF/VW/VWFeatureSourceExternalFeatures.h @@ -23,12 +23,12 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { const Features& features = *m_tls.GetStored(); for (size_t i = 0; i < features.size(); i++) { - classifier.AddLabelIndependentFeature("srcext^" + features[i]); + outFeatures.push_back(classifier.AddLabelIndependentFeature("srcext^" + features[i])); } } diff --git a/moses/FF/VW/VWFeatureSourceIndicator.h b/moses/FF/VW/VWFeatureSourceIndicator.h index fda929f13..b0d43eb0f 100644 --- a/moses/FF/VW/VWFeatureSourceIndicator.h +++ b/moses/FF/VW/VWFeatureSourceIndicator.h @@ -20,9 +20,9 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { size_t begin = sourceRange.GetStartPos(); size_t end = sourceRange.GetEndPos() + 1; @@ -31,7 +31,7 @@ public: for (size_t i = 0; i < end - begin; i++) words[i] = GetWord(input, begin + i); - classifier.AddLabelIndependentFeature("sind^" + Join(" ", words)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("sind^" + Join(" ", words))); } virtual void SetParameter(const std::string& key, const std::string& value) { diff --git a/moses/FF/VW/VWFeatureSourcePhraseInternal.h b/moses/FF/VW/VWFeatureSourcePhraseInternal.h index 4e7f6e8d1..b346660a0 100644 --- a/moses/FF/VW/VWFeatureSourcePhraseInternal.h +++ b/moses/FF/VW/VWFeatureSourcePhraseInternal.h @@ -20,14 +20,14 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { size_t begin = sourceRange.GetStartPos(); size_t end = sourceRange.GetEndPos() + 1; while (begin < end) { - classifier.AddLabelIndependentFeature("sin^" + GetWord(input, begin++)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("sin^" + GetWord(input, begin++))); } } diff --git a/moses/FF/VW/VWFeatureSourceSenseWindow.h b/moses/FF/VW/VWFeatureSourceSenseWindow.h index 614f7ff52..e7b1e1a71 100644 --- a/moses/FF/VW/VWFeatureSourceSenseWindow.h +++ b/moses/FF/VW/VWFeatureSourceSenseWindow.h @@ -51,9 +51,9 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { int begin = sourceRange.GetStartPos(); int end = sourceRange.GetEndPos() + 1; int inputLen = input.GetSize(); @@ -64,24 +64,24 @@ public: // before current phrase for (int i = std::max(0, begin - m_size); i < begin; i++) { BOOST_FOREACH(const Sense &sense, senses[i]) { - classifier.AddLabelIndependentFeature("snsb^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); - classifier.AddLabelIndependentFeature("snsb^" + forms[i] + sense.m_label, sense.m_prob); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsb^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsb^" + forms[i] + sense.m_label, sense.m_prob)); } } // within current phrase for (int i = begin; i < end; i++) { BOOST_FOREACH(const Sense &sense, senses[i]) { - classifier.AddLabelIndependentFeature("snsin^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); - classifier.AddLabelIndependentFeature("snsin^" + forms[i] + sense.m_label, sense.m_prob); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsin^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsin^" + forms[i] + sense.m_label, sense.m_prob)); } } // after current phrase for (int i = end; i < std::min(end + m_size, inputLen); i++) { BOOST_FOREACH(const Sense &sense, senses[i]) { - classifier.AddLabelIndependentFeature("snsa^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob); - classifier.AddLabelIndependentFeature("snsa^" + forms[i] + sense.m_label, sense.m_prob); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsa^" + forms[i] + SPrint(i - begin) + "^" + sense.m_label, sense.m_prob)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("snsa^" + forms[i] + sense.m_label, sense.m_prob)); } } } diff --git a/moses/FF/VW/VWFeatureSourceWindow.h b/moses/FF/VW/VWFeatureSourceWindow.h index 5205e4f2f..14c617586 100644 --- a/moses/FF/VW/VWFeatureSourceWindow.h +++ b/moses/FF/VW/VWFeatureSourceWindow.h @@ -20,19 +20,19 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { int begin = sourceRange.GetStartPos(); int end = sourceRange.GetEndPos() + 1; int inputLen = input.GetSize(); for (int i = std::max(0, begin - m_size); i < begin; i++) { - classifier.AddLabelIndependentFeature("c^" + SPrint(i - begin) + "^" + GetWord(input, i)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("c^" + SPrint(i - begin) + "^" + GetWord(input, i))); } for (int i = end; i < std::min(end + m_size, inputLen); i++) { - classifier.AddLabelIndependentFeature("c^" + SPrint(i - end + 1) + "^" + GetWord(input, i)); + outFeatures.push_back(classifier.AddLabelIndependentFeature("c^" + SPrint(i - end + 1) + "^" + GetWord(input, i))); } } diff --git a/moses/FF/VW/VWFeatureTarget.h b/moses/FF/VW/VWFeatureTarget.h index 2935b2b4e..ed936ebf3 100644 --- a/moses/FF/VW/VWFeatureTarget.h +++ b/moses/FF/VW/VWFeatureTarget.h @@ -17,15 +17,22 @@ class VWFeatureTarget : public VWFeatureBase { public: VWFeatureTarget(const std::string &line) - : VWFeatureBase(line, false) { + : VWFeatureBase(line, vwft_target) { } // Gets its pure virtual functions from VWFeatureBase virtual void operator()(const InputType &input - , const InputPath &inputPath , const Range &sourceRange - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + } + + virtual void operator()(const InputType &input + , const Phrase &contextPhrase + , const AlignmentInfo &alignmentInfo + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { } virtual void SetParameter(const std::string& key, const std::string& value) { diff --git a/moses/FF/VW/VWFeatureTargetBigrams.h b/moses/FF/VW/VWFeatureTargetBigrams.h index 6f3f35270..30264dbf5 100644 --- a/moses/FF/VW/VWFeatureTargetBigrams.h +++ b/moses/FF/VW/VWFeatureTargetBigrams.h @@ -17,11 +17,11 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { for (size_t i = 1; i < targetPhrase.GetSize(); i++) { - classifier.AddLabelDependentFeature("tbigram^" + GetWord(targetPhrase, i - 1) + "^" + GetWord(targetPhrase, i)); + outFeatures.push_back(classifier.AddLabelDependentFeature("tbigram^" + GetWord(targetPhrase, i - 1) + "^" + GetWord(targetPhrase, i))); } } diff --git a/moses/FF/VW/VWFeatureTargetIndicator.h b/moses/FF/VW/VWFeatureTargetIndicator.h index 39d8a37a0..0195990d0 100644 --- a/moses/FF/VW/VWFeatureTargetIndicator.h +++ b/moses/FF/VW/VWFeatureTargetIndicator.h @@ -17,10 +17,10 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const { - classifier.AddLabelDependentFeature("tind^" + targetPhrase.GetStringRep(m_targetFactors)); + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { + outFeatures.push_back(classifier.AddLabelDependentFeature("tind^" + targetPhrase.GetStringRep(m_targetFactors))); } virtual void SetParameter(const std::string& key, const std::string& value) { diff --git a/moses/FF/VW/VWFeatureTargetPhraseInternal.h b/moses/FF/VW/VWFeatureTargetPhraseInternal.h index e376a1ed3..8a9928aaa 100644 --- a/moses/FF/VW/VWFeatureTargetPhraseInternal.h +++ b/moses/FF/VW/VWFeatureTargetPhraseInternal.h @@ -17,11 +17,11 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { for (size_t i = 0; i < targetPhrase.GetSize(); i++) { - classifier.AddLabelDependentFeature("tin^" + GetWord(targetPhrase, i)); + outFeatures.push_back(classifier.AddLabelDependentFeature("tin^" + GetWord(targetPhrase, i))); } } diff --git a/moses/FF/VW/VWFeatureTargetPhraseScores.h b/moses/FF/VW/VWFeatureTargetPhraseScores.h index 5a4519fb1..6c9ab63d2 100644 --- a/moses/FF/VW/VWFeatureTargetPhraseScores.h +++ b/moses/FF/VW/VWFeatureTargetPhraseScores.h @@ -20,9 +20,9 @@ public: } void operator()(const InputType &input - , const InputPath &inputPath , const TargetPhrase &targetPhrase - , Discriminative::Classifier &classifier) const { + , Discriminative::Classifier &classifier + , Discriminative::FeatureVector &outFeatures) const { std::vector<FeatureFunction*> features = FeatureFunction::GetFeatureFunctions(); for (size_t i = 0; i < features.size(); i++) { std::string fname = features[i]->GetScoreProducerDescription(); @@ -31,7 +31,7 @@ public: std::vector<float> scores = targetPhrase.GetScoreBreakdown().GetScoresForProducer(features[i]); for(size_t j = 0; j < scores.size(); ++j) - classifier.AddLabelDependentFeature(fname + "^" + boost::lexical_cast<std::string>(j), scores[j]); + outFeatures.push_back(classifier.AddLabelDependentFeature(fname + "^" + boost::lexical_cast<std::string>(j), scores[j])); } } diff --git a/moses/FF/VW/VWState.cpp b/moses/FF/VW/VWState.cpp new file mode 100644 index 000000000..000b8532b --- /dev/null +++ b/moses/FF/VW/VWState.cpp @@ -0,0 +1,77 @@ +#include "VWState.h" + +#include "moses/FF/FFState.h" +#include "moses/Phrase.h" +#include "moses/Hypothesis.h" +#include "moses/Util.h" +#include "moses/TypeDef.h" +#include "moses/StaticData.h" +#include "moses/TranslationOption.h" +#include <boost/functional/hash.hpp> + +namespace Moses +{ + +VWState::VWState() : m_spanStart(0), m_spanEnd(0) +{ + ComputeHash(); +} + +VWState::VWState(const Phrase &phrase) + : m_phrase(phrase), m_spanStart(0), m_spanEnd(0) +{ + ComputeHash(); +} + +VWState::VWState(const VWState &prevState, const Hypothesis &curHypo) +{ + VERBOSE(3, "VW :: updating state\n>> previous state: " << prevState << "\n"); + + // copy phrase from previous state + Phrase phrase = prevState.GetPhrase(); + size_t contextSize = phrase.GetSize(); // identical to VWFeatureBase::GetMaximumContextSize() + + // add words from current hypothesis + phrase.Append(curHypo.GetCurrTargetPhrase()); + + VERBOSE(3, ">> current hypo: " << curHypo.GetCurrTargetPhrase() << "\n"); + + // get a slice of appropriate length + Range range(phrase.GetSize() - contextSize, phrase.GetSize() - 1); + m_phrase = phrase.GetSubString(range); + + // set current span start/end + m_spanStart = curHypo.GetTranslationOption().GetStartPos(); + m_spanEnd = curHypo.GetTranslationOption().GetEndPos(); + + // compute our hash + ComputeHash(); + + VERBOSE(3, ">> updated state: " << *this << "\n"); +} + +bool VWState::operator==(const FFState& o) const +{ + const VWState &other = static_cast<const VWState &>(o); + + return m_phrase == other.GetPhrase() + && m_spanStart == other.GetSpanStart() + && m_spanEnd == other.GetSpanEnd(); +} + +void VWState::ComputeHash() +{ + m_hash = 0; + + boost::hash_combine(m_hash, m_phrase); + boost::hash_combine(m_hash, m_spanStart); + boost::hash_combine(m_hash, m_spanEnd); +} + +std::ostream &operator<<(std::ostream &out, const VWState &state) +{ + out << state.GetPhrase() << "::" << state.GetSpanStart() << "-" << state.GetSpanEnd(); + return out; +} + +} diff --git a/moses/FF/VW/VWState.h b/moses/FF/VW/VWState.h new file mode 100644 index 000000000..d83035553 --- /dev/null +++ b/moses/FF/VW/VWState.h @@ -0,0 +1,56 @@ +#pragma once + +#include <ostream> + +#include "moses/FF/FFState.h" +#include "moses/Phrase.h" +#include "moses/Hypothesis.h" + +namespace Moses +{ + +/** + * VW state, used in decoding (when target context is enabled). + */ +class VWState : public FFState +{ +public: + // empty state, used only when VWState is ignored + VWState(); + + // used for construction of the initial VW state + VWState(const Phrase &phrase); + + // continue from previous VW state with a new hypothesis + VWState(const VWState &prevState, const Hypothesis &curHypo); + + virtual bool operator==(const FFState& o) const; + + inline virtual size_t hash() const { + return m_hash; + } + + inline const Phrase &GetPhrase() const { + return m_phrase; + } + + inline size_t GetSpanStart() const { + return m_spanStart; + } + + inline size_t GetSpanEnd() const { + return m_spanEnd; + } + +private: + void ComputeHash(); + + Phrase m_phrase; + size_t m_spanStart, m_spanEnd; + size_t m_hash; +}; + +// how to print a VW state +std::ostream &operator<<(std::ostream &out, const VWState &state); + +} diff --git a/moses/FF/VW/VWTargetSentence.h b/moses/FF/VW/VWTargetSentence.h new file mode 100644 index 000000000..1387bc042 --- /dev/null +++ b/moses/FF/VW/VWTargetSentence.h @@ -0,0 +1,55 @@ +#pragma once + +#include <vector> + +#include "moses/AlignmentInfo.h" +#include "moses/Phrase.h" + +#include "AlignmentConstraint.h" + +namespace Moses +{ + +/** + * VW thread-specific data about target sentence. + */ +class VWTargetSentence +{ +public: + VWTargetSentence() : m_sentence(NULL), m_alignment(NULL) {} + + void Clear() { + if (m_sentence) delete m_sentence; + if (m_alignment) delete m_alignment; + } + + ~VWTargetSentence() { + Clear(); + } + + void SetConstraints(size_t sourceSize) { + // initialize to unconstrained + m_sourceConstraints.assign(sourceSize, AlignmentConstraint()); + m_targetConstraints.assign(m_sentence->GetSize(), AlignmentConstraint()); + + // set constraints according to alignment points + AlignmentInfo::const_iterator it; + for (it = m_alignment->begin(); it != m_alignment->end(); it++) { + int src = it->first; + int tgt = it->second; + + if (src >= m_sourceConstraints.size() || tgt >= m_targetConstraints.size()) { + UTIL_THROW2("VW :: alignment point out of bounds: " << src << "-" << tgt); + } + + m_sourceConstraints[src].Update(tgt); + m_targetConstraints[tgt].Update(src); + } + } + + Phrase *m_sentence; + AlignmentInfo *m_alignment; + std::vector<AlignmentConstraint> m_sourceConstraints, m_targetConstraints; +}; + +} diff --git a/moses/FactorCollection.h b/moses/FactorCollection.h index 5d94e1197..c2e3362aa 100644 --- a/moses/FactorCollection.h +++ b/moses/FactorCollection.h @@ -41,6 +41,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "util/pool.hh" #include "Factor.h" +class System; + namespace Moses { @@ -66,6 +68,7 @@ struct FactorFriend { class FactorCollection { friend std::ostream& operator<<(std::ostream&, const FactorCollection&); + friend class ::System; struct HashFactor : public std::unary_function<const FactorFriend &, std::size_t> { std::size_t operator()(const FactorFriend &factor) const { diff --git a/moses/Hypothesis.cpp b/moses/Hypothesis.cpp index 519042577..0dc591ab3 100644 --- a/moses/Hypothesis.cpp +++ b/moses/Hypothesis.cpp @@ -43,6 +43,8 @@ using namespace std; namespace Moses { +//size_t g_numHypos = 0; + Hypothesis:: Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt, const Bitmap &bitmap, int id) : m_prevHypo(NULL) @@ -61,6 +63,7 @@ Hypothesis(Manager& manager, InputType const& source, const TranslationOption &i , m_manager(manager) , m_id(id) { +// ++g_numHypos; // used for initial seeding of trans process // initialize scores //_hash_computed = false; @@ -91,6 +94,8 @@ Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt, const , m_manager(prevHypo.GetManager()) , m_id(id) { +// ++g_numHypos; + m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown()); m_wordDeleted = transOpt.IsDeletionOption(); } diff --git a/moses/InputType.h b/moses/InputType.h index 80080a3f2..ce25cbc96 100644 --- a/moses/InputType.h +++ b/moses/InputType.h @@ -68,6 +68,9 @@ public: size_t m_frontSpanCoveredLength; // how many words from the beginning are covered + // Coordinates in user-defined spaces (see "coord" XML tag) + SPTR<std::map<size_t const, std::vector<float> > > m_coordMap; + InputType(AllOptions::ptr const& opts, long translationId = 0); virtual ~InputType(); diff --git a/moses/Jamfile b/moses/Jamfile index f7e05b85b..49aab9025 100644 --- a/moses/Jamfile +++ b/moses/Jamfile @@ -109,6 +109,7 @@ lib moses : FF/*.cpp FF/bilingual-lm/*.cpp FF/OSM-Feature/*.cpp + FF/Dsg-Feature/*.cpp FF/LexicalReordering/*.cpp PP/*.cpp : #exceptions diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index 8de121f2f..964fb90b5 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -1,4 +1,4 @@ -// $Id$ +// $Id$\ /*********************************************************************** Moses - factored phrase-based language decoder @@ -93,6 +93,8 @@ private: template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string &file, util::LoadMethod load_method) { + m_lmIdLookup.clear(); + lm::ngram::Config config; if(this->m_verbosity >= 1) { config.messages = &std::cerr; @@ -105,13 +107,12 @@ template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string config.load_method = load_method; m_ngram.reset(new Model(file.c_str(), config)); - - m_beginSentenceFactor = collection.AddFactor(BOS_); } template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method) :LanguageModel(line) ,m_factorType(factorType) + ,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_)) { ReadParameters(); LoadModel(file, load_method); diff --git a/moses/LM/PointerState.h b/moses/LM/PointerState.h index 8bc615c40..c0e59bb45 100644 --- a/moses/LM/PointerState.h +++ b/moses/LM/PointerState.h @@ -7,6 +7,11 @@ namespace Moses struct PointerState : public FFState { const void* lmstate; + + explicit PointerState() { + // uninitialised + } + PointerState(const void* lms) { lmstate = lms; } diff --git a/moses/PP/TargetConstituentBoundariesLeftPhraseProperty.cpp b/moses/PP/TargetConstituentBoundariesLeftPhraseProperty.cpp index b39c79036..b23126c6c 100644 --- a/moses/PP/TargetConstituentBoundariesLeftPhraseProperty.cpp +++ b/moses/PP/TargetConstituentBoundariesLeftPhraseProperty.cpp @@ -1,3 +1,4 @@ +#include <set> #include "moses/PP/TargetConstituentBoundariesLeftPhraseProperty.h" #include "moses/FactorCollection.h" #include "moses/Util.h" diff --git a/moses/PP/TargetConstituentBoundariesRightAdjacentPhraseProperty.cpp b/moses/PP/TargetConstituentBoundariesRightAdjacentPhraseProperty.cpp index 268f145cf..198ecfa38 100644 --- a/moses/PP/TargetConstituentBoundariesRightAdjacentPhraseProperty.cpp +++ b/moses/PP/TargetConstituentBoundariesRightAdjacentPhraseProperty.cpp @@ -3,6 +3,7 @@ #include "moses/Util.h" #include <iostream> #include <queue> +#include <set> #include <ostream> namespace Moses diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp index 348f8bb31..a1430ff61 100644 --- a/moses/Parameter.cpp +++ b/moses/Parameter.cpp @@ -59,6 +59,7 @@ Parameter::Parameter() AddParam(main_opts,"version", "show version of Moses and libraries used"); AddParam(main_opts,"show-weights", "print feature weights and exit"); AddParam(main_opts,"time-out", "seconds after which is interrupted (-1=no time-out, default is -1)"); + AddParam(main_opts,"segment-time-out", "seconds for single segment after which is interrupted (-1=no time-out, default is -1)"); /////////////////////////////////////////////////////////////////////////////////////// // factorization options @@ -263,7 +264,6 @@ Parameter::Parameter() AddParam(misc_opts,"references", "Reference file(s) - used for bleu score feature"); AddParam(misc_opts,"recover-input-path", "r", "(conf net/word lattice only) - recover input path corresponding to the best translation"); AddParam(misc_opts,"link-param-count", "Number of parameters on word links when using confusion networks or lattices (default = 1)"); - AddParam(misc_opts,"description", "Source language, target language, description"); AddParam(misc_opts,"feature-name-overwrite", "Override feature name (NOT arguments). Eg. SRILM-->KENLM, PhraseDictionaryMemory-->PhraseDictionaryScope3"); AddParam(misc_opts,"feature", "All the feature functions should be here"); @@ -1015,9 +1015,7 @@ ConvertWeightArgsLM() + "order=" + modelToks[2] + " " // order + "num-features=" + SPrint(numFF) + " "; if (lmType == 9) { - featureLine += "lazyken=1 "; - } else if (lmType == 8) { - featureLine += "lazyken=0 "; + featureLine += "load=lazy "; } if(oovWeights.size() > lmIndex) diff --git a/moses/ReorderingConstraint.cpp b/moses/ReorderingConstraint.cpp index a5627508f..c4950daad 100644 --- a/moses/ReorderingConstraint.cpp +++ b/moses/ReorderingConstraint.cpp @@ -54,8 +54,8 @@ void ReorderingConstraint::SetWall( size_t pos, bool value ) void ReorderingConstraint::FinalizeWalls() { for(size_t z = 0; z < m_zone.size(); z++ ) { - const size_t startZone = m_zone[z][0]; - const size_t endZone = m_zone[z][1];// note: wall after endZone is not local + const size_t startZone = m_zone[z].first; + const size_t endZone = m_zone[z].second;// note: wall after endZone is not local for( size_t pos = startZone; pos < endZone; pos++ ) { if (m_wall[ pos ]) { m_localWall[ pos ] = z; @@ -65,8 +65,8 @@ void ReorderingConstraint::FinalizeWalls() // enforce that local walls only apply to innermost zone else if (m_localWall[ pos ] != NOT_A_ZONE) { size_t assigned_z = m_localWall[ pos ]; - if ((m_zone[assigned_z][0] < startZone) || - (m_zone[assigned_z][1] > endZone)) { + if ((m_zone[assigned_z].first < startZone) || + (m_zone[assigned_z].second > endZone)) { m_localWall[ pos ] = z; } } @@ -97,9 +97,9 @@ void ReorderingConstraint::SetMonotoneAtPunctuation( const Phrase &sentence ) void ReorderingConstraint::SetZone( size_t startPos, size_t endPos ) { VERBOSE(3,"SETTING zone " << startPos << "-" << endPos << std::endl); - std::vector< size_t > newZone; - newZone.push_back( startPos ); - newZone.push_back( endPos ); + std::pair<size_t,size_t> newZone; + newZone.first = startPos; + newZone.second = endPos; m_zone.push_back( newZone ); m_active = true; } @@ -138,8 +138,8 @@ bool ReorderingConstraint::Check( const Bitmap &bitmap, size_t startPos, size_t // check zones for(size_t z = 0; z < m_zone.size(); z++ ) { - const size_t startZone = m_zone[z][0]; - const size_t endZone = m_zone[z][1]; + const size_t startZone = m_zone[z].first; + const size_t endZone = m_zone[z].second; // fine, if translation has not reached zone yet and phrase outside zone if (lastPos < startZone && ( endPos < startZone || startPos > endZone ) ) { @@ -236,4 +236,25 @@ bool ReorderingConstraint::Check( const Bitmap &bitmap, size_t startPos, size_t return true; } +std::ostream& operator<<(std::ostream& out, const ReorderingConstraint &obj) +{ + out << "Zones:"; + for (size_t i = 0; i < obj.m_zone.size(); ++i) { + const std::pair<size_t,size_t> &zone1 = obj.m_zone[i]; + out << zone1.first << "-" << zone1.second << " "; + } + + out << "Walls:"; + for (size_t i = 0; i < obj.m_size; ++i) { + out << obj.m_wall[i]; + } + + out << " Local walls:"; + for (size_t i = 0; i < obj.m_size; ++i) { + out << obj.m_localWall[i] << " "; + } + + return out; +} + } diff --git a/moses/ReorderingConstraint.h b/moses/ReorderingConstraint.h index fc74dea7d..047382076 100644 --- a/moses/ReorderingConstraint.h +++ b/moses/ReorderingConstraint.h @@ -45,13 +45,13 @@ class Bitmap; */ class ReorderingConstraint { - friend std::ostream& operator<<(std::ostream& out, const ReorderingConstraint& reorderingConstraint); + friend std::ostream& operator<<(std::ostream& out, const ReorderingConstraint &obj); protected: // const size_t m_size; /**< number of words in sentence */ size_t m_size; /**< number of words in sentence */ bool *m_wall; /**< flag for each word if it is a wall */ size_t *m_localWall; /**< flag for each word if it is a local wall */ - std::vector< std::vector< size_t > > m_zone; /** zones that limit reordering */ + std::vector< std::pair<size_t,size_t> > m_zone; /** zones that limit reordering */ bool m_active; /**< flag indicating, if there are any active constraints */ int m_max_distortion; public: @@ -93,7 +93,7 @@ public: void SetZone( size_t startPos, size_t endPos ); //! returns the vector of zones - std::vector< std::vector< size_t > > & GetZones() { + std::vector< std::pair<size_t,size_t> > & GetZones() { return m_zone; } diff --git a/moses/RuleCube.cpp b/moses/RuleCube.cpp index 3808cf2cd..874a1f57d 100644 --- a/moses/RuleCube.cpp +++ b/moses/RuleCube.cpp @@ -33,7 +33,6 @@ using namespace std; namespace Moses { -extern bool g_mosesDebug; // initialise the RuleCube by creating the top-left corner item RuleCube::RuleCube(const ChartTranslationOptions &transOpt, diff --git a/moses/Search.cpp b/moses/Search.cpp index 2d8c74b5f..caf9425cf 100644 --- a/moses/Search.cpp +++ b/moses/Search.cpp @@ -17,21 +17,34 @@ Search::Search(Manager& manager) , interrupted_flag(0) { m_initialTransOpt.SetInputPath(m_inputPath); + m_timer.start(); } - bool Search:: out_of_time() { int const& timelimit = m_options.search.timeout; - if (!timelimit) return false; - double elapsed_time = GetUserTime(); - if (elapsed_time <= timelimit) return false; - VERBOSE(1,"Decoding is out of time (" << elapsed_time << "," - << timelimit << ")" << std::endl); - interrupted_flag = 1; - return true; + if (timelimit > 0) { + double elapsed_time = GetUserTime(); + if (elapsed_time > timelimit) { + VERBOSE(1,"Decoding is out of time (" << elapsed_time << "," + << timelimit << ")" << std::endl); + interrupted_flag = 1; + return true; + } + } + int const& segment_timelimit = m_options.search.segment_timeout; + if (segment_timelimit > 0) { + double elapsed_time = m_timer.get_elapsed_time(); + if (elapsed_time > segment_timelimit) { + VERBOSE(1,"Decoding for segment is out of time (" << elapsed_time << "," + << segment_timelimit << ")" << std::endl); + interrupted_flag = 1; + return true; + } + } + return false; } } diff --git a/moses/Search.h b/moses/Search.h index a0e07870d..7797f07a0 100644 --- a/moses/Search.h +++ b/moses/Search.h @@ -7,6 +7,7 @@ #include "Phrase.h" #include "InputPath.h" #include "Bitmaps.h" +#include "Timer.h" namespace Moses { @@ -48,6 +49,7 @@ protected: /** flag indicating that decoder ran out of time (see switch -time-out) */ size_t interrupted_flag; + Timer m_timer; bool out_of_time(); }; diff --git a/moses/SearchCubePruning.cpp b/moses/SearchCubePruning.cpp index 9984ecadb..f921b9860 100644 --- a/moses/SearchCubePruning.cpp +++ b/moses/SearchCubePruning.cpp @@ -97,7 +97,6 @@ void SearchCubePruning::Decode() // go through each stack size_t stackNo = 1; - int timelimit = m_options.search.timeout; std::vector < HypothesisStack* >::iterator iterStack; for (iterStack = m_hypoStackColl.begin() + 1 ; iterStack != m_hypoStackColl.end() ; ++iterStack) { // BOOST_FOREACH(HypothesisStack* hstack, m_hypoStackColl) { diff --git a/moses/SearchNormal.cpp b/moses/SearchNormal.cpp index bed4edc5d..7bd89a7f2 100644 --- a/moses/SearchNormal.cpp +++ b/moses/SearchNormal.cpp @@ -240,6 +240,10 @@ ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos const Bitmap &sourceCompleted = hypothesis.GetWordsBitmap(); float estimatedScore = m_transOptColl.GetEstimatedScores().CalcEstimatedScore( sourceCompleted, startPos, endPos ); + const Range &hypoRange = hypothesis.GetCurrSourceWordsRange(); + //cerr << "DOING " << sourceCompleted << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]" + // " [" << startPos << " " << endPos << "]" << endl; + if (m_options.search.UseEarlyDiscarding()) { // expected score is based on score of current hypothesis expectedScore = hypothesis.GetScore(); diff --git a/moses/Sentence.cpp b/moses/Sentence.cpp index 4db022e5e..b7e24a85f 100644 --- a/moses/Sentence.cpp +++ b/moses/Sentence.cpp @@ -154,8 +154,11 @@ aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls, bool OK = ProcessAndStripXMLTags(*m_options, line, m_xmlOptions, m_reorderingConstraint, - xmlWalls, placeholders); - UTIL_THROW_IF2(!OK, "Unable to parse XML in line: " << line); + xmlWalls, placeholders, + *this); + if (!OK) { + TRACE_ERR("Unable to parse XML in line: " << line); + } } } diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index e15e1b355..85268a7b2 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -62,8 +62,6 @@ using namespace boost::algorithm; namespace Moses { -bool g_mosesDebug = false; - StaticData StaticData::s_instance; StaticData::StaticData() @@ -71,6 +69,7 @@ StaticData::StaticData() , m_requireSortingAfterSourceContext(false) , m_currentWeightSetting("default") , m_treeStructure(NULL) + , m_coordSpaceNextID(1) { Phrase::InitializeMemPool(); } @@ -938,4 +937,25 @@ void StaticData::ResetWeights(const std::string &denseWeights, const std::string } } +size_t StaticData::GetCoordSpace(string space) const +{ + map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space); + if(m == m_coordSpaceMap.end()) { + return 0; + } + return m->second; +} + +size_t StaticData::MapCoordSpace(string space) +{ + map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space); + if (m != m_coordSpaceMap.end()) { + return m->second; + } + size_t id = m_coordSpaceNextID; + m_coordSpaceNextID += 1; + m_coordSpaceMap[space] = id; + return id; +} + } // namespace diff --git a/moses/StaticData.h b/moses/StaticData.h index 871b82641..ea3d14025 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -60,7 +60,7 @@ class PhraseDictionaryDynamicCacheBased; typedef std::pair<std::string, float> UnknownLHSEntry; typedef std::vector<UnknownLHSEntry> UnknownLHSList; -/** Contains global variables and contants. +/** Contains global variables and constants. * Only 1 object of this class should be instantiated. * A const object of this class is accessible by any function during decoding by calling StaticData::Instance(); */ @@ -152,6 +152,12 @@ protected: bool ini_performance_options(); void initialize_features(); + + // Coordinate space name map for matching spaces across XML input ("coord" + // tag) and feature functions that assign or use coordinates on target phrases + std::map< std::string const, size_t > m_coordSpaceMap; + size_t m_coordSpaceNextID; + public: //! destructor @@ -394,6 +400,9 @@ public: return m_requireSortingAfterSourceContext; } + // Coordinate spaces + size_t GetCoordSpace(std::string space) const; + size_t MapCoordSpace(std::string space); }; } diff --git a/moses/TargetPhrase.cpp b/moses/TargetPhrase.cpp index 89575c462..09a733718 100644 --- a/moses/TargetPhrase.cpp +++ b/moses/TargetPhrase.cpp @@ -125,6 +125,7 @@ TargetPhrase::TargetPhrase(const Phrase &phrase, const PhraseDictionary *pt) TargetPhrase::TargetPhrase(const TargetPhrase ©) : Phrase(copy) + , m_cached_coord(copy.m_cached_coord) , m_cached_scores(copy.m_cached_scores) , m_scope(copy.m_scope) , m_futureScore(copy.m_futureScore) @@ -333,6 +334,31 @@ SetExtraScores(FeatureFunction const* ff, m_cached_scores[ff] = s; } +vector<SPTR<vector<float> > > const* +TargetPhrase:: +GetCoordList(size_t const spaceID) const +{ + if(!m_cached_coord) { + return NULL; + } + CoordCache_t::const_iterator m = m_cached_coord->find(spaceID); + if(m == m_cached_coord->end()) { + return NULL; + } + return &m->second; +} + +void +TargetPhrase:: +PushCoord(size_t const spaceID, + SPTR<vector<float> > const coord) +{ + if (!m_cached_coord) { + m_cached_coord.reset(new CoordCache_t); + } + vector<SPTR<vector<float> > >& coordList = (*m_cached_coord)[spaceID]; + coordList.push_back(coord); +} void TargetPhrase::SetProperties(const StringPiece &str) { diff --git a/moses/TargetPhrase.h b/moses/TargetPhrase.h index a2772cdae..3d621dbe9 100644 --- a/moses/TargetPhrase.h +++ b/moses/TargetPhrase.h @@ -56,9 +56,13 @@ public: Scores const* GetExtraScores(FeatureFunction const* ff) const; void SetExtraScores(FeatureFunction const* ff,boost::shared_ptr<Scores> const& scores); + typedef std::map<size_t const, std::vector<SPTR<std::vector<float> > > > CoordCache_t; + std::vector<SPTR<std::vector<float> > > const* GetCoordList(size_t const spaceID) const; + void PushCoord(size_t const spaceID, SPTR<std::vector<float> > const coord); private: ScoreCache_t m_cached_scores; + SPTR<CoordCache_t> m_cached_coord; WPTR<ContextScope> m_scope; private: diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp index 04f8db219..3a54fbac2 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp @@ -108,7 +108,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( // search for terminal symbol if (startPos == absEndPos) { - OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceWordLabel.GetLabel()); + OnDiskPt::Word *sourceWordBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceWordLabel.GetLabel()); if (sourceWordBerkeleyDb != NULL) { const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper); @@ -154,7 +154,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( for (iterSourceLHS = sourceLHSSet.begin(); iterSourceLHS != sourceLHSSet.end(); ++iterSourceLHS) { const Word &sourceLHS = *iterSourceLHS; - OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS); + OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS); if (sourceLHSBerkeleyDb == NULL) { delete sourceLHSBerkeleyDb; @@ -190,7 +190,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( if (doSearch) { - OnDiskPt::Word *chartNonTermBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_outputFactorsVec, cellLabel.GetLabel()); + OnDiskPt::Word *chartNonTermBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_outputFactorsVec, cellLabel.GetLabel()); if (chartNonTermBerkeleyDb == NULL) continue; @@ -234,7 +234,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( for (iterLabelSet = lhsSet.begin(); iterLabelSet != lhsSet.end(); ++iterLabelSet) { const Word &sourceLHS = *iterLabelSet; - OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS); + OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS); if (sourceLHSBerkeleyDb == NULL) continue; @@ -251,12 +251,13 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( std::vector<float> weightT = staticData.GetWeights(&m_dictionary); targetPhraseCollection - = tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec - ,m_outputFactorsVec - ,m_dictionary - ,weightT - ,m_dbWrapper.GetVocab() - ,true); + = m_dictionary.ConvertToMoses(tpcollBerkeleyDb + ,m_inputFactorsVec + ,m_outputFactorsVec + ,m_dictionary + ,weightT + ,m_dbWrapper.GetVocab() + ,true); tpcollBerkeleyDb.reset(); m_cache[tpCollFilePos] = targetPhraseCollection; diff --git a/moses/TranslationModel/CompactPT/CanonicalHuffman.h b/moses/TranslationModel/CompactPT/CanonicalHuffman.h index 9f6c14e56..10f3019b1 100644 --- a/moses/TranslationModel/CompactPT/CanonicalHuffman.h +++ b/moses/TranslationModel/CompactPT/CanonicalHuffman.h @@ -76,8 +76,9 @@ private: MinHeapSorter hs(A); std::make_heap(A.begin(), A.begin() + n, hs); - size_t h = n; - size_t m1, m2; + // marked volatile to prevent the intel compiler from generating bad code + volatile size_t h = n; + volatile size_t m1, m2; while(h > 1) { m1 = A[0]; std::pop_heap(A.begin(), A.begin() + h, hs); diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp index adb3f36c1..7766c897a 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp @@ -30,6 +30,8 @@ #include "OnDiskPt/OnDiskWrapper.h" #include "OnDiskPt/Word.h" +#include "util/tokenize_piece.hh" + using namespace std; @@ -148,7 +150,7 @@ void PhraseDictionaryOnDisk::GetTargetPhraseCollectionBatch(InputPath &inputPath if (prevPtNode) { Word lastWord = phrase.GetWord(phrase.GetSize() - 1); lastWord.OnlyTheseFactors(m_inputFactors); - OnDiskPt::Word *lastWordOnDisk = wrapper.ConvertFromMoses(m_input, lastWord); + OnDiskPt::Word *lastWordOnDisk = ConvertFromMoses(wrapper, m_input, lastWord); TargetPhraseCollection::shared_ptr tpc; if (lastWordOnDisk == NULL) { @@ -206,14 +208,180 @@ GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const OnDiskPt::TargetPhraseCollection::shared_ptr targetPhrasesOnDisk = ptNode->GetTargetPhraseCollection(m_tableLimit, wrapper); TargetPhraseCollection::shared_ptr targetPhrases - = targetPhrasesOnDisk->ConvertToMoses(m_input, m_output, *this, - weightT, vocab, false); + = ConvertToMoses(targetPhrasesOnDisk, m_input, m_output, *this, + weightT, vocab, false); // delete targetPhrasesOnDisk; return targetPhrases; } +Moses::TargetPhraseCollection::shared_ptr +PhraseDictionaryOnDisk::ConvertToMoses( + const OnDiskPt::TargetPhraseCollection::shared_ptr targetPhrasesOnDisk + , const std::vector<Moses::FactorType> &inputFactors + , const std::vector<Moses::FactorType> &outputFactors + , const Moses::PhraseDictionary &phraseDict + , const std::vector<float> &weightT + , OnDiskPt::Vocab &vocab + , bool isSyntax) const +{ + Moses::TargetPhraseCollection::shared_ptr ret; + ret.reset(new Moses::TargetPhraseCollection); + + for (size_t i = 0; i < targetPhrasesOnDisk->GetSize(); ++i) { + const OnDiskPt::TargetPhrase &tp = targetPhrasesOnDisk->GetTargetPhrase(i); + Moses::TargetPhrase *mosesPhrase + = ConvertToMoses(tp, inputFactors, outputFactors, vocab, + phraseDict, weightT, isSyntax); + + /* + // debugging output + stringstream strme; + strme << filePath << " " << *mosesPhrase; + mosesPhrase->SetDebugOutput(strme.str()); + */ + + ret->Add(mosesPhrase); + } + + ret->Sort(true, phraseDict.GetTableLimit()); + + return ret; +} + +Moses::TargetPhrase *PhraseDictionaryOnDisk::ConvertToMoses(const OnDiskPt::TargetPhrase &targetPhraseOnDisk + , const std::vector<Moses::FactorType> &inputFactors + , const std::vector<Moses::FactorType> &outputFactors + , const OnDiskPt::Vocab &vocab + , const Moses::PhraseDictionary &phraseDict + , const std::vector<float> &weightT + , bool isSyntax) const +{ + Moses::TargetPhrase *ret = new Moses::TargetPhrase(&phraseDict); + + // words + size_t phraseSize = targetPhraseOnDisk.GetSize(); + UTIL_THROW_IF2(phraseSize == 0, "Target phrase cannot be empty"); // last word is lhs + if (isSyntax) { + --phraseSize; + } + + for (size_t pos = 0; pos < phraseSize; ++pos) { + const OnDiskPt::Word &wordOnDisk = targetPhraseOnDisk.GetWord(pos); + ConvertToMoses(wordOnDisk, outputFactors, vocab, ret->AddWord()); + } + + // alignments + // int index = 0; + Moses::AlignmentInfo::CollType alignTerm, alignNonTerm; + std::set<std::pair<size_t, size_t> > alignmentInfo; + const OnDiskPt::PhrasePtr sp = targetPhraseOnDisk.GetSourcePhrase(); + for (size_t ind = 0; ind < targetPhraseOnDisk.GetAlign().size(); ++ind) { + const std::pair<size_t, size_t> &entry = targetPhraseOnDisk.GetAlign()[ind]; + alignmentInfo.insert(entry); + size_t sourcePos = entry.first; + size_t targetPos = entry.second; + + if (targetPhraseOnDisk.GetWord(targetPos).IsNonTerminal()) { + alignNonTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos)); + } else { + alignTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos)); + } + + } + ret->SetAlignTerm(alignTerm); + ret->SetAlignNonTerm(alignNonTerm); + + if (isSyntax) { + Moses::Word *lhsTarget = new Moses::Word(true); + const OnDiskPt::Word &lhsOnDisk = targetPhraseOnDisk.GetWord(targetPhraseOnDisk.GetSize() - 1); + ConvertToMoses(lhsOnDisk, outputFactors, vocab, *lhsTarget); + ret->SetTargetLHS(lhsTarget); + } + + // set source phrase + Moses::Phrase mosesSP(Moses::Input); + for (size_t pos = 0; pos < sp->GetSize(); ++pos) { + ConvertToMoses(sp->GetWord(pos), inputFactors, vocab, mosesSP.AddWord()); + } + + // scores + ret->GetScoreBreakdown().Assign(&phraseDict, targetPhraseOnDisk.GetScores()); + + // sparse features + ret->GetScoreBreakdown().Assign(&phraseDict, targetPhraseOnDisk.GetSparseFeatures()); + + // property + ret->SetProperties(targetPhraseOnDisk.GetProperty()); + + ret->EvaluateInIsolation(mosesSP, phraseDict.GetFeaturesToApply()); + + return ret; +} + +void PhraseDictionaryOnDisk::ConvertToMoses( + const OnDiskPt::Word &wordOnDisk, + const std::vector<Moses::FactorType> &outputFactorsVec, + const OnDiskPt::Vocab &vocab, + Moses::Word &overwrite) const +{ + Moses::FactorCollection &factorColl = Moses::FactorCollection::Instance(); + overwrite = Moses::Word(wordOnDisk.IsNonTerminal()); + + if (wordOnDisk.IsNonTerminal()) { + const std::string &tok = vocab.GetString(wordOnDisk.GetVocabId()); + overwrite.SetFactor(0, factorColl.AddFactor(tok, wordOnDisk.IsNonTerminal())); + } else { + // TODO: this conversion should have been done at load time. + util::TokenIter<util::SingleCharacter> tok(vocab.GetString(wordOnDisk.GetVocabId()), '|'); + + for (std::vector<Moses::FactorType>::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) { + UTIL_THROW_IF2(!tok, "Too few factors in \"" << vocab.GetString(wordOnDisk.GetVocabId()) << "\"; was expecting " << outputFactorsVec.size()); + overwrite.SetFactor(*t, factorColl.AddFactor(*tok, wordOnDisk.IsNonTerminal())); + } + UTIL_THROW_IF2(tok, "Too many factors in \"" << vocab.GetString(wordOnDisk.GetVocabId()) << "\"; was expecting " << outputFactorsVec.size()); + } +} + +OnDiskPt::Word *PhraseDictionaryOnDisk::ConvertFromMoses(OnDiskPt::OnDiskWrapper &wrapper, const std::vector<Moses::FactorType> &factorsVec + , const Moses::Word &origWord) const +{ + bool isNonTerminal = origWord.IsNonTerminal(); + OnDiskPt::Word *newWord = new OnDiskPt::Word(isNonTerminal); + + util::StringStream strme; + + size_t factorType = factorsVec[0]; + const Moses::Factor *factor = origWord.GetFactor(factorType); + UTIL_THROW_IF2(factor == NULL, "Expecting factor " << factorType); + strme << factor->GetString(); + + for (size_t ind = 1 ; ind < factorsVec.size() ; ++ind) { + size_t factorType = factorsVec[ind]; + const Moses::Factor *factor = origWord.GetFactor(factorType); + if (factor == NULL) { + // can have less factors than factorType.size() + break; + } + UTIL_THROW_IF2(factor == NULL, + "Expecting factor " << factorType << " at position " << ind); + strme << "|" << factor->GetString(); + } // for (size_t factorType + + bool found; + uint64_t vocabId = wrapper.GetVocab().GetVocabId(strme.str(), found); + if (!found) { + // factor not in phrase table -> phrse definately not in. exit + delete newWord; + return NULL; + } else { + newWord->SetVocabId(vocabId); + return newWord; + } + +} + void PhraseDictionaryOnDisk::SetParameter(const std::string& key, const std::string& value) { if (key == "max-span-default") { diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h index 1bd357d05..4b5f3cae9 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h @@ -64,6 +64,19 @@ protected: void GetTargetPhraseCollectionBatch(InputPath &inputPath) const; + Moses::TargetPhrase *ConvertToMoses(const OnDiskPt::TargetPhrase &targetPhraseOnDisk + , const std::vector<Moses::FactorType> &inputFactors + , const std::vector<Moses::FactorType> &outputFactors + , const OnDiskPt::Vocab &vocab + , const Moses::PhraseDictionary &phraseDict + , const std::vector<float> &weightT + , bool isSyntax) const; + + void ConvertToMoses(const OnDiskPt::Word &wordOnDisk, + const std::vector<Moses::FactorType> &outputFactorsVec, + const OnDiskPt::Vocab &vocab, + Moses::Word &overwrite) const; + public: PhraseDictionaryOnDisk(const std::string &line); ~PhraseDictionaryOnDisk(); @@ -84,6 +97,19 @@ public: TargetPhraseCollection::shared_ptr GetTargetPhraseCollectionNonCache(const OnDiskPt::PhraseNode *ptNode) const; + Moses::TargetPhraseCollection::shared_ptr + ConvertToMoses( + const OnDiskPt::TargetPhraseCollection::shared_ptr targetPhrasesOnDisk + , const std::vector<Moses::FactorType> &inputFactors + , const std::vector<Moses::FactorType> &outputFactors + , const Moses::PhraseDictionary &phraseDict + , const std::vector<float> &weightT + , OnDiskPt::Vocab &vocab + , bool isSyntax) const; + + OnDiskPt::Word *ConvertFromMoses(OnDiskPt::OnDiskWrapper &wrapper, const std::vector<Moses::FactorType> &factorsVec + , const Moses::Word &origWord) const; + void SetParameter(const std::string& key, const std::string& value); }; diff --git a/moses/TranslationModel/UG/check-coverage3.cc b/moses/TranslationModel/UG/check-coverage3.cc index b41ca9025..5eb18164a 100644 --- a/moses/TranslationModel/UG/check-coverage3.cc +++ b/moses/TranslationModel/UG/check-coverage3.cc @@ -13,6 +13,8 @@ #include <sstream> #include "mm/ug_bitext_sampler.h" +#include <boost/program_options.hpp> +namespace po=boost::program_options; using namespace Moses; using namespace sapt; using namespace std; @@ -21,6 +23,13 @@ using namespace boost; typedef sapt::L2R_Token<sapt::SimpleWordId> Token; typedef mmBitext<Token> bitext_t; +size_t topN; +string docname; +string reference_file; +string domain_name; +string bname, L1, L2; +string ifile; + struct mycmp { bool operator() (pair<string,uint32_t> const& a, @@ -30,36 +39,70 @@ struct mycmp } }; + + +void interpret_args(int ac, char* av[]); + string -basename(string const path, string const suffix) +basename(string const path) { size_t p = path.find_last_of("/"); - size_t k = path.size() - suffix.size(); - cout << path << " " << suffix << endl; - cout << path.substr(0,p) << " " << path.substr(k) << endl; - return path.substr(p+1, suffix == &path[k] ? k-p-1 : path.size() - p); + string dot = "."; + size_t k = path.find((dot + L1),p+1); + if (k == string::npos) k = path.find(dot + L1 + ".gz"); + if (k == string::npos) return path.substr(p+1); + return path.substr(p+1, k-p-1); +} + +void +print_evidence_list(bitext_t const& B, std::map<uint32_t, uint32_t> const& indoc) +{ + typedef std::map<uint32_t, uint32_t>::const_iterator iter; + typedef pair<size_t,string> item; + vector<item> where; + where.reserve(indoc.size()); + + for (iter d = indoc.begin(); d != indoc.end(); ++d) + where.push_back(item(d->second, B.docid2name(d->first))); + sort(where.begin(),where.end(),greater<item>()); + BOOST_FOREACH(item const& doc, where) + if (domain_name == doc.second) + cout << (boost::format("\t\t%4d ! %s") % doc.first % doc.second) << endl; + else + cout << (boost::format("\t\t%4d %s") % doc.first % doc.second) << endl; } int main(int argc, char* argv[]) { - boost::intrusive_ptr<bitext_t> B(new bitext_t); - B->open(argv[1],argv[2],argv[3]); - string line; - string ifile = argv[4]; - string docname = basename(ifile, string(".") + argv[2] + ".gz"); - id_type docid = B->docname2docid(docname); - boost::iostreams::filtering_istream in; + boost::shared_ptr<bitext_t> B(new bitext_t); + interpret_args(argc,argv); + + B->open(bname, L1, L2); + string line, refline; + if (domain_name == "" && ifile != "-") + domain_name = basename(ifile); + + id_type docid = B->docname2docid(domain_name); + boost::iostreams::filtering_istream in, ref; ugdiss::open_input_stream(ifile,in); + if (reference_file.size()) + ugdiss::open_input_stream(reference_file,ref); + while(getline(in,line)) { - cout << line << " [" << docname << "]" << endl; + if (reference_file.size()) getline(ref, refline); + cout << string(80,'-') << endl; + cout << " [" << domain_name << "]" << endl; + cout << line << endl; + if (refline.size()) cout << refline << endl; + cout << string(80,'-') << endl; vector<id_type> snt; B->V1->fillIdSeq(line,snt); for (size_t i = 0; i < snt.size(); ++i) { bitext_t::iter m(B->I1.get()); for (size_t k = i; k < snt.size() && m.extend(snt[k]); ++k); - for (size_t num_occurrences = m.ca(); m.size(); m.up()) + for (size_t num_occurrences = 0; m.size(); m.up()) { if (size_t(m.ca()) == num_occurrences) continue; num_occurrences = m.ca(); @@ -68,40 +111,45 @@ int main(int argc, char* argv[]) sapt::random_sampling); s(); if (s.stats()->trg.size() == 0) continue; - // if (s.stats()->indoc[docname] > 10) continue; sapt::pstats::indoc_map_t::const_iterator d = s.stats()->indoc.find(docid); size_t indoccnt = d != s.stats()->indoc.end() ? d->second : 0; - cout << m.size() << " : " << m.str(B->V1.get()) << " (" + cout << m.str(B->V1.get()) << " (" << s.stats()->trg.size() << " entries; " << indoccnt << "/" << s.stats()->good - << " samples in domain)" << endl; + << " samples in domain; " << num_occurrences + << " occ.)" << endl; vector<PhrasePair<Token> > ppairs; PhrasePair<Token>::SortDescendingByJointCount sorter; expand(m,*B,*s.stats(),ppairs,NULL); sort(ppairs.begin(),ppairs.end(),sorter); boost::format fmt("%4d/%d/%d |%s| (%4.2f : %4.2f)"); + size_t ctr = 0; + bool skipped_some = false; BOOST_FOREACH(PhrasePair<Token>& ppair, ppairs) { - if (ppair.joint * 100 < ppair.good1) break; + if (++ctr > topN && ppair.indoc.find(docid) == ppair.indoc.end()) + { + skipped_some = true; + continue; + } + if (skipped_some) + { + cout << string(17,' ') << "..." << endl; + skipped_some = false; + } + // if (ppair.joint * 100 < ppair.good1) break; ppair.good2 = ppair.raw2 * float(ppair.good1)/ppair.raw1; ppair.good2 = max(ppair.good2, ppair.joint); -#if 0 +#if 1 cout << "\t" << (fmt % ppair.joint % ppair.good1 % ppair.good2 % B->T2->pid2str(B->V2.get(),ppair.p2) % (float(ppair.joint)/ppair.good1) % (float(ppair.joint)/ppair.good2) ) << "\n"; - typedef std::map<uint32_t, uint32_t>::const_iterator iter; - for (iter d = ppair.indoc.begin(); d != ppair.indoc.end(); ++d) - { - // if (d != ppair.indoc.begin()) cout << "; "; - cout << (boost::format("\t\t%4d %s") % d->second - % B->docid2name(d->first)) - << endl; - } + print_evidence_list(*B, ppair.indoc); cout << endl; #else cout << "\t" @@ -126,3 +174,48 @@ int main(int argc, char* argv[]) } } } + +void +interpret_args(int ac, char* av[]) +{ + po::variables_map vm; + po::options_description o("Options"); + o.add_options() + + ("help,h", "print this message") + ("top,n", po::value<size_t>(&topN)->default_value(5), + "max. number of entries to show") + ("domain,D", po::value<string>(&domain_name), + "domain name (when reading from stdin)") + ("reference,r", po::value<string>(&reference_file), + "reference file") + ; + + po::options_description h("Hidden Options"); + h.add_options() + ("bname", po::value<string>(&bname), "base name of corpus") + ("L1", po::value<string>(&L1), "L1 tag") + ("L2", po::value<string>(&L2), "L2 tag") + ("input", po::value<string>(&ifile), "input file") + ; + + h.add(o); + po::positional_options_description a; + a.add("bname",1); + a.add("L1",1); + a.add("L2",1); + a.add("input",1); + + po::store(po::command_line_parser(ac,av) + .options(h) + .positional(a) + .run(),vm); + po::notify(vm); + if (vm.count("help")) + { + std::cout << "\nusage:\n\t" << av[0] + << " [options] <model file stem> <L1> <L2> <input file>" << std::endl; + std::cout << o << std::endl; + exit(0); + } +} diff --git a/moses/TranslationModel/UG/mm/ug_bitext.h b/moses/TranslationModel/UG/mm/ug_bitext.h index 6c3a73457..2c835af7e 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext.h +++ b/moses/TranslationModel/UG/mm/ug_bitext.h @@ -130,7 +130,6 @@ namespace sapt mutable pplist_cache_t m_pplist_cache1, m_pplist_cache2; // caches for unbiased sampling; biased sampling uses the caches that // are stored locally on the translation task - public: SPTR<Ttrack<char> > Tx; // word alignments SPTR<Ttrack<Token> > T1; // token track @@ -164,7 +163,8 @@ namespace sapt #ifndef NO_MOSES SPTR<pstats> - prep2(ttasksptr const& ttask, iter const& phrase, int max_sample = -1) const; + prep2(ttasksptr const& ttask, iter const& phrase, bool const track_sids, + int max_sample = -1) const; #endif protected: @@ -189,7 +189,7 @@ namespace sapt SPTR<pstats> lookup(ttasksptr const& ttask, iter const& phrase, int max_sample = -1) const; - void prep(ttasksptr const& ttask, iter const& phrase) const; + void prep(ttasksptr const& ttask, iter const& phrase, bool const track_sids) const; #endif void setDefaultSampleSize(size_t const max_samples); diff --git a/moses/TranslationModel/UG/mm/ug_bitext_agenda.h b/moses/TranslationModel/UG/mm/ug_bitext_agenda.h index bc038bd03..8865d4cd1 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_agenda.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_agenda.h @@ -33,7 +33,8 @@ public: SPTR<pstats> add_job(Bitext<Token> const* const theBitext, typename TSA<Token>::tree_iterator const& phrase, - size_t const max_samples, SPTR<SamplingBias const> const& bias); + size_t const max_samples, SPTR<SamplingBias const> const& bias, + bool const track_sids); // add_job(Bitext<Token> const* const theBitext, // typename TSA<Token>::tree_iterator const& phrase, // size_t const max_samples, SamplingBias const* const bias); @@ -93,13 +94,14 @@ SPTR<pstats> Bitext<Token> ::agenda ::add_job(Bitext<Token> const* const theBitext, typename TSA<Token>::tree_iterator const& phrase, - size_t const max_samples, SPTR<SamplingBias const> const& bias) + size_t const max_samples, SPTR<SamplingBias const> const& bias, + bool const track_sids) { boost::unique_lock<boost::mutex> lk(this->lock); static boost::posix_time::time_duration nodelay(0,0,0,0); bool fwd = phrase.root == bt.I1.get(); SPTR<job> j(new job(theBitext, phrase, fwd ? bt.I1 : bt.I2, - max_samples, fwd, bias)); + max_samples, fwd, bias, track_sids)); j->stats->register_worker(); joblist.push_back(j); diff --git a/moses/TranslationModel/UG/mm/ug_bitext_agenda_job.h b/moses/TranslationModel/UG/mm/ug_bitext_agenda_job.h index 7312ecef4..2ac7a5c35 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_agenda_job.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_agenda_job.h @@ -35,6 +35,8 @@ public: SPTR<pstats> stats; // stores statistics collected during sampling SPTR<SamplingBias const> const m_bias; // sentence-level bias for sampling float bias_total; + bool m_track_sids; // track sentence ids in sample? + bool nextSample(uint64_t & sid, uint64_t & offset); // select next occurrence int @@ -46,7 +48,7 @@ public: job(Bitext<Token> const* const theBitext, typename TSA<Token>::tree_iterator const& m, SPTR<TSA<Token> > const& r, size_t maxsmpl, bool isfwd, - SPTR<SamplingBias const> const& bias); + SPTR<SamplingBias const> const& bias, bool const track_sids); ~job(); }; @@ -66,7 +68,8 @@ Bitext<Token>::agenda::job ::job(Bitext<Token> const* const theBitext, typename TSA<Token>::tree_iterator const& m, SPTR<TSA<Token> > const& r, size_t maxsmpl, - bool isfwd, SPTR<SamplingBias const> const& bias) + bool isfwd, SPTR<SamplingBias const> const& bias, + bool const track_sids) : m_bitext(theBitext) , rnd(0) , rnddenom(rnd.max() + 1.) @@ -80,8 +83,9 @@ Bitext<Token>::agenda::job , len(m.size()) , fwd(isfwd) , m_bias(bias) + , m_track_sids(track_sids) { - stats.reset(new pstats()); + stats.reset(new pstats(m_track_sids)); stats->raw_cnt = m.approxOccurrenceCount(); bias_total = 0; diff --git a/moses/TranslationModel/UG/mm/ug_bitext_agenda_worker.h b/moses/TranslationModel/UG/mm/ug_bitext_agenda_worker.h index af862bc2d..4ab02648a 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_agenda_worker.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_agenda_worker.h @@ -90,7 +90,7 @@ Bitext<Token>::agenda size_t raw2 = b->approxOccurrenceCount(); float bwgt = j->m_bias ? (*j->m_bias)[sid] : 1; j->stats->add(tpid, sample_weight, bwgt, aln, raw2, - po_fwd, po_bwd, docid); + po_fwd, po_bwd, docid, sid); bool ok = (i == e2) || b->extend(o[i].id()); UTIL_THROW_IF2(!ok, "Could not extend target phrase."); } diff --git a/moses/TranslationModel/UG/mm/ug_bitext_jstats.cc b/moses/TranslationModel/UG/mm/ug_bitext_jstats.cc index 2fa9a49f5..1441f85ec 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_jstats.cc +++ b/moses/TranslationModel/UG/mm/ug_bitext_jstats.cc @@ -29,6 +29,7 @@ namespace sapt my_wcnt = other.wcnt(); my_bcnt = other.bcnt(); my_aln = other.aln(); + sids = other.sids; indoc = other.indoc; for (int i = 0; i <= LRModel::NONE; i++) { @@ -56,7 +57,8 @@ namespace sapt size_t jstats:: add(float w, float b, std::vector<unsigned char> const& a, uint32_t const cnt2, - uint32_t fwd_orient, uint32_t bwd_orient, int const docid) + uint32_t fwd_orient, uint32_t bwd_orient, int const docid, + uint32_t const sid, bool const track_sid) { boost::lock_guard<boost::mutex> lk(this->lock); my_cnt2 = cnt2; @@ -76,6 +78,13 @@ namespace sapt } ++ofwd[fwd_orient]; ++obwd[bwd_orient]; + // Record sentence id if requested + if (track_sid) + { + if (!sids) + sids.reset(new std::vector<uint32_t>); + sids->push_back(sid); + } if (docid >= 0) { // while (int(indoc.size()) <= docid) indoc.push_back(0); diff --git a/moses/TranslationModel/UG/mm/ug_bitext_jstats.h b/moses/TranslationModel/UG/mm/ug_bitext_jstats.h index d8e0bb18a..95c640ad7 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_jstats.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_jstats.h @@ -28,6 +28,7 @@ namespace sapt uint32_t obwd[LRModel::NONE+1]; // backward distortion type counts public: + SPTR<std::vector<uint32_t> > sids; // list of sentence ids in this sample std::map<uint32_t,uint32_t> indoc; // std::vector<uint32_t> indoc; // counts origin of samples (for biased sampling) jstats(); @@ -41,7 +42,8 @@ namespace sapt size_t add(float w, float b, std::vector<unsigned char> const& a, uint32_t const cnt2, - uint32_t fwd_orient, uint32_t bwd_orient, int const docid); + uint32_t fwd_orient, uint32_t bwd_orient, int const docid, uint32_t const sid, + bool const track_sid); void invalidate(); void validate(); diff --git a/moses/TranslationModel/UG/mm/ug_bitext_moses.h b/moses/TranslationModel/UG/mm/ug_bitext_moses.h index c04d87bfd..c024d073a 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_moses.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_moses.h @@ -30,9 +30,9 @@ lookup(ttasksptr const& ttask, iter const& phrase, int max_sample) const template<typename Token> void Bitext<Token>:: -prep(ttasksptr const& ttask, iter const& phrase) const +prep(ttasksptr const& ttask, iter const& phrase, bool const track_sids) const { - prep2(ttask, phrase, m_default_sample_size); + prep2(ttask, phrase, track_sids, m_default_sample_size); } @@ -44,7 +44,8 @@ template<typename Token> SPTR<pstats> Bitext<Token> ::prep2 -( ttasksptr const& ttask, iter const& phrase, int max_sample) const +( ttasksptr const& ttask, iter const& phrase, bool const track_sids, + int max_sample) const { if (max_sample < 0) max_sample = m_default_sample_size; SPTR<SamplingBias> bias; @@ -74,7 +75,7 @@ Bitext<Token> if (m_num_workers > 1) ag->add_workers(m_num_workers); } - ret = ag->add_job(this, phrase, max_sample, bias); + ret = ag->add_job(this, phrase, max_sample, bias, track_sids); if (cache) cache->set(phrase.getPid(),ret); UTIL_THROW_IF2(ret == NULL, "Couldn't schedule sampling job."); return ret; diff --git a/moses/TranslationModel/UG/mm/ug_bitext_pstats.cc b/moses/TranslationModel/UG/mm/ug_bitext_pstats.cc index f1602ab96..f8c93fe3c 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_pstats.cc +++ b/moses/TranslationModel/UG/mm/ug_bitext_pstats.cc @@ -10,7 +10,7 @@ namespace sapt #endif pstats:: - pstats() : raw_cnt(0), sample_cnt(0), good(0), sum_pairs(0), in_progress(0) + pstats(bool const track_sids) : raw_cnt(0), sample_cnt(0), good(0), sum_pairs(0), in_progress(0), track_sids(track_sids) { for (int i = 0; i <= LRModel::NONE; ++i) ofwd[i] = obwd[i] = 0; @@ -69,11 +69,11 @@ namespace sapt std::vector<unsigned char> const& a, uint32_t const cnt2, uint32_t fwd_o, - uint32_t bwd_o, int const docid) + uint32_t bwd_o, int const docid, uint32_t const sid) { boost::lock_guard<boost::mutex> guard(this->lock); jstats& entry = this->trg[pid]; - size_t ret = entry.add(w, b, a, cnt2, fwd_o, bwd_o, docid); + size_t ret = entry.add(w, b, a, cnt2, fwd_o, bwd_o, docid, sid, track_sids); if (this->good < entry.rcnt()) { UTIL_THROW(util::Exception, "more joint counts than good counts:" diff --git a/moses/TranslationModel/UG/mm/ug_bitext_pstats.h b/moses/TranslationModel/UG/mm/ug_bitext_pstats.h index cdc4f0c3d..47ec33afb 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_pstats.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_pstats.h @@ -35,7 +35,8 @@ namespace sapt indoc_map_t indoc; trg_map_t trg; - pstats(); + bool track_sids; + pstats(bool const track_sids); ~pstats(); void release(); void register_worker(); @@ -49,7 +50,8 @@ namespace sapt uint32_t const cnt2, // raw target phrase count uint32_t fwd_o, // fwd. phrase orientation uint32_t bwd_o, // bwd. phrase orientation - int const docid); // document where sample was found + int const docid, // document where sample was found + uint32_t const sid); // index of sentence where sample was found void count_sample(int const docid, // document where sample was found diff --git a/moses/TranslationModel/UG/mm/ug_bitext_sampler.h b/moses/TranslationModel/UG/mm/ug_bitext_sampler.h index e62d32e48..ea27f18e8 100644 --- a/moses/TranslationModel/UG/mm/ug_bitext_sampler.h +++ b/moses/TranslationModel/UG/mm/ug_bitext_sampler.h @@ -70,6 +70,7 @@ BitextSampler : public Moses::reference_counter size_t m_num_occurrences; // estimated number of phrase occurrences in corpus boost::taus88 m_rnd; // every job has its own pseudo random generator double m_bias_total; + bool m_track_sids; // track sentence ids in stats? size_t consider_sample(TokenPosition const& p); size_t perform_random_sampling(); @@ -86,7 +87,8 @@ public: SPTR<SamplingBias const> const& bias, size_t const min_samples, size_t const max_samples, - sampling_method const method); + sampling_method const method, + bool const track_sids); ~BitextSampler(); SPTR<pstats> stats(); bool done() const; @@ -185,7 +187,7 @@ BitextSampler<Token>:: BitextSampler(SPTR<Bitext<Token> const> const& bitext, typename bitext::iter const& phrase, SPTR<SamplingBias const> const& bias, size_t const min_samples, size_t const max_samples, - sampling_method const method) + sampling_method const method, bool const track_sids) : m_bitext(bitext) , m_plen(phrase.size()) , m_fwd(phrase.root == bitext->I1.get()) @@ -201,8 +203,9 @@ BitextSampler(SPTR<Bitext<Token> const> const& bitext, , m_finished(false) , m_num_occurrences(phrase.ca()) , m_rnd(0) + , m_track_sids(track_sids) { - m_stats.reset(new pstats); + m_stats.reset(new pstats(m_track_sids)); m_stats->raw_cnt = phrase.ca(); m_stats->register_worker(); } @@ -332,7 +335,8 @@ consider_sample(TokenPosition const& p) size_t raw2 = b->approxOccurrenceCount(); size_t evid = m_stats->add(tpid, sample_weight, m_bias ? (*m_bias)[p.sid] : 1, - aln, raw2, rec.po_fwd, rec.po_bwd, docid); + aln, raw2, rec.po_fwd, rec.po_bwd, docid, + p.sid); max_evidence = std::max(max_evidence, evid); bool ok = (i == rec.e2) || b->extend(o[i].id()); UTIL_THROW_IF2(!ok, "Could not extend target phrase."); diff --git a/moses/TranslationModel/UG/mm/ug_phrasepair.h b/moses/TranslationModel/UG/mm/ug_phrasepair.h index 7087c20af..803f54e34 100644 --- a/moses/TranslationModel/UG/mm/ug_phrasepair.h +++ b/moses/TranslationModel/UG/mm/ug_phrasepair.h @@ -31,6 +31,8 @@ namespace sapt std::vector<unsigned char> aln; float score; bool inverse; + SPTR<std::vector<uint32_t> > sids; // list of sampled sentence ids where + // this phrase pair was found // std::vector<uint32_t> indoc; std::map<uint32_t,uint32_t> indoc; PhrasePair() { }; @@ -132,6 +134,7 @@ namespace sapt dbwd[i] = js.dcnt_bwd(po); } + sids = js.sids; indoc = js.indoc; return *this; } @@ -182,6 +185,8 @@ namespace sapt sample2 += o.sample2; cum_bias += o.cum_bias; // todo: add distortion counts + if (sids && o.sids) + sids->insert(sids->end(), o.sids->begin(), o.sids->end()); return *this; } @@ -199,6 +204,7 @@ namespace sapt , aln(o.aln) , score(o.score) , inverse(o.inverse) + , sids(o.sids) , indoc(o.indoc) { for (int i = 0; i <= LRModel::NONE; ++i) diff --git a/moses/TranslationModel/UG/mmsapt.cpp b/moses/TranslationModel/UG/mmsapt.cpp index 024ae44d3..0120227d7 100644 --- a/moses/TranslationModel/UG/mmsapt.cpp +++ b/moses/TranslationModel/UG/mmsapt.cpp @@ -275,6 +275,40 @@ namespace Moses m = param.find("name"); if (m != param.end()) m_name = m->second; + // Optional coordinates for training corpus + // Takes form coord=name1:file1.gz,name2:file2.gz,... + // Names should match with XML input (coord tag) + param.insert(pair<string,string>("coord","0")); + if(param["coord"] != "0") + { + m_track_coord = true; + vector<string> coord_instances = Tokenize(param["coord"], ","); + BOOST_FOREACH(std::string instance, coord_instances) + { + vector<string> toks = Moses::Tokenize(instance, ":"); + string space = toks[0]; + string file = toks[1]; + // Register that this model uses the given space + m_coord_spaces.push_back(StaticData::InstanceNonConst().MapCoordSpace(space)); + // Load sid coordinates from file + m_sid_coord_list.push_back(vector<SPTR<vector<float> > >()); + vector<SPTR<vector<float> > >& sid_coord = m_sid_coord_list[m_sid_coord_list.size() - 1]; + //TODO: support extra data for btdyn, here? extra? + sid_coord.reserve(btfix->T1->size()); + string line; + cerr << "Loading coordinate lines for space \"" << space << "\" from " << file << endl; + iostreams::filtering_istream in; + ugdiss::open_input_stream(file, in); + while(getline(in, line)) + { + SPTR<vector<float> > coord(new vector<float>); + Scan<float>(*coord, Tokenize(line)); + sid_coord.push_back(coord); + } + cerr << "Loaded " << sid_coord.size() << " lines" << endl; + } + } + // check for unknown parameters vector<string> known_parameters; known_parameters.reserve(50); known_parameters.push_back("L1"); @@ -290,6 +324,7 @@ namespace Moses known_parameters.push_back("cache"); known_parameters.push_back("coh"); known_parameters.push_back("config"); + known_parameters.push_back("coord"); known_parameters.push_back("cumb"); known_parameters.push_back("extra"); known_parameters.push_back("feature-sets"); @@ -616,6 +651,29 @@ namespace Moses } #endif + // Track coordinates if requested + if (m_track_coord) + { + BOOST_FOREACH(uint32_t const sid, *pool.sids) + { + for(size_t i = 0; i < m_coord_spaces.size(); ++i) + { + tp->PushCoord(m_coord_spaces[i], m_sid_coord_list[i][sid]); + } + } + /* + cerr << btfix->toString(pool.p1, 0) << " ::: " << btfix->toString(pool.p2, 1); + BOOST_FOREACH(size_t id, m_coord_spaces) + { + cerr << " [" << id << "]"; + vector<vector<float> const*> const* coordList = tp->GetCoordList(id); + BOOST_FOREACH(vector<float> const* coord, *coordList) + cerr << " : " << Join(" ", *coord); + } + cerr << endl; + */ + } + return tp; } @@ -691,7 +749,7 @@ namespace Moses SPTR<ContextScope> const& scope = ttask->GetScope(); SPTR<TPCollCache> cache = scope->get<TPCollCache>(cache_key); if (!cache) cache = m_cache; // no context-specific cache, use global one - + ret = cache->get(phrasekey, dyn->revision()); // TO DO: we should revise the revision mechanism: we take the // length of the dynamic bitext (in sentences) at the time the PT @@ -705,12 +763,12 @@ namespace Moses // std::cerr << ret << " with " << ret->refCount << " references at " // << HERE << std::endl; boost::upgrade_lock<boost::shared_mutex> rlock(ret->lock); - if (ret->GetSize()) return ret; + if (ret->GetSize()) return ret; // new TPC (not found or old one was not up to date) boost::upgrade_to_unique_lock<boost::shared_mutex> wlock(rlock); // maybe another thread did the work while we waited for the lock ? - if (ret->GetSize()) return ret; + if (ret->GetSize()) return ret; // OK: pt entry NOT found or NOT up to date // lookup and expansion could be done in parallel threads, @@ -730,7 +788,8 @@ namespace Moses BitextSampler<Token> s(btfix, mfix, context->bias, m_min_sample_size, m_default_sample_size, - m_sampling_method); + m_sampling_method, + m_track_coord); s(); sfix = s.stats(); } @@ -918,7 +977,7 @@ namespace Moses { BitextSampler<Token> s(btfix, mfix, context->bias, m_min_sample_size, m_default_sample_size, - m_sampling_method); + m_sampling_method, m_track_coord); if (*context->cache1->get(pid, s.stats()) == s.stats()) m_thread_pool->add(s); } @@ -939,7 +998,7 @@ namespace Moses for (size_t i = 0; mdyn.size() == i && i < myphrase.size(); ++i) mdyn.extend(myphrase[i]); // let's assume a uniform bias over the foreground corpus - if (mdyn.size() == myphrase.size()) dyn->prep(ttask, mdyn); + if (mdyn.size() == myphrase.size()) dyn->prep(ttask, mdyn, m_track_coord); } return mdyn.size() == myphrase.size(); } diff --git a/moses/TranslationModel/UG/mmsapt.h b/moses/TranslationModel/UG/mmsapt.h index 4a8393c11..c573f2181 100644 --- a/moses/TranslationModel/UG/mmsapt.h +++ b/moses/TranslationModel/UG/mmsapt.h @@ -119,6 +119,12 @@ namespace Moses std::vector<SPTR<pscorer > > m_active_ff_common; // activated feature functions (dyn) + bool m_track_coord = false; // track coordinates? Track sids when sampling + // from bitext, append coords to target phrases + // Space < Sid < sptr sentence coords > > + std::vector<std::vector<SPTR<std::vector<float> > > > m_sid_coord_list; + std::vector<size_t> m_coord_spaces; + void parse_factor_spec(std::vector<FactorType>& flist, std::string const key); diff --git a/moses/TranslationModel/UG/ptable-lookup-corpus.cc b/moses/TranslationModel/UG/ptable-lookup-corpus.cc index 6bc515658..8818570d4 100644 --- a/moses/TranslationModel/UG/ptable-lookup-corpus.cc +++ b/moses/TranslationModel/UG/ptable-lookup-corpus.cc @@ -29,6 +29,13 @@ int main(int argc, char const* argv[]) // Only lookup each phrase once unordered_set<string> seen; + string context_weight_spec; + params.SetParameter(context_weight_spec,"context-weights",string("")); + boost::shared_ptr<ContextScope> scope(new ContextScope); + boost::shared_ptr<IOWrapper> none; + if (context_weight_spec.size()) + scope->SetContextWeights(context_weight_spec); + string line; while (true) { // Input line @@ -57,7 +64,8 @@ int main(int argc, char const* argv[]) // Setup task for phrase boost::shared_ptr<TranslationTask> ttask; - ttask = TranslationTask::create(phrase); + ttask = TranslationTask::create(phrase, none, scope); + // Support model combinations (PhraseDictionaryGroup) BOOST_FOREACH(PhraseDictionary* p, PhraseDictionary::GetColl()) { p->InitializeForInput(ttask); diff --git a/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h b/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h index b195be290..42689b37b 100644 --- a/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h +++ b/moses/TranslationModel/UG/sapt_pscore_cumulative_bias.h @@ -28,8 +28,8 @@ namespace sapt { void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; (*dest)[this->m_index] = log(std::max(m_floor,pp.cum_bias)); diff --git a/moses/TranslationModel/UG/sapt_pscore_length_ratio.h b/moses/TranslationModel/UG/sapt_pscore_length_ratio.h index 356217caa..e79b5517c 100644 --- a/moses/TranslationModel/UG/sapt_pscore_length_ratio.h +++ b/moses/TranslationModel/UG/sapt_pscore_length_ratio.h @@ -48,8 +48,8 @@ namespace sapt { void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; float p = float(bt.T1->numTokens()); diff --git a/moses/TranslationModel/UG/sapt_pscore_lex1.h b/moses/TranslationModel/UG/sapt_pscore_lex1.h index 4ae94502b..b861334f9 100644 --- a/moses/TranslationModel/UG/sapt_pscore_lex1.h +++ b/moses/TranslationModel/UG/sapt_pscore_lex1.h @@ -36,8 +36,8 @@ namespace sapt void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; // uint32_t sid1=0,sid2=0,off1=0,off2=0,len1=0,len2=0; diff --git a/moses/TranslationModel/UG/sapt_pscore_logcnt.h b/moses/TranslationModel/UG/sapt_pscore_logcnt.h index d079a0af8..f9573079f 100644 --- a/moses/TranslationModel/UG/sapt_pscore_logcnt.h +++ b/moses/TranslationModel/UG/sapt_pscore_logcnt.h @@ -37,8 +37,8 @@ namespace sapt { void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; assert(pp.raw1); @@ -56,7 +56,7 @@ namespace sapt { if (m_specs.find("j") != std::string::npos) (*dest)[i++] = log(pp.joint); if (m_specs.find("r2") != std::string::npos) - (*dest)[++i] = log(pp.raw2); + (*dest)[i] = log(pp.raw2); } }; } // namespace sapt diff --git a/moses/TranslationModel/UG/sapt_pscore_pbwd.h b/moses/TranslationModel/UG/sapt_pscore_pbwd.h index 35c7e1fa9..b6421d976 100644 --- a/moses/TranslationModel/UG/sapt_pscore_pbwd.h +++ b/moses/TranslationModel/UG/sapt_pscore_pbwd.h @@ -38,8 +38,8 @@ namespace sapt void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; // we use the denominator specification to scale the raw counts on the diff --git a/moses/TranslationModel/UG/sapt_pscore_pfwd.h b/moses/TranslationModel/UG/sapt_pscore_pfwd.h index bfa8027d1..7a790ee8e 100644 --- a/moses/TranslationModel/UG/sapt_pscore_pfwd.h +++ b/moses/TranslationModel/UG/sapt_pscore_pfwd.h @@ -38,8 +38,9 @@ namespace sapt } void - operator()(Bitext<Token> const& bt, PhrasePair<Token> & pp, - std::vector<float> * dest = NULL) const + operator()(Bitext<Token> const& bt, + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; if (pp.joint > pp.good1) diff --git a/moses/TranslationModel/UG/sapt_pscore_phrasecount.h b/moses/TranslationModel/UG/sapt_pscore_phrasecount.h index a1426426a..b0ffbab28 100644 --- a/moses/TranslationModel/UG/sapt_pscore_phrasecount.h +++ b/moses/TranslationModel/UG/sapt_pscore_phrasecount.h @@ -22,8 +22,8 @@ namespace sapt void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; (*dest)[this->m_index] = 1; diff --git a/moses/TranslationModel/UG/sapt_pscore_provenance.h b/moses/TranslationModel/UG/sapt_pscore_provenance.h index 67ee74850..44ac81bc3 100644 --- a/moses/TranslationModel/UG/sapt_pscore_provenance.h +++ b/moses/TranslationModel/UG/sapt_pscore_provenance.h @@ -28,8 +28,8 @@ namespace sapt { void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; size_t i = this->m_index; diff --git a/moses/TranslationModel/UG/sapt_pscore_rareness.h b/moses/TranslationModel/UG/sapt_pscore_rareness.h index c36da1913..cee9fa08f 100644 --- a/moses/TranslationModel/UG/sapt_pscore_rareness.h +++ b/moses/TranslationModel/UG/sapt_pscore_rareness.h @@ -26,8 +26,8 @@ namespace sapt { void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; size_t i = this->m_index; diff --git a/moses/TranslationModel/UG/sapt_pscore_unaligned.h b/moses/TranslationModel/UG/sapt_pscore_unaligned.h index 4201b839c..0315772e4 100644 --- a/moses/TranslationModel/UG/sapt_pscore_unaligned.h +++ b/moses/TranslationModel/UG/sapt_pscore_unaligned.h @@ -37,8 +37,8 @@ namespace sapt void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; // uint32_t sid1=0,sid2=0,off1=0,off2=0,len1=0,len2=0; diff --git a/moses/TranslationModel/UG/sapt_pscore_wordcount.h b/moses/TranslationModel/UG/sapt_pscore_wordcount.h index 6cd9e7c0c..5088567c7 100644 --- a/moses/TranslationModel/UG/sapt_pscore_wordcount.h +++ b/moses/TranslationModel/UG/sapt_pscore_wordcount.h @@ -22,8 +22,8 @@ namespace sapt void operator()(Bitext<Token> const& bt, - PhrasePair<Token>& pp, - std::vector<float> * dest = NULL) const + PhrasePair<Token>& pp, + std::vector<float> * dest = NULL) const { if (!dest) dest = &pp.fvals; (*dest)[this->m_index] = pp.len2; diff --git a/moses/TranslationModel/UG/util/tokenindex.dump.cc b/moses/TranslationModel/UG/util/tokenindex.dump.cc index 0e885630f..01dfeb03e 100644 --- a/moses/TranslationModel/UG/util/tokenindex.dump.cc +++ b/moses/TranslationModel/UG/util/tokenindex.dump.cc @@ -12,7 +12,7 @@ #include <iomanip> using namespace std; -using namespace ugdiss; +using namespace sapt; int main(int argc,char* argv[]) { diff --git a/moses/TranslationOptionCollection.cpp b/moses/TranslationOptionCollection.cpp index fe17c28b9..d1c9be7a2 100644 --- a/moses/TranslationOptionCollection.cpp +++ b/moses/TranslationOptionCollection.cpp @@ -59,6 +59,7 @@ TranslationOptionCollection(ttasksptr const& ttask, , m_maxNoTransOptPerCoverage(ttask->options()->search.max_trans_opt_per_cov) , m_translationOptionThreshold(ttask->options()->search.trans_opt_threshold) , m_max_phrase_length(ttask->options()->search.max_phrase_length) + , max_partial_trans_opt(ttask->options()->search.max_partial_trans_opt) { // create 2-d vector size_t size = src.GetSize(); @@ -391,7 +392,7 @@ CreateTranslationOptionsForRange || !HasXmlOptionsOverlappingRange(sPos,ePos)) { // partial trans opt stored in here - PartialTranslOptColl* oldPtoc = new PartialTranslOptColl(m_max_phrase_length); + PartialTranslOptColl* oldPtoc = new PartialTranslOptColl(max_partial_trans_opt); size_t totalEarlyPruned = 0; // initial translation step diff --git a/moses/TranslationOptionCollection.h b/moses/TranslationOptionCollection.h index f3a82c5f6..4529ad680 100644 --- a/moses/TranslationOptionCollection.h +++ b/moses/TranslationOptionCollection.h @@ -72,6 +72,7 @@ protected: const size_t m_maxNoTransOptPerCoverage; /*< maximum number of translation options per input span */ const float m_translationOptionThreshold; /*< threshold for translation options with regard to best option for input span */ size_t m_max_phrase_length; + size_t max_partial_trans_opt; std::vector<const Phrase*> m_unksrcs; InputPathList m_inputPathQueue; diff --git a/moses/XmlOption.cpp b/moses/XmlOption.cpp index 8a517386f..6e089c774 100644 --- a/moses/XmlOption.cpp +++ b/moses/XmlOption.cpp @@ -163,7 +163,8 @@ ProcessAndStripXMLTags(AllOptions const& opts, string &line, vector<XmlOption const*> &res, ReorderingConstraint &reorderingConstraint, vector< size_t > &walls, - std::vector< std::pair<size_t, std::string> > &placeholders) + std::vector< std::pair<size_t, std::string> > &placeholders, + InputType &input) { //parse XML markup in translation line @@ -401,6 +402,28 @@ ProcessAndStripXMLTags(AllOptions const& opts, string &line, StaticData::InstanceNonConst().SetAllWeights(allWeights); } + // Coord: coordinates of the input sentence in a user-defined space + // <coord space="NAME" coord="X Y Z ..." /> + // where NAME is the name of the space and X Y Z ... are floats. See + // PhraseDistanceFeature for an example of using this information for + // feature scoring. + else if (tagName == "coord") { + // Parse tag + string space = ParseXmlTagAttribute(tagContent, "space"); + vector<string> tok = Tokenize(ParseXmlTagAttribute(tagContent, "coord")); + size_t id = StaticData::Instance().GetCoordSpace(space); + if (!id) { + TRACE_ERR("ERROR: no models use space " << space << ", will be ignored" << endl); + } else { + // Init if needed + if (!input.m_coordMap) { + input.m_coordMap.reset(new map<size_t const, vector<float> >); + } + vector<float>& coord = (*input.m_coordMap)[id]; + Scan<float>(coord, tok); + } + } + // default: opening tag that specifies translation options else { if (startPos > endPos) { diff --git a/moses/XmlOption.h b/moses/XmlOption.h index 00bebc2e6..da3ee8044 100644 --- a/moses/XmlOption.h +++ b/moses/XmlOption.h @@ -34,7 +34,8 @@ bool ProcessAndStripXMLTags(AllOptions const& opts, std::string &line, std::vector<XmlOption const*> &res, ReorderingConstraint &reorderingConstraint, std::vector< size_t > &walls, - std::vector< std::pair<size_t, std::string> > &placeholders); + std::vector< std::pair<size_t, std::string> > &placeholders, + InputType &input); } diff --git a/moses/parameters/BookkeepingOptions.h b/moses/parameters/BookkeepingOptions.h index d54bccc0a..3bfcb0292 100644 --- a/moses/parameters/BookkeepingOptions.h +++ b/moses/parameters/BookkeepingOptions.h @@ -1,4 +1,5 @@ // -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*- +#pragma once #include "moses/Parameter.h" #include "OptionsBaseClass.h" diff --git a/moses/parameters/SearchOptions.cpp b/moses/parameters/SearchOptions.cpp index 678f9bfe0..958569e94 100644 --- a/moses/parameters/SearchOptions.cpp +++ b/moses/parameters/SearchOptions.cpp @@ -38,6 +38,7 @@ namespace Moses param.SetParameter(early_discarding_threshold, "early-discarding-threshold", DEFAULT_EARLY_DISCARDING_THRESHOLD); param.SetParameter(timeout, "time-out", 0); + param.SetParameter(segment_timeout, "segment-time-out", 0); param.SetParameter(max_phrase_length, "max-phrase-length", DEFAULT_MAX_PHRASE_LENGTH); param.SetParameter(trans_opt_threshold, "translation-option-threshold", diff --git a/moses/parameters/SearchOptions.h b/moses/parameters/SearchOptions.h index 46c53e95b..30a612f05 100644 --- a/moses/parameters/SearchOptions.h +++ b/moses/parameters/SearchOptions.h @@ -25,6 +25,7 @@ namespace Moses float beam_width; int timeout; + int segment_timeout; bool consensus; //! Use Consensus decoding (DeNero et al 2009) diff --git a/moses/server/TranslationRequest.cpp b/moses/server/TranslationRequest.cpp index 173ad5d40..6f6fed1cf 100644 --- a/moses/server/TranslationRequest.cpp +++ b/moses/server/TranslationRequest.cpp @@ -3,7 +3,6 @@ #include "moses/ContextScope.h" #include <boost/foreach.hpp> #include "moses/Util.h" -#include "moses/TreeInput.h" #include "moses/Hypothesis.h" namespace MosesServer @@ -25,7 +24,6 @@ using Moses::FValue; using Moses::PhraseDictionaryMultiModel; using Moses::FindPhraseDictionary; using Moses::Sentence; -using Moses::TreeInput; boost::shared_ptr<TranslationRequest> TranslationRequest:: @@ -235,6 +233,10 @@ check(std::map<std::string, xmlrpc_c::value> const& param, { std::map<std::string, xmlrpc_c::value>::const_iterator m = param.find(key); if(m == param.end()) return false; + + if (m->second.type() == xmlrpc_c::value::TYPE_BOOLEAN) + return xmlrpc_c::value_boolean(m->second); + std::string val = string(xmlrpc_c::value_string(m->second)); if(val == "true" || val == "True" || val == "TRUE" || val == "1") return true; return false; @@ -320,9 +322,7 @@ parse_request(std::map<std::string, xmlrpc_c::value> const& params) // m_bias[xmlrpc_c::value_int(tmp[i-1])] = xmlrpc_c::value_double(tmp[i]); // } if (is_syntax(m_options->search.algo)) { - m_source.reset(new TreeInput(m_options)); - istringstream in(m_source_string + "\n"); - m_source->Read(in); + m_source.reset(new Sentence(m_options,0,m_source_string)); } else { m_source.reset(new Sentence(m_options,0,m_source_string)); } @@ -333,10 +333,6 @@ void TranslationRequest:: run_chart_decoder() { - Moses::TreeInput tinput(m_options); - istringstream buf(m_source_string + "\n"); - tinput.Read(buf); - Moses::ChartManager manager(this->self()); manager.Decode(); |