diff options
Diffstat (limited to 'contrib/moses2/FF/LexicalReordering/LexicalReordering.cpp')
| -rw-r--r-- | contrib/moses2/FF/LexicalReordering/LexicalReordering.cpp | 222 |
1 files changed, 222 insertions, 0 deletions
diff --git a/contrib/moses2/FF/LexicalReordering/LexicalReordering.cpp b/contrib/moses2/FF/LexicalReordering/LexicalReordering.cpp new file mode 100644 index 000000000..97394ce84 --- /dev/null +++ b/contrib/moses2/FF/LexicalReordering/LexicalReordering.cpp @@ -0,0 +1,222 @@ +/* + * LexicalReordering.cpp + * + * Created on: 15 Dec 2015 + * Author: hieu + */ + +#include <boost/foreach.hpp> +#include "util/exception.hh" +#include "LexicalReordering.h" +#include "LRModel.h" +#include "PhraseBasedReorderingState.h" +#include "BidirectionalReorderingState.h" +#include "../../TranslationModel/PhraseTable.h" +#include "../../TranslationModel/CompactPT/LexicalReorderingTableCompact.h" +#include "../../System.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../PhraseBased/Manager.h" +#include "../../PhraseBased/Hypothesis.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../legacy/InputFileStream.h" +#include "../../legacy/Util2.h" + +using namespace std; + +namespace Moses2 +{ + +/////////////////////////////////////////////////////////////////////// + +LexicalReordering::LexicalReordering(size_t startInd, const std::string &line) : + StatefulFeatureFunction(startInd, line), m_compactModel(NULL), m_blank( + NULL), m_propertyInd(-1), m_coll(NULL), m_configuration(NULL) +{ + ReadParameters(); + assert(m_configuration); + //assert(m_numScores == 6); +} + +LexicalReordering::~LexicalReordering() +{ + delete m_compactModel; + delete m_coll; + delete m_configuration; +} + +void LexicalReordering::Load(System &system) +{ + MemPool &pool = system.GetSystemPool(); + + if (m_propertyInd >= 0) { + // Using integrate Lex RO. No loading needed + } + else if (FileExists(m_path + ".minlexr")) { + m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr", + m_FactorsF, m_FactorsE, m_FactorsC); + m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0); + } + else { + m_coll = new Coll(); + InputFileStream file(m_path); + string line; + size_t lineNum = 0; + + while (getline(file, line)) { + if (++lineNum % 1000000 == 0) { + cerr << lineNum << " "; + } + + std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||"); + assert(toks.size() == 3); + PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(), + system, toks[0]); + PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(), + system, toks[1]); + std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]); + std::transform(scores.begin(), scores.end(), scores.begin(), + TransformScore); + std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore); + + Key key(source, target); + (*m_coll)[key] = scores; + } + } +} + +void LexicalReordering::SetParameter(const std::string& key, + const std::string& value) +{ + if (key == "path") { + m_path = value; + } + else if (key == "type") { + m_configuration = new LRModel(value, *this); + } + else if (key == "input-factor") { + m_FactorsF = Tokenize<FactorType>(value); + } + else if (key == "output-factor") { + m_FactorsE = Tokenize<FactorType>(value); + } + else if (key == "property-index") { + m_propertyInd = Scan<int>(value); + } + else { + StatefulFeatureFunction::SetParameter(key, value); + } +} + +FFState* LexicalReordering::BlankState(MemPool &pool, const System &sys) const +{ + FFState *ret = m_configuration->CreateLRState(pool); + return ret; +} + +void LexicalReordering::EmptyHypothesisState(FFState &state, + const ManagerBase &mgr, const InputType &input, + const Hypothesis &hypo) const +{ + BidirectionalReorderingState &stateCast = + static_cast<BidirectionalReorderingState&>(state); + stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true, + &hypo.GetBitmap()); +} + +void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system, + const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ +} + +void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, + const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ + UTIL_THROW2("Don't use with SCFG models"); +} + + +void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool, + const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const +{ + BOOST_FOREACH(const TargetPhraseImpl *tp, tps){ + EvaluateAfterTablePruning(pool, *tp, sourcePhrase); +} +} + +void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool, + const TargetPhraseImpl &targetPhrase, const Phrase<Moses2::Word> &sourcePhrase) const +{ + if (m_propertyInd >= 0) { + SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd); + targetPhrase.ffData[m_PhraseTableInd] = scoreArr; + } + else if (m_compactModel) { + // using external compact binary model + const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase, + *m_blank); + if (values.size()) { + assert(values.size() == m_numScores); + + SCORE *scoreArr = pool.Allocate<SCORE>(m_numScores); + for (size_t i = 0; i < m_numScores; ++i) { + scoreArr[i] = values[i]; + } + targetPhrase.ffData[m_PhraseTableInd] = scoreArr; + } + else { + targetPhrase.ffData[m_PhraseTableInd] = NULL; + } + } + else if (m_coll) { + // using external memory model + + // cache data in target phrase + const Values *values = GetValues(sourcePhrase, targetPhrase); + assert(values->size() == m_numScores); + + if (values) { + SCORE *scoreArr = pool.Allocate<SCORE>(m_numScores); + for (size_t i = 0; i < m_numScores; ++i) { + scoreArr[i] = (*values)[i]; + } + targetPhrase.ffData[m_PhraseTableInd] = scoreArr; + } + else { + targetPhrase.ffData[m_PhraseTableInd] = NULL; + } + } +} + +void LexicalReordering::EvaluateWhenApplied(const ManagerBase &mgr, + const Hypothesis &hypo, const FFState &prevState, Scores &scores, + FFState &state) const +{ + const LRState &prevStateCast = static_cast<const LRState&>(prevState); + prevStateCast.Expand(mgr, *this, hypo, m_PhraseTableInd, scores, state); +} + +const LexicalReordering::Values *LexicalReordering::GetValues( + const Phrase<Moses2::Word> &source, const Phrase<Moses2::Word> &target) const +{ + Key key(&source, &target); + Coll::const_iterator iter; + iter = m_coll->find(key); + if (iter == m_coll->end()) { + return NULL; + } + else { + return &iter->second; + } +} + +void LexicalReordering::EvaluateWhenApplied(const SCFG::Manager &mgr, + const SCFG::Hypothesis &hypo, int featureID, Scores &scores, + FFState &state) const +{ + UTIL_THROW2("Not implemented"); +} + +} /* namespace Moses2 */ |