diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2016-06-28 12:46:51 +0300 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2016-06-28 12:46:51 +0300 |
commit | ad240a9f5b13c807e3e5dbe3a6d51f4e40e5ad53 (patch) | |
tree | 2ae54a0047db3d6731782f4a664e5bb558705582 /moses/TranslationModel | |
parent | cef30724d5fc48c0a4258cf67c95f22912603b00 (diff) |
move Moses classes out of OnDiskPt - Word
Diffstat (limited to 'moses/TranslationModel')
-rw-r--r-- | moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp | 33 | ||||
-rw-r--r-- | moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h | 5 |
2 files changed, 35 insertions, 3 deletions
diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp index 06c3bd262..04e401080 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp @@ -30,6 +30,8 @@ #include "OnDiskPt/OnDiskWrapper.h" #include "OnDiskPt/Word.h" +#include "util/tokenize_piece.hh" + using namespace std; @@ -266,7 +268,8 @@ Moses::TargetPhrase *PhraseDictionaryOnDisk::ConvertToMoses(const OnDiskPt::Targ } for (size_t pos = 0; pos < phraseSize; ++pos) { - targetPhraseOnDisk.GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord()); + const OnDiskPt::Word &wordOnDisk = targetPhraseOnDisk.GetWord(pos); + ConvertToMoses(wordOnDisk, outputFactors, vocab, ret->AddWord()); } // alignments @@ -292,14 +295,15 @@ Moses::TargetPhrase *PhraseDictionaryOnDisk::ConvertToMoses(const OnDiskPt::Targ if (isSyntax) { Moses::Word *lhsTarget = new Moses::Word(true); - targetPhraseOnDisk.GetWord(targetPhraseOnDisk.GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget); + const OnDiskPt::Word &lhsOnDisk = targetPhraseOnDisk.GetWord(targetPhraseOnDisk.GetSize() - 1); + ConvertToMoses(lhsOnDisk, outputFactors, vocab, *lhsTarget); ret->SetTargetLHS(lhsTarget); } // set source phrase Moses::Phrase mosesSP(Moses::Input); for (size_t pos = 0; pos < sp->GetSize(); ++pos) { - sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord()); + ConvertToMoses(sp->GetWord(pos), inputFactors, vocab, mosesSP.AddWord()); } // scores @@ -314,7 +318,30 @@ Moses::TargetPhrase *PhraseDictionaryOnDisk::ConvertToMoses(const OnDiskPt::Targ ret->EvaluateInIsolation(mosesSP, phraseDict.GetFeaturesToApply()); return ret; +} + +void PhraseDictionaryOnDisk::ConvertToMoses( + const OnDiskPt::Word &wordOnDisk, + const std::vector<Moses::FactorType> &outputFactorsVec, + const OnDiskPt::Vocab &vocab, + Moses::Word &overwrite) const +{ + Moses::FactorCollection &factorColl = Moses::FactorCollection::Instance(); + overwrite = Moses::Word(wordOnDisk.IsNonTerminal()); + if (wordOnDisk.IsNonTerminal()) { + const std::string &tok = vocab.GetString(wordOnDisk.GetVocabId()); + overwrite.SetFactor(0, factorColl.AddFactor(tok, wordOnDisk.IsNonTerminal())); + } else { + // TODO: this conversion should have been done at load time. + util::TokenIter<util::SingleCharacter> tok(vocab.GetString(wordOnDisk.GetVocabId()), '|'); + + for (std::vector<Moses::FactorType>::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) { + UTIL_THROW_IF2(!tok, "Too few factors in \"" << vocab.GetString(wordOnDisk.GetVocabId()) << "\"; was expecting " << outputFactorsVec.size()); + overwrite.SetFactor(*t, factorColl.AddFactor(*tok, wordOnDisk.IsNonTerminal())); + } + UTIL_THROW_IF2(tok, "Too many factors in \"" << vocab.GetString(wordOnDisk.GetVocabId()) << "\"; was expecting " << outputFactorsVec.size()); + } } void PhraseDictionaryOnDisk::SetParameter(const std::string& key, const std::string& value) diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h index f3c923b72..dc47936f6 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h @@ -72,6 +72,11 @@ protected: , const std::vector<float> &weightT , bool isSyntax) const; + void ConvertToMoses(const OnDiskPt::Word &wordOnDisk, + const std::vector<Moses::FactorType> &outputFactorsVec, + const OnDiskPt::Vocab &vocab, + Moses::Word &overwrite) const; + public: PhraseDictionaryOnDisk(const std::string &line); ~PhraseDictionaryOnDisk(); |