diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2016-06-28 13:15:40 +0300 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2016-06-28 13:15:40 +0300 |
commit | ebae7ce52063612b18ec169b397376f6b876f4d0 (patch) | |
tree | 88c97a958bf7466390050ef86cdaf893bc3db6dd /moses/TranslationModel | |
parent | ad240a9f5b13c807e3e5dbe3a6d51f4e40e5ad53 (diff) |
move ConvertFromMoses() to Moses
Diffstat (limited to 'moses/TranslationModel')
3 files changed, 46 insertions, 5 deletions
diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp index 5a5a1368a..c4959ab38 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp @@ -108,7 +108,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( // search for terminal symbol if (startPos == absEndPos) { - OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceWordLabel.GetLabel()); + OnDiskPt::Word *sourceWordBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceWordLabel.GetLabel()); if (sourceWordBerkeleyDb != NULL) { const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper); @@ -154,7 +154,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( for (iterSourceLHS = sourceLHSSet.begin(); iterSourceLHS != sourceLHSSet.end(); ++iterSourceLHS) { const Word &sourceLHS = *iterSourceLHS; - OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS); + OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS); if (sourceLHSBerkeleyDb == NULL) { delete sourceLHSBerkeleyDb; @@ -190,7 +190,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( if (doSearch) { - OnDiskPt::Word *chartNonTermBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_outputFactorsVec, cellLabel.GetLabel()); + OnDiskPt::Word *chartNonTermBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_outputFactorsVec, cellLabel.GetLabel()); if (chartNonTermBerkeleyDb == NULL) continue; @@ -234,7 +234,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( for (iterLabelSet = lhsSet.begin(); iterLabelSet != lhsSet.end(); ++iterLabelSet) { const Word &sourceLHS = *iterLabelSet; - OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS); + OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS); if (sourceLHSBerkeleyDb == NULL) continue; diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp index 04e401080..e331a8a99 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp @@ -150,7 +150,7 @@ void PhraseDictionaryOnDisk::GetTargetPhraseCollectionBatch(InputPath &inputPath if (prevPtNode) { Word lastWord = phrase.GetWord(phrase.GetSize() - 1); lastWord.OnlyTheseFactors(m_inputFactors); - OnDiskPt::Word *lastWordOnDisk = wrapper.ConvertFromMoses(m_input, lastWord); + OnDiskPt::Word *lastWordOnDisk = ConvertFromMoses(wrapper, m_input, lastWord); TargetPhraseCollection::shared_ptr tpc; if (lastWordOnDisk == NULL) { @@ -344,6 +344,44 @@ void PhraseDictionaryOnDisk::ConvertToMoses( } } +OnDiskPt::Word *PhraseDictionaryOnDisk::ConvertFromMoses(OnDiskPt::OnDiskWrapper &wrapper, const std::vector<Moses::FactorType> &factorsVec + , const Moses::Word &origWord) const +{ + bool isNonTerminal = origWord.IsNonTerminal(); + OnDiskPt::Word *newWord = new OnDiskPt::Word(isNonTerminal); + + util::StringStream strme; + + size_t factorType = factorsVec[0]; + const Moses::Factor *factor = origWord.GetFactor(factorType); + UTIL_THROW_IF2(factor == NULL, "Expecting factor " << factorType); + strme << factor->GetString(); + + for (size_t ind = 1 ; ind < factorsVec.size() ; ++ind) { + size_t factorType = factorsVec[ind]; + const Moses::Factor *factor = origWord.GetFactor(factorType); + if (factor == NULL) { + // can have less factors than factorType.size() + break; + } + UTIL_THROW_IF2(factor == NULL, + "Expecting factor " << factorType << " at position " << ind); + strme << "|" << factor->GetString(); + } // for (size_t factorType + + bool found; + uint64_t vocabId = wrapper.GetVocab().GetVocabId(strme.str(), found); + if (!found) { + // factor not in phrase table -> phrse definately not in. exit + delete newWord; + return NULL; + } else { + newWord->SetVocabId(vocabId); + return newWord; + } + +} + void PhraseDictionaryOnDisk::SetParameter(const std::string& key, const std::string& value) { if (key == "max-span-default") { diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h index dc47936f6..03af73d08 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h @@ -107,6 +107,9 @@ public: , OnDiskPt::Vocab &vocab , bool isSyntax) const; + OnDiskPt::Word *ConvertFromMoses(OnDiskPt::OnDiskWrapper &wrapper, const std::vector<Moses::FactorType> &factorsVec + , const Moses::Word &origWord) const; + void SetParameter(const std::string& key, const std::string& value); }; |