diff options
author | Rico Sennrich <rico.sennrich@gmx.ch> | 2014-03-21 14:53:15 +0400 |
---|---|---|
committer | Rico Sennrich <rico.sennrich@gmx.ch> | 2014-03-21 15:12:24 +0400 |
commit | 45630a5851fad3bdd6953e88727e4b7c8d0c4c18 (patch) | |
tree | cd95a3175eb7fc4bb1027cf92d2fa880112b3fee /moses/Word.cpp | |
parent | 1c6061e78174d09ea4a7a8125ee0df7f41d88ae5 (diff) |
various optimizations to make CYK+ parser several times faster and eat less memory.
speed-up of decoding depends on how much time is spent in parser:
10-50% speed-up for string-to-tree systems observed (more on long sentences and with high max-chart-span).
if you only use hiero or string-to-tree models (but none with source syntax), use compile-option --unlabelled-source for (small) efficiency gains.
Diffstat (limited to 'moses/Word.cpp')
-rw-r--r-- | moses/Word.cpp | 10 |
1 files changed, 6 insertions, 4 deletions
diff --git a/moses/Word.cpp b/moses/Word.cpp index 95dd3fdf4..16e184ebc 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -107,7 +107,7 @@ void Word::CreateFromString(FactorDirection direction util::TokenIter<util::MultiCharacter> fit(str, StaticData::Instance().GetFactorDelimiter()); for (size_t ind = 0; ind < factorOrder.size() && fit; ++ind, ++fit) { - m_factorArray[factorOrder[ind]] = factorCollection.AddFactor(*fit); + m_factorArray[factorOrder[ind]] = factorCollection.AddFactor(*fit, isNonTerminal); } UTIL_THROW_IF(fit, StrayFactorException, "You have configured " << factorOrder.size() << " factors but the word " << str << " contains factor delimiter " << StaticData::Instance().GetFactorDelimiter() << " too many times."); @@ -119,16 +119,18 @@ void Word::CreateUnknownWord(const Word &sourceWord) { FactorCollection &factorCollection = FactorCollection::Instance(); + m_isNonTerminal = sourceWord.IsNonTerminal(); + for (unsigned int currFactor = 0 ; currFactor < MAX_NUM_FACTORS ; currFactor++) { FactorType factorType = static_cast<FactorType>(currFactor); const Factor *sourceFactor = sourceWord[currFactor]; if (sourceFactor == NULL) - SetFactor(factorType, factorCollection.AddFactor(Output, factorType, UNKNOWN_FACTOR)); + SetFactor(factorType, factorCollection.AddFactor(Output, factorType, UNKNOWN_FACTOR, m_isNonTerminal)); else - SetFactor(factorType, factorCollection.AddFactor(Output, factorType, sourceFactor->GetString())); + SetFactor(factorType, factorCollection.AddFactor(Output, factorType, sourceFactor->GetString(), m_isNonTerminal)); } - m_isNonTerminal = sourceWord.IsNonTerminal(); + m_isOOV = true; } |