diff options
author | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2014-05-31 18:22:37 +0400 |
---|---|---|
committer | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2014-05-31 18:22:37 +0400 |
commit | d1e5a544f19c5e1f3f1a3b8804dbfa592b84669b (patch) | |
tree | c13c077dedfd1c6843426027e70791c59eb7831b /moses/Word.cpp | |
parent | 249c40ec74f182cf92600341ea07de6666a6281a (diff) |
Word.CreateFromString() now bypasses factor splitting if the factor delimiter is th empty string (specify as: none).
Diffstat (limited to 'moses/Word.cpp')
-rw-r--r-- | moses/Word.cpp | 51 |
1 files changed, 28 insertions, 23 deletions
diff --git a/moses/Word.cpp b/moses/Word.cpp index 986d667da..800d0bb33 100644 --- a/moses/Word.cpp +++ b/moses/Word.cpp @@ -108,35 +108,40 @@ CreateFromString(FactorDirection direction { FactorCollection &factorCollection = FactorCollection::Instance(); vector<StringPiece> bits(MAX_NUM_FACTORS); - util::TokenIter<util::MultiCharacter> - fit(str, StaticData::Instance().GetFactorDelimiter()); - size_t i = 0; - for (; i < MAX_NUM_FACTORS && fit; ++i,++fit) - bits[i] = *fit; - if (i == MAX_NUM_FACTORS) - UTIL_THROW_IF(fit, StrayFactorException, - "The hard limit for factors is " << MAX_NUM_FACTORS - << ". The word " << str << " contains factor delimiter " - << StaticData::Instance().GetFactorDelimiter() - << " too many times."); - if (strict) - UTIL_THROW_IF(fit, StrayFactorException, - "You have configured " << factorOrder.size() - << " factors but the word " << str - << " contains factor delimiter " - << StaticData::Instance().GetFactorDelimiter() - << " too many times."); - - UTIL_THROW_IF(i < factorOrder.size(),util::Exception, - "Too few factors in string '" << str << "'."); - + string factorDelimiter = StaticData::Instance().GetFactorDelimiter(); + if (factorDelimiter.size()) + { + util::TokenIter<util::MultiCharacter> fit(str, factorDelimiter); + size_t i = 0; + for (; i < MAX_NUM_FACTORS && fit; ++i,++fit) + bits[i] = *fit; + if (i == MAX_NUM_FACTORS) + UTIL_THROW_IF(fit, StrayFactorException, + "The hard limit for factors is " << MAX_NUM_FACTORS + << ". The word " << str << " contains factor delimiter " + << StaticData::Instance().GetFactorDelimiter() + << " too many times."); + if (strict) + UTIL_THROW_IF(fit, StrayFactorException, + "You have configured " << factorOrder.size() + << " factors but the word " << str + << " contains factor delimiter " + << StaticData::Instance().GetFactorDelimiter() + << " too many times."); + + UTIL_THROW_IF(i < factorOrder.size(),util::Exception, + "Too few factors in string '" << str << "'."); + } + else + { + bits[0] = str; + } for (size_t k = 0; k < factorOrder.size(); ++k) { UTIL_THROW_IF(factorOrder[k] >= MAX_NUM_FACTORS, util::Exception, "Factor order out of bounds."); m_factorArray[factorOrder[k]] = factorCollection.AddFactor(bits[k], isNonTerminal); } - // assume term/non-term same for all factors m_isNonTerminal = isNonTerminal; } |