diff options
author | Kenneth Heafield <github@kheafield.com> | 2013-04-10 21:27:25 +0400 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2013-04-10 21:27:25 +0400 |
commit | 73035543d6086d98185b38d313a670176adde1f2 (patch) | |
tree | b0d3ae61382c5f9dddb954236ca3b3059c984989 /moses/PDTAimp.h | |
parent | 44a0e52e3052371a850f7ee463279f4cc0522ea5 (diff) |
Binary phrase table does string ops, at least make them fast
Diffstat (limited to 'moses/PDTAimp.h')
-rw-r--r-- | moses/PDTAimp.h | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/moses/PDTAimp.h b/moses/PDTAimp.h index 25131b98a..5680b8ecb 100644 --- a/moses/PDTAimp.h +++ b/moses/PDTAimp.h @@ -11,6 +11,7 @@ #include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h" #include "SparsePhraseDictionaryFeature.h" #include "Util.h" +#include "util/tokenize_piece.hh" namespace Moses { @@ -284,11 +285,10 @@ protected: FactorCollection &factorCollection = FactorCollection::Instance(); for(size_t k=0; k<factorStrings.size(); ++k) { - std::vector<std::string> factors=TokenizeMultiCharSeparator(*factorStrings[k],StaticData::Instance().GetFactorDelimiter()); - CHECK(factors.size()==m_output.size()); + util::TokenIter<util::MultiCharacter, false> word(*factorStrings[k], StaticData::Instance().GetFactorDelimiter()); Word& w=targetPhrase.AddWord(); - for(size_t l=0; l<m_output.size(); ++l) { - w[m_output[l]]= factorCollection.AddFactor(Output, m_output[l], factors[l]); + for(size_t l=0; l<m_output.size(); ++l, ++word) { + w[m_output[l]]= factorCollection.AddFactor(*word); } } |