Binary phrase table does string ops, at least make them fast

1 files changed, 4 insertions, 4 deletions

diff --git a/moses/PDTAimp.h b/moses/PDTAimp.h
index 25131b98a..5680b8ecb 100644
--- a/moses/PDTAimp.h
+++ b/moses/PDTAimp.h
@@ -11,6 +11,7 @@
 #include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
 #include "SparsePhraseDictionaryFeature.h"
 #include "Util.h"
+#include "util/tokenize_piece.hh"
 
 namespace Moses
 {
@@ -284,11 +285,10 @@ protected:
     FactorCollection &factorCollection = FactorCollection::Instance();
 
     for(size_t k=0; k<factorStrings.size(); ++k) {
-      std::vector<std::string> factors=TokenizeMultiCharSeparator(*factorStrings[k],StaticData::Instance().GetFactorDelimiter());
-      CHECK(factors.size()==m_output.size());
+      util::TokenIter<util::MultiCharacter, false> word(*factorStrings[k], StaticData::Instance().GetFactorDelimiter());
       Word& w=targetPhrase.AddWord();
-      for(size_t l=0; l<m_output.size(); ++l) {
-        w[m_output[l]]= factorCollection.AddFactor(Output, m_output[l], factors[l]);
+      for(size_t l=0; l<m_output.size(); ++l, ++word) {
+        w[m_output[l]]= factorCollection.AddFactor(*word);
       }
     }