signatures changed to cpp strings

4 files changed, 230 insertions, 213 deletions

diff --git a/contrib/python/binpt/binpt.cpp b/contrib/python/binpt/binpt.cpp
index b5b2d0f27..da06b4707 100644
--- a/contrib/python/binpt/binpt.cpp
+++ b/contrib/python/binpt/binpt.cpp
@@ -1,4 +1,4 @@
-/* Generated by Cython 0.16 on Fri Sep 14 12:51:05 2012 */
+/* Generated by Cython 0.16 on Fri Sep 14 15:34:30 2012 */
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
@@ -378,7 +378,7 @@ typedef std::string *__pyx_t_5binpt_str_pointer;
 /* "binpt.pyx":59
  *         return repr((repr(self._words), repr(self._scores), repr(self._wa)))
  * 
- * cdef QueryResult get_query_result(StringTgtCand& cand, wa = None):             # <<<<<<<<<<<<<<
+ * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None):             # <<<<<<<<<<<<<<
  *     '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string)
  *     to a QueryResult (python object).'''
  */
@@ -415,7 +415,7 @@ struct __pyx_obj_5binpt_BinaryPhraseTable {
   PyObject *_path;
   unsigned int _nscores;
   int _wa;
-  PyObject *_delimiter;
+  PyObject *_delimiters;
 };
 
 #ifndef CYTHON_REFNANNY
@@ -488,6 +488,9 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
     PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \
     const char* function_name); /*proto*/
 
+static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
+    const char *name, int exact); /*proto*/
+
 static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level); /*proto*/
 
 static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *);
@@ -572,27 +575,28 @@ static PyObject *__pyx_pf_5binpt_11QueryResult_4scores(struct __pyx_obj_5binpt_Q
 static PyObject *__pyx_pf_5binpt_11QueryResult_6wa(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5binpt_11QueryResult_8__str__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5binpt_11QueryResult_10__repr__(struct __pyx_obj_5binpt_QueryResult *__pyx_v_self); /* proto */
-static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, char *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiter); /* proto */
+static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiters); /* proto */
 static void __pyx_pf_5binpt_17BinaryPhraseTable_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObject *__pyx_v_stem, int __pyx_v_wa); /* proto */
 static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_6path(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_8nscores(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_10wa(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiter(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, char *__pyx_v_line); /* proto */
+static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_line); /* proto */
 static char __pyx_k_1[] = "UTF-8";
 static char __pyx_k_3[] = "Cannot convert %s to string";
 static char __pyx_k_4[] = " ||| ";
 static char __pyx_k_5[] = " ";
-static char __pyx_k_6[] = "'%s' doesn't seem a valid binary table.";
-static char __pyx_k_7[] = ".binphr.idx";
-static char __pyx_k_8[] = ".binphr.srctree.wa";
-static char __pyx_k_9[] = ".binphr.srcvoc";
-static char __pyx_k_10[] = ".binphr.tgtdata.wa";
-static char __pyx_k_11[] = ".binphr.tgtvoc";
-static char __pyx_k_12[] = ".binphr.srctree";
-static char __pyx_k_13[] = ".binphr.tgtdata";
-static char __pyx_k_16[] = "/media/Data/tools/moses/mosesdecoder/contrib/python/binpt/binpt.pyx";
+static char __pyx_k_6[] = " \t";
+static char __pyx_k_7[] = "'%s' doesn't seem a valid binary table.";
+static char __pyx_k_8[] = ".binphr.idx";
+static char __pyx_k_9[] = ".binphr.srctree.wa";
+static char __pyx_k_10[] = ".binphr.srcvoc";
+static char __pyx_k_11[] = ".binphr.tgtdata.wa";
+static char __pyx_k_12[] = ".binphr.tgtvoc";
+static char __pyx_k_13[] = ".binphr.srctree";
+static char __pyx_k_14[] = ".binphr.tgtdata";
+static char __pyx_k_17[] = "/media/Data/tools/moses/mosesdecoder/contrib/python/binpt/binpt.pyx";
 static char __pyx_k__os[] = "os";
 static char __pyx_k__wa[] = "wa";
 static char __pyx_k__join[] = "join";
@@ -609,8 +613,8 @@ static char __pyx_k____main__[] = "__main__";
 static char __pyx_k____test__[] = "__test__";
 static char __pyx_k__property[] = "property";
 static char __pyx_k__TypeError[] = "TypeError";
-static char __pyx_k__delimiter[] = "delimiter";
 static char __pyx_k__ValueError[] = "ValueError";
+static char __pyx_k__delimiters[] = "delimiters";
 static char __pyx_k__staticmethod[] = "staticmethod";
 static char __pyx_k__isValidBinaryTable[] = "isValidBinaryTable";
 static PyObject *__pyx_kp_s_1;
@@ -618,7 +622,8 @@ static PyObject *__pyx_kp_s_10;
 static PyObject *__pyx_kp_s_11;
 static PyObject *__pyx_kp_s_12;
 static PyObject *__pyx_kp_s_13;
-static PyObject *__pyx_kp_s_16;
+static PyObject *__pyx_kp_s_14;
+static PyObject *__pyx_kp_s_17;
 static PyObject *__pyx_kp_s_3;
 static PyObject *__pyx_kp_s_4;
 static PyObject *__pyx_kp_s_5;
@@ -631,7 +636,7 @@ static PyObject *__pyx_n_s__ValueError;
 static PyObject *__pyx_n_s____main__;
 static PyObject *__pyx_n_s____test__;
 static PyObject *__pyx_n_s__binpt;
-static PyObject *__pyx_n_s__delimiter;
+static PyObject *__pyx_n_s__delimiters;
 static PyObject *__pyx_n_s__encode;
 static PyObject *__pyx_n_s__isValidBinaryTable;
 static PyObject *__pyx_n_s__isfile;
@@ -647,8 +652,8 @@ static PyObject *__pyx_n_s__stem;
 static PyObject *__pyx_n_s__wa;
 static PyObject *__pyx_n_s__words;
 static PyObject *__pyx_k_tuple_2;
-static PyObject *__pyx_k_tuple_14;
-static PyObject *__pyx_k_codeobj_15;
+static PyObject *__pyx_k_tuple_15;
+static PyObject *__pyx_k_codeobj_16;
 
 /* "binpt.pyx":5
  * import os
@@ -1327,7 +1332,7 @@ static PyObject *__pyx_pf_5binpt_11QueryResult_10__repr__(struct __pyx_obj_5binp
  *     def __repr__(self):
  *         return repr((repr(self._words), repr(self._scores), repr(self._wa)))             # <<<<<<<<<<<<<<
  * 
- * cdef QueryResult get_query_result(StringTgtCand& cand, wa = None):
+ * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None):
  */
   __Pyx_XDECREF(__pyx_r);
   __pyx_t_1 = ((PyObject *)__pyx_v_self->_words);
@@ -1381,7 +1386,7 @@ static PyObject *__pyx_pf_5binpt_11QueryResult_10__repr__(struct __pyx_obj_5binp
 /* "binpt.pyx":59
  *         return repr((repr(self._words), repr(self._scores), repr(self._wa)))
  * 
- * cdef QueryResult get_query_result(StringTgtCand& cand, wa = None):             # <<<<<<<<<<<<<<
+ * cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None):             # <<<<<<<<<<<<<<
  *     '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string)
  *     to a QueryResult (python object).'''
  */
@@ -1497,17 +1502,17 @@ static struct __pyx_obj_5binpt_QueryResult *__pyx_f_5binpt_get_query_result(Mose
 /* Python wrapper */
 static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
 static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
-  char *__pyx_v_path;
+  PyObject *__pyx_v_path = 0;
   unsigned int __pyx_v_nscores;
   int __pyx_v_wa;
-  PyObject *__pyx_v_delimiter = 0;
-  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__path,&__pyx_n_s__nscores,&__pyx_n_s__wa,&__pyx_n_s__delimiter,0};
+  PyObject *__pyx_v_delimiters = 0;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__path,&__pyx_n_s__nscores,&__pyx_n_s__wa,&__pyx_n_s__delimiters,0};
   int __pyx_r;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
   {
     PyObject* values[4] = {0,0,0,0};
-    values[3] = ((PyObject *)__pyx_kp_s_5);
+    values[3] = ((PyObject *)__pyx_kp_s_6);
     if (unlikely(__pyx_kwds)) {
       Py_ssize_t kw_args;
       const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
@@ -1537,7 +1542,7 @@ static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self
         }
         case  3:
         if (kw_args > 0) {
-          PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__delimiter);
+          PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__delimiters);
           if (value) { values[3] = value; kw_args--; }
         }
       }
@@ -1552,9 +1557,9 @@ static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self
       } else {
 
         /* "binpt.pyx":76
- *     cdef bytes _delimiter
+ *     cdef bytes _delimiters
  * 
- *     def __cinit__(self, char* path, unsigned nscores = 5, bint wa = False, delimiter = ' '):             # <<<<<<<<<<<<<<
+ *     def __cinit__(self, bytes path, unsigned nscores = 5, bint wa = False, delimiters = ' \t'):             # <<<<<<<<<<<<<<
  *         '''It requies a path to binary phrase table (stem of the table, e.g europarl.fr-en
  *         is the stem for europar.fr-en.binphr.*).
  */
@@ -1570,7 +1575,7 @@ static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self
         default: goto __pyx_L5_argtuple_error;
       }
     }
-    __pyx_v_path = PyBytes_AsString(values[0]); if (unlikely((!__pyx_v_path) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_path = ((PyObject*)values[0]);
     if (values[1]) {
       __pyx_v_nscores = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_nscores == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
     } else {
@@ -1581,7 +1586,7 @@ static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self
     } else {
       __pyx_v_wa = ((int)0);
     }
-    __pyx_v_delimiter = values[3];
+    __pyx_v_delimiters = values[3];
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
@@ -1591,27 +1596,32 @@ static int __pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(PyObject *__pyx_v_self
   __Pyx_RefNannyFinishContext();
   return -1;
   __pyx_L4_argument_unpacking_done:;
-  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), __pyx_v_path, __pyx_v_nscores, __pyx_v_wa, __pyx_v_delimiter);
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_path), (&PyBytes_Type), 1, "path", 1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), __pyx_v_path, __pyx_v_nscores, __pyx_v_wa, __pyx_v_delimiters);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __pyx_r = -1;
+  __pyx_L0:;
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, char *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiter) {
+static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_path, unsigned int __pyx_v_nscores, int __pyx_v_wa, PyObject *__pyx_v_delimiters) {
   int __pyx_r;
   __Pyx_RefNannyDeclarations
   PyObject *__pyx_t_1 = NULL;
   PyObject *__pyx_t_2 = NULL;
   PyObject *__pyx_t_3 = NULL;
-  PyObject *__pyx_t_4 = NULL;
+  int __pyx_t_4;
   int __pyx_t_5;
-  int __pyx_t_6;
+  char *__pyx_t_6;
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__cinit__", 0);
 
   /* "binpt.pyx":83
- *         One can also specify the token delimiter, for Moses::Tokenize(text, delimiter), which is one space by default.'''
+ *         One can also specify the token delimiters, for Moses::Tokenize(text, delimiters), which is space or tab by default.'''
  * 
  *         if not BinaryPhraseTable.isValidBinaryTable(path, wa):             # <<<<<<<<<<<<<<
  *             raise ValueError, "'%s' doesn't seem a valid binary table." % path
@@ -1619,26 +1629,24 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
  */
   __pyx_t_1 = PyObject_GetAttr(((PyObject *)((PyObject*)__pyx_ptype_5binpt_BinaryPhraseTable)), __pyx_n_s__isValidBinaryTable); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = PyBytes_FromString(__pyx_v_path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(((PyObject *)__pyx_t_2));
-  __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_wa); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_wa); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
-  __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_4);
-  PyTuple_SET_ITEM(__pyx_t_4, 0, ((PyObject *)__pyx_t_2));
-  __Pyx_GIVEREF(((PyObject *)__pyx_t_2));
-  PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_3);
-  __Pyx_GIVEREF(__pyx_t_3);
+  __Pyx_INCREF(((PyObject *)__pyx_v_path));
+  PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_path));
+  __Pyx_GIVEREF(((PyObject *)__pyx_v_path));
+  PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2);
+  __Pyx_GIVEREF(__pyx_t_2);
   __pyx_t_2 = 0;
-  __pyx_t_3 = 0;
-  __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_3);
+  __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0;
-  __pyx_t_5 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_5 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-  __pyx_t_6 = (!__pyx_t_5);
-  if (__pyx_t_6) {
+  __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
+  __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  __pyx_t_5 = (!__pyx_t_4);
+  if (__pyx_t_5) {
 
     /* "binpt.pyx":84
  * 
@@ -1647,13 +1655,10 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
  *         self._path = path
  *         self._nscores = nscores
  */
-    __pyx_t_3 = PyBytes_FromString(__pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_GOTREF(((PyObject *)__pyx_t_3));
-    __pyx_t_4 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_6), ((PyObject *)__pyx_t_3)); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_GOTREF(((PyObject *)__pyx_t_4));
-    __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
-    __Pyx_Raise(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_4), 0, 0);
-    __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0;
+    __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_7), ((PyObject *)__pyx_v_path)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(((PyObject *)__pyx_t_2));
+    __Pyx_Raise(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_2), 0, 0);
+    __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
     {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     goto __pyx_L3;
   }
@@ -1666,20 +1671,18 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
  *         self._nscores = nscores
  *         self._wa = wa
  */
-  __pyx_t_4 = PyBytes_FromString(__pyx_v_path); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(((PyObject *)__pyx_t_4));
-  __Pyx_GIVEREF(((PyObject *)__pyx_t_4));
+  __Pyx_INCREF(((PyObject *)__pyx_v_path));
+  __Pyx_GIVEREF(((PyObject *)__pyx_v_path));
   __Pyx_GOTREF(__pyx_v_self->_path);
   __Pyx_DECREF(((PyObject *)__pyx_v_self->_path));
-  __pyx_v_self->_path = __pyx_t_4;
-  __pyx_t_4 = 0;
+  __pyx_v_self->_path = __pyx_v_path;
 
   /* "binpt.pyx":86
  *             raise ValueError, "'%s' doesn't seem a valid binary table." % path
  *         self._path = path
  *         self._nscores = nscores             # <<<<<<<<<<<<<<
  *         self._wa = wa
- *         self._delimiter = delimiter
+ *         self._delimiters = delimiters
  */
   __pyx_v_self->_nscores = __pyx_v_nscores;
 
@@ -1687,7 +1690,7 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
  *         self._path = path
  *         self._nscores = nscores
  *         self._wa = wa             # <<<<<<<<<<<<<<
- *         self._delimiter = delimiter
+ *         self._delimiters = delimiters
  *         self.__tree = new PhraseDictionaryTree(nscores)
  */
   __pyx_v_self->_wa = __pyx_v_wa;
@@ -1695,31 +1698,31 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
   /* "binpt.pyx":88
  *         self._nscores = nscores
  *         self._wa = wa
- *         self._delimiter = delimiter             # <<<<<<<<<<<<<<
+ *         self._delimiters = delimiters             # <<<<<<<<<<<<<<
  *         self.__tree = new PhraseDictionaryTree(nscores)
  *         self.__tree.UseWordAlignment(wa)
  */
-  if (!(likely(PyBytes_CheckExact(__pyx_v_delimiter))||((__pyx_v_delimiter) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiter)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_INCREF(__pyx_v_delimiter);
-  __Pyx_GIVEREF(__pyx_v_delimiter);
-  __Pyx_GOTREF(__pyx_v_self->_delimiter);
-  __Pyx_DECREF(((PyObject *)__pyx_v_self->_delimiter));
-  __pyx_v_self->_delimiter = ((PyObject*)__pyx_v_delimiter);
+  if (!(likely(PyBytes_CheckExact(__pyx_v_delimiters))||((__pyx_v_delimiters) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_delimiters)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_INCREF(__pyx_v_delimiters);
+  __Pyx_GIVEREF(__pyx_v_delimiters);
+  __Pyx_GOTREF(__pyx_v_self->_delimiters);
+  __Pyx_DECREF(((PyObject *)__pyx_v_self->_delimiters));
+  __pyx_v_self->_delimiters = ((PyObject*)__pyx_v_delimiters);
 
   /* "binpt.pyx":89
  *         self._wa = wa
- *         self._delimiter = delimiter
+ *         self._delimiters = delimiters
  *         self.__tree = new PhraseDictionaryTree(nscores)             # <<<<<<<<<<<<<<
  *         self.__tree.UseWordAlignment(wa)
- *         self.__tree.Read(path)
+ *         self.__tree.Read(string(path))
  */
   __pyx_v_self->__pyx___tree = new Moses::PhraseDictionaryTree(__pyx_v_nscores);
 
   /* "binpt.pyx":90
- *         self._delimiter = delimiter
+ *         self._delimiters = delimiters
  *         self.__tree = new PhraseDictionaryTree(nscores)
  *         self.__tree.UseWordAlignment(wa)             # <<<<<<<<<<<<<<
- *         self.__tree.Read(path)
+ *         self.__tree.Read(string(path))
  * 
  */
   __pyx_v_self->__pyx___tree->UseWordAlignment(__pyx_v_wa);
@@ -1727,11 +1730,12 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
   /* "binpt.pyx":91
  *         self.__tree = new PhraseDictionaryTree(nscores)
  *         self.__tree.UseWordAlignment(wa)
- *         self.__tree.Read(path)             # <<<<<<<<<<<<<<
+ *         self.__tree.Read(string(path))             # <<<<<<<<<<<<<<
  * 
  *     def __dealloc__(self):
  */
-  __pyx_v_self->__pyx___tree->Read(__pyx_v_path);
+  __pyx_t_6 = PyBytes_AsString(((PyObject *)__pyx_v_path)); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_v_self->__pyx___tree->Read(std::string(__pyx_t_6));
 
   __pyx_r = 0;
   goto __pyx_L0;
@@ -1739,7 +1743,6 @@ static int __pyx_pf_5binpt_17BinaryPhraseTable___cinit__(struct __pyx_obj_5binpt
   __Pyx_XDECREF(__pyx_t_1);
   __Pyx_XDECREF(__pyx_t_2);
   __Pyx_XDECREF(__pyx_t_3);
-  __Pyx_XDECREF(__pyx_t_4);
   __Pyx_AddTraceback("binpt.BinaryPhraseTable.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __pyx_r = -1;
   __pyx_L0:;
@@ -1757,7 +1760,7 @@ static void __pyx_pw_5binpt_17BinaryPhraseTable_3__dealloc__(PyObject *__pyx_v_s
 }
 
 /* "binpt.pyx":93
- *         self.__tree.Read(path)
+ *         self.__tree.Read(string(path))
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
  *         del self.__tree
@@ -1914,7 +1917,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
  *                 and os.path.isfile(stem + ".binphr.srctree.wa") \
  *                 and os.path.isfile(stem + ".binphr.srcvoc") \
  */
-    __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_7)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_8)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_2);
     __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
@@ -1944,7 +1947,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
       __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_3);
       __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-      __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_8)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_1);
       __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_5);
@@ -1974,7 +1977,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
         __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_5);
         __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-        __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_3);
         __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_6);
@@ -2004,7 +2007,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
           __pyx_t_6 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_6);
           __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
-          __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_5);
           __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_7);
@@ -2034,7 +2037,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
             __pyx_t_7 = PyObject_GetAttr(__pyx_t_6, __pyx_n_s__isfile); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_7);
             __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
-            __pyx_t_6 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+            __pyx_t_6 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_6);
             __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_8);
@@ -2108,7 +2111,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
  *                 and os.path.isfile(stem + ".binphr.srctree") \
  *                 and os.path.isfile(stem + ".binphr.srcvoc") \
  */
-    __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_7)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_8)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_2);
     __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
@@ -2138,7 +2141,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
       __pyx_t_3 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__isfile); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_3);
       __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-      __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_1 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_1);
       __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_5);
@@ -2168,7 +2171,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
         __pyx_t_5 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__isfile); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_5);
         __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-        __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_9)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_3 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_10)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_3);
         __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_8);
@@ -2198,7 +2201,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
           __pyx_t_8 = PyObject_GetAttr(__pyx_t_5, __pyx_n_s__isfile); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_8);
           __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
-          __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_13)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_5 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_14)); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_5);
           __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           __Pyx_GOTREF(__pyx_t_6);
@@ -2228,7 +2231,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_4isValidBinaryTable(PyObjec
             __pyx_t_6 = PyObject_GetAttr(__pyx_t_8, __pyx_n_s__isfile); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_6);
             __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
-            __pyx_t_8 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_11)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+            __pyx_t_8 = PyNumber_Add(__pyx_v_stem, ((PyObject *)__pyx_kp_s_12)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_8);
             __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
             __Pyx_GOTREF(__pyx_t_7);
@@ -2439,12 +2442,12 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_10wa(struct __pyx_obj_5binp
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiter(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
-static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiter(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) {
+static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
+static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
-  __Pyx_RefNannySetupContext("delimiter (wrapper)", 0);
-  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_12delimiter(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self));
+  __Pyx_RefNannySetupContext("delimiters (wrapper)", 0);
+  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
@@ -2452,26 +2455,26 @@ static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_13delimiter(PyObject *__pyx
 /* "binpt.pyx":126
  * 
  *     @property
- *     def delimiter(self):             # <<<<<<<<<<<<<<
- *         return self._delimiter
+ *     def delimiters(self):             # <<<<<<<<<<<<<<
+ *         return self._delimiters
  * 
  */
 
-static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiter(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) {
+static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiters(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self) {
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
-  __Pyx_RefNannySetupContext("delimiter", 0);
+  __Pyx_RefNannySetupContext("delimiters", 0);
 
   /* "binpt.pyx":127
  *     @property
- *     def delimiter(self):
- *         return self._delimiter             # <<<<<<<<<<<<<<
+ *     def delimiters(self):
+ *         return self._delimiters             # <<<<<<<<<<<<<<
  * 
- *     def query(self, char* line):
+ *     def query(self, line):
  */
   __Pyx_XDECREF(__pyx_r);
-  __Pyx_INCREF(((PyObject *)__pyx_v_self->_delimiter));
-  __pyx_r = ((PyObject *)__pyx_v_self->_delimiter);
+  __Pyx_INCREF(((PyObject *)__pyx_v_self->_delimiters));
+  __pyx_r = ((PyObject *)__pyx_v_self->_delimiters);
   goto __pyx_L0;
 
   __pyx_r = Py_None; __Pyx_INCREF(Py_None);
@@ -2482,51 +2485,41 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_12delimiter(struct __pyx_ob
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_arg_line); /*proto*/
+static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_v_line); /*proto*/
 static char __pyx_doc_5binpt_17BinaryPhraseTable_14query[] = "Queries the phrase table and returns a list of matches.\n        Each match is a QueryResult.";
-static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_arg_line) {
-  char *__pyx_v_line;
+static PyObject *__pyx_pw_5binpt_17BinaryPhraseTable_15query(PyObject *__pyx_v_self, PyObject *__pyx_v_line) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("query (wrapper)", 0);
-  assert(__pyx_arg_line); {
-    __pyx_v_line = PyBytes_AsString(__pyx_arg_line); if (unlikely((!__pyx_v_line) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
-  }
-  goto __pyx_L4_argument_unpacking_done;
-  __pyx_L3_error:;
-  __Pyx_AddTraceback("binpt.BinaryPhraseTable.query", __pyx_clineno, __pyx_lineno, __pyx_filename);
-  __Pyx_RefNannyFinishContext();
-  return NULL;
-  __pyx_L4_argument_unpacking_done:;
-  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_14query(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), ((char *)__pyx_v_line));
+  __pyx_r = __pyx_pf_5binpt_17BinaryPhraseTable_14query(((struct __pyx_obj_5binpt_BinaryPhraseTable *)__pyx_v_self), ((PyObject *)__pyx_v_line));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
 /* "binpt.pyx":129
- *         return self._delimiter
+ *         return self._delimiters
  * 
- *     def query(self, char* line):             # <<<<<<<<<<<<<<
+ *     def query(self, line):             # <<<<<<<<<<<<<<
  *         '''Queries the phrase table and returns a list of matches.
  *         Each match is a QueryResult.'''
  */
 
-static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, char *__pyx_v_line) {
+static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5binpt_BinaryPhraseTable *__pyx_v_self, PyObject *__pyx_v_line) {
   PyObject *__pyx_v_text = 0;
   std::vector<std::string> __pyx_v_fphrase;
   std::vector<Moses::StringTgtCand> *__pyx_v_rv;
   std::vector<std::string> *__pyx_v_wa;
-  PyObject *__pyx_v_phrases = 0;
+  PyObject *__pyx_v_phrases = NULL;
   size_t __pyx_v_i;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   PyObject *__pyx_t_1 = NULL;
-  PyObject *__pyx_t_2 = NULL;
+  char *__pyx_t_2;
   char *__pyx_t_3;
-  char *__pyx_t_4;
-  int __pyx_t_5;
+  int __pyx_t_4;
+  size_t __pyx_t_5;
   size_t __pyx_t_6;
-  size_t __pyx_t_7;
+  PyObject *__pyx_t_7 = NULL;
   PyObject *__pyx_t_8 = NULL;
   struct __pyx_opt_args_5binpt_get_query_result __pyx_t_9;
   int __pyx_lineno = 0;
@@ -2538,52 +2531,55 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5b
  *         '''Queries the phrase table and returns a list of matches.
  *         Each match is a QueryResult.'''
  *         cdef bytes text = as_str(line)             # <<<<<<<<<<<<<<
- *         cdef vector[string] fphrase = Tokenize(text, self.delimiter)
+ *         cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters))
  *         cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]()
  */
-  __pyx_t_1 = PyBytes_FromString(__pyx_v_line); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
-  __pyx_t_2 = ((PyObject *)__pyx_f_5binpt_as_str(((PyObject *)__pyx_t_1))); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_2);
-  __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
-  __pyx_v_text = ((PyObject*)__pyx_t_2);
-  __pyx_t_2 = 0;
+  __pyx_t_1 = ((PyObject *)__pyx_f_5binpt_as_str(__pyx_v_line)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_text = ((PyObject*)__pyx_t_1);
+  __pyx_t_1 = 0;
 
   /* "binpt.pyx":133
  *         Each match is a QueryResult.'''
  *         cdef bytes text = as_str(line)
- *         cdef vector[string] fphrase = Tokenize(text, self.delimiter)             # <<<<<<<<<<<<<<
+ *         cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters))             # <<<<<<<<<<<<<<
  *         cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]()
- *         cdef vector[string]* wa
+ *         cdef vector[string]* wa = NULL
  */
-  __pyx_t_3 = PyBytes_AsString(((PyObject *)__pyx_v_text)); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__delimiter); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_2);
-  __pyx_t_4 = PyBytes_AsString(__pyx_t_2); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
-  __pyx_v_fphrase = Moses::Tokenize(__pyx_t_3, __pyx_t_4);
+  __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_text)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyBytes_AsString(((PyObject *)__pyx_v_self->_delimiters)); if (unlikely((!__pyx_t_3) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_v_fphrase = Moses::Tokenize(std::string(__pyx_t_2), std::string(__pyx_t_3));
 
   /* "binpt.pyx":134
  *         cdef bytes text = as_str(line)
- *         cdef vector[string] fphrase = Tokenize(text, self.delimiter)
+ *         cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters))
  *         cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]()             # <<<<<<<<<<<<<<
- *         cdef vector[string]* wa
- *         cdef list phrases
+ *         cdef vector[string]* wa = NULL
+ * 
  */
   __pyx_v_rv = new std::vector<Moses::StringTgtCand>();
 
+  /* "binpt.pyx":135
+ *         cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters))
+ *         cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]()
+ *         cdef vector[string]* wa = NULL             # <<<<<<<<<<<<<<
+ * 
+ *         if not self.__tree.UseWordAlignment():
+ */
+  __pyx_v_wa = NULL;
+
   /* "binpt.pyx":137
- *         cdef vector[string]* wa
- *         cdef list phrases
+ *         cdef vector[string]* wa = NULL
+ * 
  *         if not self.__tree.UseWordAlignment():             # <<<<<<<<<<<<<<
  *             self.__tree.GetTargetCandidates(fphrase, rv[0])
  *             phrases = [get_query_result(rv[0][i]) for i in range(rv.size())]
  */
-  __pyx_t_5 = (!__pyx_v_self->__pyx___tree->UseWordAlignment());
-  if (__pyx_t_5) {
+  __pyx_t_4 = (!__pyx_v_self->__pyx___tree->UseWordAlignment());
+  if (__pyx_t_4) {
 
     /* "binpt.pyx":138
- *         cdef list phrases
+ * 
  *         if not self.__tree.UseWordAlignment():
  *             self.__tree.GetTargetCandidates(fphrase, rv[0])             # <<<<<<<<<<<<<<
  *             phrases = [get_query_result(rv[0][i]) for i in range(rv.size())]
@@ -2598,19 +2594,19 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5b
  *         else:
  *             wa = new vector[string]()
  */
-    __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_GOTREF(__pyx_t_2);
-    __pyx_t_6 = __pyx_v_rv->size();
-    for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) {
-      __pyx_v_i = __pyx_t_7;
-      __pyx_t_1 = ((PyObject *)__pyx_f_5binpt_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), NULL)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __Pyx_GOTREF(__pyx_t_1);
-      if (unlikely(PyList_Append(__pyx_t_2, (PyObject*)__pyx_t_1))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+    __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_5 = __pyx_v_rv->size();
+    for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) {
+      __pyx_v_i = __pyx_t_6;
+      __pyx_t_7 = ((PyObject *)__pyx_f_5binpt_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), NULL)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_7);
+      if (unlikely(PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_7))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
     }
-    __Pyx_INCREF(((PyObject *)__pyx_t_2));
-    __pyx_v_phrases = __pyx_t_2;
-    __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+    __Pyx_INCREF(((PyObject *)__pyx_t_1));
+    __pyx_v_phrases = __pyx_t_1;
+    __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
     goto __pyx_L3;
   }
   /*else*/ {
@@ -2640,24 +2636,24 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5b
  *             del wa
  *         del rv
  */
-    __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_GOTREF(__pyx_t_2);
-    __pyx_t_6 = __pyx_v_rv->size();
-    for (__pyx_t_7 = 0; __pyx_t_7 < __pyx_t_6; __pyx_t_7+=1) {
-      __pyx_v_i = __pyx_t_7;
-      __pyx_t_1 = PyBytes_FromString(((__pyx_v_wa[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+    __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_5 = __pyx_v_rv->size();
+    for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) {
+      __pyx_v_i = __pyx_t_6;
+      __pyx_t_7 = PyBytes_FromString(((__pyx_v_wa[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(((PyObject *)__pyx_t_7));
       __pyx_t_9.__pyx_n = 1;
-      __pyx_t_9.wa = ((PyObject *)__pyx_t_1);
+      __pyx_t_9.wa = ((PyObject *)__pyx_t_7);
       __pyx_t_8 = ((PyObject *)__pyx_f_5binpt_get_query_result(((__pyx_v_rv[0])[__pyx_v_i]), &__pyx_t_9)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_8);
-      __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
-      if (unlikely(PyList_Append(__pyx_t_2, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0;
+      if (unlikely(PyList_Append(__pyx_t_1, (PyObject*)__pyx_t_8))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
     }
-    __Pyx_INCREF(((PyObject *)__pyx_t_2));
-    __pyx_v_phrases = __pyx_t_2;
-    __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+    __Pyx_INCREF(((PyObject *)__pyx_t_1));
+    __pyx_v_phrases = __pyx_t_1;
+    __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
 
     /* "binpt.pyx":144
  *             self.__tree.GetTargetCandidates(fphrase, rv[0], wa[0])
@@ -2694,7 +2690,7 @@ static PyObject *__pyx_pf_5binpt_17BinaryPhraseTable_14query(struct __pyx_obj_5b
   goto __pyx_L0;
   __pyx_L1_error:;
   __Pyx_XDECREF(__pyx_t_1);
-  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_7);
   __Pyx_XDECREF(__pyx_t_8);
   __Pyx_AddTraceback("binpt.BinaryPhraseTable.query", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __pyx_r = NULL;
@@ -2925,7 +2921,7 @@ static PyObject *__pyx_tp_new_5binpt_BinaryPhraseTable(PyTypeObject *t, PyObject
   if (!o) return 0;
   p = ((struct __pyx_obj_5binpt_BinaryPhraseTable *)o);
   p->_path = ((PyObject*)Py_None); Py_INCREF(Py_None);
-  p->_delimiter = ((PyObject*)Py_None); Py_INCREF(Py_None);
+  p->_delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None);
   if (__pyx_pw_5binpt_17BinaryPhraseTable_1__cinit__(o, a, k) < 0) {
     Py_DECREF(o); o = 0;
   }
@@ -2944,7 +2940,7 @@ static void __pyx_tp_dealloc_5binpt_BinaryPhraseTable(PyObject *o) {
     PyErr_Restore(etype, eval, etb);
   }
   Py_XDECREF(((PyObject *)p->_path));
-  Py_XDECREF(((PyObject *)p->_delimiter));
+  Py_XDECREF(((PyObject *)p->_delimiters));
   (*Py_TYPE(o)->tp_free)(o);
 }
 
@@ -2954,8 +2950,8 @@ static int __pyx_tp_traverse_5binpt_BinaryPhraseTable(PyObject *o, visitproc v,
   if (p->_path) {
     e = (*v)(p->_path, a); if (e) return e;
   }
-  if (p->_delimiter) {
-    e = (*v)(p->_delimiter, a); if (e) return e;
+  if (p->_delimiters) {
+    e = (*v)(p->_delimiters, a); if (e) return e;
   }
   return 0;
 }
@@ -2966,8 +2962,8 @@ static int __pyx_tp_clear_5binpt_BinaryPhraseTable(PyObject *o) {
   tmp = ((PyObject*)p->_path);
   p->_path = ((PyObject*)Py_None); Py_INCREF(Py_None);
   Py_XDECREF(tmp);
-  tmp = ((PyObject*)p->_delimiter);
-  p->_delimiter = ((PyObject*)Py_None); Py_INCREF(Py_None);
+  tmp = ((PyObject*)p->_delimiters);
+  p->_delimiters = ((PyObject*)Py_None); Py_INCREF(Py_None);
   Py_XDECREF(tmp);
   return 0;
 }
@@ -2977,7 +2973,7 @@ static PyMethodDef __pyx_methods_5binpt_BinaryPhraseTable[] = {
   {__Pyx_NAMESTR("path"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_7path, METH_NOARGS, __Pyx_DOCSTR(0)},
   {__Pyx_NAMESTR("nscores"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_9nscores, METH_NOARGS, __Pyx_DOCSTR(0)},
   {__Pyx_NAMESTR("wa"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_11wa, METH_NOARGS, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("delimiter"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_13delimiter, METH_NOARGS, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("delimiters"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_13delimiters, METH_NOARGS, __Pyx_DOCSTR(0)},
   {__Pyx_NAMESTR("query"), (PyCFunction)__pyx_pw_5binpt_17BinaryPhraseTable_15query, METH_O, __Pyx_DOCSTR(__pyx_doc_5binpt_17BinaryPhraseTable_14query)},
   {0, 0, 0, 0}
 };
@@ -3160,7 +3156,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_kp_s_11, __pyx_k_11, sizeof(__pyx_k_11), 0, 0, 1, 0},
   {&__pyx_kp_s_12, __pyx_k_12, sizeof(__pyx_k_12), 0, 0, 1, 0},
   {&__pyx_kp_s_13, __pyx_k_13, sizeof(__pyx_k_13), 0, 0, 1, 0},
-  {&__pyx_kp_s_16, __pyx_k_16, sizeof(__pyx_k_16), 0, 0, 1, 0},
+  {&__pyx_kp_s_14, __pyx_k_14, sizeof(__pyx_k_14), 0, 0, 1, 0},
+  {&__pyx_kp_s_17, __pyx_k_17, sizeof(__pyx_k_17), 0, 0, 1, 0},
   {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0},
   {&__pyx_kp_s_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 1, 0},
   {&__pyx_kp_s_5, __pyx_k_5, sizeof(__pyx_k_5), 0, 0, 1, 0},
@@ -3173,7 +3170,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1},
   {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1},
   {&__pyx_n_s__binpt, __pyx_k__binpt, sizeof(__pyx_k__binpt), 0, 0, 1, 1},
-  {&__pyx_n_s__delimiter, __pyx_k__delimiter, sizeof(__pyx_k__delimiter), 0, 0, 1, 1},
+  {&__pyx_n_s__delimiters, __pyx_k__delimiters, sizeof(__pyx_k__delimiters), 0, 0, 1, 1},
   {&__pyx_n_s__encode, __pyx_k__encode, sizeof(__pyx_k__encode), 0, 0, 1, 1},
   {&__pyx_n_s__isValidBinaryTable, __pyx_k__isValidBinaryTable, sizeof(__pyx_k__isValidBinaryTable), 0, 0, 1, 1},
   {&__pyx_n_s__isfile, __pyx_k__isfile, sizeof(__pyx_k__isfile), 0, 0, 1, 1},
@@ -3226,16 +3223,16 @@ static int __Pyx_InitCachedConstants(void) {
  *         '''This sanity check was added to the constructor, but you can access it from outside this class
  *         to determine whether or not you are providing a valid stem to BinaryPhraseTable.'''
  */
-  __pyx_k_tuple_14 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_14);
+  __pyx_k_tuple_15 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_15);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__stem));
-  PyTuple_SET_ITEM(__pyx_k_tuple_14, 0, ((PyObject *)__pyx_n_s__stem));
+  PyTuple_SET_ITEM(__pyx_k_tuple_15, 0, ((PyObject *)__pyx_n_s__stem));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stem));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__wa));
-  PyTuple_SET_ITEM(__pyx_k_tuple_14, 1, ((PyObject *)__pyx_n_s__wa));
+  PyTuple_SET_ITEM(__pyx_k_tuple_15, 1, ((PyObject *)__pyx_n_s__wa));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__wa));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_14));
-  __pyx_k_codeobj_15 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_16, __pyx_n_s__isValidBinaryTable, 97, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_15));
+  __pyx_k_codeobj_16 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_17, __pyx_n_s__isValidBinaryTable, 97, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_16)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_RefNannyFinishContext();
   return 0;
   __pyx_L1_error:;
@@ -3518,11 +3515,11 @@ PyMODINIT_FUNC PyInit_binpt(void)
   /* "binpt.pyx":126
  * 
  *     @property
- *     def delimiter(self):             # <<<<<<<<<<<<<<
- *         return self._delimiter
+ *     def delimiters(self):             # <<<<<<<<<<<<<<
+ *         return self._delimiters
  * 
  */
-  __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__delimiter); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_GetName((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable, __pyx_n_s__delimiters); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
@@ -3532,7 +3529,7 @@ PyMODINIT_FUNC PyInit_binpt(void)
   __pyx_t_1 = PyObject_Call(__pyx_builtin_property, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
-  if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__delimiter, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyDict_SetItem((PyObject *)__pyx_ptype_5binpt_BinaryPhraseTable->tp_dict, __pyx_n_s__delimiters, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   PyType_Modified(__pyx_ptype_5binpt_BinaryPhraseTable);
 
@@ -3864,6 +3861,26 @@ bad:
     return -1;
 }
 
+static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
+    const char *name, int exact)
+{
+    if (!type) {
+        PyErr_Format(PyExc_SystemError, "Missing type object");
+        return 0;
+    }
+    if (none_allowed && obj == Py_None) return 1;
+    else if (exact) {
+        if (Py_TYPE(obj) == type) return 1;
+    }
+    else {
+        if (PyObject_TypeCheck(obj, type)) return 1;
+    }
+    PyErr_Format(PyExc_TypeError,
+        "Argument '%s' has incorrect type (expected %s, got %s)",
+        name, type->tp_name, Py_TYPE(obj)->tp_name);
+    return 0;
+}
+
 static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level) {
     PyObject *py_import = 0;
     PyObject *empty_list = 0;
diff --git a/contrib/python/binpt/binpt.pxd b/contrib/python/binpt/binpt.pxd
index fc7fb0352..33661ceaf 100644
--- a/contrib/python/binpt/binpt.pxd
+++ b/contrib/python/binpt/binpt.pxd
@@ -13,7 +13,7 @@ cdef extern from 'PhraseDictionaryTree.h' namespace 'Moses':
         PhraseDictionaryTree(unsigned nscores)
         void UseWordAlignment(bint use)
         bint UseWordAlignment()
-        int Read(char* path)
+        int Read(string& path)
         void GetTargetCandidates(vector[string]& fs, 
                 vector[StringTgtCand]& rv)
         void GetTargetCandidates(vector[string]& fs, 
@@ -21,5 +21,5 @@ cdef extern from 'PhraseDictionaryTree.h' namespace 'Moses':
                 vector[string]& wa)
 
 cdef extern from 'Util.h' namespace 'Moses':
-    cdef vector[string] Tokenize(char* text, char* delimiters)
+    cdef vector[string] Tokenize(string& text, string& delimiters)
 
diff --git a/contrib/python/binpt/binpt.pyx b/contrib/python/binpt/binpt.pyx
index 0f8d06c42..7a5f21dd3 100644
--- a/contrib/python/binpt/binpt.pyx
+++ b/contrib/python/binpt/binpt.pyx
@@ -56,7 +56,7 @@ cdef class QueryResult(object):
     def __repr__(self):
         return repr((repr(self._words), repr(self._scores), repr(self._wa)))
 
-cdef QueryResult get_query_result(StringTgtCand& cand, wa = None):
+cdef QueryResult get_query_result(StringTgtCand& cand, object wa = None):
     '''Converts a StringTgtCandidate (c++ object) and possibly a word-alignment info (string)
     to a QueryResult (python object).'''
     cdef tuple words = tuple([cand.first[i].c_str() for i in range(cand.first.size())])
@@ -71,24 +71,24 @@ cdef class BinaryPhraseTable(object):
     cdef bytes _path
     cdef unsigned _nscores
     cdef bint _wa
-    cdef bytes _delimiter
+    cdef bytes _delimiters
 
-    def __cinit__(self, char* path, unsigned nscores = 5, bint wa = False, delimiter = ' '):
+    def __cinit__(self, bytes path, unsigned nscores = 5, bint wa = False, delimiters = ' \t'):
         '''It requies a path to binary phrase table (stem of the table, e.g europarl.fr-en 
         is the stem for europar.fr-en.binphr.*).
         Moses::PhraseDictionaryTree also needs to be aware of the number of scores (usually 5),
         and whether or not there is word-alignment info in the table (usually not).
-        One can also specify the token delimiter, for Moses::Tokenize(text, delimiter), which is one space by default.'''
+        One can also specify the token delimiters, for Moses::Tokenize(text, delimiters), which is space or tab by default.'''
 
         if not BinaryPhraseTable.isValidBinaryTable(path, wa):
             raise ValueError, "'%s' doesn't seem a valid binary table." % path
         self._path = path
         self._nscores = nscores
         self._wa = wa
-        self._delimiter = delimiter
+        self._delimiters = delimiters
         self.__tree = new PhraseDictionaryTree(nscores)
         self.__tree.UseWordAlignment(wa)
-        self.__tree.Read(path)
+        self.__tree.Read(string(path))
 
     def __dealloc__(self):
         del self.__tree
@@ -123,17 +123,17 @@ cdef class BinaryPhraseTable(object):
         return self._wa
 
     @property
-    def delimiter(self):
-        return self._delimiter
+    def delimiters(self):
+        return self._delimiters
 
-    def query(self, char* line):
+    def query(self, line):
         '''Queries the phrase table and returns a list of matches.
         Each match is a QueryResult.'''
         cdef bytes text = as_str(line)
-        cdef vector[string] fphrase = Tokenize(text, self.delimiter)
+        cdef vector[string] fphrase = Tokenize(string(text), string(self._delimiters))
         cdef vector[StringTgtCand]* rv = new vector[StringTgtCand]()
-        cdef vector[string]* wa
-        cdef list phrases
+        cdef vector[string]* wa = NULL
+        
         if not self.__tree.UseWordAlignment():
             self.__tree.GetTargetCandidates(fphrase, rv[0])
             phrases = [get_query_result(rv[0][i]) for i in range(rv.size())]
diff --git a/contrib/python/example.py b/contrib/python/example.py
index 21080be5c..08f17ac33 100644
--- a/contrib/python/example.py
+++ b/contrib/python/example.py
@@ -1,5 +1,5 @@
 from binpt import BinaryPhraseTable
-from binpt import QueryResult
+#from binpt import QueryResult
 
 import sys
 
@@ -12,7 +12,7 @@ nscores = int(sys.argv[2])
 wa = len(sys.argv) == 4
 
 pt = BinaryPhraseTable(pt_file, nscores, wa)
-print >> sys.stderr, "-ttable %s -nscores %d -alignment-info %s -delimiter '%s'\n" %(pt.path, pt.nscores, str(pt.wa), pt.delimiter)
+print >> sys.stderr, "-ttable %s -nscores %d -alignment-info %s -delimiter '%s'\n" %(pt.path, pt.nscores, str(pt.wa), pt.delimiters)
 
 for line in sys.stdin:
     f = line.strip()
@@ -26,6 +26,6 @@ for line in sys.stdin:
         if e.wa:
             print e.wa # string
     '''
-        
- 
+            
+