diff options
| author | Victor Chahuneau <vchahune@cs.cmu.edu> | 2013-06-05 21:05:36 +0400 |
|---|---|---|
| committer | Victor Chahuneau <vchahune@cs.cmu.edu> | 2013-06-05 21:05:36 +0400 |
| commit | a92a691a2d4289738f8eae41ae15ff6c0c8dd1d8 (patch) | |
| tree | bcc7eafbec6d4e350ac25cef62076e10826bb508 /python | |
| parent | 8791fa7b8a3dcff31423c9d01ad5a3e7abb48614 (diff) | |
| parent | 10dd0f0b43375569cf4f175814aabf18ad24bc9c (diff) | |
Merged Kenneth's changes to the Python module (stack alloc)
+ added proper IOError exception
+ fixed -lrt problem for Darwin
Diffstat (limited to 'python')
| -rw-r--r-- | python/kenlm.cpp | 1621 | ||||
| -rw-r--r-- | python/kenlm.pxd | 29 | ||||
| -rw-r--r-- | python/kenlm.pyx | 50 |
3 files changed, 935 insertions, 765 deletions
diff --git a/python/kenlm.cpp b/python/kenlm.cpp index 5379fd4..d401047 100644 --- a/python/kenlm.cpp +++ b/python/kenlm.cpp @@ -1,6 +1,18 @@ -/* Generated by Cython 0.18 on Thu Apr 4 10:54:01 2013 */ +/* Generated by Cython 0.19.1 on Wed Jun 5 13:00:19 2013 */ #define PY_SSIZE_T_CLEAN +#ifndef CYTHON_USE_PYLONG_INTERNALS +#ifdef PYLONG_BITS_IN_DIGIT +#define CYTHON_USE_PYLONG_INTERNALS 0 +#else +#include "pyconfig.h" +#ifdef PYLONG_BITS_IN_DIGIT +#define CYTHON_USE_PYLONG_INTERNALS 1 +#else +#define CYTHON_USE_PYLONG_INTERNALS 0 +#endif +#endif +#endif #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. @@ -116,6 +128,9 @@ #if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) #define Py_TPFLAGS_HAVE_NEWBUFFER 0 #endif +#if PY_VERSION_HEX < 0x02060000 + #define Py_TPFLAGS_HAVE_VERSION_TAG 0 +#endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ @@ -155,6 +170,14 @@ #define PyBytes_Concat PyString_Concat #define PyBytes_ConcatAndDel PyString_ConcatAndDel #endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj) || \ + PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (Py_TYPE(obj) == &PyBaseString_Type) +#endif #if PY_VERSION_HEX < 0x02060000 #define PySet_Check(obj) PyObject_TypeCheck(obj, &PySet_Type) #define PyFrozenSet_Check(obj) PyObject_TypeCheck(obj, &PyFrozenSet_Type) @@ -227,6 +250,40 @@ #define __Pyx_NAMESTR(n) (n) #define __Pyx_DOCSTR(n) (n) #endif +#ifndef CYTHON_INLINE + #if defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and + a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is + a quiet NaN. */ + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif #if PY_MAJOR_VERSION >= 3 @@ -252,6 +309,9 @@ #define __PYX_HAVE__kenlm #define __PYX_HAVE_API__kenlm #include "lm/word_index.hh" +#include "lm/return.hh" +#include "lm/state.hh" +#include "lm/virtual_interface.hh" #include "lm/model.hh" #include "ios" #include "new" @@ -265,17 +325,6 @@ #define CYTHON_WITHOUT_ASSERTIONS #endif -#ifndef CYTHON_INLINE - #if defined(__GNUC__) - #define CYTHON_INLINE __inline__ - #elif defined(_MSC_VER) - #define CYTHON_INLINE __inline - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_INLINE inline - #else - #define CYTHON_INLINE - #endif -#endif #ifndef CYTHON_UNUSED # if defined(__GNUC__) # if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) @@ -289,10 +338,44 @@ # define CYTHON_UNUSED # endif #endif -typedef struct {PyObject **p; char *s; const long n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ - -#define __Pyx_PyBytes_FromUString(s) PyBytes_FromString((char*)s) -#define __Pyx_PyBytes_AsUString(s) ((unsigned char*) PyBytes_AsString(s)) +typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromUString(s) __Pyx_PyObject_FromString((char*)s) +#define __Pyx_PyBytes_FromUString(s) __Pyx_PyBytes_FromString((char*)s) +#define __Pyx_PyStr_FromUString(s) __Pyx_PyStr_FromString((char*)s) +#define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return u_end - u - 1; +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode #define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None) #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); @@ -306,6 +389,78 @@ static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); #define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) #endif #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params() { + PyObject* sys = NULL; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + sys = PyImport_ImportModule("sys"); + if (sys == NULL) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + if (default_encoding == NULL) goto bad; + if (strcmp(PyBytes_AsString(default_encoding), "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + const char* default_encoding_c = PyBytes_AS_STRING(default_encoding); + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (ascii_chars_u == NULL) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (ascii_chars_b == NULL || strncmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + } + Py_XDECREF(sys); + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return 0; +bad: + Py_XDECREF(sys); + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params() { + PyObject* sys = NULL; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (sys == NULL) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + if (default_encoding == NULL) goto bad; + default_encoding_c = PyBytes_AS_STRING(default_encoding); + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(sys); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(sys); + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif #ifdef __GNUC__ @@ -321,8 +476,9 @@ static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ - + static PyObject *__pyx_m; +static PyObject *__pyx_d; static PyObject *__pyx_b; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; @@ -349,26 +505,26 @@ struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores; */ struct __pyx_obj_5kenlm_LanguageModel { PyObject_HEAD - lm::ngram::Model *model; + lm::base::Model *model; PyObject *path; - const lm::ngram::Vocabulary *vocab; + const lm::base::Vocabulary *vocab; }; -/* "kenlm.pyx":42 +/* "kenlm.pyx":44 * return total * * def full_scores(self, sentence): # <<<<<<<<<<<<<< * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) + * cdef State state */ struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores { PyObject_HEAD - lm::ngram::State *__pyx_v_out_state; - lm::FullScoreReturn *__pyx_v_ret; + struct lm::ngram::State __pyx_v_out_state; + struct lm::FullScoreReturn __pyx_v_ret; struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self; PyObject *__pyx_v_sentence; - lm::ngram::State *__pyx_v_state; + struct lm::ngram::State __pyx_v_state; float __pyx_v_total; PyObject *__pyx_v_word; PyObject *__pyx_v_words; @@ -430,7 +586,22 @@ struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores { #define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) #define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ @@ -446,8 +617,12 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); /*proto*/ + static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ +#include <string.h> + static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ @@ -524,6 +699,26 @@ static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); +static PyObject* __Pyx_PyObject_CallMethodTuple(PyObject* obj, PyObject* method_name, PyObject* args) { + PyObject *method, *result = NULL; + if (unlikely(!args)) return NULL; + method = __Pyx_PyObject_GetAttrStr(obj, method_name); + if (unlikely(!method)) goto bad; + result = PyObject_Call(method, args, NULL); + Py_DECREF(method); +bad: + Py_DECREF(args); + return result; +} +#define __Pyx_PyObject_CallMethod3(obj, name, arg1, arg2, arg3) \ + __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(3, arg1, arg2, arg3)) +#define __Pyx_PyObject_CallMethod2(obj, name, arg1, arg2) \ + __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(2, arg1, arg2)) +#define __Pyx_PyObject_CallMethod1(obj, name, arg1) \ + __Pyx_PyObject_CallMethodTuple(obj, name, PyTuple_Pack(1, arg1)) +#define __Pyx_PyObject_CallMethod0(obj, name) \ + __Pyx_PyObject_CallMethodTuple(obj, name, (Py_INCREF(__pyx_empty_tuple), __pyx_empty_tuple)) + static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ #define __Pyx_Generator_USED @@ -597,39 +792,58 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_13__reduce__(struct __pyx_obj_5 static PyObject *__pyx_pf_5kenlm_13LanguageModel_4path___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */ static int __pyx_pf_5kenlm_13LanguageModel_4path_2__set__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_value); /* proto */ static int __pyx_pf_5kenlm_13LanguageModel_4path_4__del__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */ +static PyObject *__pyx_tp_new_5kenlm_LanguageModel(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static char __pyx_k_2[] = "Cannot convert %s to string"; -static char __pyx_k_3[] = "Cannot read model '{}' ({})"; -static char __pyx_k_4[] = "<LanguageModel from {0}>"; +static char __pyx_k_3[] = "\n"; +static char __pyx_k_4[] = " "; +static char __pyx_k_6[] = "Cannot read model '{}' ({})"; +static char __pyx_k_7[] = "<LanguageModel from {0}>"; static char __pyx_k__os[] = "os"; +static char __pyx_k__args[] = "args"; static char __pyx_k__path[] = "path"; +static char __pyx_k__send[] = "send"; static char __pyx_k__utf8[] = "utf8"; +static char __pyx_k__close[] = "close"; static char __pyx_k__split[] = "split"; +static char __pyx_k__throw[] = "throw"; static char __pyx_k__encode[] = "encode"; static char __pyx_k__format[] = "format"; static char __pyx_k__IOError[] = "IOError"; static char __pyx_k__abspath[] = "abspath"; +static char __pyx_k__replace[] = "replace"; static char __pyx_k____main__[] = "__main__"; static char __pyx_k____test__[] = "__test__"; static char __pyx_k__basename[] = "basename"; static char __pyx_k__TypeError[] = "TypeError"; +static char __pyx_k____import__[] = "__import__"; static char __pyx_k__RuntimeError[] = "RuntimeError"; static PyObject *__pyx_kp_s_2; static PyObject *__pyx_kp_s_3; static PyObject *__pyx_kp_s_4; +static PyObject *__pyx_kp_s_6; +static PyObject *__pyx_kp_s_7; static PyObject *__pyx_n_s__IOError; static PyObject *__pyx_n_s__RuntimeError; static PyObject *__pyx_n_s__TypeError; +static PyObject *__pyx_n_s____import__; static PyObject *__pyx_n_s____main__; static PyObject *__pyx_n_s____test__; static PyObject *__pyx_n_s__abspath; +static PyObject *__pyx_n_s__args; static PyObject *__pyx_n_s__basename; +static PyObject *__pyx_n_s__close; static PyObject *__pyx_n_s__encode; static PyObject *__pyx_n_s__format; static PyObject *__pyx_n_s__os; static PyObject *__pyx_n_s__path; +static PyObject *__pyx_n_s__replace; +static PyObject *__pyx_n_s__send; static PyObject *__pyx_n_s__split; +static PyObject *__pyx_n_s__throw; static PyObject *__pyx_n_s__utf8; static PyObject *__pyx_k_tuple_1; +static PyObject *__pyx_k_tuple_5; /* "kenlm.pyx":3 * import os @@ -643,8 +857,9 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; + int __pyx_t_2; PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; @@ -658,7 +873,8 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { * elif isinstance(data, unicode): */ __pyx_t_1 = PyBytes_Check(__pyx_v_data); - if (__pyx_t_1) { + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { /* "kenlm.pyx":5 * cdef bytes as_str(data): @@ -682,7 +898,8 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { * return data.encode('utf8') * raise TypeError('Cannot convert %s to string' % type(data)) */ - __pyx_t_1 = PyUnicode_Check(__pyx_v_data); + __pyx_t_2 = PyUnicode_Check(__pyx_v_data); + __pyx_t_1 = (__pyx_t_2 != 0); if (__pyx_t_1) { /* "kenlm.pyx":7 @@ -693,14 +910,14 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { * */ __Pyx_XDECREF(((PyObject *)__pyx_r)); - __pyx_t_2 = PyObject_GetAttr(__pyx_v_data, __pyx_n_s__encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_data, __pyx_n_s__encode); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_3)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = ((PyObject*)__pyx_t_3); - __pyx_t_3 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_k_tuple_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_4)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; goto __pyx_L0; goto __pyx_L3; } @@ -713,25 +930,25 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { * * cdef class LanguageModel: */ - __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_2), ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_3)); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_t_3)); - __Pyx_GIVEREF(((PyObject *)__pyx_t_3)); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_2), ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_Raise(__pyx_t_3, 0, 0, 0); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_4)); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = ((PyObject*)Py_None); __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("kenlm.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -744,6 +961,9 @@ static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) { static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_path = 0; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); @@ -797,6 +1017,7 @@ static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyO static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_v_exception = NULL; + PyObject *__pyx_v_exception_message = NULL; int __pyx_r; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; @@ -806,7 +1027,7 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang PyObject *__pyx_t_5 = NULL; PyObject *__pyx_t_6 = NULL; char *__pyx_t_7; - lm::ngram::Model *__pyx_t_8; + lm::base::Model *__pyx_t_8; int __pyx_t_9; PyObject *__pyx_t_10 = NULL; PyObject *__pyx_t_11 = NULL; @@ -821,14 +1042,14 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang * def __init__(self, path): * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<< * try: - * self.model = new Model(self.path) + * self.model = LoadVirtual(self.path) */ - __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s__path); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__abspath); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s__abspath); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_v_path)); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -853,7 +1074,7 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang * def __init__(self, path): * self.path = os.path.abspath(as_str(path)) * try: # <<<<<<<<<<<<<< - * self.model = new Model(self.path) + * self.model = LoadVirtual(self.path) * except RuntimeError as exception: */ { @@ -866,13 +1087,13 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang /* "kenlm.pyx":18 * self.path = os.path.abspath(as_str(path)) * try: - * self.model = new Model(self.path) # <<<<<<<<<<<<<< + * self.model = LoadVirtual(self.path) # <<<<<<<<<<<<<< * except RuntimeError as exception: - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception)) from exception + * exception_message = str(exception).replace('\n', ' ') */ - __pyx_t_7 = PyBytes_AsString(((PyObject *)__pyx_v_self->path)); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_t_7 = __Pyx_PyObject_AsString(((PyObject *)__pyx_v_self->path)); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L3_error;} try { - __pyx_t_8 = new lm::ngram::Model(__pyx_t_7); + __pyx_t_8 = lm::ngram::LoadVirtual(__pyx_t_7); } catch(...) { __Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L3_error;} @@ -890,10 +1111,10 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang /* "kenlm.pyx":19 * try: - * self.model = new Model(self.path) + * self.model = LoadVirtual(self.path) * except RuntimeError as exception: # <<<<<<<<<<<<<< - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception)) from exception - * self.vocab = &self.model.GetVocabulary() + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ */ __pyx_t_9 = PyErr_ExceptionMatches(__pyx_builtin_RuntimeError); if (__pyx_t_9) { @@ -906,37 +1127,69 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang __pyx_v_exception = __pyx_t_3; /* "kenlm.pyx":20 - * self.model = new Model(self.path) + * self.model = LoadVirtual(self.path) * except RuntimeError as exception: - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception)) from exception # <<<<<<<<<<<<<< - * self.vocab = &self.model.GetVocabulary() - * + * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception */ - __pyx_t_10 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_3), __pyx_n_s__format); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __pyx_t_10 = PyTuple_New(1); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_10); - __pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_11); - __Pyx_INCREF(__pyx_v_path); - PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_v_path); - __Pyx_GIVEREF(__pyx_v_path); __Pyx_INCREF(__pyx_v_exception); - PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_v_exception); + PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_v_exception); __Pyx_GIVEREF(__pyx_v_exception); - __pyx_t_12 = PyObject_Call(__pyx_t_10, ((PyObject *)__pyx_t_11), NULL); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} - __Pyx_GOTREF(__pyx_t_12); + __pyx_t_11 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_10), NULL); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(((PyObject *)__pyx_t_10)); __pyx_t_10 = 0; + __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_t_11, __pyx_n_s__replace); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_11 = PyObject_Call(__pyx_t_10, ((PyObject *)__pyx_k_tuple_5), NULL); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_11)); __pyx_t_11 = 0; - __pyx_t_11 = PyTuple_New(1); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __pyx_v_exception_message = __pyx_t_11; + __pyx_t_11 = 0; + + /* "kenlm.pyx":21 + * except RuntimeError as exception: + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<< + * from exception + * self.vocab = &self.model.BaseVocabulary() + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_kp_s_6), __pyx_n_s__format); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_11); - PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_12); + __pyx_t_10 = PyTuple_New(2); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_10); + __Pyx_INCREF(__pyx_v_path); + PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_v_path); + __Pyx_GIVEREF(__pyx_v_path); + __Pyx_INCREF(__pyx_v_exception_message); + PyTuple_SET_ITEM(__pyx_t_10, 1, __pyx_v_exception_message); + __Pyx_GIVEREF(__pyx_v_exception_message); + __pyx_t_12 = PyObject_Call(__pyx_t_11, ((PyObject *)__pyx_t_10), NULL); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_10)); __pyx_t_10 = 0; + __pyx_t_10 = PyTuple_New(1); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __Pyx_GOTREF(__pyx_t_10); + PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_12); __Pyx_GIVEREF(__pyx_t_12); __pyx_t_12 = 0; - __pyx_t_12 = PyObject_Call(__pyx_builtin_IOError, ((PyObject *)__pyx_t_11), NULL); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + __pyx_t_12 = PyObject_Call(__pyx_builtin_IOError, ((PyObject *)__pyx_t_10), NULL); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_GOTREF(__pyx_t_12); - __Pyx_DECREF(((PyObject *)__pyx_t_11)); __pyx_t_11 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_10)); __pyx_t_10 = 0; + + /* "kenlm.pyx":22 + * exception_message = str(exception).replace('\n', ' ') + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception # <<<<<<<<<<<<<< + * self.vocab = &self.model.BaseVocabulary() + * + */ __Pyx_Raise(__pyx_t_12, 0, 0, __pyx_v_exception); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -956,14 +1209,14 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang __pyx_L10_try_end:; } - /* "kenlm.pyx":21 - * except RuntimeError as exception: - * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception)) from exception - * self.vocab = &self.model.GetVocabulary() # <<<<<<<<<<<<<< + /* "kenlm.pyx":23 + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception + * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __pyx_v_self->vocab = (&__pyx_v_self->model->GetVocabulary()); + __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary()); __pyx_r = 0; goto __pyx_L0; @@ -978,6 +1231,7 @@ static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_Lang __pyx_r = -1; __pyx_L0:; __Pyx_XDECREF(__pyx_v_exception); + __Pyx_XDECREF(__pyx_v_exception_message); __Pyx_RefNannyFinishContext(); return __pyx_r; } @@ -991,8 +1245,8 @@ static void __pyx_pw_5kenlm_13LanguageModel_3__dealloc__(PyObject *__pyx_v_self) __Pyx_RefNannyFinishContext(); } -/* "kenlm.pyx":23 - * self.vocab = &self.model.GetVocabulary() +/* "kenlm.pyx":25 + * self.vocab = &self.model.BaseVocabulary() * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.model @@ -1003,7 +1257,7 @@ static void __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(CYTHON_UNUSED struct __ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "kenlm.pyx":24 + /* "kenlm.pyx":26 * * def __dealloc__(self): * del self.model # <<<<<<<<<<<<<< @@ -1026,7 +1280,7 @@ static PyObject *__pyx_pw_5kenlm_13LanguageModel_5order_1__get__(PyObject *__pyx return __pyx_r; } -/* "kenlm.pyx":27 +/* "kenlm.pyx":29 * * property order: * def __get__(self): # <<<<<<<<<<<<<< @@ -1043,7 +1297,7 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "kenlm.pyx":28 + /* "kenlm.pyx":30 * property order: * def __get__(self): * return self.model.Order() # <<<<<<<<<<<<<< @@ -1051,7 +1305,7 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj * def score(self, sentence): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyLong_FromUnsignedLong(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -1080,18 +1334,18 @@ static PyObject *__pyx_pw_5kenlm_13LanguageModel_5score(PyObject *__pyx_v_self, return __pyx_r; } -/* "kenlm.pyx":30 +/* "kenlm.pyx":32 * return self.model.Order() * * def score(self, sentence): # <<<<<<<<<<<<<< * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) + * cdef State state */ static PyObject *__pyx_pf_5kenlm_13LanguageModel_4score(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence) { PyObject *__pyx_v_words = 0; - lm::ngram::State *__pyx_v_state; - lm::ngram::State *__pyx_v_out_state; + struct lm::ngram::State __pyx_v_state; + struct lm::ngram::State __pyx_v_out_state; float __pyx_v_total; PyObject *__pyx_v_word = NULL; PyObject *__pyx_r = NULL; @@ -1105,124 +1359,105 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_4score(struct __pyx_obj_5kenlm_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("score", 0); - /* "kenlm.pyx":31 + /* "kenlm.pyx":33 * * def score(self, sentence): * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef State* state = new State(self.model.BeginSentenceState()) - * cdef State* out_state = new State() + * cdef State state + * self.model.BeginSentenceWrite(&state) */ - __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_v_sentence)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_v_sentence)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__split); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s__split); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected list, got %.200s", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected list, got %.200s", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_words = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":32 - * def score(self, sentence): - * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) # <<<<<<<<<<<<<< - * cdef State* out_state = new State() - * cdef float total = 0 - */ - __pyx_v_state = new lm::ngram::State(__pyx_v_self->model->BeginSentenceState()); - - /* "kenlm.pyx":33 + /* "kenlm.pyx":35 * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) - * cdef State* out_state = new State() # <<<<<<<<<<<<<< + * cdef State state + * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< + * cdef State out_state * cdef float total = 0 - * for word in words: */ - __pyx_v_out_state = new lm::ngram::State(); + __pyx_v_self->model->BeginSentenceWrite((&__pyx_v_state)); - /* "kenlm.pyx":34 - * cdef State* state = new State(self.model.BeginSentenceState()) - * cdef State* out_state = new State() + /* "kenlm.pyx":37 + * self.model.BeginSentenceWrite(&state) + * cdef State out_state * cdef float total = 0 # <<<<<<<<<<<<<< * for word in words: - * total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) + * total += self.model.Score(&state, self.vocab.Index(word), &out_state) */ __pyx_v_total = 0.0; - /* "kenlm.pyx":35 - * cdef State* out_state = new State() + /* "kenlm.pyx":38 + * cdef State out_state * cdef float total = 0 * for word in words: # <<<<<<<<<<<<<< - * total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) - * state[0] = out_state[0] + * total += self.model.Score(&state, self.vocab.Index(word), &out_state) + * state = out_state */ if (unlikely(((PyObject *)__pyx_v_words) == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_t_1 = ((PyObject *)__pyx_v_words); __Pyx_INCREF(__pyx_t_1); __pyx_t_3 = 0; for (;;) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_1)) break; #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;} #endif __Pyx_XDECREF(__pyx_v_word); __pyx_v_word = __pyx_t_2; __pyx_t_2 = 0; - /* "kenlm.pyx":36 + /* "kenlm.pyx":39 * cdef float total = 0 * for word in words: - * total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) # <<<<<<<<<<<<<< - * state[0] = out_state[0] - * total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) + * total += self.model.Score(&state, self.vocab.Index(word), &out_state) # <<<<<<<<<<<<<< + * state = out_state + * total += self.model.Score(&state, self.vocab.EndSentence(), &out_state) */ - __pyx_t_4 = PyBytes_AsString(__pyx_v_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->Score((__pyx_v_state[0]), __pyx_v_self->vocab->Index(__pyx_t_4), (__pyx_v_out_state[0]))); + __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_v_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->Score((&__pyx_v_state), __pyx_v_self->vocab->Index(__pyx_t_4), (&__pyx_v_out_state))); - /* "kenlm.pyx":37 + /* "kenlm.pyx":40 * for word in words: - * total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) - * state[0] = out_state[0] # <<<<<<<<<<<<<< - * total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) - * del state, out_state + * total += self.model.Score(&state, self.vocab.Index(word), &out_state) + * state = out_state # <<<<<<<<<<<<<< + * total += self.model.Score(&state, self.vocab.EndSentence(), &out_state) + * return total */ - (__pyx_v_state[0]) = (__pyx_v_out_state[0]); + __pyx_v_state = __pyx_v_out_state; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":38 - * total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) - * state[0] = out_state[0] - * total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) # <<<<<<<<<<<<<< - * del state, out_state - * return total - */ - __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->Score((__pyx_v_state[0]), __pyx_v_self->vocab->EndSentence(), (__pyx_v_out_state[0]))); - - /* "kenlm.pyx":39 - * state[0] = out_state[0] - * total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) - * del state, out_state # <<<<<<<<<<<<<< + /* "kenlm.pyx":41 + * total += self.model.Score(&state, self.vocab.Index(word), &out_state) + * state = out_state + * total += self.model.Score(&state, self.vocab.EndSentence(), &out_state) # <<<<<<<<<<<<<< * return total * */ - delete __pyx_v_state; - delete __pyx_v_out_state; + __pyx_v_total = (__pyx_v_total + __pyx_v_self->model->Score((&__pyx_v_state), __pyx_v_self->vocab->EndSentence(), (&__pyx_v_out_state))); - /* "kenlm.pyx":40 - * total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) - * del state, out_state + /* "kenlm.pyx":42 + * state = out_state + * total += self.model.Score(&state, self.vocab.EndSentence(), &out_state) * return total # <<<<<<<<<<<<<< * * def full_scores(self, sentence): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_total); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -1255,12 +1490,12 @@ static PyObject *__pyx_pw_5kenlm_13LanguageModel_7full_scores(PyObject *__pyx_v_ return __pyx_r; } -/* "kenlm.pyx":42 +/* "kenlm.pyx":44 * return total * * def full_scores(self, sentence): # <<<<<<<<<<<<<< * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) + * cdef State state */ static PyObject *__pyx_pf_5kenlm_13LanguageModel_6full_scores(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence) { @@ -1271,7 +1506,7 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_6full_scores(struct __pyx_obj_5 const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("full_scores", 0); - __pyx_cur_scope = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_ptype_5kenlm___pyx_scope_struct__full_scores->tp_new(__pyx_ptype_5kenlm___pyx_scope_struct__full_scores, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(__pyx_ptype_5kenlm___pyx_scope_struct__full_scores, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -1284,7 +1519,7 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_6full_scores(struct __pyx_obj_5 __Pyx_INCREF(__pyx_cur_scope->__pyx_v_sentence); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_sentence); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5kenlm_13LanguageModel_8generator, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5kenlm_13LanguageModel_8generator, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -1312,326 +1547,179 @@ static PyObject *__pyx_gb_5kenlm_13LanguageModel_8generator(__pyx_GeneratorObjec char *__pyx_t_4; PyObject *__pyx_t_5 = NULL; PyObject *__pyx_t_6 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("None", 0); switch (__pyx_generator->resume_label) { case 0: goto __pyx_L3_first_run; - case 1: goto __pyx_L14_resume_from_yield; - case 2: goto __pyx_L19_resume_from_yield; + case 1: goto __pyx_L6_resume_from_yield; + case 2: goto __pyx_L7_resume_from_yield; default: /* CPython raises the right error here */ __Pyx_RefNannyFinishContext(); return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "kenlm.pyx":43 + /* "kenlm.pyx":45 * * def full_scores(self, sentence): * cdef list words = as_str(sentence).split() # <<<<<<<<<<<<<< - * cdef State* state = new State(self.model.BeginSentenceState()) - * cdef State* out_state = new State() + * cdef State state + * self.model.BeginSentenceWrite(&state) */ - __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_cur_scope->__pyx_v_sentence)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_cur_scope->__pyx_v_sentence)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__split); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s__split); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected list, got %.200s", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyList_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected list, got %.200s", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GIVEREF(__pyx_t_1); __pyx_cur_scope->__pyx_v_words = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":44 - * def full_scores(self, sentence): - * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) # <<<<<<<<<<<<<< - * cdef State* out_state = new State() - * cdef FullScoreReturn* ret - */ - __pyx_cur_scope->__pyx_v_state = new lm::ngram::State(__pyx_cur_scope->__pyx_v_self->model->BeginSentenceState()); - - /* "kenlm.pyx":45 + /* "kenlm.pyx":47 * cdef list words = as_str(sentence).split() - * cdef State* state = new State(self.model.BeginSentenceState()) - * cdef State* out_state = new State() # <<<<<<<<<<<<<< - * cdef FullScoreReturn* ret - * cdef float total = 0 + * cdef State state + * self.model.BeginSentenceWrite(&state) # <<<<<<<<<<<<<< + * cdef State out_state + * cdef FullScoreReturn ret */ - __pyx_cur_scope->__pyx_v_out_state = new lm::ngram::State(); + __pyx_cur_scope->__pyx_v_self->model->BeginSentenceWrite((&__pyx_cur_scope->__pyx_v_state)); - /* "kenlm.pyx":47 - * cdef State* out_state = new State() - * cdef FullScoreReturn* ret + /* "kenlm.pyx":50 + * cdef State out_state + * cdef FullScoreReturn ret * cdef float total = 0 # <<<<<<<<<<<<<< - * try: - * for word in words: + * for word in words: + * ret = self.model.FullScore(&state, */ __pyx_cur_scope->__pyx_v_total = 0.0; - /* "kenlm.pyx":48 - * cdef FullScoreReturn* ret + /* "kenlm.pyx":51 + * cdef FullScoreReturn ret * cdef float total = 0 - * try: # <<<<<<<<<<<<<< - * for word in words: - * ret = new FullScoreReturn(self.model.FullScore(state[0], - */ - /*try:*/ { - - /* "kenlm.pyx":49 - * cdef float total = 0 - * try: - * for word in words: # <<<<<<<<<<<<<< - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.Index(word), out_state[0])) - */ - if (unlikely(((PyObject *)__pyx_cur_scope->__pyx_v_words) == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L5;} - } - __pyx_t_1 = ((PyObject *)__pyx_cur_scope->__pyx_v_words); __Pyx_INCREF(__pyx_t_1); __pyx_t_3 = 0; - for (;;) { - if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L5;} - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L5;} - #endif - __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_word); - __Pyx_XDECREF(__pyx_cur_scope->__pyx_v_word); - __Pyx_GIVEREF(__pyx_t_2); - __pyx_cur_scope->__pyx_v_word = __pyx_t_2; - __pyx_t_2 = 0; - - /* "kenlm.pyx":51 - * for word in words: - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.Index(word), out_state[0])) # <<<<<<<<<<<<<< - * try: - * yield (ret.prob, ret.ngram_length) - */ - __pyx_t_4 = PyBytes_AsString(__pyx_cur_scope->__pyx_v_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L5;} - __pyx_cur_scope->__pyx_v_ret = new lm::FullScoreReturn(__pyx_cur_scope->__pyx_v_self->model->FullScore((__pyx_cur_scope->__pyx_v_state[0]), __pyx_cur_scope->__pyx_v_self->vocab->Index(__pyx_t_4), (__pyx_cur_scope->__pyx_v_out_state[0]))); - - /* "kenlm.pyx":52 - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.Index(word), out_state[0])) - * try: # <<<<<<<<<<<<<< - * yield (ret.prob, ret.ngram_length) - * finally: - */ - /*try:*/ { - - /* "kenlm.pyx":53 - * self.vocab.Index(word), out_state[0])) - * try: - * yield (ret.prob, ret.ngram_length) # <<<<<<<<<<<<<< - * finally: - * del ret - */ - __pyx_t_2 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret->prob); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L12;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_5 = PyInt_FromLong(__pyx_cur_scope->__pyx_v_ret->ngram_length); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L12;} - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L12;} - __Pyx_GOTREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_2); - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_5); - __Pyx_GIVEREF(__pyx_t_5); - __pyx_t_2 = 0; - __pyx_t_5 = 0; - __pyx_r = ((PyObject *)__pyx_t_6); - __pyx_t_6 = 0; - __Pyx_XGIVEREF(__pyx_t_1); - __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 1; - return __pyx_r; - __pyx_L14_resume_from_yield:; - __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; - __pyx_cur_scope->__pyx_t_0 = 0; - __Pyx_XGOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L12;} - } - - /* "kenlm.pyx":55 - * yield (ret.prob, ret.ngram_length) - * finally: - * del ret # <<<<<<<<<<<<<< - * state[0] = out_state[0] - * ret = new FullScoreReturn(self.model.FullScore(state[0], + * for word in words: # <<<<<<<<<<<<<< + * ret = self.model.FullScore(&state, + * self.vocab.Index(word), &out_state) */ - /*finally:*/ { - int __pyx_why; - PyObject *__pyx_exc_type, *__pyx_exc_value, *__pyx_exc_tb; - int __pyx_exc_lineno; - __pyx_exc_type = 0; __pyx_exc_value = 0; __pyx_exc_tb = 0; __pyx_exc_lineno = 0; - __pyx_why = 0; goto __pyx_L13; - __pyx_L12: { - __pyx_why = 4; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_ErrFetch(&__pyx_exc_type, &__pyx_exc_value, &__pyx_exc_tb); - __pyx_exc_lineno = __pyx_lineno; - goto __pyx_L13; - } - __pyx_L13:; - delete __pyx_cur_scope->__pyx_v_ret; - switch (__pyx_why) { - case 4: { - __Pyx_ErrRestore(__pyx_exc_type, __pyx_exc_value, __pyx_exc_tb); - __pyx_lineno = __pyx_exc_lineno; - __pyx_exc_type = 0; - __pyx_exc_value = 0; - __pyx_exc_tb = 0; - goto __pyx_L5; - } - } - } + if (unlikely(((PyObject *)__pyx_cur_scope->__pyx_v_words) == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_t_1 = ((PyObject *)__pyx_cur_scope->__pyx_v_words); __Pyx_INCREF(__pyx_t_1); __pyx_t_3 = 0; + for (;;) { + if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #else + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif + __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_word); + __Pyx_XDECREF(__pyx_cur_scope->__pyx_v_word); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_cur_scope->__pyx_v_word = __pyx_t_2; + __pyx_t_2 = 0; - /* "kenlm.pyx":56 - * finally: - * del ret - * state[0] = out_state[0] # <<<<<<<<<<<<<< - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.EndSentence(), out_state[0])) - */ - (__pyx_cur_scope->__pyx_v_state[0]) = (__pyx_cur_scope->__pyx_v_out_state[0]); - } - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "kenlm.pyx":58 - * state[0] = out_state[0] - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.EndSentence(), out_state[0])) # <<<<<<<<<<<<<< - * try: - * yield (ret.prob, ret.ngram_length) + /* "kenlm.pyx":53 + * for word in words: + * ret = self.model.FullScore(&state, + * self.vocab.Index(word), &out_state) # <<<<<<<<<<<<<< + * yield (ret.prob, ret.ngram_length) + * state = out_state */ - __pyx_cur_scope->__pyx_v_ret = new lm::FullScoreReturn(__pyx_cur_scope->__pyx_v_self->model->FullScore((__pyx_cur_scope->__pyx_v_state[0]), __pyx_cur_scope->__pyx_v_self->vocab->EndSentence(), (__pyx_cur_scope->__pyx_v_out_state[0]))); - - /* "kenlm.pyx":59 - * ret = new FullScoreReturn(self.model.FullScore(state[0], - * self.vocab.EndSentence(), out_state[0])) - * try: # <<<<<<<<<<<<<< - * yield (ret.prob, ret.ngram_length) - * finally: + __pyx_t_4 = __Pyx_PyObject_AsString(__pyx_cur_scope->__pyx_v_word); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->FullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_self->vocab->Index(__pyx_t_4), (&__pyx_cur_scope->__pyx_v_out_state)); + + /* "kenlm.pyx":54 + * ret = self.model.FullScore(&state, + * self.vocab.Index(word), &out_state) + * yield (ret.prob, ret.ngram_length) # <<<<<<<<<<<<<< + * state = out_state + * ret = self.model.FullScore(&state, */ - /*try:*/ { - - /* "kenlm.pyx":60 - * self.vocab.EndSentence(), out_state[0])) - * try: - * yield (ret.prob, ret.ngram_length) # <<<<<<<<<<<<<< - * finally: - * del ret + __pyx_t_2 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = PyInt_FromLong(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_t_5); + __Pyx_GIVEREF(__pyx_t_5); + __pyx_t_2 = 0; + __pyx_t_5 = 0; + __pyx_r = ((PyObject *)__pyx_t_6); + __pyx_t_6 = 0; + __Pyx_XGIVEREF(__pyx_t_1); + __pyx_cur_scope->__pyx_t_0 = __pyx_t_1; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L6_resume_from_yield:; + __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; + __pyx_cur_scope->__pyx_t_0 = 0; + __Pyx_XGOTREF(__pyx_t_1); + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + + /* "kenlm.pyx":55 + * self.vocab.Index(word), &out_state) + * yield (ret.prob, ret.ngram_length) + * state = out_state # <<<<<<<<<<<<<< + * ret = self.model.FullScore(&state, + * self.vocab.EndSentence(), &out_state) */ - __pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret->prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L17;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = PyInt_FromLong(__pyx_cur_scope->__pyx_v_ret->ngram_length); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L17;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L17;} - __Pyx_GOTREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_6); - __Pyx_GIVEREF(__pyx_t_6); - __pyx_t_1 = 0; - __pyx_t_6 = 0; - __pyx_r = ((PyObject *)__pyx_t_5); - __pyx_t_5 = 0; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - /* return from generator, yielding value */ - __pyx_generator->resume_label = 2; - return __pyx_r; - __pyx_L19_resume_from_yield:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L17;} - } + __pyx_cur_scope->__pyx_v_state = __pyx_cur_scope->__pyx_v_out_state; + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":62 - * yield (ret.prob, ret.ngram_length) - * finally: - * del ret # <<<<<<<<<<<<<< - * finally: - * del state, out_state + /* "kenlm.pyx":57 + * state = out_state + * ret = self.model.FullScore(&state, + * self.vocab.EndSentence(), &out_state) # <<<<<<<<<<<<<< + * yield (ret.prob, ret.ngram_length) + * */ - /*finally:*/ { - int __pyx_why; - PyObject *__pyx_exc_type, *__pyx_exc_value, *__pyx_exc_tb; - int __pyx_exc_lineno; - __pyx_exc_type = 0; __pyx_exc_value = 0; __pyx_exc_tb = 0; __pyx_exc_lineno = 0; - __pyx_why = 0; goto __pyx_L18; - __pyx_L17: { - __pyx_why = 4; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_ErrFetch(&__pyx_exc_type, &__pyx_exc_value, &__pyx_exc_tb); - __pyx_exc_lineno = __pyx_lineno; - goto __pyx_L18; - } - __pyx_L18:; - delete __pyx_cur_scope->__pyx_v_ret; - switch (__pyx_why) { - case 4: { - __Pyx_ErrRestore(__pyx_exc_type, __pyx_exc_value, __pyx_exc_tb); - __pyx_lineno = __pyx_exc_lineno; - __pyx_exc_type = 0; - __pyx_exc_value = 0; - __pyx_exc_tb = 0; - goto __pyx_L5; - } - } - } - } + __pyx_cur_scope->__pyx_v_ret = __pyx_cur_scope->__pyx_v_self->model->FullScore((&__pyx_cur_scope->__pyx_v_state), __pyx_cur_scope->__pyx_v_self->vocab->EndSentence(), (&__pyx_cur_scope->__pyx_v_out_state)); - /* "kenlm.pyx":64 - * del ret - * finally: - * del state, out_state # <<<<<<<<<<<<<< + /* "kenlm.pyx":58 + * ret = self.model.FullScore(&state, + * self.vocab.EndSentence(), &out_state) + * yield (ret.prob, ret.ngram_length) # <<<<<<<<<<<<<< * * def __contains__(self, word): */ - /*finally:*/ { - int __pyx_why; - PyObject *__pyx_exc_type, *__pyx_exc_value, *__pyx_exc_tb; - int __pyx_exc_lineno; - __pyx_exc_type = 0; __pyx_exc_value = 0; __pyx_exc_tb = 0; __pyx_exc_lineno = 0; - __pyx_why = 0; goto __pyx_L6; - __pyx_L5: { - __pyx_why = 4; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_ErrFetch(&__pyx_exc_type, &__pyx_exc_value, &__pyx_exc_tb); - __pyx_exc_lineno = __pyx_lineno; - goto __pyx_L6; - } - __pyx_L6:; - delete __pyx_cur_scope->__pyx_v_state; - delete __pyx_cur_scope->__pyx_v_out_state; - switch (__pyx_why) { - case 4: { - __Pyx_ErrRestore(__pyx_exc_type, __pyx_exc_value, __pyx_exc_tb); - __pyx_lineno = __pyx_exc_lineno; - __pyx_exc_type = 0; - __pyx_exc_value = 0; - __pyx_exc_tb = 0; - goto __pyx_L1_error; - } - } - } + __pyx_t_1 = PyFloat_FromDouble(__pyx_cur_scope->__pyx_v_ret.prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = PyInt_FromLong(__pyx_cur_scope->__pyx_v_ret.ngram_length); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_6); + __pyx_t_1 = 0; + __pyx_t_6 = 0; + __pyx_r = ((PyObject *)__pyx_t_5); + __pyx_t_5 = 0; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 2; + return __pyx_r; + __pyx_L7_resume_from_yield:; + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; __pyx_L1_error:; @@ -1659,8 +1747,8 @@ static int __pyx_pw_5kenlm_13LanguageModel_10__contains__(PyObject *__pyx_v_self return __pyx_r; } -/* "kenlm.pyx":66 - * del state, out_state +/* "kenlm.pyx":60 + * yield (ret.prob, ret.ngram_length) * * def __contains__(self, word): # <<<<<<<<<<<<<< * cdef bytes w = as_str(word) @@ -1678,26 +1766,26 @@ static int __pyx_pf_5kenlm_13LanguageModel_9__contains__(struct __pyx_obj_5kenlm int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__contains__", 0); - /* "kenlm.pyx":67 + /* "kenlm.pyx":61 * * def __contains__(self, word): * cdef bytes w = as_str(word) # <<<<<<<<<<<<<< * return (self.vocab.Index(w) != 0) * */ - __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_v_word)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = ((PyObject *)__pyx_f_5kenlm_as_str(__pyx_v_word)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_v_w = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "kenlm.pyx":68 + /* "kenlm.pyx":62 * def __contains__(self, word): * cdef bytes w = as_str(word) * return (self.vocab.Index(w) != 0) # <<<<<<<<<<<<<< * * def __repr__(self): */ - __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_w)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_AsString(((PyObject *)__pyx_v_w)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = (__pyx_v_self->vocab->Index(__pyx_t_2) != 0); goto __pyx_L0; @@ -1724,7 +1812,7 @@ static PyObject *__pyx_pw_5kenlm_13LanguageModel_12__repr__(PyObject *__pyx_v_se return __pyx_r; } -/* "kenlm.pyx":70 +/* "kenlm.pyx":64 * return (self.vocab.Index(w) != 0) * * def __repr__(self): # <<<<<<<<<<<<<< @@ -1744,7 +1832,7 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_11__repr__(struct __pyx_obj_5ke int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__repr__", 0); - /* "kenlm.pyx":71 + /* "kenlm.pyx":65 * * def __repr__(self): * return '<LanguageModel from {0}>'.format(os.path.basename(self.path)) # <<<<<<<<<<<<<< @@ -1752,31 +1840,31 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_11__repr__(struct __pyx_obj_5ke * def __reduce__(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_kp_s_4), __pyx_n_s__format); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_kp_s_7), __pyx_n_s__format); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s__os); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s__path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__basename); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s__basename); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(((PyObject *)__pyx_v_self->path)); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_self->path)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self->path)); - __pyx_t_4 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; @@ -1810,7 +1898,7 @@ static PyObject *__pyx_pw_5kenlm_13LanguageModel_14__reduce__(PyObject *__pyx_v_ return __pyx_r; } -/* "kenlm.pyx":73 +/* "kenlm.pyx":67 * return '<LanguageModel from {0}>'.format(os.path.basename(self.path)) * * def __reduce__(self): # <<<<<<<<<<<<<< @@ -1827,18 +1915,18 @@ static PyObject *__pyx_pf_5kenlm_13LanguageModel_13__reduce__(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__reduce__", 0); - /* "kenlm.pyx":74 + /* "kenlm.pyx":68 * * def __reduce__(self): * return (LanguageModel, (self.path,)) # <<<<<<<<<<<<<< */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(((PyObject *)__pyx_v_self->path)); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self->path)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self->path)); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(((PyObject *)((PyObject*)__pyx_ptype_5kenlm_LanguageModel))); PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)((PyObject*)__pyx_ptype_5kenlm_LanguageModel))); @@ -1961,8 +2049,9 @@ static int __pyx_pf_5kenlm_13LanguageModel_4path_4__del__(struct __pyx_obj_5kenl static PyObject *__pyx_tp_new_5kenlm_LanguageModel(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { struct __pyx_obj_5kenlm_LanguageModel *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; + PyObject *o; + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; p = ((struct __pyx_obj_5kenlm_LanguageModel *)o); p->path = ((PyObject*)Py_None); Py_INCREF(Py_None); return o; @@ -1981,7 +2070,6 @@ static void __pyx_tp_dealloc_5kenlm_LanguageModel(PyObject *o) { PyErr_Restore(etype, eval, etb); } Py_CLEAR(p->path); - PyObject_GC_Track(o); (*Py_TYPE(o)->tp_free)(o); } @@ -2033,64 +2121,6 @@ static struct PyGetSetDef __pyx_getsets_5kenlm_LanguageModel[] = { {0, 0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number_LanguageModel = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_long*/ - #else - 0, /*reserved*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if PY_VERSION_HEX >= 0x02050000 - 0, /*nb_index*/ - #endif -}; - static PySequenceMethods __pyx_tp_as_sequence_LanguageModel = { 0, /*sq_length*/ 0, /*sq_concat*/ @@ -2104,33 +2134,6 @@ static PySequenceMethods __pyx_tp_as_sequence_LanguageModel = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping_LanguageModel = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_LanguageModel = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - static PyTypeObject __pyx_type_5kenlm_LanguageModel = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("kenlm.LanguageModel"), /*tp_name*/ @@ -2146,16 +2149,16 @@ static PyTypeObject __pyx_type_5kenlm_LanguageModel = { 0, /*reserved*/ #endif __pyx_pw_5kenlm_13LanguageModel_12__repr__, /*tp_repr*/ - &__pyx_tp_as_number_LanguageModel, /*tp_as_number*/ + 0, /*tp_as_number*/ &__pyx_tp_as_sequence_LanguageModel, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_LanguageModel, /*tp_as_mapping*/ + 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_LanguageModel, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ __pyx_tp_traverse_5kenlm_LanguageModel, /*tp_traverse*/ __pyx_tp_clear_5kenlm_LanguageModel, /*tp_clear*/ @@ -2187,10 +2190,21 @@ static PyTypeObject __pyx_type_5kenlm_LanguageModel = { #endif }; +static struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *__pyx_freelist_5kenlm___pyx_scope_struct__full_scores[8]; +static int __pyx_freecount_5kenlm___pyx_scope_struct__full_scores = 0; + static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; + PyObject *o; + if (likely((__pyx_freecount_5kenlm___pyx_scope_struct__full_scores > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores)))) { + o = (PyObject*)__pyx_freelist_5kenlm___pyx_scope_struct__full_scores[--__pyx_freecount_5kenlm___pyx_scope_struct__full_scores]; + memset(o, 0, sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores)); + PyObject_INIT(o, t); + PyObject_GC_Track(o); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } p = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o); p->__pyx_v_self = 0; p->__pyx_v_sentence = 0; @@ -2208,8 +2222,11 @@ static void __pyx_tp_dealloc_5kenlm___pyx_scope_struct__full_scores(PyObject *o) Py_CLEAR(p->__pyx_v_word); Py_CLEAR(p->__pyx_v_words); Py_CLEAR(p->__pyx_t_0); - PyObject_GC_Track(o); - (*Py_TYPE(o)->tp_free)(o); + if ((__pyx_freecount_5kenlm___pyx_scope_struct__full_scores < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores))) { + __pyx_freelist_5kenlm___pyx_scope_struct__full_scores[__pyx_freecount_5kenlm___pyx_scope_struct__full_scores++] = ((struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } } static int __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores(PyObject *o, visitproc v, void *a) { @@ -2258,104 +2275,6 @@ static PyMethodDef __pyx_methods_5kenlm___pyx_scope_struct__full_scores[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct__full_scores = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_long*/ - #else - 0, /*reserved*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if PY_VERSION_HEX >= 0x02050000 - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct__full_scores = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct__full_scores = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct__full_scores = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - static PyTypeObject __pyx_type_5kenlm___pyx_scope_struct__full_scores = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("kenlm.__pyx_scope_struct__full_scores"), /*tp_name*/ @@ -2371,16 +2290,16 @@ static PyTypeObject __pyx_type_5kenlm___pyx_scope_struct__full_scores = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct__full_scores, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct__full_scores, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct__full_scores, /*tp_as_mapping*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct__full_scores, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ __pyx_tp_traverse_5kenlm___pyx_scope_struct__full_scores, /*tp_traverse*/ __pyx_tp_clear_5kenlm___pyx_scope_struct__full_scores, /*tp_clear*/ @@ -2438,25 +2357,33 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 1, 0}, {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0}, {&__pyx_kp_s_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 1, 0}, + {&__pyx_kp_s_6, __pyx_k_6, sizeof(__pyx_k_6), 0, 0, 1, 0}, + {&__pyx_kp_s_7, __pyx_k_7, sizeof(__pyx_k_7), 0, 0, 1, 0}, {&__pyx_n_s__IOError, __pyx_k__IOError, sizeof(__pyx_k__IOError), 0, 0, 1, 1}, {&__pyx_n_s__RuntimeError, __pyx_k__RuntimeError, sizeof(__pyx_k__RuntimeError), 0, 0, 1, 1}, {&__pyx_n_s__TypeError, __pyx_k__TypeError, sizeof(__pyx_k__TypeError), 0, 0, 1, 1}, + {&__pyx_n_s____import__, __pyx_k____import__, sizeof(__pyx_k____import__), 0, 0, 1, 1}, {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1}, {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1}, {&__pyx_n_s__abspath, __pyx_k__abspath, sizeof(__pyx_k__abspath), 0, 0, 1, 1}, + {&__pyx_n_s__args, __pyx_k__args, sizeof(__pyx_k__args), 0, 0, 1, 1}, {&__pyx_n_s__basename, __pyx_k__basename, sizeof(__pyx_k__basename), 0, 0, 1, 1}, + {&__pyx_n_s__close, __pyx_k__close, sizeof(__pyx_k__close), 0, 0, 1, 1}, {&__pyx_n_s__encode, __pyx_k__encode, sizeof(__pyx_k__encode), 0, 0, 1, 1}, {&__pyx_n_s__format, __pyx_k__format, sizeof(__pyx_k__format), 0, 0, 1, 1}, {&__pyx_n_s__os, __pyx_k__os, sizeof(__pyx_k__os), 0, 0, 1, 1}, {&__pyx_n_s__path, __pyx_k__path, sizeof(__pyx_k__path), 0, 0, 1, 1}, + {&__pyx_n_s__replace, __pyx_k__replace, sizeof(__pyx_k__replace), 0, 0, 1, 1}, + {&__pyx_n_s__send, __pyx_k__send, sizeof(__pyx_k__send), 0, 0, 1, 1}, {&__pyx_n_s__split, __pyx_k__split, sizeof(__pyx_k__split), 0, 0, 1, 1}, + {&__pyx_n_s__throw, __pyx_k__throw, sizeof(__pyx_k__throw), 0, 0, 1, 1}, {&__pyx_n_s__utf8, __pyx_k__utf8, sizeof(__pyx_k__utf8), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_RuntimeError = __Pyx_GetName(__pyx_b, __pyx_n_s__RuntimeError); if (!__pyx_builtin_RuntimeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_IOError = __Pyx_GetName(__pyx_b, __pyx_n_s__IOError); if (!__pyx_builtin_IOError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s__RuntimeError); if (!__pyx_builtin_RuntimeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s__IOError); if (!__pyx_builtin_IOError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -2476,6 +2403,17 @@ static int __Pyx_InitCachedConstants(void) { __pyx_k_tuple_1 = PyTuple_Pack(1, ((PyObject *)__pyx_n_s__utf8)); if (unlikely(!__pyx_k_tuple_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_1); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_1)); + + /* "kenlm.pyx":20 + * self.model = LoadVirtual(self.path) + * except RuntimeError as exception: + * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<< + * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ + * from exception + */ + __pyx_k_tuple_5 = PyTuple_Pack(2, ((PyObject *)__pyx_kp_s_3), ((PyObject *)__pyx_kp_s_4)); if (unlikely(!__pyx_k_tuple_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_k_tuple_5); + __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_5)); __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -2499,6 +2437,9 @@ PyMODINIT_FUNC PyInit_kenlm(void) #endif { PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; __Pyx_RefNannyDeclarations #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); @@ -2536,6 +2477,8 @@ PyMODINIT_FUNC PyInit_kenlm(void) __pyx_m = PyModule_Create(&__pyx_moduledef); #endif if (unlikely(!__pyx_m)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + Py_INCREF(__pyx_d); #if PY_MAJOR_VERSION >= 3 { PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -2551,6 +2494,9 @@ PyMODINIT_FUNC PyInit_kenlm(void) if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; /*--- Initialize various global constants etc. ---*/ if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + #endif if (__pyx_module_is_main_kenlm) { if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_n_s____main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; } @@ -2565,7 +2511,7 @@ PyMODINIT_FUNC PyInit_kenlm(void) if (PyType_Ready(&__pyx_type_5kenlm_LanguageModel) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "LanguageModel", (PyObject *)&__pyx_type_5kenlm_LanguageModel) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5kenlm_LanguageModel = &__pyx_type_5kenlm_LanguageModel; - if (PyType_Ready(&__pyx_type_5kenlm___pyx_scope_struct__full_scores) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5kenlm___pyx_scope_struct__full_scores) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5kenlm___pyx_scope_struct__full_scores = &__pyx_type_5kenlm___pyx_scope_struct__full_scores; /*--- Type import code ---*/ /*--- Variable import code ---*/ @@ -2579,7 +2525,7 @@ PyMODINIT_FUNC PyInit_kenlm(void) */ __pyx_t_1 = __Pyx_Import(((PyObject *)__pyx_n_s__os), 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (PyObject_SetAttr(__pyx_m, __pyx_n_s__os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s__os, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "kenlm.pyx":12 @@ -2591,7 +2537,7 @@ PyMODINIT_FUNC PyInit_kenlm(void) */ __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - if (PyObject_SetAttr(__pyx_m, __pyx_n_s____test__, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_d, __pyx_n_s____test__, ((PyObject *)__pyx_t_1)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; goto __pyx_L0; __pyx_L1_error:; @@ -2628,17 +2574,15 @@ end: } #endif /* CYTHON_REFNANNY */ -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { - PyObject *result; - result = PyObject_GetAttr(dict, name); - if (!result) { - if (dict != __pyx_b) { - PyErr_Clear(); - result = PyObject_GetAttr(__pyx_b, name); - } - if (!result) { - PyErr_SetObject(PyExc_NameError, name); - } +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%s' is not defined", PyString_AS_STRING(name)); +#endif } return result; } @@ -2712,24 +2656,23 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, } value = type; #if PY_VERSION_HEX < 0x02050000 - if (PyInstance_Check(type)) { - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - Py_INCREF(type); - } - else { - type = 0; - PyErr_SetString(PyExc_TypeError, - "raise: exception must be an old-style class or instance"); - goto raise_error; - } - #else - type = (PyObject*) Py_TYPE(type); + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } + } else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } #endif } __Pyx_ErrRestore(type, value, tb); @@ -2767,8 +2710,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject else if (PyTuple_Check(value)) { Py_INCREF(value); args = value; - } - else + } else args = PyTuple_Pack(1, value); if (!args) goto bad; @@ -2789,18 +2731,22 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject "raise: exception class must be a subclass of BaseException"); goto bad; } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else if (cause && cause != Py_None) { +#endif PyObject *fixed_cause; - if (PyExceptionClass_Check(cause)) { + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { fixed_cause = PyObject_CallObject(cause, NULL); if (fixed_cause == NULL) goto bad; - } - else if (PyExceptionInstance_Check(cause)) { + } else if (PyExceptionInstance_Check(cause)) { fixed_cause = cause; Py_INCREF(fixed_cause); - } - else { + } else { PyErr_SetString(PyExc_TypeError, "exception causes must derive from " "BaseException"); @@ -2963,6 +2909,23 @@ static void __Pyx_RaiseArgtupleInvalid( (num_expected == 1) ? "" : "s", num_found); } +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if CYTHON_COMPILING_IN_CPYTHON + result = PyDict_GetItem(__pyx_d, name); + if (result) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { PyObject *local_type, *local_value, *local_tb; #if CYTHON_COMPILING_IN_CPYTHON @@ -3058,8 +3021,8 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_dict = 0; PyObject *list; #if PY_VERSION_HEX < 0x03030000 - PyObject *py_import = 0; - py_import = __Pyx_GetAttrString(__pyx_b, "__import__"); + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s____import__); if (!py_import) goto bad; #endif @@ -3323,10 +3286,15 @@ static CYTHON_INLINE int __Pyx_PyInt_AsLongDouble(PyObject* x) { return (int)__Pyx_PyInt_AsLong(x); } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { const unsigned long neg_one = (unsigned long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3339,6 +3307,16 @@ static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(unsigned long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (unsigned long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned long"); @@ -3346,6 +3324,17 @@ static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { } return (unsigned long)PyLong_AsUnsignedLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(unsigned long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(unsigned long) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(unsigned long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (unsigned long)PyLong_AsLong(x); } } else { @@ -3358,10 +3347,15 @@ static CYTHON_INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { } } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { const unsigned PY_LONG_LONG neg_one = (unsigned PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3374,6 +3368,16 @@ static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObje #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to unsigned PY_LONG_LONG"); @@ -3381,6 +3385,17 @@ static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObje } return (unsigned PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(unsigned PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(unsigned PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (unsigned PY_LONG_LONG)PyLong_AsLongLong(x); } } else { @@ -3393,10 +3408,15 @@ static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObje } } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { const long neg_one = (long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3409,6 +3429,16 @@ static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to long"); @@ -3416,6 +3446,17 @@ static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { } return (long)PyLong_AsUnsignedLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(long) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (long)PyLong_AsLong(x); } } else { @@ -3428,10 +3469,15 @@ static CYTHON_INLINE long __Pyx_PyInt_AsLong(PyObject* x) { } } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { const PY_LONG_LONG neg_one = (PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3444,6 +3490,16 @@ static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to PY_LONG_LONG"); @@ -3451,6 +3507,17 @@ static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { } return (PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (PY_LONG_LONG)PyLong_AsLongLong(x); } } else { @@ -3463,10 +3530,15 @@ static CYTHON_INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { } } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { const signed long neg_one = (signed long)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3479,6 +3551,16 @@ static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(signed long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (signed long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed long"); @@ -3486,6 +3568,17 @@ static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { } return (signed long)PyLong_AsUnsignedLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(signed long)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(signed long) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(signed long) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (signed long)PyLong_AsLong(x); } } else { @@ -3498,10 +3591,15 @@ static CYTHON_INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { } } +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS +#include "longintrepr.h" +#endif +#endif static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { const signed PY_LONG_LONG neg_one = (signed PY_LONG_LONG)-1, const_zero = 0; const int is_unsigned = neg_one > const_zero; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { long val = PyInt_AS_LONG(x); if (is_unsigned && unlikely(val < 0)) { @@ -3514,6 +3612,16 @@ static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* #endif if (likely(PyLong_Check(x))) { if (is_unsigned) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return (signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif if (unlikely(Py_SIZE(x) < 0)) { PyErr_SetString(PyExc_OverflowError, "can't convert negative value to signed PY_LONG_LONG"); @@ -3521,6 +3629,17 @@ static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* } return (signed PY_LONG_LONG)PyLong_AsUnsignedLongLong(x); } else { +#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +#if CYTHON_USE_PYLONG_INTERNALS + if (sizeof(digit) <= sizeof(signed PY_LONG_LONG)) { + switch (Py_SIZE(x)) { + case 0: return 0; + case 1: return +(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + case -1: return -(signed PY_LONG_LONG) ((PyLongObject*)x)->ob_digit[0]; + } + } +#endif +#endif return (signed PY_LONG_LONG)PyLong_AsLongLong(x); } } else { @@ -3601,7 +3720,7 @@ static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { Py_DECREF(ev); #else { - PyObject* args = PyObject_GetAttrString(ev, "args"); + PyObject* args = PyObject_GetAttr(ev, __pyx_n_s__args); Py_DECREF(ev); if (likely(args)) { value = PyObject_GetItem(args, 0); @@ -3737,7 +3856,7 @@ static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value) { if (value == Py_None) ret = PyIter_Next(yf); else - ret = PyObject_CallMethod(yf, (char*)"send", (char*)"O", value); + ret = __Pyx_PyObject_CallMethod1(yf, __pyx_n_s__send, value); } gen->is_running = 0; if (likely(ret)) { @@ -3757,7 +3876,7 @@ static int __Pyx_Generator_CloseIter(__pyx_GeneratorObject *gen, PyObject *yf) { } else { PyObject *meth; gen->is_running = 1; - meth = PyObject_GetAttrString(yf, "close"); + meth = PyObject_GetAttr(yf, __pyx_n_s__close); if (unlikely(!meth)) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_WriteUnraisable(yf); @@ -3842,7 +3961,7 @@ static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args) { if (__Pyx_Generator_CheckExact(yf)) { ret = __Pyx_Generator_Throw(yf, args); } else { - PyObject *meth = PyObject_GetAttrString(yf, "throw"); + PyObject *meth = PyObject_GetAttr(yf, __pyx_n_s__throw); if (unlikely(!meth)) { Py_DECREF(yf); if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -3932,7 +4051,7 @@ static void __Pyx_Generator_del(PyObject *self) { _Py_NewReference(self); self->ob_refcnt = refcnt; } -#if CYTHON_COMPILING_FOR_CPYTHON +#if CYTHON_COMPILING_IN_CPYTHON assert(PyType_IS_GC(self->ob_type) && _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so @@ -4264,6 +4383,65 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { return 0; } +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif /*__PYX_DEFAULT_STRING_ENCODING_IS_ASCII*/ + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else /* PY_VERSION_HEX < 0x03030000 */ + if (PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_DATA_SIZE(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ + return PyUnicode_AsUTF8AndSize(o, length); +#endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ +#endif /* PY_VERSION_HEX < 0x03030000 */ + } else +#endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (r < 0) { + return NULL; + } else { + return result; + } + } +} static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { int is_true = x == Py_True; if (is_true | (x == Py_False) | (x == Py_None)) return is_true; @@ -4273,14 +4451,14 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { PyNumberMethods *m; const char *name = NULL; PyObject *res = NULL; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (PyInt_Check(x) || PyLong_Check(x)) #else if (PyLong_Check(x)) #endif return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); @@ -4296,7 +4474,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { } #endif if (res) { -#if PY_VERSION_HEX < 0x03000000 +#if PY_MAJOR_VERSION < 3 if (!PyInt_Check(res) && !PyLong_Check(res)) { #else if (!PyLong_Check(res)) { @@ -4337,11 +4515,10 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { } static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); - if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { - return (size_t)-1; - } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); + if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + if ((val != (unsigned PY_LONG_LONG)-1) || !PyErr_Occurred()) + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); return (size_t)-1; } return (size_t)val; diff --git a/python/kenlm.pxd b/python/kenlm.pxd index 35195dd..9a397f3 100644 --- a/python/kenlm.pxd +++ b/python/kenlm.pxd @@ -1,30 +1,31 @@ cdef extern from "lm/word_index.hh": ctypedef unsigned WordIndex -cdef extern from "lm/model.hh" namespace "lm": - cdef cppclass FullScoreReturn: - FullScoreReturn(FullScoreReturn) +cdef extern from "lm/return.hh" namespace "lm": + cdef struct FullScoreReturn: float prob unsigned char ngram_length -cdef extern from "lm/model.hh" namespace "lm::ngram": - cdef cppclass State: - State() - State(State& state) +cdef extern from "lm/state.hh" namespace "lm::ngram": + cdef struct State: + pass +cdef extern from "lm/virtual_interface.hh" namespace "lm::base": cdef cppclass Vocabulary: WordIndex Index(char*) WordIndex BeginSentence() WordIndex EndSentence() WordIndex NotFound() - ctypedef Vocabulary const_Vocabulary "const lm::ngram::Vocabulary" + ctypedef Vocabulary const_Vocabulary "const lm::base::Vocabulary" cdef cppclass Model: - Model(char*) except + - State &BeginSentenceState() - State &NullContextState() + void BeginSentenceWrite(void *) + void NullContextWrite(void *) unsigned int Order() - const_Vocabulary& GetVocabulary() - float Score(State &in_state, WordIndex new_word, State &out_state) - FullScoreReturn FullScore(State &in_state, WordIndex new_word, State &out_state) + const_Vocabulary& BaseVocabulary() + float Score(void *in_state, WordIndex new_word, void *out_state) + FullScoreReturn FullScore(void *in_state, WordIndex new_word, void *out_state) + +cdef extern from "lm/model.hh" namespace "lm::ngram": + cdef Model *LoadVirtual(char *) except + diff --git a/python/kenlm.pyx b/python/kenlm.pyx index 243b879..7f965d4 100644 --- a/python/kenlm.pyx +++ b/python/kenlm.pyx @@ -15,12 +15,12 @@ cdef class LanguageModel: def __init__(self, path): self.path = os.path.abspath(as_str(path)) try: - self.model = new Model(self.path) + self.model = LoadVirtual(self.path) except RuntimeError as exception: - exception_message = str(exception_message).replace('\n', ' ') + exception_message = str(exception).replace('\n', ' ') raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ from exception - self.vocab = &self.model.GetVocabulary() + self.vocab = &self.model.BaseVocabulary() def __dealloc__(self): del self.model @@ -31,39 +31,31 @@ cdef class LanguageModel: def score(self, sentence): cdef list words = as_str(sentence).split() - cdef State* state = new State(self.model.BeginSentenceState()) - cdef State* out_state = new State() + cdef State state + self.model.BeginSentenceWrite(&state) + cdef State out_state cdef float total = 0 for word in words: - total += self.model.Score(state[0], self.vocab.Index(word), out_state[0]) - state[0] = out_state[0] - total += self.model.Score(state[0], self.vocab.EndSentence(), out_state[0]) - del state, out_state + total += self.model.Score(&state, self.vocab.Index(word), &out_state) + state = out_state + total += self.model.Score(&state, self.vocab.EndSentence(), &out_state) return total def full_scores(self, sentence): cdef list words = as_str(sentence).split() - cdef State* state = new State(self.model.BeginSentenceState()) - cdef State* out_state = new State() - cdef FullScoreReturn* ret + cdef State state + self.model.BeginSentenceWrite(&state) + cdef State out_state + cdef FullScoreReturn ret cdef float total = 0 - try: - for word in words: - ret = new FullScoreReturn(self.model.FullScore(state[0], - self.vocab.Index(word), out_state[0])) - try: - yield (ret.prob, ret.ngram_length) - finally: - del ret - state[0] = out_state[0] - ret = new FullScoreReturn(self.model.FullScore(state[0], - self.vocab.EndSentence(), out_state[0])) - try: - yield (ret.prob, ret.ngram_length) - finally: - del ret - finally: - del state, out_state + for word in words: + ret = self.model.FullScore(&state, + self.vocab.Index(word), &out_state) + yield (ret.prob, ret.ngram_length) + state = out_state + ret = self.model.FullScore(&state, + self.vocab.EndSentence(), &out_state) + yield (ret.prob, ret.ngram_length) def __contains__(self, word): cdef bytes w = as_str(word) |