From 5bddfde217b1477c02de9dcfa8c078969d432519 Mon Sep 17 00:00:00 2001 From: Ray Molenkamp Date: Wed, 10 Feb 2021 09:24:47 +1100 Subject: cmake/deps: update Python to 3.9.1 The following packages also have received updates: - IDNA 2.10 - CHARDET 4.0.0 - URLLIB3 1.26.3 - CERTIFI 2020.12.5 - REQUESTS 2.25.1 - NUMPY 1.19.5 numpy has gained a hard dependency on cython: - CYTHON 0.29.21 Notes: - This only updates the build environment files, once these are built, Blender can default to Python 3.9. - The 'm' suffix for Python binaries/libs has been removed. - The macOS patch in Python 3.7 is has been removed. Reviewed By: sybren, campbellbarton, sebbas Ref D10257 --- build_files/build_environment/cmake/harvest.cmake | 2 +- build_files/build_environment/cmake/numpy.cmake | 1 + build_files/build_environment/cmake/python.cmake | 3 +- .../cmake/python_site_packages.cmake | 2 +- build_files/build_environment/cmake/versions.cmake | 25 +- .../build_environment/patches/python_linux.diff | 26 +- .../build_environment/patches/python_macos.diff | 289 --------------------- 7 files changed, 30 insertions(+), 318 deletions(-) delete mode 100644 build_files/build_environment/patches/python_macos.diff (limited to 'build_files') diff --git a/build_files/build_environment/cmake/harvest.cmake b/build_files/build_environment/cmake/harvest.cmake index 6785b727a69..308cb77144c 100644 --- a/build_files/build_environment/cmake/harvest.cmake +++ b/build_files/build_environment/cmake/harvest.cmake @@ -162,7 +162,7 @@ harvest(png/include png/include "*.h") harvest(png/lib png/lib "*.a") harvest(pugixml/include pugixml/include "*.hpp") harvest(pugixml/lib pugixml/lib "*.a") -harvest(python/bin python/bin "python${PYTHON_SHORT_VERSION}m") +harvest(python/bin python/bin "python${PYTHON_SHORT_VERSION}") harvest(python/include python/include "*h") harvest(python/lib python/lib "*") harvest(sdl/include/SDL2 sdl/include "*.h") diff --git a/build_files/build_environment/cmake/numpy.cmake b/build_files/build_environment/cmake/numpy.cmake index 03316a8fc63..7a1b00895f4 100644 --- a/build_files/build_environment/cmake/numpy.cmake +++ b/build_files/build_environment/cmake/numpy.cmake @@ -47,4 +47,5 @@ ExternalProject_Add(external_numpy add_dependencies( external_numpy external_python + external_python_site_packages ) diff --git a/build_files/build_environment/cmake/python.cmake b/build_files/build_environment/cmake/python.cmake index dfb2437d3cb..5731a0e0ae8 100644 --- a/build_files/build_environment/cmake/python.cmake +++ b/build_files/build_environment/cmake/python.cmake @@ -43,7 +43,7 @@ if(WIN32) PREFIX ${BUILD_DIR}/python CONFIGURE_COMMAND "" BUILD_COMMAND cd ${BUILD_DIR}/python/src/external_python/pcbuild/ && set IncludeTkinter=false && call build.bat -e -p x64 -c ${BUILD_MODE} - INSTALL_COMMAND ${PYTHON_BINARY_INTERNAL} ${PYTHON_SRC}/PC/layout/main.py -b ${PYTHON_SRC}/PCbuild/amd64 -s ${PYTHON_SRC} -t ${PYTHON_SRC}/tmp/ --include-underpth --include-stable --include-pip --include-dev --include-launchers --include-venv --include-symbols ${PYTHON_EXTRA_INSTLAL_FLAGS} --copy ${LIBDIR}/python + INSTALL_COMMAND ${PYTHON_BINARY_INTERNAL} ${PYTHON_SRC}/PC/layout/main.py -b ${PYTHON_SRC}/PCbuild/amd64 -s ${PYTHON_SRC} -t ${PYTHON_SRC}/tmp/ --include-stable --include-pip --include-dev --include-launchers --include-venv --include-symbols ${PYTHON_EXTRA_INSTLAL_FLAGS} --copy ${LIBDIR}/python ) else() @@ -74,7 +74,6 @@ else() endif() set(PYTHON_CONFIGURE_ENV ${CONFIGURE_ENV} && ${PYTHON_FUNC_CONFIGS}) set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python.exe) - set(PYTHON_PATCH ${PATCH_CMD} --verbose -p1 -d ${BUILD_DIR}/python/src/external_python < ${PATCH_DIR}/python_macos.diff) else() set(PYTHON_CONFIGURE_ENV ${CONFIGURE_ENV}) set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python) diff --git a/build_files/build_environment/cmake/python_site_packages.cmake b/build_files/build_environment/cmake/python_site_packages.cmake index d17f65a152b..58d95326ee9 100644 --- a/build_files/build_environment/cmake/python_site_packages.cmake +++ b/build_files/build_environment/cmake/python_site_packages.cmake @@ -21,7 +21,7 @@ ExternalProject_Add(external_python_site_packages CONFIGURE_COMMAND "" BUILD_COMMAND "" PREFIX ${BUILD_DIR}/site_packages - INSTALL_COMMAND ${PYTHON_BINARY} -m pip install idna==${IDNA_VERSION} chardet==${CHARDET_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} --no-binary :all: + INSTALL_COMMAND ${PYTHON_BINARY} -m pip install cython==${CYTHON_VERSION} idna==${IDNA_VERSION} chardet==${CHARDET_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} --no-binary :all: ) add_dependencies( diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake index 98558721cb3..76417b59bb3 100644 --- a/build_files/build_environment/cmake/versions.cmake +++ b/build_files/build_environment/cmake/versions.cmake @@ -138,11 +138,11 @@ set(OSL_VERSION 1.10.10) set(OSL_URI https://github.com/imageworks/OpenShadingLanguage/archive/Release-${OSL_VERSION}.tar.gz) set(OSL_HASH 00dec08a93c8084e53848b9ad047889f) -set(PYTHON_VERSION 3.7.7) -set(PYTHON_SHORT_VERSION 3.7) -set(PYTHON_SHORT_VERSION_NO_DOTS 37) +set(PYTHON_VERSION 3.9.1) +set(PYTHON_SHORT_VERSION 3.9) +set(PYTHON_SHORT_VERSION_NO_DOTS 39) set(PYTHON_URI https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz) -set(PYTHON_HASH 172c650156f7bea68ce31b2fd01fa766) +set(PYTHON_HASH 61981498e75ac8f00adcb908281fadb6) set(TBB_VERSION 2019_U9) set(TBB_URI https://github.com/oneapi-src/oneTBB/archive/${TBB_VERSION}.tar.gz) @@ -156,16 +156,17 @@ set(NANOVDB_GIT_UID e62f7a0bf1e27397223c61ddeaaf57edf111b77f) set(NANOVDB_URI https://github.com/AcademySoftwareFoundation/openvdb/archive/${NANOVDB_GIT_UID}.tar.gz) set(NANOVDB_HASH 90919510bc6ccd630fedc56f748cb199) -set(IDNA_VERSION 2.9) -set(CHARDET_VERSION 3.0.4) -set(URLLIB3_VERSION 1.25.9) -set(CERTIFI_VERSION 2020.4.5.2) -set(REQUESTS_VERSION 2.23.0) +set(IDNA_VERSION 2.10) +set(CHARDET_VERSION 4.0.0) +set(URLLIB3_VERSION 1.26.3) +set(CERTIFI_VERSION 2020.12.5) +set(REQUESTS_VERSION 2.25.1) +set(CYTHON_VERSION 0.29.21) -set(NUMPY_VERSION 1.17.5) -set(NUMPY_SHORT_VERSION 1.17) +set(NUMPY_VERSION 1.19.5) +set(NUMPY_SHORT_VERSION 1.19) set(NUMPY_URI https://github.com/numpy/numpy/releases/download/v${NUMPY_VERSION}/numpy-${NUMPY_VERSION}.zip) -set(NUMPY_HASH 763a5646fa6eef7a22f4895bca0524f2) +set(NUMPY_HASH f6a1b48717c552bbc18f1adc3cc1fe0e) set(LAME_VERSION 3.100) set(LAME_URI http://downloads.sourceforge.net/project/lame/lame/3.100/lame-${LAME_VERSION}.tar.gz) diff --git a/build_files/build_environment/patches/python_linux.diff b/build_files/build_environment/patches/python_linux.diff index 24d625c7ceb..a9baae8ce32 100644 --- a/build_files/build_environment/patches/python_linux.diff +++ b/build_files/build_environment/patches/python_linux.diff @@ -2,23 +2,23 @@ diff --git a/setup.py.orig b/setup.py index a97a755..07ce853 100644 --- a/setup.py.orig +++ b/setup.py -@@ -1422,13 +1422,13 @@ class PyBuildExt(build_ext): +@@ -1603,13 +1603,13 @@ version = line.split()[2] break if version >= version_req: -- if (self.compiler.find_library_file(lib_dirs, 'z')): -+ if (self.compiler.find_library_file(lib_dirs, 'z_pic')): - if host_platform == "darwin": +- if (self.compiler.find_library_file(self.lib_dirs, 'z')): ++ if (self.compiler.find_library_file(self.lib_dirs, 'z_pic')): + if MACOS: zlib_extra_link_args = ('-Wl,-search_paths_first',) else: zlib_extra_link_args = () - exts.append( Extension('zlib', ['zlibmodule.c'], -- libraries = ['z'], -+ libraries = ['z_pic'], - extra_link_args = zlib_extra_link_args)) + self.add(Extension('zlib', ['zlibmodule.c'], +- libraries=['z'], ++ libraries=['z_pic'], + extra_link_args=zlib_extra_link_args)) have_zlib = True else: -@@ -1442,7 +1442,7 @@ class PyBuildExt(build_ext): +@@ -1623,7 +1623,7 @@ # crc32 if we have it. Otherwise binascii uses its own. if have_zlib: extra_compile_args = ['-DUSE_ZLIB_CRC32'] @@ -27,12 +27,12 @@ index a97a755..07ce853 100644 extra_link_args = zlib_extra_link_args else: extra_compile_args = [] -@@ -1991,7 +1991,7 @@ class PyBuildExt(build_ext): +@@ -2168,7 +2168,7 @@ + ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: - for lib_name in ('ffi', 'ffi_pic'): + for lib_name in ('ffi_pic', ): - if (self.compiler.find_library_file(lib_dirs, lib_name)): + if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break diff --git a/build_files/build_environment/patches/python_macos.diff b/build_files/build_environment/patches/python_macos.diff deleted file mode 100644 index 22ccbebee2f..00000000000 --- a/build_files/build_environment/patches/python_macos.diff +++ /dev/null @@ -1,289 +0,0 @@ -diff -ru a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst ---- a/Doc/library/ctypes.rst 2020-03-10 07:11:12.000000000 +0100 -+++ b/Doc/library/ctypes.rst 2020-07-14 08:10:10.000000000 +0200 -@@ -1551,6 +1551,13 @@ - value usable as argument (integer, string, ctypes instance). This allows - defining adapters that can adapt custom objects as function parameters. - -+ .. attribute:: variadic -+ -+ Assign a boolean to specify that the function takes a variable number of -+ arguments. This does not matter on most platforms, but for Apple arm64 -+ platforms variadic functions have a different calling convention than -+ normal functions. -+ - .. attribute:: errcheck - - Assign a Python function or another callable to this attribute. The -diff -ru a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c ---- a/Modules/_ctypes/_ctypes.c 2020-03-10 07:11:12.000000000 +0100 -+++ b/Modules/_ctypes/_ctypes.c 2020-07-14 08:14:41.000000000 +0200 -@@ -3175,6 +3175,35 @@ - } - - static int -+PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) -+{ -+ StgDictObject *dict = PyObject_stgdict((PyObject *)self); -+ assert(dict); -+ int r = PyObject_IsTrue(ob); -+ if (r == 1) { -+ dict->flags |= FUNCFLAG_VARIADIC; -+ return 0; -+ } else if (r == 0) { -+ dict->flags &= ~FUNCFLAG_VARIADIC; -+ return 0; -+ } else { -+ return -1; -+ } -+} -+ -+static PyObject * -+PyCFuncPtr_get_variadic(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) -+{ -+ StgDictObject *dict = PyObject_stgdict((PyObject *)self); -+ assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ -+ if (dict->flags & FUNCFLAG_VARIADIC) -+ Py_RETURN_TRUE; -+ else -+ Py_RETURN_FALSE; -+} -+ -+ -+static int - PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored)) - { - PyObject *converters; -@@ -5632,6 +5661,7 @@ - PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO)); - PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR)); - PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI)); -+ PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC)); - PyModule_AddStringConstant(m, "__version__", "1.1.0"); - - PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove)); -diff -ru a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c ---- a/Modules/_ctypes/callproc.c 2020-03-10 07:11:12.000000000 +0100 -+++ b/Modules/_ctypes/callproc.c 2020-07-14 08:18:33.000000000 +0200 -@@ -767,7 +767,8 @@ - ffi_type **atypes, - ffi_type *restype, - void *resmem, -- int argcount) -+ int argcount, -+ int argtypecount) - { - PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ - PyObject *error_object = NULL; -@@ -793,15 +794,38 @@ - if ((flags & FUNCFLAG_CDECL) == 0) - cc = FFI_STDCALL; - #endif -- if (FFI_OK != ffi_prep_cif(&cif, -- cc, -- argcount, -- restype, -- atypes)) { -- PyErr_SetString(PyExc_RuntimeError, -- "ffi_prep_cif failed"); -- return -1; -- } -+#if HAVE_FFI_PREP_CIF_VAR -+ /* Everyone SHOULD set f.variadic=True on variadic function pointers, but -+ * lots of existing code will not. If there's at least one arg and more -+ * args are passed than are defined in the prototype, then it must be a -+ * variadic function. */ -+ if ((flags & FUNCFLAG_VARIADIC) || -+ (argtypecount != 0 && argcount > argtypecount)) -+ { -+ if (FFI_OK != ffi_prep_cif_var(&cif, -+ cc, -+ argtypecount, -+ argcount, -+ restype, -+ atypes)) { -+ PyErr_SetString(PyExc_RuntimeError, -+ "ffi_prep_cif_var failed"); -+ return -1; -+ } -+ } else { -+#endif -+ if (FFI_OK != ffi_prep_cif(&cif, -+ cc, -+ argcount, -+ restype, -+ atypes)) { -+ PyErr_SetString(PyExc_RuntimeError, -+ "ffi_prep_cif failed"); -+ return -1; -+ } -+#if HAVE_FFI_PREP_CIF_VAR -+ } -+#endif - - if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { - error_object = _ctypes_get_errobj(&space); -@@ -1185,9 +1209,8 @@ - - if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, - rtype, resbuf, -- Py_SAFE_DOWNCAST(argcount, -- Py_ssize_t, -- int))) -+ Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), -+ Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) - goto cleanup; - - #ifdef WORDS_BIGENDIAN -diff -ru a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h ---- a/Modules/_ctypes/ctypes.h 2020-03-10 07:11:12.000000000 +0100 -+++ b/Modules/_ctypes/ctypes.h 2020-07-14 08:30:53.000000000 +0200 -@@ -285,6 +285,7 @@ - #define FUNCFLAG_PYTHONAPI 0x4 - #define FUNCFLAG_USE_ERRNO 0x8 - #define FUNCFLAG_USE_LASTERROR 0x10 -+#define FUNCFLAG_VARIADIC 0x20 - - #define TYPEFLAG_ISPOINTER 0x100 - #define TYPEFLAG_HASPOINTER 0x200 -diff -ru a/configure b/configure ---- a/configure 2020-03-10 07:11:12.000000000 +0100 -+++ b/configure 2020-07-14 08:03:27.000000000 +0200 -@@ -3374,7 +3374,7 @@ - # has no effect, don't bother defining them - Darwin/[6789].*) - define_xopen_source=no;; -- Darwin/1[0-9].*) -+ Darwin/[12][0-9].*) - define_xopen_source=no;; - # On AIX 4 and 5.1, mbstate_t is defined only when _XOPEN_SOURCE == 500 but - # used in wcsnrtombs() and mbsnrtowcs() even if _XOPEN_SOURCE is not defined -@@ -9251,6 +9251,9 @@ - ppc) - MACOSX_DEFAULT_ARCH="ppc64" - ;; -+ arm64) -+ MACOSX_DEFAULT_ARCH="arm64" -+ ;; - *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 - ;; -diff -ru a/configure.ac b/configure.ac ---- a/configure.ac 2020-03-10 07:11:12.000000000 +0100 -+++ b/configure.ac 2020-07-14 08:03:27.000000000 +0200 -@@ -2456,6 +2456,9 @@ - ppc) - MACOSX_DEFAULT_ARCH="ppc64" - ;; -+ arm64) -+ MACOSX_DEFAULT_ARCH="arm64" -+ ;; - *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) - ;; -diff -ru a/setup.py b/setup.py ---- a/setup.py 2020-03-10 07:11:12.000000000 +0100 -+++ b/setup.py 2020-07-14 08:28:12.000000000 +0200 -@@ -141,6 +141,13 @@ - os.unlink(tmpfile) - - return MACOS_SDK_ROOT -+ -+def is_macosx_at_least(vers): -+ if host_platform == 'darwin': -+ dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') -+ if dep_target: -+ return tuple(map(int, dep_target.split('.'))) >= vers -+ return False - - def is_macosx_sdk_path(path): - """ -@@ -150,6 +157,13 @@ - or path.startswith('/System/') - or path.startswith('/Library/') ) - -+def grep_headers_for(function, headers): -+ for header in headers: -+ with open(header, 'r') as f: -+ if function in f.read(): -+ return True -+ return False -+ - def find_file(filename, std_dirs, paths): - """Searches for the directory where a given file is located, - and returns a possibly-empty list of additional directories, or None -@@ -1972,7 +1986,11 @@ - return True - - def detect_ctypes(self, inc_dirs, lib_dirs): -- self.use_system_libffi = False -+ if not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and is_macosx_at_least((10,15)): -+ self.use_system_libffi = True -+ else: -+ self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") -+ - include_dirs = [] - extra_compile_args = [] - extra_link_args = [] -@@ -2016,32 +2034,48 @@ - ext_test = Extension('_ctypes_test', - sources=['_ctypes/_ctypes_test.c'], - libraries=['m']) -+ ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") -+ ffi_lib = None -+ - self.extensions.extend([ext, ext_test]) - - if host_platform == 'darwin': -- if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): -+ if not self.use_system_libffi: - return -- # OS X 10.5 comes with libffi.dylib; the include files are -- # in /usr/include/ffi -- inc_dirs.append('/usr/include/ffi') -- -- ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] -- if not ffi_inc or ffi_inc[0] == '': -- ffi_inc = find_file('ffi.h', [], inc_dirs) -- if ffi_inc is not None: -- ffi_h = ffi_inc[0] + '/ffi.h' -+ ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") -+ if os.path.exists(ffi_in_sdk): -+ ffi_inc = ffi_in_sdk -+ ffi_lib = 'ffi' -+ else: -+ # OS X 10.5 comes with libffi.dylib; the include files are -+ # in /usr/include/ffi -+ ffi_inc_dirs.append('/usr/include/ffi') -+ -+ if not ffi_inc: -+ found = find_file('ffi.h', [], ffi_inc_dirs) -+ if found: -+ ffi_inc = found[0] -+ if ffi_inc: -+ ffi_h = ffi_inc + '/ffi.h' - if not os.path.exists(ffi_h): - ffi_inc = None - print('Header file {} does not exist'.format(ffi_h)) -- ffi_lib = None -- if ffi_inc is not None: -+ if ffi_lib is None and ffi_inc: - for lib_name in ('ffi', 'ffi_pic'): - if (self.compiler.find_library_file(lib_dirs, lib_name)): - ffi_lib = lib_name - break - - if ffi_inc and ffi_lib: -- ext.include_dirs.extend(ffi_inc) -+ ffi_headers = glob(os.path.join(ffi_inc, '*.h')) -+ if grep_headers_for('ffi_closure_alloc', ffi_headers): -+ try: -+ sources.remove('_ctypes/malloc_closure.c') -+ except ValueError: -+ pass -+ if grep_headers_for('ffi_prep_cif_var', ffi_headers): -+ ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") -+ ext.include_dirs.append(ffi_inc) - ext.libraries.append(ffi_lib) - self.use_system_libffi = True - -- cgit v1.2.3