cmake/deps: update Python to 3.9.1

The following packages also have received updates:

- IDNA 2.10
- CHARDET 4.0.0
- URLLIB3 1.26.3
- CERTIFI 2020.12.5
- REQUESTS 2.25.1
- NUMPY 1.19.5

numpy has gained a hard dependency on cython:

- CYTHON 0.29.21

Notes:

- This only updates the build environment files,
  once these are built, Blender can default to Python 3.9.
- The 'm' suffix for Python binaries/libs has been removed.
- The macOS patch in Python 3.7 is has been removed.

Reviewed By: sybren, campbellbarton, sebbas

Ref D10257

7 files changed, 30 insertions, 318 deletions

diff --git a/build_files/build_environment/cmake/harvest.cmake b/build_files/build_environment/cmake/harvest.cmake
index 6785b727a69..308cb77144c 100644
--- a/build_files/build_environment/cmake/harvest.cmake
+++ b/build_files/build_environment/cmake/harvest.cmake
@@ -162,7 +162,7 @@ harvest(png/include png/include "*.h")
 harvest(png/lib png/lib "*.a")
 harvest(pugixml/include pugixml/include "*.hpp")
 harvest(pugixml/lib pugixml/lib "*.a")
-harvest(python/bin python/bin "python${PYTHON_SHORT_VERSION}m")
+harvest(python/bin python/bin "python${PYTHON_SHORT_VERSION}")
 harvest(python/include python/include "*h")
 harvest(python/lib python/lib "*")
 harvest(sdl/include/SDL2 sdl/include "*.h")
diff --git a/build_files/build_environment/cmake/numpy.cmake b/build_files/build_environment/cmake/numpy.cmake
index 03316a8fc63..7a1b00895f4 100644
--- a/build_files/build_environment/cmake/numpy.cmake
+++ b/build_files/build_environment/cmake/numpy.cmake
@@ -47,4 +47,5 @@ ExternalProject_Add(external_numpy
 add_dependencies(
   external_numpy
   external_python
+  external_python_site_packages
 )
diff --git a/build_files/build_environment/cmake/python.cmake b/build_files/build_environment/cmake/python.cmake
index dfb2437d3cb..5731a0e0ae8 100644
--- a/build_files/build_environment/cmake/python.cmake
+++ b/build_files/build_environment/cmake/python.cmake
@@ -43,7 +43,7 @@ if(WIN32)
     PREFIX ${BUILD_DIR}/python
     CONFIGURE_COMMAND ""
     BUILD_COMMAND cd ${BUILD_DIR}/python/src/external_python/pcbuild/ && set IncludeTkinter=false && call build.bat -e -p x64 -c ${BUILD_MODE}
-    INSTALL_COMMAND ${PYTHON_BINARY_INTERNAL} ${PYTHON_SRC}/PC/layout/main.py -b ${PYTHON_SRC}/PCbuild/amd64 -s ${PYTHON_SRC} -t ${PYTHON_SRC}/tmp/  --include-underpth --include-stable --include-pip --include-dev --include-launchers  --include-venv --include-symbols ${PYTHON_EXTRA_INSTLAL_FLAGS} --copy ${LIBDIR}/python
+    INSTALL_COMMAND ${PYTHON_BINARY_INTERNAL} ${PYTHON_SRC}/PC/layout/main.py -b ${PYTHON_SRC}/PCbuild/amd64 -s ${PYTHON_SRC} -t ${PYTHON_SRC}/tmp/ --include-stable --include-pip --include-dev --include-launchers  --include-venv --include-symbols ${PYTHON_EXTRA_INSTLAL_FLAGS} --copy ${LIBDIR}/python
   )
 
 else()
@@ -74,7 +74,6 @@ else()
     endif()
     set(PYTHON_CONFIGURE_ENV ${CONFIGURE_ENV} && ${PYTHON_FUNC_CONFIGS})
     set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python.exe)
-    set(PYTHON_PATCH ${PATCH_CMD} --verbose -p1 -d ${BUILD_DIR}/python/src/external_python < ${PATCH_DIR}/python_macos.diff)
   else()
     set(PYTHON_CONFIGURE_ENV ${CONFIGURE_ENV})
     set(PYTHON_BINARY ${BUILD_DIR}/python/src/external_python/python)
diff --git a/build_files/build_environment/cmake/python_site_packages.cmake b/build_files/build_environment/cmake/python_site_packages.cmake
index d17f65a152b..58d95326ee9 100644
--- a/build_files/build_environment/cmake/python_site_packages.cmake
+++ b/build_files/build_environment/cmake/python_site_packages.cmake
@@ -21,7 +21,7 @@ ExternalProject_Add(external_python_site_packages
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   PREFIX ${BUILD_DIR}/site_packages
-  INSTALL_COMMAND ${PYTHON_BINARY} -m pip install idna==${IDNA_VERSION} chardet==${CHARDET_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} --no-binary :all:
+  INSTALL_COMMAND ${PYTHON_BINARY} -m pip install cython==${CYTHON_VERSION} idna==${IDNA_VERSION} chardet==${CHARDET_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} --no-binary :all:
 )
 
 add_dependencies(
diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake
index 98558721cb3..76417b59bb3 100644
--- a/build_files/build_environment/cmake/versions.cmake
+++ b/build_files/build_environment/cmake/versions.cmake
@@ -138,11 +138,11 @@ set(OSL_VERSION 1.10.10)
 set(OSL_URI https://github.com/imageworks/OpenShadingLanguage/archive/Release-${OSL_VERSION}.tar.gz)
 set(OSL_HASH 00dec08a93c8084e53848b9ad047889f)
 
-set(PYTHON_VERSION 3.7.7)
-set(PYTHON_SHORT_VERSION 3.7)
-set(PYTHON_SHORT_VERSION_NO_DOTS 37)
+set(PYTHON_VERSION 3.9.1)
+set(PYTHON_SHORT_VERSION 3.9)
+set(PYTHON_SHORT_VERSION_NO_DOTS 39)
 set(PYTHON_URI https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz)
-set(PYTHON_HASH 172c650156f7bea68ce31b2fd01fa766)
+set(PYTHON_HASH 61981498e75ac8f00adcb908281fadb6)
 
 set(TBB_VERSION 2019_U9)
 set(TBB_URI https://github.com/oneapi-src/oneTBB/archive/${TBB_VERSION}.tar.gz)
@@ -156,16 +156,17 @@ set(NANOVDB_GIT_UID e62f7a0bf1e27397223c61ddeaaf57edf111b77f)
 set(NANOVDB_URI https://github.com/AcademySoftwareFoundation/openvdb/archive/${NANOVDB_GIT_UID}.tar.gz)
 set(NANOVDB_HASH 90919510bc6ccd630fedc56f748cb199)
 
-set(IDNA_VERSION 2.9)
-set(CHARDET_VERSION 3.0.4)
-set(URLLIB3_VERSION 1.25.9)
-set(CERTIFI_VERSION 2020.4.5.2)
-set(REQUESTS_VERSION 2.23.0)
+set(IDNA_VERSION 2.10)
+set(CHARDET_VERSION 4.0.0)
+set(URLLIB3_VERSION 1.26.3)
+set(CERTIFI_VERSION 2020.12.5)
+set(REQUESTS_VERSION 2.25.1)
+set(CYTHON_VERSION 0.29.21)
 
-set(NUMPY_VERSION 1.17.5)
-set(NUMPY_SHORT_VERSION 1.17)
+set(NUMPY_VERSION 1.19.5)
+set(NUMPY_SHORT_VERSION 1.19)
 set(NUMPY_URI https://github.com/numpy/numpy/releases/download/v${NUMPY_VERSION}/numpy-${NUMPY_VERSION}.zip)
-set(NUMPY_HASH 763a5646fa6eef7a22f4895bca0524f2)
+set(NUMPY_HASH f6a1b48717c552bbc18f1adc3cc1fe0e)
 
 set(LAME_VERSION 3.100)
 set(LAME_URI http://downloads.sourceforge.net/project/lame/lame/3.100/lame-${LAME_VERSION}.tar.gz)
diff --git a/build_files/build_environment/patches/python_linux.diff b/build_files/build_environment/patches/python_linux.diff
index 24d625c7ceb..a9baae8ce32 100644
--- a/build_files/build_environment/patches/python_linux.diff
+++ b/build_files/build_environment/patches/python_linux.diff
@@ -2,23 +2,23 @@ diff --git a/setup.py.orig b/setup.py
 index a97a755..07ce853 100644
 --- a/setup.py.orig
 +++ b/setup.py
-@@ -1422,13 +1422,13 @@ class PyBuildExt(build_ext):
+@@ -1603,13 +1603,13 @@
                          version = line.split()[2]
                          break
              if version >= version_req:
--                if (self.compiler.find_library_file(lib_dirs, 'z')):
-+                if (self.compiler.find_library_file(lib_dirs, 'z_pic')):
-                     if host_platform == "darwin":
+-                if (self.compiler.find_library_file(self.lib_dirs, 'z')):
++                if (self.compiler.find_library_file(self.lib_dirs, 'z_pic')):
+                     if MACOS:
                          zlib_extra_link_args = ('-Wl,-search_paths_first',)
                      else:
                          zlib_extra_link_args = ()
-                     exts.append( Extension('zlib', ['zlibmodule.c'],
--                                           libraries = ['z'],
-+                                           libraries = ['z_pic'],
-                                            extra_link_args = zlib_extra_link_args))
+                     self.add(Extension('zlib', ['zlibmodule.c'],
+-                                       libraries=['z'],
++                                       libraries=['z_pic'],
+                                        extra_link_args=zlib_extra_link_args))
                      have_zlib = True
                  else:
-@@ -1442,7 +1442,7 @@ class PyBuildExt(build_ext):
+@@ -1623,7 +1623,7 @@
          # crc32 if we have it.  Otherwise binascii uses its own.
          if have_zlib:
              extra_compile_args = ['-DUSE_ZLIB_CRC32']
@@ -27,12 +27,12 @@ index a97a755..07ce853 100644
              extra_link_args = zlib_extra_link_args
          else:
              extra_compile_args = []
-@@ -1991,7 +1991,7 @@ class PyBuildExt(build_ext):
+@@ -2168,7 +2168,7 @@
+                 ffi_inc = None
                  print('Header file {} does not exist'.format(ffi_h))
-         ffi_lib = None
-         if ffi_inc is not None:
+         if ffi_lib is None and ffi_inc:
 -            for lib_name in ('ffi', 'ffi_pic'):
 +            for lib_name in ('ffi_pic', ):
-                 if (self.compiler.find_library_file(lib_dirs, lib_name)):
+                 if (self.compiler.find_library_file(self.lib_dirs, lib_name)):
                      ffi_lib = lib_name
                      break
diff --git a/build_files/build_environment/patches/python_macos.diff b/build_files/build_environment/patches/python_macos.diff
deleted file mode 100644
index 22ccbebee2f..00000000000
--- a/build_files/build_environment/patches/python_macos.diff
+++ /dev/null
@@ -1,289 +0,0 @@
-diff -ru a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
---- a/Doc/library/ctypes.rst	2020-03-10 07:11:12.000000000 +0100
-+++ b/Doc/library/ctypes.rst	2020-07-14 08:10:10.000000000 +0200
-@@ -1551,6 +1551,13 @@
-       value usable as argument (integer, string, ctypes instance).  This allows
-       defining adapters that can adapt custom objects as function parameters.
- 
-+   .. attribute:: variadic
-+
-+      Assign a boolean to specify that the function takes a variable number of
-+      arguments.   This does not matter on most platforms, but for Apple arm64
-+      platforms variadic functions have a different calling convention than
-+      normal functions.
-+
-    .. attribute:: errcheck
- 
-       Assign a Python function or another callable to this attribute. The
-diff -ru a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
---- a/Modules/_ctypes/_ctypes.c	2020-03-10 07:11:12.000000000 +0100
-+++ b/Modules/_ctypes/_ctypes.c	2020-07-14 08:14:41.000000000 +0200
-@@ -3175,6 +3175,35 @@
- }
- 
- static int
-+PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
-+{
-+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
-+    assert(dict);
-+    int r = PyObject_IsTrue(ob);
-+    if (r == 1) {
-+        dict->flags |= FUNCFLAG_VARIADIC;
-+        return 0;
-+    } else if (r == 0) {
-+        dict->flags &= ~FUNCFLAG_VARIADIC;
-+        return 0;
-+    } else {
-+        return -1;
-+    }
-+}
-+
-+static PyObject *
-+PyCFuncPtr_get_variadic(PyCFuncPtrObject *self, void *Py_UNUSED(ignored))
-+{
-+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
-+    assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */
-+    if (dict->flags & FUNCFLAG_VARIADIC)
-+        Py_RETURN_TRUE;
-+    else
-+        Py_RETURN_FALSE;
-+}
-+
-+
-+static int
- PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
- {
-     PyObject *converters;
-@@ -5632,6 +5661,7 @@
-     PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO));
-     PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR));
-     PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI));
-+    PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC));
-     PyModule_AddStringConstant(m, "__version__", "1.1.0");
- 
-     PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove));
-diff -ru a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
---- a/Modules/_ctypes/callproc.c	2020-03-10 07:11:12.000000000 +0100
-+++ b/Modules/_ctypes/callproc.c	2020-07-14 08:18:33.000000000 +0200
-@@ -767,7 +767,8 @@
-                                   ffi_type **atypes,
-                                   ffi_type *restype,
-                                   void *resmem,
--                                  int argcount)
-+                                  int argcount,
-+                                  int argtypecount)
- {
-     PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */
-     PyObject *error_object = NULL;
-@@ -793,15 +794,38 @@
-     if ((flags & FUNCFLAG_CDECL) == 0)
-         cc = FFI_STDCALL;
- #endif
--    if (FFI_OK != ffi_prep_cif(&cif,
--                               cc,
--                               argcount,
--                               restype,
--                               atypes)) {
--        PyErr_SetString(PyExc_RuntimeError,
--                        "ffi_prep_cif failed");
--        return -1;
--    }
-+#if HAVE_FFI_PREP_CIF_VAR
-+    /* Everyone SHOULD set f.variadic=True on variadic function pointers, but
-+     * lots of existing code will not.  If there's at least one arg and more
-+     * args are passed than are defined in the prototype, then it must be a
-+     * variadic function. */
-+    if ((flags & FUNCFLAG_VARIADIC) ||
-+        (argtypecount != 0 && argcount > argtypecount))
-+    {
-+        if (FFI_OK != ffi_prep_cif_var(&cif,
-+                                       cc,
-+                                       argtypecount,
-+                                       argcount,
-+                                       restype,
-+                                       atypes)) {
-+            PyErr_SetString(PyExc_RuntimeError,
-+                            "ffi_prep_cif_var failed");
-+            return -1;
-+        }
-+    } else {
-+#endif
-+        if (FFI_OK != ffi_prep_cif(&cif,
-+                                   cc,
-+                                   argcount,
-+                                   restype,
-+                                   atypes)) {
-+            PyErr_SetString(PyExc_RuntimeError,
-+                            "ffi_prep_cif failed");
-+            return -1;
-+        }
-+#if HAVE_FFI_PREP_CIF_VAR
-+     }
-+#endif
- 
-     if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) {
-         error_object = _ctypes_get_errobj(&space);
-@@ -1185,9 +1209,8 @@
- 
-     if (-1 == _call_function_pointer(flags, pProc, avalues, atypes,
-                                      rtype, resbuf,
--                                     Py_SAFE_DOWNCAST(argcount,
--                                                      Py_ssize_t,
--                                                      int)))
-+                                     Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int),
-+                                     Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int)))
-         goto cleanup;
- 
- #ifdef WORDS_BIGENDIAN
-diff -ru a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h
---- a/Modules/_ctypes/ctypes.h	2020-03-10 07:11:12.000000000 +0100
-+++ b/Modules/_ctypes/ctypes.h	2020-07-14 08:30:53.000000000 +0200
-@@ -285,6 +285,7 @@
- #define FUNCFLAG_PYTHONAPI 0x4
- #define FUNCFLAG_USE_ERRNO 0x8
- #define FUNCFLAG_USE_LASTERROR 0x10
-+#define FUNCFLAG_VARIADIC 0x20
- 
- #define TYPEFLAG_ISPOINTER 0x100
- #define TYPEFLAG_HASPOINTER 0x200
-diff -ru a/configure b/configure
---- a/configure	2020-03-10 07:11:12.000000000 +0100
-+++ b/configure	2020-07-14 08:03:27.000000000 +0200
-@@ -3374,7 +3374,7 @@
-   # has no effect, don't bother defining them
-   Darwin/[6789].*)
-     define_xopen_source=no;;
--  Darwin/1[0-9].*)
-+  Darwin/[12][0-9].*)
-     define_xopen_source=no;;
-   # On AIX 4 and 5.1, mbstate_t is defined only when _XOPEN_SOURCE == 500 but
-   # used in wcsnrtombs() and mbsnrtowcs() even if _XOPEN_SOURCE is not defined
-@@ -9251,6 +9251,9 @@
-     	ppc)
-     		MACOSX_DEFAULT_ARCH="ppc64"
-     		;;
-+        arm64)
-+            MACOSX_DEFAULT_ARCH="arm64"
-+            ;;
-     	*)
-     		as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5
-     		;;
-diff -ru a/configure.ac b/configure.ac
---- a/configure.ac	2020-03-10 07:11:12.000000000 +0100
-+++ b/configure.ac	2020-07-14 08:03:27.000000000 +0200
-@@ -2456,6 +2456,9 @@
-     	ppc)
-     		MACOSX_DEFAULT_ARCH="ppc64"
-     		;;
-+        arm64)
-+            MACOSX_DEFAULT_ARCH="arm64"
-+            ;;
-     	*)
-     		AC_MSG_ERROR([Unexpected output of 'arch' on OSX])
-     		;;
-diff -ru a/setup.py b/setup.py
---- a/setup.py	2020-03-10 07:11:12.000000000 +0100
-+++ b/setup.py	2020-07-14 08:28:12.000000000 +0200
-@@ -141,6 +141,13 @@
-             os.unlink(tmpfile)
- 
-     return MACOS_SDK_ROOT
-+ 
-+def is_macosx_at_least(vers):
-+    if host_platform == 'darwin':
-+        dep_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
-+        if dep_target:
-+            return tuple(map(int, dep_target.split('.'))) >= vers
-+    return False
- 
- def is_macosx_sdk_path(path):
-     """
-@@ -150,6 +157,13 @@
-                 or path.startswith('/System/')
-                 or path.startswith('/Library/') )
- 
-+def grep_headers_for(function, headers):
-+    for header in headers:
-+        with open(header, 'r') as f:
-+            if function in f.read():
-+                return True
-+    return False
-+
- def find_file(filename, std_dirs, paths):
-     """Searches for the directory where a given file is located,
-     and returns a possibly-empty list of additional directories, or None
-@@ -1972,7 +1986,11 @@
-         return True
- 
-     def detect_ctypes(self, inc_dirs, lib_dirs):
--        self.use_system_libffi = False
-+        if not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and is_macosx_at_least((10,15)):
-+            self.use_system_libffi = True
-+        else:
-+            self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS")
-+
-         include_dirs = []
-         extra_compile_args = []
-         extra_link_args = []
-@@ -2016,32 +2034,48 @@
-         ext_test = Extension('_ctypes_test',
-                      sources=['_ctypes/_ctypes_test.c'],
-                      libraries=['m'])
-+        ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR")
-+        ffi_lib = None
-+
-         self.extensions.extend([ext, ext_test])
- 
-         if host_platform == 'darwin':
--            if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"):
-+            if not self.use_system_libffi:
-                 return
--            # OS X 10.5 comes with libffi.dylib; the include files are
--            # in /usr/include/ffi
--            inc_dirs.append('/usr/include/ffi')
--
--        ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")]
--        if not ffi_inc or ffi_inc[0] == '':
--            ffi_inc = find_file('ffi.h', [], inc_dirs)
--        if ffi_inc is not None:
--            ffi_h = ffi_inc[0] + '/ffi.h'
-+            ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi")
-+            if os.path.exists(ffi_in_sdk):
-+                ffi_inc = ffi_in_sdk
-+                ffi_lib = 'ffi'
-+            else:
-+                # OS X 10.5 comes with libffi.dylib; the include files are
-+                # in /usr/include/ffi
-+                ffi_inc_dirs.append('/usr/include/ffi')
-+
-+        if not ffi_inc:
-+            found = find_file('ffi.h', [], ffi_inc_dirs)
-+            if found:
-+                ffi_inc = found[0]
-+        if ffi_inc:
-+            ffi_h = ffi_inc + '/ffi.h'
-             if not os.path.exists(ffi_h):
-                 ffi_inc = None
-                 print('Header file {} does not exist'.format(ffi_h))
--        ffi_lib = None
--        if ffi_inc is not None:
-+        if ffi_lib is None and ffi_inc:
-             for lib_name in ('ffi', 'ffi_pic'):
-                 if (self.compiler.find_library_file(lib_dirs, lib_name)):
-                     ffi_lib = lib_name
-                     break
- 
-         if ffi_inc and ffi_lib:
--            ext.include_dirs.extend(ffi_inc)
-+            ffi_headers = glob(os.path.join(ffi_inc, '*.h'))
-+            if grep_headers_for('ffi_closure_alloc', ffi_headers):
-+                try:
-+                    sources.remove('_ctypes/malloc_closure.c')
-+                except ValueError:
-+                    pass
-+            if grep_headers_for('ffi_prep_cif_var', ffi_headers):
-+                ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1")
-+            ext.include_dirs.append(ffi_inc)
-             ext.libraries.append(ffi_lib)
-             self.use_system_libffi = True
-