Deps: Python, bundle zstandard package

This package allows Python scripts to handle compressed blend files (see
rB2ea66af742bc). This is for example needed by Blender Asset Tracer to
send files to a Flamenco render farm.

This change includes a new `WITH_PYTHON_INSTALL_ZSTANDARD` build-time
option, to control whether to actually install the package. For this the
already-existing approach for Requests was copied.

Reviewed By: LazyDodo, mont29, brecht

Differential Revision: https://developer.blender.org/D12777

5 files changed, 44 insertions, 2 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ac11b495948..94a5ff27491 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -398,6 +398,10 @@ if(WITH_PYTHON_INSTALL)
     set(PYTHON_REQUESTS_PATH "" CACHE PATH "Path to python site-packages or dist-packages containing 'requests' module")
     mark_as_advanced(PYTHON_REQUESTS_PATH)
   endif()
+
+  option(WITH_PYTHON_INSTALL_ZSTANDARD "Copy zstandard into the blender install folder" ON)
+  set(PYTHON_ZSTANDARD_PATH "" CACHE PATH "Path to python site-packages or dist-packages containing 'zstandard' module")
+  mark_as_advanced(PYTHON_ZSTANDARD_PATH)
 endif()
 
 option(WITH_CPU_SIMD              "Enable SIMD instruction if they're detected on the host machine" ON)
@@ -1734,6 +1738,12 @@ if(WITH_PYTHON)
   elseif(WITH_PYTHON_INSTALL_REQUESTS)
     find_python_package(requests "")
   endif()
+
+  if(WIN32 OR APPLE)
+    # pass, we have this in lib/python/site-packages
+  elseif(WITH_PYTHON_INSTALL_ZSTANDARD)
+    find_python_package(zstandard "")
+  endif()
 endif()
 
 # Select C++17 as the standard for C++ projects.
@@ -2005,6 +2015,7 @@ if(FIRST_RUN)
   endif()
   info_cfg_option(WITH_PYTHON_INSTALL)
   info_cfg_option(WITH_PYTHON_INSTALL_NUMPY)
+  info_cfg_option(WITH_PYTHON_INSTALL_ZSTANDARD)
   info_cfg_option(WITH_PYTHON_MODULE)
   info_cfg_option(WITH_PYTHON_SAFETY)
 
diff --git a/build_files/build_environment/cmake/python_site_packages.cmake b/build_files/build_environment/cmake/python_site_packages.cmake
index 6bc47fa5c45..a8918fdb784 100644
--- a/build_files/build_environment/cmake/python_site_packages.cmake
+++ b/build_files/build_environment/cmake/python_site_packages.cmake
@@ -18,14 +18,20 @@
 
 if(WIN32 AND BUILD_MODE STREQUAL Debug)
   set(SITE_PACKAGES_EXTRA --global-option build --global-option --debug)
+  # zstandard is determined to build and link release mode libs in a debug
+  # configuration, the only way to make it happy is to bend to its will
+  # and give it a library to link with.
+  set(PIP_CONFIGURE_COMMAND ${CMAKE_COMMAND} -E copy ${LIBDIR}/python/libs/python${PYTHON_SHORT_VERSION_NO_DOTS}_d.lib ${LIBDIR}/python/libs/python${PYTHON_SHORT_VERSION_NO_DOTS}.lib)
+else()
+  set(PIP_CONFIGURE_COMMAND echo ".")
 endif()
 
 ExternalProject_Add(external_python_site_packages
   DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND ""
+  CONFIGURE_COMMAND ${PIP_CONFIGURE_COMMAND}
   BUILD_COMMAND ""
   PREFIX ${BUILD_DIR}/site_packages
-  INSTALL_COMMAND ${PYTHON_BINARY} -m pip install ${SITE_PACKAGES_EXTRA} cython==${CYTHON_VERSION} idna==${IDNA_VERSION} charset-normalizer==${CHARSET_NORMALIZER_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} --no-binary :all:
+  INSTALL_COMMAND ${PYTHON_BINARY} -m pip install ${SITE_PACKAGES_EXTRA} cython==${CYTHON_VERSION} idna==${IDNA_VERSION} charset-normalizer==${CHARSET_NORMALIZER_VERSION} urllib3==${URLLIB3_VERSION} certifi==${CERTIFI_VERSION} requests==${REQUESTS_VERSION} zstandard==${ZSTANDARD_VERSION} --no-binary :all:
 )
 
 if(USE_PIP_NUMPY)
diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake
index ec83de4866f..f2c245dc380 100644
--- a/build_files/build_environment/cmake/versions.cmake
+++ b/build_files/build_environment/cmake/versions.cmake
@@ -221,6 +221,7 @@ set(URLLIB3_VERSION 1.26.7)
 set(CERTIFI_VERSION 2021.10.8)
 set(REQUESTS_VERSION 2.26.0)
 set(CYTHON_VERSION 0.29.24)
+set(ZSTANDARD_VERSION 0.15.2 )
 
 set(NUMPY_VERSION 1.21.2)
 set(NUMPY_SHORT_VERSION 1.21)
diff --git a/build_files/build_environment/install_deps.sh b/build_files/build_environment/install_deps.sh
index a278fabb6b8..2bb7175da00 100755
--- a/build_files/build_environment/install_deps.sh
+++ b/build_files/build_environment/install_deps.sh
@@ -414,6 +414,11 @@ PYTHON_REQUESTS_VERSION_MIN="2.0"
 PYTHON_REQUESTS_VERSION_MEX="3.0"
 PYTHON_REQUESTS_NAME="requests"
 
+PYTHON_ZSTANDARD_VERSION="0.15.2"
+PYTHON_ZSTANDARD_VERSION_MIN="0.15.2"
+PYTHON_ZSTANDARD_VERSION_MEX="0.16.0"
+PYTHON_ZSTANDARD_NAME="zstandard"
+
 PYTHON_NUMPY_VERSION="1.21.2"
 PYTHON_NUMPY_VERSION_MIN="1.14"
 PYTHON_NUMPY_VERSION_MEX="2.0"
@@ -426,6 +431,7 @@ PYTHON_MODULES_PACKAGES=(
   "$PYTHON_URLLIB3_NAME $PYTHON_URLLIB3_VERSION_MIN $PYTHON_URLLIB3_VERSION_MEX"
   "$PYTHON_CERTIFI_NAME $PYTHON_CERTIFI_VERSION_MIN $PYTHON_CERTIFI_VERSION_MEX"
   "$PYTHON_REQUESTS_NAME $PYTHON_REQUESTS_VERSION_MIN $PYTHON_REQUESTS_VERSION_MEX"
+  "$PYTHON_ZSTANDARD_NAME $PYTHON_ZSTANDARD_VERSION_MIN $PYTHON_ZSTANDARD_VERSION_MEX"
   "$PYTHON_NUMPY_NAME $PYTHON_NUMPY_VERSION_MIN $PYTHON_NUMPY_VERSION_MEX"
 )
 
diff --git a/source/creator/CMakeLists.txt b/source/creator/CMakeLists.txt
index 1d5d1491c7a..5ee9a3f4342 100644
--- a/source/creator/CMakeLists.txt
+++ b/source/creator/CMakeLists.txt
@@ -642,6 +642,24 @@ if(UNIX AND NOT APPLE)
         unset(_suffix)
       endif()
 
+      if(WITH_PYTHON_INSTALL_ZSTANDARD)
+        # Install to the same directory as the source, so debian-like
+        # distros are happy with their policy.
+        set(_suffix "site-packages")
+        if(${PYTHON_NUMPY_PATH} MATCHES "dist-packages")
+          set(_suffix "dist-packages")
+        endif()
+        install(
+          DIRECTORY ${PYTHON_NUMPY_PATH}/zstandard
+          DESTINATION ${TARGETDIR_VER}/python/${_target_LIB}/python${PYTHON_VERSION}/${_suffix}
+          PATTERN ".svn" EXCLUDE
+          PATTERN "__pycache__" EXCLUDE           # * any cache *
+          PATTERN "*.pyc" EXCLUDE                 # * any cache *
+          PATTERN "*.pyo" EXCLUDE                 # * any cache *
+        )
+        unset(_suffix)
+      endif()
+
       # Copy requests, we need to generalize site-packages
       if(WITH_PYTHON_INSTALL_REQUESTS)
         set(_suffix "site-packages")