Deps builder: OIIO/OSL/ISPC/OIDN/LLVM/Flex updates

This diff somewhat snowballed out of updating OIDN to 1.4.1 it had some
changes that allowed us to remove the arm hacks we had in place and
revert to using identical versions for a whole bunch of deps. But that
required an update to ISPC which needed a newer LLVM and if we're
updating LLVM we may as well update OSL, and when we update OSL, OIIO
may as well be dragged in soo......anyhow...

This diff updates:

LLVM 9.0.0 (11.0.1 for mac/arm) -> 12.0.0
OIIO 2.1.15.0 -> 2.2.15.1
OSL 1.11.10.0 -> 1.11.14.1
winflex_bison 2.5.5-> 2.5.24 (ispc needed newer bison, windows only dep)
OIDN 1.4.0 -> 1.4.1
ISPC v1.14.1(random hash for mac/arm) -> v1.16.0
Flex 2.6.4 (ISPC needed newer Flex than available on CentOS 7)

and removes most of the "special arm/mac" versions. I think just ssl and
embree are left with special versions.

notable changes:
@LazyDodo included some clang headers in the linux/mac harvest which are
needed to start writing custom clang based tooling like D9465 these were
already shipping on windows, but not the other platforms.

[macOS] Change the `LC_ID_DYLIB` of OpenMP  for {D11997}. This changes
where the executables look for dylibs.

Reviewed By: sebbas, LazyDodo

Differential Revision: https://developer.blender.org/D11748

8 files changed, 85 insertions, 60 deletions

diff --git a/build_files/build_environment/cmake/download.cmake b/build_files/build_environment/cmake/download.cmake
index b4d96ea856f..2a84202ea02 100644
--- a/build_files/build_environment/cmake/download.cmake
+++ b/build_files/build_environment/cmake/download.cmake
@@ -93,3 +93,4 @@ download_source(GMP)
 download_source(POTRACE)
 download_source(HARU)
 download_source(ZSTD)
+download_source(FLEX)
diff --git a/build_files/build_environment/cmake/flex.cmake b/build_files/build_environment/cmake/flex.cmake
new file mode 100644
index 00000000000..5daad6ce64b
--- /dev/null
+++ b/build_files/build_environment/cmake/flex.cmake
@@ -0,0 +1,28 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ***** END GPL LICENSE BLOCK *****
+
+ExternalProject_Add(external_flex
+  URL file://${PACKAGE_DIR}/${FLEX_FILE}
+  URL_HASH ${FLEX_HASH_TYPE}=${FLEX_HASH}
+  DOWNLOAD_DIR ${DOWNLOAD_DIR}
+  PREFIX ${BUILD_DIR}/flex
+  CONFIGURE_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && ${CONFIGURE_COMMAND} --prefix=${LIBDIR}/flex
+  BUILD_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && make -j${MAKE_THREADS}
+  INSTALL_COMMAND ${CONFIGURE_ENV} && cd ${BUILD_DIR}/flex/src/external_flex/ && make install
+  INSTALL_DIR ${LIBDIR}/flex
+)
diff --git a/build_files/build_environment/cmake/harvest.cmake b/build_files/build_environment/cmake/harvest.cmake
index a19c332457d..a3740559adc 100644
--- a/build_files/build_environment/cmake/harvest.cmake
+++ b/build_files/build_environment/cmake/harvest.cmake
@@ -106,6 +106,7 @@ harvest(llvm/include llvm/include "*")
 harvest(llvm/bin llvm/bin "llvm-config")
 harvest(llvm/lib llvm/lib "libLLVM*.a")
 harvest(llvm/lib llvm/lib "libclang*.a")
+harvest(llvm/lib/clang llvm/lib/clang "*.h")
 if(APPLE)
   harvest(openmp/lib openmp/lib "*")
   harvest(openmp/include openmp/include "*.h")
diff --git a/build_files/build_environment/cmake/ispc.cmake b/build_files/build_environment/cmake/ispc.cmake
index e8a4dedcdaa..93fc9dc4846 100644
--- a/build_files/build_environment/cmake/ispc.cmake
+++ b/build_files/build_environment/cmake/ispc.cmake
@@ -29,12 +29,13 @@ elseif(APPLE)
   if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "arm64")
     set(ISPC_EXTRA_ARGS_APPLE
       -DBISON_EXECUTABLE=/opt/homebrew/opt/bison/bin/bison
-      -DFLEX_EXECUTABLE=/opt/homebrew/opt/flex/bin/flex
+      -DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
       -DARM_ENABLED=On
     )
   else()
     set(ISPC_EXTRA_ARGS_APPLE
       -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison
+      -DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
       -DARM_ENABLED=Off
     )
   endif()
@@ -43,6 +44,7 @@ elseif(UNIX)
     -DCMAKE_C_COMPILER=${LIBDIR}/llvm/bin/clang
     -DCMAKE_CXX_COMPILER=${LIBDIR}/llvm/bin/clang++
     -DARM_ENABLED=Off
+    -DFLEX_EXECUTABLE=${LIBDIR}/flex/bin/flex
   )
 endif()
 
@@ -82,4 +84,9 @@ if(WIN32)
     external_ispc
     external_flexbison
   )
+else()
+  add_dependencies(
+    external_ispc
+    external_flex
+  )
 endif()
diff --git a/build_files/build_environment/cmake/openimagedenoise.cmake b/build_files/build_environment/cmake/openimagedenoise.cmake
index f927d6c231c..aca2b4cef9f 100644
--- a/build_files/build_environment/cmake/openimagedenoise.cmake
+++ b/build_files/build_environment/cmake/openimagedenoise.cmake
@@ -45,7 +45,6 @@ ExternalProject_Add(external_openimagedenoise
   DOWNLOAD_DIR ${DOWNLOAD_DIR}
   URL_HASH ${OIDN_HASH_TYPE}=${OIDN_HASH}
   PREFIX ${BUILD_DIR}/openimagedenoise
-  PATCH_COMMAND ${PATCH_CMD} -p 1 -N -d ${BUILD_DIR}/openimagedenoise/src/external_openimagedenoise < ${PATCH_DIR}/oidn.diff
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openimagedenoise ${DEFAULT_CMAKE_FLAGS} ${OIDN_EXTRA_ARGS}
   INSTALL_DIR ${LIBDIR}/openimagedenoise
 )
diff --git a/build_files/build_environment/cmake/openmp.cmake b/build_files/build_environment/cmake/openmp.cmake
index 96081c7fc49..0e5323ca513 100644
--- a/build_files/build_environment/cmake/openmp.cmake
+++ b/build_files/build_environment/cmake/openmp.cmake
@@ -22,9 +22,8 @@ ExternalProject_Add(external_openmp
   DOWNLOAD_DIR ${DOWNLOAD_DIR}
   URL_HASH ${OPENMP_HASH_TYPE}=${OPENMP_HASH}
   PREFIX ${BUILD_DIR}/openmp
-  PATCH_COMMAND ${PATCH_CMD} -p 1 -d ${BUILD_DIR}/openmp/src/external_openmp < ${PATCH_DIR}/openmp.diff
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBDIR}/openmp ${DEFAULT_CMAKE_FLAGS}
-  INSTALL_COMMAND cd ${BUILD_DIR}/openmp/src/external_openmp-build && install_name_tool -id @executable_path/../Resources/lib/libomp.dylib runtime/src/libomp.dylib && make install
+  INSTALL_COMMAND cd ${BUILD_DIR}/openmp/src/external_openmp-build && install_name_tool -id @rpath/libomp.dylib runtime/src/libomp.dylib && make install
   INSTALL_DIR ${LIBDIR}/openmp
 )
 
diff --git a/build_files/build_environment/cmake/osl.cmake b/build_files/build_environment/cmake/osl.cmake
index 54e924b348a..5f300544e2b 100644
--- a/build_files/build_environment/cmake/osl.cmake
+++ b/build_files/build_environment/cmake/osl.cmake
@@ -51,6 +51,7 @@ set(OSL_EXTRA_ARGS
   -DOpenImageIO_ROOT=${LIBDIR}/openimageio/
   -DOSL_BUILD_TESTS=OFF
   -DOSL_BUILD_MATERIALX=OFF
+  -DPNG_ROOT=${LIBDIR}/png
   -DZLIB_LIBRARY=${LIBDIR}/zlib/lib/${ZLIB_LIBRARY}
   -DZLIB_INCLUDE_DIR=${LIBDIR}/zlib/include/
   ${OSL_FLEX_BISON}
@@ -69,13 +70,9 @@ set(OSL_EXTRA_ARGS
   ${OSL_SIMD_FLAGS}
   -Dpugixml_ROOT=${LIBDIR}/pugixml
   -DUSE_PYTHON=OFF
+  -DCMAKE_CXX_STANDARD=14
 )
 
-# Apple arm64 uses LLVM 11, LLVM 10+ requires C++14
-if (APPLE AND "${CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
-  list(APPEND OSL_EXTRA_ARGS -DCMAKE_CXX_STANDARD=14)
-endif()
-
 ExternalProject_Add(external_osl
   URL file://${PACKAGE_DIR}/${OSL_FILE}
   DOWNLOAD_DIR ${DOWNLOAD_DIR}
@@ -93,10 +90,20 @@ add_dependencies(
   ll
   external_openexr
   external_zlib
-  external_flexbison
   external_openimageio
   external_pugixml
 )
+if(WIN32)
+  add_dependencies(
+    external_osl
+    external_flexbison
+  )
+else()
+  add_dependencies(
+    external_osl
+    external_flex
+  )
+endif()
 
 if(WIN32)
   if(BUILD_MODE STREQUAL Release)
diff --git a/build_files/build_environment/cmake/versions.cmake b/build_files/build_environment/cmake/versions.cmake
index a71fc4b85e4..bc220c596c1 100644
--- a/build_files/build_environment/cmake/versions.cmake
+++ b/build_files/build_environment/cmake/versions.cmake
@@ -152,35 +152,20 @@ set(OPENCOLORIO_HASH 1a2e3478b6cd9a1549f24e1b2205e3f0)
 set(OPENCOLORIO_HASH_TYPE MD5)
 set(OPENCOLORIO_FILE OpenColorIO-${OPENCOLORIO_VERSION}.tar.gz)
 
-if(BLENDER_PLATFORM_ARM)
-  # Newer version required by ISPC with arm support.
-  set(LLVM_VERSION 11.0.1)
-  set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz)
-  set(LLVM_HASH e700af40ab83463e4e9ab0ba3708312e)
-  set(LLVM_HASH_TYPE MD5)
-  set(LLVM_FILE llvm-project-${LLVM_VERSION}.src.tar.xz)
-
-  set(OPENMP_VERSION 9.0.1)
-  set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${OPENMP_VERSION}/openmp-${OPENMP_VERSION}.src.tar.xz)
-  set(OPENMP_HASH 6eade16057edbdecb3c4eef9daa2bfcf)
-  set(OPENMP_HASH_TYPE MD5)
-  set(OPENMP_FILE openmp-${OPENMP_VERSION}.src.tar.xz)
-else()
-  set(LLVM_VERSION 9.0.1)
-  set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.tar.xz)
-  set(LLVM_HASH b4268e733dfe352960140dc07ef2efcb)
-  set(LLVM_HASH_TYPE MD5)
-  set(LLVM_FILE llvm-project-${LLVM_VERSION}.tar.xz)
-
-  set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/openmp-${LLVM_VERSION}.src.tar.xz)
-  set(OPENMP_HASH 6eade16057edbdecb3c4eef9daa2bfcf)
-  set(OPENMP_HASH_TYPE MD5)
-  set(OPENMP_FILE openmp-${LLVM_VERSION}.src.tar.xz)
-endif()
-
-set(OPENIMAGEIO_VERSION 2.1.15.0)
+set(LLVM_VERSION 12.0.0)
+set(LLVM_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz)
+set(LLVM_HASH 5a4fab4d7fc84aefffb118ac2c8a4fc0)
+set(LLVM_HASH_TYPE MD5)
+set(LLVM_FILE llvm-project-${LLVM_VERSION}.src.tar.xz)
+
+set(OPENMP_URI https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/openmp-${LLVM_VERSION}.src.tar.xz)
+set(OPENMP_HASH ac48ce3e4582ccb82f81ab59eb3fc9dc)
+set(OPENMP_HASH_TYPE MD5)
+set(OPENMP_FILE openmp-${LLVM_VERSION}.src.tar.xz)
+
+set(OPENIMAGEIO_VERSION 2.2.15.1)
 set(OPENIMAGEIO_URI https://github.com/OpenImageIO/oiio/archive/Release-${OPENIMAGEIO_VERSION}.tar.gz)
-set(OPENIMAGEIO_HASH f03aa5e3ac4795af04771ee4146e9832)
+set(OPENIMAGEIO_HASH 3db5c5f0b3dc91597c75e5df09eb9072)
 set(OPENIMAGEIO_HASH_TYPE MD5)
 set(OPENIMAGEIO_FILE OpenImageIO-${OPENIMAGEIO_VERSION}.tar.gz)
 
@@ -190,9 +175,9 @@ set(TIFF_HASH 2165e7aba557463acc0664e71a3ed424)
 set(TIFF_HASH_TYPE MD5)
 set(TIFF_FILE tiff-${TIFF_VERSION}.tar.gz)
 
-set(OSL_VERSION 1.11.10.0)
+set(OSL_VERSION 1.11.14.1)
 set(OSL_URI https://github.com/imageworks/OpenShadingLanguage/archive/Release-${OSL_VERSION}.tar.gz)
-set(OSL_HASH dfdc23597aeef083832cbada62211756)
+set(OSL_HASH 1abd7ce40481771a9fa937f19595d2f2)
 set(OSL_HASH_TYPE MD5)
 set(OSL_FILE OpenShadingLanguage-${OSL_VERSION}.tar.gz)
 
@@ -370,12 +355,18 @@ set(PUGIXML_HASH 0c208b0664c7fb822bf1b49ad035e8fd)
 set(PUGIXML_HASH_TYPE MD5)
 set(PUGIXML_FILE pugixml-${PUGIXML_VERSION}.tar.gz)
 
-set(FLEXBISON_VERSION 2.5.5)
+set(FLEXBISON_VERSION 2.5.24)
 set(FLEXBISON_URI http://prdownloads.sourceforge.net/winflexbison/win_flex_bison-${FLEXBISON_VERSION}.zip)
-set(FLEXBISON_HASH d87a3938194520d904013abef3df10ce)
+set(FLEXBISON_HASH 6b549d43e34ece0e8ed05af92daa31c4)
 set(FLEXBISON_HASH_TYPE MD5)
 set(FLEXBISON_FILE win_flex_bison-${FLEXBISON_VERSION}.zip)
 
+set(FLEX_VERSION 2.6.4)
+set(FLEX_URI https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz)
+set(FLEX_HASH 2882e3179748cc9f9c23ec593d6adc8d)
+set(FLEX_HASH_TYPE MD5)
+set(FLEX_FILE flex-${FLEX_VERSION}.tar.gz)
+
 # Libraries to keep Python modules static on Linux.
 
 # NOTE: bzip.org domain does no longer belong to BZip 2 project, so we download
@@ -432,9 +423,9 @@ set(USD_HASH 1dd1e2092d085ed393c1f7c450a4155a)
 set(USD_HASH_TYPE MD5)
 set(USD_FILE usd-v${USD_VERSION}.tar.gz)
 
-set(OIDN_VERSION 1.4.0)
+set(OIDN_VERSION 1.4.1)
 set(OIDN_URI https://github.com/OpenImageDenoise/oidn/releases/download/v${OIDN_VERSION}/oidn-${OIDN_VERSION}.src.tar.gz)
-set(OIDN_HASH 421824019becc5b664a22a2b98332bc5)
+set(OIDN_HASH df4007b0ab93b1c41cdf223b075d01c0)
 set(OIDN_HASH_TYPE MD5)
 set(OIDN_FILE oidn-${OIDN_VERSION}.src.tar.gz)
 
@@ -444,10 +435,10 @@ set(LIBGLU_HASH 151aef599b8259efe9acd599c96ea2a3)
 set(LIBGLU_HASH_TYPE MD5)
 set(LIBGLU_FILE glu-${LIBGLU_VERSION}.tar.xz)
 
-set(MESA_VERSION 20.3.4)
+set(MESA_VERSION 21.1.5)
 set(MESA_URI ftp://ftp.freedesktop.org/pub/mesa/mesa-${MESA_VERSION}.tar.xz)
-set(MESA_HASH 556338446aef8ae947a789b3e0b5e056)
-set(MESA_HASH_TYPE MD5)
+set(MESA_HASH 022c7293074aeeced2278c872db4fa693147c70f8595b076cf3f1ef81520766d)
+set(MESA_HASH_TYPE SHA256)
 set(MESA_FILE mesa-${MESA_VERSION}.tar.xz)
 
 set(NASM_VERSION 2.15.02)
@@ -468,19 +459,11 @@ set(WL_PROTOCOLS_URI https://gitlab.freedesktop.org/wayland/wayland-protocols/-/
 set(WL_PROTOCOLS_HASH af5ca07e13517cdbab33504492cef54a)
 set(WL_PROTOCOLS_HASH_TYPE MD5)
 
-if(BLENDER_PLATFORM_ARM)
-  # Unreleased version with macOS arm support.
-  set(ISPC_URI https://github.com/ispc/ispc/archive/f5949c055eb9eeb93696978a3da4bfb3a6a30b35.zip)
-  set(ISPC_HASH d382fea18d01dbd0cd05d9e1ede36d7d)
-  set(ISPC_HASH_TYPE MD5)
-  set(ISPC_FILE f5949c055eb9eeb93696978a3da4bfb3a6a30b35.zip)
-else()
-  set(ISPC_VERSION v1.14.1)
-  set(ISPC_URI https://github.com/ispc/ispc/archive/${ISPC_VERSION}.tar.gz)
-  set(ISPC_HASH 968fbc8dfd16a60ba4e32d2e0e03ea7a)
-  set(ISPC_HASH_TYPE MD5)
-  set(ISPC_FILE ispc-${ISPC_VERSION}.tar.gz)
-endif()
+set(ISPC_VERSION v1.16.0)
+set(ISPC_URI https://github.com/ispc/ispc/archive/${ISPC_VERSION}.tar.gz)
+set(ISPC_HASH 2e3abedbc0ea9aaec17d6562c632454d)
+set(ISPC_HASH_TYPE MD5)
+set(ISPC_FILE ispc-${ISPC_VERSION}.tar.gz)
 
 set(GMP_VERSION 6.2.0)
 set(GMP_URI https://gmplib.org/download/gmp/gmp-${GMP_VERSION}.tar.xz)