Build: add system for shipping with dynamic libraries on Linux and macOS

PLATFORM_BUNDLED_LIBRARIES gathers shared libraries that will be installed
to the lib/ folder. The Blender executable gets a relative rpath pointing to
this folder as part of the install step.

The build rpath is different and uses absolute paths, so that it works for
executables like tests that are in different locations, and to support the
case where the build and install folders are different.

The system is already used for the OpenMP library on macOS. But on Linux it
will only kick in once we start using shared libraries for dependencies.

This also removes Mesa libraries from the old location, as these would cause
Blender to start with software OpenGL.

Ref T99618

5 files changed, 121 insertions, 51 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9f4c5e80b17..784a8ad555a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -731,6 +731,13 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   endif()
 endif()
 
+# Effective install path including config folder, as a generator expression.
+get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+if(GENERATOR_IS_MULTI_CONFIG)
+  string(REPLACE "\${BUILD_TYPE}" "$<CONFIG>" CMAKE_INSTALL_PREFIX_WITH_CONFIG ${CMAKE_INSTALL_PREFIX})
+else()
+  string(REPLACE "\${BUILD_TYPE}" "" CMAKE_INSTALL_PREFIX_WITH_CONFIG ${CMAKE_INSTALL_PREFIX})
+endif()
 
 
 # Apple
diff --git a/build_files/cmake/platform/platform_apple.cmake b/build_files/cmake/platform/platform_apple.cmake
index f39cb7a4951..bc5baf43530 100644
--- a/build_files/cmake/platform/platform_apple.cmake
+++ b/build_files/cmake/platform/platform_apple.cmake
@@ -21,6 +21,18 @@ function(print_found_status
   endif()
 endfunction()
 
+# Utility to install precompiled shared libraries.
+macro(add_bundled_libraries library)
+  if(EXISTS ${LIBDIR})
+    set(_library_dir ${LIBDIR}/${library}/lib)
+    file(GLOB _all_library_versions ${_library_dir}/*\.dylib*)
+    list(APPEND PLATFORM_BUNDLED_LIBRARIES ${_all_library_versions})
+    list(APPEND PLATFORM_BUNDLED_LIBRARY_DIRS ${_library_dir})
+    unset(_all_library_versions)
+    unset(_library_dir)
+ endif()
+endmacro()
+
 # ------------------------------------------------------------------------
 # Find system provided libraries.
 
@@ -415,6 +427,7 @@ if(WITH_OPENMP)
     set(OpenMP_LIBRARY_DIR "${LIBDIR}/openmp/lib/")
     set(OpenMP_LINKER_FLAGS "-L'${OpenMP_LIBRARY_DIR}' -lomp")
     set(OpenMP_LIBRARY "${OpenMP_LIBRARY_DIR}/libomp.dylib")
+    add_bundled_libraries(openmp)
   endif()
 endif()
 
@@ -504,17 +517,27 @@ if(WITH_COMPILER_CCACHE)
   endif()
 endif()
 
-# For binaries that are built but not installed (also not distributed) (datatoc,
-# makesdna, tests, etc.), we add an rpath to the OpenMP library dir through
-# CMAKE_BUILD_RPATH. This avoids having to make many copies of the dylib next to each binary.
-#
-# For the installed Python module and installed Blender executable, CMAKE_INSTALL_RPATH
-# is modified to find the dylib in an adjacent folder. Install step puts the libraries there.
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-list(APPEND CMAKE_BUILD_RPATH "${OpenMP_LIBRARY_DIR}")
+if(WITH_COMPILER_ASAN)
+  list(APPEND PLATFORM_BUNDLED_LIBRARIES ${COMPILER_ASAN_LIBRARY})
+endif()
 
-set(CMAKE_SKIP_INSTALL_RPATH FALSE)
-list(APPEND CMAKE_INSTALL_RPATH "@loader_path/../Resources/${BLENDER_VERSION}/lib")
+if(PLATFORM_BUNDLED_LIBRARIES)
+  # For the installed Python module and installed Blender executable, we set the
+  # rpath to the location where install step will copy the shared libraries.
+  set(CMAKE_SKIP_INSTALL_RPATH FALSE)
+  if(WITH_PYTHON_MODULE)
+    list(APPEND CMAKE_INSTALL_RPATH "@loader_path/lib")
+  else()
+    list(APPEND CMAKE_INSTALL_RPATH "@loader_path/../Resources/lib")
+  endif()
+
+  # For binaries that are built but not installed (like makesdan or tests), we add
+  # the original directory of all shared libraries to the rpath. This is needed because
+  # these can be in different folders, and because the build and install folder may be
+  # different.
+  set(CMAKE_SKIP_BUILD_RPATH FALSE)
+  list(APPEND CMAKE_BUILD_RPATH ${PLATFORM_BUNDLED_LIBRARY_DIRS})
+endif()
 
 # Same as `CFBundleIdentifier` in Info.plist.
 set(CMAKE_XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER "org.blenderfoundation.blender")
diff --git a/build_files/cmake/platform/platform_unix.cmake b/build_files/cmake/platform/platform_unix.cmake
index f18638dfa6b..080cbfd6a16 100644
--- a/build_files/cmake/platform/platform_unix.cmake
+++ b/build_files/cmake/platform/platform_unix.cmake
@@ -81,6 +81,15 @@ macro(find_package_wrapper)
   endif()
 endmacro()
 
+# Utility to install precompiled shared libraries.
+macro(add_bundled_libraries library)
+  if(EXISTS ${LIBDIR})
+    file(GLOB _all_library_versions ${LIBDIR}/${library}/lib/*\.so*)
+    list(APPEND PLATFORM_BUNDLED_LIBRARIES ${_all_library_versions})
+    unset(_all_library_versions)
+ endif()
+endmacro()
+
 # ----------------------------------------------------------------------------
 # Precompiled Libraries
 #
@@ -971,3 +980,16 @@ function(CONFIGURE_ATOMIC_LIB_IF_NEEDED)
 endfunction()
 
 CONFIGURE_ATOMIC_LIB_IF_NEEDED()
+
+if(PLATFORM_BUNDLED_LIBRARIES)
+  # For the installed Python module and installed Blender executable, we set the
+  # rpath to the relative path where the install step will copy the shared libraries.
+  set(CMAKE_SKIP_INSTALL_RPATH FALSE)
+  list(APPEND CMAKE_INSTALL_RPATH $ORIGIN/lib)
+
+  # For executables that are built but not installed (mainly tests) we set an absolute
+  # rpath to the lib folder. This is needed because these can be in different folders,
+  # and because the build and install folder may be different.
+  set(CMAKE_SKIP_BUILD_RPATH FALSE)
+  list(APPEND CMAKE_BUILD_RPATH $ORIGIN/lib ${CMAKE_INSTALL_PREFIX_WITH_CONFIG}/lib)
+endif()
diff --git a/source/creator/CMakeLists.txt b/source/creator/CMakeLists.txt
index 6d217b54488..0e9c3a853aa 100644
--- a/source/creator/CMakeLists.txt
+++ b/source/creator/CMakeLists.txt
@@ -308,14 +308,17 @@ if(UNIX AND NOT APPLE)
     if(WITH_INSTALL_PORTABLE)
       set(TARGETDIR_BPY .)
       set(TARGETDIR_VER ${BLENDER_VERSION})
+      set(TARGETDIR_LIB lib)
     else()
       set(TARGETDIR_BPY ${PYTHON_SITE_PACKAGES})
       set(TARGETDIR_VER ${PYTHON_SITE_PACKAGES}/${BLENDER_VERSION})
+      set(TARGETDIR_LIB ${PYTHON_SITE_PACKAGES}/lib)
     endif()
   else()
     if(WITH_INSTALL_PORTABLE)
       set(TARGETDIR_VER ${BLENDER_VERSION})
       set(TARGETDIR_TEXT .)
+      set(TARGETDIR_LIB lib)
     else()
       set(TARGETDIR_VER share/blender/${BLENDER_VERSION})
       set(TARGETDIR_TEXT share/doc/blender)
@@ -325,19 +328,23 @@ if(UNIX AND NOT APPLE)
 elseif(WIN32)
   set(TARGETDIR_VER ${BLENDER_VERSION})
   set(TARGETDIR_TEXT .)
+  set(TARGETDIR_LIB .)
 
 elseif(APPLE)
   if(WITH_PYTHON_MODULE)
     if(WITH_INSTALL_PORTABLE)
       set(TARGETDIR_VER $<TARGET_FILE_DIR:blender>/../Resources/${BLENDER_VERSION})
+      set(TARGETDIR_LIB lib)
     else()
       # Paths defined in terms of site-packages since the site-packages
       # directory can be a symlink (brew for example).
       set(TARGETDIR_BPY ${PYTHON_LIBPATH}/site-packages)
       set(TARGETDIR_VER ${TARGETDIR_BPY}/../Resources/${BLENDER_VERSION})
+      set(TARGETDIR_LIB ${TARGETDIR_BPY}/lib)
     endif()
   else()
     set(TARGETDIR_VER Blender.app/Contents/Resources/${BLENDER_VERSION})
+    set(TARGETDIR_LIB Blender.app/Contents/Resources/lib)
     set(TARGETDIR_TEXT Blender.app/Contents/Resources/text)
   endif()
 
@@ -474,6 +481,13 @@ if(UNIX AND NOT APPLE)
     endif()
   endif()
 
+  if(PLATFORM_BUNDLED_LIBRARIES AND TARGETDIR_LIB)
+    install(
+      FILES ${PLATFORM_BUNDLED_LIBRARIES}
+      DESTINATION ${TARGETDIR_LIB}
+    )
+  endif()
+
   # there are a few differences between portable and system install
   if(WITH_PYTHON_MODULE)
     if(WITH_INSTALL_PORTABLE)
@@ -523,6 +537,22 @@ if(UNIX AND NOT APPLE)
         ${CMAKE_SOURCE_DIR}/release/bin/blender-softwaregl
         DESTINATION "."
       )
+
+      # Remove from old location, so existing builds don't start with software
+      # OpenGL now that the lib/ folder is used for other libraries.
+      install(
+        CODE
+        "file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGL.so)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGL.so.1)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGL.so.1.5.0)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGLU.so)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGLU.so.1)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libGLU.so.1.3.1)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libglapi.so)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libglapi.so.0)\n
+         file(REMOVE ${CMAKE_BINARY_DIR}/bin/lib/libglapi.so.0.0.0)\n
+        "
+      )
     endif()
   else()
     # main blender binary
@@ -718,20 +748,20 @@ if(UNIX AND NOT APPLE)
 elseif(WIN32)
   install(
     FILES ${LIBDIR}/epoxy/bin/epoxy-0.dll
-    DESTINATION "."
+    DESTINATION ${TARGETDIR_LIB}
   )
 
   if(WITH_OPENMP AND MSVC_CLANG)
     install(
       FILES ${CLANG_OPENMP_DLL}
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
   if(WITH_FFTW3)
     install(
       FILES ${LIBDIR}/fftw3/lib/libfftw3-3.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
   if(MSVC_ASAN)
@@ -743,12 +773,12 @@ elseif(WIN32)
     endif()
     install(
       FILES ${ASAN_DLL}
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
     )
     install(
       FILES ${ASAN_DEBUG_DLL}
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Debug
     )
     unset(ASAN_DLL)
@@ -758,16 +788,16 @@ elseif(WIN32)
   if(WITH_GMP)
     install(
       FILES ${LIBDIR}/gmp/lib/libgmp-10.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
     install(
       FILES ${LIBDIR}/gmp/lib/libgmpxx.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
     )
     install(
       FILES ${LIBDIR}/gmp/lib/libgmpxx_d.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Debug
     )
   endif()
@@ -786,12 +816,12 @@ elseif(WIN32)
   if(WITH_OPENVDB)
     install(
       FILES ${LIBDIR}/openvdb/bin/openvdb.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
     )
     install(
       FILES ${LIBDIR}/openvdb/bin/openvdb_d.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Debug
     )
   endif()
@@ -803,14 +833,14 @@ elseif(WIN32)
       install(
         FILES ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/bin/python${_PYTHON_VERSION_NO_DOTS}.dll
               ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/bin/python3.dll
-        DESTINATION "."
+        DESTINATION ${TARGETDIR_LIB}
         CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
       )
 
       install(
         FILES ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/bin/python${_PYTHON_VERSION_NO_DOTS}_d.dll
               ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/bin/python3_d.dll
-        DESTINATION "."
+        DESTINATION ${TARGETDIR_LIB}
         CONFIGURATIONS Debug
       )
     endif()
@@ -872,13 +902,13 @@ elseif(WIN32)
       if(WINDOWS_PYTHON_DEBUG)
         install(
           FILES ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/libs/python${_PYTHON_VERSION_NO_DOTS}.pdb
-          DESTINATION "."
+          DESTINATION ${TARGETDIR_LIB}
           CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
         )
 
         install(
           FILES ${LIBDIR}/python/${_PYTHON_VERSION_NO_DOTS}/libs/python${_PYTHON_VERSION_NO_DOTS}_d.pdb
-          DESTINATION "."
+          DESTINATION ${TARGETDIR_LIB}
           CONFIGURATIONS Debug
         )
       endif()
@@ -900,7 +930,7 @@ elseif(WIN32)
           ${LIBDIR}/ffmpeg/lib/avutil-57.dll
           ${LIBDIR}/ffmpeg/lib/swscale-6.dll
           ${LIBDIR}/ffmpeg/lib/swresample-4.dll
-        DESTINATION "."
+        DESTINATION ${TARGETDIR_LIB}
       )
     else()
       install(
@@ -911,7 +941,7 @@ elseif(WIN32)
           ${LIBDIR}/ffmpeg/lib/avutil-56.dll
           ${LIBDIR}/ffmpeg/lib/swscale-5.dll
           ${LIBDIR}/ffmpeg/lib/swresample-3.dll
-        DESTINATION "."
+        DESTINATION ${TARGETDIR_LIB}
       )
     endif()
   endif()
@@ -919,13 +949,13 @@ elseif(WIN32)
     install(
       FILES
         ${LIBDIR}/tbb/bin/tbb.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
     )
     install(
       FILES
         ${LIBDIR}/tbb/bin/tbb_debug.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Debug
     )
   endif()
@@ -934,14 +964,14 @@ elseif(WIN32)
       FILES
         ${LIBDIR}/tbb/bin/tbbmalloc.dll
         ${LIBDIR}/tbb/bin/tbbmalloc_proxy.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Release;RelWithDebInfo;MinSizeRel
     )
     install(
       FILES
         ${LIBDIR}/tbb/bin/tbbmalloc_debug.dll
         ${LIBDIR}/tbb/bin/tbbmalloc_proxy_debug.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
       CONFIGURATIONS Debug
     )
     list(APPEND LIB ${TBB_MALLOC_LIBRARIES})
@@ -950,7 +980,7 @@ elseif(WIN32)
   if(WITH_CODEC_SNDFILE)
     install(
       FILES ${LIBDIR}/sndfile/lib/libsndfile-1.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
@@ -958,14 +988,14 @@ elseif(WIN32)
     install(
       FILES
         ${LIBDIR}/openal/lib/OpenAL32.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
   if(WITH_SDL)
     install(
       FILES ${LIBDIR}/sdl/lib/SDL2.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
@@ -975,7 +1005,7 @@ elseif(WIN32)
         ${LIBDIR}/audaspace/lib/audaspace.dll
         ${LIBDIR}/audaspace/lib/audaspace-c.dll
         ${LIBDIR}/audaspace/lib/audaspace-py.dll
-      DESTINATION "."
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
@@ -987,7 +1017,7 @@ elseif(WIN32)
       ${CMAKE_SOURCE_DIR}/release/windows/batch/blender_factory_startup.cmd
       ${CMAKE_SOURCE_DIR}/release/windows/batch/blender_oculus.cmd
       ${CMAKE_SOURCE_DIR}/release/windows/batch/oculus.json
-    DESTINATION "."
+    DESTINATION ${TARGETDIR_LIB}
   )
 
   if(WITH_BLENDER_THUMBNAILER)
@@ -1085,17 +1115,10 @@ elseif(APPLE)
       )
   endif()
 
-  if(WITH_OPENMP AND OPENMP_CUSTOM)
-    install(
-      FILES "${OpenMP_LIBRARY}"
-      DESTINATION "${MAC_BLENDER_TARGET_DYLIBS_DIR}"
-    )
-  endif()
-
-  if(WITH_COMPILER_ASAN)
+  if(PLATFORM_BUNDLED_LIBRARIES AND TARGETDIR_LIB)
     install(
-      FILES "${COMPILER_ASAN_LIBRARY}"
-      DESTINATION "${MAC_BLENDER_TARGET_DYLIBS_DIR}"
+      FILES ${PLATFORM_BUNDLED_LIBRARIES}
+      DESTINATION ${TARGETDIR_LIB}
     )
   endif()
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 484ffd17046..6d1c838ad6d 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -5,12 +5,7 @@
 #
 # Getting the install path of the executable is somewhat involved, as there are
 # no direct CMake generator expressions to get the install paths of executables.
-get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
-if(GENERATOR_IS_MULTI_CONFIG)
-  string(REPLACE "\${BUILD_TYPE}" "$<CONFIG>" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
-else()
-  string(REPLACE "\${BUILD_TYPE}" "" TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
-endif()
+set(TEST_INSTALL_DIR ${CMAKE_INSTALL_PREFIX_WITH_CONFIG})
 
 # Path to Blender and Python executables for all platforms.
 if(MSVC)