From e045fe53f1b03458cfb8ffb93246d9ad7737597c Mon Sep 17 00:00:00 2001 From: Sergey Sharybin Date: Thu, 27 Jun 2019 12:41:52 +0200 Subject: Buildbot: Port configuration to Blender2.7 branch --- build_files/buildbot/config/blender_linux.cmake | 145 ++---------------------- build_files/buildbot/slave_compile.py | 46 +++----- build_files/buildbot/slave_pack.py | 29 ++--- 3 files changed, 42 insertions(+), 178 deletions(-) diff --git a/build_files/buildbot/config/blender_linux.cmake b/build_files/buildbot/config/blender_linux.cmake index 9ba91a9a463..5c9327bcfbe 100644 --- a/build_files/buildbot/config/blender_linux.cmake +++ b/build_files/buildbot/config/blender_linux.cmake @@ -2,33 +2,22 @@ include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/config/blender_release.cmake") -# Detect which libc we'll be linking against. -# Some of the paths will depend on this - +# For libc-2.24 we are using chroot which runs on a 64bit system. +# There we can not use CPU bitness check since it is always 64bit. So instead +# we check for a specific libraries. +# +# Other builders we are runnign in a bare virtual machine, and the libraries +# are installed to /opt/. +# We assume that only 64bit builders exists in such configuration. if(EXISTS "/lib/x86_64-linux-gnu/libc-2.24.so") message(STATUS "Building in GLibc-2.24 environment") - set(GLIBC "2.24") - set(MULTILIB "/x86_64-linux-gnu") set(LIBDIR_NAME "linux_x86_64") elseif(EXISTS "/lib/i386-linux-gnu//libc-2.24.so") message(STATUS "Building in GLibc-2.24 environment") - set(GLIBC "2.24") - set(MULTILIB "/i386-linux-gnu") set(LIBDIR_NAME "linux_i686") -elseif(EXISTS "/lib/x86_64-linux-gnu/libc-2.19.so") - message(STATUS "Building in GLibc-2.19 environment") - set(GLIBC "2.19") - set(MULTILIB "/x86_64-linux-gnu") -elseif(EXISTS "/lib/i386-linux-gnu//libc-2.19.so") - message(STATUS "Building in GLibc-2.19 environment") - set(GLIBC "2.19") - set(MULTILIB "/i386-linux-gnu") -elseif(EXISTS "/lib/libc-2.11.3.so") - message(STATUS "Building in GLibc-2.11 environment") - set(GLIBC "2.11") - set(MULTILIB "") else() - message(FATAL_ERROR "Unknown build environment") + message(STATUS "Building in generic 64bit environment") + set(LIBDIR_NAME "linux_x86_64") endif() # Default to only build Blender, not the player @@ -52,122 +41,8 @@ set(WITH_PYTHON_INSTALL_REQUESTS ON CACHE BOOL "" FORCE) # ######## Release environment specific settings ######## -if (NOT ${GLIBC} STREQUAL "2.24") - -# All the hardcoded library paths and such - -# LLVM libraries -set(LLVM_VERSION "3.4" CACHE STRING "" FORCE) -set(LLVM_ROOT_DIR "/opt/lib/llvm-${LLVM_VERSION}" CACHE STRING "" FORCE) -set(LLVM_STATIC ON CACHE BOOL "" FORCE) - -# BOOST libraries -set(BOOST_ROOT "/opt/lib/boost" CACHE STRING "" FORCE) -set(Boost_USE_STATIC_LIBS ON CACHE BOOL "" FORCE) - -# FFmpeg libraries -set(FFMPEG "/opt/lib/ffmpeg" CACHE STRING "" FORCE) -set(FFMPEG_LIBRARIES - avdevice avformat avcodec avutil avfilter swscale swresample - /usr/lib${MULTILIB}/libxvidcore.a - /usr/lib${MULTILIB}/libx264.a - /usr/lib${MULTILIB}/libmp3lame.a - /usr/lib${MULTILIB}/libvpx.a - /usr/lib${MULTILIB}/libvorbis.a - /usr/lib${MULTILIB}/libogg.a - /usr/lib${MULTILIB}/libvorbisenc.a - /usr/lib${MULTILIB}/libtheora.a - /usr/lib${MULTILIB}/libschroedinger-1.0.a - /usr/lib${MULTILIB}/liborc-0.4.a - CACHE STRING "" FORCE -) - -# SndFile libraries -set(SNDFILE_LIBRARY "/usr/lib${MULTILIB}/libsndfile.a;/usr/lib${MULTILIB}/libFLAC.a" CACHE STRING "" FORCE) - -# OpenAL libraries -set(OPENAL_ROOT_DIR "/opt/lib/openal" CACHE STRING "" FORCE) -set(OPENAL_INCLUDE_DIR "${OPENAL_ROOT_DIR}/include/AL" CACHE STRING "" FORCE) -set(OPENAL_LIBRARY - ${OPENAL_ROOT_DIR}/lib/libopenal.a - ${OPENAL_ROOT_DIR}/lib/libcommon.a - CACHE STRING "" FORCE -) - -# OpenCollada libraries -set(OPENCOLLADA_UTF_LIBRARY "" CACHE STRING "" FORCE) -set(PCRE_INCLUDE_DIR "/usr/include" CACHE STRING "" FORCE) -set(PCRE_LIBRARY "/usr/lib${MULTILIB}/libpcre.a" CACHE STRING "" FORCE) -set(XML2_INCLUDE_DIR "/usr/include" CACHE STRING "" FORCE) -set(XML2_LIBRARY "/usr/lib${MULTILIB}/libxml2.a" CACHE STRING "" FORCE) - -# OpenColorIO libraries -set(OPENCOLORIO_ROOT_DIR "/opt/lib/ocio" CACHE STRING "" FORCE) -set(OPENCOLORIO_OPENCOLORIO_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libOpenColorIO.a" CACHE STRING "" FORCE) -set(OPENCOLORIO_TINYXML_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libtinyxml.a" CACHE STRING "" FORCE) -set(OPENCOLORIO_YAML-CPP_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libyaml-cpp.a" CACHE STRING "" FORCE) - -# Freetype -set(FREETYPE_INCLUDE_DIRS "/usr/include/freetype2" CACHE STRING "" FORCE) -set(FREETYPE_LIBRARY "/usr/lib${MULTILIB}/libfreetype.a" CACHE STRING "" FORCE) - -# OpenImageIO -if(GLIBC EQUAL "2.19") - set(OPENIMAGEIO_LIBRARY - /opt/lib/oiio/lib/libOpenImageIO.a - /opt/lib/oiio/lib/libOpenImageIO_Util.a - /usr/lib${MULTILIB}/libwebp.a - /usr/lib${MULTILIB}/liblzma.a - /usr/lib${MULTILIB}/libjbig.a - ${FREETYPE_LIBRARY} - CACHE STRING "" FORCE - ) -endif() - -# OpenSubdiv libraries -set(OPENSUBDIV_ROOT_DIR "/opt/lib/opensubdiv" CACHE STRING "" FORCE) -set(OPENSUBDIV_OSDCPU_LIBRARY "${OPENSUBDIV_ROOT_DIR}/lib/libosdCPU.a" CACHE STRING "" FORCE) -set(OPENSUBDIV_OSDGPU_LIBRARY "${OPENSUBDIV_ROOT_DIR}/lib/libosdGPU.a" CACHE STRING "" FORCE) - -# OpenEXR libraries -set(OPENEXR_ROOT_DIR "/opt/lib/openexr" CACHE STRING "" FORCE) -set(OPENEXR_HALF_LIBRARY "/opt/lib/openexr/lib/libHalf.a" CACHE STRING "" FORCE) -set(OPENEXR_IEX_LIBRARY "/opt/lib/openexr/lib/libIex.a" CACHE STRING "" FORCE) -set(OPENEXR_ILMIMF_LIBRARY "/opt/lib/openexr/lib/libIlmImf.a" CACHE STRING "" FORCE) -set(OPENEXR_ILMTHREAD_LIBRARY "/opt/lib/openexr/lib/libIlmThread.a" CACHE STRING "" FORCE) -set(OPENEXR_IMATH_LIBRARY "/opt/lib/openexr/lib/libImath.a" CACHE STRING "" FORCE) - -# JeMalloc library -set(JEMALLOC_LIBRARY "/opt/lib/jemalloc/lib/libjemalloc.a" CACHE STRING "" FORCE) - -# Space navigation -set(SPACENAV_ROOT_DIR "/opt/lib/libspnav" CACHE STRING "" FORCE) - -# Force some system libraries to be static -set(FFTW3_LIBRARY "/usr/lib${MULTILIB}/libfftw3.a" CACHE STRING "" FORCE) -set(JPEG_LIBRARY "/usr/lib${MULTILIB}/libjpeg.a" CACHE STRING "" FORCE) -set(PNG_LIBRARY "/usr/lib${MULTILIB}/libpng.a" CACHE STRING "" FORCE) -set(TIFF_LIBRARY "/usr/lib${MULTILIB}/libtiff.a" CACHE STRING "" FORCE) -set(ZLIB_LIBRARY "/usr/lib${MULTILIB}/libz.a" CACHE STRING "" FORCE) - -# OpenVDB -set(OPENVDB_LIBRARY - /opt/lib/openvdb/lib/libopenvdb.a - CACHE BOOL "" FORCE -) - -set(BLOSC_LIBRARY - /opt/lib/blosc/lib/libblosc.a - CACHE BOOL "" FORCE -) - -else() - set(LIBDIR "/opt/blender-deps/${LIBDIR_NAME}" CACHE BOOL "" FORCE) -# TODO(sergey): Remove once Python is oficially bumped to 3.7. -set(PYTHON_VERSION 3.7 CACHE BOOL "" FORCE) - # Platform specific configuration, to ensure static linking against everything. set(Boost_USE_STATIC_LIBS ON CACHE BOOL "" FORCE) @@ -178,7 +53,5 @@ set(Boost_USE_STATIC_LIBS ON CACHE BOOL "" FORCE) set(PCRE_INCLUDE_DIR "/usr/include" CACHE STRING "" FORCE) set(PCRE_LIBRARY "${LIBDIR}/opencollada/lib/libpcre.a" CACHE STRING "" FORCE) -endif() - # Additional linking libraries set(CMAKE_EXE_LINKER_FLAGS "-lrt -static-libstdc++ -no-pie" CACHE STRING "" FORCE) diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py index a339688c06a..685c769fb0c 100644 --- a/build_files/buildbot/slave_compile.py +++ b/build_files/buildbot/slave_compile.py @@ -52,9 +52,8 @@ if 'cmake' in builder: build_dir = os.path.abspath(os.path.join('..', 'build', builder)) install_dir = os.path.abspath(os.path.join('..', 'install', builder)) targets = ['blender'] + command_prefix = [] - chroot_name = None # If not None command will be delegated to that chroot - build_cubins = True # Whether to build Cycles CUDA kernels bits = 64 # Config file to be used (relative to blender's sources root) @@ -79,31 +78,26 @@ if 'cmake' in builder: cmake_options.extend(['-G', 'Visual Studio 15 2017']) elif builder.startswith('linux'): + cmake_config_file = "build_files/buildbot/config/blender_linux.cmake" + cmake_player_config_file = "build_files/buildbot/config/blender_linux_player.cmake" + targets = ['player', 'blender'] tokens = builder.split("_") glibc = tokens[1] if glibc == 'glibc224': deb_name = "stretch" - elif glibc == 'glibc219': - deb_name = "jessie" - elif glibc == 'glibc211': - deb_name = "squeeze" - cmake_config_file = "build_files/buildbot/config/blender_linux.cmake" - cmake_player_config_file = "build_files/buildbot/config/blender_linux_player.cmake" - if builder.endswith('x86_64_cmake'): - chroot_name = 'buildbot_' + deb_name + '_x86_64' - targets = ['player', 'blender'] - elif builder.endswith('i686_cmake'): - bits = 32 - chroot_name = 'buildbot_' + deb_name + '_i686' - targets = ['player', 'blender'] - if deb_name != "stretch": - cmake_extra_options.extend(["-DCMAKE_C_COMPILER=/usr/bin/gcc-7", - "-DCMAKE_CXX_COMPILER=/usr/bin/g++-7"]) + if builder.endswith('x86_64_cmake'): + chroot_name = 'buildbot_' + deb_name + '_x86_64' + elif builder.endswith('i686_cmake'): + bits = 32 + chroot_name = 'buildbot_' + deb_name + '_i686' + command_prefix = ['schroot', '-c', chroot_name, '--'] + elif glibc == 'glibc217': + command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file)) # Prepare CMake options needed to configure cuda binaries compilation, 64bit only. - if bits == 64 and build_cubins: + if bits == 64: cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=ON") cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON") else: @@ -113,12 +107,6 @@ if 'cmake' in builder: cmake_options += cmake_extra_options - # Prepare chroot command prefix if needed - if chroot_name: - chroot_prefix = ['schroot', '-c', chroot_name, '--'] - else: - chroot_prefix = [] - # Make sure no garbage remained from the previous run if os.path.isdir(install_dir): shutil.rmtree(install_dir) @@ -127,7 +115,7 @@ if 'cmake' in builder: print("Building target %s" % (target)) # Construct build directory name based on the target target_build_dir = build_dir - target_chroot_prefix = chroot_prefix[:] + target_command_prefix = command_prefix[:] if target != 'blender': target_build_dir += '_' + target target_name = 'install' @@ -141,7 +129,7 @@ if 'cmake' in builder: print("Fetching remotes") command = ['git', 'fetch', '--all'] print(command) - retcode = subprocess.call(target_chroot_prefix + command) + retcode = subprocess.call(target_command_prefix + command) if retcode != 0: sys.exit(retcode) # Make sure build directory exists and enter it @@ -160,7 +148,7 @@ if 'cmake' in builder: if os.path.exists(full_path): print("Removing {}" . format(buildinfo)) os.remove(full_path) - retcode = subprocess.call(target_chroot_prefix + ['cmake', blender_dir] + target_cmake_options) + retcode = subprocess.call(target_command_prefix + ['cmake', blender_dir] + target_cmake_options) if retcode != 0: print('Configuration FAILED!') sys.exit(retcode) @@ -172,7 +160,7 @@ if 'cmake' in builder: print("Executing command:") print(command) - retcode = subprocess.call(target_chroot_prefix + command) + retcode = subprocess.call(target_command_prefix + command) if retcode != 0: sys.exit(retcode) diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py index 27bebeb109e..b29b078ca85 100644 --- a/build_files/buildbot/slave_pack.py +++ b/build_files/buildbot/slave_pack.py @@ -126,7 +126,6 @@ if builder.find('cmake') != -1: elif builder.startswith('linux_'): blender = os.path.join(install_dir, 'blender') - blenderplayer = os.path.join(install_dir, 'blenderplayer') buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h") blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") @@ -136,24 +135,28 @@ if builder.find('cmake') != -1: blender_version = "%d.%d" % (blender_version // 100, blender_version % 100) blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1] blender_glibc = builder.split('_')[1] - - if builder.endswith('x86_64_cmake'): - chroot_name = 'buildbot_jessie_x86_64' - bits = 64 - blender_arch = 'x86_64' - elif builder.endswith('i686_cmake'): - chroot_name = 'buildbot_jessie_i686' - bits = 32 - blender_arch = 'i686' + command_prefix = [] + bits = 64 + blender_arch = 'x86_64' + + if blender_glibc == 'glibc224': + if builder.endswith('x86_64_cmake'): + chroot_name = 'buildbot_stretch_x86_64' + elif builder.endswith('i686_cmake'): + chroot_name = 'buildbot_stretch_i686' + bits = 32 + blender_arch = 'i686' + command_prefix = ['schroot', '-c', chroot_name, '--'] + elif blender_glibc == 'glibc217': + command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] # Strip all unused symbols from the binaries print("Stripping binaries...") - chroot_prefix = ['schroot', '-c', chroot_name, '--'] - subprocess.call(chroot_prefix + ['strip', '--strip-all', blender, blenderplayer]) + subprocess.call(command_prefix + ['strip', '--strip-all', blender]) print("Stripping python...") py_target = os.path.join(install_dir, blender_version) - subprocess.call(chroot_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) + subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) # Copy all specific files which are too specific to be copied by # the CMake rules themselves -- cgit v1.2.3