Buildbot: Initial work to move linux build environment to CMake

This is so called "seems to work in dry tests" commit which is aimed to switch
linux release environment to CMake.

Some notes:

- There's no special handle of libstdc++, but it wasn't really static for quite
  some time in SCons configuration and nobody really complained.

- It was quite tricky to get OpenMP linked statically with just using some
  configuration so we went ahead and added a special option to CMake now which is
  only exist on Linux and advertised as shouldn't be used.

- Packing is happening manually in slave_pack.py. This is because we have to add
  some really special files to the archive (mesa libraries for example) which we
  can't really handle from CMake/CPack in a nice generic way.

  Don't think it's bad approach, at least crappynness is localized and it's not
  _that_ crappy anyway.

- Windows buildbot should keep working, but needs doublechecing. It's just a
  build folder changed, but you never know what it might imply.

- Some further tweaks are likely needed to ensure all builders are working.

Thanks Campbell for assistance in this patch!

5 files changed, 337 insertions, 61 deletions

diff --git a/build_files/buildbot/config/blender_linux.cmake b/build_files/buildbot/config/blender_linux.cmake
new file mode 100644
index 00000000000..e88896628a8
--- /dev/null
+++ b/build_files/buildbot/config/blender_linux.cmake
@@ -0,0 +1,94 @@
+# ######## Global feature set settings ########
+
+include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/config/blender_full.cmake")
+
+# ######## Linux-specific build options ########
+# Options which are specific to Linux-only platforms
+set(WITH_DOC_MANPAGE         OFF CACHE BOOL "" FORCE)
+
+# ######## Official release-specific build options ########
+# Options which are specific to Linux release builds only
+set(WITH_JACK_DYNLOAD        ON  CACHE BOOL "" FORCE)
+set(WITH_SDL_DYNLOAD         ON  CACHE BOOL "" FORCE)
+set(WITH_SYSTEM_GLEW         OFF CACHE BOOL "" FORCE)
+
+set(WITH_OPENMP_STATIC       ON  CACHE BOOL "" FORCE)
+
+set(WITH_PYTHON_INSTALL_NUMPY    ON CACHE BOOL "" FORCE)
+set(WITH_PYTHON_INSTALL_REQUESTS ON CACHE BOOL "" FORCE)
+
+# ######## Release environment specific settings ########
+# All the hardcoded libraru paths and such
+
+# LLVM libraries
+set(LLVM_VERSION             "3.4"  CACHE STRING "" FORCE)
+set(LLVM_ROOT_DIR            "/opt/lib/llvm-${LLVM_VERSION}"  CACHE STRING "" FORCE)
+set(LLVM_STATIC              ON  CACHE BOOL "" FORCE)
+
+# BOOST libraries
+set(BOOST_ROOT               "/opt/lib/boost" CACHE STRING "" FORCE)
+set(Boost_USE_STATIC_LIBS    ON CACHE BOOL "" FORCE)
+
+# FFmpeg libraries
+set(FFMPEG                   "/opt/lib/ffmpeg" CACHE STRING "" FORCE)
+set(FFMPEG_LIBRARIES
+    avdevice avformat avcodec avutil avfilter swscale swresample
+    /usr/lib/libxvidcore.a
+    /usr/lib/libx264.a
+    /usr/lib/libmp3lame.a
+    /usr/lib/libvpx.a
+    /usr/lib/libvorbis.a
+    /usr/lib/libogg.a
+    /usr/lib/libvorbisenc.a
+    /usr/lib/libtheora.a
+    /usr/lib/libschroedinger-1.0.a
+    /usr/lib/liborc-0.4.a
+    CACHE STRING "" FORCE
+)
+
+# SndFile libraries
+set(SNDFILE_LIBRARY          "/usr/lib/libsndfile.a;/usr/lib/libFLAC.a" CACHE STRING "" FORCE)
+
+# OpenAL libraries
+set(OPENAL_ROOT_DIR           "/opt/lib/openal" CACHE STRING "" FORCE)
+set(OPENAL_INCLUDE_DIR        "${OPENAL_ROOT_DIR}/include" CACHE STRING "" FORCE)
+set(OPENAL_LIBRARY
+    ${OPENAL_ROOT_DIR}/lib/libopenal.a
+    ${OPENAL_ROOT_DIR}/lib/libcommon.a
+    CACHE STRING "" FORCE
+)
+
+# OpenCollada libraries
+set(OPENCOLLADA_UTF_LIBRARY   "" CACHE STRING "" FORCE)
+
+# OpenColorIO libraries
+set(OPENCOLORIO_ROOT_DIR      "/opt/lib/ocio" CACHE STRING "" FORCE)
+set(OPENCOLORIO_OPENCOLORIO_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libOpenColorIO.a" CACHE STRING "" FORCE)
+set(OPENCOLORIO_TINYXML_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libtinyxml.a"         CACHE STRING "" FORCE)
+set(OPENCOLORIO_YAML-CPP_LIBRARY "${OPENCOLORIO_ROOT_DIR}/lib/libyaml-cpp.a"       CACHE STRING "" FORCE)
+
+# OpenSubdiv libraries
+set(OPENSUBDIV_ROOT_DIR "/opt/lib/opensubdiv" CACHE STRING "" FORCE)
+set(OPENSUBDIV_OSDCPU_LIBRARY "${OPENSUBDIV_ROOT_DIR}/lib/libosdCPU.a" CACHE STRING "" FORCE)
+set(OPENSUBDIV_OSDGPU_LIBRARY "${OPENSUBDIV_ROOT_DIR}/lib/libosdGPU.a" CACHE STRING "" FORCE)
+
+# OpenEXR libraries
+set(OPENEXR_ROOT_DIR          "/opt/lib/openexr"                    CACHE STRING "" FORCE)
+set(OPENEXR_HALF_LIBRARY      "/opt/lib/openexr/lib/libHalf.a"      CACHE STRING "" FORCE)
+set(OPENEXR_IEX_LIBRARY       "/opt/lib/openexr/lib/libIex.a"       CACHE STRING "" FORCE)
+set(OPENEXR_ILMIMF_LIBRARY    "/opt/lib/openexr/lib/libIlmImf.a"    CACHE STRING "" FORCE)
+set(OPENEXR_ILMTHREAD_LIBRARY "/opt/lib/openexr/lib/libIlmThread.a" CACHE STRING "" FORCE)
+set(OPENEXR_IMATH_LIBRARY     "/opt/lib/openexr/lib/libImath.a"     CACHE STRING "" FORCE)
+
+# JeMalloc library
+set(JEMALLOC_LIBRARY    "/opt/lib/jemalloc/lib/libjemalloc.a" CACHE STRING "" FORCE)
+
+# Foce some system libraries to be static
+set(FFTW3_LIBRARY       "/usr/lib/libfftw3.a" CACHE STRING "" FORCE)
+set(JPEG_LIBRARY        "/usr/lib/libjpeg.a"  CACHE STRING "" FORCE)
+set(PNG_LIBRARY         "/usr/lib/libpng.a"   CACHE STRING "" FORCE)
+set(TIFF_LIBRARY        "/usr/lib/libtiff.a"  CACHE STRING "" FORCE)
+set(ZLIB_LIBRARY        "/usr/lib/libz.a"     CACHE STRING "" FORCE)
+
+# Additional linking libraries
+set(CMAKE_EXE_LINKER_FLAGS   "-lrt"  CACHE STRING "" FORCE)
diff --git a/build_files/buildbot/config/blender_player_linux.cmake b/build_files/buildbot/config/blender_player_linux.cmake
new file mode 100644
index 00000000000..2fb31192002
--- /dev/null
+++ b/build_files/buildbot/config/blender_player_linux.cmake
@@ -0,0 +1,12 @@
+# This is applied as an ovveride on top of blender_linux.config
+# Disables all the areas which are not needed for the player.
+set(WITH_COMPOSITOR          OFF CACHE BOOL "" FORCE)
+set(WITH_CYCLES              OFF CACHE BOOL "" FORCE)
+set(WITH_FREESTYLE           OFF CACHE BOOL "" FORCE)
+set(WITH_GHOST_XDND          OFF CACHE BOOL "" FORCE)
+set(WITH_OPENCOLLADA         OFF CACHE BOOL "" FORCE)
+set(WITH_OPENSUBDIV          OFF CACHE BOOL "" FORCE)
+set(WITH_LIBMV               OFF CACHE BOOL "" FORCE)
+
+set(WITH_BLENDER             OFF CACHE BOOL "" FORCE)
+set(WITH_PLAYER              ON  CACHE BOOL "" FORCE)
diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py
index ee89bc90225..d055822f69d 100644
--- a/build_files/buildbot/slave_compile.py
+++ b/build_files/buildbot/slave_compile.py
@@ -31,43 +31,121 @@ if len(sys.argv) < 2:
 builder = sys.argv[1]
 
 # we run from build/ directory
-blender_dir = '../blender.git'
+blender_dir = os.path.join('..', 'blender.git')
 
 if 'cmake' in builder:
     # cmake
 
-    # set build options
+    # Some fine-tuning configuration
+    blender_dir = os.path.join('..', blender_dir)
+    build_dir = os.path.abspath(os.path.join('..', 'build', builder))
+    install_dir = os.path.abspath(os.path.join('..', 'install', builder))
+    targets = ['blender']
+
+    chroot_name = None  # If not None command will be delegated to that chroot
+    build_cubins = True  # Whether to build Cycles CUDA kernels
+    remove_cache = False  # Remove CMake cache to be sure config is totally up-to-date
+    remove_install_dir = False  # Remove installation folder before building
+
+    # Config file to be used (relative to blender's sources root)
+    cmake_config_file = "build_files/cmake/config/blender_full.cmake"
+    cmake_player_config_file = None
+    cmake_cuda_config_file = None
+
+    # Set build options.
     cmake_options = ['-DCMAKE_BUILD_TYPE:STRING=Release']
 
-    if builder.endswith('mac_x86_64_cmake'):
-        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
-    elif builder.endswith('mac_i386_cmake'):
-        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386')
-    elif builder.endswith('mac_ppc_cmake'):
-        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc')
-
-    if 'win64' in builder:
-        cmake_options.append(['-G', '"Visual Studio 12 2013 Win64"'])
-    elif 'win32' in builder:
-        cmake_options.append(['-G', '"Visual Studio 12 2013"'])
-
-    cmake_options.append("-C../blender.git/build_files/cmake/config/blender_full.cmake")
-    if 'win32' not in builder:
-        cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=1")
-    else:
-        cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=0")
-    # configure and make
-    retcode = subprocess.call(['cmake', blender_dir] + cmake_options)
-    if retcode != 0:
-        sys.exit(retcode)
+    if builder.startswith('mac'):
+        # Set up OSX architecture
+        if builder.endswith('x86_64_cmake'):
+            cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
+        elif builder.endswith('i386_cmake'):
+            cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386')
+        elif builder.endswith('ppc_cmake'):
+            cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc')
+
+    elif builder.startswith('win'):
+        if builder.startwith('win64'):
+            cmake_options.append(['-G', '"Visual Studio 12 2013 Win64"'])
+        elif builder.startswith('win32'):
+            cmake_options.append(['-G', '"Visual Studio 12 2013"'])
+            build_cubins = False
+
+    elif builder.startswith('linux'):
+        remove_cache = True
+        remove_install_dir = True
+        cmake_config_file = "build_files/buildbot/config/blender_linux.cmake"
+        cmake_player_config_file = "build_files/buildbot/config/blender_player_linux.cmake"
+        cmake_cuda_config_file = "build_files/buildbot/config/blender_cuda_linux.cmake"
+        if builder.endswith('x86_64_cmake'):
+            chroot_name = 'buildbot_squeeze_x86_64'
+            build_cubins = False
+            targets = ['player', 'blender']
+        elif builder.endswith('i386_cmake'):
+            chroot_name = 'buildbot_squeeze_i686'
+            build_cubins = False
+            targets = ['player', 'blender']
+
+    cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=%d" % (build_cubins))
+
+    if install_dir:
+        cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir))
+
+    cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file))
+
+    # Prepare chroot command prefix if needed
 
-    if 'win32' in builder:
-        retcode = subprocess.call(['msbuild', 'INSTALL.vcxproj', '/Property:PlatformToolset=v120_xp', '/p:Configuration=Release'])
-    elif 'win64' in builder:
-        retcode = subprocess.call(['msbuild', 'INSTALL.vcxproj', '/p:Configuration=Release'])
+    if chroot_name:
+        chroot_prefix = ['schroot', '-c', chroot_name, '--']
     else:
-        retcode = subprocess.call(['make', '-s', '-j4', 'install'])
-    sys.exit(retcode)
+        chroot_prefix = []
+
+    # Make sure no garbage remained from the previous run
+    # (only do it if builder requested this)
+    if remove_install_dir:
+        if os.path.isdir(install_dir):
+            shutil.rmtree(install_dir)
+
+    for target in targets:
+        print("Building target %s" % (target))
+        # Construct build directory name based on the target
+        target_build_dir = build_dir
+        if target != 'blender':
+            target_build_dir += '_' + target
+        # Make sure build directory exists and enter it
+        if not os.path.isdir(target_build_dir):
+            os.mkdir(target_build_dir)
+        os.chdir(target_build_dir)
+        # Tweaking CMake options to respect the target
+        target_cmake_options = cmake_options[:]
+        if target == 'player':
+            target_cmake_options.append("-C" + os.path.join(blender_dir, cmake_player_config_file))
+        elif target == 'cuda':
+            target_cmake_options.append("-C" + os.path.join(blender_dir, cmake_cuda_config_file))
+        # Configure the build
+        print("CMake options:")
+        print(target_cmake_options)
+        if remove_cache and os.path.exists('CMakeCache.txt'):
+            print("Removing CMake cache")
+            os.remove('CMakeCache.txt')
+        retcode = subprocess.call(chroot_prefix + ['cmake', blender_dir] + target_cmake_options)
+        if retcode != 0:
+            print('Condifuration FAILED!')
+            sys.exit(retcode)
+
+        if 'win32' in builder:
+            command = ['msbuild', 'INSTALL.vcxproj', '/Property:PlatformToolset=v120_xp', '/p:Configuration=Release']
+        elif 'win64' in builder:
+            command = ['msbuild', 'INSTALL.vcxproj', '/p:Configuration=Release']
+        else:
+            command = chroot_prefix + ['make', '-s', '-j2', 'install']
+
+        print("Executing command:")
+        print(command)
+        retcode = subprocess.call(command)
+
+        if retcode != 0:
+            sys.exit(retcode)
 else:
     python_bin = 'python'
     if builder.find('linux') != -1:
diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py
index aec7cdca80a..8c7e732216f 100644
--- a/build_files/buildbot/slave_pack.py
+++ b/build_files/buildbot/slave_pack.py
@@ -36,6 +36,24 @@ builder = sys.argv[1]
 # Never write branch if it is master.
 branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
 
+upload_filename = None  # Name of the archive to be uploaded
+                        # (this is the name of archive which will appear on the
+                        # download page)
+upload_filepath = None  # Filepath to be uploaded to the server
+                        # (this folder will be packed)
+
+
+def parse_header_file(filename, define):
+    import re
+    regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
+    with open(filename, "r") as file:
+        for l in file:
+            match = regex.match(l)
+            if match:
+                return match.group(1)
+    return None
+
+
 # scons does own packaging
 if builder.find('scons') != -1:
     python_bin = 'python'
@@ -76,8 +94,8 @@ if builder.find('scons') != -1:
         blenderplayer = os.path.join(install_dir, 'blenderplayer')
         subprocess.call(['schroot', '-c', chroot_name, '--', 'strip', '--strip-all', blender, blenderplayer])
 
-        extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
-        mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
+        extra = "/home/sources/release-builder/extra/"
+        mesalibs = os.path.join(extra, 'mesalibs%d.tar.bz2' % bits)
         software_gl = os.path.join(extra, 'blender-softwaregl')
 
         os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
@@ -124,7 +142,7 @@ else:
         retcode = subprocess.call(['cpack', '-G', 'ZIP'])
         result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
 
-        # TODO(sergey): Such magic usually happens in SCon's packaging bu we don't have it
+        # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
         # in the CMake yet. For until then we do some magic here.
         tokens = result_file.split('-')
         blender_version = tokens[1].split('.')
@@ -149,40 +167,100 @@ else:
             sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
             sys.exit(1)
 
+    elif builder.startswith('linux_'):
+        blender_dir = os.path.join('..', 'blender.git')
+        build_dir = os.path.join('..', 'build', builder)
+        install_dir = os.path.join('..', 'install', builder)
 
-# clean release directory if it already exists
-release_dir = 'release'
+        blender = os.path.join(install_dir, 'blender')
+        blenderplayer = os.path.join(install_dir, 'blenderplayer')
 
-if os.path.exists(release_dir):
-    for f in os.listdir(release_dir):
-        if os.path.isfile(os.path.join(release_dir, f)):
-            os.remove(os.path.join(release_dir, f))
+        buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
+        blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender.h")
 
-# create release package
-try:
-    subprocess.call(['make', 'package_archive'])
-except Exception as ex:
-    sys.stderr.write('Make package release failed' + str(ex) + '\n')
-    sys.exit(1)
+        if builder.endswith('x86_64_cmake'):
+            chroot_name = 'buildbot_squeeze_x86_64'
+            bits = 64
+            blender_arch = 'x86_64'
+        elif builder.endswith('i686_cmake'):
+            chroot_name = 'buildbot_squeeze_i686'
+            bits = 32
+            blender_arch = 'i686'
 
-# find release directory, must exist this time
-if not os.path.exists(release_dir):
-    sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
-    sys.exit(1)
+        # Strip all unused symbols from the binaries
+        print("Stripping binaries...")
+        chroot_prefix = ['schroot', '-c', chroot_name, '--']
+        subprocess.call(chroot_prefix + ['strip', '--strip-all', blender, blenderplayer])
 
-# find release package
-file = None
-filepath = None
+        # Copy all specific files which are too specific to be copied by
+        # the CMake rules themselves
+        print("Copying extra scripts and libs...")
 
-for f in os.listdir(release_dir):
-    rf = os.path.join(release_dir, f)
-    if os.path.isfile(rf) and f.startswith('blender'):
-        file = f
-        filepath = rf
+        extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
+        mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
+        software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
+        icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
 
-if not file:
-    sys.stderr.write("Failed to find release package.\n")
-    sys.exit(1)
+        os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
+        os.system('cp %s %s' % (software_gl, install_dir))
+        os.system('cp -r %s %s' % (icons, install_dir))
+        os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
+
+        # Get version information for the archive name
+        blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
+        blender_version = "%d.%d" % (blender_version / 100, blender_version % 100)
+        blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
+        blender_glibc = builder.split('_')[1]
+
+        upload_filename = 'blender-%s-%s-linux-%s-%s.tar.bz2' % (blender_version,
+                                                                 blender_hash,
+                                                                 blender_glibc,
+                                                                 blender_arch)
+        if branch != '':
+            upload_filename = branch + "-" + upload_filename
+
+        print("Creating .tar.bz2 archive")
+        os.system('tar -C../install -cjf %s.tar.bz2 %s' % (builder, builder))
+        upload_filepath = install_dir + '.tar.bz2'
+
+
+if upload_filepath is None:
+    # clean release directory if it already exists
+    release_dir = 'release'
+
+    if os.path.exists(release_dir):
+        for f in os.listdir(release_dir):
+            if os.path.isfile(os.path.join(release_dir, f)):
+                os.remove(os.path.join(release_dir, f))
+
+    # create release package
+    try:
+        subprocess.call(['make', 'package_archive'])
+    except Exception as ex:
+        sys.stderr.write('Make package release failed' + str(ex) + '\n')
+        sys.exit(1)
+
+    # find release directory, must exist this time
+    if not os.path.exists(release_dir):
+        sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
+        sys.exit(1)
+
+    # find release package
+    file = None
+    filepath = None
+
+    for f in os.listdir(release_dir):
+        rf = os.path.join(release_dir, f)
+        if os.path.isfile(rf) and f.startswith('blender'):
+            file = f
+            filepath = rf
+
+    if not file:
+        sys.stderr.write("Failed to find release package.\n")
+        sys.exit(1)
+
+    upload_filename = file
+    upload_filepath = filepath
 
 # create zip file
 try:
@@ -190,7 +268,7 @@ try:
     if os.path.exists(upload_zip):
         os.remove(upload_zip)
     z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
-    z.write(filepath, arcname=file)
+    z.write(upload_filepath, arcname=upload_filename)
     z.close()
 except Exception as ex:
     sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
diff --git a/build_files/buildbot/slave_test.py b/build_files/buildbot/slave_test.py
index ef7a4b23d04..3932c0f992b 100644
--- a/build_files/buildbot/slave_test.py
+++ b/build_files/buildbot/slave_test.py
@@ -19,6 +19,7 @@
 # <pep8 compliant>
 
 import subprocess
+import os
 import sys
 
 # get builder name
@@ -33,7 +34,20 @@ blender_dir = '../blender.git'
 
 if "cmake" in builder:
     # cmake
-    retcode = subprocess.call(['ctest', '.' '--output-on-failure'])
+
+    build_dir = os.path.abspath(os.path.join('..', 'build', builder))
+    chroot_name = None
+    chroot_prefix = []
+
+    if builder.endswith('x86_64_cmake'):
+        chroot_name = 'buildbot_squeeze_x86_64'
+    elif builder.endswith('i686_cmake'):
+        chroot_name = 'buildbot_squeeze_i686'
+    if chroot_name:
+        chroot_prefix = ['schroot', '-c', chroot_name, '--']
+
+    os.chdir(build_dir)
+    retcode = subprocess.call(chroot_prefix + ['ctest', '--output-on-failure'])
     sys.exit(retcode)
 else:
     # scons