Buildbot: Initial work to move linux build environment to CMake

This is so called "seems to work in dry tests" commit which is aimed to switch
linux release environment to CMake.

Some notes:

- There's no special handle of libstdc++, but it wasn't really static for quite
  some time in SCons configuration and nobody really complained.

- It was quite tricky to get OpenMP linked statically with just using some
  configuration so we went ahead and added a special option to CMake now which is
  only exist on Linux and advertised as shouldn't be used.

- Packing is happening manually in slave_pack.py. This is because we have to add
  some really special files to the archive (mesa libraries for example) which we
  can't really handle from CMake/CPack in a nice generic way.

  Don't think it's bad approach, at least crappynness is localized and it's not
  _that_ crappy anyway.

- Windows buildbot should keep working, but needs doublechecing. It's just a
  build folder changed, but you never know what it might imply.

- Some further tweaks are likely needed to ensure all builders are working.

Thanks Campbell for assistance in this patch!

1 files changed, 109 insertions, 31 deletions

diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py
index aec7cdca80a..8c7e732216f 100644
--- a/build_files/buildbot/slave_pack.py
+++ b/build_files/buildbot/slave_pack.py
@@ -36,6 +36,24 @@ builder = sys.argv[1]
 # Never write branch if it is master.
 branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else ''
 
+upload_filename = None  # Name of the archive to be uploaded
+                        # (this is the name of archive which will appear on the
+                        # download page)
+upload_filepath = None  # Filepath to be uploaded to the server
+                        # (this folder will be packed)
+
+
+def parse_header_file(filename, define):
+    import re
+    regex = re.compile("^#\s*define\s+%s\s+(.*)" % define)
+    with open(filename, "r") as file:
+        for l in file:
+            match = regex.match(l)
+            if match:
+                return match.group(1)
+    return None
+
+
 # scons does own packaging
 if builder.find('scons') != -1:
     python_bin = 'python'
@@ -76,8 +94,8 @@ if builder.find('scons') != -1:
         blenderplayer = os.path.join(install_dir, 'blenderplayer')
         subprocess.call(['schroot', '-c', chroot_name, '--', 'strip', '--strip-all', blender, blenderplayer])
 
-        extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
-        mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
+        extra = "/home/sources/release-builder/extra/"
+        mesalibs = os.path.join(extra, 'mesalibs%d.tar.bz2' % bits)
         software_gl = os.path.join(extra, 'blender-softwaregl')
 
         os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
@@ -124,7 +142,7 @@ else:
         retcode = subprocess.call(['cpack', '-G', 'ZIP'])
         result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0]
 
-        # TODO(sergey): Such magic usually happens in SCon's packaging bu we don't have it
+        # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it
         # in the CMake yet. For until then we do some magic here.
         tokens = result_file.split('-')
         blender_version = tokens[1].split('.')
@@ -149,40 +167,100 @@ else:
             sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
             sys.exit(1)
 
+    elif builder.startswith('linux_'):
+        blender_dir = os.path.join('..', 'blender.git')
+        build_dir = os.path.join('..', 'build', builder)
+        install_dir = os.path.join('..', 'install', builder)
 
-# clean release directory if it already exists
-release_dir = 'release'
+        blender = os.path.join(install_dir, 'blender')
+        blenderplayer = os.path.join(install_dir, 'blenderplayer')
 
-if os.path.exists(release_dir):
-    for f in os.listdir(release_dir):
-        if os.path.isfile(os.path.join(release_dir, f)):
-            os.remove(os.path.join(release_dir, f))
+        buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h")
+        blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender.h")
 
-# create release package
-try:
-    subprocess.call(['make', 'package_archive'])
-except Exception as ex:
-    sys.stderr.write('Make package release failed' + str(ex) + '\n')
-    sys.exit(1)
+        if builder.endswith('x86_64_cmake'):
+            chroot_name = 'buildbot_squeeze_x86_64'
+            bits = 64
+            blender_arch = 'x86_64'
+        elif builder.endswith('i686_cmake'):
+            chroot_name = 'buildbot_squeeze_i686'
+            bits = 32
+            blender_arch = 'i686'
 
-# find release directory, must exist this time
-if not os.path.exists(release_dir):
-    sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
-    sys.exit(1)
+        # Strip all unused symbols from the binaries
+        print("Stripping binaries...")
+        chroot_prefix = ['schroot', '-c', chroot_name, '--']
+        subprocess.call(chroot_prefix + ['strip', '--strip-all', blender, blenderplayer])
 
-# find release package
-file = None
-filepath = None
+        # Copy all specific files which are too specific to be copied by
+        # the CMake rules themselves
+        print("Copying extra scripts and libs...")
 
-for f in os.listdir(release_dir):
-    rf = os.path.join(release_dir, f)
-    if os.path.isfile(rf) and f.startswith('blender'):
-        file = f
-        filepath = rf
+        extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra')
+        mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2')
+        software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl')
+        icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons')
 
-if not file:
-    sys.stderr.write("Failed to find release package.\n")
-    sys.exit(1)
+        os.system('tar -xpf %s -C %s' % (mesalibs, install_dir))
+        os.system('cp %s %s' % (software_gl, install_dir))
+        os.system('cp -r %s %s' % (icons, install_dir))
+        os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl')))
+
+        # Get version information for the archive name
+        blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION'))
+        blender_version = "%d.%d" % (blender_version / 100, blender_version % 100)
+        blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1]
+        blender_glibc = builder.split('_')[1]
+
+        upload_filename = 'blender-%s-%s-linux-%s-%s.tar.bz2' % (blender_version,
+                                                                 blender_hash,
+                                                                 blender_glibc,
+                                                                 blender_arch)
+        if branch != '':
+            upload_filename = branch + "-" + upload_filename
+
+        print("Creating .tar.bz2 archive")
+        os.system('tar -C../install -cjf %s.tar.bz2 %s' % (builder, builder))
+        upload_filepath = install_dir + '.tar.bz2'
+
+
+if upload_filepath is None:
+    # clean release directory if it already exists
+    release_dir = 'release'
+
+    if os.path.exists(release_dir):
+        for f in os.listdir(release_dir):
+            if os.path.isfile(os.path.join(release_dir, f)):
+                os.remove(os.path.join(release_dir, f))
+
+    # create release package
+    try:
+        subprocess.call(['make', 'package_archive'])
+    except Exception as ex:
+        sys.stderr.write('Make package release failed' + str(ex) + '\n')
+        sys.exit(1)
+
+    # find release directory, must exist this time
+    if not os.path.exists(release_dir):
+        sys.stderr.write("Failed to find release directory %r.\n" % release_dir)
+        sys.exit(1)
+
+    # find release package
+    file = None
+    filepath = None
+
+    for f in os.listdir(release_dir):
+        rf = os.path.join(release_dir, f)
+        if os.path.isfile(rf) and f.startswith('blender'):
+            file = f
+            filepath = rf
+
+    if not file:
+        sys.stderr.write("Failed to find release package.\n")
+        sys.exit(1)
+
+    upload_filename = file
+    upload_filepath = filepath
 
 # create zip file
 try:
@@ -190,7 +268,7 @@ try:
     if os.path.exists(upload_zip):
         os.remove(upload_zip)
     z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
-    z.write(filepath, arcname=file)
+    z.write(upload_filepath, arcname=upload_filename)
     z.close()
 except Exception as ex:
     sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')