diff options
Diffstat (limited to 'build_files/buildbot')
| -rw-r--r-- | build_files/buildbot/buildbot_utils.py | 112 | ||||
| -rw-r--r-- | build_files/buildbot/slave_compile.py | 209 | ||||
| -rw-r--r-- | build_files/buildbot/slave_pack.py | 320 | ||||
| -rw-r--r-- | build_files/buildbot/slave_rsync.py | 24 | ||||
| -rw-r--r-- | build_files/buildbot/slave_test.py | 66 | ||||
| -rw-r--r-- | build_files/buildbot/slave_update.py | 16 |
6 files changed, 355 insertions, 392 deletions
diff --git a/build_files/buildbot/buildbot_utils.py b/build_files/buildbot/buildbot_utils.py new file mode 100644 index 00000000000..6891b91aa1e --- /dev/null +++ b/build_files/buildbot/buildbot_utils.py @@ -0,0 +1,112 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +import argparse +import os +import subprocess +import sys + +class Builder: + def __init__(self, name, branch): + self.name = name + self.branch = branch + + # Buildbot runs from build/ directory + self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git')) + self.build_dir = os.path.abspath(os.path.join('..', 'build', name)) + self.install_dir = os.path.abspath(os.path.join('..', 'install', name)) + self.upload_dir = os.path.abspath(os.path.join('..', 'install')) + + # Detect platform + if name.startswith('mac'): + self.platform = 'mac' + self.command_prefix = [] + elif name.startswith('linux'): + self.platform = 'linux' + self.command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] + elif name.startswith('win'): + self.platform = 'win' + self.command_prefix = [] + else: + raise ValueError('Unkonw platform for builder ' + self.platform) + + # Always 64 bit now + self.bits = 64 + +def create_builder_from_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument('builder_name') + parser.add_argument('branch', default='master', nargs='?') + args = parser.parse_args() + return Builder(args.builder_name, args.branch) + + +class VersionInfo: + def __init__(self, builder): + # Get version information + buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h") + blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") + + version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION')) + self.version = "%d.%d" % (version_number // 100, version_number % 100) + self.version_char = self._parse_header_file(blender_h, 'BLENDER_VERSION_CHAR') + self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE') + self.version_cycle_number = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE_NUMBER') + self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1] + + if self.version_cycle == "release": + # Final release + self.full_version = self.version + self.version_char + self.is_development_build = False + elif self.version_cycle == "rc": + # Release candidate + version_cycle = self.version_cycle + self.version_cycle_number + if len(self.version_char) == 0: + self.full_version = self.version + version_cycle + else: + self.full_version = self.version + self.version_char + '-' + version_cycle + self.is_development_build = False + else: + # Development build + self.full_version = self.version + '-' + self.hash + self.is_development_build = True + + def _parse_header_file(self, filename, define): + import re + regex = re.compile("^#\s*define\s+%s\s+(.*)" % define) + with open(filename, "r") as file: + for l in file: + match = regex.match(l) + if match: + return match.group(1) + return None + + +def call(cmd, env=None, exit_on_error=True): + print(' '.join(cmd)) + + # Flush to ensure correct order output on Windows. + sys.stdout.flush() + sys.stderr.flush() + + retcode = subprocess.call(cmd, env=env) + if exit_on_error and retcode != 0: + sys.exit(retcode) + return retcode diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py index 55543055e5b..84667e663f6 100644 --- a/build_files/buildbot/slave_compile.py +++ b/build_files/buildbot/slave_compile.py @@ -18,149 +18,80 @@ # <pep8 compliant> +import buildbot_utils import os -import subprocess -import sys import shutil -# get builder name -if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting builder name\n") - sys.exit(1) - -builder = sys.argv[1] - -# we run from build/ directory -blender_dir = os.path.join('..', 'blender.git') - - -def parse_header_file(filename, define): - import re - regex = re.compile("^#\s*define\s+%s\s+(.*)" % define) - with open(filename, "r") as file: - for l in file: - match = regex.match(l) - if match: - return match.group(1) - return None - -if 'cmake' in builder: - # cmake - - # Some fine-tuning configuration - blender_dir = os.path.abspath(blender_dir) - build_dir = os.path.abspath(os.path.join('..', 'build', builder)) - install_dir = os.path.abspath(os.path.join('..', 'install', builder)) - targets = ['blender'] - command_prefix = [] - - bits = 64 - - # Config file to be used (relative to blender's sources root) - cmake_config_file = "build_files/cmake/config/blender_release.cmake" - - # Set build options. - cmake_options = [] - cmake_extra_options = ['-DCMAKE_BUILD_TYPE:STRING=Release', - '-DWITH_GTESTS=ON'] - - if builder.startswith('mac'): - # Set up OSX architecture - if builder.endswith('x86_64_10_9_cmake'): - cmake_extra_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') - cmake_extra_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') - - elif builder.startswith('win'): - if builder.startswith('win64'): - cmake_options.extend(['-G', 'Visual Studio 15 2017 Win64']) - elif builder.startswith('win32'): - bits = 32 - cmake_options.extend(['-G', 'Visual Studio 15 2017']) - - elif builder.startswith('linux'): - cmake_config_file = "build_files/buildbot/config/blender_linux.cmake" - tokens = builder.split("_") - glibc = tokens[1] - if glibc == 'glibc224': - deb_name = "stretch" - if builder.endswith('x86_64_cmake'): - chroot_name = 'buildbot_' + deb_name + '_x86_64' - elif builder.endswith('i686_cmake'): - bits = 32 - chroot_name = 'buildbot_' + deb_name + '_i686' - command_prefix = ['schroot', '-c', chroot_name, '--'] - elif glibc == 'glibc217': - command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] - - cmake_options.append("-C" + os.path.join(blender_dir, cmake_config_file)) - - # Prepare CMake options needed to configure cuda binaries compilation, 64bit only. - if bits == 64: - cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=ON") - cmake_options.append("-DCUDA_64_BIT_DEVICE_CODE=ON") - else: - cmake_options.append("-DWITH_CYCLES_CUDA_BINARIES=OFF") +def get_cmake_options(builder): + config_file = "build_files/cmake/config/blender_release.cmake" + options = ['-DCMAKE_BUILD_TYPE:STRING=Release', '-DWITH_GTESTS=ON'] + + if builder.platform == 'mac': + options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') + options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') + elif builder.platform == 'win': + options.extend(['-G', 'Visual Studio 15 2017 Win64']) + elif builder.platform == 'linux': + config_file = "build_files/buildbot/config/blender_linux.cmake" + + options.append("-C" + os.path.join(builder.blender_dir, config_file)) + options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir)) + + return options - cmake_options.append("-DCMAKE_INSTALL_PREFIX=%s" % (install_dir)) +def update_git(builder): + # Do extra git fetch because not all platform/git/buildbot combinations + # update the origin remote, causing buildinfo to detect local changes. + os.chdir(builder.blender_dir) - cmake_options += cmake_extra_options + print("Fetching remotes") + command = ['git', 'fetch', '--all'] + buildbot_utils.call(builder.command_prefix + command) +def clean_directories(builder): # Make sure no garbage remained from the previous run - if os.path.isdir(install_dir): - shutil.rmtree(install_dir) - - for target in targets: - print("Building target %s" % (target)) - # Construct build directory name based on the target - target_build_dir = build_dir - target_command_prefix = command_prefix[:] - if target != 'blender': - target_build_dir += '_' + target - target_name = 'install' - # Tweaking CMake options to respect the target - target_cmake_options = cmake_options[:] - # Do extra git fetch because not all platform/git/buildbot combinations - # update the origin remote, causing buildinfo to detect local changes. - os.chdir(blender_dir) - print("Fetching remotes") - command = ['git', 'fetch', '--all'] - print(command) - retcode = subprocess.call(target_command_prefix + command) - if retcode != 0: - sys.exit(retcode) - # Make sure build directory exists and enter it - if not os.path.isdir(target_build_dir): - os.mkdir(target_build_dir) - os.chdir(target_build_dir) - # Configure the build - print("CMake options:") - print(target_cmake_options) - if os.path.exists('CMakeCache.txt'): - print("Removing CMake cache") - os.remove('CMakeCache.txt') - # Remove buildinfo files to force buildbot to re-generate them. - for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): - full_path = os.path.join('source', 'creator', buildinfo) - if os.path.exists(full_path): - print("Removing {}" . format(buildinfo)) - os.remove(full_path) - retcode = subprocess.call(target_command_prefix + ['cmake', blender_dir] + target_cmake_options) - if retcode != 0: - print('Configuration FAILED!') - sys.exit(retcode) - - if 'win32' in builder or 'win64' in builder: - command = ['cmake', '--build', '.', '--target', target_name, '--config', 'Release'] - else: - command = ['make', '-s', '-j2', target_name] - - print("Executing command:") - print(command) - retcode = subprocess.call(target_command_prefix + command) - - if retcode != 0: - sys.exit(retcode) - -else: - print("Unknown building system") - sys.exit(1) + if os.path.isdir(builder.install_dir): + shutil.rmtree(builder.install_dir) + + # Make sure build directory exists and enter it + os.makedirs(builder.build_dir, exist_ok=True) + + # Remove buildinfo files to force buildbot to re-generate them. + for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): + full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo) + if os.path.exists(full_path): + print("Removing {}" . format(buildinfo)) + os.remove(full_path) + +def cmake_configure(builder): + # CMake configuration + os.chdir(builder.build_dir) + + cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt') + if os.path.exists(cmake_cache): + print("Removing CMake cache") + os.remove(cmake_cache) + + print("CMake configure:") + cmake_options = get_cmake_options(builder) + command = ['cmake', builder.blender_dir] + cmake_options + buildbot_utils.call(builder.command_prefix + command) + +def cmake_build(builder): + # CMake build + os.chdir(builder.build_dir) + + if builder.platform == 'win': + command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] + else: + command = ['make', '-s', '-j2', 'install'] + + print("CMake build:") + buildbot_utils.call(builder.command_prefix + command) + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + update_git(builder) + clean_directories(builder) + cmake_configure(builder) + cmake_build(builder) diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py index b29b078ca85..45331cce61f 100644 --- a/build_files/buildbot/slave_pack.py +++ b/build_files/buildbot/slave_pack.py @@ -22,47 +22,36 @@ # system and zipping it into buildbot_upload.zip. This is then uploaded # to the master in the next buildbot step. +import buildbot_utils import os -import subprocess import sys -import zipfile -# get builder name -if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting builder name\n") - sys.exit(1) +def get_package_name(builder, platform=None): + info = buildbot_utils.VersionInfo(builder) -builder = sys.argv[1] -# Never write branch if it is master. -branch = sys.argv[2] if (len(sys.argv) >= 3 and sys.argv[2] != 'master') else '' + package_name = 'blender-' + info.full_version + if platform: + package_name += '-' + platform + if builder.branch != 'master' and info.is_development_build: + package_name = builder.branch + "-" + package_name -blender_dir = os.path.join('..', 'blender.git') -build_dir = os.path.join('..', 'build', builder) -install_dir = os.path.join('..', 'install', builder) -buildbot_upload_zip = os.path.abspath(os.path.join(os.path.dirname(install_dir), "buildbot_upload.zip")) + return package_name -upload_filename = None # Name of the archive to be uploaded - # (this is the name of archive which will appear on the - # download page) -upload_filepath = None # Filepath to be uploaded to the server - # (this folder will be packed) +def create_buildbot_upload_zip(builder, package_files): + import zipfile + buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip") + if os.path.exists(buildbot_upload_zip): + os.remove(buildbot_upload_zip) -def parse_header_file(filename, define): - import re - regex = re.compile("^#\s*define\s+%s\s+(.*)" % define) - with open(filename, "r") as file: - for l in file: - match = regex.match(l) - if match: - return match.group(1) - return None - - -# Make sure install directory always exists -if not os.path.exists(install_dir): - os.makedirs(install_dir) - + try: + z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) + for filepath, filename in package_files: + z.write(filepath, arcname=filename) + z.close() + except Exception as ex: + sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n') + sys.exit(1) def create_tar_bz2(src, dest, package_name): # One extra to remove leading os.sep when cleaning root for package_root @@ -80,163 +69,120 @@ def create_tar_bz2(src, dest, package_name): package.add(entry[0], entry[1], recursive=False) package.close() +def cleanup_files(dirpath, extension): + for f in os.listdir(dirpath): + filepath = os.path.join(dirpath, f) + if os.path.isfile(filepath) and f.endswith(extension): + os.remove(filepath) -if builder.find('cmake') != -1: - # CMake - if 'win' in builder or 'mac' in builder: - os.chdir(build_dir) - - files = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')] - for f in files: - os.remove(f) - retcode = subprocess.call(['cpack', '-G', 'ZIP']) - result_file = [f for f in os.listdir('.') if os.path.isfile(f) and f.endswith('.zip')][0] - - # TODO(sergey): Such magic usually happens in SCon's packaging but we don't have it - # in the CMake yet. For until then we do some magic here. - tokens = result_file.split('-') - blender_version = tokens[1].split('.') - blender_full_version = '.'.join(blender_version[0:2]) - git_hash = tokens[2].split('.')[1] - platform = builder.split('_')[0] - if platform == 'mac': - # Special exception for OSX - platform = 'OSX-10.9-' - if builder.endswith('x86_64_10_9_cmake'): - platform += 'x86_64' - if builder.endswith('vc2015'): - platform += "-vc14" - builderified_name = 'blender-{}-{}-{}'.format(blender_full_version, git_hash, platform) - # NOTE: Blender 2.7 is already respected by blender_full_version. - if branch != '' and branch != 'blender2.7': - builderified_name = branch + "-" + builderified_name - - os.rename(result_file, "{}.zip".format(builderified_name)) - # create zip file - try: - if os.path.exists(buildbot_upload_zip): - os.remove(buildbot_upload_zip) - z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) - z.write("{}.zip".format(builderified_name)) - z.close() - sys.exit(retcode) - except Exception as ex: - sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n') - sys.exit(1) - - elif builder.startswith('linux_'): - blender = os.path.join(install_dir, 'blender') - - buildinfo_h = os.path.join(build_dir, "source", "creator", "buildinfo.h") - blender_h = os.path.join(blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") - - # Get version information - blender_version = int(parse_header_file(blender_h, 'BLENDER_VERSION')) - blender_version = "%d.%d" % (blender_version // 100, blender_version % 100) - blender_hash = parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1] - blender_glibc = builder.split('_')[1] - command_prefix = [] - bits = 64 - blender_arch = 'x86_64' - - if blender_glibc == 'glibc224': - if builder.endswith('x86_64_cmake'): - chroot_name = 'buildbot_stretch_x86_64' - elif builder.endswith('i686_cmake'): - chroot_name = 'buildbot_stretch_i686' - bits = 32 - blender_arch = 'i686' - command_prefix = ['schroot', '-c', chroot_name, '--'] - elif blender_glibc == 'glibc217': - command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] - - # Strip all unused symbols from the binaries - print("Stripping binaries...") - subprocess.call(command_prefix + ['strip', '--strip-all', blender]) - - print("Stripping python...") - py_target = os.path.join(install_dir, blender_version) - subprocess.call(command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) - - # Copy all specific files which are too specific to be copied by - # the CMake rules themselves - print("Copying extra scripts and libs...") - - extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra') - mesalibs = os.path.join(extra, 'mesalibs' + str(bits) + '.tar.bz2') - software_gl = os.path.join(blender_dir, 'release', 'bin', 'blender-softwaregl') - icons = os.path.join(blender_dir, 'release', 'freedesktop', 'icons') - - os.system('tar -xpf %s -C %s' % (mesalibs, install_dir)) - os.system('cp %s %s' % (software_gl, install_dir)) - os.system('cp -r %s %s' % (icons, install_dir)) - os.system('chmod 755 %s' % (os.path.join(install_dir, 'blender-softwaregl'))) - - # Construct archive name - package_name = 'blender-%s-%s-linux-%s-%s' % (blender_version, - blender_hash, - blender_glibc, - blender_arch) - # NOTE: Blender 2.7 is already respected by blender_full_version. - if branch != '' and branch != 'blender2.7': - package_name = branch + "-" + package_name - - upload_filename = package_name + ".tar.bz2" - - print("Creating .tar.bz2 archive") - upload_filepath = install_dir + '.tar.bz2' - create_tar_bz2(install_dir, upload_filepath, package_name) -else: - print("Unknown building system") - sys.exit(1) - - -if upload_filepath is None: - # clean release directory if it already exists - release_dir = 'release' - - if os.path.exists(release_dir): - for f in os.listdir(release_dir): - if os.path.isfile(os.path.join(release_dir, f)): - os.remove(os.path.join(release_dir, f)) - - # create release package - try: - subprocess.call(['make', 'package_archive']) - except Exception as ex: - sys.stderr.write('Make package release failed' + str(ex) + '\n') - sys.exit(1) - # find release directory, must exist this time - if not os.path.exists(release_dir): - sys.stderr.write("Failed to find release directory %r.\n" % release_dir) - sys.exit(1) +def pack_mac(builder): + info = buildbot_utils.VersionInfo(builder) - # find release package - file = None - filepath = None + os.chdir(builder.build_dir) + cleanup_files(builder.build_dir, '.dmg') - for f in os.listdir(release_dir): - rf = os.path.join(release_dir, f) - if os.path.isfile(rf) and f.startswith('blender'): - file = f - filepath = rf + package_name = get_package_name(builder, 'macOS') + package_filename = package_name + '.dmg' + package_filepath = os.path.join(builder.build_dir, package_filename) - if not file: - sys.stderr.write("Failed to find release package.\n") - sys.exit(1) + release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') + bundle_sh = os.path.join(release_dir, 'bundle.sh') + if info.is_development_build: + background_image = os.path.join(release_dir, 'buildbot', 'background.tif') + + command = [bundle_sh] + command += ['--source', builder.install_dir] + command += ['--dmg', package_filepath] + command += ['--background-image', background_image] + buildbot_utils.call(command) + + create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) + + +def pack_win(builder): + info = buildbot_utils.VersionInfo(builder) + + os.chdir(builder.build_dir) + cleanup_files(builder.build_dir, '.zip') + + # CPack will add the platform name + cpack_name = get_package_name(builder, None) + package_name = get_package_name(builder, 'windows' + str(builder.bits)) + + command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.'] + buildbot_utils.call(builder.command_prefix + command) + command = ['cpack', '-G', 'ZIP'] + buildbot_utils.call(builder.command_prefix + command) + + package_filename = package_name + '.zip' + package_filepath = os.path.join(builder.build_dir, package_filename) + package_files = [(package_filepath, package_filename)] + + if info.version_cycle == 'release': + # Installer only for final release builds, otherwise will get + # 'this product is already installed' messages. + command = ['cpack', '-G', 'WIX'] + buildbot_utils.call(builder.command_prefix + command) + + package_filename = package_name + '.msi' + package_filepath = os.path.join(builder.build_dir, package_filename) + package_files += [(package_filepath, package_filename)] + + create_buildbot_upload_zip(builder, package_files) + + +def pack_linux(builder): + blender_executable = os.path.join(builder.install_dir, 'blender') + + info = buildbot_utils.VersionInfo(builder) + blender_glibc = builder.name.split('_')[1] + blender_arch = 'x86_64' + + # Strip all unused symbols from the binaries + print("Stripping binaries...") + buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable]) + + print("Stripping python...") + py_target = os.path.join(builder.install_dir, info.version) + buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) + + # Copy all specific files which are too specific to be copied by + # the CMake rules themselves + print("Copying extra scripts and libs...") + + extra = '/' + os.path.join('home', 'sources', 'release-builder', 'extra') + mesalibs = os.path.join(extra, 'mesalibs' + str(builder.bits) + '.tar.bz2') + software_gl = os.path.join(builder.blender_dir, 'release', 'bin', 'blender-softwaregl') + icons = os.path.join(builder.blender_dir, 'release', 'freedesktop', 'icons') + + os.system('tar -xpf %s -C %s' % (mesalibs, builder.install_dir)) + os.system('cp %s %s' % (software_gl, builder.install_dir)) + os.system('cp -r %s %s' % (icons, builder.install_dir)) + os.system('chmod 755 %s' % (os.path.join(builder.install_dir, 'blender-softwaregl'))) + + # Construct package name + platform_name = 'linux-' + blender_glibc + '-' + blender_arch + package_name = get_package_name(builder, platform_name) + package_filename = package_name + ".tar.bz2" + + print("Creating .tar.bz2 archive") + package_filepath = builder.install_dir + '.tar.bz2' + create_tar_bz2(builder.install_dir, package_filepath, package_name) + + # Create buildbot_upload.zip + create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) + + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + + # Make sure install directory always exists + os.makedirs(builder.install_dir, exist_ok=True) - upload_filename = file - upload_filepath = filepath - -# create zip file -try: - upload_zip = os.path.join(buildbot_upload_zip) - if os.path.exists(upload_zip): - os.remove(upload_zip) - z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED) - z.write(upload_filepath, arcname=upload_filename) - z.close() -except Exception as ex: - sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n') - sys.exit(1) + if builder.platform == 'mac': + pack_mac(builder) + elif builder.platform == 'win': + pack_win(builder) + elif builder.platform == 'linux': + pack_linux(builder) diff --git a/build_files/buildbot/slave_rsync.py b/build_files/buildbot/slave_rsync.py index 6936232a495..19f1e67408d 100644 --- a/build_files/buildbot/slave_rsync.py +++ b/build_files/buildbot/slave_rsync.py @@ -21,23 +21,17 @@ # Runs on buildbot slave, rsync zip directly to buildbot server rather # than using upload which is much slower +import buildbot_utils import os import sys -# get builder name -if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting builder name\n") - sys.exit(1) +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() -builder = sys.argv[1] + # rsync, this assumes ssh keys are setup so no password is needed + local_zip = "buildbot_upload.zip" + remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/" + remote_zip = remote_folder + "buildbot_upload_" + builder.name + ".zip" -# rsync, this assumes ssh keys are setup so no password is needed -local_zip = "buildbot_upload.zip" -remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/" -remote_zip = remote_folder + "buildbot_upload_" + builder + ".zip" -command = "rsync -avz %s %s" % (local_zip, remote_zip) - -print(command) - -ret = os.system(command) -sys.exit(ret) + command = ["rsync", "-avz", local_zip, remote_zip] + buildbot_utils.call(command) diff --git a/build_files/buildbot/slave_test.py b/build_files/buildbot/slave_test.py index ff186d8cd45..1b6b426b621 100644 --- a/build_files/buildbot/slave_test.py +++ b/build_files/buildbot/slave_test.py @@ -18,59 +18,39 @@ # <pep8 compliant> -import subprocess +import buildbot_utils import os import sys -# get builder name -if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting builder name\n") - sys.exit(1) +def get_ctest_environment(builder): + info = buildbot_utils.VersionInfo(builder) + blender_version_dir = os.path.join(builder.install_dir, info.version) -builder = sys.argv[1] + env = os.environ.copy() + env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts') + env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles') + return env -# we run from build/ directory -blender_dir = '../blender.git' +def get_ctest_arguments(builder): + args = ['--output-on-failure'] + if builder.platform == 'win': + args += ['-C', 'Release'] + return args -if "cmake" in builder: - print("Automated tests are still DISABLED!") - sys.exit(0) - - build_dir = os.path.abspath(os.path.join('..', 'build', builder)) - install_dir = os.path.abspath(os.path.join('..', 'install', builder)) - # NOTE: For quick test only to see if the approach work. - # n the future must be replaced with an actual blender version. - blender_version = '2.80' - blender_version_dir = os.path.join(install_dir, blender_version) - command_prefix = [] - extra_ctest_args = [] +def test(builder): + os.chdir(builder.build_dir) - if builder.startswith('win'): - extra_ctest_args += ['-C', 'Release'] - elif builder.startswith('linux'): - tokens = builder.split("_") - glibc = tokens[1] - if glibc == 'glibc224': - deb_name = "stretch" - if builder.endswith('x86_64_cmake'): - chroot_name = 'buildbot_' + deb_name + '_x86_64' - elif builder.endswith('i686_cmake'): - chroot_name = 'buildbot_' + deb_name + '_i686' - command_prefix = ['schroot', '--preserve-environment', '-c', chroot_name, '--'] - elif glibc == 'glibc217': - command_prefix = ['scl', 'enable', 'devtoolset-6', '--'] + command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder) + ctest_env = get_ctest_environment(builder) + buildbot_utils.call(command, env=ctest_env, exit_on_error=False) - ctest_env = os.environ.copy() - ctest_env['BLENDER_SYSTEM_SCRIPTS'] = os.path.join(blender_version_dir, 'scripts') - ctest_env['BLENDER_SYSTEM_DATAFILES'] = os.path.join(blender_version_dir, 'datafiles') +if __name__ == "__main__": + print("Automated tests are still DISABLED!") + sys.exit(0) - os.chdir(build_dir) - retcode = subprocess.call(command_prefix + ['ctest', '--output-on-failure'] + extra_ctest_args, - env=ctest_env) + builder = buildbot_utils.create_builder_from_arguments() + test(builder) # Always exit with a success, for until we know all the tests are passing # on all builders. sys.exit(0) -else: - print("Unknown building system") - sys.exit(1) diff --git a/build_files/buildbot/slave_update.py b/build_files/buildbot/slave_update.py index b602cdb9b2f..16b18de3a5d 100644 --- a/build_files/buildbot/slave_update.py +++ b/build_files/buildbot/slave_update.py @@ -18,14 +18,14 @@ # <pep8 compliant> +import buildbot_utils import os -import runpy +import sys -# We run from build/ directory. -blender_dir = os.path.join('..', 'blender.git') -blender_dir = os.path.abspath(blender_dir) -os.chdir(blender_dir) +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + os.chdir(builder.blender_dir) -# Run make update which handles all libraries and submodules. -make_update = os.path.join(blender_dir, "build_files", "utils", "make_update.py") -runpy.run_path(make_update) + # Run make update which handles all libraries and submodules. + make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py") + buildbot_utils.call([sys.executable, make_update]) |