From 296e3ee62c284c601e12c010dc1ec7cbb95af663 Mon Sep 17 00:00:00 2001 From: Sergey Sharybin Date: Wed, 17 Jun 2020 17:39:17 +0200 Subject: Buildbot: Cleanup, remove unused script and change naming Follow upstream convention. --- build_files/buildbot/README.md | 2 +- build_files/buildbot/slave_bundle_dmg.py | 542 ----------------------------- build_files/buildbot/slave_codesign.cmake | 44 --- build_files/buildbot/slave_codesign.py | 74 ---- build_files/buildbot/slave_compile.py | 116 ------ build_files/buildbot/slave_pack.py | 197 ----------- build_files/buildbot/slave_rsync.py | 37 -- build_files/buildbot/slave_test.py | 39 --- build_files/buildbot/slave_update.py | 31 -- build_files/buildbot/worker_bundle_dmg.py | 542 +++++++++++++++++++++++++++++ build_files/buildbot/worker_codesign.cmake | 44 +++ build_files/buildbot/worker_codesign.py | 74 ++++ build_files/buildbot/worker_compile.py | 116 ++++++ build_files/buildbot/worker_pack.py | 197 +++++++++++ build_files/buildbot/worker_test.py | 39 +++ build_files/buildbot/worker_update.py | 31 ++ 16 files changed, 1044 insertions(+), 1081 deletions(-) delete mode 100755 build_files/buildbot/slave_bundle_dmg.py delete mode 100644 build_files/buildbot/slave_codesign.cmake delete mode 100755 build_files/buildbot/slave_codesign.py delete mode 100644 build_files/buildbot/slave_compile.py delete mode 100644 build_files/buildbot/slave_pack.py delete mode 100644 build_files/buildbot/slave_rsync.py delete mode 100644 build_files/buildbot/slave_test.py delete mode 100644 build_files/buildbot/slave_update.py create mode 100755 build_files/buildbot/worker_bundle_dmg.py create mode 100644 build_files/buildbot/worker_codesign.cmake create mode 100755 build_files/buildbot/worker_codesign.py create mode 100644 build_files/buildbot/worker_compile.py create mode 100644 build_files/buildbot/worker_pack.py create mode 100644 build_files/buildbot/worker_test.py create mode 100644 build_files/buildbot/worker_update.py diff --git a/build_files/buildbot/README.md b/build_files/buildbot/README.md index cf129f83b39..06733c9a42d 100644 --- a/build_files/buildbot/README.md +++ b/build_files/buildbot/README.md @@ -8,7 +8,7 @@ Code signing is done as part of INSTALL target, which makes it possible to sign files which are aimed into a bundle and coming from a non-signed source (such as libraries SVN). -This is achieved by specifying `slave_codesign.cmake` as a post-install script +This is achieved by specifying `worker_codesign.cmake` as a post-install script run by CMake. This CMake script simply involves an utility script written in Python which takes care of an actual signing. diff --git a/build_files/buildbot/slave_bundle_dmg.py b/build_files/buildbot/slave_bundle_dmg.py deleted file mode 100755 index 11d2c3cb602..00000000000 --- a/build_files/buildbot/slave_bundle_dmg.py +++ /dev/null @@ -1,542 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -import argparse -import re -import shutil -import subprocess -import sys -import time - -from pathlib import Path -from tempfile import TemporaryDirectory, NamedTemporaryFile -from typing import List - -BUILDBOT_DIRECTORY = Path(__file__).absolute().parent -CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'slave_codesign.py' -BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent -DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' - - -# Extra size which is added on top of actual files size when estimating size -# of destination DNG. -EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 - -################################################################################ -# Common utilities - - -def get_directory_size(root_directory: Path) -> int: - """ - Get size of directory on disk - """ - - total_size = 0 - for file in root_directory.glob('**/*'): - total_size += file.lstat().st_size - return total_size - - -################################################################################ -# DMG bundling specific logic - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'source_dir', - type=Path, - help='Source directory which points to either existing .app bundle' - 'or to a directory with .app bundles.') - parser.add_argument( - '--background-image', - type=Path, - help="Optional background picture which will be set on the DMG." - "If not provided default Blender's one is used.") - parser.add_argument( - '--volume-name', - type=str, - help='Optional name of a volume which will be used for DMG.') - parser.add_argument( - '--dmg', - type=Path, - help='Optional argument which points to a final DMG file name.') - parser.add_argument( - '--applescript', - type=Path, - help="Optional path to applescript to set up folder looks of DMG." - "If not provided default Blender's one is used.") - return parser - - -def collect_app_bundles(source_dir: Path) -> List[Path]: - """ - Collect all app bundles which are to be put into DMG - - If the source directory points to FOO.app it will be the only app bundle - packed. - - Otherwise all .app bundles from given directory are placed to a single - DMG. - """ - - if source_dir.name.endswith('.app'): - return [source_dir] - - app_bundles = [] - for filename in source_dir.glob('*'): - if not filename.is_dir(): - continue - if not filename.name.endswith('.app'): - continue - - app_bundles.append(filename) - - return app_bundles - - -def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: - app_bundles = collect_app_bundles(source_dir) - - if not app_bundles: - print('No app bundles found for packing') - return - - print(f'Found {len(app_bundles)} to pack:') - for app_bundle in app_bundles: - print(f'- {app_bundle}') - - return app_bundles - - -def estimate_dmg_size(app_bundles: List[Path]) -> int: - """ - Estimate size of DMG to hold requested app bundles - - The size is based on actual size of all files in all bundles plus some - space to compensate for different size-on-disk plus some space to hold - codesign signatures. - - Is better to be on a high side since the empty space is compressed, but - lack of space might cause silent failures later on. - """ - - app_bundles_size = 0 - for app_bundle in app_bundles: - app_bundles_size += get_directory_size(app_bundle) - - return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES - - -def copy_app_bundles_to_directory(app_bundles: List[Path], - directory: Path) -> None: - """ - Copy all bundles to a given directory - - This directory is what the DMG will be created from. - """ - for app_bundle in app_bundles: - print(f'Copying {app_bundle.name}...') - shutil.copytree(app_bundle, directory / app_bundle.name) - - -def get_main_app_bundle(app_bundles: List[Path]) -> Path: - """ - Get application bundle main for the installation - """ - return app_bundles[0] - - -def create_dmg_image(app_bundles: List[Path], - dmg_filepath: Path, - volume_name: str) -> None: - """ - Create DMG disk image and put app bundles in it - - No DMG configuration or codesigning is happening here. - """ - - if dmg_filepath.exists(): - print(f'Removing existing writable DMG {dmg_filepath}...') - dmg_filepath.unlink() - - print('Preparing directory with app bundles for the DMG...') - with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: - # Copy all bundles to a clean directory. - content_dir = Path(content_dir_str) - copy_app_bundles_to_directory(app_bundles, content_dir) - - # Estimate size of the DMG. - dmg_size = estimate_dmg_size(app_bundles) - print(f'Estimated DMG size: {dmg_size:,} bytes.') - - # Create the DMG. - print(f'Creating writable DMG {dmg_filepath}') - command = ('hdiutil', - 'create', - '-size', str(dmg_size), - '-fs', 'HFS+', - '-srcfolder', content_dir, - '-volname', volume_name, - '-format', 'UDRW', - dmg_filepath) - subprocess.run(command) - - -def get_writable_dmg_filepath(dmg_filepath: Path): - """ - Get file path for writable DMG image - """ - parent = dmg_filepath.parent - return parent / (dmg_filepath.stem + '-temp.dmg') - - -def mount_readwrite_dmg(dmg_filepath: Path) -> None: - """ - Mount writable DMG - - Mounting point would be /Volumes/ - """ - - print(f'Mounting read-write DMG ${dmg_filepath}') - command = ('hdiutil', - 'attach', '-readwrite', - '-noverify', - '-noautoopen', - dmg_filepath) - subprocess.run(command) - - -def get_mount_directory_for_volume_name(volume_name: str) -> Path: - """ - Get directory under which the volume will be mounted - """ - - return Path('/Volumes') / volume_name - - -def eject_volume(volume_name: str) -> None: - """ - Eject given volume, if mounted - """ - mount_directory = get_mount_directory_for_volume_name(volume_name) - if not mount_directory.exists(): - return - mount_directory_str = str(mount_directory) - - print(f'Ejecting volume {volume_name}') - - # Figure out which device to eject. - mount_output = subprocess.check_output(['mount']).decode() - device = '' - for line in mount_output.splitlines(): - if f'on {mount_directory_str} (' not in line: - continue - tokens = line.split(' ', 3) - if len(tokens) < 3: - continue - if tokens[1] != 'on': - continue - if device: - raise Exception( - f'Multiple devices found for mounting point {mount_directory}') - device = tokens[0] - - if not device: - raise Exception( - f'No device found for mounting point {mount_directory}') - - print(f'{mount_directory} is mounted as device {device}, ejecting...') - subprocess.run(['diskutil', 'eject', device]) - - -def copy_background_if_needed(background_image_filepath: Path, - mount_directory: Path) -> None: - """ - Copy background to the DMG - - If the background image is not specified it will not be copied. - """ - - if not background_image_filepath: - print('No background image provided.') - return - - print(f'Copying background image {background_image_filepath}') - - destination_dir = mount_directory / '.background' - destination_dir.mkdir(exist_ok=True) - - destination_filepath = destination_dir / background_image_filepath.name - shutil.copy(background_image_filepath, destination_filepath) - - -def create_applications_link(mount_directory: Path) -> None: - """ - Create link to /Applications in the given location - """ - - print('Creating link to /Applications') - - command = ('ln', '-s', '/Applications', mount_directory / ' ') - subprocess.run(command) - - -def run_applescript(applescript: Path, - volume_name: str, - app_bundles: List[Path], - background_image_filepath: Path) -> None: - """ - Run given applescript to adjust look and feel of the DMG - """ - - main_app_bundle = get_main_app_bundle(app_bundles) - - with NamedTemporaryFile( - mode='w', suffix='.applescript') as temp_applescript: - print('Adjusting applescript for volume name...') - # Adjust script to the specific volume name. - with open(applescript, mode='r') as input: - for line in input.readlines(): - stripped_line = line.strip() - if stripped_line.startswith('tell disk'): - line = re.sub('tell disk ".*"', - f'tell disk "{volume_name}"', - line) - elif stripped_line.startswith('set background picture'): - if not background_image_filepath: - continue - else: - background_image_short = \ - '.background:' + background_image_filepath.name - line = re.sub('to file ".*"', - f'to file "{background_image_short}"', - line) - line = line.replace('blender.app', main_app_bundle.name) - temp_applescript.write(line) - - temp_applescript.flush() - - print('Running applescript...') - command = ('osascript', temp_applescript.name) - subprocess.run(command) - - print('Waiting for applescript...') - - # NOTE: This is copied from bundle.sh. The exact reason for sleep is - # still remained a mystery. - time.sleep(5) - - -def codesign(subject: Path): - """ - Codesign file or directory - - NOTE: For DMG it will also notarize. - """ - - command = (CODESIGN_SCRIPT, subject) - subprocess.run(command) - - -def codesign_app_bundles_in_dmg(mount_directory: str) -> None: - """ - Code sign all binaries and bundles in the mounted directory - """ - - print(f'Codesigning all app bundles in {mount_directory}') - codesign(mount_directory) - - -def codesign_and_notarize_dmg(dmg_filepath: Path) -> None: - """ - Run codesign and notarization pipeline on the DMG - """ - - print(f'Codesigning and notarizing DMG {dmg_filepath}') - codesign(dmg_filepath) - - -def compress_dmg(writable_dmg_filepath: Path, - final_dmg_filepath: Path) -> None: - """ - Compress temporary read-write DMG - """ - command = ('hdiutil', 'convert', - writable_dmg_filepath, - '-format', 'UDZO', - '-o', final_dmg_filepath) - - if final_dmg_filepath.exists(): - print(f'Removing old compressed DMG {final_dmg_filepath}') - final_dmg_filepath.unlink() - - print('Compressing disk image...') - subprocess.run(command) - - -def create_final_dmg(app_bundles: List[Path], - dmg_filepath: Path, - background_image_filepath: Path, - volume_name: str, - applescript: Path) -> None: - """ - Create DMG with all app bundles - - Will take care configuring background, signing all binaries and app bundles - and notarizing the DMG. - """ - - print('Running all routines to create final DMG') - - writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath) - mount_directory = get_mount_directory_for_volume_name(volume_name) - - # Make sure volume is not mounted. - # If it is mounted it will prevent removing old DMG files and could make - # it so app bundles are copied to the wrong place. - eject_volume(volume_name) - - create_dmg_image(app_bundles, writable_dmg_filepath, volume_name) - - mount_readwrite_dmg(writable_dmg_filepath) - - # Run codesign first, prior to copying amything else. - # - # This allows to recurs into the content of bundles without worrying about - # possible interfereice of Application symlink. - codesign_app_bundles_in_dmg(mount_directory) - - copy_background_if_needed(background_image_filepath, mount_directory) - create_applications_link(mount_directory) - run_applescript(applescript, volume_name, app_bundles, - background_image_filepath) - - print('Ejecting read-write DMG image...') - eject_volume(volume_name) - - compress_dmg(writable_dmg_filepath, dmg_filepath) - writable_dmg_filepath.unlink() - - codesign_and_notarize_dmg(dmg_filepath) - - -def ensure_dmg_extension(filepath: Path) -> Path: - """ - Make sure given file have .dmg extension - """ - - if filepath.suffix != '.dmg': - return filepath.with_suffix(f'{filepath.suffix}.dmg') - return filepath - - -def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path: - """ - Get full file path for the final DMG image - - Will use the provided one when possible, otherwise will deduct it from - app bundles. - - If the name is deducted, the DMG is stored in the current directory. - """ - - if requested_name: - return ensure_dmg_extension(requested_name.absolute()) - - # TODO(sergey): This is not necessarily the main one. - main_bundle = app_bundles[0] - # Strip .app from the name - return Path(main_bundle.name[:-4] + '.dmg').absolute() - - -def get_background_image(requested_background_image: Path) -> Path: - """ - Get effective filepath for the background image - """ - - if requested_background_image: - return requested_background_image.absolute() - - return DARWIN_DIRECTORY / 'background.tif' - - -def get_applescript(requested_applescript: Path) -> Path: - """ - Get effective filepath for the applescript - """ - - if requested_applescript: - return requested_applescript.absolute() - - return DARWIN_DIRECTORY / 'blender.applescript' - - -def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str: - """ - Deduct volume name from the DMG path - - Will use first part of the DMG file name prior to dash. - """ - - tokens = dmg_filepath.stem.split('-') - words = tokens[0].split() - - return ' '.join(word.capitalize() for word in words) - - -def get_volume_name(requested_volume_name: str, - dmg_filepath: Path) -> str: - """ - Get effective name for DMG volume - """ - - if requested_volume_name: - return requested_volume_name - - return get_volume_name_from_dmg_filepath(dmg_filepath) - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - - # Get normalized input parameters. - source_dir = args.source_dir.absolute() - background_image_filepath = get_background_image(args.background_image) - applescript = get_applescript(args.applescript) - - app_bundles = collect_and_log_app_bundles(source_dir) - if not app_bundles: - return - - dmg_filepath = get_dmg_filepath(args.dmg, app_bundles) - volume_name = get_volume_name(args.volume_name, dmg_filepath) - - print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)') - - create_final_dmg(app_bundles, - dmg_filepath, - background_image_filepath, - volume_name, - applescript) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/slave_codesign.cmake b/build_files/buildbot/slave_codesign.cmake deleted file mode 100644 index fd2beae11a0..00000000000 --- a/build_files/buildbot/slave_codesign.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# This is a script which is used as POST-INSTALL one for regular CMake's -# INSTALL target. -# It is used by buildbot workers to sign every binary which is going into -# the final buundle. - -# On Windows Python 3 there only is python.exe, no python3.exe. -# -# On other platforms it is possible to have python2 and python3, and a -# symbolic link to python to either of them. So on those platforms use -# an explicit Python version. -if(WIN32) - set(PYTHON_EXECUTABLE python) -else() - set(PYTHON_EXECUTABLE python3) -endif() - -execute_process( - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/slave_codesign.py" - "${CMAKE_INSTALL_PREFIX}" - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE exit_code -) - -if(NOT exit_code EQUAL "0") - message(FATAL_ERROR "Non-zero exit code of codesign tool") -endif() diff --git a/build_files/buildbot/slave_codesign.py b/build_files/buildbot/slave_codesign.py deleted file mode 100755 index a82ee98b1b5..00000000000 --- a/build_files/buildbot/slave_codesign.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# Helper script which takes care of signing provided location. -# -# The location can either be a directory (in which case all eligible binaries -# will be signed) or a single file (in which case a single file will be signed). -# -# This script takes care of all the complexity of communicating between process -# which requests file to be signed and the code signing server. -# -# NOTE: Signing happens in-place. - -import argparse -import sys - -from pathlib import Path - -from codesign.simple_code_signer import SimpleCodeSigner - - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('path_to_sign', type=Path) - return parser - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - path_to_sign = args.path_to_sign.absolute() - - if sys.platform == 'win32': - # When WIX packed is used to generate .msi on Windows the CPack will - # install two different projects and install them to different - # installation prefix: - # - # - C:\b\build\_CPack_Packages\WIX\Blender - # - C:\b\build\_CPack_Packages\WIX\Unspecified - # - # Annoying part is: CMake's post-install script will only be run - # once, with the install prefix which corresponds to a project which - # was installed last. But we want to sign binaries from all projects. - # So in order to do so we detect that we are running for a CPack's - # project used for WIX and force parent directory (which includes both - # projects) to be signed. - # - # Here we force both projects to be signed. - if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign): - path_to_sign = path_to_sign.parent - - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(path_to_sign) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py deleted file mode 100644 index 65cadea587b..00000000000 --- a/build_files/buildbot/slave_compile.py +++ /dev/null @@ -1,116 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# - -import os -import shutil - -import buildbot_utils - -def get_cmake_options(builder): - post_install_script = os.path.join( - builder.blender_dir, 'build_files', 'buildbot', 'slave_codesign.cmake') - - config_file = "build_files/cmake/config/blender_release.cmake" - options = ['-DCMAKE_BUILD_TYPE:STRING=Release', - '-DWITH_GTESTS=ON'] - - if builder.platform == 'mac': - options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') - options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') - elif builder.platform == 'win': - options.extend(['-G', 'Visual Studio 15 2017 Win64']) - options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + post_install_script]) - elif builder.platform == 'linux': - config_file = "build_files/buildbot/config/blender_linux.cmake" - - optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK') - options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir) - - options.append("-C" + os.path.join(builder.blender_dir, config_file)) - options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir)) - - return options - -def update_git(builder): - # Do extra git fetch because not all platform/git/buildbot combinations - # update the origin remote, causing buildinfo to detect local changes. - os.chdir(builder.blender_dir) - - print("Fetching remotes") - command = ['git', 'fetch', '--all'] - buildbot_utils.call(builder.command_prefix + command) - -def clean_directories(builder): - # Make sure no garbage remained from the previous run - if os.path.isdir(builder.install_dir): - shutil.rmtree(builder.install_dir) - - # Make sure build directory exists and enter it - os.makedirs(builder.build_dir, exist_ok=True) - - # Remove buildinfo files to force buildbot to re-generate them. - for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): - full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo) - if os.path.exists(full_path): - print("Removing {}" . format(buildinfo)) - os.remove(full_path) - -def cmake_configure(builder): - # CMake configuration - os.chdir(builder.build_dir) - - cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt') - if os.path.exists(cmake_cache): - print("Removing CMake cache") - os.remove(cmake_cache) - - print("CMake configure:") - cmake_options = get_cmake_options(builder) - command = ['cmake', builder.blender_dir] + cmake_options - buildbot_utils.call(builder.command_prefix + command) - -def cmake_build(builder): - # CMake build - os.chdir(builder.build_dir) - - # NOTE: CPack will build an INSTALL target, which would mean that code - # signing will happen twice when using `make install` and CPack. - # The tricky bit here is that it is not possible to know whether INSTALL - # target is used by CPack or by a buildbot itaself. Extra level on top of - # this is that on Windows it is required to build INSTALL target in order - # to have unit test binaries to run. - # So on the one hand we do an extra unneeded code sign on Windows, but on - # a positive side we don't add complexity and don't make build process more - # fragile trying to avoid this. The signing process is way faster than just - # a clean build of buildbot, especially with regression tests enabled. - if builder.platform == 'win': - command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] - else: - command = ['make', '-s', '-j16', 'install'] - - print("CMake build:") - buildbot_utils.call(builder.command_prefix + command) - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - update_git(builder) - clean_directories(builder) - cmake_configure(builder) - cmake_build(builder) diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py deleted file mode 100644 index 8549a7881e6..00000000000 --- a/build_files/buildbot/slave_pack.py +++ /dev/null @@ -1,197 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# - -# Runs on buildbot slave, creating a release package using the build -# system and zipping it into buildbot_upload.zip. This is then uploaded -# to the master in the next buildbot step. - -import os -import sys - -from pathlib import Path - -import buildbot_utils - -def get_package_name(builder, platform=None): - info = buildbot_utils.VersionInfo(builder) - - package_name = 'blender-' + info.full_version - if platform: - package_name += '-' + platform - if not (builder.branch == 'master' or builder.is_release_branch): - if info.is_development_build: - package_name = builder.branch + "-" + package_name - - return package_name - -def sign_file_or_directory(path): - from codesign.simple_code_signer import SimpleCodeSigner - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(Path(path)) - - -def create_buildbot_upload_zip(builder, package_files): - import zipfile - - buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip") - if os.path.exists(buildbot_upload_zip): - os.remove(buildbot_upload_zip) - - try: - z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) - for filepath, filename in package_files: - print("Packaged", filename) - z.write(filepath, arcname=filename) - z.close() - except Exception as ex: - sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n') - sys.exit(1) - -def create_tar_xz(src, dest, package_name): - # One extra to remove leading os.sep when cleaning root for package_root - ln = len(src) + 1 - flist = list() - - # Create list of tuples containing file and archive name - for root, dirs, files in os.walk(src): - package_root = os.path.join(package_name, root[ln:]) - flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files]) - - import tarfile - - # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever - # user compiled the package. If root then unpacks it to /usr/local/ you get - # a security issue. - def _fakeroot(tarinfo): - tarinfo.gid = 0 - tarinfo.gname = "root" - tarinfo.uid = 0 - tarinfo.uname = "root" - return tarinfo - - package = tarfile.open(dest, 'w:xz', preset=9) - for entry in flist: - package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) - package.close() - -def cleanup_files(dirpath, extension): - for f in os.listdir(dirpath): - filepath = os.path.join(dirpath, f) - if os.path.isfile(filepath) and f.endswith(extension): - os.remove(filepath) - - -def pack_mac(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.dmg') - - package_name = get_package_name(builder, 'macOS') - package_filename = package_name + '.dmg' - package_filepath = os.path.join(builder.build_dir, package_filename) - - release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') - buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot') - bundle_script = os.path.join(buildbot_dir, 'slave_bundle_dmg.py') - - command = [bundle_script] - command += ['--dmg', package_filepath] - if info.is_development_build: - background_image = os.path.join(release_dir, 'buildbot', 'background.tif') - command += ['--background-image', background_image] - command += [builder.install_dir] - buildbot_utils.call(command) - - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -def pack_win(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.zip') - - # CPack will add the platform name - cpack_name = get_package_name(builder, None) - package_name = get_package_name(builder, 'windows' + str(builder.bits)) - - command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.'] - buildbot_utils.call(builder.command_prefix + command) - command = ['cpack', '-G', 'ZIP'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.zip' - package_filepath = os.path.join(builder.build_dir, package_filename) - package_files = [(package_filepath, package_filename)] - - if info.version_cycle == 'release': - # Installer only for final release builds, otherwise will get - # 'this product is already installed' messages. - command = ['cpack', '-G', 'WIX'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.msi' - package_filepath = os.path.join(builder.build_dir, package_filename) - sign_file_or_directory(package_filepath) - - package_files += [(package_filepath, package_filename)] - - create_buildbot_upload_zip(builder, package_files) - - -def pack_linux(builder): - blender_executable = os.path.join(builder.install_dir, 'blender') - - info = buildbot_utils.VersionInfo(builder) - - # Strip all unused symbols from the binaries - print("Stripping binaries...") - buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable]) - - print("Stripping python...") - py_target = os.path.join(builder.install_dir, info.short_version) - buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) - - # Construct package name - platform_name = 'linux64' - package_name = get_package_name(builder, platform_name) - package_filename = package_name + ".tar.xz" - - print("Creating .tar.xz archive") - package_filepath = builder.install_dir + '.tar.xz' - create_tar_xz(builder.install_dir, package_filepath, package_name) - - # Create buildbot_upload.zip - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - - # Make sure install directory always exists - os.makedirs(builder.install_dir, exist_ok=True) - - if builder.platform == 'mac': - pack_mac(builder) - elif builder.platform == 'win': - pack_win(builder) - elif builder.platform == 'linux': - pack_linux(builder) diff --git a/build_files/buildbot/slave_rsync.py b/build_files/buildbot/slave_rsync.py deleted file mode 100644 index 19f1e67408d..00000000000 --- a/build_files/buildbot/slave_rsync.py +++ /dev/null @@ -1,37 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# - -# Runs on buildbot slave, rsync zip directly to buildbot server rather -# than using upload which is much slower - -import buildbot_utils -import os -import sys - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - - # rsync, this assumes ssh keys are setup so no password is needed - local_zip = "buildbot_upload.zip" - remote_folder = "builder.blender.org:/data/buildbot-master/uploaded/" - remote_zip = remote_folder + "buildbot_upload_" + builder.name + ".zip" - - command = ["rsync", "-avz", local_zip, remote_zip] - buildbot_utils.call(command) diff --git a/build_files/buildbot/slave_test.py b/build_files/buildbot/slave_test.py deleted file mode 100644 index b959568a5c6..00000000000 --- a/build_files/buildbot/slave_test.py +++ /dev/null @@ -1,39 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# - -import buildbot_utils -import os -import sys - -def get_ctest_arguments(builder): - args = ['--output-on-failure'] - if builder.platform == 'win': - args += ['-C', 'Release'] - return args - -def test(builder): - os.chdir(builder.build_dir) - - command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder) - buildbot_utils.call(command) - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - test(builder) diff --git a/build_files/buildbot/slave_update.py b/build_files/buildbot/slave_update.py deleted file mode 100644 index 36a7ae31c84..00000000000 --- a/build_files/buildbot/slave_update.py +++ /dev/null @@ -1,31 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# - -import buildbot_utils -import os -import sys - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - os.chdir(builder.blender_dir) - - # Run make update which handles all libraries and submodules. - make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py") - buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"]) diff --git a/build_files/buildbot/worker_bundle_dmg.py b/build_files/buildbot/worker_bundle_dmg.py new file mode 100755 index 00000000000..cd3da85e12a --- /dev/null +++ b/build_files/buildbot/worker_bundle_dmg.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +import argparse +import re +import shutil +import subprocess +import sys +import time + +from pathlib import Path +from tempfile import TemporaryDirectory, NamedTemporaryFile +from typing import List + +BUILDBOT_DIRECTORY = Path(__file__).absolute().parent +CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py' +BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent +DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' + + +# Extra size which is added on top of actual files size when estimating size +# of destination DNG. +EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 + +################################################################################ +# Common utilities + + +def get_directory_size(root_directory: Path) -> int: + """ + Get size of directory on disk + """ + + total_size = 0 + for file in root_directory.glob('**/*'): + total_size += file.lstat().st_size + return total_size + + +################################################################################ +# DMG bundling specific logic + +def create_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'source_dir', + type=Path, + help='Source directory which points to either existing .app bundle' + 'or to a directory with .app bundles.') + parser.add_argument( + '--background-image', + type=Path, + help="Optional background picture which will be set on the DMG." + "If not provided default Blender's one is used.") + parser.add_argument( + '--volume-name', + type=str, + help='Optional name of a volume which will be used for DMG.') + parser.add_argument( + '--dmg', + type=Path, + help='Optional argument which points to a final DMG file name.') + parser.add_argument( + '--applescript', + type=Path, + help="Optional path to applescript to set up folder looks of DMG." + "If not provided default Blender's one is used.") + return parser + + +def collect_app_bundles(source_dir: Path) -> List[Path]: + """ + Collect all app bundles which are to be put into DMG + + If the source directory points to FOO.app it will be the only app bundle + packed. + + Otherwise all .app bundles from given directory are placed to a single + DMG. + """ + + if source_dir.name.endswith('.app'): + return [source_dir] + + app_bundles = [] + for filename in source_dir.glob('*'): + if not filename.is_dir(): + continue + if not filename.name.endswith('.app'): + continue + + app_bundles.append(filename) + + return app_bundles + + +def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: + app_bundles = collect_app_bundles(source_dir) + + if not app_bundles: + print('No app bundles found for packing') + return + + print(f'Found {len(app_bundles)} to pack:') + for app_bundle in app_bundles: + print(f'- {app_bundle}') + + return app_bundles + + +def estimate_dmg_size(app_bundles: List[Path]) -> int: + """ + Estimate size of DMG to hold requested app bundles + + The size is based on actual size of all files in all bundles plus some + space to compensate for different size-on-disk plus some space to hold + codesign signatures. + + Is better to be on a high side since the empty space is compressed, but + lack of space might cause silent failures later on. + """ + + app_bundles_size = 0 + for app_bundle in app_bundles: + app_bundles_size += get_directory_size(app_bundle) + + return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES + + +def copy_app_bundles_to_directory(app_bundles: List[Path], + directory: Path) -> None: + """ + Copy all bundles to a given directory + + This directory is what the DMG will be created from. + """ + for app_bundle in app_bundles: + print(f'Copying {app_bundle.name}...') + shutil.copytree(app_bundle, directory / app_bundle.name) + + +def get_main_app_bundle(app_bundles: List[Path]) -> Path: + """ + Get application bundle main for the installation + """ + return app_bundles[0] + + +def create_dmg_image(app_bundles: List[Path], + dmg_filepath: Path, + volume_name: str) -> None: + """ + Create DMG disk image and put app bundles in it + + No DMG configuration or codesigning is happening here. + """ + + if dmg_filepath.exists(): + print(f'Removing existing writable DMG {dmg_filepath}...') + dmg_filepath.unlink() + + print('Preparing directory with app bundles for the DMG...') + with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: + # Copy all bundles to a clean directory. + content_dir = Path(content_dir_str) + copy_app_bundles_to_directory(app_bundles, content_dir) + + # Estimate size of the DMG. + dmg_size = estimate_dmg_size(app_bundles) + print(f'Estimated DMG size: {dmg_size:,} bytes.') + + # Create the DMG. + print(f'Creating writable DMG {dmg_filepath}') + command = ('hdiutil', + 'create', + '-size', str(dmg_size), + '-fs', 'HFS+', + '-srcfolder', content_dir, + '-volname', volume_name, + '-format', 'UDRW', + dmg_filepath) + subprocess.run(command) + + +def get_writable_dmg_filepath(dmg_filepath: Path): + """ + Get file path for writable DMG image + """ + parent = dmg_filepath.parent + return parent / (dmg_filepath.stem + '-temp.dmg') + + +def mount_readwrite_dmg(dmg_filepath: Path) -> None: + """ + Mount writable DMG + + Mounting point would be /Volumes/ + """ + + print(f'Mounting read-write DMG ${dmg_filepath}') + command = ('hdiutil', + 'attach', '-readwrite', + '-noverify', + '-noautoopen', + dmg_filepath) + subprocess.run(command) + + +def get_mount_directory_for_volume_name(volume_name: str) -> Path: + """ + Get directory under which the volume will be mounted + """ + + return Path('/Volumes') / volume_name + + +def eject_volume(volume_name: str) -> None: + """ + Eject given volume, if mounted + """ + mount_directory = get_mount_directory_for_volume_name(volume_name) + if not mount_directory.exists(): + return + mount_directory_str = str(mount_directory) + + print(f'Ejecting volume {volume_name}') + + # Figure out which device to eject. + mount_output = subprocess.check_output(['mount']).decode() + device = '' + for line in mount_output.splitlines(): + if f'on {mount_directory_str} (' not in line: + continue + tokens = line.split(' ', 3) + if len(tokens) < 3: + continue + if tokens[1] != 'on': + continue + if device: + raise Exception( + f'Multiple devices found for mounting point {mount_directory}') + device = tokens[0] + + if not device: + raise Exception( + f'No device found for mounting point {mount_directory}') + + print(f'{mount_directory} is mounted as device {device}, ejecting...') + subprocess.run(['diskutil', 'eject', device]) + + +def copy_background_if_needed(background_image_filepath: Path, + mount_directory: Path) -> None: + """ + Copy background to the DMG + + If the background image is not specified it will not be copied. + """ + + if not background_image_filepath: + print('No background image provided.') + return + + print(f'Copying background image {background_image_filepath}') + + destination_dir = mount_directory / '.background' + destination_dir.mkdir(exist_ok=True) + + destination_filepath = destination_dir / background_image_filepath.name + shutil.copy(background_image_filepath, destination_filepath) + + +def create_applications_link(mount_directory: Path) -> None: + """ + Create link to /Applications in the given location + """ + + print('Creating link to /Applications') + + command = ('ln', '-s', '/Applications', mount_directory / ' ') + subprocess.run(command) + + +def run_applescript(applescript: Path, + volume_name: str, + app_bundles: List[Path], + background_image_filepath: Path) -> None: + """ + Run given applescript to adjust look and feel of the DMG + """ + + main_app_bundle = get_main_app_bundle(app_bundles) + + with NamedTemporaryFile( + mode='w', suffix='.applescript') as temp_applescript: + print('Adjusting applescript for volume name...') + # Adjust script to the specific volume name. + with open(applescript, mode='r') as input: + for line in input.readlines(): + stripped_line = line.strip() + if stripped_line.startswith('tell disk'): + line = re.sub('tell disk ".*"', + f'tell disk "{volume_name}"', + line) + elif stripped_line.startswith('set background picture'): + if not background_image_filepath: + continue + else: + background_image_short = \ + '.background:' + background_image_filepath.name + line = re.sub('to file ".*"', + f'to file "{background_image_short}"', + line) + line = line.replace('blender.app', main_app_bundle.name) + temp_applescript.write(line) + + temp_applescript.flush() + + print('Running applescript...') + command = ('osascript', temp_applescript.name) + subprocess.run(command) + + print('Waiting for applescript...') + + # NOTE: This is copied from bundle.sh. The exact reason for sleep is + # still remained a mystery. + time.sleep(5) + + +def codesign(subject: Path): + """ + Codesign file or directory + + NOTE: For DMG it will also notarize. + """ + + command = (CODESIGN_SCRIPT, subject) + subprocess.run(command) + + +def codesign_app_bundles_in_dmg(mount_directory: str) -> None: + """ + Code sign all binaries and bundles in the mounted directory + """ + + print(f'Codesigning all app bundles in {mount_directory}') + codesign(mount_directory) + + +def codesign_and_notarize_dmg(dmg_filepath: Path) -> None: + """ + Run codesign and notarization pipeline on the DMG + """ + + print(f'Codesigning and notarizing DMG {dmg_filepath}') + codesign(dmg_filepath) + + +def compress_dmg(writable_dmg_filepath: Path, + final_dmg_filepath: Path) -> None: + """ + Compress temporary read-write DMG + """ + command = ('hdiutil', 'convert', + writable_dmg_filepath, + '-format', 'UDZO', + '-o', final_dmg_filepath) + + if final_dmg_filepath.exists(): + print(f'Removing old compressed DMG {final_dmg_filepath}') + final_dmg_filepath.unlink() + + print('Compressing disk image...') + subprocess.run(command) + + +def create_final_dmg(app_bundles: List[Path], + dmg_filepath: Path, + background_image_filepath: Path, + volume_name: str, + applescript: Path) -> None: + """ + Create DMG with all app bundles + + Will take care configuring background, signing all binaries and app bundles + and notarizing the DMG. + """ + + print('Running all routines to create final DMG') + + writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath) + mount_directory = get_mount_directory_for_volume_name(volume_name) + + # Make sure volume is not mounted. + # If it is mounted it will prevent removing old DMG files and could make + # it so app bundles are copied to the wrong place. + eject_volume(volume_name) + + create_dmg_image(app_bundles, writable_dmg_filepath, volume_name) + + mount_readwrite_dmg(writable_dmg_filepath) + + # Run codesign first, prior to copying amything else. + # + # This allows to recurs into the content of bundles without worrying about + # possible interfereice of Application symlink. + codesign_app_bundles_in_dmg(mount_directory) + + copy_background_if_needed(background_image_filepath, mount_directory) + create_applications_link(mount_directory) + run_applescript(applescript, volume_name, app_bundles, + background_image_filepath) + + print('Ejecting read-write DMG image...') + eject_volume(volume_name) + + compress_dmg(writable_dmg_filepath, dmg_filepath) + writable_dmg_filepath.unlink() + + codesign_and_notarize_dmg(dmg_filepath) + + +def ensure_dmg_extension(filepath: Path) -> Path: + """ + Make sure given file have .dmg extension + """ + + if filepath.suffix != '.dmg': + return filepath.with_suffix(f'{filepath.suffix}.dmg') + return filepath + + +def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path: + """ + Get full file path for the final DMG image + + Will use the provided one when possible, otherwise will deduct it from + app bundles. + + If the name is deducted, the DMG is stored in the current directory. + """ + + if requested_name: + return ensure_dmg_extension(requested_name.absolute()) + + # TODO(sergey): This is not necessarily the main one. + main_bundle = app_bundles[0] + # Strip .app from the name + return Path(main_bundle.name[:-4] + '.dmg').absolute() + + +def get_background_image(requested_background_image: Path) -> Path: + """ + Get effective filepath for the background image + """ + + if requested_background_image: + return requested_background_image.absolute() + + return DARWIN_DIRECTORY / 'background.tif' + + +def get_applescript(requested_applescript: Path) -> Path: + """ + Get effective filepath for the applescript + """ + + if requested_applescript: + return requested_applescript.absolute() + + return DARWIN_DIRECTORY / 'blender.applescript' + + +def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str: + """ + Deduct volume name from the DMG path + + Will use first part of the DMG file name prior to dash. + """ + + tokens = dmg_filepath.stem.split('-') + words = tokens[0].split() + + return ' '.join(word.capitalize() for word in words) + + +def get_volume_name(requested_volume_name: str, + dmg_filepath: Path) -> str: + """ + Get effective name for DMG volume + """ + + if requested_volume_name: + return requested_volume_name + + return get_volume_name_from_dmg_filepath(dmg_filepath) + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + + # Get normalized input parameters. + source_dir = args.source_dir.absolute() + background_image_filepath = get_background_image(args.background_image) + applescript = get_applescript(args.applescript) + + app_bundles = collect_and_log_app_bundles(source_dir) + if not app_bundles: + return + + dmg_filepath = get_dmg_filepath(args.dmg, app_bundles) + volume_name = get_volume_name(args.volume_name, dmg_filepath) + + print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)') + + create_final_dmg(app_bundles, + dmg_filepath, + background_image_filepath, + volume_name, + applescript) + + +if __name__ == "__main__": + main() diff --git a/build_files/buildbot/worker_codesign.cmake b/build_files/buildbot/worker_codesign.cmake new file mode 100644 index 00000000000..f37feaef407 --- /dev/null +++ b/build_files/buildbot/worker_codesign.cmake @@ -0,0 +1,44 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# This is a script which is used as POST-INSTALL one for regular CMake's +# INSTALL target. +# It is used by buildbot workers to sign every binary which is going into +# the final buundle. + +# On Windows Python 3 there only is python.exe, no python3.exe. +# +# On other platforms it is possible to have python2 and python3, and a +# symbolic link to python to either of them. So on those platforms use +# an explicit Python version. +if(WIN32) + set(PYTHON_EXECUTABLE python) +else() + set(PYTHON_EXECUTABLE python3) +endif() + +execute_process( + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py" + "${CMAKE_INSTALL_PREFIX}" + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE exit_code +) + +if(NOT exit_code EQUAL "0") + message(FATAL_ERROR "Non-zero exit code of codesign tool") +endif() diff --git a/build_files/buildbot/worker_codesign.py b/build_files/buildbot/worker_codesign.py new file mode 100755 index 00000000000..a82ee98b1b5 --- /dev/null +++ b/build_files/buildbot/worker_codesign.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# Helper script which takes care of signing provided location. +# +# The location can either be a directory (in which case all eligible binaries +# will be signed) or a single file (in which case a single file will be signed). +# +# This script takes care of all the complexity of communicating between process +# which requests file to be signed and the code signing server. +# +# NOTE: Signing happens in-place. + +import argparse +import sys + +from pathlib import Path + +from codesign.simple_code_signer import SimpleCodeSigner + + +def create_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('path_to_sign', type=Path) + return parser + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + path_to_sign = args.path_to_sign.absolute() + + if sys.platform == 'win32': + # When WIX packed is used to generate .msi on Windows the CPack will + # install two different projects and install them to different + # installation prefix: + # + # - C:\b\build\_CPack_Packages\WIX\Blender + # - C:\b\build\_CPack_Packages\WIX\Unspecified + # + # Annoying part is: CMake's post-install script will only be run + # once, with the install prefix which corresponds to a project which + # was installed last. But we want to sign binaries from all projects. + # So in order to do so we detect that we are running for a CPack's + # project used for WIX and force parent directory (which includes both + # projects) to be signed. + # + # Here we force both projects to be signed. + if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign): + path_to_sign = path_to_sign.parent + + code_signer = SimpleCodeSigner() + code_signer.sign_file_or_directory(path_to_sign) + + +if __name__ == "__main__": + main() diff --git a/build_files/buildbot/worker_compile.py b/build_files/buildbot/worker_compile.py new file mode 100644 index 00000000000..705614660cf --- /dev/null +++ b/build_files/buildbot/worker_compile.py @@ -0,0 +1,116 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# + +import os +import shutil + +import buildbot_utils + +def get_cmake_options(builder): + post_install_script = os.path.join( + builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake') + + config_file = "build_files/cmake/config/blender_release.cmake" + options = ['-DCMAKE_BUILD_TYPE:STRING=Release', + '-DWITH_GTESTS=ON'] + + if builder.platform == 'mac': + options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') + options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') + elif builder.platform == 'win': + options.extend(['-G', 'Visual Studio 15 2017 Win64']) + options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + post_install_script]) + elif builder.platform == 'linux': + config_file = "build_files/buildbot/config/blender_linux.cmake" + + optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK') + options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir) + + options.append("-C" + os.path.join(builder.blender_dir, config_file)) + options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir)) + + return options + +def update_git(builder): + # Do extra git fetch because not all platform/git/buildbot combinations + # update the origin remote, causing buildinfo to detect local changes. + os.chdir(builder.blender_dir) + + print("Fetching remotes") + command = ['git', 'fetch', '--all'] + buildbot_utils.call(builder.command_prefix + command) + +def clean_directories(builder): + # Make sure no garbage remained from the previous run + if os.path.isdir(builder.install_dir): + shutil.rmtree(builder.install_dir) + + # Make sure build directory exists and enter it + os.makedirs(builder.build_dir, exist_ok=True) + + # Remove buildinfo files to force buildbot to re-generate them. + for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): + full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo) + if os.path.exists(full_path): + print("Removing {}" . format(buildinfo)) + os.remove(full_path) + +def cmake_configure(builder): + # CMake configuration + os.chdir(builder.build_dir) + + cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt') + if os.path.exists(cmake_cache): + print("Removing CMake cache") + os.remove(cmake_cache) + + print("CMake configure:") + cmake_options = get_cmake_options(builder) + command = ['cmake', builder.blender_dir] + cmake_options + buildbot_utils.call(builder.command_prefix + command) + +def cmake_build(builder): + # CMake build + os.chdir(builder.build_dir) + + # NOTE: CPack will build an INSTALL target, which would mean that code + # signing will happen twice when using `make install` and CPack. + # The tricky bit here is that it is not possible to know whether INSTALL + # target is used by CPack or by a buildbot itaself. Extra level on top of + # this is that on Windows it is required to build INSTALL target in order + # to have unit test binaries to run. + # So on the one hand we do an extra unneeded code sign on Windows, but on + # a positive side we don't add complexity and don't make build process more + # fragile trying to avoid this. The signing process is way faster than just + # a clean build of buildbot, especially with regression tests enabled. + if builder.platform == 'win': + command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] + else: + command = ['make', '-s', '-j16', 'install'] + + print("CMake build:") + buildbot_utils.call(builder.command_prefix + command) + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + update_git(builder) + clean_directories(builder) + cmake_configure(builder) + cmake_build(builder) diff --git a/build_files/buildbot/worker_pack.py b/build_files/buildbot/worker_pack.py new file mode 100644 index 00000000000..87ee49c87d8 --- /dev/null +++ b/build_files/buildbot/worker_pack.py @@ -0,0 +1,197 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# + +# Runs on buildbot worker, creating a release package using the build +# system and zipping it into buildbot_upload.zip. This is then uploaded +# to the master in the next buildbot step. + +import os +import sys + +from pathlib import Path + +import buildbot_utils + +def get_package_name(builder, platform=None): + info = buildbot_utils.VersionInfo(builder) + + package_name = 'blender-' + info.full_version + if platform: + package_name += '-' + platform + if not (builder.branch == 'master' or builder.is_release_branch): + if info.is_development_build: + package_name = builder.branch + "-" + package_name + + return package_name + +def sign_file_or_directory(path): + from codesign.simple_code_signer import SimpleCodeSigner + code_signer = SimpleCodeSigner() + code_signer.sign_file_or_directory(Path(path)) + + +def create_buildbot_upload_zip(builder, package_files): + import zipfile + + buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip") + if os.path.exists(buildbot_upload_zip): + os.remove(buildbot_upload_zip) + + try: + z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) + for filepath, filename in package_files: + print("Packaged", filename) + z.write(filepath, arcname=filename) + z.close() + except Exception as ex: + sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n') + sys.exit(1) + +def create_tar_xz(src, dest, package_name): + # One extra to remove leading os.sep when cleaning root for package_root + ln = len(src) + 1 + flist = list() + + # Create list of tuples containing file and archive name + for root, dirs, files in os.walk(src): + package_root = os.path.join(package_name, root[ln:]) + flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files]) + + import tarfile + + # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever + # user compiled the package. If root then unpacks it to /usr/local/ you get + # a security issue. + def _fakeroot(tarinfo): + tarinfo.gid = 0 + tarinfo.gname = "root" + tarinfo.uid = 0 + tarinfo.uname = "root" + return tarinfo + + package = tarfile.open(dest, 'w:xz', preset=9) + for entry in flist: + package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) + package.close() + +def cleanup_files(dirpath, extension): + for f in os.listdir(dirpath): + filepath = os.path.join(dirpath, f) + if os.path.isfile(filepath) and f.endswith(extension): + os.remove(filepath) + + +def pack_mac(builder): + info = buildbot_utils.VersionInfo(builder) + + os.chdir(builder.build_dir) + cleanup_files(builder.build_dir, '.dmg') + + package_name = get_package_name(builder, 'macOS') + package_filename = package_name + '.dmg' + package_filepath = os.path.join(builder.build_dir, package_filename) + + release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') + buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot') + bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py') + + command = [bundle_script] + command += ['--dmg', package_filepath] + if info.is_development_build: + background_image = os.path.join(release_dir, 'buildbot', 'background.tif') + command += ['--background-image', background_image] + command += [builder.install_dir] + buildbot_utils.call(command) + + create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) + + +def pack_win(builder): + info = buildbot_utils.VersionInfo(builder) + + os.chdir(builder.build_dir) + cleanup_files(builder.build_dir, '.zip') + + # CPack will add the platform name + cpack_name = get_package_name(builder, None) + package_name = get_package_name(builder, 'windows' + str(builder.bits)) + + command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.'] + buildbot_utils.call(builder.command_prefix + command) + command = ['cpack', '-G', 'ZIP'] + buildbot_utils.call(builder.command_prefix + command) + + package_filename = package_name + '.zip' + package_filepath = os.path.join(builder.build_dir, package_filename) + package_files = [(package_filepath, package_filename)] + + if info.version_cycle == 'release': + # Installer only for final release builds, otherwise will get + # 'this product is already installed' messages. + command = ['cpack', '-G', 'WIX'] + buildbot_utils.call(builder.command_prefix + command) + + package_filename = package_name + '.msi' + package_filepath = os.path.join(builder.build_dir, package_filename) + sign_file_or_directory(package_filepath) + + package_files += [(package_filepath, package_filename)] + + create_buildbot_upload_zip(builder, package_files) + + +def pack_linux(builder): + blender_executable = os.path.join(builder.install_dir, 'blender') + + info = buildbot_utils.VersionInfo(builder) + + # Strip all unused symbols from the binaries + print("Stripping binaries...") + buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable]) + + print("Stripping python...") + py_target = os.path.join(builder.install_dir, info.short_version) + buildbot_utils.call(builder.command_prefix + ['find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';']) + + # Construct package name + platform_name = 'linux64' + package_name = get_package_name(builder, platform_name) + package_filename = package_name + ".tar.xz" + + print("Creating .tar.xz archive") + package_filepath = builder.install_dir + '.tar.xz' + create_tar_xz(builder.install_dir, package_filepath, package_name) + + # Create buildbot_upload.zip + create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) + + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + + # Make sure install directory always exists + os.makedirs(builder.install_dir, exist_ok=True) + + if builder.platform == 'mac': + pack_mac(builder) + elif builder.platform == 'win': + pack_win(builder) + elif builder.platform == 'linux': + pack_linux(builder) diff --git a/build_files/buildbot/worker_test.py b/build_files/buildbot/worker_test.py new file mode 100644 index 00000000000..b959568a5c6 --- /dev/null +++ b/build_files/buildbot/worker_test.py @@ -0,0 +1,39 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# + +import buildbot_utils +import os +import sys + +def get_ctest_arguments(builder): + args = ['--output-on-failure'] + if builder.platform == 'win': + args += ['-C', 'Release'] + return args + +def test(builder): + os.chdir(builder.build_dir) + + command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder) + buildbot_utils.call(command) + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + test(builder) diff --git a/build_files/buildbot/worker_update.py b/build_files/buildbot/worker_update.py new file mode 100644 index 00000000000..36a7ae31c84 --- /dev/null +++ b/build_files/buildbot/worker_update.py @@ -0,0 +1,31 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# + +import buildbot_utils +import os +import sys + +if __name__ == "__main__": + builder = buildbot_utils.create_builder_from_arguments() + os.chdir(builder.blender_dir) + + # Run make update which handles all libraries and submodules. + make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py") + buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"]) -- cgit v1.2.3