diff options
Diffstat (limited to 'build_files/buildbot')
25 files changed, 3 insertions, 3227 deletions
diff --git a/build_files/buildbot/README.md b/build_files/buildbot/README.md index 06733c9a42d..f6fd07d9246 100644 --- a/build_files/buildbot/README.md +++ b/build_files/buildbot/README.md @@ -1,70 +1,4 @@ -Blender Buildbot -================ +Buildbot Configuration +===================== -Code signing ------------- - -Code signing is done as part of INSTALL target, which makes it possible to sign -files which are aimed into a bundle and coming from a non-signed source (such as -libraries SVN). - -This is achieved by specifying `worker_codesign.cmake` as a post-install script -run by CMake. This CMake script simply involves an utility script written in -Python which takes care of an actual signing. - -### Configuration - -Client configuration doesn't need anything special, other than variable -`SHARED_STORAGE_DIR` pointing to a location which is watched by a server. -This is done in `config_builder.py` file and is stored in Git (which makes it -possible to have almost zero-configuration buildbot machines). - -Server configuration requires copying `config_server_template.py` under the -name of `config_server.py` and tweaking values, which are platform-specific. - -#### Windows configuration - -There are two things which are needed on Windows in order to have code signing -to work: - -- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com -- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx). - -## Tips - -### Self-signed certificate on Windows - -It is easiest to test configuration using self-signed certificate. - -The certificate manipulation utilities are coming with Windows SDK. -Unfortunately, they are not added to PATH. Here is an example of how to make -sure they are easily available: - -``` -set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% -set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH% -``` - -Generate CA: - -``` -makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^ - -cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer -``` - -Import the generated CA: - -``` -certutil -user -addstore Root BlenderTestCA.cer -``` - -Create self-signed certificate and pack it into PKCS #12: - -``` -makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^ - -sky signature ^ - -ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^ - -sv BlenderTestSPC.pvk BlenderTestSPC.cer - -pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx -```
\ No newline at end of file +Files used by Buildbot's `compile-code` step. diff --git a/build_files/buildbot/buildbot_utils.py b/build_files/buildbot/buildbot_utils.py deleted file mode 100644 index 7e9858d9268..00000000000 --- a/build_files/buildbot/buildbot_utils.py +++ /dev/null @@ -1,127 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import argparse -import os -import re -import subprocess -import sys - - -def is_tool(name): - """Check whether `name` is on PATH and marked as executable.""" - - # from whichcraft import which - from shutil import which - - return which(name) is not None - - -class Builder: - def __init__(self, name, branch, codesign): - self.name = name - self.branch = branch - self.is_release_branch = re.match("^blender-v(.*)-release$", branch) is not None - self.codesign = codesign - - # Buildbot runs from build/ directory - self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git')) - self.build_dir = os.path.abspath(os.path.join('..', 'build')) - self.install_dir = os.path.abspath(os.path.join('..', 'install')) - self.upload_dir = os.path.abspath(os.path.join('..', 'install')) - - # Detect platform - if name.startswith('mac'): - self.platform = 'mac' - self.command_prefix = [] - elif name.startswith('linux'): - self.platform = 'linux' - if is_tool('scl'): - self.command_prefix = ['scl', 'enable', 'devtoolset-9', '--'] - else: - self.command_prefix = [] - elif name.startswith('win'): - self.platform = 'win' - self.command_prefix = [] - else: - raise ValueError('Unkonw platform for builder ' + self.platform) - - # Always 64 bit now - self.bits = 64 - - -def create_builder_from_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument('builder_name') - parser.add_argument('branch', default='master', nargs='?') - parser.add_argument("--codesign", action="store_true") - args = parser.parse_args() - return Builder(args.builder_name, args.branch, args.codesign) - - -class VersionInfo: - def __init__(self, builder): - # Get version information - buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h") - blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") - - version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION')) - version_number_patch = int(self._parse_header_file(blender_h, 'BLENDER_VERSION_PATCH')) - version_numbers = (version_number // 100, version_number % 100, version_number_patch) - self.short_version = "%d.%d" % (version_numbers[0], version_numbers[1]) - self.version = "%d.%d.%d" % version_numbers - self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE') - self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1] - - if self.version_cycle == "release": - # Final release - self.full_version = self.version - self.is_development_build = False - elif self.version_cycle == "rc": - # Release candidate - self.full_version = self.version + self.version_cycle - self.is_development_build = False - else: - # Development build - self.full_version = self.version + '-' + self.hash - self.is_development_build = True - - def _parse_header_file(self, filename, define): - import re - regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define) - with open(filename, "r") as file: - for l in file: - match = regex.match(l) - if match: - return match.group(1) - return None - - -def call(cmd, env=None, exit_on_error=True): - print(' '.join(cmd)) - - # Flush to ensure correct order output on Windows. - sys.stdout.flush() - sys.stderr.flush() - - retcode = subprocess.call(cmd, env=env) - if exit_on_error and retcode != 0: - sys.exit(retcode) - return retcode diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py deleted file mode 100644 index cb42710e785..00000000000 --- a/build_files/buildbot/codesign/absolute_and_relative_filename.py +++ /dev/null @@ -1,81 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from dataclasses import dataclass -from pathlib import Path -from typing import List - - -@dataclass -class AbsoluteAndRelativeFileName: - """ - Helper class which keeps track of absolute file path for a direct access and - corresponding relative path against given base. - - The relative part is used to construct a file name within an archive which - contains files which are to be signed or which has been signed already - (depending on whether the archive is addressed to signing server or back - to the buildbot worker). - """ - - # Base directory which is where relative_filepath is relative to. - base_dir: Path - - # Full absolute path of the corresponding file. - absolute_filepath: Path - - # Derived from full file path, contains part of the path which is relative - # to a desired base path. - relative_filepath: Path - - def __init__(self, base_dir: Path, filepath: Path): - self.base_dir = base_dir - self.absolute_filepath = filepath.resolve() - self.relative_filepath = self.absolute_filepath.relative_to( - self.base_dir) - - @classmethod - def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName': - assert path.is_absolute() - assert path.is_file() - - base_dir = path.parent - return AbsoluteAndRelativeFileName(base_dir, path) - - @classmethod - def recursively_from_directory(cls, base_dir: Path) \ - -> List['AbsoluteAndRelativeFileName']: - """ - Create list of AbsoluteAndRelativeFileName for all the files in the - given directory. - - NOTE: Result will be pointing to a resolved paths. - """ - assert base_dir.is_absolute() - assert base_dir.is_dir() - - base_dir = base_dir.resolve() - - result = [] - for filename in base_dir.glob('**/*'): - if not filename.is_file(): - continue - result.append(AbsoluteAndRelativeFileName(base_dir, filename)) - return result diff --git a/build_files/buildbot/codesign/archive_with_indicator.py b/build_files/buildbot/codesign/archive_with_indicator.py deleted file mode 100644 index aebf5a15417..00000000000 --- a/build_files/buildbot/codesign/archive_with_indicator.py +++ /dev/null @@ -1,245 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import dataclasses -import json -import os - -from pathlib import Path -from typing import Optional - -import codesign.util as util - - -class ArchiveStateError(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) - - -@dataclasses.dataclass -class ArchiveState: - """ - Additional information (state) of the archive - - Includes information like expected file size of the archive file in the case - the archive file is expected to be successfully created. - - If the archive can not be created, this state will contain error message - indicating details of error. - """ - - # Size in bytes of the corresponding archive. - file_size: Optional[int] = None - - # Non-empty value indicates that error has happenned. - error_message: str = '' - - def has_error(self) -> bool: - """ - Check whether the archive is at error state - """ - - return self.error_message - - def serialize_to_string(self) -> str: - payload = dataclasses.asdict(self) - return json.dumps(payload, sort_keys=True, indent=4) - - def serialize_to_file(self, filepath: Path) -> None: - string = self.serialize_to_string() - filepath.write_text(string) - - @classmethod - def deserialize_from_string(cls, string: str) -> 'ArchiveState': - try: - object_as_dict = json.loads(string) - except json.decoder.JSONDecodeError: - raise ArchiveStateError('Error parsing JSON') - - return cls(**object_as_dict) - - @classmethod - def deserialize_from_file(cls, filepath: Path): - string = filepath.read_text() - return cls.deserialize_from_string(string) - - -class ArchiveWithIndicator: - """ - The idea of this class is to wrap around logic which takes care of keeping - track of a name of an archive and synchronization routines between buildbot - worker and signing server. - - The synchronization is done based on creating a special file after the - archive file is knowingly ready for access. - """ - - # Base directory where the archive is stored (basically, a basename() of - # the absolute archive file name). - # - # For example, 'X:\\TEMP\\'. - base_dir: Path - - # Absolute file name of the archive. - # - # For example, 'X:\\TEMP\\FOO.ZIP'. - archive_filepath: Path - - # Absolute name of a file which acts as an indication of the fact that the - # archive is ready and is available for access. - # - # This is how synchronization between buildbot worker and signing server is - # done: - # - First, the archive is created under archive_filepath name. - # - Second, the indication file is created under ready_indicator_filepath - # name. - # - Third, the colleague of whoever created the indicator name watches for - # the indication file to appear, and once it's there it access the - # archive. - ready_indicator_filepath: Path - - def __init__( - self, base_dir: Path, archive_name: str, ready_indicator_name: str): - """ - Construct the object from given base directory and name of the archive - file: - ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY') - """ - - self.base_dir = base_dir - self.archive_filepath = self.base_dir / archive_name - self.ready_indicator_filepath = self.base_dir / ready_indicator_name - - def is_ready_unsafe(self) -> bool: - """ - Check whether the archive is ready for access. - - No guarding about possible network failres is done here. - """ - if not self.ready_indicator_filepath.exists(): - return False - - try: - archive_state = ArchiveState.deserialize_from_file( - self.ready_indicator_filepath) - except ArchiveStateError as error: - print(f'Error deserializing archive state: {error.message}') - return False - - if archive_state.has_error(): - # If the error did happen during codesign procedure there will be no - # corresponding archive file. - # The caller code will deal with the error check further. - return True - - # Sometimes on macOS indicator file appears prior to the actual archive - # despite the order of creation and os.sync() used in tag_ready(). - # So consider archive not ready if there is an indicator without an - # actual archive. - if not self.archive_filepath.exists(): - print('Found indicator without actual archive, waiting for archive ' - f'({self.archive_filepath}) to appear.') - return False - - # Wait for until archive is fully stored. - actual_archive_size = self.archive_filepath.stat().st_size - if actual_archive_size != archive_state.file_size: - print('Partial/invalid archive size (expected ' - f'{archive_state.file_size} got {actual_archive_size})') - return False - - return True - - def is_ready(self) -> bool: - """ - Check whether the archive is ready for access. - - Will tolerate possible network failures: if there is a network failure - or if there is still no proper permission on a file False is returned. - """ - - # There are some intermitten problem happening at a random which is - # translates to "OSError : [WinError 59] An unexpected network error occurred". - # Some reports suggests it might be due to lack of permissions to the file, - # which might be applicable in our case since it's possible that file is - # initially created with non-accessible permissions and gets chmod-ed - # after initial creation. - try: - return self.is_ready_unsafe() - except OSError as e: - print(f'Exception checking archive: {e}') - return False - - def tag_ready(self, error_message='') -> None: - """ - Tag the archive as ready by creating the corresponding indication file. - - NOTE: It is expected that the archive was never tagged as ready before - and that there are no subsequent tags of the same archive. - If it is violated, an assert will fail. - """ - assert not self.is_ready() - - # Try the best to make sure everything is synced to the file system, - # to avoid any possibility of stamp appearing on a network share prior to - # an actual file. - if util.get_current_platform() != util.Platform.WINDOWS: - os.sync() - - archive_size = -1 - if self.archive_filepath.exists(): - archive_size = self.archive_filepath.stat().st_size - - archive_info = ArchiveState( - file_size=archive_size, error_message=error_message) - - self.ready_indicator_filepath.write_text( - archive_info.serialize_to_string()) - - def get_state(self) -> ArchiveState: - """ - Get state object for this archive - - The state is read from the corresponding state file. - """ - - try: - return ArchiveState.deserialize_from_file(self.ready_indicator_filepath) - except ArchiveStateError as error: - return ArchiveState(error_message=f'Error in information format: {error}') - - def clean(self) -> None: - """ - Remove both archive and the ready indication file. - """ - util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath) - util.ensure_file_does_not_exist_or_die(self.archive_filepath) - - def is_fully_absent(self) -> bool: - """ - Check whether both archive and its ready indicator are absent. - Is used for a sanity check during code signing process by both - buildbot worker and signing server. - """ - return (not self.archive_filepath.exists() and - not self.ready_indicator_filepath.exists()) diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py deleted file mode 100644 index f045e9c8242..00000000000 --- a/build_files/buildbot/codesign/base_code_signer.py +++ /dev/null @@ -1,501 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Signing process overview. -# -# From buildbot worker side: -# - Files which needs to be signed are collected from either a directory to -# sign all signable files in there, or by filename of a single file to sign. -# - Those files gets packed into an archive and stored in a location location -# which is watched by the signing server. -# - A marker READY file is created which indicates the archive is ready for -# access. -# - Wait for the server to provide an archive with signed files. -# This is done by watching for the READY file which corresponds to an archive -# coming from the signing server. -# - Unpack the signed signed files from the archives and replace original ones. -# -# From code sign server: -# - Watch special location for a READY file which indicates the there is an -# archive with files which are to be signed. -# - Unpack the archive to a temporary location. -# - Run codesign tool and make sure all the files are signed. -# - Pack the signed files and store them in a location which is watched by -# the buildbot worker. -# - Create a READY file which indicates that the archive with signed files is -# ready. - -import abc -import logging -import shutil -import subprocess -import time -import tarfile -import uuid - -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Iterable, List - -import codesign.util as util - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.archive_with_indicator import ArchiveWithIndicator -from codesign.exception import CodeSignException - - -logger = logging.getLogger(__name__) -logger_builder = logger.getChild('builder') -logger_server = logger.getChild('server') - - -def pack_files(files: Iterable[AbsoluteAndRelativeFileName], - archive_filepath: Path) -> None: - """ - Create tar archive from given files for the signing pipeline. - Is used by buildbot worker to create an archive of files which are to be - signed, and by signing server to send signed files back to the worker. - """ - with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle: - for file_info in files: - tar_file_handle.add(file_info.absolute_filepath, - arcname=file_info.relative_filepath) - - -def extract_files(archive_filepath: Path, - extraction_dir: Path) -> None: - """ - Extract all files form the given archive into the given direcotry. - """ - - # TODO(sergey): Verify files in the archive have relative path. - - with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle: - tar_file_handle.extractall(path=extraction_dir) - - -class BaseCodeSigner(metaclass=abc.ABCMeta): - """ - Base class for a platform-specific signer of binaries. - - Contains all the logic shared across platform-specific implementations, such - as synchronization and notification logic. - - Platform specific bits (such as actual command for signing the binary) are - to be implemented as a subclass. - - Provides utilities code signing as a whole, including functionality needed - by a signing server and a buildbot worker. - - The signer and builder may run on separate machines, the only requirement is - that they have access to a directory which is shared between them. For the - security concerns this is to be done as a separate machine (or as a Shared - Folder configuration in VirtualBox configuration). This directory might be - mounted under different base paths, but its underlying storage is to be - the same. - - The code signer is short-lived on a buildbot worker side, and is living - forever on a code signing server side. - """ - - # TODO(sergey): Find a neat way to have config annotated. - # config: Config - - # Storage directory where builder puts files which are requested to be - # signed. - # Consider this an input of the code signing server. - unsigned_storage_dir: Path - - # Storage where signed files are stored. - # Consider this an output of the code signer server. - signed_storage_dir: Path - - # Platform the code is currently executing on. - platform: util.Platform - - def __init__(self, config): - self.config = config - - absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() - - # Unsigned (signing server input) configuration. - self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' - - # Signed (signing server output) configuration. - self.signed_storage_dir = absolute_shared_storage_dir / 'signed' - - self.platform = util.get_current_platform() - - def cleanup_environment_for_builder(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def cleanup_environment_for_signing_server(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def generate_request_id(self) -> str: - """ - Generate an unique identifier for code signing request. - """ - return str(uuid.uuid4()) - - def archive_info_for_request_id( - self, path: Path, request_id: str) -> ArchiveWithIndicator: - return ArchiveWithIndicator( - path, f'{request_id}.tar', f'{request_id}.ready') - - def signed_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.signed_storage_dir, request_id) - - def unsigned_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.unsigned_storage_dir, request_id) - - ############################################################################ - # Buildbot worker side helpers. - - @abc.abstractmethod - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is to be signed. - - Is used by both single file signing pipeline and recursive directory - signing pipeline. - - This is where code signer is to check whether file is to be signed or - not. This check might be based on a simple extension test or on actual - test whether file have a digital signature already or not. - """ - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - """ - Get all files which need to be signed from the given path. - - NOTE: The path might either be a file or directory. - - This function is run from the buildbot worker side. - """ - - # If there is a single file provided trust the buildbot worker that it - # is eligible for signing. - if path.is_file(): - file = AbsoluteAndRelativeFileName.from_path(path) - if not self.check_file_is_to_be_signed(file): - return [] - return [file] - - all_files = AbsoluteAndRelativeFileName.recursively_from_directory( - path) - files_to_be_signed = [file for file in all_files - if self.check_file_is_to_be_signed(file)] - return files_to_be_signed - - def wait_for_signed_archive_or_die(self, request_id) -> None: - """ - Wait until archive with signed files is available. - - Will only return if the archive with signed files is available. If there - was an error during code sign procedure the SystemExit exception is - raised, with the message set to the error reported by the codesign - server. - - Will only wait for the configured time. If that time exceeds and there - is still no responce from the signing server the application will exit - with a non-zero exit code. - - """ - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS - time_start = time.monotonic() - while not signed_archive_info.is_ready(): - time.sleep(1) - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit("Signing server didn't finish signing in " - f'{timeout_in_seconds} seconds, dying :(') - - archive_state = signed_archive_info.get_state() - if archive_state.has_error(): - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit( - f'Error happenned during codesign procedure: {archive_state.error_message}') - - def copy_signed_files_to_directory( - self, signed_dir: Path, destination_dir: Path) -> None: - """ - Copy all files from signed_dir to destination_dir. - - This function will overwrite any existing file. Permissions are copied - from the source files, but other metadata, such as timestamps, are not. - """ - for signed_filepath in signed_dir.glob('**/*'): - if not signed_filepath.is_file(): - continue - - relative_filepath = signed_filepath.relative_to(signed_dir) - destination_filepath = destination_dir / relative_filepath - destination_filepath.parent.mkdir(parents=True, exist_ok=True) - - shutil.copy(signed_filepath, destination_filepath) - - def run_buildbot_path_sign_pipeline(self, path: Path) -> None: - """ - Run all steps needed to make given path signed. - - Path points to an unsigned file or a directory which contains unsigned - files. - - If the path points to a single file then this file will be signed. - This is used to sign a final bundle such as .msi on Windows or .dmg on - macOS. - - NOTE: The code signed implementation might actually reject signing the - file, in which case the file will be left unsigned. This isn't anything - to be considered a failure situation, just might happen when buildbot - worker can not detect whether signing is really required in a specific - case or not. - - If the path points to a directory then code signer will sign all - signable files from it (finding them recursively). - """ - - self.cleanup_environment_for_builder() - - # Make sure storage directory exists. - self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) - - # Collect all files which needs to be signed and pack them into a single - # archive which will be sent to the signing server. - logger_builder.info('Collecting files which are to be signed...') - files = self.collect_files_to_sign(path) - if not files: - logger_builder.info('No files to be signed, ignoring.') - return - logger_builder.info('Found %d files to sign.', len(files)) - - request_id = self.generate_request_id() - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - pack_files(files=files, - archive_filepath=unsigned_archive_info.archive_filepath) - unsigned_archive_info.tag_ready() - - # Wait for the signing server to finish signing. - logger_builder.info('Waiting signing server to sign the files...') - self.wait_for_signed_archive_or_die(request_id) - - # Extract signed files from archive and move files to final location. - with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: - unpacked_signed_files_dir = Path(temp_dir_str) - - logger_builder.info('Extracting signed files from archive...') - extract_files( - archive_filepath=signed_archive_info.archive_filepath, - extraction_dir=unpacked_signed_files_dir) - - destination_dir = path - if destination_dir.is_file(): - destination_dir = destination_dir.parent - self.copy_signed_files_to_directory( - unpacked_signed_files_dir, destination_dir) - - logger_builder.info('Removing archive with signed files...') - signed_archive_info.clean() - - ############################################################################ - # Signing server side helpers. - - def wait_for_sign_request(self) -> str: - """ - Wait for the buildbot to request signing of an archive. - - Returns an identifier of signing request. - """ - - # TOOD(sergey): Support graceful shutdown on Ctrl-C. - - logger_server.info( - f'Waiting for a request directory {self.unsigned_storage_dir} to appear.') - while not self.unsigned_storage_dir.exists(): - time.sleep(1) - - logger_server.info( - 'Waiting for a READY indicator of any signing request.') - request_id = None - while request_id is None: - for file in self.unsigned_storage_dir.iterdir(): - if file.suffix != '.ready': - continue - request_id = file.stem - logger_server.info(f'Found READY for request ID {request_id}.') - if request_id is None: - time.sleep(1) - - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - while not unsigned_archive_info.is_ready(): - time.sleep(1) - - return request_id - - @abc.abstractmethod - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Sign all files in the given directory. - - NOTE: Signing should happen in-place. - """ - - def run_signing_pipeline(self, request_id: str): - """ - Run the full signing pipeline starting from the point when buildbot - worker have requested signing. - """ - - # Make sure storage directory exists. - self.signed_storage_dir.mkdir(parents=True, exist_ok=True) - - with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: - temp_dir = Path(temp_dir_str) - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - logger_server.info('Extracting unsigned files from archive...') - extract_files( - archive_filepath=unsigned_archive_info.archive_filepath, - extraction_dir=temp_dir) - - logger_server.info('Collecting all files which needs signing...') - files = AbsoluteAndRelativeFileName.recursively_from_directory( - temp_dir) - - logger_server.info('Signing all requested files...') - try: - self.sign_all_files(files) - except CodeSignException as error: - signed_archive_info.tag_ready(error_message=error.message) - unsigned_archive_info.clean() - logger_server.info('Signing is complete with errors.') - return - - logger_server.info('Packing signed files...') - pack_files(files=files, - archive_filepath=signed_archive_info.archive_filepath) - signed_archive_info.tag_ready() - - logger_server.info('Removing signing request...') - unsigned_archive_info.clean() - - logger_server.info('Signing is complete.') - - def run_signing_server(self): - logger_server.info('Starting new code signing server...') - self.cleanup_environment_for_signing_server() - logger_server.info('Code signing server is ready') - while True: - logger_server.info('Waiting for the signing request in %s...', - self.unsigned_storage_dir) - request_id = self.wait_for_sign_request() - - logger_server.info( - f'Beging signign procedure for request ID {request_id}.') - self.run_signing_pipeline(request_id) - - ############################################################################ - # Command executing. - # - # Abstracted to a degree that allows to run commands from a foreign - # platform. - # The goal with this is to allow performing dry-run tests of code signer - # server from other platforms (for example, to test that macOS code signer - # does what it is supposed to after doing a refactor on Linux). - - # TODO(sergey): What is the type annotation for the command? - def run_command_or_mock(self, command, platform: util.Platform) -> None: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - logger_server.info(f'Running command: {command}') - subprocess.run(command) - - # TODO(sergey): What is the type annotation for the command? - def check_output_or_mock(self, command, - platform: util.Platform, - allow_nonzero_exit_code=False) -> str: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - - If allow_nonzero_exit_code is truth then the output will be returned - even if application quit with non-zero exit code. - Otherwise an subprocess.CalledProcessError exception will be raised - in such case. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - if allow_nonzero_exit_code: - process = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = process.communicate()[0] - return output.decode() - - logger_server.info(f'Running command: {command}') - return subprocess.check_output( - command, stderr=subprocess.STDOUT).decode() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py deleted file mode 100644 index 1f41619ba13..00000000000 --- a/build_files/buildbot/codesign/config_builder.py +++ /dev/null @@ -1,62 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code running from -# buildbot's worker. - -import sys - -from pathlib import Path - -import codesign.util as util - -from codesign.config_common import * - -platform = util.get_current_platform() -if platform == util.Platform.LINUX: - SHARED_STORAGE_DIR = Path('/data/codesign') -elif platform == util.Platform.WINDOWS: - SHARED_STORAGE_DIR = Path('Z:\\codesign') -elif platform == util.Platform.MACOS: - SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign') - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py deleted file mode 100644 index a37bc731dc0..00000000000 --- a/build_files/buildbot/codesign/config_common.py +++ /dev/null @@ -1,36 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from pathlib import Path - -# Timeout in seconds for the signing process. -# -# This is how long buildbot packing step will wait signing server to -# perform signing. -# -# NOTE: Notarization could take a long time, hence the rather high value -# here. Might consider using different timeout for different platforms. -TIMEOUT_IN_SECONDS = 45 * 60 * 60 - -# Directory which is shared across buildbot worker and signing server. -# -# This is where worker puts files requested for signing as well as where -# server puts signed files. -SHARED_STORAGE_DIR: Path diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py deleted file mode 100644 index ff97ed15fa5..00000000000 --- a/build_files/buildbot/codesign/config_server_template.py +++ /dev/null @@ -1,101 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code signing server. -# -# NOTE: DO NOT put any sensitive information here, put it in an actual -# configuration on the signing machine. - -from pathlib import Path - -from codesign.config_common import * - -CODESIGN_DIRECTORY = Path(__file__).absolute().parent -BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent - -################################################################################ -# Common configuration. - -# Directory where folders for codesign requests and signed result are stored. -# For example, /data/codesign -SHARED_STORAGE_DIR: Path - -################################################################################ -# macOS-specific configuration. - -MACOS_ENTITLEMENTS_FILE = \ - BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist' - -# Identity of the Developer ID Application certificate which is to be used for -# codesign tool. -# Use `security find-identity -v -p codesigning` to find the identity. -# -# NOTE: This identity is just an example from release/darwin/README.txt. -MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97' - -# User name (Apple ID) which will be used to request notarization. -MACOS_XCRUN_USERNAME = 'me@example.com' - -# One-time application password which will be used to request notarization. -MACOS_XCRUN_PASSWORD = '@keychain:altool-password' - -# Timeout in seconds within which the notarial office is supposed to reply. -MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60 - -################################################################################ -# Windows-specific configuration. - -# URL to the timestamping authority. -WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' - -# Full path to the certificate used for signing. -# -# The path and expected file format might vary depending on a platform. -# -# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look -# like Path('C:\\Secret\\Blender.pfx'). -WIN_CERTIFICATE_FILEPATH: Path - -################################################################################ -# Logging configuration, common for all platforms. - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/exception.py b/build_files/buildbot/codesign/exception.py deleted file mode 100644 index 6c8a9f262a5..00000000000 --- a/build_files/buildbot/codesign/exception.py +++ /dev/null @@ -1,26 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -class CodeSignException(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py deleted file mode 100644 index 04935f67832..00000000000 --- a/build_files/buildbot/codesign/linux_code_signer.py +++ /dev/null @@ -1,72 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# NOTE: This is a no-op signer (since there isn't really a procedure to sign -# Linux binaries yet). Used to debug and verify the code signing routines on -# a Linux environment. - -import logging - -from pathlib import Path -from typing import List - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - - -class LinuxCodeSigner(BaseCodeSigner): - def is_active(self) -> bool: - """ - Check whether this signer is active. - - if it is inactive, no files will be signed. - - Is used to be able to debug code signing pipeline on Linux, where there - is no code signing happening in the actual buildbot and release - environment. - """ - return False - - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath == Path('blender'): - return True - if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and - file.relative_filepath.name.startwith('python')): - return True - if file.relative_filepath.suffix == '.so': - return True - return False - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - if not self.is_active(): - return [] - - return super().collect_files_to_sign(path) - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - num_files = len(files) - for file_index, file in enumerate(files): - logger.info('Server: Signed file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) diff --git a/build_files/buildbot/codesign/macos_code_signer.py b/build_files/buildbot/codesign/macos_code_signer.py deleted file mode 100644 index f03dad8e1d6..00000000000 --- a/build_files/buildbot/codesign/macos_code_signer.py +++ /dev/null @@ -1,456 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging -import re -import stat -import subprocess -import time - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'} - -# Prefixes of a file (not directory) name which are to be signed. -# Used to sign extra executable files in Contents/Resources. -NAME_PREFIXES_TO_BE_SIGNED = {'python'} - - -class NotarizationException(CodeSignException): - pass - - -def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is coming from an .app bundle - """ - parts = file.relative_filepath.parts - if not parts: - return False - if not parts[0].endswith('.app'): - return False - return True - - -def get_bundle_from_file( - file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName: - """ - Get AbsoluteAndRelativeFileName descriptor of bundle - """ - assert(is_file_from_bundle(file)) - - parts = file.relative_filepath.parts - bundle_name = parts[0] - - base_dir = file.base_dir - bundle_filepath = file.base_dir / bundle_name - return AbsoluteAndRelativeFileName(base_dir, bundle_filepath) - - -def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether given file is an executable within an app bundle - """ - if not is_file_from_bundle(file): - return False - - parts = file.relative_filepath.parts - num_parts = len(parts) - if num_parts < 3: - return False - - if parts[1:3] != ('Contents', 'MacOS'): - return False - - return True - - -def xcrun_field_value_from_output(field: str, output: str) -> str: - """ - Get value of a given field from xcrun output. - - If field is not found empty string is returned. - """ - - field_prefix = field + ': ' - for line in output.splitlines(): - line = line.strip() - if line.startswith(field_prefix): - return line[len(field_prefix):] - return '' - - -class MacOSCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath.name.startswith('.'): - return False - - if is_bundle_executable_file(file): - return True - - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in NAME_PREFIXES_TO_BE_SIGNED): - return True - - mode = file.absolute_filepath.lstat().st_mode - if mode & stat.S_IXUSR != 0: - file_output = subprocess.check_output( - ("file", file.absolute_filepath)).decode() - if "64-bit executable" in file_output: - return True - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - # Include all files when signing app or dmg bundle: all the files are - # needed to do valid signature of bundle. - if path.name.endswith('.app'): - return AbsoluteAndRelativeFileName.recursively_from_directory(path) - if path.is_dir(): - files = [] - for child in path.iterdir(): - if child.name.endswith('.app'): - current_files = AbsoluteAndRelativeFileName.recursively_from_directory( - child) - else: - current_files = super().collect_files_to_sign(child) - for current_file in current_files: - files.append(AbsoluteAndRelativeFileName( - path, current_file.absolute_filepath)) - return files - return super().collect_files_to_sign(path) - - ############################################################################ - # Codesign. - - def codesign_remove_signature( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Make sure given file does not have codesign signature - - This is needed because codesigning is not possible for file which has - signature already. - """ - - logger_server.info( - 'Removing codesign signature from %s...', file.relative_filepath) - - command = ['codesign', '--remove-signature', file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_file( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Sign given file - - NOTE: File must not have any signatures. - """ - - logger_server.info( - 'Codesigning %s...', file.relative_filepath) - - entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE - command = ['codesign', - '--timestamp', - '--options', 'runtime', - f'--entitlements={entitlements_file}', - '--sign', self.config.MACOS_CODESIGN_IDENTITY, - file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Run codesign tool on all eligible files in the given list. - - Will ignore all files which are not to be signed. For the rest will - remove possible existing signature and add a new signature. - """ - - num_files = len(files) - have_ignored_files = False - signed_files = [] - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of a bundle and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - have_ignored_files = True - continue - - logger_server.info( - 'Running codesigning routines for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - - self.codesign_remove_signature(file) - self.codesign_file(file) - - signed_files.append(file) - - if have_ignored_files: - logger_server.info('Signed %d files:', len(signed_files)) - num_signed_files = len(signed_files) - for file_index, signed_file in enumerate(signed_files): - logger_server.info( - '- [%d/%d] %s', - file_index + 1, num_signed_files, - signed_file.relative_filepath) - - def codesign_bundles( - self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Codesign all .app bundles in the given list of files. - - Bundle is deducted from paths of the files, and every bundle is only - signed once. - """ - - signed_bundles = set() - extra_files = [] - - for file in files: - if not is_file_from_bundle(file): - continue - bundle = get_bundle_from_file(file) - bundle_name = bundle.relative_filepath - if bundle_name in signed_bundles: - continue - - logger_server.info('Running codesign routines on bundle %s', - bundle_name) - - # It is not possible to remove signature from DMG. - if bundle.relative_filepath.name.endswith('.app'): - self.codesign_remove_signature(bundle) - self.codesign_file(bundle) - - signed_bundles.add(bundle_name) - - # Codesign on a bundle adds an extra folder with information. - # It needs to be compied to the source. - code_signature_directory = \ - bundle.absolute_filepath / 'Contents' / '_CodeSignature' - code_signature_files = \ - AbsoluteAndRelativeFileName.recursively_from_directory( - code_signature_directory) - for code_signature_file in code_signature_files: - bundle_relative_file = AbsoluteAndRelativeFileName( - bundle.base_dir, - code_signature_directory / - code_signature_file.relative_filepath) - extra_files.append(bundle_relative_file) - - files.extend(extra_files) - - ############################################################################ - # Notarization. - - def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str: - """ - Get bundle ID which will be used to notarize DMG - """ - name = file.relative_filepath.name - app_name = name.split('-', 2)[0].lower() - - app_name_words = app_name.split() - if len(app_name_words) > 1: - app_name_id = ''.join(word.capitalize() for word in app_name_words) - else: - app_name_id = app_name_words[0] - - # TODO(sergey): Consider using "alpha" for buildbot builds. - return f'org.blenderfoundation.{app_name_id}.release' - - def notarize_request(self, file) -> str: - """ - Request notarization of the given file. - - Returns UUID of the notarization request. If error occurred None is - returned instead of UUID. - """ - - bundle_id = self.notarize_get_bundle_id(file) - logger_server.info('Bundle ID: %s', bundle_id) - - logger_server.info('Submitting file to the notarial office.') - command = [ - 'xcrun', 'altool', '--notarize-app', '--verbose', - '-f', file.absolute_filepath, - '--primary-bundle-id', bundle_id, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - for line in output.splitlines(): - line = line.strip() - if line.startswith('RequestUUID = '): - request_uuid = line[14:] - return request_uuid - - # Check whether the package has been already submitted. - if 'The software asset has already been uploaded.' in line: - request_uuid = re.sub( - '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line) - logger_server.warning( - f'The package has been already submitted under UUID {request_uuid}') - return request_uuid - - logger_server.error(output) - logger_server.error('xcrun command did not report RequestUUID') - return None - - def notarize_review_status(self, xcrun_output: str) -> bool: - """ - Review status returned by xcrun's notarization info - - Returns truth if the notarization process has finished. - If there are errors during notarization, a NotarizationException() - exception is thrown with status message from the notarial office. - """ - - # Parse status and message - status = xcrun_field_value_from_output('Status', xcrun_output) - status_message = xcrun_field_value_from_output( - 'Status Message', xcrun_output) - - if status == 'success': - logger_server.info( - 'Package successfully notarized: %s', status_message) - return True - - if status == 'invalid': - logger_server.error(xcrun_output) - logger_server.error( - 'Package notarization has failed: %s', status_message) - raise NotarizationException(status_message) - - if status == 'in progress': - return False - - logger_server.info( - 'Unknown notarization status %s (%s)', status, status_message) - - return False - - def notarize_wait_result(self, request_uuid: str) -> None: - """ - Wait for until notarial office have a reply - """ - - logger_server.info( - 'Waiting for a result from the notarization office.') - - command = ['xcrun', 'altool', - '--notarization-info', request_uuid, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - time_start = time.monotonic() - timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS - - while True: - xcrun_output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - if self.notarize_review_status(xcrun_output): - break - - logger_server.info('Keep waiting for notarization office.') - time.sleep(30) - - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - logger_server.error( - "Notarial office didn't reply in %f seconds.", - timeout_in_seconds) - - def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Staple notarial label on the file - """ - - logger_server.info('Stapling notarial stamp.') - - command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath] - self.check_output_or_mock(command, util.Platform.MACOS) - - def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Run entire pipeline to get DMG notarized. - """ - logger_server.info('Begin notarization routines on %s', - file.relative_filepath) - - # Submit file for notarization. - request_uuid = self.notarize_request(file) - if not request_uuid: - return False - logger_server.info('Received Request UUID: %s', request_uuid) - - # Wait for the status from the notarization office. - if not self.notarize_wait_result(request_uuid): - return False - - # Staple. - self.notarize_staple(file) - - def notarize_all_dmg( - self, files: List[AbsoluteAndRelativeFileName]) -> bool: - """ - Notarize all DMG images from the input. - - Images are supposed to be codesigned already. - """ - for file in files: - if not file.relative_filepath.name.endswith('.dmg'): - continue - if not self.check_file_is_to_be_signed(file): - continue - - self.notarize_dmg(file) - - ############################################################################ - # Entry point. - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # TODO(sergey): Handle errors somehow. - - self.codesign_all_files(files) - self.codesign_bundles(files) - self.notarize_all_dmg(files) diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py deleted file mode 100644 index 674d9e9ce9e..00000000000 --- a/build_files/buildbot/codesign/simple_code_signer.py +++ /dev/null @@ -1,52 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - - -import logging.config -import sys - -from pathlib import Path -from typing import Optional - -import codesign.config_builder -import codesign.util as util -from codesign.base_code_signer import BaseCodeSigner - - -class SimpleCodeSigner: - code_signer: Optional[BaseCodeSigner] - - def __init__(self): - platform = util.get_current_platform() - if platform == util.Platform.LINUX: - from codesign.linux_code_signer import LinuxCodeSigner - self.code_signer = LinuxCodeSigner(codesign.config_builder) - elif platform == util.Platform.MACOS: - from codesign.macos_code_signer import MacOSCodeSigner - self.code_signer = MacOSCodeSigner(codesign.config_builder) - elif platform == util.Platform.WINDOWS: - from codesign.windows_code_signer import WindowsCodeSigner - self.code_signer = WindowsCodeSigner(codesign.config_builder) - else: - self.code_signer = None - - def sign_file_or_directory(self, path: Path) -> None: - logging.config.dictConfig(codesign.config_builder.LOGGING) - self.code_signer.run_buildbot_path_sign_pipeline(path) diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py deleted file mode 100644 index e67292dd049..00000000000 --- a/build_files/buildbot/codesign/util.py +++ /dev/null @@ -1,54 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import sys - -from enum import Enum -from pathlib import Path - - -class Platform(Enum): - LINUX = 1 - MACOS = 2 - WINDOWS = 3 - - -def get_current_platform() -> Platform: - if sys.platform == 'linux': - return Platform.LINUX - elif sys.platform == 'darwin': - return Platform.MACOS - elif sys.platform == 'win32': - return Platform.WINDOWS - raise Exception(f'Unknown platform {sys.platform}') - - -def ensure_file_does_not_exist_or_die(filepath: Path) -> None: - """ - If the file exists, unlink it. - If the file path exists and is not a file an assert will trigger. - If the file path does not exists nothing happens. - """ - if not filepath.exists(): - return - if not filepath.is_file(): - # TODO(sergey): Provide information about what the filepath actually is. - raise SystemExit(f'{filepath} is expected to be a file, but is not') - filepath.unlink() diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py deleted file mode 100644 index db185788a56..00000000000 --- a/build_files/buildbot/codesign/windows_code_signer.py +++ /dev/null @@ -1,117 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'} - -BLACKLIST_FILE_PREFIXES = ( - 'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime') - - -class SigntoolException(CodeSignException): - pass - -class WindowsCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in BLACKLIST_FILE_PREFIXES): - return False - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - - def get_sign_command_prefix(self) -> List[str]: - return [ - 'signtool', 'sign', '/v', - '/f', self.config.WIN_CERTIFICATE_FILEPATH, - '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL] - - - def run_codesign_tool(self, filepath: Path) -> None: - command = self.get_sign_command_prefix() + [filepath] - - try: - codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS) - except subprocess.CalledProcessError as e: - raise SigntoolException(f'Error running signtool {e}') - - logger_server.info(f'signtool output:\n{codesign_output}') - - got_number_of_success = False - - for line in codesign_output.split('\n'): - line_clean = line.strip() - line_clean_lower = line_clean.lower() - - if line_clean_lower.startswith('number of warnings') or \ - line_clean_lower.startswith('number of errors'): - number = int(line_clean_lower.split(':')[1]) - if number != 0: - raise SigntoolException('Non-clean success of signtool') - - if line_clean_lower.startswith('number of files successfully signed'): - got_number_of_success = True - number = int(line_clean_lower.split(':')[1]) - if number != 1: - raise SigntoolException('Signtool did not consider codesign a success') - - if not got_number_of_success: - raise SigntoolException('Signtool did not report number of files signed') - - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # NOTE: Sign files one by one to avoid possible command line length - # overflow (which could happen if we ever decide to sign every binary - # in the install folder, for example). - # - # TODO(sergey): Consider doing batched signing of handful of files in - # one go (but only if this actually known to be much faster). - num_files = len(files) - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of package and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - continue - - logger_server.info( - 'Running signtool command for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - self.run_codesign_tool(file.absolute_filepath) diff --git a/build_files/buildbot/codesign_server_linux.py b/build_files/buildbot/codesign_server_linux.py deleted file mode 100755 index be3065e640d..00000000000 --- a/build_files/buildbot/codesign_server_linux.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# NOTE: This is a no-op signer (since there isn't really a procedure to sign -# Linux binaries yet). Used to debug and verify the code signing routines on -# a Linux environment. - -import logging.config -from pathlib import Path -from typing import List - -from codesign.linux_code_signer import LinuxCodeSigner -import codesign.config_server - -if __name__ == "__main__": - logging.config.dictConfig(codesign.config_server.LOGGING) - code_signer = LinuxCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_macos.py b/build_files/buildbot/codesign_server_macos.py deleted file mode 100755 index 1bdb012fe67..00000000000 --- a/build_files/buildbot/codesign_server_macos.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging.config -from pathlib import Path -from typing import List - -from codesign.macos_code_signer import MacOSCodeSigner -import codesign.config_server - -if __name__ == "__main__": - entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE - if not entitlements_file.exists(): - raise SystemExit( - 'Entitlements file {entitlements_file} does not exist.') - if not entitlements_file.is_file(): - raise SystemExit( - 'Entitlements file {entitlements_file} is not a file.') - - logging.config.dictConfig(codesign.config_server.LOGGING) - code_signer = MacOSCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_windows.bat b/build_files/buildbot/codesign_server_windows.bat deleted file mode 100644 index 82680f30eb4..00000000000 --- a/build_files/buildbot/codesign_server_windows.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off - -rem This is an entry point of the codesign server for Windows. -rem It makes sure that signtool.exe is within the current PATH and can be -rem used by the Python script. - -SETLOCAL - -set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% - -codesign_server_windows.py diff --git a/build_files/buildbot/codesign_server_windows.py b/build_files/buildbot/codesign_server_windows.py deleted file mode 100755 index 97ea4fd6756..00000000000 --- a/build_files/buildbot/codesign_server_windows.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Implementation of codesign server for Windows. -# -# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat - -import logging.config -import shutil - -from pathlib import Path -from typing import List - -import codesign.util as util - -from codesign.windows_code_signer import WindowsCodeSigner -import codesign.config_server - -if __name__ == "__main__": - logging.config.dictConfig(codesign.config_server.LOGGING) - - logger = logging.getLogger(__name__) - logger_server = logger.getChild('server') - - # TODO(sergey): Consider moving such sanity checks into - # CodeSigner.check_environment_or_die(). - if not shutil.which('signtool.exe'): - if util.get_current_platform() == util.Platform.WINDOWS: - raise SystemExit("signtool.exe is not found in %PATH%") - logger_server.info( - 'signtool.exe not found, ' - 'but will not be used on this foreign platform') - - code_signer = WindowsCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/worker_bundle_dmg.py b/build_files/buildbot/worker_bundle_dmg.py deleted file mode 100755 index 56e0d7da88e..00000000000 --- a/build_files/buildbot/worker_bundle_dmg.py +++ /dev/null @@ -1,551 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -import argparse -import re -import shutil -import subprocess -import sys -import time - -from pathlib import Path -from tempfile import TemporaryDirectory, NamedTemporaryFile -from typing import List - -BUILDBOT_DIRECTORY = Path(__file__).absolute().parent -CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py' -BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent -DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' - - -# Extra size which is added on top of actual files size when estimating size -# of destination DNG. -EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 - -################################################################################ -# Common utilities - - -def get_directory_size(root_directory: Path) -> int: - """ - Get size of directory on disk - """ - - total_size = 0 - for file in root_directory.glob('**/*'): - total_size += file.lstat().st_size - return total_size - - -################################################################################ -# DMG bundling specific logic - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'source_dir', - type=Path, - help='Source directory which points to either existing .app bundle' - 'or to a directory with .app bundles.') - parser.add_argument( - '--background-image', - type=Path, - help="Optional background picture which will be set on the DMG." - "If not provided default Blender's one is used.") - parser.add_argument( - '--volume-name', - type=str, - help='Optional name of a volume which will be used for DMG.') - parser.add_argument( - '--dmg', - type=Path, - help='Optional argument which points to a final DMG file name.') - parser.add_argument( - '--applescript', - type=Path, - help="Optional path to applescript to set up folder looks of DMG." - "If not provided default Blender's one is used.") - parser.add_argument( - '--codesign', - action="store_true", - help="Code sign and notarize DMG contents.") - return parser - - -def collect_app_bundles(source_dir: Path) -> List[Path]: - """ - Collect all app bundles which are to be put into DMG - - If the source directory points to FOO.app it will be the only app bundle - packed. - - Otherwise all .app bundles from given directory are placed to a single - DMG. - """ - - if source_dir.name.endswith('.app'): - return [source_dir] - - app_bundles = [] - for filename in source_dir.glob('*'): - if not filename.is_dir(): - continue - if not filename.name.endswith('.app'): - continue - - app_bundles.append(filename) - - return app_bundles - - -def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: - app_bundles = collect_app_bundles(source_dir) - - if not app_bundles: - print('No app bundles found for packing') - return - - print(f'Found {len(app_bundles)} to pack:') - for app_bundle in app_bundles: - print(f'- {app_bundle}') - - return app_bundles - - -def estimate_dmg_size(app_bundles: List[Path]) -> int: - """ - Estimate size of DMG to hold requested app bundles - - The size is based on actual size of all files in all bundles plus some - space to compensate for different size-on-disk plus some space to hold - codesign signatures. - - Is better to be on a high side since the empty space is compressed, but - lack of space might cause silent failures later on. - """ - - app_bundles_size = 0 - for app_bundle in app_bundles: - app_bundles_size += get_directory_size(app_bundle) - - return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES - - -def copy_app_bundles_to_directory(app_bundles: List[Path], - directory: Path) -> None: - """ - Copy all bundles to a given directory - - This directory is what the DMG will be created from. - """ - for app_bundle in app_bundles: - print(f'Copying {app_bundle.name}...') - shutil.copytree(app_bundle, directory / app_bundle.name) - - -def get_main_app_bundle(app_bundles: List[Path]) -> Path: - """ - Get application bundle main for the installation - """ - return app_bundles[0] - - -def create_dmg_image(app_bundles: List[Path], - dmg_filepath: Path, - volume_name: str) -> None: - """ - Create DMG disk image and put app bundles in it - - No DMG configuration or codesigning is happening here. - """ - - if dmg_filepath.exists(): - print(f'Removing existing writable DMG {dmg_filepath}...') - dmg_filepath.unlink() - - print('Preparing directory with app bundles for the DMG...') - with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: - # Copy all bundles to a clean directory. - content_dir = Path(content_dir_str) - copy_app_bundles_to_directory(app_bundles, content_dir) - - # Estimate size of the DMG. - dmg_size = estimate_dmg_size(app_bundles) - print(f'Estimated DMG size: {dmg_size:,} bytes.') - - # Create the DMG. - print(f'Creating writable DMG {dmg_filepath}') - command = ('hdiutil', - 'create', - '-size', str(dmg_size), - '-fs', 'HFS+', - '-srcfolder', content_dir, - '-volname', volume_name, - '-format', 'UDRW', - dmg_filepath) - subprocess.run(command) - - -def get_writable_dmg_filepath(dmg_filepath: Path): - """ - Get file path for writable DMG image - """ - parent = dmg_filepath.parent - return parent / (dmg_filepath.stem + '-temp.dmg') - - -def mount_readwrite_dmg(dmg_filepath: Path) -> None: - """ - Mount writable DMG - - Mounting point would be /Volumes/<volume name> - """ - - print(f'Mounting read-write DMG ${dmg_filepath}') - command = ('hdiutil', - 'attach', '-readwrite', - '-noverify', - '-noautoopen', - dmg_filepath) - subprocess.run(command) - - -def get_mount_directory_for_volume_name(volume_name: str) -> Path: - """ - Get directory under which the volume will be mounted - """ - - return Path('/Volumes') / volume_name - - -def eject_volume(volume_name: str) -> None: - """ - Eject given volume, if mounted - """ - mount_directory = get_mount_directory_for_volume_name(volume_name) - if not mount_directory.exists(): - return - mount_directory_str = str(mount_directory) - - print(f'Ejecting volume {volume_name}') - - # Figure out which device to eject. - mount_output = subprocess.check_output(['mount']).decode() - device = '' - for line in mount_output.splitlines(): - if f'on {mount_directory_str} (' not in line: - continue - tokens = line.split(' ', 3) - if len(tokens) < 3: - continue - if tokens[1] != 'on': - continue - if device: - raise Exception( - f'Multiple devices found for mounting point {mount_directory}') - device = tokens[0] - - if not device: - raise Exception( - f'No device found for mounting point {mount_directory}') - - print(f'{mount_directory} is mounted as device {device}, ejecting...') - subprocess.run(['diskutil', 'eject', device]) - - -def copy_background_if_needed(background_image_filepath: Path, - mount_directory: Path) -> None: - """ - Copy background to the DMG - - If the background image is not specified it will not be copied. - """ - - if not background_image_filepath: - print('No background image provided.') - return - - print(f'Copying background image {background_image_filepath}') - - destination_dir = mount_directory / '.background' - destination_dir.mkdir(exist_ok=True) - - destination_filepath = destination_dir / background_image_filepath.name - shutil.copy(background_image_filepath, destination_filepath) - - -def create_applications_link(mount_directory: Path) -> None: - """ - Create link to /Applications in the given location - """ - - print('Creating link to /Applications') - - command = ('ln', '-s', '/Applications', mount_directory / ' ') - subprocess.run(command) - - -def run_applescript(applescript: Path, - volume_name: str, - app_bundles: List[Path], - background_image_filepath: Path) -> None: - """ - Run given applescript to adjust look and feel of the DMG - """ - - main_app_bundle = get_main_app_bundle(app_bundles) - - with NamedTemporaryFile( - mode='w', suffix='.applescript') as temp_applescript: - print('Adjusting applescript for volume name...') - # Adjust script to the specific volume name. - with open(applescript, mode='r') as input: - for line in input.readlines(): - stripped_line = line.strip() - if stripped_line.startswith('tell disk'): - line = re.sub('tell disk ".*"', - f'tell disk "{volume_name}"', - line) - elif stripped_line.startswith('set background picture'): - if not background_image_filepath: - continue - else: - background_image_short = \ - '.background:' + background_image_filepath.name - line = re.sub('to file ".*"', - f'to file "{background_image_short}"', - line) - line = line.replace('blender.app', main_app_bundle.name) - temp_applescript.write(line) - - temp_applescript.flush() - - print('Running applescript...') - command = ('osascript', temp_applescript.name) - subprocess.run(command) - - print('Waiting for applescript...') - - # NOTE: This is copied from bundle.sh. The exact reason for sleep is - # still remained a mystery. - time.sleep(5) - - -def codesign(subject: Path): - """ - Codesign file or directory - - NOTE: For DMG it will also notarize. - """ - - command = (CODESIGN_SCRIPT, subject) - subprocess.run(command) - - -def codesign_app_bundles_in_dmg(mount_directory: str) -> None: - """ - Code sign all binaries and bundles in the mounted directory - """ - - print(f'Codesigning all app bundles in {mount_directory}') - codesign(mount_directory) - - -def codesign_and_notarize_dmg(dmg_filepath: Path) -> None: - """ - Run codesign and notarization pipeline on the DMG - """ - - print(f'Codesigning and notarizing DMG {dmg_filepath}') - codesign(dmg_filepath) - - -def compress_dmg(writable_dmg_filepath: Path, - final_dmg_filepath: Path) -> None: - """ - Compress temporary read-write DMG - """ - command = ('hdiutil', 'convert', - writable_dmg_filepath, - '-format', 'UDZO', - '-o', final_dmg_filepath) - - if final_dmg_filepath.exists(): - print(f'Removing old compressed DMG {final_dmg_filepath}') - final_dmg_filepath.unlink() - - print('Compressing disk image...') - subprocess.run(command) - - -def create_final_dmg(app_bundles: List[Path], - dmg_filepath: Path, - background_image_filepath: Path, - volume_name: str, - applescript: Path, - codesign: bool) -> None: - """ - Create DMG with all app bundles - - Will take care configuring background, signing all binaries and app bundles - and notarizing the DMG. - """ - - print('Running all routines to create final DMG') - - writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath) - mount_directory = get_mount_directory_for_volume_name(volume_name) - - # Make sure volume is not mounted. - # If it is mounted it will prevent removing old DMG files and could make - # it so app bundles are copied to the wrong place. - eject_volume(volume_name) - - create_dmg_image(app_bundles, writable_dmg_filepath, volume_name) - - mount_readwrite_dmg(writable_dmg_filepath) - - # Run codesign first, prior to copying amything else. - # - # This allows to recurs into the content of bundles without worrying about - # possible interfereice of Application symlink. - if codesign: - codesign_app_bundles_in_dmg(mount_directory) - - copy_background_if_needed(background_image_filepath, mount_directory) - create_applications_link(mount_directory) - run_applescript(applescript, volume_name, app_bundles, - background_image_filepath) - - print('Ejecting read-write DMG image...') - eject_volume(volume_name) - - compress_dmg(writable_dmg_filepath, dmg_filepath) - writable_dmg_filepath.unlink() - - if codesign: - codesign_and_notarize_dmg(dmg_filepath) - - -def ensure_dmg_extension(filepath: Path) -> Path: - """ - Make sure given file have .dmg extension - """ - - if filepath.suffix != '.dmg': - return filepath.with_suffix(f'{filepath.suffix}.dmg') - return filepath - - -def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path: - """ - Get full file path for the final DMG image - - Will use the provided one when possible, otherwise will deduct it from - app bundles. - - If the name is deducted, the DMG is stored in the current directory. - """ - - if requested_name: - return ensure_dmg_extension(requested_name.absolute()) - - # TODO(sergey): This is not necessarily the main one. - main_bundle = app_bundles[0] - # Strip .app from the name - return Path(main_bundle.name[:-4] + '.dmg').absolute() - - -def get_background_image(requested_background_image: Path) -> Path: - """ - Get effective filepath for the background image - """ - - if requested_background_image: - return requested_background_image.absolute() - - return DARWIN_DIRECTORY / 'background.tif' - - -def get_applescript(requested_applescript: Path) -> Path: - """ - Get effective filepath for the applescript - """ - - if requested_applescript: - return requested_applescript.absolute() - - return DARWIN_DIRECTORY / 'blender.applescript' - - -def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str: - """ - Deduct volume name from the DMG path - - Will use first part of the DMG file name prior to dash. - """ - - tokens = dmg_filepath.stem.split('-') - words = tokens[0].split() - - return ' '.join(word.capitalize() for word in words) - - -def get_volume_name(requested_volume_name: str, - dmg_filepath: Path) -> str: - """ - Get effective name for DMG volume - """ - - if requested_volume_name: - return requested_volume_name - - return get_volume_name_from_dmg_filepath(dmg_filepath) - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - - # Get normalized input parameters. - source_dir = args.source_dir.absolute() - background_image_filepath = get_background_image(args.background_image) - applescript = get_applescript(args.applescript) - codesign = args.codesign - - app_bundles = collect_and_log_app_bundles(source_dir) - if not app_bundles: - return - - dmg_filepath = get_dmg_filepath(args.dmg, app_bundles) - volume_name = get_volume_name(args.volume_name, dmg_filepath) - - print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)') - - create_final_dmg(app_bundles, - dmg_filepath, - background_image_filepath, - volume_name, - applescript, - codesign) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/worker_codesign.cmake b/build_files/buildbot/worker_codesign.cmake deleted file mode 100644 index f37feaef407..00000000000 --- a/build_files/buildbot/worker_codesign.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# This is a script which is used as POST-INSTALL one for regular CMake's -# INSTALL target. -# It is used by buildbot workers to sign every binary which is going into -# the final buundle. - -# On Windows Python 3 there only is python.exe, no python3.exe. -# -# On other platforms it is possible to have python2 and python3, and a -# symbolic link to python to either of them. So on those platforms use -# an explicit Python version. -if(WIN32) - set(PYTHON_EXECUTABLE python) -else() - set(PYTHON_EXECUTABLE python3) -endif() - -execute_process( - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py" - "${CMAKE_INSTALL_PREFIX}" - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE exit_code -) - -if(NOT exit_code EQUAL "0") - message(FATAL_ERROR "Non-zero exit code of codesign tool") -endif() diff --git a/build_files/buildbot/worker_codesign.py b/build_files/buildbot/worker_codesign.py deleted file mode 100755 index a82ee98b1b5..00000000000 --- a/build_files/buildbot/worker_codesign.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# Helper script which takes care of signing provided location. -# -# The location can either be a directory (in which case all eligible binaries -# will be signed) or a single file (in which case a single file will be signed). -# -# This script takes care of all the complexity of communicating between process -# which requests file to be signed and the code signing server. -# -# NOTE: Signing happens in-place. - -import argparse -import sys - -from pathlib import Path - -from codesign.simple_code_signer import SimpleCodeSigner - - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('path_to_sign', type=Path) - return parser - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - path_to_sign = args.path_to_sign.absolute() - - if sys.platform == 'win32': - # When WIX packed is used to generate .msi on Windows the CPack will - # install two different projects and install them to different - # installation prefix: - # - # - C:\b\build\_CPack_Packages\WIX\Blender - # - C:\b\build\_CPack_Packages\WIX\Unspecified - # - # Annoying part is: CMake's post-install script will only be run - # once, with the install prefix which corresponds to a project which - # was installed last. But we want to sign binaries from all projects. - # So in order to do so we detect that we are running for a CPack's - # project used for WIX and force parent directory (which includes both - # projects) to be signed. - # - # Here we force both projects to be signed. - if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign): - path_to_sign = path_to_sign.parent - - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(path_to_sign) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/worker_compile.py b/build_files/buildbot/worker_compile.py deleted file mode 100644 index 8c6b44c5866..00000000000 --- a/build_files/buildbot/worker_compile.py +++ /dev/null @@ -1,135 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import os -import shutil - -import buildbot_utils - - -def get_cmake_options(builder): - codesign_script = os.path.join( - builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake') - - config_file = "build_files/cmake/config/blender_release.cmake" - options = ['-DCMAKE_BUILD_TYPE:STRING=Release', - '-DWITH_GTESTS=ON'] - - if builder.platform == 'mac': - options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') - options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') - elif builder.platform == 'win': - options.extend(['-G', 'Visual Studio 16 2019', '-A', 'x64']) - if builder.codesign: - options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + codesign_script]) - elif builder.platform == 'linux': - config_file = "build_files/buildbot/config/blender_linux.cmake" - - optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK-7.1') - options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir) - - # Workaround to build sm_30 kernels with CUDA 10, since CUDA 11 no longer supports that architecture - if builder.platform == 'win': - options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1') - options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe') - options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1') - options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1/bin/nvcc.exe') - elif builder.platform == 'linux': - options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.1') - options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.1/bin/nvcc') - options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-11.1') - options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-11.1/bin/nvcc') - - options.append("-C" + os.path.join(builder.blender_dir, config_file)) - options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir)) - - return options - - -def update_git(builder): - # Do extra git fetch because not all platform/git/buildbot combinations - # update the origin remote, causing buildinfo to detect local changes. - os.chdir(builder.blender_dir) - - print("Fetching remotes") - command = ['git', 'fetch', '--all'] - buildbot_utils.call(builder.command_prefix + command) - - -def clean_directories(builder): - # Make sure no garbage remained from the previous run - if os.path.isdir(builder.install_dir): - shutil.rmtree(builder.install_dir) - - # Make sure build directory exists and enter it - os.makedirs(builder.build_dir, exist_ok=True) - - # Remove buildinfo files to force buildbot to re-generate them. - for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): - full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo) - if os.path.exists(full_path): - print("Removing {}" . format(buildinfo)) - os.remove(full_path) - - -def cmake_configure(builder): - # CMake configuration - os.chdir(builder.build_dir) - - cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt') - if os.path.exists(cmake_cache): - print("Removing CMake cache") - os.remove(cmake_cache) - - print("CMake configure:") - cmake_options = get_cmake_options(builder) - command = ['cmake', builder.blender_dir] + cmake_options - buildbot_utils.call(builder.command_prefix + command) - - -def cmake_build(builder): - # CMake build - os.chdir(builder.build_dir) - - # NOTE: CPack will build an INSTALL target, which would mean that code - # signing will happen twice when using `make install` and CPack. - # The tricky bit here is that it is not possible to know whether INSTALL - # target is used by CPack or by a buildbot itaself. Extra level on top of - # this is that on Windows it is required to build INSTALL target in order - # to have unit test binaries to run. - # So on the one hand we do an extra unneeded code sign on Windows, but on - # a positive side we don't add complexity and don't make build process more - # fragile trying to avoid this. The signing process is way faster than just - # a clean build of buildbot, especially with regression tests enabled. - if builder.platform == 'win': - command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] - else: - command = ['make', '-s', '-j16', 'install'] - - print("CMake build:") - buildbot_utils.call(builder.command_prefix + command) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - update_git(builder) - clean_directories(builder) - cmake_configure(builder) - cmake_build(builder) diff --git a/build_files/buildbot/worker_pack.py b/build_files/buildbot/worker_pack.py deleted file mode 100644 index a5727a66e09..00000000000 --- a/build_files/buildbot/worker_pack.py +++ /dev/null @@ -1,208 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Runs on buildbot worker, creating a release package using the build -# system and zipping it into buildbot_upload.zip. This is then uploaded -# to the master in the next buildbot step. - -import os -import sys - -from pathlib import Path - -import buildbot_utils - - -def get_package_name(builder, platform=None): - info = buildbot_utils.VersionInfo(builder) - - package_name = 'blender-' + info.full_version - if platform: - package_name += '-' + platform - if not (builder.branch == 'master' or builder.is_release_branch): - if info.is_development_build: - package_name = builder.branch + "-" + package_name - - return package_name - - -def sign_file_or_directory(path): - from codesign.simple_code_signer import SimpleCodeSigner - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(Path(path)) - - -def create_buildbot_upload_zip(builder, package_files): - import zipfile - - buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip") - if os.path.exists(buildbot_upload_zip): - os.remove(buildbot_upload_zip) - - try: - z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) - for filepath, filename in package_files: - print("Packaged", filename) - z.write(filepath, arcname=filename) - z.close() - except Exception as ex: - sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n') - sys.exit(1) - - -def create_tar_xz(src, dest, package_name): - # One extra to remove leading os.sep when cleaning root for package_root - ln = len(src) + 1 - flist = list() - - # Create list of tuples containing file and archive name - for root, dirs, files in os.walk(src): - package_root = os.path.join(package_name, root[ln:]) - flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files]) - - import tarfile - - # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever - # user compiled the package. If root then unpacks it to /usr/local/ you get - # a security issue. - def _fakeroot(tarinfo): - tarinfo.gid = 0 - tarinfo.gname = "root" - tarinfo.uid = 0 - tarinfo.uname = "root" - return tarinfo - - package = tarfile.open(dest, 'w:xz', preset=9) - for entry in flist: - package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) - package.close() - - -def cleanup_files(dirpath, extension): - for f in os.listdir(dirpath): - filepath = os.path.join(dirpath, f) - if os.path.isfile(filepath) and f.endswith(extension): - os.remove(filepath) - - -def pack_mac(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.dmg') - - package_name = get_package_name(builder, 'macOS') - package_filename = package_name + '.dmg' - package_filepath = os.path.join(builder.build_dir, package_filename) - - release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') - buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot') - bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py') - - command = [bundle_script] - command += ['--dmg', package_filepath] - if info.is_development_build: - background_image = os.path.join(release_dir, 'buildbot', 'background.tif') - command += ['--background-image', background_image] - if builder.codesign: - command += ['--codesign'] - command += [builder.install_dir] - buildbot_utils.call(command) - - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -def pack_win(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.zip') - - # CPack will add the platform name - cpack_name = get_package_name(builder, None) - package_name = get_package_name(builder, 'windows' + str(builder.bits)) - - command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.'] - buildbot_utils.call(builder.command_prefix + command) - command = ['cpack', '-G', 'ZIP'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.zip' - package_filepath = os.path.join(builder.build_dir, package_filename) - package_files = [(package_filepath, package_filename)] - - if info.version_cycle == 'release': - # Installer only for final release builds, otherwise will get - # 'this product is already installed' messages. - command = ['cpack', '-G', 'WIX'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.msi' - package_filepath = os.path.join(builder.build_dir, package_filename) - if builder.codesign: - sign_file_or_directory(package_filepath) - - package_files += [(package_filepath, package_filename)] - - create_buildbot_upload_zip(builder, package_files) - - -def pack_linux(builder): - blender_executable = os.path.join(builder.install_dir, 'blender') - - info = buildbot_utils.VersionInfo(builder) - - # Strip all unused symbols from the binaries - print("Stripping binaries...") - buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable]) - - print("Stripping python...") - py_target = os.path.join(builder.install_dir, info.short_version) - buildbot_utils.call( - builder.command_prefix + [ - 'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';', - ], - ) - - # Construct package name - platform_name = 'linux64' - package_name = get_package_name(builder, platform_name) - package_filename = package_name + ".tar.xz" - - print("Creating .tar.xz archive") - package_filepath = builder.install_dir + '.tar.xz' - create_tar_xz(builder.install_dir, package_filepath, package_name) - - # Create buildbot_upload.zip - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - - # Make sure install directory always exists - os.makedirs(builder.install_dir, exist_ok=True) - - if builder.platform == 'mac': - pack_mac(builder) - elif builder.platform == 'win': - pack_win(builder) - elif builder.platform == 'linux': - pack_linux(builder) diff --git a/build_files/buildbot/worker_test.py b/build_files/buildbot/worker_test.py deleted file mode 100644 index 8a6e7d4bb69..00000000000 --- a/build_files/buildbot/worker_test.py +++ /dev/null @@ -1,42 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import buildbot_utils -import os -import sys - - -def get_ctest_arguments(builder): - args = ['--output-on-failure'] - if builder.platform == 'win': - args += ['-C', 'Release'] - return args - - -def test(builder): - os.chdir(builder.build_dir) - - command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder) - buildbot_utils.call(command) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - test(builder) diff --git a/build_files/buildbot/worker_update.py b/build_files/buildbot/worker_update.py deleted file mode 100644 index 36a7ae31c84..00000000000 --- a/build_files/buildbot/worker_update.py +++ /dev/null @@ -1,31 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import buildbot_utils -import os -import sys - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - os.chdir(builder.blender_dir) - - # Run make update which handles all libraries and submodules. - make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py") - buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"]) |