diff options
44 files changed, 134 insertions, 4435 deletions
diff --git a/build_files/buildbot/README.md b/build_files/buildbot/README.md index 06733c9a42d..f6fd07d9246 100644 --- a/build_files/buildbot/README.md +++ b/build_files/buildbot/README.md @@ -1,70 +1,4 @@ -Blender Buildbot -================ +Buildbot Configuration +===================== -Code signing ------------- - -Code signing is done as part of INSTALL target, which makes it possible to sign -files which are aimed into a bundle and coming from a non-signed source (such as -libraries SVN). - -This is achieved by specifying `worker_codesign.cmake` as a post-install script -run by CMake. This CMake script simply involves an utility script written in -Python which takes care of an actual signing. - -### Configuration - -Client configuration doesn't need anything special, other than variable -`SHARED_STORAGE_DIR` pointing to a location which is watched by a server. -This is done in `config_builder.py` file and is stored in Git (which makes it -possible to have almost zero-configuration buildbot machines). - -Server configuration requires copying `config_server_template.py` under the -name of `config_server.py` and tweaking values, which are platform-specific. - -#### Windows configuration - -There are two things which are needed on Windows in order to have code signing -to work: - -- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com -- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx). - -## Tips - -### Self-signed certificate on Windows - -It is easiest to test configuration using self-signed certificate. - -The certificate manipulation utilities are coming with Windows SDK. -Unfortunately, they are not added to PATH. Here is an example of how to make -sure they are easily available: - -``` -set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% -set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH% -``` - -Generate CA: - -``` -makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^ - -cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer -``` - -Import the generated CA: - -``` -certutil -user -addstore Root BlenderTestCA.cer -``` - -Create self-signed certificate and pack it into PKCS #12: - -``` -makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^ - -sky signature ^ - -ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^ - -sv BlenderTestSPC.pvk BlenderTestSPC.cer - -pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx -```
\ No newline at end of file +Files used by Buildbot's `compile-code` step. diff --git a/build_files/buildbot/buildbot_utils.py b/build_files/buildbot/buildbot_utils.py deleted file mode 100644 index 7e9858d9268..00000000000 --- a/build_files/buildbot/buildbot_utils.py +++ /dev/null @@ -1,127 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import argparse -import os -import re -import subprocess -import sys - - -def is_tool(name): - """Check whether `name` is on PATH and marked as executable.""" - - # from whichcraft import which - from shutil import which - - return which(name) is not None - - -class Builder: - def __init__(self, name, branch, codesign): - self.name = name - self.branch = branch - self.is_release_branch = re.match("^blender-v(.*)-release$", branch) is not None - self.codesign = codesign - - # Buildbot runs from build/ directory - self.blender_dir = os.path.abspath(os.path.join('..', 'blender.git')) - self.build_dir = os.path.abspath(os.path.join('..', 'build')) - self.install_dir = os.path.abspath(os.path.join('..', 'install')) - self.upload_dir = os.path.abspath(os.path.join('..', 'install')) - - # Detect platform - if name.startswith('mac'): - self.platform = 'mac' - self.command_prefix = [] - elif name.startswith('linux'): - self.platform = 'linux' - if is_tool('scl'): - self.command_prefix = ['scl', 'enable', 'devtoolset-9', '--'] - else: - self.command_prefix = [] - elif name.startswith('win'): - self.platform = 'win' - self.command_prefix = [] - else: - raise ValueError('Unkonw platform for builder ' + self.platform) - - # Always 64 bit now - self.bits = 64 - - -def create_builder_from_arguments(): - parser = argparse.ArgumentParser() - parser.add_argument('builder_name') - parser.add_argument('branch', default='master', nargs='?') - parser.add_argument("--codesign", action="store_true") - args = parser.parse_args() - return Builder(args.builder_name, args.branch, args.codesign) - - -class VersionInfo: - def __init__(self, builder): - # Get version information - buildinfo_h = os.path.join(builder.build_dir, "source", "creator", "buildinfo.h") - blender_h = os.path.join(builder.blender_dir, "source", "blender", "blenkernel", "BKE_blender_version.h") - - version_number = int(self._parse_header_file(blender_h, 'BLENDER_VERSION')) - version_number_patch = int(self._parse_header_file(blender_h, 'BLENDER_VERSION_PATCH')) - version_numbers = (version_number // 100, version_number % 100, version_number_patch) - self.short_version = "%d.%d" % (version_numbers[0], version_numbers[1]) - self.version = "%d.%d.%d" % version_numbers - self.version_cycle = self._parse_header_file(blender_h, 'BLENDER_VERSION_CYCLE') - self.hash = self._parse_header_file(buildinfo_h, 'BUILD_HASH')[1:-1] - - if self.version_cycle == "release": - # Final release - self.full_version = self.version - self.is_development_build = False - elif self.version_cycle == "rc": - # Release candidate - self.full_version = self.version + self.version_cycle - self.is_development_build = False - else: - # Development build - self.full_version = self.version + '-' + self.hash - self.is_development_build = True - - def _parse_header_file(self, filename, define): - import re - regex = re.compile(r"^#\s*define\s+%s\s+(.*)" % define) - with open(filename, "r") as file: - for l in file: - match = regex.match(l) - if match: - return match.group(1) - return None - - -def call(cmd, env=None, exit_on_error=True): - print(' '.join(cmd)) - - # Flush to ensure correct order output on Windows. - sys.stdout.flush() - sys.stderr.flush() - - retcode = subprocess.call(cmd, env=env) - if exit_on_error and retcode != 0: - sys.exit(retcode) - return retcode diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py deleted file mode 100644 index cb42710e785..00000000000 --- a/build_files/buildbot/codesign/absolute_and_relative_filename.py +++ /dev/null @@ -1,81 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from dataclasses import dataclass -from pathlib import Path -from typing import List - - -@dataclass -class AbsoluteAndRelativeFileName: - """ - Helper class which keeps track of absolute file path for a direct access and - corresponding relative path against given base. - - The relative part is used to construct a file name within an archive which - contains files which are to be signed or which has been signed already - (depending on whether the archive is addressed to signing server or back - to the buildbot worker). - """ - - # Base directory which is where relative_filepath is relative to. - base_dir: Path - - # Full absolute path of the corresponding file. - absolute_filepath: Path - - # Derived from full file path, contains part of the path which is relative - # to a desired base path. - relative_filepath: Path - - def __init__(self, base_dir: Path, filepath: Path): - self.base_dir = base_dir - self.absolute_filepath = filepath.resolve() - self.relative_filepath = self.absolute_filepath.relative_to( - self.base_dir) - - @classmethod - def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName': - assert path.is_absolute() - assert path.is_file() - - base_dir = path.parent - return AbsoluteAndRelativeFileName(base_dir, path) - - @classmethod - def recursively_from_directory(cls, base_dir: Path) \ - -> List['AbsoluteAndRelativeFileName']: - """ - Create list of AbsoluteAndRelativeFileName for all the files in the - given directory. - - NOTE: Result will be pointing to a resolved paths. - """ - assert base_dir.is_absolute() - assert base_dir.is_dir() - - base_dir = base_dir.resolve() - - result = [] - for filename in base_dir.glob('**/*'): - if not filename.is_file(): - continue - result.append(AbsoluteAndRelativeFileName(base_dir, filename)) - return result diff --git a/build_files/buildbot/codesign/archive_with_indicator.py b/build_files/buildbot/codesign/archive_with_indicator.py deleted file mode 100644 index aebf5a15417..00000000000 --- a/build_files/buildbot/codesign/archive_with_indicator.py +++ /dev/null @@ -1,245 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import dataclasses -import json -import os - -from pathlib import Path -from typing import Optional - -import codesign.util as util - - -class ArchiveStateError(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) - - -@dataclasses.dataclass -class ArchiveState: - """ - Additional information (state) of the archive - - Includes information like expected file size of the archive file in the case - the archive file is expected to be successfully created. - - If the archive can not be created, this state will contain error message - indicating details of error. - """ - - # Size in bytes of the corresponding archive. - file_size: Optional[int] = None - - # Non-empty value indicates that error has happenned. - error_message: str = '' - - def has_error(self) -> bool: - """ - Check whether the archive is at error state - """ - - return self.error_message - - def serialize_to_string(self) -> str: - payload = dataclasses.asdict(self) - return json.dumps(payload, sort_keys=True, indent=4) - - def serialize_to_file(self, filepath: Path) -> None: - string = self.serialize_to_string() - filepath.write_text(string) - - @classmethod - def deserialize_from_string(cls, string: str) -> 'ArchiveState': - try: - object_as_dict = json.loads(string) - except json.decoder.JSONDecodeError: - raise ArchiveStateError('Error parsing JSON') - - return cls(**object_as_dict) - - @classmethod - def deserialize_from_file(cls, filepath: Path): - string = filepath.read_text() - return cls.deserialize_from_string(string) - - -class ArchiveWithIndicator: - """ - The idea of this class is to wrap around logic which takes care of keeping - track of a name of an archive and synchronization routines between buildbot - worker and signing server. - - The synchronization is done based on creating a special file after the - archive file is knowingly ready for access. - """ - - # Base directory where the archive is stored (basically, a basename() of - # the absolute archive file name). - # - # For example, 'X:\\TEMP\\'. - base_dir: Path - - # Absolute file name of the archive. - # - # For example, 'X:\\TEMP\\FOO.ZIP'. - archive_filepath: Path - - # Absolute name of a file which acts as an indication of the fact that the - # archive is ready and is available for access. - # - # This is how synchronization between buildbot worker and signing server is - # done: - # - First, the archive is created under archive_filepath name. - # - Second, the indication file is created under ready_indicator_filepath - # name. - # - Third, the colleague of whoever created the indicator name watches for - # the indication file to appear, and once it's there it access the - # archive. - ready_indicator_filepath: Path - - def __init__( - self, base_dir: Path, archive_name: str, ready_indicator_name: str): - """ - Construct the object from given base directory and name of the archive - file: - ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY') - """ - - self.base_dir = base_dir - self.archive_filepath = self.base_dir / archive_name - self.ready_indicator_filepath = self.base_dir / ready_indicator_name - - def is_ready_unsafe(self) -> bool: - """ - Check whether the archive is ready for access. - - No guarding about possible network failres is done here. - """ - if not self.ready_indicator_filepath.exists(): - return False - - try: - archive_state = ArchiveState.deserialize_from_file( - self.ready_indicator_filepath) - except ArchiveStateError as error: - print(f'Error deserializing archive state: {error.message}') - return False - - if archive_state.has_error(): - # If the error did happen during codesign procedure there will be no - # corresponding archive file. - # The caller code will deal with the error check further. - return True - - # Sometimes on macOS indicator file appears prior to the actual archive - # despite the order of creation and os.sync() used in tag_ready(). - # So consider archive not ready if there is an indicator without an - # actual archive. - if not self.archive_filepath.exists(): - print('Found indicator without actual archive, waiting for archive ' - f'({self.archive_filepath}) to appear.') - return False - - # Wait for until archive is fully stored. - actual_archive_size = self.archive_filepath.stat().st_size - if actual_archive_size != archive_state.file_size: - print('Partial/invalid archive size (expected ' - f'{archive_state.file_size} got {actual_archive_size})') - return False - - return True - - def is_ready(self) -> bool: - """ - Check whether the archive is ready for access. - - Will tolerate possible network failures: if there is a network failure - or if there is still no proper permission on a file False is returned. - """ - - # There are some intermitten problem happening at a random which is - # translates to "OSError : [WinError 59] An unexpected network error occurred". - # Some reports suggests it might be due to lack of permissions to the file, - # which might be applicable in our case since it's possible that file is - # initially created with non-accessible permissions and gets chmod-ed - # after initial creation. - try: - return self.is_ready_unsafe() - except OSError as e: - print(f'Exception checking archive: {e}') - return False - - def tag_ready(self, error_message='') -> None: - """ - Tag the archive as ready by creating the corresponding indication file. - - NOTE: It is expected that the archive was never tagged as ready before - and that there are no subsequent tags of the same archive. - If it is violated, an assert will fail. - """ - assert not self.is_ready() - - # Try the best to make sure everything is synced to the file system, - # to avoid any possibility of stamp appearing on a network share prior to - # an actual file. - if util.get_current_platform() != util.Platform.WINDOWS: - os.sync() - - archive_size = -1 - if self.archive_filepath.exists(): - archive_size = self.archive_filepath.stat().st_size - - archive_info = ArchiveState( - file_size=archive_size, error_message=error_message) - - self.ready_indicator_filepath.write_text( - archive_info.serialize_to_string()) - - def get_state(self) -> ArchiveState: - """ - Get state object for this archive - - The state is read from the corresponding state file. - """ - - try: - return ArchiveState.deserialize_from_file(self.ready_indicator_filepath) - except ArchiveStateError as error: - return ArchiveState(error_message=f'Error in information format: {error}') - - def clean(self) -> None: - """ - Remove both archive and the ready indication file. - """ - util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath) - util.ensure_file_does_not_exist_or_die(self.archive_filepath) - - def is_fully_absent(self) -> bool: - """ - Check whether both archive and its ready indicator are absent. - Is used for a sanity check during code signing process by both - buildbot worker and signing server. - """ - return (not self.archive_filepath.exists() and - not self.ready_indicator_filepath.exists()) diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py deleted file mode 100644 index f045e9c8242..00000000000 --- a/build_files/buildbot/codesign/base_code_signer.py +++ /dev/null @@ -1,501 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Signing process overview. -# -# From buildbot worker side: -# - Files which needs to be signed are collected from either a directory to -# sign all signable files in there, or by filename of a single file to sign. -# - Those files gets packed into an archive and stored in a location location -# which is watched by the signing server. -# - A marker READY file is created which indicates the archive is ready for -# access. -# - Wait for the server to provide an archive with signed files. -# This is done by watching for the READY file which corresponds to an archive -# coming from the signing server. -# - Unpack the signed signed files from the archives and replace original ones. -# -# From code sign server: -# - Watch special location for a READY file which indicates the there is an -# archive with files which are to be signed. -# - Unpack the archive to a temporary location. -# - Run codesign tool and make sure all the files are signed. -# - Pack the signed files and store them in a location which is watched by -# the buildbot worker. -# - Create a READY file which indicates that the archive with signed files is -# ready. - -import abc -import logging -import shutil -import subprocess -import time -import tarfile -import uuid - -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Iterable, List - -import codesign.util as util - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.archive_with_indicator import ArchiveWithIndicator -from codesign.exception import CodeSignException - - -logger = logging.getLogger(__name__) -logger_builder = logger.getChild('builder') -logger_server = logger.getChild('server') - - -def pack_files(files: Iterable[AbsoluteAndRelativeFileName], - archive_filepath: Path) -> None: - """ - Create tar archive from given files for the signing pipeline. - Is used by buildbot worker to create an archive of files which are to be - signed, and by signing server to send signed files back to the worker. - """ - with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle: - for file_info in files: - tar_file_handle.add(file_info.absolute_filepath, - arcname=file_info.relative_filepath) - - -def extract_files(archive_filepath: Path, - extraction_dir: Path) -> None: - """ - Extract all files form the given archive into the given direcotry. - """ - - # TODO(sergey): Verify files in the archive have relative path. - - with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle: - tar_file_handle.extractall(path=extraction_dir) - - -class BaseCodeSigner(metaclass=abc.ABCMeta): - """ - Base class for a platform-specific signer of binaries. - - Contains all the logic shared across platform-specific implementations, such - as synchronization and notification logic. - - Platform specific bits (such as actual command for signing the binary) are - to be implemented as a subclass. - - Provides utilities code signing as a whole, including functionality needed - by a signing server and a buildbot worker. - - The signer and builder may run on separate machines, the only requirement is - that they have access to a directory which is shared between them. For the - security concerns this is to be done as a separate machine (or as a Shared - Folder configuration in VirtualBox configuration). This directory might be - mounted under different base paths, but its underlying storage is to be - the same. - - The code signer is short-lived on a buildbot worker side, and is living - forever on a code signing server side. - """ - - # TODO(sergey): Find a neat way to have config annotated. - # config: Config - - # Storage directory where builder puts files which are requested to be - # signed. - # Consider this an input of the code signing server. - unsigned_storage_dir: Path - - # Storage where signed files are stored. - # Consider this an output of the code signer server. - signed_storage_dir: Path - - # Platform the code is currently executing on. - platform: util.Platform - - def __init__(self, config): - self.config = config - - absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() - - # Unsigned (signing server input) configuration. - self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' - - # Signed (signing server output) configuration. - self.signed_storage_dir = absolute_shared_storage_dir / 'signed' - - self.platform = util.get_current_platform() - - def cleanup_environment_for_builder(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def cleanup_environment_for_signing_server(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def generate_request_id(self) -> str: - """ - Generate an unique identifier for code signing request. - """ - return str(uuid.uuid4()) - - def archive_info_for_request_id( - self, path: Path, request_id: str) -> ArchiveWithIndicator: - return ArchiveWithIndicator( - path, f'{request_id}.tar', f'{request_id}.ready') - - def signed_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.signed_storage_dir, request_id) - - def unsigned_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.unsigned_storage_dir, request_id) - - ############################################################################ - # Buildbot worker side helpers. - - @abc.abstractmethod - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is to be signed. - - Is used by both single file signing pipeline and recursive directory - signing pipeline. - - This is where code signer is to check whether file is to be signed or - not. This check might be based on a simple extension test or on actual - test whether file have a digital signature already or not. - """ - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - """ - Get all files which need to be signed from the given path. - - NOTE: The path might either be a file or directory. - - This function is run from the buildbot worker side. - """ - - # If there is a single file provided trust the buildbot worker that it - # is eligible for signing. - if path.is_file(): - file = AbsoluteAndRelativeFileName.from_path(path) - if not self.check_file_is_to_be_signed(file): - return [] - return [file] - - all_files = AbsoluteAndRelativeFileName.recursively_from_directory( - path) - files_to_be_signed = [file for file in all_files - if self.check_file_is_to_be_signed(file)] - return files_to_be_signed - - def wait_for_signed_archive_or_die(self, request_id) -> None: - """ - Wait until archive with signed files is available. - - Will only return if the archive with signed files is available. If there - was an error during code sign procedure the SystemExit exception is - raised, with the message set to the error reported by the codesign - server. - - Will only wait for the configured time. If that time exceeds and there - is still no responce from the signing server the application will exit - with a non-zero exit code. - - """ - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS - time_start = time.monotonic() - while not signed_archive_info.is_ready(): - time.sleep(1) - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit("Signing server didn't finish signing in " - f'{timeout_in_seconds} seconds, dying :(') - - archive_state = signed_archive_info.get_state() - if archive_state.has_error(): - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit( - f'Error happenned during codesign procedure: {archive_state.error_message}') - - def copy_signed_files_to_directory( - self, signed_dir: Path, destination_dir: Path) -> None: - """ - Copy all files from signed_dir to destination_dir. - - This function will overwrite any existing file. Permissions are copied - from the source files, but other metadata, such as timestamps, are not. - """ - for signed_filepath in signed_dir.glob('**/*'): - if not signed_filepath.is_file(): - continue - - relative_filepath = signed_filepath.relative_to(signed_dir) - destination_filepath = destination_dir / relative_filepath - destination_filepath.parent.mkdir(parents=True, exist_ok=True) - - shutil.copy(signed_filepath, destination_filepath) - - def run_buildbot_path_sign_pipeline(self, path: Path) -> None: - """ - Run all steps needed to make given path signed. - - Path points to an unsigned file or a directory which contains unsigned - files. - - If the path points to a single file then this file will be signed. - This is used to sign a final bundle such as .msi on Windows or .dmg on - macOS. - - NOTE: The code signed implementation might actually reject signing the - file, in which case the file will be left unsigned. This isn't anything - to be considered a failure situation, just might happen when buildbot - worker can not detect whether signing is really required in a specific - case or not. - - If the path points to a directory then code signer will sign all - signable files from it (finding them recursively). - """ - - self.cleanup_environment_for_builder() - - # Make sure storage directory exists. - self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) - - # Collect all files which needs to be signed and pack them into a single - # archive which will be sent to the signing server. - logger_builder.info('Collecting files which are to be signed...') - files = self.collect_files_to_sign(path) - if not files: - logger_builder.info('No files to be signed, ignoring.') - return - logger_builder.info('Found %d files to sign.', len(files)) - - request_id = self.generate_request_id() - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - pack_files(files=files, - archive_filepath=unsigned_archive_info.archive_filepath) - unsigned_archive_info.tag_ready() - - # Wait for the signing server to finish signing. - logger_builder.info('Waiting signing server to sign the files...') - self.wait_for_signed_archive_or_die(request_id) - - # Extract signed files from archive and move files to final location. - with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: - unpacked_signed_files_dir = Path(temp_dir_str) - - logger_builder.info('Extracting signed files from archive...') - extract_files( - archive_filepath=signed_archive_info.archive_filepath, - extraction_dir=unpacked_signed_files_dir) - - destination_dir = path - if destination_dir.is_file(): - destination_dir = destination_dir.parent - self.copy_signed_files_to_directory( - unpacked_signed_files_dir, destination_dir) - - logger_builder.info('Removing archive with signed files...') - signed_archive_info.clean() - - ############################################################################ - # Signing server side helpers. - - def wait_for_sign_request(self) -> str: - """ - Wait for the buildbot to request signing of an archive. - - Returns an identifier of signing request. - """ - - # TOOD(sergey): Support graceful shutdown on Ctrl-C. - - logger_server.info( - f'Waiting for a request directory {self.unsigned_storage_dir} to appear.') - while not self.unsigned_storage_dir.exists(): - time.sleep(1) - - logger_server.info( - 'Waiting for a READY indicator of any signing request.') - request_id = None - while request_id is None: - for file in self.unsigned_storage_dir.iterdir(): - if file.suffix != '.ready': - continue - request_id = file.stem - logger_server.info(f'Found READY for request ID {request_id}.') - if request_id is None: - time.sleep(1) - - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - while not unsigned_archive_info.is_ready(): - time.sleep(1) - - return request_id - - @abc.abstractmethod - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Sign all files in the given directory. - - NOTE: Signing should happen in-place. - """ - - def run_signing_pipeline(self, request_id: str): - """ - Run the full signing pipeline starting from the point when buildbot - worker have requested signing. - """ - - # Make sure storage directory exists. - self.signed_storage_dir.mkdir(parents=True, exist_ok=True) - - with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: - temp_dir = Path(temp_dir_str) - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - logger_server.info('Extracting unsigned files from archive...') - extract_files( - archive_filepath=unsigned_archive_info.archive_filepath, - extraction_dir=temp_dir) - - logger_server.info('Collecting all files which needs signing...') - files = AbsoluteAndRelativeFileName.recursively_from_directory( - temp_dir) - - logger_server.info('Signing all requested files...') - try: - self.sign_all_files(files) - except CodeSignException as error: - signed_archive_info.tag_ready(error_message=error.message) - unsigned_archive_info.clean() - logger_server.info('Signing is complete with errors.') - return - - logger_server.info('Packing signed files...') - pack_files(files=files, - archive_filepath=signed_archive_info.archive_filepath) - signed_archive_info.tag_ready() - - logger_server.info('Removing signing request...') - unsigned_archive_info.clean() - - logger_server.info('Signing is complete.') - - def run_signing_server(self): - logger_server.info('Starting new code signing server...') - self.cleanup_environment_for_signing_server() - logger_server.info('Code signing server is ready') - while True: - logger_server.info('Waiting for the signing request in %s...', - self.unsigned_storage_dir) - request_id = self.wait_for_sign_request() - - logger_server.info( - f'Beging signign procedure for request ID {request_id}.') - self.run_signing_pipeline(request_id) - - ############################################################################ - # Command executing. - # - # Abstracted to a degree that allows to run commands from a foreign - # platform. - # The goal with this is to allow performing dry-run tests of code signer - # server from other platforms (for example, to test that macOS code signer - # does what it is supposed to after doing a refactor on Linux). - - # TODO(sergey): What is the type annotation for the command? - def run_command_or_mock(self, command, platform: util.Platform) -> None: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - logger_server.info(f'Running command: {command}') - subprocess.run(command) - - # TODO(sergey): What is the type annotation for the command? - def check_output_or_mock(self, command, - platform: util.Platform, - allow_nonzero_exit_code=False) -> str: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - - If allow_nonzero_exit_code is truth then the output will be returned - even if application quit with non-zero exit code. - Otherwise an subprocess.CalledProcessError exception will be raised - in such case. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - if allow_nonzero_exit_code: - process = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = process.communicate()[0] - return output.decode() - - logger_server.info(f'Running command: {command}') - return subprocess.check_output( - command, stderr=subprocess.STDOUT).decode() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py deleted file mode 100644 index 1f41619ba13..00000000000 --- a/build_files/buildbot/codesign/config_builder.py +++ /dev/null @@ -1,62 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code running from -# buildbot's worker. - -import sys - -from pathlib import Path - -import codesign.util as util - -from codesign.config_common import * - -platform = util.get_current_platform() -if platform == util.Platform.LINUX: - SHARED_STORAGE_DIR = Path('/data/codesign') -elif platform == util.Platform.WINDOWS: - SHARED_STORAGE_DIR = Path('Z:\\codesign') -elif platform == util.Platform.MACOS: - SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign') - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py deleted file mode 100644 index a37bc731dc0..00000000000 --- a/build_files/buildbot/codesign/config_common.py +++ /dev/null @@ -1,36 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from pathlib import Path - -# Timeout in seconds for the signing process. -# -# This is how long buildbot packing step will wait signing server to -# perform signing. -# -# NOTE: Notarization could take a long time, hence the rather high value -# here. Might consider using different timeout for different platforms. -TIMEOUT_IN_SECONDS = 45 * 60 * 60 - -# Directory which is shared across buildbot worker and signing server. -# -# This is where worker puts files requested for signing as well as where -# server puts signed files. -SHARED_STORAGE_DIR: Path diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py deleted file mode 100644 index ff97ed15fa5..00000000000 --- a/build_files/buildbot/codesign/config_server_template.py +++ /dev/null @@ -1,101 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code signing server. -# -# NOTE: DO NOT put any sensitive information here, put it in an actual -# configuration on the signing machine. - -from pathlib import Path - -from codesign.config_common import * - -CODESIGN_DIRECTORY = Path(__file__).absolute().parent -BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent - -################################################################################ -# Common configuration. - -# Directory where folders for codesign requests and signed result are stored. -# For example, /data/codesign -SHARED_STORAGE_DIR: Path - -################################################################################ -# macOS-specific configuration. - -MACOS_ENTITLEMENTS_FILE = \ - BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist' - -# Identity of the Developer ID Application certificate which is to be used for -# codesign tool. -# Use `security find-identity -v -p codesigning` to find the identity. -# -# NOTE: This identity is just an example from release/darwin/README.txt. -MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97' - -# User name (Apple ID) which will be used to request notarization. -MACOS_XCRUN_USERNAME = 'me@example.com' - -# One-time application password which will be used to request notarization. -MACOS_XCRUN_PASSWORD = '@keychain:altool-password' - -# Timeout in seconds within which the notarial office is supposed to reply. -MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60 - -################################################################################ -# Windows-specific configuration. - -# URL to the timestamping authority. -WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' - -# Full path to the certificate used for signing. -# -# The path and expected file format might vary depending on a platform. -# -# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look -# like Path('C:\\Secret\\Blender.pfx'). -WIN_CERTIFICATE_FILEPATH: Path - -################################################################################ -# Logging configuration, common for all platforms. - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/exception.py b/build_files/buildbot/codesign/exception.py deleted file mode 100644 index 6c8a9f262a5..00000000000 --- a/build_files/buildbot/codesign/exception.py +++ /dev/null @@ -1,26 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -class CodeSignException(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py deleted file mode 100644 index 04935f67832..00000000000 --- a/build_files/buildbot/codesign/linux_code_signer.py +++ /dev/null @@ -1,72 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# NOTE: This is a no-op signer (since there isn't really a procedure to sign -# Linux binaries yet). Used to debug and verify the code signing routines on -# a Linux environment. - -import logging - -from pathlib import Path -from typing import List - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - - -class LinuxCodeSigner(BaseCodeSigner): - def is_active(self) -> bool: - """ - Check whether this signer is active. - - if it is inactive, no files will be signed. - - Is used to be able to debug code signing pipeline on Linux, where there - is no code signing happening in the actual buildbot and release - environment. - """ - return False - - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath == Path('blender'): - return True - if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and - file.relative_filepath.name.startwith('python')): - return True - if file.relative_filepath.suffix == '.so': - return True - return False - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - if not self.is_active(): - return [] - - return super().collect_files_to_sign(path) - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - num_files = len(files) - for file_index, file in enumerate(files): - logger.info('Server: Signed file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) diff --git a/build_files/buildbot/codesign/macos_code_signer.py b/build_files/buildbot/codesign/macos_code_signer.py deleted file mode 100644 index f03dad8e1d6..00000000000 --- a/build_files/buildbot/codesign/macos_code_signer.py +++ /dev/null @@ -1,456 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging -import re -import stat -import subprocess -import time - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'} - -# Prefixes of a file (not directory) name which are to be signed. -# Used to sign extra executable files in Contents/Resources. -NAME_PREFIXES_TO_BE_SIGNED = {'python'} - - -class NotarizationException(CodeSignException): - pass - - -def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is coming from an .app bundle - """ - parts = file.relative_filepath.parts - if not parts: - return False - if not parts[0].endswith('.app'): - return False - return True - - -def get_bundle_from_file( - file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName: - """ - Get AbsoluteAndRelativeFileName descriptor of bundle - """ - assert(is_file_from_bundle(file)) - - parts = file.relative_filepath.parts - bundle_name = parts[0] - - base_dir = file.base_dir - bundle_filepath = file.base_dir / bundle_name - return AbsoluteAndRelativeFileName(base_dir, bundle_filepath) - - -def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether given file is an executable within an app bundle - """ - if not is_file_from_bundle(file): - return False - - parts = file.relative_filepath.parts - num_parts = len(parts) - if num_parts < 3: - return False - - if parts[1:3] != ('Contents', 'MacOS'): - return False - - return True - - -def xcrun_field_value_from_output(field: str, output: str) -> str: - """ - Get value of a given field from xcrun output. - - If field is not found empty string is returned. - """ - - field_prefix = field + ': ' - for line in output.splitlines(): - line = line.strip() - if line.startswith(field_prefix): - return line[len(field_prefix):] - return '' - - -class MacOSCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath.name.startswith('.'): - return False - - if is_bundle_executable_file(file): - return True - - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in NAME_PREFIXES_TO_BE_SIGNED): - return True - - mode = file.absolute_filepath.lstat().st_mode - if mode & stat.S_IXUSR != 0: - file_output = subprocess.check_output( - ("file", file.absolute_filepath)).decode() - if "64-bit executable" in file_output: - return True - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - # Include all files when signing app or dmg bundle: all the files are - # needed to do valid signature of bundle. - if path.name.endswith('.app'): - return AbsoluteAndRelativeFileName.recursively_from_directory(path) - if path.is_dir(): - files = [] - for child in path.iterdir(): - if child.name.endswith('.app'): - current_files = AbsoluteAndRelativeFileName.recursively_from_directory( - child) - else: - current_files = super().collect_files_to_sign(child) - for current_file in current_files: - files.append(AbsoluteAndRelativeFileName( - path, current_file.absolute_filepath)) - return files - return super().collect_files_to_sign(path) - - ############################################################################ - # Codesign. - - def codesign_remove_signature( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Make sure given file does not have codesign signature - - This is needed because codesigning is not possible for file which has - signature already. - """ - - logger_server.info( - 'Removing codesign signature from %s...', file.relative_filepath) - - command = ['codesign', '--remove-signature', file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_file( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Sign given file - - NOTE: File must not have any signatures. - """ - - logger_server.info( - 'Codesigning %s...', file.relative_filepath) - - entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE - command = ['codesign', - '--timestamp', - '--options', 'runtime', - f'--entitlements={entitlements_file}', - '--sign', self.config.MACOS_CODESIGN_IDENTITY, - file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Run codesign tool on all eligible files in the given list. - - Will ignore all files which are not to be signed. For the rest will - remove possible existing signature and add a new signature. - """ - - num_files = len(files) - have_ignored_files = False - signed_files = [] - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of a bundle and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - have_ignored_files = True - continue - - logger_server.info( - 'Running codesigning routines for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - - self.codesign_remove_signature(file) - self.codesign_file(file) - - signed_files.append(file) - - if have_ignored_files: - logger_server.info('Signed %d files:', len(signed_files)) - num_signed_files = len(signed_files) - for file_index, signed_file in enumerate(signed_files): - logger_server.info( - '- [%d/%d] %s', - file_index + 1, num_signed_files, - signed_file.relative_filepath) - - def codesign_bundles( - self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Codesign all .app bundles in the given list of files. - - Bundle is deducted from paths of the files, and every bundle is only - signed once. - """ - - signed_bundles = set() - extra_files = [] - - for file in files: - if not is_file_from_bundle(file): - continue - bundle = get_bundle_from_file(file) - bundle_name = bundle.relative_filepath - if bundle_name in signed_bundles: - continue - - logger_server.info('Running codesign routines on bundle %s', - bundle_name) - - # It is not possible to remove signature from DMG. - if bundle.relative_filepath.name.endswith('.app'): - self.codesign_remove_signature(bundle) - self.codesign_file(bundle) - - signed_bundles.add(bundle_name) - - # Codesign on a bundle adds an extra folder with information. - # It needs to be compied to the source. - code_signature_directory = \ - bundle.absolute_filepath / 'Contents' / '_CodeSignature' - code_signature_files = \ - AbsoluteAndRelativeFileName.recursively_from_directory( - code_signature_directory) - for code_signature_file in code_signature_files: - bundle_relative_file = AbsoluteAndRelativeFileName( - bundle.base_dir, - code_signature_directory / - code_signature_file.relative_filepath) - extra_files.append(bundle_relative_file) - - files.extend(extra_files) - - ############################################################################ - # Notarization. - - def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str: - """ - Get bundle ID which will be used to notarize DMG - """ - name = file.relative_filepath.name - app_name = name.split('-', 2)[0].lower() - - app_name_words = app_name.split() - if len(app_name_words) > 1: - app_name_id = ''.join(word.capitalize() for word in app_name_words) - else: - app_name_id = app_name_words[0] - - # TODO(sergey): Consider using "alpha" for buildbot builds. - return f'org.blenderfoundation.{app_name_id}.release' - - def notarize_request(self, file) -> str: - """ - Request notarization of the given file. - - Returns UUID of the notarization request. If error occurred None is - returned instead of UUID. - """ - - bundle_id = self.notarize_get_bundle_id(file) - logger_server.info('Bundle ID: %s', bundle_id) - - logger_server.info('Submitting file to the notarial office.') - command = [ - 'xcrun', 'altool', '--notarize-app', '--verbose', - '-f', file.absolute_filepath, - '--primary-bundle-id', bundle_id, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - for line in output.splitlines(): - line = line.strip() - if line.startswith('RequestUUID = '): - request_uuid = line[14:] - return request_uuid - - # Check whether the package has been already submitted. - if 'The software asset has already been uploaded.' in line: - request_uuid = re.sub( - '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line) - logger_server.warning( - f'The package has been already submitted under UUID {request_uuid}') - return request_uuid - - logger_server.error(output) - logger_server.error('xcrun command did not report RequestUUID') - return None - - def notarize_review_status(self, xcrun_output: str) -> bool: - """ - Review status returned by xcrun's notarization info - - Returns truth if the notarization process has finished. - If there are errors during notarization, a NotarizationException() - exception is thrown with status message from the notarial office. - """ - - # Parse status and message - status = xcrun_field_value_from_output('Status', xcrun_output) - status_message = xcrun_field_value_from_output( - 'Status Message', xcrun_output) - - if status == 'success': - logger_server.info( - 'Package successfully notarized: %s', status_message) - return True - - if status == 'invalid': - logger_server.error(xcrun_output) - logger_server.error( - 'Package notarization has failed: %s', status_message) - raise NotarizationException(status_message) - - if status == 'in progress': - return False - - logger_server.info( - 'Unknown notarization status %s (%s)', status, status_message) - - return False - - def notarize_wait_result(self, request_uuid: str) -> None: - """ - Wait for until notarial office have a reply - """ - - logger_server.info( - 'Waiting for a result from the notarization office.') - - command = ['xcrun', 'altool', - '--notarization-info', request_uuid, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - time_start = time.monotonic() - timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS - - while True: - xcrun_output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - if self.notarize_review_status(xcrun_output): - break - - logger_server.info('Keep waiting for notarization office.') - time.sleep(30) - - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - logger_server.error( - "Notarial office didn't reply in %f seconds.", - timeout_in_seconds) - - def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Staple notarial label on the file - """ - - logger_server.info('Stapling notarial stamp.') - - command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath] - self.check_output_or_mock(command, util.Platform.MACOS) - - def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Run entire pipeline to get DMG notarized. - """ - logger_server.info('Begin notarization routines on %s', - file.relative_filepath) - - # Submit file for notarization. - request_uuid = self.notarize_request(file) - if not request_uuid: - return False - logger_server.info('Received Request UUID: %s', request_uuid) - - # Wait for the status from the notarization office. - if not self.notarize_wait_result(request_uuid): - return False - - # Staple. - self.notarize_staple(file) - - def notarize_all_dmg( - self, files: List[AbsoluteAndRelativeFileName]) -> bool: - """ - Notarize all DMG images from the input. - - Images are supposed to be codesigned already. - """ - for file in files: - if not file.relative_filepath.name.endswith('.dmg'): - continue - if not self.check_file_is_to_be_signed(file): - continue - - self.notarize_dmg(file) - - ############################################################################ - # Entry point. - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # TODO(sergey): Handle errors somehow. - - self.codesign_all_files(files) - self.codesign_bundles(files) - self.notarize_all_dmg(files) diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py deleted file mode 100644 index 674d9e9ce9e..00000000000 --- a/build_files/buildbot/codesign/simple_code_signer.py +++ /dev/null @@ -1,52 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - - -import logging.config -import sys - -from pathlib import Path -from typing import Optional - -import codesign.config_builder -import codesign.util as util -from codesign.base_code_signer import BaseCodeSigner - - -class SimpleCodeSigner: - code_signer: Optional[BaseCodeSigner] - - def __init__(self): - platform = util.get_current_platform() - if platform == util.Platform.LINUX: - from codesign.linux_code_signer import LinuxCodeSigner - self.code_signer = LinuxCodeSigner(codesign.config_builder) - elif platform == util.Platform.MACOS: - from codesign.macos_code_signer import MacOSCodeSigner - self.code_signer = MacOSCodeSigner(codesign.config_builder) - elif platform == util.Platform.WINDOWS: - from codesign.windows_code_signer import WindowsCodeSigner - self.code_signer = WindowsCodeSigner(codesign.config_builder) - else: - self.code_signer = None - - def sign_file_or_directory(self, path: Path) -> None: - logging.config.dictConfig(codesign.config_builder.LOGGING) - self.code_signer.run_buildbot_path_sign_pipeline(path) diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py deleted file mode 100644 index e67292dd049..00000000000 --- a/build_files/buildbot/codesign/util.py +++ /dev/null @@ -1,54 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import sys - -from enum import Enum -from pathlib import Path - - -class Platform(Enum): - LINUX = 1 - MACOS = 2 - WINDOWS = 3 - - -def get_current_platform() -> Platform: - if sys.platform == 'linux': - return Platform.LINUX - elif sys.platform == 'darwin': - return Platform.MACOS - elif sys.platform == 'win32': - return Platform.WINDOWS - raise Exception(f'Unknown platform {sys.platform}') - - -def ensure_file_does_not_exist_or_die(filepath: Path) -> None: - """ - If the file exists, unlink it. - If the file path exists and is not a file an assert will trigger. - If the file path does not exists nothing happens. - """ - if not filepath.exists(): - return - if not filepath.is_file(): - # TODO(sergey): Provide information about what the filepath actually is. - raise SystemExit(f'{filepath} is expected to be a file, but is not') - filepath.unlink() diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py deleted file mode 100644 index db185788a56..00000000000 --- a/build_files/buildbot/codesign/windows_code_signer.py +++ /dev/null @@ -1,117 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'} - -BLACKLIST_FILE_PREFIXES = ( - 'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime') - - -class SigntoolException(CodeSignException): - pass - -class WindowsCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in BLACKLIST_FILE_PREFIXES): - return False - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - - def get_sign_command_prefix(self) -> List[str]: - return [ - 'signtool', 'sign', '/v', - '/f', self.config.WIN_CERTIFICATE_FILEPATH, - '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL] - - - def run_codesign_tool(self, filepath: Path) -> None: - command = self.get_sign_command_prefix() + [filepath] - - try: - codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS) - except subprocess.CalledProcessError as e: - raise SigntoolException(f'Error running signtool {e}') - - logger_server.info(f'signtool output:\n{codesign_output}') - - got_number_of_success = False - - for line in codesign_output.split('\n'): - line_clean = line.strip() - line_clean_lower = line_clean.lower() - - if line_clean_lower.startswith('number of warnings') or \ - line_clean_lower.startswith('number of errors'): - number = int(line_clean_lower.split(':')[1]) - if number != 0: - raise SigntoolException('Non-clean success of signtool') - - if line_clean_lower.startswith('number of files successfully signed'): - got_number_of_success = True - number = int(line_clean_lower.split(':')[1]) - if number != 1: - raise SigntoolException('Signtool did not consider codesign a success') - - if not got_number_of_success: - raise SigntoolException('Signtool did not report number of files signed') - - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # NOTE: Sign files one by one to avoid possible command line length - # overflow (which could happen if we ever decide to sign every binary - # in the install folder, for example). - # - # TODO(sergey): Consider doing batched signing of handful of files in - # one go (but only if this actually known to be much faster). - num_files = len(files) - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of package and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - continue - - logger_server.info( - 'Running signtool command for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - self.run_codesign_tool(file.absolute_filepath) diff --git a/build_files/buildbot/codesign_server_linux.py b/build_files/buildbot/codesign_server_linux.py deleted file mode 100755 index be3065e640d..00000000000 --- a/build_files/buildbot/codesign_server_linux.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# NOTE: This is a no-op signer (since there isn't really a procedure to sign -# Linux binaries yet). Used to debug and verify the code signing routines on -# a Linux environment. - -import logging.config -from pathlib import Path -from typing import List - -from codesign.linux_code_signer import LinuxCodeSigner -import codesign.config_server - -if __name__ == "__main__": - logging.config.dictConfig(codesign.config_server.LOGGING) - code_signer = LinuxCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_macos.py b/build_files/buildbot/codesign_server_macos.py deleted file mode 100755 index 1bdb012fe67..00000000000 --- a/build_files/buildbot/codesign_server_macos.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging.config -from pathlib import Path -from typing import List - -from codesign.macos_code_signer import MacOSCodeSigner -import codesign.config_server - -if __name__ == "__main__": - entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE - if not entitlements_file.exists(): - raise SystemExit( - 'Entitlements file {entitlements_file} does not exist.') - if not entitlements_file.is_file(): - raise SystemExit( - 'Entitlements file {entitlements_file} is not a file.') - - logging.config.dictConfig(codesign.config_server.LOGGING) - code_signer = MacOSCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_windows.bat b/build_files/buildbot/codesign_server_windows.bat deleted file mode 100644 index 82680f30eb4..00000000000 --- a/build_files/buildbot/codesign_server_windows.bat +++ /dev/null @@ -1,11 +0,0 @@ -@echo off - -rem This is an entry point of the codesign server for Windows. -rem It makes sure that signtool.exe is within the current PATH and can be -rem used by the Python script. - -SETLOCAL - -set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% - -codesign_server_windows.py diff --git a/build_files/buildbot/codesign_server_windows.py b/build_files/buildbot/codesign_server_windows.py deleted file mode 100755 index 97ea4fd6756..00000000000 --- a/build_files/buildbot/codesign_server_windows.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Implementation of codesign server for Windows. -# -# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat - -import logging.config -import shutil - -from pathlib import Path -from typing import List - -import codesign.util as util - -from codesign.windows_code_signer import WindowsCodeSigner -import codesign.config_server - -if __name__ == "__main__": - logging.config.dictConfig(codesign.config_server.LOGGING) - - logger = logging.getLogger(__name__) - logger_server = logger.getChild('server') - - # TODO(sergey): Consider moving such sanity checks into - # CodeSigner.check_environment_or_die(). - if not shutil.which('signtool.exe'): - if util.get_current_platform() == util.Platform.WINDOWS: - raise SystemExit("signtool.exe is not found in %PATH%") - logger_server.info( - 'signtool.exe not found, ' - 'but will not be used on this foreign platform') - - code_signer = WindowsCodeSigner(codesign.config_server) - code_signer.run_signing_server() diff --git a/build_files/buildbot/worker_bundle_dmg.py b/build_files/buildbot/worker_bundle_dmg.py deleted file mode 100755 index 56e0d7da88e..00000000000 --- a/build_files/buildbot/worker_bundle_dmg.py +++ /dev/null @@ -1,551 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -import argparse -import re -import shutil -import subprocess -import sys -import time - -from pathlib import Path -from tempfile import TemporaryDirectory, NamedTemporaryFile -from typing import List - -BUILDBOT_DIRECTORY = Path(__file__).absolute().parent -CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'worker_codesign.py' -BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent -DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' - - -# Extra size which is added on top of actual files size when estimating size -# of destination DNG. -EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 - -################################################################################ -# Common utilities - - -def get_directory_size(root_directory: Path) -> int: - """ - Get size of directory on disk - """ - - total_size = 0 - for file in root_directory.glob('**/*'): - total_size += file.lstat().st_size - return total_size - - -################################################################################ -# DMG bundling specific logic - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'source_dir', - type=Path, - help='Source directory which points to either existing .app bundle' - 'or to a directory with .app bundles.') - parser.add_argument( - '--background-image', - type=Path, - help="Optional background picture which will be set on the DMG." - "If not provided default Blender's one is used.") - parser.add_argument( - '--volume-name', - type=str, - help='Optional name of a volume which will be used for DMG.') - parser.add_argument( - '--dmg', - type=Path, - help='Optional argument which points to a final DMG file name.') - parser.add_argument( - '--applescript', - type=Path, - help="Optional path to applescript to set up folder looks of DMG." - "If not provided default Blender's one is used.") - parser.add_argument( - '--codesign', - action="store_true", - help="Code sign and notarize DMG contents.") - return parser - - -def collect_app_bundles(source_dir: Path) -> List[Path]: - """ - Collect all app bundles which are to be put into DMG - - If the source directory points to FOO.app it will be the only app bundle - packed. - - Otherwise all .app bundles from given directory are placed to a single - DMG. - """ - - if source_dir.name.endswith('.app'): - return [source_dir] - - app_bundles = [] - for filename in source_dir.glob('*'): - if not filename.is_dir(): - continue - if not filename.name.endswith('.app'): - continue - - app_bundles.append(filename) - - return app_bundles - - -def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: - app_bundles = collect_app_bundles(source_dir) - - if not app_bundles: - print('No app bundles found for packing') - return - - print(f'Found {len(app_bundles)} to pack:') - for app_bundle in app_bundles: - print(f'- {app_bundle}') - - return app_bundles - - -def estimate_dmg_size(app_bundles: List[Path]) -> int: - """ - Estimate size of DMG to hold requested app bundles - - The size is based on actual size of all files in all bundles plus some - space to compensate for different size-on-disk plus some space to hold - codesign signatures. - - Is better to be on a high side since the empty space is compressed, but - lack of space might cause silent failures later on. - """ - - app_bundles_size = 0 - for app_bundle in app_bundles: - app_bundles_size += get_directory_size(app_bundle) - - return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES - - -def copy_app_bundles_to_directory(app_bundles: List[Path], - directory: Path) -> None: - """ - Copy all bundles to a given directory - - This directory is what the DMG will be created from. - """ - for app_bundle in app_bundles: - print(f'Copying {app_bundle.name}...') - shutil.copytree(app_bundle, directory / app_bundle.name) - - -def get_main_app_bundle(app_bundles: List[Path]) -> Path: - """ - Get application bundle main for the installation - """ - return app_bundles[0] - - -def create_dmg_image(app_bundles: List[Path], - dmg_filepath: Path, - volume_name: str) -> None: - """ - Create DMG disk image and put app bundles in it - - No DMG configuration or codesigning is happening here. - """ - - if dmg_filepath.exists(): - print(f'Removing existing writable DMG {dmg_filepath}...') - dmg_filepath.unlink() - - print('Preparing directory with app bundles for the DMG...') - with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: - # Copy all bundles to a clean directory. - content_dir = Path(content_dir_str) - copy_app_bundles_to_directory(app_bundles, content_dir) - - # Estimate size of the DMG. - dmg_size = estimate_dmg_size(app_bundles) - print(f'Estimated DMG size: {dmg_size:,} bytes.') - - # Create the DMG. - print(f'Creating writable DMG {dmg_filepath}') - command = ('hdiutil', - 'create', - '-size', str(dmg_size), - '-fs', 'HFS+', - '-srcfolder', content_dir, - '-volname', volume_name, - '-format', 'UDRW', - dmg_filepath) - subprocess.run(command) - - -def get_writable_dmg_filepath(dmg_filepath: Path): - """ - Get file path for writable DMG image - """ - parent = dmg_filepath.parent - return parent / (dmg_filepath.stem + '-temp.dmg') - - -def mount_readwrite_dmg(dmg_filepath: Path) -> None: - """ - Mount writable DMG - - Mounting point would be /Volumes/<volume name> - """ - - print(f'Mounting read-write DMG ${dmg_filepath}') - command = ('hdiutil', - 'attach', '-readwrite', - '-noverify', - '-noautoopen', - dmg_filepath) - subprocess.run(command) - - -def get_mount_directory_for_volume_name(volume_name: str) -> Path: - """ - Get directory under which the volume will be mounted - """ - - return Path('/Volumes') / volume_name - - -def eject_volume(volume_name: str) -> None: - """ - Eject given volume, if mounted - """ - mount_directory = get_mount_directory_for_volume_name(volume_name) - if not mount_directory.exists(): - return - mount_directory_str = str(mount_directory) - - print(f'Ejecting volume {volume_name}') - - # Figure out which device to eject. - mount_output = subprocess.check_output(['mount']).decode() - device = '' - for line in mount_output.splitlines(): - if f'on {mount_directory_str} (' not in line: - continue - tokens = line.split(' ', 3) - if len(tokens) < 3: - continue - if tokens[1] != 'on': - continue - if device: - raise Exception( - f'Multiple devices found for mounting point {mount_directory}') - device = tokens[0] - - if not device: - raise Exception( - f'No device found for mounting point {mount_directory}') - - print(f'{mount_directory} is mounted as device {device}, ejecting...') - subprocess.run(['diskutil', 'eject', device]) - - -def copy_background_if_needed(background_image_filepath: Path, - mount_directory: Path) -> None: - """ - Copy background to the DMG - - If the background image is not specified it will not be copied. - """ - - if not background_image_filepath: - print('No background image provided.') - return - - print(f'Copying background image {background_image_filepath}') - - destination_dir = mount_directory / '.background' - destination_dir.mkdir(exist_ok=True) - - destination_filepath = destination_dir / background_image_filepath.name - shutil.copy(background_image_filepath, destination_filepath) - - -def create_applications_link(mount_directory: Path) -> None: - """ - Create link to /Applications in the given location - """ - - print('Creating link to /Applications') - - command = ('ln', '-s', '/Applications', mount_directory / ' ') - subprocess.run(command) - - -def run_applescript(applescript: Path, - volume_name: str, - app_bundles: List[Path], - background_image_filepath: Path) -> None: - """ - Run given applescript to adjust look and feel of the DMG - """ - - main_app_bundle = get_main_app_bundle(app_bundles) - - with NamedTemporaryFile( - mode='w', suffix='.applescript') as temp_applescript: - print('Adjusting applescript for volume name...') - # Adjust script to the specific volume name. - with open(applescript, mode='r') as input: - for line in input.readlines(): - stripped_line = line.strip() - if stripped_line.startswith('tell disk'): - line = re.sub('tell disk ".*"', - f'tell disk "{volume_name}"', - line) - elif stripped_line.startswith('set background picture'): - if not background_image_filepath: - continue - else: - background_image_short = \ - '.background:' + background_image_filepath.name - line = re.sub('to file ".*"', - f'to file "{background_image_short}"', - line) - line = line.replace('blender.app', main_app_bundle.name) - temp_applescript.write(line) - - temp_applescript.flush() - - print('Running applescript...') - command = ('osascript', temp_applescript.name) - subprocess.run(command) - - print('Waiting for applescript...') - - # NOTE: This is copied from bundle.sh. The exact reason for sleep is - # still remained a mystery. - time.sleep(5) - - -def codesign(subject: Path): - """ - Codesign file or directory - - NOTE: For DMG it will also notarize. - """ - - command = (CODESIGN_SCRIPT, subject) - subprocess.run(command) - - -def codesign_app_bundles_in_dmg(mount_directory: str) -> None: - """ - Code sign all binaries and bundles in the mounted directory - """ - - print(f'Codesigning all app bundles in {mount_directory}') - codesign(mount_directory) - - -def codesign_and_notarize_dmg(dmg_filepath: Path) -> None: - """ - Run codesign and notarization pipeline on the DMG - """ - - print(f'Codesigning and notarizing DMG {dmg_filepath}') - codesign(dmg_filepath) - - -def compress_dmg(writable_dmg_filepath: Path, - final_dmg_filepath: Path) -> None: - """ - Compress temporary read-write DMG - """ - command = ('hdiutil', 'convert', - writable_dmg_filepath, - '-format', 'UDZO', - '-o', final_dmg_filepath) - - if final_dmg_filepath.exists(): - print(f'Removing old compressed DMG {final_dmg_filepath}') - final_dmg_filepath.unlink() - - print('Compressing disk image...') - subprocess.run(command) - - -def create_final_dmg(app_bundles: List[Path], - dmg_filepath: Path, - background_image_filepath: Path, - volume_name: str, - applescript: Path, - codesign: bool) -> None: - """ - Create DMG with all app bundles - - Will take care configuring background, signing all binaries and app bundles - and notarizing the DMG. - """ - - print('Running all routines to create final DMG') - - writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath) - mount_directory = get_mount_directory_for_volume_name(volume_name) - - # Make sure volume is not mounted. - # If it is mounted it will prevent removing old DMG files and could make - # it so app bundles are copied to the wrong place. - eject_volume(volume_name) - - create_dmg_image(app_bundles, writable_dmg_filepath, volume_name) - - mount_readwrite_dmg(writable_dmg_filepath) - - # Run codesign first, prior to copying amything else. - # - # This allows to recurs into the content of bundles without worrying about - # possible interfereice of Application symlink. - if codesign: - codesign_app_bundles_in_dmg(mount_directory) - - copy_background_if_needed(background_image_filepath, mount_directory) - create_applications_link(mount_directory) - run_applescript(applescript, volume_name, app_bundles, - background_image_filepath) - - print('Ejecting read-write DMG image...') - eject_volume(volume_name) - - compress_dmg(writable_dmg_filepath, dmg_filepath) - writable_dmg_filepath.unlink() - - if codesign: - codesign_and_notarize_dmg(dmg_filepath) - - -def ensure_dmg_extension(filepath: Path) -> Path: - """ - Make sure given file have .dmg extension - """ - - if filepath.suffix != '.dmg': - return filepath.with_suffix(f'{filepath.suffix}.dmg') - return filepath - - -def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path: - """ - Get full file path for the final DMG image - - Will use the provided one when possible, otherwise will deduct it from - app bundles. - - If the name is deducted, the DMG is stored in the current directory. - """ - - if requested_name: - return ensure_dmg_extension(requested_name.absolute()) - - # TODO(sergey): This is not necessarily the main one. - main_bundle = app_bundles[0] - # Strip .app from the name - return Path(main_bundle.name[:-4] + '.dmg').absolute() - - -def get_background_image(requested_background_image: Path) -> Path: - """ - Get effective filepath for the background image - """ - - if requested_background_image: - return requested_background_image.absolute() - - return DARWIN_DIRECTORY / 'background.tif' - - -def get_applescript(requested_applescript: Path) -> Path: - """ - Get effective filepath for the applescript - """ - - if requested_applescript: - return requested_applescript.absolute() - - return DARWIN_DIRECTORY / 'blender.applescript' - - -def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str: - """ - Deduct volume name from the DMG path - - Will use first part of the DMG file name prior to dash. - """ - - tokens = dmg_filepath.stem.split('-') - words = tokens[0].split() - - return ' '.join(word.capitalize() for word in words) - - -def get_volume_name(requested_volume_name: str, - dmg_filepath: Path) -> str: - """ - Get effective name for DMG volume - """ - - if requested_volume_name: - return requested_volume_name - - return get_volume_name_from_dmg_filepath(dmg_filepath) - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - - # Get normalized input parameters. - source_dir = args.source_dir.absolute() - background_image_filepath = get_background_image(args.background_image) - applescript = get_applescript(args.applescript) - codesign = args.codesign - - app_bundles = collect_and_log_app_bundles(source_dir) - if not app_bundles: - return - - dmg_filepath = get_dmg_filepath(args.dmg, app_bundles) - volume_name = get_volume_name(args.volume_name, dmg_filepath) - - print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)') - - create_final_dmg(app_bundles, - dmg_filepath, - background_image_filepath, - volume_name, - applescript, - codesign) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/worker_codesign.cmake b/build_files/buildbot/worker_codesign.cmake deleted file mode 100644 index f37feaef407..00000000000 --- a/build_files/buildbot/worker_codesign.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# This is a script which is used as POST-INSTALL one for regular CMake's -# INSTALL target. -# It is used by buildbot workers to sign every binary which is going into -# the final buundle. - -# On Windows Python 3 there only is python.exe, no python3.exe. -# -# On other platforms it is possible to have python2 and python3, and a -# symbolic link to python to either of them. So on those platforms use -# an explicit Python version. -if(WIN32) - set(PYTHON_EXECUTABLE python) -else() - set(PYTHON_EXECUTABLE python3) -endif() - -execute_process( - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/worker_codesign.py" - "${CMAKE_INSTALL_PREFIX}" - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE exit_code -) - -if(NOT exit_code EQUAL "0") - message(FATAL_ERROR "Non-zero exit code of codesign tool") -endif() diff --git a/build_files/buildbot/worker_codesign.py b/build_files/buildbot/worker_codesign.py deleted file mode 100755 index a82ee98b1b5..00000000000 --- a/build_files/buildbot/worker_codesign.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# Helper script which takes care of signing provided location. -# -# The location can either be a directory (in which case all eligible binaries -# will be signed) or a single file (in which case a single file will be signed). -# -# This script takes care of all the complexity of communicating between process -# which requests file to be signed and the code signing server. -# -# NOTE: Signing happens in-place. - -import argparse -import sys - -from pathlib import Path - -from codesign.simple_code_signer import SimpleCodeSigner - - -def create_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('path_to_sign', type=Path) - return parser - - -def main(): - parser = create_argument_parser() - args = parser.parse_args() - path_to_sign = args.path_to_sign.absolute() - - if sys.platform == 'win32': - # When WIX packed is used to generate .msi on Windows the CPack will - # install two different projects and install them to different - # installation prefix: - # - # - C:\b\build\_CPack_Packages\WIX\Blender - # - C:\b\build\_CPack_Packages\WIX\Unspecified - # - # Annoying part is: CMake's post-install script will only be run - # once, with the install prefix which corresponds to a project which - # was installed last. But we want to sign binaries from all projects. - # So in order to do so we detect that we are running for a CPack's - # project used for WIX and force parent directory (which includes both - # projects) to be signed. - # - # Here we force both projects to be signed. - if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign): - path_to_sign = path_to_sign.parent - - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(path_to_sign) - - -if __name__ == "__main__": - main() diff --git a/build_files/buildbot/worker_compile.py b/build_files/buildbot/worker_compile.py deleted file mode 100644 index 8c6b44c5866..00000000000 --- a/build_files/buildbot/worker_compile.py +++ /dev/null @@ -1,135 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import os -import shutil - -import buildbot_utils - - -def get_cmake_options(builder): - codesign_script = os.path.join( - builder.blender_dir, 'build_files', 'buildbot', 'worker_codesign.cmake') - - config_file = "build_files/cmake/config/blender_release.cmake" - options = ['-DCMAKE_BUILD_TYPE:STRING=Release', - '-DWITH_GTESTS=ON'] - - if builder.platform == 'mac': - options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') - options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9') - elif builder.platform == 'win': - options.extend(['-G', 'Visual Studio 16 2019', '-A', 'x64']) - if builder.codesign: - options.extend(['-DPOSTINSTALL_SCRIPT:PATH=' + codesign_script]) - elif builder.platform == 'linux': - config_file = "build_files/buildbot/config/blender_linux.cmake" - - optix_sdk_dir = os.path.join(builder.blender_dir, '..', '..', 'NVIDIA-Optix-SDK-7.1') - options.append('-DOPTIX_ROOT_DIR:PATH=' + optix_sdk_dir) - - # Workaround to build sm_30 kernels with CUDA 10, since CUDA 11 no longer supports that architecture - if builder.platform == 'win': - options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1') - options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe') - options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1') - options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.1/bin/nvcc.exe') - elif builder.platform == 'linux': - options.append('-DCUDA10_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-10.1') - options.append('-DCUDA10_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-10.1/bin/nvcc') - options.append('-DCUDA11_TOOLKIT_ROOT_DIR:PATH=/usr/local/cuda-11.1') - options.append('-DCUDA11_NVCC_EXECUTABLE:FILEPATH=/usr/local/cuda-11.1/bin/nvcc') - - options.append("-C" + os.path.join(builder.blender_dir, config_file)) - options.append("-DCMAKE_INSTALL_PREFIX=%s" % (builder.install_dir)) - - return options - - -def update_git(builder): - # Do extra git fetch because not all platform/git/buildbot combinations - # update the origin remote, causing buildinfo to detect local changes. - os.chdir(builder.blender_dir) - - print("Fetching remotes") - command = ['git', 'fetch', '--all'] - buildbot_utils.call(builder.command_prefix + command) - - -def clean_directories(builder): - # Make sure no garbage remained from the previous run - if os.path.isdir(builder.install_dir): - shutil.rmtree(builder.install_dir) - - # Make sure build directory exists and enter it - os.makedirs(builder.build_dir, exist_ok=True) - - # Remove buildinfo files to force buildbot to re-generate them. - for buildinfo in ('buildinfo.h', 'buildinfo.h.txt', ): - full_path = os.path.join(builder.build_dir, 'source', 'creator', buildinfo) - if os.path.exists(full_path): - print("Removing {}" . format(buildinfo)) - os.remove(full_path) - - -def cmake_configure(builder): - # CMake configuration - os.chdir(builder.build_dir) - - cmake_cache = os.path.join(builder.build_dir, 'CMakeCache.txt') - if os.path.exists(cmake_cache): - print("Removing CMake cache") - os.remove(cmake_cache) - - print("CMake configure:") - cmake_options = get_cmake_options(builder) - command = ['cmake', builder.blender_dir] + cmake_options - buildbot_utils.call(builder.command_prefix + command) - - -def cmake_build(builder): - # CMake build - os.chdir(builder.build_dir) - - # NOTE: CPack will build an INSTALL target, which would mean that code - # signing will happen twice when using `make install` and CPack. - # The tricky bit here is that it is not possible to know whether INSTALL - # target is used by CPack or by a buildbot itaself. Extra level on top of - # this is that on Windows it is required to build INSTALL target in order - # to have unit test binaries to run. - # So on the one hand we do an extra unneeded code sign on Windows, but on - # a positive side we don't add complexity and don't make build process more - # fragile trying to avoid this. The signing process is way faster than just - # a clean build of buildbot, especially with regression tests enabled. - if builder.platform == 'win': - command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] - else: - command = ['make', '-s', '-j16', 'install'] - - print("CMake build:") - buildbot_utils.call(builder.command_prefix + command) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - update_git(builder) - clean_directories(builder) - cmake_configure(builder) - cmake_build(builder) diff --git a/build_files/buildbot/worker_pack.py b/build_files/buildbot/worker_pack.py deleted file mode 100644 index a5727a66e09..00000000000 --- a/build_files/buildbot/worker_pack.py +++ /dev/null @@ -1,208 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Runs on buildbot worker, creating a release package using the build -# system and zipping it into buildbot_upload.zip. This is then uploaded -# to the master in the next buildbot step. - -import os -import sys - -from pathlib import Path - -import buildbot_utils - - -def get_package_name(builder, platform=None): - info = buildbot_utils.VersionInfo(builder) - - package_name = 'blender-' + info.full_version - if platform: - package_name += '-' + platform - if not (builder.branch == 'master' or builder.is_release_branch): - if info.is_development_build: - package_name = builder.branch + "-" + package_name - - return package_name - - -def sign_file_or_directory(path): - from codesign.simple_code_signer import SimpleCodeSigner - code_signer = SimpleCodeSigner() - code_signer.sign_file_or_directory(Path(path)) - - -def create_buildbot_upload_zip(builder, package_files): - import zipfile - - buildbot_upload_zip = os.path.join(builder.upload_dir, "buildbot_upload.zip") - if os.path.exists(buildbot_upload_zip): - os.remove(buildbot_upload_zip) - - try: - z = zipfile.ZipFile(buildbot_upload_zip, "w", compression=zipfile.ZIP_STORED) - for filepath, filename in package_files: - print("Packaged", filename) - z.write(filepath, arcname=filename) - z.close() - except Exception as ex: - sys.stderr.write('Create buildbot_upload.zip failed: ' + str(ex) + '\n') - sys.exit(1) - - -def create_tar_xz(src, dest, package_name): - # One extra to remove leading os.sep when cleaning root for package_root - ln = len(src) + 1 - flist = list() - - # Create list of tuples containing file and archive name - for root, dirs, files in os.walk(src): - package_root = os.path.join(package_name, root[ln:]) - flist.extend([(os.path.join(root, file), os.path.join(package_root, file)) for file in files]) - - import tarfile - - # Set UID/GID of archived files to 0, otherwise they'd be owned by whatever - # user compiled the package. If root then unpacks it to /usr/local/ you get - # a security issue. - def _fakeroot(tarinfo): - tarinfo.gid = 0 - tarinfo.gname = "root" - tarinfo.uid = 0 - tarinfo.uname = "root" - return tarinfo - - package = tarfile.open(dest, 'w:xz', preset=9) - for entry in flist: - package.add(entry[0], entry[1], recursive=False, filter=_fakeroot) - package.close() - - -def cleanup_files(dirpath, extension): - for f in os.listdir(dirpath): - filepath = os.path.join(dirpath, f) - if os.path.isfile(filepath) and f.endswith(extension): - os.remove(filepath) - - -def pack_mac(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.dmg') - - package_name = get_package_name(builder, 'macOS') - package_filename = package_name + '.dmg' - package_filepath = os.path.join(builder.build_dir, package_filename) - - release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') - buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot') - bundle_script = os.path.join(buildbot_dir, 'worker_bundle_dmg.py') - - command = [bundle_script] - command += ['--dmg', package_filepath] - if info.is_development_build: - background_image = os.path.join(release_dir, 'buildbot', 'background.tif') - command += ['--background-image', background_image] - if builder.codesign: - command += ['--codesign'] - command += [builder.install_dir] - buildbot_utils.call(command) - - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -def pack_win(builder): - info = buildbot_utils.VersionInfo(builder) - - os.chdir(builder.build_dir) - cleanup_files(builder.build_dir, '.zip') - - # CPack will add the platform name - cpack_name = get_package_name(builder, None) - package_name = get_package_name(builder, 'windows' + str(builder.bits)) - - command = ['cmake', '-DCPACK_OVERRIDE_PACKAGENAME:STRING=' + cpack_name, '.'] - buildbot_utils.call(builder.command_prefix + command) - command = ['cpack', '-G', 'ZIP'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.zip' - package_filepath = os.path.join(builder.build_dir, package_filename) - package_files = [(package_filepath, package_filename)] - - if info.version_cycle == 'release': - # Installer only for final release builds, otherwise will get - # 'this product is already installed' messages. - command = ['cpack', '-G', 'WIX'] - buildbot_utils.call(builder.command_prefix + command) - - package_filename = package_name + '.msi' - package_filepath = os.path.join(builder.build_dir, package_filename) - if builder.codesign: - sign_file_or_directory(package_filepath) - - package_files += [(package_filepath, package_filename)] - - create_buildbot_upload_zip(builder, package_files) - - -def pack_linux(builder): - blender_executable = os.path.join(builder.install_dir, 'blender') - - info = buildbot_utils.VersionInfo(builder) - - # Strip all unused symbols from the binaries - print("Stripping binaries...") - buildbot_utils.call(builder.command_prefix + ['strip', '--strip-all', blender_executable]) - - print("Stripping python...") - py_target = os.path.join(builder.install_dir, info.short_version) - buildbot_utils.call( - builder.command_prefix + [ - 'find', py_target, '-iname', '*.so', '-exec', 'strip', '-s', '{}', ';', - ], - ) - - # Construct package name - platform_name = 'linux64' - package_name = get_package_name(builder, platform_name) - package_filename = package_name + ".tar.xz" - - print("Creating .tar.xz archive") - package_filepath = builder.install_dir + '.tar.xz' - create_tar_xz(builder.install_dir, package_filepath, package_name) - - # Create buildbot_upload.zip - create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - - # Make sure install directory always exists - os.makedirs(builder.install_dir, exist_ok=True) - - if builder.platform == 'mac': - pack_mac(builder) - elif builder.platform == 'win': - pack_win(builder) - elif builder.platform == 'linux': - pack_linux(builder) diff --git a/build_files/buildbot/worker_test.py b/build_files/buildbot/worker_test.py deleted file mode 100644 index 8a6e7d4bb69..00000000000 --- a/build_files/buildbot/worker_test.py +++ /dev/null @@ -1,42 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import buildbot_utils -import os -import sys - - -def get_ctest_arguments(builder): - args = ['--output-on-failure'] - if builder.platform == 'win': - args += ['-C', 'Release'] - return args - - -def test(builder): - os.chdir(builder.build_dir) - - command = builder.command_prefix + ['ctest'] + get_ctest_arguments(builder) - buildbot_utils.call(command) - - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - test(builder) diff --git a/build_files/buildbot/worker_update.py b/build_files/buildbot/worker_update.py deleted file mode 100644 index 36a7ae31c84..00000000000 --- a/build_files/buildbot/worker_update.py +++ /dev/null @@ -1,31 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import buildbot_utils -import os -import sys - -if __name__ == "__main__": - builder = buildbot_utils.create_builder_from_arguments() - os.chdir(builder.blender_dir) - - # Run make update which handles all libraries and submodules. - make_update = os.path.join(builder.blender_dir, "build_files", "utils", "make_update.py") - buildbot_utils.call([sys.executable, make_update, '--no-blender', "--use-tests", "--use-centos-libraries"]) diff --git a/build_files/config/README.md b/build_files/config/README.md new file mode 100644 index 00000000000..6c429e4e58d --- /dev/null +++ b/build_files/config/README.md @@ -0,0 +1,8 @@ +Pipeline Config +=============== + +This configuration file is used by buildbot new pipeline for the `update-code` step. + +It will soon be used by the ../utils/make_update.py script. + +Both buildbot and developers will eventually use the same configuration file. diff --git a/build_files/config/pipeline_config.json b/build_files/config/pipeline_config.json new file mode 100644 index 00000000000..47318b2f7dc --- /dev/null +++ b/build_files/config/pipeline_config.json @@ -0,0 +1,87 @@ +{ + "update-code": + { + "git" : + { + "submodules": + [ + { "path": "release/scripts/addons", "branch": "master", "commit_id": "HEAD" }, + { "path": "release/scripts/addons_contrib", "branch": "master", "commit_id": "HEAD" }, + { "path": "release/datafiles/locale", "branch": "master", "commit_id": "HEAD" }, + { "path": "source/tools", "branch": "master", "commit_id": "HEAD" } + ] + }, + "svn": + { + "tests": { "path": "lib/tests", "branch": "trunk", "commit_id": "HEAD" }, + "libraries": + { + "darwin-x86_64": { "path": "lib/darwin", "branch": "trunk", "commit_id": "HEAD" }, + "darwin-arm64": { "path": "lib/darwin_arm64", "branch": "trunk", "commit_id": "HEAD" }, + "linux-x86_64": { "path": "lib/linux_centos7_x86_64", "branch": "trunk", "commit_id": "HEAD" }, + "windows-amd64": { "path": "lib/win64_vc15", "branch": "trunk", "commit_id": "HEAD" } + } + } + }, + "buildbot": + { + "gcc": + { + "version": "9.0" + }, + "sdks": + { + "optix": + { + "version": "7.1.0" + }, + "cuda10": + { + "version": "10.1" + }, + "cuda11": + { + "version": "11.3" + } + }, + "cmake": + { + "default": + { + "version": "any", + "overrides": + { + + } + }, + "darwin-x86_64": + { + "overrides": + { + + } + }, + "darwin-arm64": + { + "overrides": + { + + } + }, + "linux-x86_64": + { + "overrides": + { + + } + }, + "windows-amd64": + { + "overrides": + { + + } + } + } + } +} diff --git a/build_files/utils/README.md b/build_files/utils/README.md new file mode 100644 index 00000000000..e78d05b0c69 --- /dev/null +++ b/build_files/utils/README.md @@ -0,0 +1,5 @@ +Make Utility Scripts +==================== + +Scripts used only by developers for now + diff --git a/release/darwin/README.md b/release/darwin/README.md new file mode 100644 index 00000000000..f1f02543ff3 --- /dev/null +++ b/release/darwin/README.md @@ -0,0 +1,5 @@ +Buildbot Configuration +====================== + +Files used by Buildbot's `package-code-binaires` step for the darwin platform. + diff --git a/release/darwin/README.txt b/release/darwin/README.txt deleted file mode 100644 index 626ce8820ab..00000000000 --- a/release/darwin/README.txt +++ /dev/null @@ -1,55 +0,0 @@ - -macOS app bundling guide -======================== - -Install Code Signing Certificate --------------------------------- - -* Go to https://developer.apple.com/account/resources/certificates/list -* Download the Developer ID Application certificate. -* Double click the file and add to key chain (default options). -* Delete the file from the Downloads folder. - -* You will also need to install a .p12 public/private key file for the - certificate. This is only available for the owner of the Blender account, - or can be exported and copied from another system that already has code - signing set up. - -Find the codesigning identity by running: - -$ security find-identity -v -p codesigning - -"Developer ID Application: Stichting Blender Foundation" is the identity needed. -The long code at the start of the line is used as <identity> below. - -Setup Apple ID --------------- - -* The Apple ID must have two step verification enabled. -* Create an app specific password for the code signing app (label can be anything): -https://support.apple.com/en-us/HT204397 -* Add the app specific password to keychain: - -$ security add-generic-password -a <apple-id> -w <app-specific-password> -s altool-password - -When running the bundle script, there will be a popup. To avoid that either: -* Click Always Allow in the popup -* In the Keychain Access app, change the Access Control settings on altool-password - -Bundle ------- - -Then the bundle is created as follows: - -$ ./bundle.sh --source <sourcedir> --dmg <dmg> --bundle-id <bundleid> --username <apple-id> --password "@keychain:altool-password" --codesign <identity> - -<sourcedir> directory where built Blender.app is -<dmg> location and name of the final disk image -<bundleid> id on notarization, for example org.blenderfoundation.blender.release -<apple-id> your appleid email -<identity> codesigning identity - -When specifying only --sourcedir and --dmg, the build will not be signed. - -Example : -$ ./bundle.sh --source /data/build/bin --dmg /data/Blender-2.8-alpha-macOS-10.11.dmg --bundle-id org.blenderfoundation.blender.release --username "foo@mac.com" --password "@keychain:altool-password" --codesign AE825E26F12D08B692F360133210AF46F4CF7B97 diff --git a/release/darwin/blender.applescript b/release/darwin/blender.applescript deleted file mode 100644 index 130dc2eb64c..00000000000 --- a/release/darwin/blender.applescript +++ /dev/null @@ -1,18 +0,0 @@ -tell application "Finder" - tell disk "Blender" - open - set current view of container window to icon view - set toolbar visible of container window to false - set statusbar visible of container window to false - set the bounds of container window to {100, 100, 640, 472} - set theViewOptions to icon view options of container window - set arrangement of theViewOptions to not arranged - set icon size of theViewOptions to 128 - set background picture of theViewOptions to file ".background:background.tif" - set position of item " " of container window to {400, 190} - set position of item "blender.app" of container window to {135, 190} - update without registering applications - delay 5 - close - end tell -end tell diff --git a/release/darwin/bundle.sh b/release/darwin/bundle.sh deleted file mode 100755 index 6d8695a441d..00000000000 --- a/release/darwin/bundle.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env bash -# -# Script to create a macOS dmg file for Blender builds, including code -# signing and notarization for releases. - -# Check that we have all needed tools. -for i in osascript git codesign hdiutil xcrun ; do - if [ ! -x "$(which ${i})" ]; then - echo "Unable to execute command $i, macOS broken?" - exit 1 - fi -done - -# Defaults settings. -_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -_volume_name="Blender" -_tmp_dir="$(mktemp -d)" -_tmp_dmg="/tmp/blender-tmp.dmg" -_background_image="${_script_dir}/background.tif" -_mount_dir="/Volumes/${_volume_name}" -_entitlements="${_script_dir}/entitlements.plist" - -# Handle arguments. -while [[ $# -gt 0 ]]; do - key=$1 - case $key in - -s|--source) - SRC_DIR="$2" - shift - shift - ;; - -d|--dmg) - DEST_DMG="$2" - shift - shift - ;; - -b|--bundle-id) - N_BUNDLE_ID="$2" - shift - shift - ;; - -u|--username) - N_USERNAME="$2" - shift - shift - ;; - -p|--password) - N_PASSWORD="$2" - shift - shift - ;; - -c|--codesign) - C_CERT="$2" - shift - shift - ;; - --background-image) - _background_image="$2" - shift - shift - ;; - -h|--help) - echo "Usage:" - echo " $(basename "$0") --source DIR --dmg IMAGENAME " - echo " optional arguments:" - echo " --codesign <certname>" - echo " --username <username>" - echo " --password <password>" - echo " --bundle-id <bundleid>" - echo " Check https://developer.apple.com/documentation/security/notarizing_your_app_before_distribution/customizing_the_notarization_workflow " - exit 1 - ;; - esac -done - -if [ ! -d "${SRC_DIR}/Blender.app" ]; then - echo "use --source parameter to set source directory where Blender.app can be found" - exit 1 -fi - -if [ -z "${DEST_DMG}" ]; then - echo "use --dmg parameter to set output dmg name" - exit 1 -fi - -# Destroy destination dmg if there is any. -test -f "${DEST_DMG}" && rm "${DEST_DMG}" -if [ -d "${_mount_dir}" ]; then - echo -n "Ejecting existing blender volume.." - DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }') - diskutil eject "${DEV_FILE}" || exit 1 - echo -fi - -# Copy dmg contents. -echo -n "Copying Blender.app..." -cp -r "${SRC_DIR}/Blender.app" "${_tmp_dir}/" || exit 1 -echo - -# Create the disk image. -_directory_size=$(du -sh ${_tmp_dir} | awk -F'[^0-9]*' '$0=$1') -_image_size=$(echo "${_directory_size}" + 400 | bc) # extra 400 need for codesign to work (why on earth?) - -echo -echo -n "Creating disk image of size ${_image_size}M.." -test -f "${_tmp_dmg}" && rm "${_tmp_dmg}" -hdiutil create -size "${_image_size}m" -fs HFS+ -srcfolder "${_tmp_dir}" -volname "${_volume_name}" -format UDRW "${_tmp_dmg}" -mode 755 - -echo "Mounting readwrite image..." -hdiutil attach -readwrite -noverify -noautoopen "${_tmp_dmg}" - -echo "Setting background picture.." -if ! test -z "${_background_image}"; then - echo "Copying background image ..." - test -d "${_mount_dir}/.background" || mkdir "${_mount_dir}/.background" - _background_image_NAME=$(basename "${_background_image}") - cp "${_background_image}" "${_mount_dir}/.background/${_background_image_NAME}" -fi - -echo "Creating link to /Applications ..." -ln -s /Applications "${_mount_dir}/Applications" -echo "Renaming Applications to empty string." -mv ${_mount_dir}/Applications "${_mount_dir}/ " - -echo "Running applescript to set folder looks ..." -cat "${_script_dir}/blender.applescript" | osascript - -echo "Waiting after applescript ..." -sleep 5 - -if [ ! -z "${C_CERT}" ]; then - # Codesigning requires all libs and binaries to be signed separately. - echo -n "Codesigning Python" - for f in $(find "${_mount_dir}/Blender.app/Contents/Resources" -name "python*"); do - if [ -x ${f} ] && [ ! -d ${f} ]; then - codesign --remove-signature "${f}" - codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}" - fi - done - echo ; echo -n "Codesigning .dylib and .so libraries" - for f in $(find "${_mount_dir}/Blender.app" -name "*.dylib" -o -name "*.so"); do - codesign --remove-signature "${f}" - codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${f}" - done - echo ; echo -n "Codesigning Blender.app" - codesign --remove-signature "${_mount_dir}/Blender.app" - codesign --timestamp --options runtime --entitlements="${_entitlements}" --sign "${C_CERT}" "${_mount_dir}/Blender.app" - echo -else - echo "No codesigning cert given, skipping..." -fi - -# Need to eject dev files to remove /dev files and free .dmg for converting -echo "Unmounting rw disk image ..." -DEV_FILE=$(mount | grep "${_mount_dir}" | awk '{ print $1 }') -diskutil eject "${DEV_FILE}" - -sleep 3 - -echo "Compressing disk image ..." -hdiutil convert "${_tmp_dmg}" -format UDZO -o "${DEST_DMG}" - -# Codesign the dmg -if [ ! -z "${C_CERT}" ]; then - echo -n "Codesigning dmg..." - codesign --timestamp --force --sign "${C_CERT}" "${DEST_DMG}" - echo -fi - -# Cleanup -rm -rf "${_tmp_dir}" -rm "${_tmp_dmg}" - -# Notarize -if [ ! -z "${N_USERNAME}" ] && [ ! -z "${N_PASSWORD}" ] && [ ! -z "${N_BUNDLE_ID}" ]; then - # Send to Apple - echo "Sending ${DEST_DMG} for notarization..." - _tmpout=$(mktemp) - echo xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}" - xcrun altool --notarize-app --verbose -f "${DEST_DMG}" --primary-bundle-id "${N_BUNDLE_ID}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1 - - # Parse request uuid - _requuid=$(cat "${_tmpout}" | grep "RequestUUID" | awk '{ print $3 }') - echo "RequestUUID: ${_requuid}" - if [ ! -z "${_requuid}" ]; then - # Wait for Apple to confirm notarization is complete - echo "Waiting for notarization to be complete.." - for c in {20..0};do - sleep 600 - xcrun altool --notarization-info "${_requuid}" --username "${N_USERNAME}" --password "${N_PASSWORD}" >${_tmpout} 2>&1 - _status=$(cat "${_tmpout}" | grep "Status:" | awk '{ print $2 }') - if [ "${_status}" == "invalid" ]; then - echo "Got invalid notarization!" - break; - fi - - if [ "${_status}" == "success" ]; then - echo -n "Notarization successful! Stapling..." - xcrun stapler staple -v "${DEST_DMG}" - break; - fi - echo "Notarization in progress, waiting..." - done - else - cat ${_tmpout} - echo "Error getting RequestUUID, notarization unsuccessful" - fi -else - echo "No notarization credentials supplied, skipping..." -fi - -echo "..done. You should have ${DEST_DMG} ready to upload" diff --git a/release/freedesktop/snap/README.md b/release/freedesktop/snap/README.md new file mode 100644 index 00000000000..742b265ada6 --- /dev/null +++ b/release/freedesktop/snap/README.md @@ -0,0 +1,17 @@ +Snap Configuration +=================== + +Files used by Buildbot's `package-code-store-snap` and `deliver-code-store-snap` steps. + +Build pipeline snap tracks and channels + +``` + <track>/stable + - Latest stable release for the specified track + <track>/candidate + - Test builds for the upcoming stable release - *not used for now* + <track>/beta + - Nightly automated builds provided by a release branch + <track>/egde/<branch> + - Nightly or on demand builds - will also make use of branch +``` diff --git a/release/freedesktop/snap/README.txt b/release/freedesktop/snap/README.txt deleted file mode 100644 index 2e8822f32dc..00000000000 --- a/release/freedesktop/snap/README.txt +++ /dev/null @@ -1,38 +0,0 @@ - -Snap Package Instructions -========================= - -This folder contains the scripts for creating and uploading the snap on: -https://snapcraft.io/blender - - -Setup ------ - -This has only been tested to work on Ubuntu. - -# Install required packages -sudo apt install snapd snapcraft - - -Steps ------ - -# Build the snap file -python3 bundle.py --version 2.XX --url https://download.blender.org/release/Blender2.XX/blender-2.XX-x86_64.tar.bz2 - -# Install snap to test -# --dangerous is needed since the snap has not been signed yet -# --classic is required for installing Blender in general -sudo snap install --dangerous --classic blender_2.XX_amd64.snap - -# Upload -snapcraft push --release=stable blender_2.XX_amd64.snap - - -Release Values --------------- - -stable: final release -candidate: release candidates - diff --git a/release/freedesktop/snap/snapcraft.yaml.in b/release/freedesktop/snap/blender-snapcraft-template.yaml index eb3ef97eba8..882f9081c09 100644 --- a/release/freedesktop/snap/snapcraft.yaml.in +++ b/release/freedesktop/snap/blender-snapcraft-template.yaml @@ -10,12 +10,7 @@ description: | scientists, students, VFX experts, animators, game artists, modders, and the list goes on. - The standard snap channels are used in the following way: - - stable - Latest stable release. - candidate - Test builds for the upcoming stable release. - -icon: ../icons/scalable/apps/blender.svg +icon: @ICON_PATH@ passthrough: license: GPL-3.0 @@ -27,13 +22,14 @@ apps: command: ./blender-wrapper desktop: ./blender.desktop +base: core18 version: '@VERSION@' grade: @GRADE@ parts: blender: plugin: dump - source: @URL@ + source: @PACKAGE_PATH@ build-attributes: [keep-execstack, no-patchelf] override-build: | snapcraftctl build @@ -47,7 +43,7 @@ parts: - libxrender1 - libxxf86vm1 wrapper: - plugin: copy + plugin: dump source: . - files: - blender-wrapper: blender-wrapper + stage: + - ./blender-wrapper diff --git a/release/freedesktop/snap/bundle.py b/release/freedesktop/snap/bundle.py deleted file mode 100755 index c3ecc5af561..00000000000 --- a/release/freedesktop/snap/bundle.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import pathlib -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument("--version", required=True) -parser.add_argument("--url", required=True) -parser.add_argument("--grade", default="stable", choices=["stable", "devel"]) -args = parser.parse_args() - -yaml_text = pathlib.Path("snapcraft.yaml.in").read_text() -yaml_text = yaml_text.replace("@VERSION@", args.version) -yaml_text = yaml_text.replace("@URL@", args.url) -yaml_text = yaml_text.replace("@GRADE@", args.grade) -pathlib.Path("snapcraft.yaml").write_text(yaml_text) - -subprocess.call(["snapcraft", "clean"]) -subprocess.call(["snapcraft", "snap"]) diff --git a/release/steam/README.md b/release/steam/README.md deleted file mode 100644 index 05eda799c3f..00000000000 --- a/release/steam/README.md +++ /dev/null @@ -1,70 +0,0 @@ -Creating Steam builds for Blender -================================= - -This script automates creation of the Steam files: download of the archives, -extraction of the archives, preparation of the build scripts (VDF files), actual -building of the Steam game files. - -Requirements -============ - -* MacOS machine - Tested on Catalina 10.15.6. Extracting contents from the DMG - archive did not work Windows nor on Linux using 7-zip. All DMG archives tested - failed to be extracted. As such only MacOS is known to work. -* Steam SDK downloaded from SteamWorks - The `steamcmd` is used to generate the - Steam game files. The path to the `steamcmd` is what is actually needed. -* SteamWorks credentials - Needed to log in using `steamcmd`. -* Login to SteamWorks with the `steamcmd` from the command-line at least once - - Needded to ensure the user is properly logged in. On a new machine the user - will have to go through two-factor authentication. -* App ID and Depot IDs - Needed to create the VDF files. -* Python 3.x - 3.7 was tested. -* Base URL - for downloading the archives. - -Usage -===== - -```bash -$ export STEAMUSER=SteamUserName -$ export STEAMPW=SteamUserPW -$ export BASEURL=https://download.blender.org/release/Blender2.83/ -$ export VERSION=2.83.3 -$ export APPID=appidnr -$ export WINID=winidnr -$ export LINID=linuxidnr -$ export MACOSID=macosidnr - -# log in to SteamWorks from command-line at least once - -$ ../sdk/tools/ContentBuilder/builder_osx/steamcmd +login $STEAMUSER $STEAMPW - -# once that has been done we can now actually start our tool - -$ python3.7 create_steam_builds.py --baseurl $BASEURL --version $VERSION --appid $APPID --winid $WINID --linuxid $LINID --macosid $MACOSID --steamuser $STEAMUSER --steampw $STEAMPW --steamcmd ../sdk/tools/ContentBuilder/builder_osx/steamcmd -``` - -All arguments in the above example are required. - -At the start the tool will login using `steamcmd`. This is necessary to let the -Steam SDK update itself if necessary. - -There are a few optional arguments: - -* `--dryrun`: If set building the game files will not actually happen. A set of - log files and a preview manifest per depot will be created in the output folder. - This can be used to double-check everything works as expected. -* `--skipdl`: If set will skip downloading of the archives. The tool expects the - archives to already exist in the correct content location. -* `--skipextract`: If set will skip extraction of all archives. The tool expects - the archives to already have been correctly extracted in the content location. - -Run the tool with `-h` for detailed information on each argument. - -The content and output folders are generated through appending the version -without dots to the words `content` and `output` respectively, e.g. `content2833` -and `output2833`. These folders are created next to the tool. - -From all `.template` files the Steam build scripts will be generated also in the -same directory as the tool. The files will have the extension `.vdf`. - -In case of errors the tool will have a non-zero return code.
\ No newline at end of file diff --git a/release/steam/blender_app_build.vdf.template b/release/steam/blender_app_build.vdf.template deleted file mode 100644 index 9e2d0625d72..00000000000 --- a/release/steam/blender_app_build.vdf.template +++ /dev/null @@ -1,17 +0,0 @@ -"appbuild" -{ - "appid" "[APPID]" - "desc" "Blender [VERSION]" // description for this build - "buildoutput" "./[OUTPUT]" // build output folder for .log, .csm & .csd files, relative to location of this file - "contentroot" "./[CONTENT]" // root content folder, relative to location of this file - "setlive" "" // branch to set live after successful build, non if empty - "preview" "[DRYRUN]" // 1 to enable preview builds, 0 to commit build to steampipe - "local" "" // set to flie path of local content server - - "depots" - { - "[WINID]" "depot_build_win.vdf" - "[LINUXID]" "depot_build_linux.vdf" - "[MACOSID]" "depot_build_macos.vdf" - } -} diff --git a/release/steam/create_steam_builds.py b/release/steam/create_steam_builds.py deleted file mode 100644 index 2ecd0c347f7..00000000000 --- a/release/steam/create_steam_builds.py +++ /dev/null @@ -1,397 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib -import requests -import shutil -import subprocess -from typing import Callable, Iterator, List, Tuple - -# supported archive and platform endings, used to create actual archive names -archive_endings = ["windows64.zip", "linux64.tar.xz", "macOS.dmg"] - - -def add_optional_argument(option: str, help: str) -> None: - global parser - """Add an optional argument - - Args: - option (str): Option to add - help (str): Help description for the argument - """ - parser.add_argument(option, help=help, action='store_const', const=1) - - -def blender_archives(version: str) -> Iterator[str]: - """Generator for Blender archives for version. - - Yields for items in archive_endings an archive name in the form of - blender-{version}-{ending}. - - Args: - version (str): Version string of the form 2.83.2 - - - Yields: - Iterator[str]: Name in the form of blender-{version}-{ending} - """ - global archive_endings - - for ending in archive_endings: - yield f"blender-{version}-{ending}" - - -def get_archive_type(archive_type: str, version: str) -> str: - """Return the archive of given type and version. - - Args: - archive_type (str): extension for archive type to check for - version (str): Version string in the form 2.83.2 - - Raises: - Exception: Execption when archive type isn't found - - Returns: - str: archive name for given type - """ - - for archive in blender_archives(version): - if archive.endswith(archive_type): - return archive - raise Exception("Unknown archive type") - - -def execute_command(cmd: List[str], name: str, errcode: int, cwd=".", capture_output=True) -> str: - """Execute the given command. - - Returns the process stdout upon success if any. - - On error print message the command with name that has failed. Print stdout - and stderr of the process if any, and then exit with given error code. - - Args: - cmd (List[str]): Command in list format, each argument as their own item - name (str): Name of command to use when printing to command-line - errcode (int): Error code to use in case of exit() - cwd (str, optional): Folder to use as current work directory for command - execution. Defaults to ".". - capture_output (bool, optional): Whether to capture command output or not. - Defaults to True. - - Returns: - str: stdout if any, or empty string - """ - cmd_process = subprocess.run( - cmd, capture_output=capture_output, encoding="UTF-8", cwd=cwd) - if cmd_process.returncode == 0: - if cmd_process.stdout: - return cmd_process.stdout - else: - return "" - else: - print(f"ERROR: {name} failed.") - if cmd_process.stdout: - print(cmd_process.stdout) - if cmd_process.stderr: - print(cmd_process.stderr) - exit(errcode) - return "" - - -def download_archives(base_url: str, archives: Callable[[str], Iterator[str]], version: str, dst_dir: pathlib.Path): - """Download archives from the given base_url. - - Archives is a generator for Blender archive names based on version. - - Archive names are appended to the base_url to load from, and appended to - dst_dir to save to. - - Args: - base_url (str): Base URL to load archives from - archives (Callable[[str], Iterator[str]]): Generator for Blender archive - names based on version - version (str): Version string in the form of 2.83.2 - dst_dir (pathlib.Path): Download destination - """ - - if base_url[-1] != '/': - base_url = base_url + '/' - - for archive in archives(version): - download_url = f"{base_url}{archive}" - target_file = dst_dir.joinpath(archive) - download_file(download_url, target_file) - - -def download_file(from_url: str, to_file: pathlib.Path) -> None: - """Download from_url as to_file. - - Actual downloading will be skipped if --skipdl is given on the command-line. - - Args: - from_url (str): Full URL to resource to download - to_file (pathlib.Path): Full path to save downloaded resource as - """ - global args - - if not args.skipdl or not to_file.exists(): - print(f"Downloading {from_url}") - with open(to_file, "wb") as download_zip: - response = requests.get(from_url) - if response.status_code != requests.codes.ok: - print(f"ERROR: failed to download {from_url} (status code: {response.status_code})") - exit(1313) - download_zip.write(response.content) - else: - print(f"Downloading {from_url} skipped") - print(" ... OK") - - -def copy_contents_from_dmg_to_path(dmg_file: pathlib.Path, dst: pathlib.Path) -> None: - """Copy the contents of the given DMG file to the destination folder. - - Args: - dmg_file (pathlib.Path): Full path to DMG archive to extract from - dst (pathlib.Path): Full path to destination to extract to - """ - hdiutil_attach = ["hdiutil", - "attach", - "-readonly", - f"{dmg_file}" - ] - attached = execute_command(hdiutil_attach, "hdiutil attach", 1) - - # Last line of output is what we want, it is of the form - # /dev/somedisk Apple_HFS /Volumes/Blender - # We want to retain the mount point, and the folder the mount is - # created on. The mounted disk we need for detaching, the folder we - # need to be able to copy the contents to where we can use them - attachment_items = attached.splitlines()[-1].split() - mounted_disk = attachment_items[0] - source_location = pathlib.Path(attachment_items[2], "Blender.app") - - print(f"{source_location} -> {dst}") - - shutil.copytree(source_location, dst) - - hdiutil_detach = ["hdiutil", - "detach", - f"{mounted_disk}" - ] - execute_command(hdiutil_detach, "hdiutil detach", 2) - - -def create_build_script(template_name: str, vars: List[Tuple[str, str]]) -> pathlib.Path: - """ - Create the Steam build script - - Use the given template and template variable tuple list. - - Returns pathlib.Path to the created script. - - Args: - template_name (str): [description] - vars (List[Tuple[str, str]]): [description] - - Returns: - pathlib.Path: Full path to the generated script - """ - build_script = pathlib.Path(".", template_name).read_text() - for var in vars: - build_script = build_script.replace(var[0], var[1]) - build_script_file = template_name.replace(".template", "") - build_script_path = pathlib.Path(".", build_script_file) - build_script_path.write_text(build_script) - return build_script_path - - -def clean_up() -> None: - """Remove intermediate files depending on given command-line arguments - """ - global content_location, args - - if not args.leavearch and not args.leaveextracted: - shutil.rmtree(content_location) - - if args.leavearch and not args.leaveextracted: - shutil.rmtree(content_location.joinpath(zip_extract_folder)) - shutil.rmtree(content_location.joinpath(tarxz_extract_folder)) - shutil.rmtree(content_location.joinpath(dmg_extract_folder)) - - if args.leaveextracted and not args.leavearch: - import os - os.remove(content_location.joinpath(zipped_blender)) - os.remove(content_location.joinpath(tarxz_blender)) - os.remove(content_location.joinpath(dmg_blender)) - - -def extract_archive(archive: str, extract_folder_name: str, - cmd: List[str], errcode: int) -> None: - """Extract all files from archive to given folder name. - - Will not extract if - target folder already exists, or if --skipextract was given on the - command-line. - - Args: - archive (str): Archive name to extract - extract_folder_name (str): Folder name to extract to - cmd (List[str]): Command with arguments to use - errcode (int): Error code to use for exit() - """ - global args, content_location - - extract_location = content_location.joinpath(extract_folder_name) - - pre_extract = set(content_location.glob("*")) - - if not args.skipextract or not extract_location.exists(): - print(f"Extracting files from {archive}...") - cmd.append(content_location.joinpath(archive)) - execute_command(cmd, cmd[0], errcode, cwd=content_location) - # in case we use a non-release archive the naming will be incorrect. - # simply rename to expected target name - post_extract = set(content_location.glob("*")) - diff_extract = post_extract - pre_extract - if not extract_location in diff_extract: - folder_to_rename = list(diff_extract)[0] - folder_to_rename.rename(extract_location) - print(" OK") - else: - print(f"Skipping extraction {archive}!") - -# ============================================================================== - - -parser = argparse.ArgumentParser() - -parser.add_argument("--baseurl", required=True, - help="The base URL for files to download, " - "i.e. https://download.blender.org/release/Blender2.83/") - -parser.add_argument("--version", required=True, - help="The Blender version to release, in the form 2.83.3") - -parser.add_argument("--appid", required=True, - help="The Blender App ID on Steam") -parser.add_argument("--winid", required=True, - help="The Windows depot ID") -parser.add_argument("--linuxid", required=True, - help="The Linux depot ID") -parser.add_argument("--macosid", required=True, - help="The MacOS depot ID") - -parser.add_argument("--steamcmd", required=True, - help="Path to the steamcmd") -parser.add_argument("--steamuser", required=True, - help="The login for the Steam builder user") -parser.add_argument("--steampw", required=True, - help="Login password for the Steam builder user") - -add_optional_argument("--dryrun", - "If set the Steam files will not be uploaded") -add_optional_argument("--leavearch", - help="If set don't clean up the downloaded archives") -add_optional_argument("--leaveextracted", - help="If set don't clean up the extraction folders") -add_optional_argument("--skipdl", - help="If set downloading the archives is skipped if it already exists locally.") -add_optional_argument("--skipextract", - help="If set skips extracting of archives. The tool assumes the archives" - "have already been extracted to their correct locations") - -args = parser.parse_args() - -VERSIONNODOTS = args.version.replace('.', '') -OUTPUT = f"output{VERSIONNODOTS}" -CONTENT = f"content{VERSIONNODOTS}" - -# ===== set up main locations - -content_location = pathlib.Path(".", CONTENT).absolute() -output_location = pathlib.Path(".", OUTPUT).absolute() - -content_location.mkdir(parents=True, exist_ok=True) -output_location.mkdir(parents=True, exist_ok=True) - -# ===== login - -# Logging into Steam once to ensure the SDK updates itself properly. If we don't -# do that the combined +login and +run_app_build_http at the end of the tool -# will fail. -steam_login = [args.steamcmd, - "+login", - args.steamuser, - args.steampw, - "+quit" - ] -print("Logging in to Steam...") -execute_command(steam_login, "Login to Steam", 10) -print(" OK") - -# ===== prepare Steam build scripts - -template_vars = [ - ("[APPID]", args.appid), - ("[OUTPUT]", OUTPUT), - ("[CONTENT]", CONTENT), - ("[VERSION]", args.version), - ("[WINID]", args.winid), - ("[LINUXID]", args.linuxid), - ("[MACOSID]", args.macosid), - ("[DRYRUN]", f"{args.dryrun}" if args.dryrun else "0") -] - -blender_app_build = create_build_script( - "blender_app_build.vdf.template", template_vars) -create_build_script("depot_build_win.vdf.template", template_vars) -create_build_script("depot_build_linux.vdf.template", template_vars) -create_build_script("depot_build_macos.vdf.template", template_vars) - -# ===== download archives - -download_archives(args.baseurl, blender_archives, - args.version, content_location) - -# ===== set up file and folder names - -zipped_blender = get_archive_type("zip", args.version) -zip_extract_folder = zipped_blender.replace(".zip", "") -tarxz_blender = get_archive_type("tar.xz", args.version) -tarxz_extract_folder = tarxz_blender.replace(".tar.xz", "") -dmg_blender = get_archive_type("dmg", args.version) -dmg_extract_folder = dmg_blender.replace(".dmg", "") - -# ===== extract - -unzip_cmd = ["unzip", "-q"] -extract_archive(zipped_blender, zip_extract_folder, unzip_cmd, 3) - -untarxz_cmd = ["tar", "-xf"] -extract_archive(tarxz_blender, tarxz_extract_folder, untarxz_cmd, 4) - -if not args.skipextract or not content_location.joinpath(dmg_extract_folder).exists(): - print("Extracting files from Blender MacOS archive...") - blender_dmg = content_location.joinpath(dmg_blender) - target_location = content_location.joinpath( - dmg_extract_folder, "Blender.app") - copy_contents_from_dmg_to_path(blender_dmg, target_location) - print(" OK") -else: - print("Skipping extraction of .dmg!") - -# ===== building - -print("Build Steam game files...") -steam_build = [args.steamcmd, - "+login", - args.steamuser, - args.steampw, - "+run_app_build_http", - blender_app_build.absolute(), - "+quit" - ] -execute_command(steam_build, "Build with steamcmd", 13) -print(" OK") - -clean_up() diff --git a/release/steam/depot_build_linux.vdf.template b/release/steam/depot_build_linux.vdf.template deleted file mode 100644 index 0f69008548e..00000000000 --- a/release/steam/depot_build_linux.vdf.template +++ /dev/null @@ -1,31 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[LINUXID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-linux64/" - - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} diff --git a/release/steam/depot_build_macos.vdf.template b/release/steam/depot_build_macos.vdf.template deleted file mode 100644 index 33dde860462..00000000000 --- a/release/steam/depot_build_macos.vdf.template +++ /dev/null @@ -1,30 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[MACOSID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-macOS/" - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} diff --git a/release/steam/depot_build_win.vdf.template b/release/steam/depot_build_win.vdf.template deleted file mode 100644 index 2c18a0f15dd..00000000000 --- a/release/steam/depot_build_win.vdf.template +++ /dev/null @@ -1,31 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[WINID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-windows64/" - - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} diff --git a/release/windows/msix/README.md b/release/windows/msix/README.md index 3f661a44066..96f753f0e78 100644 --- a/release/windows/msix/README.md +++ b/release/windows/msix/README.md @@ -1,82 +1,4 @@ -create_msix_package -=================== +Buildbot Configuration +====================== -This tool is used to create MSIX packages from a given ZiP archive. The MSIX -package is distributed mainly through the Microsoft Store. It can also be -installed when downloaded from blender.org. For that to work the MSIX package -needs to be signed. - -Requirements -============ - -* MakeAppX.exe - this tool is distributed with the Windows 10 SDK and is used to build the .appx package. -* MakePri.exe - this tool is distributed with the Windows 10 SDK and is used to generate a resources file. -* SignTool.exe - this tool is distributed with the Windows 10 SDK and is used to sign the .appx package. -* Python 3 (3.7 or later tested) - to run the create_msix_package.py script -* requests module - can be installed with `pip install requests` -* PFX file (optional, but strongly recommended) - for signing the resulting MSIX - package. **NOTE:** If the MSIX package is not signed when uploaded to the Microsoft - store the validation and certification process can take up to three full - business day. - -Usage -===== - -On the command-line: -```batch -set VERSION=2.83.4.0 -set URL=https://download.blender.org/release/Blender2.83/blender-2.83.4-windows64.zip -set PUBID=CN=PUBIDHERE -set PFX=X:\path\to\cert.pfx -set PFXPW=pwhere - -python create_msix_package.py --version %VERSION% --url %URL% --publisher %PUBID% --pfx %PFX% --password %PFXPW% -``` - -Result will be a MSIX package with the name `blender-2.83.4-windows64.msix`. -With the above usage it will be signed. If the signing options are left out the -package will not be signed. - -Optional arguments -================== - -In support of testing and developing the manifest and scripts there are a few -optional arguments: - -* `--skipdl` : If a `blender.zip` is available already next to the tool use this - to skip actual downloading of the archive designated by `--url`. The latter - option is still required -* `--overwrite` : When script fails the final clean-up may be incomplete leaving - the `Content` folder with its structure. Specify this argument to automatically - clean up this folder before starting to seed the `Content` folder -* `--leavezip` : When specified leave the `blender.zip` file while cleaning up - all other intermediate files, including the `Content` folder. This is useful - to not have to re-download the same archive from `--url` on each usage - - -What it does -============ - -The tool creates in the directory it lives a subfolder called `Content`. This is -where all necessary files are placed. - -The `Assets` folder is copied to the `Content` folder. - -From the application manifest template a version with necessary parts replaced as -their actual values as specified on the command-line is realized. This manifest controls the packaging of Blender into the MSIX format. - -Next the tool downloads the designated ZIP archive locally as blender.zip. From -this archive the files are extracted into the `Content\Blender` folder, but skip -the leading part of paths in the ZIP. We want to write the files to the -content_blender_folder where blender.exe ends up as -`Content\Blender\blender.exe`, and not -`Content\Blender\blender-2.83.4-windows64\blender.exe` - -Once the extraction is completed the MakeAppX tool is executed with the `Content` -folder as input. The result will be the MSIX package with the name in the form -`blender-X.YY.Z-windows64.msix`. - -If the PFX file and its password are given on the command-line this MSIX package -will be signed. - -All intermediate files and directories will be removed. +Files used by Buildbot's `package-code-store-windows` step. diff --git a/release/windows/msix/create_msix_package.py b/release/windows/msix/create_msix_package.py deleted file mode 100644 index 3e41484eef5..00000000000 --- a/release/windows/msix/create_msix_package.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import pathlib -import requests -import shutil -import subprocess -import zipfile - -parser = argparse.ArgumentParser() -parser.add_argument( - "--version", - required=True, - help="Version string in the form of 2.83.3.0", -) -parser.add_argument( - "--url", - required=True, - help="Location of the release ZIP archive to download", -) -parser.add_argument( - "--publisher", - required=True, - help="A string in the form of 'CN=PUBLISHER'", -) -parser.add_argument( - "--pfx", - required=False, - help="Absolute path to the PFX file used for signing the resulting MSIX package", -) -parser.add_argument( - "--password", - required=False, - default="blender", - help="Password for the PFX file", -) -parser.add_argument( - "--lts", - required=False, - help="If set this MSIX is for an LTS release", - action='store_const', - const=1, -) -parser.add_argument( - "--skipdl", - required=False, - help="If set skip downloading of the specified URL as blender.zip. The tool assumes blender.zip exists", - action='store_const', - const=1, -) -parser.add_argument( - "--leavezip", - required=False, - help="If set don't clean up the downloaded blender.zip", - action='store_const', - const=1, -) -parser.add_argument( - "--overwrite", - required=False, - help="If set remove Content folder if it already exists", - action='store_const', - const=1, -) -args = parser.parse_args() - - -def execute_command(cmd: list, name: str, errcode: int): - """ - Execute given command in cmd. Output is captured. If an error - occurs name is used to print ERROR message, along with stderr and - stdout of the process if either was captured. - """ - cmd_process = subprocess.run(cmd, capture_output=True, encoding="UTF-8") - if cmd_process.returncode != 0: - print(f"ERROR: {name} failed.") - if cmd_process.stdout: - print(cmd_process.stdout) - if cmd_process.stderr: - print(cmd_process.stderr) - exit(errcode) - - -LTSORNOT = "" -PACKAGETYPE = "" -if args.lts: - versionparts = args.version.split(".") - LTSORNOT = f" {versionparts[0]}.{versionparts[1]} LTS" - PACKAGETYPE = f"{versionparts[0]}.{versionparts[1]}LTS" - -blender_package_msix = pathlib.Path(".", f"blender-{args.version}-windows64.msix").absolute() -content_folder = pathlib.Path(".", "Content") -content_blender_folder = pathlib.Path(content_folder, "Blender").absolute() -content_assets_folder = pathlib.Path(content_folder, "Assets") -assets_original_folder = pathlib.Path(".", "Assets") - -pri_config_file = pathlib.Path(".", "priconfig.xml") -pri_resources_file = pathlib.Path(content_folder, "resources.pri") - -local_blender_zip = pathlib.Path(".", "blender.zip") - -if args.pfx: - pfx_path = pathlib.Path(args.pfx) - if not pfx_path.exists(): - print("ERROR: PFX file not found. Please ensure you give the correct path to the PFX file on the command-line.") - exit(1) - print(f"Creating MSIX package with signing using PFX file at {pfx_path}") -else: - pfx_path = None - print("Creating MSIX package without signing.") - -pri_command = ["makepri", - "new", - "/pr", f"{content_folder.absolute()}", - "/cf", f"{pri_config_file.absolute()}", - "/of", f"{pri_resources_file.absolute()}" - ] - -msix_command = ["makeappx", - "pack", - "/h", "SHA256", - "/d", f"{content_folder.absolute()}", - "/p", f"{blender_package_msix}" - ] -if pfx_path: - sign_command = ["signtool", - "sign", - "/fd", "sha256", - "/a", "/f", f"{pfx_path.absolute()}", - "/p", f"{args.password}", - f"{blender_package_msix}" - ] - -if args.overwrite: - if content_folder.joinpath("Assets").exists(): - shutil.rmtree(content_folder) -content_folder.mkdir(exist_ok=True) -shutil.copytree(assets_original_folder, content_assets_folder) - -manifest_text = pathlib.Path("AppxManifest.xml.template").read_text() -manifest_text = manifest_text.replace("[VERSION]", args.version) -manifest_text = manifest_text.replace("[PUBLISHER]", args.publisher) -manifest_text = manifest_text.replace("[LTSORNOT]", LTSORNOT) -manifest_text = manifest_text.replace("[PACKAGETYPE]", PACKAGETYPE) -pathlib.Path(content_folder, "AppxManifest.xml").write_text(manifest_text) - -if not args.skipdl: - print(f"Downloading blender archive {args.url} to {local_blender_zip}...") - - with open(local_blender_zip, "wb") as download_zip: - response = requests.get(args.url) - download_zip.write(response.content) - - print("... download complete.") -else: - print("Skipping download") - -print(f"Extracting files from ZIP to {content_blender_folder}...") - -# Extract the files from the ZIP archive, but skip the leading part of paths -# in the ZIP. We want to write the files to the content_blender_folder where -# blender.exe ends up as ./Content/Blender/blender.exe, and not -# ./Content/Blender/blender-2.83.3-windows64/blender.exe -with zipfile.ZipFile(local_blender_zip, "r") as blender_zip: - for entry in blender_zip.infolist(): - if entry.is_dir(): - continue - entry_location = pathlib.Path(entry.filename) - target_location = content_blender_folder.joinpath(*entry_location.parts[1:]) - pathlib.Path(target_location.parent).mkdir(parents=True, exist_ok=True) - extracted_entry = blender_zip.read(entry) - target_location.write_bytes(extracted_entry) - -print("... extraction complete.") - - -print(f"Generating Package Resource Index (PRI) file using command: {' '.join(pri_command)}") -execute_command(pri_command, "MakePri", 4) - -print(f"Creating MSIX package using command: {' '.join(msix_command)}") - -# Remove MSIX file if it already exists. Otherwise the MakeAppX tool -# will hang. -if blender_package_msix.exists(): - os.remove(blender_package_msix) -execute_command(msix_command, "MakeAppX", 2) - -if args.pfx: - print(f"Signing MSIX package using command: {' '.join(sign_command)}") - execute_command(sign_command, "SignTool", 3) - -if not args.leavezip: - os.remove(local_blender_zip) -shutil.rmtree(content_folder) - -print("Done.") |