diff options
Diffstat (limited to 'build_files/buildbot/codesign')
| -rw-r--r-- | build_files/buildbot/codesign/absolute_and_relative_filename.py | 81 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/archive_with_indicator.py | 245 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/base_code_signer.py | 501 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/config_builder.py | 62 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/config_common.py | 36 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/config_server_template.py | 101 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/exception.py | 26 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/linux_code_signer.py | 72 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/macos_code_signer.py | 456 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/simple_code_signer.py | 52 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/util.py | 54 | ||||
| -rw-r--r-- | build_files/buildbot/codesign/windows_code_signer.py | 117 |
12 files changed, 0 insertions, 1803 deletions
diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py deleted file mode 100644 index cb42710e785..00000000000 --- a/build_files/buildbot/codesign/absolute_and_relative_filename.py +++ /dev/null @@ -1,81 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from dataclasses import dataclass -from pathlib import Path -from typing import List - - -@dataclass -class AbsoluteAndRelativeFileName: - """ - Helper class which keeps track of absolute file path for a direct access and - corresponding relative path against given base. - - The relative part is used to construct a file name within an archive which - contains files which are to be signed or which has been signed already - (depending on whether the archive is addressed to signing server or back - to the buildbot worker). - """ - - # Base directory which is where relative_filepath is relative to. - base_dir: Path - - # Full absolute path of the corresponding file. - absolute_filepath: Path - - # Derived from full file path, contains part of the path which is relative - # to a desired base path. - relative_filepath: Path - - def __init__(self, base_dir: Path, filepath: Path): - self.base_dir = base_dir - self.absolute_filepath = filepath.resolve() - self.relative_filepath = self.absolute_filepath.relative_to( - self.base_dir) - - @classmethod - def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName': - assert path.is_absolute() - assert path.is_file() - - base_dir = path.parent - return AbsoluteAndRelativeFileName(base_dir, path) - - @classmethod - def recursively_from_directory(cls, base_dir: Path) \ - -> List['AbsoluteAndRelativeFileName']: - """ - Create list of AbsoluteAndRelativeFileName for all the files in the - given directory. - - NOTE: Result will be pointing to a resolved paths. - """ - assert base_dir.is_absolute() - assert base_dir.is_dir() - - base_dir = base_dir.resolve() - - result = [] - for filename in base_dir.glob('**/*'): - if not filename.is_file(): - continue - result.append(AbsoluteAndRelativeFileName(base_dir, filename)) - return result diff --git a/build_files/buildbot/codesign/archive_with_indicator.py b/build_files/buildbot/codesign/archive_with_indicator.py deleted file mode 100644 index aebf5a15417..00000000000 --- a/build_files/buildbot/codesign/archive_with_indicator.py +++ /dev/null @@ -1,245 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import dataclasses -import json -import os - -from pathlib import Path -from typing import Optional - -import codesign.util as util - - -class ArchiveStateError(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) - - -@dataclasses.dataclass -class ArchiveState: - """ - Additional information (state) of the archive - - Includes information like expected file size of the archive file in the case - the archive file is expected to be successfully created. - - If the archive can not be created, this state will contain error message - indicating details of error. - """ - - # Size in bytes of the corresponding archive. - file_size: Optional[int] = None - - # Non-empty value indicates that error has happenned. - error_message: str = '' - - def has_error(self) -> bool: - """ - Check whether the archive is at error state - """ - - return self.error_message - - def serialize_to_string(self) -> str: - payload = dataclasses.asdict(self) - return json.dumps(payload, sort_keys=True, indent=4) - - def serialize_to_file(self, filepath: Path) -> None: - string = self.serialize_to_string() - filepath.write_text(string) - - @classmethod - def deserialize_from_string(cls, string: str) -> 'ArchiveState': - try: - object_as_dict = json.loads(string) - except json.decoder.JSONDecodeError: - raise ArchiveStateError('Error parsing JSON') - - return cls(**object_as_dict) - - @classmethod - def deserialize_from_file(cls, filepath: Path): - string = filepath.read_text() - return cls.deserialize_from_string(string) - - -class ArchiveWithIndicator: - """ - The idea of this class is to wrap around logic which takes care of keeping - track of a name of an archive and synchronization routines between buildbot - worker and signing server. - - The synchronization is done based on creating a special file after the - archive file is knowingly ready for access. - """ - - # Base directory where the archive is stored (basically, a basename() of - # the absolute archive file name). - # - # For example, 'X:\\TEMP\\'. - base_dir: Path - - # Absolute file name of the archive. - # - # For example, 'X:\\TEMP\\FOO.ZIP'. - archive_filepath: Path - - # Absolute name of a file which acts as an indication of the fact that the - # archive is ready and is available for access. - # - # This is how synchronization between buildbot worker and signing server is - # done: - # - First, the archive is created under archive_filepath name. - # - Second, the indication file is created under ready_indicator_filepath - # name. - # - Third, the colleague of whoever created the indicator name watches for - # the indication file to appear, and once it's there it access the - # archive. - ready_indicator_filepath: Path - - def __init__( - self, base_dir: Path, archive_name: str, ready_indicator_name: str): - """ - Construct the object from given base directory and name of the archive - file: - ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY') - """ - - self.base_dir = base_dir - self.archive_filepath = self.base_dir / archive_name - self.ready_indicator_filepath = self.base_dir / ready_indicator_name - - def is_ready_unsafe(self) -> bool: - """ - Check whether the archive is ready for access. - - No guarding about possible network failres is done here. - """ - if not self.ready_indicator_filepath.exists(): - return False - - try: - archive_state = ArchiveState.deserialize_from_file( - self.ready_indicator_filepath) - except ArchiveStateError as error: - print(f'Error deserializing archive state: {error.message}') - return False - - if archive_state.has_error(): - # If the error did happen during codesign procedure there will be no - # corresponding archive file. - # The caller code will deal with the error check further. - return True - - # Sometimes on macOS indicator file appears prior to the actual archive - # despite the order of creation and os.sync() used in tag_ready(). - # So consider archive not ready if there is an indicator without an - # actual archive. - if not self.archive_filepath.exists(): - print('Found indicator without actual archive, waiting for archive ' - f'({self.archive_filepath}) to appear.') - return False - - # Wait for until archive is fully stored. - actual_archive_size = self.archive_filepath.stat().st_size - if actual_archive_size != archive_state.file_size: - print('Partial/invalid archive size (expected ' - f'{archive_state.file_size} got {actual_archive_size})') - return False - - return True - - def is_ready(self) -> bool: - """ - Check whether the archive is ready for access. - - Will tolerate possible network failures: if there is a network failure - or if there is still no proper permission on a file False is returned. - """ - - # There are some intermitten problem happening at a random which is - # translates to "OSError : [WinError 59] An unexpected network error occurred". - # Some reports suggests it might be due to lack of permissions to the file, - # which might be applicable in our case since it's possible that file is - # initially created with non-accessible permissions and gets chmod-ed - # after initial creation. - try: - return self.is_ready_unsafe() - except OSError as e: - print(f'Exception checking archive: {e}') - return False - - def tag_ready(self, error_message='') -> None: - """ - Tag the archive as ready by creating the corresponding indication file. - - NOTE: It is expected that the archive was never tagged as ready before - and that there are no subsequent tags of the same archive. - If it is violated, an assert will fail. - """ - assert not self.is_ready() - - # Try the best to make sure everything is synced to the file system, - # to avoid any possibility of stamp appearing on a network share prior to - # an actual file. - if util.get_current_platform() != util.Platform.WINDOWS: - os.sync() - - archive_size = -1 - if self.archive_filepath.exists(): - archive_size = self.archive_filepath.stat().st_size - - archive_info = ArchiveState( - file_size=archive_size, error_message=error_message) - - self.ready_indicator_filepath.write_text( - archive_info.serialize_to_string()) - - def get_state(self) -> ArchiveState: - """ - Get state object for this archive - - The state is read from the corresponding state file. - """ - - try: - return ArchiveState.deserialize_from_file(self.ready_indicator_filepath) - except ArchiveStateError as error: - return ArchiveState(error_message=f'Error in information format: {error}') - - def clean(self) -> None: - """ - Remove both archive and the ready indication file. - """ - util.ensure_file_does_not_exist_or_die(self.ready_indicator_filepath) - util.ensure_file_does_not_exist_or_die(self.archive_filepath) - - def is_fully_absent(self) -> bool: - """ - Check whether both archive and its ready indicator are absent. - Is used for a sanity check during code signing process by both - buildbot worker and signing server. - """ - return (not self.archive_filepath.exists() and - not self.ready_indicator_filepath.exists()) diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py deleted file mode 100644 index f045e9c8242..00000000000 --- a/build_files/buildbot/codesign/base_code_signer.py +++ /dev/null @@ -1,501 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Signing process overview. -# -# From buildbot worker side: -# - Files which needs to be signed are collected from either a directory to -# sign all signable files in there, or by filename of a single file to sign. -# - Those files gets packed into an archive and stored in a location location -# which is watched by the signing server. -# - A marker READY file is created which indicates the archive is ready for -# access. -# - Wait for the server to provide an archive with signed files. -# This is done by watching for the READY file which corresponds to an archive -# coming from the signing server. -# - Unpack the signed signed files from the archives and replace original ones. -# -# From code sign server: -# - Watch special location for a READY file which indicates the there is an -# archive with files which are to be signed. -# - Unpack the archive to a temporary location. -# - Run codesign tool and make sure all the files are signed. -# - Pack the signed files and store them in a location which is watched by -# the buildbot worker. -# - Create a READY file which indicates that the archive with signed files is -# ready. - -import abc -import logging -import shutil -import subprocess -import time -import tarfile -import uuid - -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Iterable, List - -import codesign.util as util - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.archive_with_indicator import ArchiveWithIndicator -from codesign.exception import CodeSignException - - -logger = logging.getLogger(__name__) -logger_builder = logger.getChild('builder') -logger_server = logger.getChild('server') - - -def pack_files(files: Iterable[AbsoluteAndRelativeFileName], - archive_filepath: Path) -> None: - """ - Create tar archive from given files for the signing pipeline. - Is used by buildbot worker to create an archive of files which are to be - signed, and by signing server to send signed files back to the worker. - """ - with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle: - for file_info in files: - tar_file_handle.add(file_info.absolute_filepath, - arcname=file_info.relative_filepath) - - -def extract_files(archive_filepath: Path, - extraction_dir: Path) -> None: - """ - Extract all files form the given archive into the given direcotry. - """ - - # TODO(sergey): Verify files in the archive have relative path. - - with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle: - tar_file_handle.extractall(path=extraction_dir) - - -class BaseCodeSigner(metaclass=abc.ABCMeta): - """ - Base class for a platform-specific signer of binaries. - - Contains all the logic shared across platform-specific implementations, such - as synchronization and notification logic. - - Platform specific bits (such as actual command for signing the binary) are - to be implemented as a subclass. - - Provides utilities code signing as a whole, including functionality needed - by a signing server and a buildbot worker. - - The signer and builder may run on separate machines, the only requirement is - that they have access to a directory which is shared between them. For the - security concerns this is to be done as a separate machine (or as a Shared - Folder configuration in VirtualBox configuration). This directory might be - mounted under different base paths, but its underlying storage is to be - the same. - - The code signer is short-lived on a buildbot worker side, and is living - forever on a code signing server side. - """ - - # TODO(sergey): Find a neat way to have config annotated. - # config: Config - - # Storage directory where builder puts files which are requested to be - # signed. - # Consider this an input of the code signing server. - unsigned_storage_dir: Path - - # Storage where signed files are stored. - # Consider this an output of the code signer server. - signed_storage_dir: Path - - # Platform the code is currently executing on. - platform: util.Platform - - def __init__(self, config): - self.config = config - - absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() - - # Unsigned (signing server input) configuration. - self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' - - # Signed (signing server output) configuration. - self.signed_storage_dir = absolute_shared_storage_dir / 'signed' - - self.platform = util.get_current_platform() - - def cleanup_environment_for_builder(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def cleanup_environment_for_signing_server(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def generate_request_id(self) -> str: - """ - Generate an unique identifier for code signing request. - """ - return str(uuid.uuid4()) - - def archive_info_for_request_id( - self, path: Path, request_id: str) -> ArchiveWithIndicator: - return ArchiveWithIndicator( - path, f'{request_id}.tar', f'{request_id}.ready') - - def signed_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.signed_storage_dir, request_id) - - def unsigned_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.unsigned_storage_dir, request_id) - - ############################################################################ - # Buildbot worker side helpers. - - @abc.abstractmethod - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is to be signed. - - Is used by both single file signing pipeline and recursive directory - signing pipeline. - - This is where code signer is to check whether file is to be signed or - not. This check might be based on a simple extension test or on actual - test whether file have a digital signature already or not. - """ - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - """ - Get all files which need to be signed from the given path. - - NOTE: The path might either be a file or directory. - - This function is run from the buildbot worker side. - """ - - # If there is a single file provided trust the buildbot worker that it - # is eligible for signing. - if path.is_file(): - file = AbsoluteAndRelativeFileName.from_path(path) - if not self.check_file_is_to_be_signed(file): - return [] - return [file] - - all_files = AbsoluteAndRelativeFileName.recursively_from_directory( - path) - files_to_be_signed = [file for file in all_files - if self.check_file_is_to_be_signed(file)] - return files_to_be_signed - - def wait_for_signed_archive_or_die(self, request_id) -> None: - """ - Wait until archive with signed files is available. - - Will only return if the archive with signed files is available. If there - was an error during code sign procedure the SystemExit exception is - raised, with the message set to the error reported by the codesign - server. - - Will only wait for the configured time. If that time exceeds and there - is still no responce from the signing server the application will exit - with a non-zero exit code. - - """ - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS - time_start = time.monotonic() - while not signed_archive_info.is_ready(): - time.sleep(1) - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit("Signing server didn't finish signing in " - f'{timeout_in_seconds} seconds, dying :(') - - archive_state = signed_archive_info.get_state() - if archive_state.has_error(): - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit( - f'Error happenned during codesign procedure: {archive_state.error_message}') - - def copy_signed_files_to_directory( - self, signed_dir: Path, destination_dir: Path) -> None: - """ - Copy all files from signed_dir to destination_dir. - - This function will overwrite any existing file. Permissions are copied - from the source files, but other metadata, such as timestamps, are not. - """ - for signed_filepath in signed_dir.glob('**/*'): - if not signed_filepath.is_file(): - continue - - relative_filepath = signed_filepath.relative_to(signed_dir) - destination_filepath = destination_dir / relative_filepath - destination_filepath.parent.mkdir(parents=True, exist_ok=True) - - shutil.copy(signed_filepath, destination_filepath) - - def run_buildbot_path_sign_pipeline(self, path: Path) -> None: - """ - Run all steps needed to make given path signed. - - Path points to an unsigned file or a directory which contains unsigned - files. - - If the path points to a single file then this file will be signed. - This is used to sign a final bundle such as .msi on Windows or .dmg on - macOS. - - NOTE: The code signed implementation might actually reject signing the - file, in which case the file will be left unsigned. This isn't anything - to be considered a failure situation, just might happen when buildbot - worker can not detect whether signing is really required in a specific - case or not. - - If the path points to a directory then code signer will sign all - signable files from it (finding them recursively). - """ - - self.cleanup_environment_for_builder() - - # Make sure storage directory exists. - self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) - - # Collect all files which needs to be signed and pack them into a single - # archive which will be sent to the signing server. - logger_builder.info('Collecting files which are to be signed...') - files = self.collect_files_to_sign(path) - if not files: - logger_builder.info('No files to be signed, ignoring.') - return - logger_builder.info('Found %d files to sign.', len(files)) - - request_id = self.generate_request_id() - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - pack_files(files=files, - archive_filepath=unsigned_archive_info.archive_filepath) - unsigned_archive_info.tag_ready() - - # Wait for the signing server to finish signing. - logger_builder.info('Waiting signing server to sign the files...') - self.wait_for_signed_archive_or_die(request_id) - - # Extract signed files from archive and move files to final location. - with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: - unpacked_signed_files_dir = Path(temp_dir_str) - - logger_builder.info('Extracting signed files from archive...') - extract_files( - archive_filepath=signed_archive_info.archive_filepath, - extraction_dir=unpacked_signed_files_dir) - - destination_dir = path - if destination_dir.is_file(): - destination_dir = destination_dir.parent - self.copy_signed_files_to_directory( - unpacked_signed_files_dir, destination_dir) - - logger_builder.info('Removing archive with signed files...') - signed_archive_info.clean() - - ############################################################################ - # Signing server side helpers. - - def wait_for_sign_request(self) -> str: - """ - Wait for the buildbot to request signing of an archive. - - Returns an identifier of signing request. - """ - - # TOOD(sergey): Support graceful shutdown on Ctrl-C. - - logger_server.info( - f'Waiting for a request directory {self.unsigned_storage_dir} to appear.') - while not self.unsigned_storage_dir.exists(): - time.sleep(1) - - logger_server.info( - 'Waiting for a READY indicator of any signing request.') - request_id = None - while request_id is None: - for file in self.unsigned_storage_dir.iterdir(): - if file.suffix != '.ready': - continue - request_id = file.stem - logger_server.info(f'Found READY for request ID {request_id}.') - if request_id is None: - time.sleep(1) - - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - while not unsigned_archive_info.is_ready(): - time.sleep(1) - - return request_id - - @abc.abstractmethod - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Sign all files in the given directory. - - NOTE: Signing should happen in-place. - """ - - def run_signing_pipeline(self, request_id: str): - """ - Run the full signing pipeline starting from the point when buildbot - worker have requested signing. - """ - - # Make sure storage directory exists. - self.signed_storage_dir.mkdir(parents=True, exist_ok=True) - - with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: - temp_dir = Path(temp_dir_str) - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - logger_server.info('Extracting unsigned files from archive...') - extract_files( - archive_filepath=unsigned_archive_info.archive_filepath, - extraction_dir=temp_dir) - - logger_server.info('Collecting all files which needs signing...') - files = AbsoluteAndRelativeFileName.recursively_from_directory( - temp_dir) - - logger_server.info('Signing all requested files...') - try: - self.sign_all_files(files) - except CodeSignException as error: - signed_archive_info.tag_ready(error_message=error.message) - unsigned_archive_info.clean() - logger_server.info('Signing is complete with errors.') - return - - logger_server.info('Packing signed files...') - pack_files(files=files, - archive_filepath=signed_archive_info.archive_filepath) - signed_archive_info.tag_ready() - - logger_server.info('Removing signing request...') - unsigned_archive_info.clean() - - logger_server.info('Signing is complete.') - - def run_signing_server(self): - logger_server.info('Starting new code signing server...') - self.cleanup_environment_for_signing_server() - logger_server.info('Code signing server is ready') - while True: - logger_server.info('Waiting for the signing request in %s...', - self.unsigned_storage_dir) - request_id = self.wait_for_sign_request() - - logger_server.info( - f'Beging signign procedure for request ID {request_id}.') - self.run_signing_pipeline(request_id) - - ############################################################################ - # Command executing. - # - # Abstracted to a degree that allows to run commands from a foreign - # platform. - # The goal with this is to allow performing dry-run tests of code signer - # server from other platforms (for example, to test that macOS code signer - # does what it is supposed to after doing a refactor on Linux). - - # TODO(sergey): What is the type annotation for the command? - def run_command_or_mock(self, command, platform: util.Platform) -> None: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - logger_server.info(f'Running command: {command}') - subprocess.run(command) - - # TODO(sergey): What is the type annotation for the command? - def check_output_or_mock(self, command, - platform: util.Platform, - allow_nonzero_exit_code=False) -> str: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - - If allow_nonzero_exit_code is truth then the output will be returned - even if application quit with non-zero exit code. - Otherwise an subprocess.CalledProcessError exception will be raised - in such case. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - if allow_nonzero_exit_code: - process = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = process.communicate()[0] - return output.decode() - - logger_server.info(f'Running command: {command}') - return subprocess.check_output( - command, stderr=subprocess.STDOUT).decode() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py deleted file mode 100644 index 1f41619ba13..00000000000 --- a/build_files/buildbot/codesign/config_builder.py +++ /dev/null @@ -1,62 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code running from -# buildbot's worker. - -import sys - -from pathlib import Path - -import codesign.util as util - -from codesign.config_common import * - -platform = util.get_current_platform() -if platform == util.Platform.LINUX: - SHARED_STORAGE_DIR = Path('/data/codesign') -elif platform == util.Platform.WINDOWS: - SHARED_STORAGE_DIR = Path('Z:\\codesign') -elif platform == util.Platform.MACOS: - SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign') - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py deleted file mode 100644 index a37bc731dc0..00000000000 --- a/build_files/buildbot/codesign/config_common.py +++ /dev/null @@ -1,36 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -from pathlib import Path - -# Timeout in seconds for the signing process. -# -# This is how long buildbot packing step will wait signing server to -# perform signing. -# -# NOTE: Notarization could take a long time, hence the rather high value -# here. Might consider using different timeout for different platforms. -TIMEOUT_IN_SECONDS = 45 * 60 * 60 - -# Directory which is shared across buildbot worker and signing server. -# -# This is where worker puts files requested for signing as well as where -# server puts signed files. -SHARED_STORAGE_DIR: Path diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py deleted file mode 100644 index ff97ed15fa5..00000000000 --- a/build_files/buildbot/codesign/config_server_template.py +++ /dev/null @@ -1,101 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Configuration of a code signer which is specific to the code signing server. -# -# NOTE: DO NOT put any sensitive information here, put it in an actual -# configuration on the signing machine. - -from pathlib import Path - -from codesign.config_common import * - -CODESIGN_DIRECTORY = Path(__file__).absolute().parent -BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent - -################################################################################ -# Common configuration. - -# Directory where folders for codesign requests and signed result are stored. -# For example, /data/codesign -SHARED_STORAGE_DIR: Path - -################################################################################ -# macOS-specific configuration. - -MACOS_ENTITLEMENTS_FILE = \ - BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist' - -# Identity of the Developer ID Application certificate which is to be used for -# codesign tool. -# Use `security find-identity -v -p codesigning` to find the identity. -# -# NOTE: This identity is just an example from release/darwin/README.txt. -MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97' - -# User name (Apple ID) which will be used to request notarization. -MACOS_XCRUN_USERNAME = 'me@example.com' - -# One-time application password which will be used to request notarization. -MACOS_XCRUN_PASSWORD = '@keychain:altool-password' - -# Timeout in seconds within which the notarial office is supposed to reply. -MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60 - -################################################################################ -# Windows-specific configuration. - -# URL to the timestamping authority. -WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' - -# Full path to the certificate used for signing. -# -# The path and expected file format might vary depending on a platform. -# -# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look -# like Path('C:\\Secret\\Blender.pfx'). -WIN_CERTIFICATE_FILEPATH: Path - -################################################################################ -# Logging configuration, common for all platforms. - -# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema -LOGGING = { - 'version': 1, - 'formatters': { - 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - 'stream': 'ext://sys.stderr', - } - }, - 'loggers': { - 'codesign': {'level': 'INFO'}, - }, - 'root': { - 'level': 'WARNING', - 'handlers': [ - 'console', - ], - } -} diff --git a/build_files/buildbot/codesign/exception.py b/build_files/buildbot/codesign/exception.py deleted file mode 100644 index 6c8a9f262a5..00000000000 --- a/build_files/buildbot/codesign/exception.py +++ /dev/null @@ -1,26 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -class CodeSignException(Exception): - message: str - - def __init__(self, message): - self.message = message - super().__init__(self.message) diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py deleted file mode 100644 index 04935f67832..00000000000 --- a/build_files/buildbot/codesign/linux_code_signer.py +++ /dev/null @@ -1,72 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# NOTE: This is a no-op signer (since there isn't really a procedure to sign -# Linux binaries yet). Used to debug and verify the code signing routines on -# a Linux environment. - -import logging - -from pathlib import Path -from typing import List - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - - -class LinuxCodeSigner(BaseCodeSigner): - def is_active(self) -> bool: - """ - Check whether this signer is active. - - if it is inactive, no files will be signed. - - Is used to be able to debug code signing pipeline on Linux, where there - is no code signing happening in the actual buildbot and release - environment. - """ - return False - - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath == Path('blender'): - return True - if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and - file.relative_filepath.name.startwith('python')): - return True - if file.relative_filepath.suffix == '.so': - return True - return False - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - if not self.is_active(): - return [] - - return super().collect_files_to_sign(path) - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - num_files = len(files) - for file_index, file in enumerate(files): - logger.info('Server: Signed file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) diff --git a/build_files/buildbot/codesign/macos_code_signer.py b/build_files/buildbot/codesign/macos_code_signer.py deleted file mode 100644 index f03dad8e1d6..00000000000 --- a/build_files/buildbot/codesign/macos_code_signer.py +++ /dev/null @@ -1,456 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging -import re -import stat -import subprocess -import time - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'} - -# Prefixes of a file (not directory) name which are to be signed. -# Used to sign extra executable files in Contents/Resources. -NAME_PREFIXES_TO_BE_SIGNED = {'python'} - - -class NotarizationException(CodeSignException): - pass - - -def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is coming from an .app bundle - """ - parts = file.relative_filepath.parts - if not parts: - return False - if not parts[0].endswith('.app'): - return False - return True - - -def get_bundle_from_file( - file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName: - """ - Get AbsoluteAndRelativeFileName descriptor of bundle - """ - assert(is_file_from_bundle(file)) - - parts = file.relative_filepath.parts - bundle_name = parts[0] - - base_dir = file.base_dir - bundle_filepath = file.base_dir / bundle_name - return AbsoluteAndRelativeFileName(base_dir, bundle_filepath) - - -def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether given file is an executable within an app bundle - """ - if not is_file_from_bundle(file): - return False - - parts = file.relative_filepath.parts - num_parts = len(parts) - if num_parts < 3: - return False - - if parts[1:3] != ('Contents', 'MacOS'): - return False - - return True - - -def xcrun_field_value_from_output(field: str, output: str) -> str: - """ - Get value of a given field from xcrun output. - - If field is not found empty string is returned. - """ - - field_prefix = field + ': ' - for line in output.splitlines(): - line = line.strip() - if line.startswith(field_prefix): - return line[len(field_prefix):] - return '' - - -class MacOSCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - if file.relative_filepath.name.startswith('.'): - return False - - if is_bundle_executable_file(file): - return True - - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in NAME_PREFIXES_TO_BE_SIGNED): - return True - - mode = file.absolute_filepath.lstat().st_mode - if mode & stat.S_IXUSR != 0: - file_output = subprocess.check_output( - ("file", file.absolute_filepath)).decode() - if "64-bit executable" in file_output: - return True - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - # Include all files when signing app or dmg bundle: all the files are - # needed to do valid signature of bundle. - if path.name.endswith('.app'): - return AbsoluteAndRelativeFileName.recursively_from_directory(path) - if path.is_dir(): - files = [] - for child in path.iterdir(): - if child.name.endswith('.app'): - current_files = AbsoluteAndRelativeFileName.recursively_from_directory( - child) - else: - current_files = super().collect_files_to_sign(child) - for current_file in current_files: - files.append(AbsoluteAndRelativeFileName( - path, current_file.absolute_filepath)) - return files - return super().collect_files_to_sign(path) - - ############################################################################ - # Codesign. - - def codesign_remove_signature( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Make sure given file does not have codesign signature - - This is needed because codesigning is not possible for file which has - signature already. - """ - - logger_server.info( - 'Removing codesign signature from %s...', file.relative_filepath) - - command = ['codesign', '--remove-signature', file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_file( - self, file: AbsoluteAndRelativeFileName) -> None: - """ - Sign given file - - NOTE: File must not have any signatures. - """ - - logger_server.info( - 'Codesigning %s...', file.relative_filepath) - - entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE - command = ['codesign', - '--timestamp', - '--options', 'runtime', - f'--entitlements={entitlements_file}', - '--sign', self.config.MACOS_CODESIGN_IDENTITY, - file.absolute_filepath] - self.run_command_or_mock(command, util.Platform.MACOS) - - def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Run codesign tool on all eligible files in the given list. - - Will ignore all files which are not to be signed. For the rest will - remove possible existing signature and add a new signature. - """ - - num_files = len(files) - have_ignored_files = False - signed_files = [] - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of a bundle and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - have_ignored_files = True - continue - - logger_server.info( - 'Running codesigning routines for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - - self.codesign_remove_signature(file) - self.codesign_file(file) - - signed_files.append(file) - - if have_ignored_files: - logger_server.info('Signed %d files:', len(signed_files)) - num_signed_files = len(signed_files) - for file_index, signed_file in enumerate(signed_files): - logger_server.info( - '- [%d/%d] %s', - file_index + 1, num_signed_files, - signed_file.relative_filepath) - - def codesign_bundles( - self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Codesign all .app bundles in the given list of files. - - Bundle is deducted from paths of the files, and every bundle is only - signed once. - """ - - signed_bundles = set() - extra_files = [] - - for file in files: - if not is_file_from_bundle(file): - continue - bundle = get_bundle_from_file(file) - bundle_name = bundle.relative_filepath - if bundle_name in signed_bundles: - continue - - logger_server.info('Running codesign routines on bundle %s', - bundle_name) - - # It is not possible to remove signature from DMG. - if bundle.relative_filepath.name.endswith('.app'): - self.codesign_remove_signature(bundle) - self.codesign_file(bundle) - - signed_bundles.add(bundle_name) - - # Codesign on a bundle adds an extra folder with information. - # It needs to be compied to the source. - code_signature_directory = \ - bundle.absolute_filepath / 'Contents' / '_CodeSignature' - code_signature_files = \ - AbsoluteAndRelativeFileName.recursively_from_directory( - code_signature_directory) - for code_signature_file in code_signature_files: - bundle_relative_file = AbsoluteAndRelativeFileName( - bundle.base_dir, - code_signature_directory / - code_signature_file.relative_filepath) - extra_files.append(bundle_relative_file) - - files.extend(extra_files) - - ############################################################################ - # Notarization. - - def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str: - """ - Get bundle ID which will be used to notarize DMG - """ - name = file.relative_filepath.name - app_name = name.split('-', 2)[0].lower() - - app_name_words = app_name.split() - if len(app_name_words) > 1: - app_name_id = ''.join(word.capitalize() for word in app_name_words) - else: - app_name_id = app_name_words[0] - - # TODO(sergey): Consider using "alpha" for buildbot builds. - return f'org.blenderfoundation.{app_name_id}.release' - - def notarize_request(self, file) -> str: - """ - Request notarization of the given file. - - Returns UUID of the notarization request. If error occurred None is - returned instead of UUID. - """ - - bundle_id = self.notarize_get_bundle_id(file) - logger_server.info('Bundle ID: %s', bundle_id) - - logger_server.info('Submitting file to the notarial office.') - command = [ - 'xcrun', 'altool', '--notarize-app', '--verbose', - '-f', file.absolute_filepath, - '--primary-bundle-id', bundle_id, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - for line in output.splitlines(): - line = line.strip() - if line.startswith('RequestUUID = '): - request_uuid = line[14:] - return request_uuid - - # Check whether the package has been already submitted. - if 'The software asset has already been uploaded.' in line: - request_uuid = re.sub( - '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line) - logger_server.warning( - f'The package has been already submitted under UUID {request_uuid}') - return request_uuid - - logger_server.error(output) - logger_server.error('xcrun command did not report RequestUUID') - return None - - def notarize_review_status(self, xcrun_output: str) -> bool: - """ - Review status returned by xcrun's notarization info - - Returns truth if the notarization process has finished. - If there are errors during notarization, a NotarizationException() - exception is thrown with status message from the notarial office. - """ - - # Parse status and message - status = xcrun_field_value_from_output('Status', xcrun_output) - status_message = xcrun_field_value_from_output( - 'Status Message', xcrun_output) - - if status == 'success': - logger_server.info( - 'Package successfully notarized: %s', status_message) - return True - - if status == 'invalid': - logger_server.error(xcrun_output) - logger_server.error( - 'Package notarization has failed: %s', status_message) - raise NotarizationException(status_message) - - if status == 'in progress': - return False - - logger_server.info( - 'Unknown notarization status %s (%s)', status, status_message) - - return False - - def notarize_wait_result(self, request_uuid: str) -> None: - """ - Wait for until notarial office have a reply - """ - - logger_server.info( - 'Waiting for a result from the notarization office.') - - command = ['xcrun', 'altool', - '--notarization-info', request_uuid, - '--username', self.config.MACOS_XCRUN_USERNAME, - '--password', self.config.MACOS_XCRUN_PASSWORD] - - time_start = time.monotonic() - timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS - - while True: - xcrun_output = self.check_output_or_mock( - command, util.Platform.MACOS, allow_nonzero_exit_code=True) - - if self.notarize_review_status(xcrun_output): - break - - logger_server.info('Keep waiting for notarization office.') - time.sleep(30) - - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - logger_server.error( - "Notarial office didn't reply in %f seconds.", - timeout_in_seconds) - - def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Staple notarial label on the file - """ - - logger_server.info('Stapling notarial stamp.') - - command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath] - self.check_output_or_mock(command, util.Platform.MACOS) - - def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Run entire pipeline to get DMG notarized. - """ - logger_server.info('Begin notarization routines on %s', - file.relative_filepath) - - # Submit file for notarization. - request_uuid = self.notarize_request(file) - if not request_uuid: - return False - logger_server.info('Received Request UUID: %s', request_uuid) - - # Wait for the status from the notarization office. - if not self.notarize_wait_result(request_uuid): - return False - - # Staple. - self.notarize_staple(file) - - def notarize_all_dmg( - self, files: List[AbsoluteAndRelativeFileName]) -> bool: - """ - Notarize all DMG images from the input. - - Images are supposed to be codesigned already. - """ - for file in files: - if not file.relative_filepath.name.endswith('.dmg'): - continue - if not self.check_file_is_to_be_signed(file): - continue - - self.notarize_dmg(file) - - ############################################################################ - # Entry point. - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # TODO(sergey): Handle errors somehow. - - self.codesign_all_files(files) - self.codesign_bundles(files) - self.notarize_all_dmg(files) diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py deleted file mode 100644 index 674d9e9ce9e..00000000000 --- a/build_files/buildbot/codesign/simple_code_signer.py +++ /dev/null @@ -1,52 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - - -import logging.config -import sys - -from pathlib import Path -from typing import Optional - -import codesign.config_builder -import codesign.util as util -from codesign.base_code_signer import BaseCodeSigner - - -class SimpleCodeSigner: - code_signer: Optional[BaseCodeSigner] - - def __init__(self): - platform = util.get_current_platform() - if platform == util.Platform.LINUX: - from codesign.linux_code_signer import LinuxCodeSigner - self.code_signer = LinuxCodeSigner(codesign.config_builder) - elif platform == util.Platform.MACOS: - from codesign.macos_code_signer import MacOSCodeSigner - self.code_signer = MacOSCodeSigner(codesign.config_builder) - elif platform == util.Platform.WINDOWS: - from codesign.windows_code_signer import WindowsCodeSigner - self.code_signer = WindowsCodeSigner(codesign.config_builder) - else: - self.code_signer = None - - def sign_file_or_directory(self, path: Path) -> None: - logging.config.dictConfig(codesign.config_builder.LOGGING) - self.code_signer.run_buildbot_path_sign_pipeline(path) diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py deleted file mode 100644 index e67292dd049..00000000000 --- a/build_files/buildbot/codesign/util.py +++ /dev/null @@ -1,54 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import sys - -from enum import Enum -from pathlib import Path - - -class Platform(Enum): - LINUX = 1 - MACOS = 2 - WINDOWS = 3 - - -def get_current_platform() -> Platform: - if sys.platform == 'linux': - return Platform.LINUX - elif sys.platform == 'darwin': - return Platform.MACOS - elif sys.platform == 'win32': - return Platform.WINDOWS - raise Exception(f'Unknown platform {sys.platform}') - - -def ensure_file_does_not_exist_or_die(filepath: Path) -> None: - """ - If the file exists, unlink it. - If the file path exists and is not a file an assert will trigger. - If the file path does not exists nothing happens. - """ - if not filepath.exists(): - return - if not filepath.is_file(): - # TODO(sergey): Provide information about what the filepath actually is. - raise SystemExit(f'{filepath} is expected to be a file, but is not') - filepath.unlink() diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py deleted file mode 100644 index db185788a56..00000000000 --- a/build_files/buildbot/codesign/windows_code_signer.py +++ /dev/null @@ -1,117 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -import logging - -from pathlib import Path -from typing import List - -import codesign.util as util - -from buildbot_utils import Builder - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.base_code_signer import BaseCodeSigner -from codesign.exception import CodeSignException - -logger = logging.getLogger(__name__) -logger_server = logger.getChild('server') - -# NOTE: Check is done as filename.endswith(), so keep the dot -EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'} - -BLACKLIST_FILE_PREFIXES = ( - 'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime') - - -class SigntoolException(CodeSignException): - pass - -class WindowsCodeSigner(BaseCodeSigner): - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - base_name = file.relative_filepath.name - if any(base_name.startswith(prefix) - for prefix in BLACKLIST_FILE_PREFIXES): - return False - - return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED - - - def get_sign_command_prefix(self) -> List[str]: - return [ - 'signtool', 'sign', '/v', - '/f', self.config.WIN_CERTIFICATE_FILEPATH, - '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL] - - - def run_codesign_tool(self, filepath: Path) -> None: - command = self.get_sign_command_prefix() + [filepath] - - try: - codesign_output = self.check_output_or_mock(command, util.Platform.WINDOWS) - except subprocess.CalledProcessError as e: - raise SigntoolException(f'Error running signtool {e}') - - logger_server.info(f'signtool output:\n{codesign_output}') - - got_number_of_success = False - - for line in codesign_output.split('\n'): - line_clean = line.strip() - line_clean_lower = line_clean.lower() - - if line_clean_lower.startswith('number of warnings') or \ - line_clean_lower.startswith('number of errors'): - number = int(line_clean_lower.split(':')[1]) - if number != 0: - raise SigntoolException('Non-clean success of signtool') - - if line_clean_lower.startswith('number of files successfully signed'): - got_number_of_success = True - number = int(line_clean_lower.split(':')[1]) - if number != 1: - raise SigntoolException('Signtool did not consider codesign a success') - - if not got_number_of_success: - raise SigntoolException('Signtool did not report number of files signed') - - - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - # NOTE: Sign files one by one to avoid possible command line length - # overflow (which could happen if we ever decide to sign every binary - # in the install folder, for example). - # - # TODO(sergey): Consider doing batched signing of handful of files in - # one go (but only if this actually known to be much faster). - num_files = len(files) - for file_index, file in enumerate(files): - # Ignore file if it is not to be signed. - # Allows to manually construct ZIP of package and get it signed. - if not self.check_file_is_to_be_signed(file): - logger_server.info( - 'Ignoring file [%d/%d] %s', - file_index + 1, num_files, file.relative_filepath) - continue - - logger_server.info( - 'Running signtool command for file [%d/%d] %s...', - file_index + 1, num_files, file.relative_filepath) - self.run_codesign_tool(file.absolute_filepath) |