diff options
Diffstat (limited to 'build_files/buildbot/codesign/base_code_signer.py')
| -rw-r--r-- | build_files/buildbot/codesign/base_code_signer.py | 501 |
1 files changed, 0 insertions, 501 deletions
diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py deleted file mode 100644 index f045e9c8242..00000000000 --- a/build_files/buildbot/codesign/base_code_signer.py +++ /dev/null @@ -1,501 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# <pep8 compliant> - -# Signing process overview. -# -# From buildbot worker side: -# - Files which needs to be signed are collected from either a directory to -# sign all signable files in there, or by filename of a single file to sign. -# - Those files gets packed into an archive and stored in a location location -# which is watched by the signing server. -# - A marker READY file is created which indicates the archive is ready for -# access. -# - Wait for the server to provide an archive with signed files. -# This is done by watching for the READY file which corresponds to an archive -# coming from the signing server. -# - Unpack the signed signed files from the archives and replace original ones. -# -# From code sign server: -# - Watch special location for a READY file which indicates the there is an -# archive with files which are to be signed. -# - Unpack the archive to a temporary location. -# - Run codesign tool and make sure all the files are signed. -# - Pack the signed files and store them in a location which is watched by -# the buildbot worker. -# - Create a READY file which indicates that the archive with signed files is -# ready. - -import abc -import logging -import shutil -import subprocess -import time -import tarfile -import uuid - -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Iterable, List - -import codesign.util as util - -from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName -from codesign.archive_with_indicator import ArchiveWithIndicator -from codesign.exception import CodeSignException - - -logger = logging.getLogger(__name__) -logger_builder = logger.getChild('builder') -logger_server = logger.getChild('server') - - -def pack_files(files: Iterable[AbsoluteAndRelativeFileName], - archive_filepath: Path) -> None: - """ - Create tar archive from given files for the signing pipeline. - Is used by buildbot worker to create an archive of files which are to be - signed, and by signing server to send signed files back to the worker. - """ - with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle: - for file_info in files: - tar_file_handle.add(file_info.absolute_filepath, - arcname=file_info.relative_filepath) - - -def extract_files(archive_filepath: Path, - extraction_dir: Path) -> None: - """ - Extract all files form the given archive into the given direcotry. - """ - - # TODO(sergey): Verify files in the archive have relative path. - - with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle: - tar_file_handle.extractall(path=extraction_dir) - - -class BaseCodeSigner(metaclass=abc.ABCMeta): - """ - Base class for a platform-specific signer of binaries. - - Contains all the logic shared across platform-specific implementations, such - as synchronization and notification logic. - - Platform specific bits (such as actual command for signing the binary) are - to be implemented as a subclass. - - Provides utilities code signing as a whole, including functionality needed - by a signing server and a buildbot worker. - - The signer and builder may run on separate machines, the only requirement is - that they have access to a directory which is shared between them. For the - security concerns this is to be done as a separate machine (or as a Shared - Folder configuration in VirtualBox configuration). This directory might be - mounted under different base paths, but its underlying storage is to be - the same. - - The code signer is short-lived on a buildbot worker side, and is living - forever on a code signing server side. - """ - - # TODO(sergey): Find a neat way to have config annotated. - # config: Config - - # Storage directory where builder puts files which are requested to be - # signed. - # Consider this an input of the code signing server. - unsigned_storage_dir: Path - - # Storage where signed files are stored. - # Consider this an output of the code signer server. - signed_storage_dir: Path - - # Platform the code is currently executing on. - platform: util.Platform - - def __init__(self, config): - self.config = config - - absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() - - # Unsigned (signing server input) configuration. - self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' - - # Signed (signing server output) configuration. - self.signed_storage_dir = absolute_shared_storage_dir / 'signed' - - self.platform = util.get_current_platform() - - def cleanup_environment_for_builder(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def cleanup_environment_for_signing_server(self) -> None: - # TODO(sergey): Revisit need of cleaning up the existing files. - # In practice it wasn't so helpful, and with multiple clients - # talking to the same server it becomes even more tricky. - pass - - def generate_request_id(self) -> str: - """ - Generate an unique identifier for code signing request. - """ - return str(uuid.uuid4()) - - def archive_info_for_request_id( - self, path: Path, request_id: str) -> ArchiveWithIndicator: - return ArchiveWithIndicator( - path, f'{request_id}.tar', f'{request_id}.ready') - - def signed_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.signed_storage_dir, request_id) - - def unsigned_archive_info_for_request_id( - self, request_id: str) -> ArchiveWithIndicator: - return self.archive_info_for_request_id( - self.unsigned_storage_dir, request_id) - - ############################################################################ - # Buildbot worker side helpers. - - @abc.abstractmethod - def check_file_is_to_be_signed( - self, file: AbsoluteAndRelativeFileName) -> bool: - """ - Check whether file is to be signed. - - Is used by both single file signing pipeline and recursive directory - signing pipeline. - - This is where code signer is to check whether file is to be signed or - not. This check might be based on a simple extension test or on actual - test whether file have a digital signature already or not. - """ - - def collect_files_to_sign(self, path: Path) \ - -> List[AbsoluteAndRelativeFileName]: - """ - Get all files which need to be signed from the given path. - - NOTE: The path might either be a file or directory. - - This function is run from the buildbot worker side. - """ - - # If there is a single file provided trust the buildbot worker that it - # is eligible for signing. - if path.is_file(): - file = AbsoluteAndRelativeFileName.from_path(path) - if not self.check_file_is_to_be_signed(file): - return [] - return [file] - - all_files = AbsoluteAndRelativeFileName.recursively_from_directory( - path) - files_to_be_signed = [file for file in all_files - if self.check_file_is_to_be_signed(file)] - return files_to_be_signed - - def wait_for_signed_archive_or_die(self, request_id) -> None: - """ - Wait until archive with signed files is available. - - Will only return if the archive with signed files is available. If there - was an error during code sign procedure the SystemExit exception is - raised, with the message set to the error reported by the codesign - server. - - Will only wait for the configured time. If that time exceeds and there - is still no responce from the signing server the application will exit - with a non-zero exit code. - - """ - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS - time_start = time.monotonic() - while not signed_archive_info.is_ready(): - time.sleep(1) - time_slept_in_seconds = time.monotonic() - time_start - if time_slept_in_seconds > timeout_in_seconds: - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit("Signing server didn't finish signing in " - f'{timeout_in_seconds} seconds, dying :(') - - archive_state = signed_archive_info.get_state() - if archive_state.has_error(): - signed_archive_info.clean() - unsigned_archive_info.clean() - raise SystemExit( - f'Error happenned during codesign procedure: {archive_state.error_message}') - - def copy_signed_files_to_directory( - self, signed_dir: Path, destination_dir: Path) -> None: - """ - Copy all files from signed_dir to destination_dir. - - This function will overwrite any existing file. Permissions are copied - from the source files, but other metadata, such as timestamps, are not. - """ - for signed_filepath in signed_dir.glob('**/*'): - if not signed_filepath.is_file(): - continue - - relative_filepath = signed_filepath.relative_to(signed_dir) - destination_filepath = destination_dir / relative_filepath - destination_filepath.parent.mkdir(parents=True, exist_ok=True) - - shutil.copy(signed_filepath, destination_filepath) - - def run_buildbot_path_sign_pipeline(self, path: Path) -> None: - """ - Run all steps needed to make given path signed. - - Path points to an unsigned file or a directory which contains unsigned - files. - - If the path points to a single file then this file will be signed. - This is used to sign a final bundle such as .msi on Windows or .dmg on - macOS. - - NOTE: The code signed implementation might actually reject signing the - file, in which case the file will be left unsigned. This isn't anything - to be considered a failure situation, just might happen when buildbot - worker can not detect whether signing is really required in a specific - case or not. - - If the path points to a directory then code signer will sign all - signable files from it (finding them recursively). - """ - - self.cleanup_environment_for_builder() - - # Make sure storage directory exists. - self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) - - # Collect all files which needs to be signed and pack them into a single - # archive which will be sent to the signing server. - logger_builder.info('Collecting files which are to be signed...') - files = self.collect_files_to_sign(path) - if not files: - logger_builder.info('No files to be signed, ignoring.') - return - logger_builder.info('Found %d files to sign.', len(files)) - - request_id = self.generate_request_id() - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - pack_files(files=files, - archive_filepath=unsigned_archive_info.archive_filepath) - unsigned_archive_info.tag_ready() - - # Wait for the signing server to finish signing. - logger_builder.info('Waiting signing server to sign the files...') - self.wait_for_signed_archive_or_die(request_id) - - # Extract signed files from archive and move files to final location. - with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: - unpacked_signed_files_dir = Path(temp_dir_str) - - logger_builder.info('Extracting signed files from archive...') - extract_files( - archive_filepath=signed_archive_info.archive_filepath, - extraction_dir=unpacked_signed_files_dir) - - destination_dir = path - if destination_dir.is_file(): - destination_dir = destination_dir.parent - self.copy_signed_files_to_directory( - unpacked_signed_files_dir, destination_dir) - - logger_builder.info('Removing archive with signed files...') - signed_archive_info.clean() - - ############################################################################ - # Signing server side helpers. - - def wait_for_sign_request(self) -> str: - """ - Wait for the buildbot to request signing of an archive. - - Returns an identifier of signing request. - """ - - # TOOD(sergey): Support graceful shutdown on Ctrl-C. - - logger_server.info( - f'Waiting for a request directory {self.unsigned_storage_dir} to appear.') - while not self.unsigned_storage_dir.exists(): - time.sleep(1) - - logger_server.info( - 'Waiting for a READY indicator of any signing request.') - request_id = None - while request_id is None: - for file in self.unsigned_storage_dir.iterdir(): - if file.suffix != '.ready': - continue - request_id = file.stem - logger_server.info(f'Found READY for request ID {request_id}.') - if request_id is None: - time.sleep(1) - - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - while not unsigned_archive_info.is_ready(): - time.sleep(1) - - return request_id - - @abc.abstractmethod - def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: - """ - Sign all files in the given directory. - - NOTE: Signing should happen in-place. - """ - - def run_signing_pipeline(self, request_id: str): - """ - Run the full signing pipeline starting from the point when buildbot - worker have requested signing. - """ - - # Make sure storage directory exists. - self.signed_storage_dir.mkdir(parents=True, exist_ok=True) - - with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: - temp_dir = Path(temp_dir_str) - - signed_archive_info = self.signed_archive_info_for_request_id( - request_id) - unsigned_archive_info = self.unsigned_archive_info_for_request_id( - request_id) - - logger_server.info('Extracting unsigned files from archive...') - extract_files( - archive_filepath=unsigned_archive_info.archive_filepath, - extraction_dir=temp_dir) - - logger_server.info('Collecting all files which needs signing...') - files = AbsoluteAndRelativeFileName.recursively_from_directory( - temp_dir) - - logger_server.info('Signing all requested files...') - try: - self.sign_all_files(files) - except CodeSignException as error: - signed_archive_info.tag_ready(error_message=error.message) - unsigned_archive_info.clean() - logger_server.info('Signing is complete with errors.') - return - - logger_server.info('Packing signed files...') - pack_files(files=files, - archive_filepath=signed_archive_info.archive_filepath) - signed_archive_info.tag_ready() - - logger_server.info('Removing signing request...') - unsigned_archive_info.clean() - - logger_server.info('Signing is complete.') - - def run_signing_server(self): - logger_server.info('Starting new code signing server...') - self.cleanup_environment_for_signing_server() - logger_server.info('Code signing server is ready') - while True: - logger_server.info('Waiting for the signing request in %s...', - self.unsigned_storage_dir) - request_id = self.wait_for_sign_request() - - logger_server.info( - f'Beging signign procedure for request ID {request_id}.') - self.run_signing_pipeline(request_id) - - ############################################################################ - # Command executing. - # - # Abstracted to a degree that allows to run commands from a foreign - # platform. - # The goal with this is to allow performing dry-run tests of code signer - # server from other platforms (for example, to test that macOS code signer - # does what it is supposed to after doing a refactor on Linux). - - # TODO(sergey): What is the type annotation for the command? - def run_command_or_mock(self, command, platform: util.Platform) -> None: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - logger_server.info(f'Running command: {command}') - subprocess.run(command) - - # TODO(sergey): What is the type annotation for the command? - def check_output_or_mock(self, command, - platform: util.Platform, - allow_nonzero_exit_code=False) -> str: - """ - Run given command if current platform matches given one - - If the platform is different then it will only be printed allowing - to verify logic of the code signing process. - - If allow_nonzero_exit_code is truth then the output will be returned - even if application quit with non-zero exit code. - Otherwise an subprocess.CalledProcessError exception will be raised - in such case. - """ - - if platform != self.platform: - logger_server.info( - f'Will run command for {platform}: {command}') - return - - if allow_nonzero_exit_code: - process = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = process.communicate()[0] - return output.decode() - - logger_server.info(f'Running command: {command}') - return subprocess.check_output( - command, stderr=subprocess.STDOUT).decode() |