diff options
Diffstat (limited to 'build_files/buildbot/codesign/base_code_signer.py')
| -rw-r--r-- | build_files/buildbot/codesign/base_code_signer.py | 385 |
1 files changed, 385 insertions, 0 deletions
diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py new file mode 100644 index 00000000000..ff4b4539658 --- /dev/null +++ b/build_files/buildbot/codesign/base_code_signer.py @@ -0,0 +1,385 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Signing process overview. +# +# From buildbot worker side: +# - Files which needs to be signed are collected from either a directory to +# sign all signable files in there, or by filename of a single file to sign. +# - Those files gets packed into an archive and stored in a location location +# which is watched by the signing server. +# - A marker READY file is created which indicates the archive is ready for +# access. +# - Wait for the server to provide an archive with signed files. +# This is done by watching for the READY file which corresponds to an archive +# coming from the signing server. +# - Unpack the signed signed files from the archives and replace original ones. +# +# From code sign server: +# - Watch special location for a READY file which indicates the there is an +# archive with files which are to be signed. +# - Unpack the archive to a temporary location. +# - Run codesign tool and make sure all the files are signed. +# - Pack the signed files and store them in a location which is watched by +# the buildbot worker. +# - Create a READY file which indicates that the archive with signed files is +# ready. + +import abc +import logging +import shutil +import time +import zipfile + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterable, List + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.archive_with_indicator import ArchiveWithIndicator + + +logger = logging.getLogger(__name__) +logger_builder = logger.getChild('builder') +logger_server = logger.getChild('server') + + +def pack_files(files: Iterable[AbsoluteAndRelativeFileName], + archive_filepath: Path) -> None: + """ + Create zip archive from given files for the signing pipeline. + Is used by buildbot worker to create an archive of files which are to be + signed, and by signing server to send signed files back to the worker. + """ + with zipfile.ZipFile(archive_filepath, 'w') as zip_file_handle: + for file_info in files: + zip_file_handle.write(file_info.absolute_filepath, + arcname=file_info.relative_filepath) + + +def extract_files(archive_filepath: Path, + extraction_dir: Path) -> None: + """ + Extract all files form the given archive into the given direcotry. + """ + + # TODO(sergey): Verify files in the archive have relative path. + + with zipfile.ZipFile(archive_filepath, mode='r') as zip_file_handle: + zip_file_handle.extractall(path=extraction_dir) + + +class BaseCodeSigner(metaclass=abc.ABCMeta): + """ + Base class for a platform-specific signer of binaries. + + Contains all the logic shared across platform-specific implementations, such + as synchronization and notification logic. + + Platform specific bits (such as actual command for signing the binary) are + to be implemented as a subclass. + + Provides utilities code signing as a whole, including functionality needed + by a signing server and a buildbot worker. + + The signer and builder may run on separate machines, the only requirement is + that they have access to a directory which is shared between them. For the + security concerns this is to be done as a separate machine (or as a Shared + Folder configuration in VirtualBox configuration). This directory might be + mounted under different base paths, but its underlying storage is to be + the same. + + The code signer is short-lived on a buildbot worker side, and is living + forever on a code signing server side. + """ + + # TODO(sergey): Find a neat way to have config annotated. + # config: Config + + # Storage directory where builder puts files which are requested to be + # signed. + # Consider this an input of the code signing server. + unsigned_storage_dir: Path + + # Information about archive which contains files which are to be signed. + # + # This archive is created by the buildbot worked and acts as an input for + # the code signing server. + unsigned_archive_info: ArchiveWithIndicator + + # Storage where signed files are stored. + # Consider this an output of the code signer server. + signed_storage_dir: Path + + # Information about archive which contains signed files. + # + # This archive is created by the code signing server. + signed_archive_info: ArchiveWithIndicator + + def __init__(self, config): + self.config = config + + absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() + + # Unsigned (signing server input) configuration. + self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' + self.unsigned_archive_info = ArchiveWithIndicator( + self.unsigned_storage_dir, 'unsigned_files.zip', 'ready.stamp') + + # Signed (signing server output) configuration. + self.signed_storage_dir = absolute_shared_storage_dir / 'signed' + self.signed_archive_info = ArchiveWithIndicator( + self.signed_storage_dir, 'signed_files.zip', 'ready.stamp') + + """ + General note on cleanup environment functions. + + It is expected that there is only one instance of the code signer server + running for a given input/output directory, and that it serves a single + buildbot worker. + By its nature, a buildbot worker only produces one build at a time and + never performs concurrent builds. + This leads to a conclusion that when starting in a clean environment + there shouldn't be any archives remaining from a previous build. + + However, it is possible to have various failure scenarios which might + leave the environment in a non-clean state: + + - Network hiccup which makes buildbot worker to stop current build + and re-start it after connection to server is re-established. + + Note, this could also happen during buildbot server maintenance. + + - Signing server might get restarted due to updates or other reasons. + + Requiring manual interaction in such cases is not something good to + require, so here we simply assume that the system is used the way it is + intended to and restore environment to a prestine clean state. + """ + + def cleanup_environment_for_builder(self) -> None: + self.unsigned_archive_info.clean() + self.signed_archive_info.clean() + + def cleanup_environment_for_signing_server(self) -> None: + # Don't clear the requested to-be-signed archive since we might be + # restarting signing machine while the buildbot is busy. + self.signed_archive_info.clean() + + ############################################################################ + # Buildbot worker side helpers. + + @abc.abstractmethod + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + """ + Check whether file is to be signed. + + Is used by both single file signing pipeline and recursive directory + signing pipeline. + + This is where code signer is to check whether file is to be signed or + not. This check might be based on a simple extension test or on actual + test whether file have a digital signature already or not. + """ + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + """ + Get all files which need to be signed from the given path. + + NOTE: The path might either be a file or directory. + + This function is run from the buildbot worker side. + """ + + # If there is a single file provided trust the buildbot worker that it + # is eligible for signing. + if path.is_file(): + file = AbsoluteAndRelativeFileName.from_path(path) + if not self.check_file_is_to_be_signed(file): + return [] + return [file] + + all_files = AbsoluteAndRelativeFileName.recursively_from_directory( + path) + files_to_be_signed = [file for file in all_files + if self.check_file_is_to_be_signed(file)] + return files_to_be_signed + + def wait_for_signed_archive_or_die(self) -> None: + """ + Wait until archive with signed files is available. + + Will only wait for the configured time. If that time exceeds and there + is still no responce from the signing server the application will exit + with a non-zero exit code. + """ + timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS + time_start = time.monotonic() + while not self.signed_archive_info.is_ready(): + time.sleep(1) + time_slept_in_seconds = time.monotonic() - time_start + if time_slept_in_seconds > timeout_in_seconds: + self.unsigned_archive_info.clean() + raise SystemExit("Signing server didn't finish signing in " + f"{timeout_in_seconds} seconds, dying :(") + + def copy_signed_files_to_directory( + self, signed_dir: Path, destination_dir: Path) -> None: + """ + Copy all files from signed_dir to destination_dir. + + This function will overwrite any existing file. Permissions are copied + from the source files, but other metadata, such as timestamps, are not. + """ + for signed_filepath in signed_dir.glob('**/*'): + if not signed_filepath.is_file(): + continue + + relative_filepath = signed_filepath.relative_to(signed_dir) + destination_filepath = destination_dir / relative_filepath + destination_filepath.parent.mkdir(parents=True, exist_ok=True) + + shutil.copy(signed_filepath, destination_filepath) + + def run_buildbot_path_sign_pipeline(self, path: Path) -> None: + """ + Run all steps needed to make given path signed. + + Path points to an unsigned file or a directory which contains unsigned + files. + + If the path points to a single file then this file will be signed. + This is used to sign a final bundle such as .msi on Windows or .dmg on + macOS. + + NOTE: The code signed implementation might actually reject signing the + file, in which case the file will be left unsigned. This isn't anything + to be considered a failure situation, just might happen when buildbot + worker can not detect whether signing is really required in a specific + case or not. + + If the path points to a directory then code signer will sign all + signable files from it (finding them recursively). + """ + + self.cleanup_environment_for_builder() + + # Make sure storage directory exists. + self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) + + # Collect all files which needs to be signed and pack them into a single + # archive which will be sent to the signing server. + logger_builder.info('Collecting files which are to be signed...') + files = self.collect_files_to_sign(path) + if not files: + logger_builder.info('No files to be signed, ignoring.') + return + logger_builder.info('Found %d files to sign.', len(files)) + + pack_files(files=files, + archive_filepath=self.unsigned_archive_info.archive_filepath) + self.unsigned_archive_info.tag_ready() + + # Wait for the signing server to finish signing. + logger_builder.info('Waiting signing server to sign the files...') + self.wait_for_signed_archive_or_die() + + # Extract signed files from archive and move files to final location. + with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: + unpacked_signed_files_dir = Path(temp_dir_str) + + logger_builder.info('Extracting signed files from archive...') + extract_files( + archive_filepath=self.signed_archive_info.archive_filepath, + extraction_dir=unpacked_signed_files_dir) + + destination_dir = path + if destination_dir.is_file(): + destination_dir = destination_dir.parent + self.copy_signed_files_to_directory( + unpacked_signed_files_dir, destination_dir) + + ############################################################################ + # Signing server side helpers. + + def wait_for_sign_request(self) -> None: + """ + Wait for the buildbot to request signing of an archive. + """ + # TOOD(sergey): Support graceful shutdown on Ctrl-C. + while not self.unsigned_archive_info.is_ready(): + time.sleep(1) + + @abc.abstractmethod + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + """ + Sign all files in the given directory. + + NOTE: Signing should happen in-place. + """ + + def run_signing_pipeline(self): + """ + Run the full signing pipeline starting from the point when buildbot + worker have requested signing. + """ + + # Make sure storage directory exists. + self.signed_storage_dir.mkdir(parents=True, exist_ok=True) + + with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: + temp_dir = Path(temp_dir_str) + + logger_server.info('Extracting unsigned files from archive...') + extract_files( + archive_filepath=self.unsigned_archive_info.archive_filepath, + extraction_dir=temp_dir) + + logger_server.info('Collecting all files which needs signing...') + files = AbsoluteAndRelativeFileName.recursively_from_directory( + temp_dir) + + logger_server.info('Signing all requested files...') + self.sign_all_files(files) + + logger_server.info('Packing signed files...') + pack_files(files=files, + archive_filepath=self.signed_archive_info.archive_filepath) + self.signed_archive_info.tag_ready() + + logger_server.info('Removing signing request...') + self.unsigned_archive_info.clean() + + logger_server.info('Signing is complete.') + + def run_signing_server(self): + logger_server.info('Starting new code signing server...') + self.cleanup_environment_for_signing_server() + logger_server.info('Code signing server is ready') + while True: + logger_server.info('Waiting for the signing request in %s...', + self.unsigned_storage_dir) + self.wait_for_sign_request() + + logger_server.info( + 'Got signing request, beging signign procedure.') + self.run_signing_pipeline() |