diff options
Diffstat (limited to 'build_files/buildbot/codesign')
-rw-r--r-- | build_files/buildbot/codesign/absolute_and_relative_filename.py | 77 | ||||
-rw-r--r-- | build_files/buildbot/codesign/archive_with_indicator.py | 101 | ||||
-rw-r--r-- | build_files/buildbot/codesign/base_code_signer.py | 385 | ||||
-rw-r--r-- | build_files/buildbot/codesign/config_builder.py | 57 | ||||
-rw-r--r-- | build_files/buildbot/codesign/config_common.py | 33 | ||||
-rw-r--r-- | build_files/buildbot/codesign/config_server_template.py | 63 | ||||
-rw-r--r-- | build_files/buildbot/codesign/linux_code_signer.py | 72 | ||||
-rw-r--r-- | build_files/buildbot/codesign/simple_code_signer.py | 47 | ||||
-rw-r--r-- | build_files/buildbot/codesign/util.py | 35 | ||||
-rw-r--r-- | build_files/buildbot/codesign/windows_code_signer.py | 75 |
10 files changed, 945 insertions, 0 deletions
diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py new file mode 100644 index 00000000000..bea9ea7e8d0 --- /dev/null +++ b/build_files/buildbot/codesign/absolute_and_relative_filename.py @@ -0,0 +1,77 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from dataclasses import dataclass +from pathlib import Path +from typing import List + + +@dataclass +class AbsoluteAndRelativeFileName: + """ + Helper class which keeps track of absolute file path for a direct access and + corresponding relative path against given base. + + The relative part is used to construct a file name within an archive which + contains files which are to be signed or which has been signed already + (depending on whether the archive is addressed to signing server or back + to the buildbot worker). + """ + + # Base directory which is where relative_filepath is relative to. + base_dir: Path + + # Full absolute path of the corresponding file. + absolute_filepath: Path + + # Derived from full file path, contains part of the path which is relative + # to a desired base path. + relative_filepath: Path + + def __init__(self, base_dir: Path, filepath: Path): + self.base_dir = base_dir + self.absolute_filepath = filepath.resolve() + self.relative_filepath = self.absolute_filepath.relative_to( + self.base_dir) + + @classmethod + def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName': + assert path.is_absolute() + assert path.is_file() + + base_dir = path.parent + return AbsoluteAndRelativeFileName(base_dir, path) + + @classmethod + def recursively_from_directory(cls, base_dir: Path) \ + -> List['AbsoluteAndRelativeFileName']: + """ + Create list of AbsoluteAndRelativeFileName for all the files in the + given directory. + """ + assert base_dir.is_absolute() + assert base_dir.is_dir() + + result = [] + for filename in base_dir.glob('**/*'): + if not filename.is_file(): + continue + result.append(AbsoluteAndRelativeFileName(base_dir, filename)) + return result diff --git a/build_files/buildbot/codesign/archive_with_indicator.py b/build_files/buildbot/codesign/archive_with_indicator.py new file mode 100644 index 00000000000..51bcc28520d --- /dev/null +++ b/build_files/buildbot/codesign/archive_with_indicator.py @@ -0,0 +1,101 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + +from codesign.util import ensure_file_does_not_exist_or_die + + +class ArchiveWithIndicator: + """ + The idea of this class is to wrap around logic which takes care of keeping + track of a name of an archive and synchronization routines between buildbot + worker and signing server. + + The synchronization is done based on creating a special file after the + archive file is knowingly ready for access. + """ + + # Base directory where the archive is stored (basically, a basename() of + # the absolute archive file name). + # + # For example, 'X:\\TEMP\\'. + base_dir: Path + + # Absolute file name of the archive. + # + # For example, 'X:\\TEMP\\FOO.ZIP'. + archive_filepath: Path + + # Absolute name of a file which acts as an indication of the fact that the + # archive is ready and is available for access. + # + # This is how synchronization between buildbot worker and signing server is + # done: + # - First, the archive is created under archive_filepath name. + # - Second, the indication file is created under ready_indicator_filepath + # name. + # - Third, the colleague of whoever created the indicator name watches for + # the indication file to appear, and once it's there it access the + # archive. + ready_indicator_filepath: Path + + def __init__( + self, base_dir: Path, archive_name: str, ready_indicator_name: str): + """ + Construct the object from given base directory and name of the archive + file: + ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY') + """ + + self.base_dir = base_dir + self.archive_filepath = self.base_dir / archive_name + self.ready_indicator_filepath = self.base_dir / ready_indicator_name + + def is_ready(self) -> bool: + """Check whether the archive is ready for access.""" + return self.ready_indicator_filepath.exists() + + def tag_ready(self) -> None: + """ + Tag the archive as ready by creating the corresponding indication file. + + NOTE: It is expected that the archive was never tagged as ready before + and that there are no subsequent tags of the same archive. + If it is violated, an assert will fail. + """ + assert not self.is_ready() + self.ready_indicator_filepath.touch() + + def clean(self) -> None: + """ + Remove both archive and the ready indication file. + """ + ensure_file_does_not_exist_or_die(self.ready_indicator_filepath) + ensure_file_does_not_exist_or_die(self.archive_filepath) + + def is_fully_absent(self) -> bool: + """ + Check whether both archive and its ready indicator are absent. + Is used for a sanity check during code signing process by both + buildbot worker and signing server. + """ + return (not self.archive_filepath.exists() and + not self.ready_indicator_filepath.exists()) diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py new file mode 100644 index 00000000000..ff4b4539658 --- /dev/null +++ b/build_files/buildbot/codesign/base_code_signer.py @@ -0,0 +1,385 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Signing process overview. +# +# From buildbot worker side: +# - Files which needs to be signed are collected from either a directory to +# sign all signable files in there, or by filename of a single file to sign. +# - Those files gets packed into an archive and stored in a location location +# which is watched by the signing server. +# - A marker READY file is created which indicates the archive is ready for +# access. +# - Wait for the server to provide an archive with signed files. +# This is done by watching for the READY file which corresponds to an archive +# coming from the signing server. +# - Unpack the signed signed files from the archives and replace original ones. +# +# From code sign server: +# - Watch special location for a READY file which indicates the there is an +# archive with files which are to be signed. +# - Unpack the archive to a temporary location. +# - Run codesign tool and make sure all the files are signed. +# - Pack the signed files and store them in a location which is watched by +# the buildbot worker. +# - Create a READY file which indicates that the archive with signed files is +# ready. + +import abc +import logging +import shutil +import time +import zipfile + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterable, List + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.archive_with_indicator import ArchiveWithIndicator + + +logger = logging.getLogger(__name__) +logger_builder = logger.getChild('builder') +logger_server = logger.getChild('server') + + +def pack_files(files: Iterable[AbsoluteAndRelativeFileName], + archive_filepath: Path) -> None: + """ + Create zip archive from given files for the signing pipeline. + Is used by buildbot worker to create an archive of files which are to be + signed, and by signing server to send signed files back to the worker. + """ + with zipfile.ZipFile(archive_filepath, 'w') as zip_file_handle: + for file_info in files: + zip_file_handle.write(file_info.absolute_filepath, + arcname=file_info.relative_filepath) + + +def extract_files(archive_filepath: Path, + extraction_dir: Path) -> None: + """ + Extract all files form the given archive into the given direcotry. + """ + + # TODO(sergey): Verify files in the archive have relative path. + + with zipfile.ZipFile(archive_filepath, mode='r') as zip_file_handle: + zip_file_handle.extractall(path=extraction_dir) + + +class BaseCodeSigner(metaclass=abc.ABCMeta): + """ + Base class for a platform-specific signer of binaries. + + Contains all the logic shared across platform-specific implementations, such + as synchronization and notification logic. + + Platform specific bits (such as actual command for signing the binary) are + to be implemented as a subclass. + + Provides utilities code signing as a whole, including functionality needed + by a signing server and a buildbot worker. + + The signer and builder may run on separate machines, the only requirement is + that they have access to a directory which is shared between them. For the + security concerns this is to be done as a separate machine (or as a Shared + Folder configuration in VirtualBox configuration). This directory might be + mounted under different base paths, but its underlying storage is to be + the same. + + The code signer is short-lived on a buildbot worker side, and is living + forever on a code signing server side. + """ + + # TODO(sergey): Find a neat way to have config annotated. + # config: Config + + # Storage directory where builder puts files which are requested to be + # signed. + # Consider this an input of the code signing server. + unsigned_storage_dir: Path + + # Information about archive which contains files which are to be signed. + # + # This archive is created by the buildbot worked and acts as an input for + # the code signing server. + unsigned_archive_info: ArchiveWithIndicator + + # Storage where signed files are stored. + # Consider this an output of the code signer server. + signed_storage_dir: Path + + # Information about archive which contains signed files. + # + # This archive is created by the code signing server. + signed_archive_info: ArchiveWithIndicator + + def __init__(self, config): + self.config = config + + absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() + + # Unsigned (signing server input) configuration. + self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' + self.unsigned_archive_info = ArchiveWithIndicator( + self.unsigned_storage_dir, 'unsigned_files.zip', 'ready.stamp') + + # Signed (signing server output) configuration. + self.signed_storage_dir = absolute_shared_storage_dir / 'signed' + self.signed_archive_info = ArchiveWithIndicator( + self.signed_storage_dir, 'signed_files.zip', 'ready.stamp') + + """ + General note on cleanup environment functions. + + It is expected that there is only one instance of the code signer server + running for a given input/output directory, and that it serves a single + buildbot worker. + By its nature, a buildbot worker only produces one build at a time and + never performs concurrent builds. + This leads to a conclusion that when starting in a clean environment + there shouldn't be any archives remaining from a previous build. + + However, it is possible to have various failure scenarios which might + leave the environment in a non-clean state: + + - Network hiccup which makes buildbot worker to stop current build + and re-start it after connection to server is re-established. + + Note, this could also happen during buildbot server maintenance. + + - Signing server might get restarted due to updates or other reasons. + + Requiring manual interaction in such cases is not something good to + require, so here we simply assume that the system is used the way it is + intended to and restore environment to a prestine clean state. + """ + + def cleanup_environment_for_builder(self) -> None: + self.unsigned_archive_info.clean() + self.signed_archive_info.clean() + + def cleanup_environment_for_signing_server(self) -> None: + # Don't clear the requested to-be-signed archive since we might be + # restarting signing machine while the buildbot is busy. + self.signed_archive_info.clean() + + ############################################################################ + # Buildbot worker side helpers. + + @abc.abstractmethod + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + """ + Check whether file is to be signed. + + Is used by both single file signing pipeline and recursive directory + signing pipeline. + + This is where code signer is to check whether file is to be signed or + not. This check might be based on a simple extension test or on actual + test whether file have a digital signature already or not. + """ + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + """ + Get all files which need to be signed from the given path. + + NOTE: The path might either be a file or directory. + + This function is run from the buildbot worker side. + """ + + # If there is a single file provided trust the buildbot worker that it + # is eligible for signing. + if path.is_file(): + file = AbsoluteAndRelativeFileName.from_path(path) + if not self.check_file_is_to_be_signed(file): + return [] + return [file] + + all_files = AbsoluteAndRelativeFileName.recursively_from_directory( + path) + files_to_be_signed = [file for file in all_files + if self.check_file_is_to_be_signed(file)] + return files_to_be_signed + + def wait_for_signed_archive_or_die(self) -> None: + """ + Wait until archive with signed files is available. + + Will only wait for the configured time. If that time exceeds and there + is still no responce from the signing server the application will exit + with a non-zero exit code. + """ + timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS + time_start = time.monotonic() + while not self.signed_archive_info.is_ready(): + time.sleep(1) + time_slept_in_seconds = time.monotonic() - time_start + if time_slept_in_seconds > timeout_in_seconds: + self.unsigned_archive_info.clean() + raise SystemExit("Signing server didn't finish signing in " + f"{timeout_in_seconds} seconds, dying :(") + + def copy_signed_files_to_directory( + self, signed_dir: Path, destination_dir: Path) -> None: + """ + Copy all files from signed_dir to destination_dir. + + This function will overwrite any existing file. Permissions are copied + from the source files, but other metadata, such as timestamps, are not. + """ + for signed_filepath in signed_dir.glob('**/*'): + if not signed_filepath.is_file(): + continue + + relative_filepath = signed_filepath.relative_to(signed_dir) + destination_filepath = destination_dir / relative_filepath + destination_filepath.parent.mkdir(parents=True, exist_ok=True) + + shutil.copy(signed_filepath, destination_filepath) + + def run_buildbot_path_sign_pipeline(self, path: Path) -> None: + """ + Run all steps needed to make given path signed. + + Path points to an unsigned file or a directory which contains unsigned + files. + + If the path points to a single file then this file will be signed. + This is used to sign a final bundle such as .msi on Windows or .dmg on + macOS. + + NOTE: The code signed implementation might actually reject signing the + file, in which case the file will be left unsigned. This isn't anything + to be considered a failure situation, just might happen when buildbot + worker can not detect whether signing is really required in a specific + case or not. + + If the path points to a directory then code signer will sign all + signable files from it (finding them recursively). + """ + + self.cleanup_environment_for_builder() + + # Make sure storage directory exists. + self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) + + # Collect all files which needs to be signed and pack them into a single + # archive which will be sent to the signing server. + logger_builder.info('Collecting files which are to be signed...') + files = self.collect_files_to_sign(path) + if not files: + logger_builder.info('No files to be signed, ignoring.') + return + logger_builder.info('Found %d files to sign.', len(files)) + + pack_files(files=files, + archive_filepath=self.unsigned_archive_info.archive_filepath) + self.unsigned_archive_info.tag_ready() + + # Wait for the signing server to finish signing. + logger_builder.info('Waiting signing server to sign the files...') + self.wait_for_signed_archive_or_die() + + # Extract signed files from archive and move files to final location. + with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: + unpacked_signed_files_dir = Path(temp_dir_str) + + logger_builder.info('Extracting signed files from archive...') + extract_files( + archive_filepath=self.signed_archive_info.archive_filepath, + extraction_dir=unpacked_signed_files_dir) + + destination_dir = path + if destination_dir.is_file(): + destination_dir = destination_dir.parent + self.copy_signed_files_to_directory( + unpacked_signed_files_dir, destination_dir) + + ############################################################################ + # Signing server side helpers. + + def wait_for_sign_request(self) -> None: + """ + Wait for the buildbot to request signing of an archive. + """ + # TOOD(sergey): Support graceful shutdown on Ctrl-C. + while not self.unsigned_archive_info.is_ready(): + time.sleep(1) + + @abc.abstractmethod + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + """ + Sign all files in the given directory. + + NOTE: Signing should happen in-place. + """ + + def run_signing_pipeline(self): + """ + Run the full signing pipeline starting from the point when buildbot + worker have requested signing. + """ + + # Make sure storage directory exists. + self.signed_storage_dir.mkdir(parents=True, exist_ok=True) + + with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: + temp_dir = Path(temp_dir_str) + + logger_server.info('Extracting unsigned files from archive...') + extract_files( + archive_filepath=self.unsigned_archive_info.archive_filepath, + extraction_dir=temp_dir) + + logger_server.info('Collecting all files which needs signing...') + files = AbsoluteAndRelativeFileName.recursively_from_directory( + temp_dir) + + logger_server.info('Signing all requested files...') + self.sign_all_files(files) + + logger_server.info('Packing signed files...') + pack_files(files=files, + archive_filepath=self.signed_archive_info.archive_filepath) + self.signed_archive_info.tag_ready() + + logger_server.info('Removing signing request...') + self.unsigned_archive_info.clean() + + logger_server.info('Signing is complete.') + + def run_signing_server(self): + logger_server.info('Starting new code signing server...') + self.cleanup_environment_for_signing_server() + logger_server.info('Code signing server is ready') + while True: + logger_server.info('Waiting for the signing request in %s...', + self.unsigned_storage_dir) + self.wait_for_sign_request() + + logger_server.info( + 'Got signing request, beging signign procedure.') + self.run_signing_pipeline() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py new file mode 100644 index 00000000000..c023b4234da --- /dev/null +++ b/build_files/buildbot/codesign/config_builder.py @@ -0,0 +1,57 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Configuration of a code signer which is specific to the code running from +# buildbot's worker. + +import sys + +from pathlib import Path + +from codesign.config_common import * + +if sys.platform == 'linux': + SHARED_STORAGE_DIR = Path('/data/codesign') +elif sys.platform == 'win32': + SHARED_STORAGE_DIR = Path('Z:\\codesign') + +# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema +LOGGING = { + 'version': 1, + 'formatters': { + 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'stream': 'ext://sys.stderr', + } + }, + 'loggers': { + 'codesign': {'level': 'INFO'}, + }, + 'root': { + 'level': 'WARNING', + 'handlers': [ + 'console', + ], + } +} diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py new file mode 100644 index 00000000000..4de71f54c7a --- /dev/null +++ b/build_files/buildbot/codesign/config_common.py @@ -0,0 +1,33 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + +# Timeout in seconds for the signing process. +# +# This is how long buildbot packing step will wait signing server to +# perform signing. +TIMEOUT_IN_SECONDS = 120 + +# Directory which is shared across buildbot worker and signing server. +# +# This is where worker puts files requested for signing as well as where +# server puts signed files. +SHARED_STORAGE_DIR: Path diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py new file mode 100644 index 00000000000..dc164634cef --- /dev/null +++ b/build_files/buildbot/codesign/config_server_template.py @@ -0,0 +1,63 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Configuration of a code signer which is specific to the code signing server. +# +# NOTE: DO NOT put any sensitive information here, put it in an actual +# configuration on the signing machine. + +from pathlib import Path + +from codesign.config_common import * + +# URL to the timestamping authority. +TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' + +# Full path to the certificate used for signing. +# +# The path and expected file format might vary depending on a platform. +# +# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look +# like Path('C:\\Secret\\Blender.pfx'). +CERTIFICATE_FILEPATH: Path + +# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema +LOGGING = { + 'version': 1, + 'formatters': { + 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'stream': 'ext://sys.stderr', + } + }, + 'loggers': { + 'codesign': {'level': 'INFO'}, + }, + 'root': { + 'level': 'WARNING', + 'handlers': [ + 'console', + ], + } +} diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py new file mode 100644 index 00000000000..f1523851eb7 --- /dev/null +++ b/build_files/buildbot/codesign/linux_code_signer.py @@ -0,0 +1,72 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# NOTE: This is a no-op signer (since there isn't really a procedure to sign +# Linux binaries yet). Used to debug and verify the code signing routines on +# a Linux environment. + +import logging + +from pathlib import Path +from typing import List + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.base_code_signer import BaseCodeSigner + +logger = logging.getLogger(__name__) +logger_server = logger.getChild('server') + + +class LinuxCodeSigner(BaseCodeSigner): + def is_active(self) -> bool: + """ + Check whether this signer is active. + + if it is inactive, no files will be signed. + + Is used to be able to debug code signing pipeline on Linux, where there + is no code signing happening in the actual buildbot and release + environment. + """ + return False + + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + if file.relative_filepath == Path('blender'): + return True + if (file.relative_filepath.parts()[-3:-1] == ('python', 'bin') and + file.relative_filepath.name.startwith('python')): + return True + if file.relative_filepath.suffix == '.so': + return True + return False + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + if not self.is_active(): + return [] + + return super().collect_files_to_sign(path) + + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + num_files = len(files) + for file_index, file in enumerate(files): + logger.info('Server: Signed file [%d/%d] %s', + file_index + 1, num_files, file.relative_filepath) diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py new file mode 100644 index 00000000000..d7bdce137c5 --- /dev/null +++ b/build_files/buildbot/codesign/simple_code_signer.py @@ -0,0 +1,47 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + + +import logging.config +import sys + +from pathlib import Path +from typing import Optional + +import codesign.config_builder +from codesign.base_code_signer import BaseCodeSigner + + +class SimpleCodeSigner: + code_signer: Optional[BaseCodeSigner] + + def __init__(self): + if sys.platform == 'linux': + from codesign.linux_code_signer import LinuxCodeSigner + self.code_signer = LinuxCodeSigner(codesign.config_builder) + elif sys.platform == 'win32': + from codesign.windows_code_signer import WindowsCodeSigner + self.code_signer = WindowsCodeSigner(codesign.config_builder) + else: + self.code_signer = None + + def sign_file_or_directory(self, path: Path) -> None: + logging.config.dictConfig(codesign.config_builder.LOGGING) + self.code_signer.run_buildbot_path_sign_pipeline(path) diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py new file mode 100644 index 00000000000..3c016fe5387 --- /dev/null +++ b/build_files/buildbot/codesign/util.py @@ -0,0 +1,35 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + + +def ensure_file_does_not_exist_or_die(filepath: Path) -> None: + """ + If the file exists, unlink it. + If the file path exists and is not a file an assert will trigger. + If the file path does not exists nothing happens. + """ + if not filepath.exists(): + return + if not filepath.is_file(): + # TODO(sergey): Provide information about what the filepath actually is. + raise SystemExit(f'{filepath} is expected to be a file, but is not') + filepath.unlink() diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py new file mode 100644 index 00000000000..9481b66ee1e --- /dev/null +++ b/build_files/buildbot/codesign/windows_code_signer.py @@ -0,0 +1,75 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +import logging +import subprocess + +from pathlib import Path +from typing import List + +from buildbot_utils import Builder + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.base_code_signer import BaseCodeSigner + +logger = logging.getLogger(__name__) +logger_server = logger.getChild('server') + +# NOTE: Check is done as filename.endswith(), so keep the dot +EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'} + +BLACKLIST_FILE_PREFIXES = ( + 'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime') + + +class WindowsCodeSigner(BaseCodeSigner): + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + base_name = file.relative_filepath.name + if any(base_name.startswith(prefix) + for prefix in BLACKLIST_FILE_PREFIXES): + return False + + return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED + + def get_sign_command_prefix(self) -> List[str]: + return [ + 'signtool', 'sign', '/v', + '/f', self.config.CERTIFICATE_FILEPATH, + '/t', self.config.TIMESTAMP_AUTHORITY_URL] + + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + # NOTE: Sign files one by one to avoid possible command line length + # overflow (which could happen if we ever decide to sign every binary + # in the install folder, for example). + # + # TODO(sergey): Consider doing batched signing of handful of files in + # one go (but only if this actually known to be much faster). + num_files = len(files) + for file_index, file in enumerate(files): + command = self.get_sign_command_prefix() + command.append(file.absolute_filepath) + logger_server.info( + 'Running signtool command for file [%d/%d] %s...', + file_index + 1, num_files, file.relative_filepath) + # TODO(sergey): Check the status somehow. With a missing certificate + # the command still exists with a zero code. + subprocess.run(command) + # TODO(sergey): Report number of signed and ignored files. |