diff options
Diffstat (limited to 'build_files/buildbot')
18 files changed, 1255 insertions, 3 deletions
diff --git a/build_files/buildbot/README.md b/build_files/buildbot/README.md new file mode 100644 index 00000000000..cf129f83b39 --- /dev/null +++ b/build_files/buildbot/README.md @@ -0,0 +1,70 @@ +Blender Buildbot +================ + +Code signing +------------ + +Code signing is done as part of INSTALL target, which makes it possible to sign +files which are aimed into a bundle and coming from a non-signed source (such as +libraries SVN). + +This is achieved by specifying `slave_codesign.cmake` as a post-install script +run by CMake. This CMake script simply involves an utility script written in +Python which takes care of an actual signing. + +### Configuration + +Client configuration doesn't need anything special, other than variable +`SHARED_STORAGE_DIR` pointing to a location which is watched by a server. +This is done in `config_builder.py` file and is stored in Git (which makes it +possible to have almost zero-configuration buildbot machines). + +Server configuration requires copying `config_server_template.py` under the +name of `config_server.py` and tweaking values, which are platform-specific. + +#### Windows configuration + +There are two things which are needed on Windows in order to have code signing +to work: + +- `TIMESTAMP_AUTHORITY_URL` which is most likely set http://timestamp.digicert.com +- `CERTIFICATE_FILEPATH` which is a full file path to a PKCS #12 key (.pfx). + +## Tips + +### Self-signed certificate on Windows + +It is easiest to test configuration using self-signed certificate. + +The certificate manipulation utilities are coming with Windows SDK. +Unfortunately, they are not added to PATH. Here is an example of how to make +sure they are easily available: + +``` +set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% +set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64;%PATH% +``` + +Generate CA: + +``` +makecert -r -pe -n "CN=Blender Test CA" -ss CA -sr CurrentUser -a sha256 ^ + -cy authority -sky signature -sv BlenderTestCA.pvk BlenderTestCA.cer +``` + +Import the generated CA: + +``` +certutil -user -addstore Root BlenderTestCA.cer +``` + +Create self-signed certificate and pack it into PKCS #12: + +``` +makecert -pe -n "CN=Blender Test SPC" -a sha256 -cy end ^ + -sky signature ^ + -ic BlenderTestCA.cer -iv BlenderTestCA.pvk ^ + -sv BlenderTestSPC.pvk BlenderTestSPC.cer + +pvk2pfx -pvk BlenderTestSPC.pvk -spc BlenderTestSPC.cer -pfx BlenderTestSPC.pfx +```
\ No newline at end of file diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py new file mode 100644 index 00000000000..bea9ea7e8d0 --- /dev/null +++ b/build_files/buildbot/codesign/absolute_and_relative_filename.py @@ -0,0 +1,77 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from dataclasses import dataclass +from pathlib import Path +from typing import List + + +@dataclass +class AbsoluteAndRelativeFileName: + """ + Helper class which keeps track of absolute file path for a direct access and + corresponding relative path against given base. + + The relative part is used to construct a file name within an archive which + contains files which are to be signed or which has been signed already + (depending on whether the archive is addressed to signing server or back + to the buildbot worker). + """ + + # Base directory which is where relative_filepath is relative to. + base_dir: Path + + # Full absolute path of the corresponding file. + absolute_filepath: Path + + # Derived from full file path, contains part of the path which is relative + # to a desired base path. + relative_filepath: Path + + def __init__(self, base_dir: Path, filepath: Path): + self.base_dir = base_dir + self.absolute_filepath = filepath.resolve() + self.relative_filepath = self.absolute_filepath.relative_to( + self.base_dir) + + @classmethod + def from_path(cls, path: Path) -> 'AbsoluteAndRelativeFileName': + assert path.is_absolute() + assert path.is_file() + + base_dir = path.parent + return AbsoluteAndRelativeFileName(base_dir, path) + + @classmethod + def recursively_from_directory(cls, base_dir: Path) \ + -> List['AbsoluteAndRelativeFileName']: + """ + Create list of AbsoluteAndRelativeFileName for all the files in the + given directory. + """ + assert base_dir.is_absolute() + assert base_dir.is_dir() + + result = [] + for filename in base_dir.glob('**/*'): + if not filename.is_file(): + continue + result.append(AbsoluteAndRelativeFileName(base_dir, filename)) + return result diff --git a/build_files/buildbot/codesign/archive_with_indicator.py b/build_files/buildbot/codesign/archive_with_indicator.py new file mode 100644 index 00000000000..51bcc28520d --- /dev/null +++ b/build_files/buildbot/codesign/archive_with_indicator.py @@ -0,0 +1,101 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + +from codesign.util import ensure_file_does_not_exist_or_die + + +class ArchiveWithIndicator: + """ + The idea of this class is to wrap around logic which takes care of keeping + track of a name of an archive and synchronization routines between buildbot + worker and signing server. + + The synchronization is done based on creating a special file after the + archive file is knowingly ready for access. + """ + + # Base directory where the archive is stored (basically, a basename() of + # the absolute archive file name). + # + # For example, 'X:\\TEMP\\'. + base_dir: Path + + # Absolute file name of the archive. + # + # For example, 'X:\\TEMP\\FOO.ZIP'. + archive_filepath: Path + + # Absolute name of a file which acts as an indication of the fact that the + # archive is ready and is available for access. + # + # This is how synchronization between buildbot worker and signing server is + # done: + # - First, the archive is created under archive_filepath name. + # - Second, the indication file is created under ready_indicator_filepath + # name. + # - Third, the colleague of whoever created the indicator name watches for + # the indication file to appear, and once it's there it access the + # archive. + ready_indicator_filepath: Path + + def __init__( + self, base_dir: Path, archive_name: str, ready_indicator_name: str): + """ + Construct the object from given base directory and name of the archive + file: + ArchiveWithIndicator(Path('X:\\TEMP'), 'FOO.ZIP', 'INPUT_READY') + """ + + self.base_dir = base_dir + self.archive_filepath = self.base_dir / archive_name + self.ready_indicator_filepath = self.base_dir / ready_indicator_name + + def is_ready(self) -> bool: + """Check whether the archive is ready for access.""" + return self.ready_indicator_filepath.exists() + + def tag_ready(self) -> None: + """ + Tag the archive as ready by creating the corresponding indication file. + + NOTE: It is expected that the archive was never tagged as ready before + and that there are no subsequent tags of the same archive. + If it is violated, an assert will fail. + """ + assert not self.is_ready() + self.ready_indicator_filepath.touch() + + def clean(self) -> None: + """ + Remove both archive and the ready indication file. + """ + ensure_file_does_not_exist_or_die(self.ready_indicator_filepath) + ensure_file_does_not_exist_or_die(self.archive_filepath) + + def is_fully_absent(self) -> bool: + """ + Check whether both archive and its ready indicator are absent. + Is used for a sanity check during code signing process by both + buildbot worker and signing server. + """ + return (not self.archive_filepath.exists() and + not self.ready_indicator_filepath.exists()) diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py new file mode 100644 index 00000000000..ff4b4539658 --- /dev/null +++ b/build_files/buildbot/codesign/base_code_signer.py @@ -0,0 +1,385 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Signing process overview. +# +# From buildbot worker side: +# - Files which needs to be signed are collected from either a directory to +# sign all signable files in there, or by filename of a single file to sign. +# - Those files gets packed into an archive and stored in a location location +# which is watched by the signing server. +# - A marker READY file is created which indicates the archive is ready for +# access. +# - Wait for the server to provide an archive with signed files. +# This is done by watching for the READY file which corresponds to an archive +# coming from the signing server. +# - Unpack the signed signed files from the archives and replace original ones. +# +# From code sign server: +# - Watch special location for a READY file which indicates the there is an +# archive with files which are to be signed. +# - Unpack the archive to a temporary location. +# - Run codesign tool and make sure all the files are signed. +# - Pack the signed files and store them in a location which is watched by +# the buildbot worker. +# - Create a READY file which indicates that the archive with signed files is +# ready. + +import abc +import logging +import shutil +import time +import zipfile + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterable, List + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.archive_with_indicator import ArchiveWithIndicator + + +logger = logging.getLogger(__name__) +logger_builder = logger.getChild('builder') +logger_server = logger.getChild('server') + + +def pack_files(files: Iterable[AbsoluteAndRelativeFileName], + archive_filepath: Path) -> None: + """ + Create zip archive from given files for the signing pipeline. + Is used by buildbot worker to create an archive of files which are to be + signed, and by signing server to send signed files back to the worker. + """ + with zipfile.ZipFile(archive_filepath, 'w') as zip_file_handle: + for file_info in files: + zip_file_handle.write(file_info.absolute_filepath, + arcname=file_info.relative_filepath) + + +def extract_files(archive_filepath: Path, + extraction_dir: Path) -> None: + """ + Extract all files form the given archive into the given direcotry. + """ + + # TODO(sergey): Verify files in the archive have relative path. + + with zipfile.ZipFile(archive_filepath, mode='r') as zip_file_handle: + zip_file_handle.extractall(path=extraction_dir) + + +class BaseCodeSigner(metaclass=abc.ABCMeta): + """ + Base class for a platform-specific signer of binaries. + + Contains all the logic shared across platform-specific implementations, such + as synchronization and notification logic. + + Platform specific bits (such as actual command for signing the binary) are + to be implemented as a subclass. + + Provides utilities code signing as a whole, including functionality needed + by a signing server and a buildbot worker. + + The signer and builder may run on separate machines, the only requirement is + that they have access to a directory which is shared between them. For the + security concerns this is to be done as a separate machine (or as a Shared + Folder configuration in VirtualBox configuration). This directory might be + mounted under different base paths, but its underlying storage is to be + the same. + + The code signer is short-lived on a buildbot worker side, and is living + forever on a code signing server side. + """ + + # TODO(sergey): Find a neat way to have config annotated. + # config: Config + + # Storage directory where builder puts files which are requested to be + # signed. + # Consider this an input of the code signing server. + unsigned_storage_dir: Path + + # Information about archive which contains files which are to be signed. + # + # This archive is created by the buildbot worked and acts as an input for + # the code signing server. + unsigned_archive_info: ArchiveWithIndicator + + # Storage where signed files are stored. + # Consider this an output of the code signer server. + signed_storage_dir: Path + + # Information about archive which contains signed files. + # + # This archive is created by the code signing server. + signed_archive_info: ArchiveWithIndicator + + def __init__(self, config): + self.config = config + + absolute_shared_storage_dir = config.SHARED_STORAGE_DIR.resolve() + + # Unsigned (signing server input) configuration. + self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' + self.unsigned_archive_info = ArchiveWithIndicator( + self.unsigned_storage_dir, 'unsigned_files.zip', 'ready.stamp') + + # Signed (signing server output) configuration. + self.signed_storage_dir = absolute_shared_storage_dir / 'signed' + self.signed_archive_info = ArchiveWithIndicator( + self.signed_storage_dir, 'signed_files.zip', 'ready.stamp') + + """ + General note on cleanup environment functions. + + It is expected that there is only one instance of the code signer server + running for a given input/output directory, and that it serves a single + buildbot worker. + By its nature, a buildbot worker only produces one build at a time and + never performs concurrent builds. + This leads to a conclusion that when starting in a clean environment + there shouldn't be any archives remaining from a previous build. + + However, it is possible to have various failure scenarios which might + leave the environment in a non-clean state: + + - Network hiccup which makes buildbot worker to stop current build + and re-start it after connection to server is re-established. + + Note, this could also happen during buildbot server maintenance. + + - Signing server might get restarted due to updates or other reasons. + + Requiring manual interaction in such cases is not something good to + require, so here we simply assume that the system is used the way it is + intended to and restore environment to a prestine clean state. + """ + + def cleanup_environment_for_builder(self) -> None: + self.unsigned_archive_info.clean() + self.signed_archive_info.clean() + + def cleanup_environment_for_signing_server(self) -> None: + # Don't clear the requested to-be-signed archive since we might be + # restarting signing machine while the buildbot is busy. + self.signed_archive_info.clean() + + ############################################################################ + # Buildbot worker side helpers. + + @abc.abstractmethod + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + """ + Check whether file is to be signed. + + Is used by both single file signing pipeline and recursive directory + signing pipeline. + + This is where code signer is to check whether file is to be signed or + not. This check might be based on a simple extension test or on actual + test whether file have a digital signature already or not. + """ + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + """ + Get all files which need to be signed from the given path. + + NOTE: The path might either be a file or directory. + + This function is run from the buildbot worker side. + """ + + # If there is a single file provided trust the buildbot worker that it + # is eligible for signing. + if path.is_file(): + file = AbsoluteAndRelativeFileName.from_path(path) + if not self.check_file_is_to_be_signed(file): + return [] + return [file] + + all_files = AbsoluteAndRelativeFileName.recursively_from_directory( + path) + files_to_be_signed = [file for file in all_files + if self.check_file_is_to_be_signed(file)] + return files_to_be_signed + + def wait_for_signed_archive_or_die(self) -> None: + """ + Wait until archive with signed files is available. + + Will only wait for the configured time. If that time exceeds and there + is still no responce from the signing server the application will exit + with a non-zero exit code. + """ + timeout_in_seconds = self.config.TIMEOUT_IN_SECONDS + time_start = time.monotonic() + while not self.signed_archive_info.is_ready(): + time.sleep(1) + time_slept_in_seconds = time.monotonic() - time_start + if time_slept_in_seconds > timeout_in_seconds: + self.unsigned_archive_info.clean() + raise SystemExit("Signing server didn't finish signing in " + f"{timeout_in_seconds} seconds, dying :(") + + def copy_signed_files_to_directory( + self, signed_dir: Path, destination_dir: Path) -> None: + """ + Copy all files from signed_dir to destination_dir. + + This function will overwrite any existing file. Permissions are copied + from the source files, but other metadata, such as timestamps, are not. + """ + for signed_filepath in signed_dir.glob('**/*'): + if not signed_filepath.is_file(): + continue + + relative_filepath = signed_filepath.relative_to(signed_dir) + destination_filepath = destination_dir / relative_filepath + destination_filepath.parent.mkdir(parents=True, exist_ok=True) + + shutil.copy(signed_filepath, destination_filepath) + + def run_buildbot_path_sign_pipeline(self, path: Path) -> None: + """ + Run all steps needed to make given path signed. + + Path points to an unsigned file or a directory which contains unsigned + files. + + If the path points to a single file then this file will be signed. + This is used to sign a final bundle such as .msi on Windows or .dmg on + macOS. + + NOTE: The code signed implementation might actually reject signing the + file, in which case the file will be left unsigned. This isn't anything + to be considered a failure situation, just might happen when buildbot + worker can not detect whether signing is really required in a specific + case or not. + + If the path points to a directory then code signer will sign all + signable files from it (finding them recursively). + """ + + self.cleanup_environment_for_builder() + + # Make sure storage directory exists. + self.unsigned_storage_dir.mkdir(parents=True, exist_ok=True) + + # Collect all files which needs to be signed and pack them into a single + # archive which will be sent to the signing server. + logger_builder.info('Collecting files which are to be signed...') + files = self.collect_files_to_sign(path) + if not files: + logger_builder.info('No files to be signed, ignoring.') + return + logger_builder.info('Found %d files to sign.', len(files)) + + pack_files(files=files, + archive_filepath=self.unsigned_archive_info.archive_filepath) + self.unsigned_archive_info.tag_ready() + + # Wait for the signing server to finish signing. + logger_builder.info('Waiting signing server to sign the files...') + self.wait_for_signed_archive_or_die() + + # Extract signed files from archive and move files to final location. + with TemporaryDirectory(prefix='blender-buildbot-') as temp_dir_str: + unpacked_signed_files_dir = Path(temp_dir_str) + + logger_builder.info('Extracting signed files from archive...') + extract_files( + archive_filepath=self.signed_archive_info.archive_filepath, + extraction_dir=unpacked_signed_files_dir) + + destination_dir = path + if destination_dir.is_file(): + destination_dir = destination_dir.parent + self.copy_signed_files_to_directory( + unpacked_signed_files_dir, destination_dir) + + ############################################################################ + # Signing server side helpers. + + def wait_for_sign_request(self) -> None: + """ + Wait for the buildbot to request signing of an archive. + """ + # TOOD(sergey): Support graceful shutdown on Ctrl-C. + while not self.unsigned_archive_info.is_ready(): + time.sleep(1) + + @abc.abstractmethod + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + """ + Sign all files in the given directory. + + NOTE: Signing should happen in-place. + """ + + def run_signing_pipeline(self): + """ + Run the full signing pipeline starting from the point when buildbot + worker have requested signing. + """ + + # Make sure storage directory exists. + self.signed_storage_dir.mkdir(parents=True, exist_ok=True) + + with TemporaryDirectory(prefix='blender-codesign-') as temp_dir_str: + temp_dir = Path(temp_dir_str) + + logger_server.info('Extracting unsigned files from archive...') + extract_files( + archive_filepath=self.unsigned_archive_info.archive_filepath, + extraction_dir=temp_dir) + + logger_server.info('Collecting all files which needs signing...') + files = AbsoluteAndRelativeFileName.recursively_from_directory( + temp_dir) + + logger_server.info('Signing all requested files...') + self.sign_all_files(files) + + logger_server.info('Packing signed files...') + pack_files(files=files, + archive_filepath=self.signed_archive_info.archive_filepath) + self.signed_archive_info.tag_ready() + + logger_server.info('Removing signing request...') + self.unsigned_archive_info.clean() + + logger_server.info('Signing is complete.') + + def run_signing_server(self): + logger_server.info('Starting new code signing server...') + self.cleanup_environment_for_signing_server() + logger_server.info('Code signing server is ready') + while True: + logger_server.info('Waiting for the signing request in %s...', + self.unsigned_storage_dir) + self.wait_for_sign_request() + + logger_server.info( + 'Got signing request, beging signign procedure.') + self.run_signing_pipeline() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py new file mode 100644 index 00000000000..c023b4234da --- /dev/null +++ b/build_files/buildbot/codesign/config_builder.py @@ -0,0 +1,57 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Configuration of a code signer which is specific to the code running from +# buildbot's worker. + +import sys + +from pathlib import Path + +from codesign.config_common import * + +if sys.platform == 'linux': + SHARED_STORAGE_DIR = Path('/data/codesign') +elif sys.platform == 'win32': + SHARED_STORAGE_DIR = Path('Z:\\codesign') + +# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema +LOGGING = { + 'version': 1, + 'formatters': { + 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'stream': 'ext://sys.stderr', + } + }, + 'loggers': { + 'codesign': {'level': 'INFO'}, + }, + 'root': { + 'level': 'WARNING', + 'handlers': [ + 'console', + ], + } +} diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py new file mode 100644 index 00000000000..4de71f54c7a --- /dev/null +++ b/build_files/buildbot/codesign/config_common.py @@ -0,0 +1,33 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + +# Timeout in seconds for the signing process. +# +# This is how long buildbot packing step will wait signing server to +# perform signing. +TIMEOUT_IN_SECONDS = 120 + +# Directory which is shared across buildbot worker and signing server. +# +# This is where worker puts files requested for signing as well as where +# server puts signed files. +SHARED_STORAGE_DIR: Path diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py new file mode 100644 index 00000000000..dc164634cef --- /dev/null +++ b/build_files/buildbot/codesign/config_server_template.py @@ -0,0 +1,63 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Configuration of a code signer which is specific to the code signing server. +# +# NOTE: DO NOT put any sensitive information here, put it in an actual +# configuration on the signing machine. + +from pathlib import Path + +from codesign.config_common import * + +# URL to the timestamping authority. +TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' + +# Full path to the certificate used for signing. +# +# The path and expected file format might vary depending on a platform. +# +# On Windows it is usually is a PKCS #12 key (.pfx), so the path will look +# like Path('C:\\Secret\\Blender.pfx'). +CERTIFICATE_FILEPATH: Path + +# https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema +LOGGING = { + 'version': 1, + 'formatters': { + 'default': {'format': '%(asctime)-15s %(levelname)8s %(name)s %(message)s'} + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'stream': 'ext://sys.stderr', + } + }, + 'loggers': { + 'codesign': {'level': 'INFO'}, + }, + 'root': { + 'level': 'WARNING', + 'handlers': [ + 'console', + ], + } +} diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py new file mode 100644 index 00000000000..f1523851eb7 --- /dev/null +++ b/build_files/buildbot/codesign/linux_code_signer.py @@ -0,0 +1,72 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# NOTE: This is a no-op signer (since there isn't really a procedure to sign +# Linux binaries yet). Used to debug and verify the code signing routines on +# a Linux environment. + +import logging + +from pathlib import Path +from typing import List + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.base_code_signer import BaseCodeSigner + +logger = logging.getLogger(__name__) +logger_server = logger.getChild('server') + + +class LinuxCodeSigner(BaseCodeSigner): + def is_active(self) -> bool: + """ + Check whether this signer is active. + + if it is inactive, no files will be signed. + + Is used to be able to debug code signing pipeline on Linux, where there + is no code signing happening in the actual buildbot and release + environment. + """ + return False + + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + if file.relative_filepath == Path('blender'): + return True + if (file.relative_filepath.parts()[-3:-1] == ('python', 'bin') and + file.relative_filepath.name.startwith('python')): + return True + if file.relative_filepath.suffix == '.so': + return True + return False + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + if not self.is_active(): + return [] + + return super().collect_files_to_sign(path) + + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + num_files = len(files) + for file_index, file in enumerate(files): + logger.info('Server: Signed file [%d/%d] %s', + file_index + 1, num_files, file.relative_filepath) diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py new file mode 100644 index 00000000000..d7bdce137c5 --- /dev/null +++ b/build_files/buildbot/codesign/simple_code_signer.py @@ -0,0 +1,47 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + + +import logging.config +import sys + +from pathlib import Path +from typing import Optional + +import codesign.config_builder +from codesign.base_code_signer import BaseCodeSigner + + +class SimpleCodeSigner: + code_signer: Optional[BaseCodeSigner] + + def __init__(self): + if sys.platform == 'linux': + from codesign.linux_code_signer import LinuxCodeSigner + self.code_signer = LinuxCodeSigner(codesign.config_builder) + elif sys.platform == 'win32': + from codesign.windows_code_signer import WindowsCodeSigner + self.code_signer = WindowsCodeSigner(codesign.config_builder) + else: + self.code_signer = None + + def sign_file_or_directory(self, path: Path) -> None: + logging.config.dictConfig(codesign.config_builder.LOGGING) + self.code_signer.run_buildbot_path_sign_pipeline(path) diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py new file mode 100644 index 00000000000..3c016fe5387 --- /dev/null +++ b/build_files/buildbot/codesign/util.py @@ -0,0 +1,35 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +from pathlib import Path + + +def ensure_file_does_not_exist_or_die(filepath: Path) -> None: + """ + If the file exists, unlink it. + If the file path exists and is not a file an assert will trigger. + If the file path does not exists nothing happens. + """ + if not filepath.exists(): + return + if not filepath.is_file(): + # TODO(sergey): Provide information about what the filepath actually is. + raise SystemExit(f'{filepath} is expected to be a file, but is not') + filepath.unlink() diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py new file mode 100644 index 00000000000..9481b66ee1e --- /dev/null +++ b/build_files/buildbot/codesign/windows_code_signer.py @@ -0,0 +1,75 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +import logging +import subprocess + +from pathlib import Path +from typing import List + +from buildbot_utils import Builder + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.base_code_signer import BaseCodeSigner + +logger = logging.getLogger(__name__) +logger_server = logger.getChild('server') + +# NOTE: Check is done as filename.endswith(), so keep the dot +EXTENSIONS_TO_BE_SIGNED = {'.exe', '.dll', '.pyd', '.msi'} + +BLACKLIST_FILE_PREFIXES = ( + 'api-ms-', 'concrt', 'msvcp', 'ucrtbase', 'vcomp', 'vcruntime') + + +class WindowsCodeSigner(BaseCodeSigner): + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + base_name = file.relative_filepath.name + if any(base_name.startswith(prefix) + for prefix in BLACKLIST_FILE_PREFIXES): + return False + + return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED + + def get_sign_command_prefix(self) -> List[str]: + return [ + 'signtool', 'sign', '/v', + '/f', self.config.CERTIFICATE_FILEPATH, + '/t', self.config.TIMESTAMP_AUTHORITY_URL] + + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + # NOTE: Sign files one by one to avoid possible command line length + # overflow (which could happen if we ever decide to sign every binary + # in the install folder, for example). + # + # TODO(sergey): Consider doing batched signing of handful of files in + # one go (but only if this actually known to be much faster). + num_files = len(files) + for file_index, file in enumerate(files): + command = self.get_sign_command_prefix() + command.append(file.absolute_filepath) + logger_server.info( + 'Running signtool command for file [%d/%d] %s...', + file_index + 1, num_files, file.relative_filepath) + # TODO(sergey): Check the status somehow. With a missing certificate + # the command still exists with a zero code. + subprocess.run(command) + # TODO(sergey): Report number of signed and ignored files. diff --git a/build_files/buildbot/codesign_server_linux.py b/build_files/buildbot/codesign_server_linux.py new file mode 100755 index 00000000000..be3065e640d --- /dev/null +++ b/build_files/buildbot/codesign_server_linux.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# NOTE: This is a no-op signer (since there isn't really a procedure to sign +# Linux binaries yet). Used to debug and verify the code signing routines on +# a Linux environment. + +import logging.config +from pathlib import Path +from typing import List + +from codesign.linux_code_signer import LinuxCodeSigner +import codesign.config_server + +if __name__ == "__main__": + logging.config.dictConfig(codesign.config_server.LOGGING) + code_signer = LinuxCodeSigner(codesign.config_server) + code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_windows.bat b/build_files/buildbot/codesign_server_windows.bat new file mode 100644 index 00000000000..82680f30eb4 --- /dev/null +++ b/build_files/buildbot/codesign_server_windows.bat @@ -0,0 +1,11 @@ +@echo off + +rem This is an entry point of the codesign server for Windows. +rem It makes sure that signtool.exe is within the current PATH and can be +rem used by the Python script. + +SETLOCAL + +set PATH=C:\Program Files (x86)\Windows Kits\10\App Certification Kit;%PATH% + +codesign_server_windows.py diff --git a/build_files/buildbot/codesign_server_windows.py b/build_files/buildbot/codesign_server_windows.py new file mode 100755 index 00000000000..2f7aab961f5 --- /dev/null +++ b/build_files/buildbot/codesign_server_windows.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +# Implementation of codesign server for Windows. +# +# NOTE: If signtool.exe is not in the PATH use codesign_server_windows.bat + +import logging.config +import shutil + +from pathlib import Path +from typing import List + +from codesign.windows_code_signer import WindowsCodeSigner +import codesign.config_server + +if __name__ == "__main__": + # TODO(sergey): Consider moving such sanity checks into + # CodeSigner.check_environment_or_die(). + if not shutil.which('signtool.exe'): + raise SystemExit("signtool.exe is not found in %PATH%") + + logging.config.dictConfig(codesign.config_server.LOGGING) + code_signer = WindowsCodeSigner(codesign.config_server) + code_signer.run_signing_server() diff --git a/build_files/buildbot/slave_codesign.cmake b/build_files/buildbot/slave_codesign.cmake new file mode 100644 index 00000000000..2c3b58c08c0 --- /dev/null +++ b/build_files/buildbot/slave_codesign.cmake @@ -0,0 +1,44 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# This is a script which is used as POST-INSTALL one for regular CMake's +# INSTALL target. +# It is used by buildbot workers to sign every binary which is going into +# the final buundle. + +# On Windows Python 3 there only is python.exe, no python3.exe. +# +# On other platforms it is possible to have python2 and python3, and a +# symbolic link to python to either of them. So on those platforms use +# an explicit Python version. +if(WIN32) + set(PYTHON_EXECUTABLE python) +else() + set(PYTHON_EXECUTABLE python3) +endif() + +execute_process( + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/slave_codesign.py" + "${CMAKE_INSTALL_PREFIX}" + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE exit_code +) + +if(NOT exit_code EQUAL "0") + message( FATAL_ERROR "Non-zero exit code of codesign tool") +endif() diff --git a/build_files/buildbot/slave_codesign.py b/build_files/buildbot/slave_codesign.py new file mode 100755 index 00000000000..8dedf5ffcd3 --- /dev/null +++ b/build_files/buildbot/slave_codesign.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# Helper script which takes care of signing provided location. +# +# The location can either be a directory (in which case all eligible binaries +# will be signed) or a single file (in which case a single file will be signed). +# +# This script takes care of all the complexity of communicating between process +# which requests file to be signed and the code signing server. +# +# NOTE: Signing happens in-place. + +import argparse +import sys + +from pathlib import Path + +from codesign.simple_code_signer import SimpleCodeSigner + + +def create_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('path_to_sign', type=Path) + return parser + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + path_to_sign = args.path_to_sign + + if sys.platform == 'win32': + # When WIX packed is used to generate .msi on Windows the CPack will + # install two different projects and install them to different + # installation prefix: + # + # - C:\b\build\_CPack_Packages\WIX\Blender + # - C:\b\build\_CPack_Packages\WIX\Unspecified + # + # Annoying part is: CMake's post-install script will only be run + # once, with the install prefix which corresponds to a project which + # was installed last. But we want to sign binaries from all projects. + # So in order to do so we detect that we are running for a CPack's + # project used for WIX and force parent directory (which includes both + # projects) to be signed. + # + # Here we force both projects to be signed. + if path_to_sign.name == 'Unspecified' and 'WIX' in str(path_to_sign): + path_to_sign = path_to_sign.parent + + code_signer = SimpleCodeSigner() + code_signer.sign_file_or_directory(path_to_sign) + + +if __name__ == "__main__": + main() diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py index 0da0ead819f..f8bab19a1e9 100644 --- a/build_files/buildbot/slave_compile.py +++ b/build_files/buildbot/slave_compile.py @@ -18,13 +18,19 @@ # <pep8 compliant> -import buildbot_utils import os import shutil +import buildbot_utils + def get_cmake_options(builder): + post_install_script = os.path.join( + builder.blender_dir, 'build_files', 'buildbot', 'slave_codesign.cmake') + config_file = "build_files/cmake/config/blender_release.cmake" - options = ['-DCMAKE_BUILD_TYPE:STRING=Release', '-DWITH_GTESTS=ON'] + options = ['-DCMAKE_BUILD_TYPE:STRING=Release', + '-DWITH_GTESTS=ON', + '-DPOSTINSTALL_SCRIPT:PATH=' + post_install_script] if builder.platform == 'mac': options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') @@ -84,6 +90,16 @@ def cmake_build(builder): # CMake build os.chdir(builder.build_dir) + # NOTE: CPack will build an INSTALL target, which would mean that code + # signing will happen twice when using `make install` and CPack. + # The tricky bit here is that it is not possible to know whether INSTALL + # target is used by CPack or by a buildbot itaself. Extra level on top of + # this is that on Windows it is required to build INSTALL target in order + # to have unit test binaries to run. + # So on the one hand we do an extra unneeded code sign on Windows, but on + # a positive side we don't add complexity and don't make build process more + # fragile trying to avoid this. The signing process is way faster than just + # a clean build of buildbot, especially with regression tests enabled. if builder.platform == 'win': command = ['cmake', '--build', '.', '--target', 'install', '--config', 'Release'] else: diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py index 5bef2b81739..19dac236762 100644 --- a/build_files/buildbot/slave_pack.py +++ b/build_files/buildbot/slave_pack.py @@ -22,10 +22,13 @@ # system and zipping it into buildbot_upload.zip. This is then uploaded # to the master in the next buildbot step. -import buildbot_utils import os import sys +from pathlib import Path + +import buildbot_utils + def get_package_name(builder, platform=None): info = buildbot_utils.VersionInfo(builder) @@ -38,6 +41,12 @@ def get_package_name(builder, platform=None): return package_name +def sign_file_or_directory(path): + from codesign.simple_code_signer import SimpleCodeSigner + code_signer = SimpleCodeSigner() + code_signer.sign_file_or_directory(Path(path)) + + def create_buildbot_upload_zip(builder, package_files): import zipfile @@ -129,6 +138,8 @@ def pack_win(builder): package_filename = package_name + '.msi' package_filepath = os.path.join(builder.build_dir, package_filename) + sign_file_or_directory(package_filepath) + package_files += [(package_filepath, package_filename)] create_buildbot_upload_zip(builder, package_files) |