diff options
21 files changed, 1290 insertions, 52 deletions
diff --git a/build_files/buildbot/codesign/absolute_and_relative_filename.py b/build_files/buildbot/codesign/absolute_and_relative_filename.py index bea9ea7e8d0..cb42710e785 100644 --- a/build_files/buildbot/codesign/absolute_and_relative_filename.py +++ b/build_files/buildbot/codesign/absolute_and_relative_filename.py @@ -65,10 +65,14 @@ class AbsoluteAndRelativeFileName: """ Create list of AbsoluteAndRelativeFileName for all the files in the given directory. + + NOTE: Result will be pointing to a resolved paths. """ assert base_dir.is_absolute() assert base_dir.is_dir() + base_dir = base_dir.resolve() + result = [] for filename in base_dir.glob('**/*'): if not filename.is_file(): diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py index ff4b4539658..0505905c6f4 100644 --- a/build_files/buildbot/codesign/base_code_signer.py +++ b/build_files/buildbot/codesign/base_code_signer.py @@ -45,13 +45,16 @@ import abc import logging import shutil +import subprocess import time -import zipfile +import tarfile from pathlib import Path from tempfile import TemporaryDirectory from typing import Iterable, List +import codesign.util as util + from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName from codesign.archive_with_indicator import ArchiveWithIndicator @@ -64,14 +67,14 @@ logger_server = logger.getChild('server') def pack_files(files: Iterable[AbsoluteAndRelativeFileName], archive_filepath: Path) -> None: """ - Create zip archive from given files for the signing pipeline. + Create tar archive from given files for the signing pipeline. Is used by buildbot worker to create an archive of files which are to be signed, and by signing server to send signed files back to the worker. """ - with zipfile.ZipFile(archive_filepath, 'w') as zip_file_handle: + with tarfile.TarFile.open(archive_filepath, 'w') as tar_file_handle: for file_info in files: - zip_file_handle.write(file_info.absolute_filepath, - arcname=file_info.relative_filepath) + tar_file_handle.add(file_info.absolute_filepath, + arcname=file_info.relative_filepath) def extract_files(archive_filepath: Path, @@ -82,8 +85,8 @@ def extract_files(archive_filepath: Path, # TODO(sergey): Verify files in the archive have relative path. - with zipfile.ZipFile(archive_filepath, mode='r') as zip_file_handle: - zip_file_handle.extractall(path=extraction_dir) + with tarfile.TarFile.open(archive_filepath, mode='r') as tar_file_handle: + tar_file_handle.extractall(path=extraction_dir) class BaseCodeSigner(metaclass=abc.ABCMeta): @@ -133,6 +136,9 @@ class BaseCodeSigner(metaclass=abc.ABCMeta): # This archive is created by the code signing server. signed_archive_info: ArchiveWithIndicator + # Platform the code is currently executing on. + platform: util.Platform + def __init__(self, config): self.config = config @@ -141,12 +147,14 @@ class BaseCodeSigner(metaclass=abc.ABCMeta): # Unsigned (signing server input) configuration. self.unsigned_storage_dir = absolute_shared_storage_dir / 'unsigned' self.unsigned_archive_info = ArchiveWithIndicator( - self.unsigned_storage_dir, 'unsigned_files.zip', 'ready.stamp') + self.unsigned_storage_dir, 'unsigned_files.tar', 'ready.stamp') # Signed (signing server output) configuration. self.signed_storage_dir = absolute_shared_storage_dir / 'signed' self.signed_archive_info = ArchiveWithIndicator( - self.signed_storage_dir, 'signed_files.zip', 'ready.stamp') + self.signed_storage_dir, 'signed_files.tar', 'ready.stamp') + + self.platform = util.get_current_platform() """ General note on cleanup environment functions. @@ -383,3 +391,61 @@ class BaseCodeSigner(metaclass=abc.ABCMeta): logger_server.info( 'Got signing request, beging signign procedure.') self.run_signing_pipeline() + + ############################################################################ + # Command executing. + # + # Abstracted to a degree that allows to run commands from a foreign + # platform. + # The goal with this is to allow performing dry-run tests of code signer + # server from other platforms (for example, to test that macOS code signer + # does what it is supposed to after doing a refactor on Linux). + + # TODO(sergey): What is the type annotation for the command? + def run_command_or_mock(self, command, platform: util.Platform) -> None: + """ + Run given command if current platform matches given one + + If the platform is different then it will only be printed allowing + to verify logic of the code signing process. + """ + + if platform != self.platform: + logger_server.info( + f'Will run command for {platform}: {command}') + return + + logger_server.info(f'Running command: {command}') + subprocess.run(command) + + # TODO(sergey): What is the type annotation for the command? + def check_output_or_mock(self, command, + platform: util.Platform, + allow_nonzero_exit_code=False) -> str: + """ + Run given command if current platform matches given one + + If the platform is different then it will only be printed allowing + to verify logic of the code signing process. + + If allow_nonzero_exit_code is truth then the output will be returned + even if application quit with non-zero exit code. + Otherwise an subprocess.CalledProcessError exception will be raised + in such case. + """ + + if platform != self.platform: + logger_server.info( + f'Will run command for {platform}: {command}') + return + + if allow_nonzero_exit_code: + process = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = process.communicate()[0] + return output.decode() + + logger_server.info(f'Running command: {command}') + return subprocess.check_output( + command, stderr=subprocess.STDOUT).decode() diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py index e1e3913b99e..1f41619ba13 100644 --- a/build_files/buildbot/codesign/config_builder.py +++ b/build_files/buildbot/codesign/config_builder.py @@ -25,13 +25,16 @@ import sys from pathlib import Path +import codesign.util as util + from codesign.config_common import * -if sys.platform == 'linux': +platform = util.get_current_platform() +if platform == util.Platform.LINUX: SHARED_STORAGE_DIR = Path('/data/codesign') -elif sys.platform == 'win32': +elif platform == util.Platform.WINDOWS: SHARED_STORAGE_DIR = Path('Z:\\codesign') -elif sys.platform == 'darwin': +elif platform == util.Platform.MACOS: SHARED_STORAGE_DIR = Path('/Volumes/codesign_macos/codesign') # https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py index 3710286c777..a37bc731dc0 100644 --- a/build_files/buildbot/codesign/config_common.py +++ b/build_files/buildbot/codesign/config_common.py @@ -24,7 +24,10 @@ from pathlib import Path # # This is how long buildbot packing step will wait signing server to # perform signing. -TIMEOUT_IN_SECONDS = 240 +# +# NOTE: Notarization could take a long time, hence the rather high value +# here. Might consider using different timeout for different platforms. +TIMEOUT_IN_SECONDS = 45 * 60 * 60 # Directory which is shared across buildbot worker and signing server. # diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py index dc164634cef..ff97ed15fa5 100644 --- a/build_files/buildbot/codesign/config_server_template.py +++ b/build_files/buildbot/codesign/config_server_template.py @@ -27,8 +27,43 @@ from pathlib import Path from codesign.config_common import * +CODESIGN_DIRECTORY = Path(__file__).absolute().parent +BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent + +################################################################################ +# Common configuration. + +# Directory where folders for codesign requests and signed result are stored. +# For example, /data/codesign +SHARED_STORAGE_DIR: Path + +################################################################################ +# macOS-specific configuration. + +MACOS_ENTITLEMENTS_FILE = \ + BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' / 'entitlements.plist' + +# Identity of the Developer ID Application certificate which is to be used for +# codesign tool. +# Use `security find-identity -v -p codesigning` to find the identity. +# +# NOTE: This identity is just an example from release/darwin/README.txt. +MACOS_CODESIGN_IDENTITY = 'AE825E26F12D08B692F360133210AF46F4CF7B97' + +# User name (Apple ID) which will be used to request notarization. +MACOS_XCRUN_USERNAME = 'me@example.com' + +# One-time application password which will be used to request notarization. +MACOS_XCRUN_PASSWORD = '@keychain:altool-password' + +# Timeout in seconds within which the notarial office is supposed to reply. +MACOS_NOTARIZE_TIMEOUT_IN_SECONDS = 60 * 60 + +################################################################################ +# Windows-specific configuration. + # URL to the timestamping authority. -TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' +WIN_TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' # Full path to the certificate used for signing. # @@ -36,7 +71,10 @@ TIMESTAMP_AUTHORITY_URL = 'http://timestamp.digicert.com' # # On Windows it is usually is a PKCS #12 key (.pfx), so the path will look # like Path('C:\\Secret\\Blender.pfx'). -CERTIFICATE_FILEPATH: Path +WIN_CERTIFICATE_FILEPATH: Path + +################################################################################ +# Logging configuration, common for all platforms. # https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema LOGGING = { diff --git a/build_files/buildbot/codesign/linux_code_signer.py b/build_files/buildbot/codesign/linux_code_signer.py index f1523851eb7..04935f67832 100644 --- a/build_files/buildbot/codesign/linux_code_signer.py +++ b/build_files/buildbot/codesign/linux_code_signer.py @@ -51,7 +51,7 @@ class LinuxCodeSigner(BaseCodeSigner): self, file: AbsoluteAndRelativeFileName) -> bool: if file.relative_filepath == Path('blender'): return True - if (file.relative_filepath.parts()[-3:-1] == ('python', 'bin') and + if (file.relative_filepath.parts[-3:-1] == ('python', 'bin') and file.relative_filepath.name.startwith('python')): return True if file.relative_filepath.suffix == '.so': diff --git a/build_files/buildbot/codesign/macos_code_signer.py b/build_files/buildbot/codesign/macos_code_signer.py new file mode 100644 index 00000000000..ce2bfb6d1b5 --- /dev/null +++ b/build_files/buildbot/codesign/macos_code_signer.py @@ -0,0 +1,454 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +import logging +import re +import stat +import subprocess +import time + +from pathlib import Path +from typing import List + +import codesign.util as util + +from buildbot_utils import Builder + +from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName +from codesign.base_code_signer import BaseCodeSigner + +logger = logging.getLogger(__name__) +logger_server = logger.getChild('server') + +# NOTE: Check is done as filename.endswith(), so keep the dot +EXTENSIONS_TO_BE_SIGNED = {'.dylib', '.so', '.dmg'} + +# Prefixes of a file (not directory) name which are to be signed. +# Used to sign extra executable files in Contents/Resources. +NAME_PREFIXES_TO_BE_SIGNED = {'python'} + + +def is_file_from_bundle(file: AbsoluteAndRelativeFileName) -> bool: + """ + Check whether file is coming from an .app bundle + """ + parts = file.relative_filepath.parts + if not parts: + return False + if not parts[0].endswith('.app'): + return False + return True + + +def get_bundle_from_file( + file: AbsoluteAndRelativeFileName) -> AbsoluteAndRelativeFileName: + """ + Get AbsoluteAndRelativeFileName descriptor of bundle + """ + assert(is_file_from_bundle(file)) + + parts = file.relative_filepath.parts + bundle_name = parts[0] + + base_dir = file.base_dir + bundle_filepath = file.base_dir / bundle_name + return AbsoluteAndRelativeFileName(base_dir, bundle_filepath) + + +def is_bundle_executable_file(file: AbsoluteAndRelativeFileName) -> bool: + """ + Check whether given file is an executable within an app bundle + """ + if not is_file_from_bundle(file): + return False + + parts = file.relative_filepath.parts + num_parts = len(parts) + if num_parts < 3: + return False + + if parts[1:3] != ('Contents', 'MacOS'): + return False + + return True + + +def xcrun_field_value_from_output(field: str, output: str) -> str: + """ + Get value of a given field from xcrun output. + + If field is not found empty string is returned. + """ + + field_prefix = field + ': ' + for line in output.splitlines(): + line = line.strip() + if line.startswith(field_prefix): + return line[len(field_prefix):] + return '' + + +class MacOSCodeSigner(BaseCodeSigner): + def check_file_is_to_be_signed( + self, file: AbsoluteAndRelativeFileName) -> bool: + if file.relative_filepath.name.startswith('.'): + return False + + if is_bundle_executable_file(file): + return True + + base_name = file.relative_filepath.name + if any(base_name.startswith(prefix) + for prefix in NAME_PREFIXES_TO_BE_SIGNED): + return True + + mode = file.absolute_filepath.lstat().st_mode + if mode & stat.S_IXUSR != 0: + file_output = subprocess.check_output( + ("file", file.absolute_filepath)).decode() + if "64-bit executable" in file_output: + return True + + return file.relative_filepath.suffix in EXTENSIONS_TO_BE_SIGNED + + def collect_files_to_sign(self, path: Path) \ + -> List[AbsoluteAndRelativeFileName]: + # Include all files when signing app or dmg bundle: all the files are + # needed to do valid signature of bundle. + if path.name.endswith('.app'): + return AbsoluteAndRelativeFileName.recursively_from_directory(path) + if path.is_dir(): + files = [] + for child in path.iterdir(): + if child.name.endswith('.app'): + current_files = AbsoluteAndRelativeFileName.recursively_from_directory( + child) + else: + current_files = super().collect_files_to_sign(child) + for current_file in current_files: + files.append(AbsoluteAndRelativeFileName( + path, current_file.absolute_filepath)) + return files + return super().collect_files_to_sign(path) + + ############################################################################ + # Codesign. + + def codesign_remove_signature( + self, file: AbsoluteAndRelativeFileName) -> None: + """ + Make sure given file does not have codesign signature + + This is needed because codesigning is not possible for file which has + signature already. + """ + + logger_server.info( + 'Removing codesign signature from %s...', file.relative_filepath) + + command = ['codesign', '--remove-signature', file.absolute_filepath] + self.run_command_or_mock(command, util.Platform.MACOS) + + def codesign_file( + self, file: AbsoluteAndRelativeFileName) -> None: + """ + Sign given file + + NOTE: File must not have any signatures. + """ + + logger_server.info( + 'Codesigning %s...', file.relative_filepath) + + entitlements_file = self.config.MACOS_ENTITLEMENTS_FILE + command = ['codesign', + '--timestamp', + '--options', 'runtime', + f'--entitlements={entitlements_file}', + '--sign', self.config.MACOS_CODESIGN_IDENTITY, + file.absolute_filepath] + self.run_command_or_mock(command, util.Platform.MACOS) + + def codesign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> bool: + """ + Run codesign tool on all eligible files in the given list. + + Will ignore all files which are not to be signed. For the rest will + remove possible existing signature and add a new signature. + """ + + num_files = len(files) + have_ignored_files = False + signed_files = [] + for file_index, file in enumerate(files): + # Ignore file if it is not to be signed. + # Allows to manually construct ZIP of a bundle and get it signed. + if not self.check_file_is_to_be_signed(file): + logger_server.info( + 'Ignoring file [%d/%d] %s', + file_index + 1, num_files, file.relative_filepath) + have_ignored_files = True + continue + + logger_server.info( + 'Running codesigning routines for file [%d/%d] %s...', + file_index + 1, num_files, file.relative_filepath) + + self.codesign_remove_signature(file) + self.codesign_file(file) + + signed_files.append(file) + + if have_ignored_files: + logger_server.info('Signed %d files:', len(signed_files)) + num_signed_files = len(signed_files) + for file_index, signed_file in enumerate(signed_files): + logger_server.info( + '- [%d/%d] %s', + file_index + 1, num_signed_files, + signed_file.relative_filepath) + + return True + + def codesign_bundles( + self, files: List[AbsoluteAndRelativeFileName]) -> None: + """ + Codesign all .app bundles in the given list of files. + + Bundle is deducted from paths of the files, and every bundle is only + signed once. + """ + + signed_bundles = set() + extra_files = [] + + for file in files: + if not is_file_from_bundle(file): + continue + bundle = get_bundle_from_file(file) + bundle_name = bundle.relative_filepath + if bundle_name in signed_bundles: + continue + + logger_server.info('Running codesign routines on bundle %s', + bundle_name) + + # It is not possible to remove signature from DMG. + if bundle.relative_filepath.name.endswith('.app'): + self.codesign_remove_signature(bundle) + self.codesign_file(bundle) + + signed_bundles.add(bundle_name) + + # Codesign on a bundle adds an extra folder with information. + # It needs to be compied to the source. + code_signature_directory = \ + bundle.absolute_filepath / 'Contents' / '_CodeSignature' + code_signature_files = \ + AbsoluteAndRelativeFileName.recursively_from_directory( + code_signature_directory) + for code_signature_file in code_signature_files: + bundle_relative_file = AbsoluteAndRelativeFileName( + bundle.base_dir, + code_signature_directory / + code_signature_file.relative_filepath) + extra_files.append(bundle_relative_file) + + files.extend(extra_files) + + return True + + ############################################################################ + # Notarization. + + def notarize_get_bundle_id(self, file: AbsoluteAndRelativeFileName) -> str: + """ + Get bundle ID which will be used to notarize DMG + """ + name = file.relative_filepath.name + app_name = name.split('-', 2)[0].lower() + + app_name_words = app_name.split() + if len(app_name_words) > 1: + app_name_id = ''.join(word.capitalize() for word in app_name_words) + else: + app_name_id = app_name_words[0] + + # TODO(sergey): Consider using "alpha" for buildbot builds. + return f'org.blenderfoundation.{app_name_id}.release' + + def notarize_request(self, file) -> str: + """ + Request notarization of the given file. + + Returns UUID of the notarization request. If error occurred None is + returned instead of UUID. + """ + + bundle_id = self.notarize_get_bundle_id(file) + logger_server.info('Bundle ID: %s', bundle_id) + + logger_server.info('Submitting file to the notarial office.') + command = [ + 'xcrun', 'altool', '--notarize-app', '--verbose', + '-f', file.absolute_filepath, + '--primary-bundle-id', bundle_id, + '--username', self.config.MACOS_XCRUN_USERNAME, + '--password', self.config.MACOS_XCRUN_PASSWORD] + + output = self.check_output_or_mock( + command, util.Platform.MACOS, allow_nonzero_exit_code=True) + + for line in output.splitlines(): + line = line.strip() + if line.startswith('RequestUUID = '): + request_uuid = line[14:] + return request_uuid + + # Check whether the package has been already submitted. + if 'The software asset has already been uploaded.' in line: + request_uuid = re.sub( + '.*The upload ID is ([A-Fa-f0-9\-]+).*', '\\1', line) + logger_server.warning( + f'The package has been already submitted under UUID {request_uuid}') + return request_uuid + + logger_server.error(output) + logger_server.error('xcrun command did not report RequestUUID') + return None + + def notarize_wait_result(self, request_uuid: str) -> bool: + """ + Wait for until notarial office have a reply + """ + + logger_server.info( + 'Waiting for a result from the notarization office.') + + command = ['xcrun', 'altool', + '--notarization-info', request_uuid, + '--username', self.config.MACOS_XCRUN_USERNAME, + '--password', self.config.MACOS_XCRUN_PASSWORD] + + time_start = time.monotonic() + timeout_in_seconds = self.config.MACOS_NOTARIZE_TIMEOUT_IN_SECONDS + + while True: + output = self.check_output_or_mock( + command, util.Platform.MACOS, allow_nonzero_exit_code=True) + # Parse status and message + status = xcrun_field_value_from_output('Status', output) + status_message = xcrun_field_value_from_output( + 'Status Message', output) + + # Review status. + if status: + if status == 'success': + logger_server.info( + 'Package successfully notarized: %s', status_message) + return True + elif status == 'invalid': + logger_server.error(output) + logger_server.error( + 'Package notarization has failed: %s', status_message) + return False + elif status == 'in progress': + pass + else: + logger_server.info( + 'Unknown notarization status %s (%s)', status, status_message) + + logger_server.info('Keep waiting for notarization office.') + time.sleep(30) + + time_slept_in_seconds = time.monotonic() - time_start + if time_slept_in_seconds > timeout_in_seconds: + logger_server.error( + "Notarial office didn't reply in %f seconds.", + timeout_in_seconds) + + def notarize_staple(self, file: AbsoluteAndRelativeFileName) -> bool: + """ + Staple notarial label on the file + """ + + logger_server.info( + 'Waiting for a result from the notarization office.') + + command = ['xcrun', 'stapler', 'staple', '-v', file.absolute_filepath] + self.check_output_or_mock(command, util.Platform.MACOS) + + return True + + def notarize_dmg(self, file: AbsoluteAndRelativeFileName) -> bool: + """ + Run entire pipeline to get DMG notarized. + """ + logger_server.info('Begin notarization routines on %s', + file.relative_filepath) + + # Submit file for notarization. + request_uuid = self.notarize_request(file) + if not request_uuid: + return False + logger_server.info('Received Request UUID: %s', request_uuid) + + # Wait for the status from the notarization office. + if not self.notarize_wait_result(request_uuid): + return False + + # Staple. + if not self.notarize_staple(file): + return False + + return True + + def notarize_all_dmg( + self, files: List[AbsoluteAndRelativeFileName]) -> bool: + """ + Notarize all DMG images from the input. + + Images are supposed to be codesigned already. + """ + for file in files: + if not file.relative_filepath.name.endswith('.dmg'): + continue + if not self.check_file_is_to_be_signed(file): + continue + + if not self.notarize_dmg(file): + return False + + return True + + ############################################################################ + # Entry point. + + def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: + # TODO(sergey): Handle errors somehow. + + if not self.codesign_all_files(files): + return + + if not self.codesign_bundles(files): + return + + if not self.notarize_all_dmg(files): + return diff --git a/build_files/buildbot/codesign/simple_code_signer.py b/build_files/buildbot/codesign/simple_code_signer.py index d7bdce137c5..674d9e9ce9e 100644 --- a/build_files/buildbot/codesign/simple_code_signer.py +++ b/build_files/buildbot/codesign/simple_code_signer.py @@ -26,6 +26,7 @@ from pathlib import Path from typing import Optional import codesign.config_builder +import codesign.util as util from codesign.base_code_signer import BaseCodeSigner @@ -33,10 +34,14 @@ class SimpleCodeSigner: code_signer: Optional[BaseCodeSigner] def __init__(self): - if sys.platform == 'linux': + platform = util.get_current_platform() + if platform == util.Platform.LINUX: from codesign.linux_code_signer import LinuxCodeSigner self.code_signer = LinuxCodeSigner(codesign.config_builder) - elif sys.platform == 'win32': + elif platform == util.Platform.MACOS: + from codesign.macos_code_signer import MacOSCodeSigner + self.code_signer = MacOSCodeSigner(codesign.config_builder) + elif platform == util.Platform.WINDOWS: from codesign.windows_code_signer import WindowsCodeSigner self.code_signer = WindowsCodeSigner(codesign.config_builder) else: diff --git a/build_files/buildbot/codesign/util.py b/build_files/buildbot/codesign/util.py index 3c016fe5387..e67292dd049 100644 --- a/build_files/buildbot/codesign/util.py +++ b/build_files/buildbot/codesign/util.py @@ -18,9 +18,28 @@ # <pep8 compliant> +import sys + +from enum import Enum from pathlib import Path +class Platform(Enum): + LINUX = 1 + MACOS = 2 + WINDOWS = 3 + + +def get_current_platform() -> Platform: + if sys.platform == 'linux': + return Platform.LINUX + elif sys.platform == 'darwin': + return Platform.MACOS + elif sys.platform == 'win32': + return Platform.WINDOWS + raise Exception(f'Unknown platform {sys.platform}') + + def ensure_file_does_not_exist_or_die(filepath: Path) -> None: """ If the file exists, unlink it. diff --git a/build_files/buildbot/codesign/windows_code_signer.py b/build_files/buildbot/codesign/windows_code_signer.py index 638f098d8bc..2557d3c0b68 100644 --- a/build_files/buildbot/codesign/windows_code_signer.py +++ b/build_files/buildbot/codesign/windows_code_signer.py @@ -19,11 +19,12 @@ # <pep8 compliant> import logging -import subprocess from pathlib import Path from typing import List +import codesign.util as util + from buildbot_utils import Builder from codesign.absolute_and_relative_filename import AbsoluteAndRelativeFileName @@ -52,8 +53,8 @@ class WindowsCodeSigner(BaseCodeSigner): def get_sign_command_prefix(self) -> List[str]: return [ 'signtool', 'sign', '/v', - '/f', self.config.CERTIFICATE_FILEPATH, - '/tr', self.config.TIMESTAMP_AUTHORITY_URL] + '/f', self.config.WIN_CERTIFICATE_FILEPATH, + '/tr', self.config.WIN_TIMESTAMP_AUTHORITY_URL] def sign_all_files(self, files: List[AbsoluteAndRelativeFileName]) -> None: # NOTE: Sign files one by one to avoid possible command line length @@ -64,6 +65,14 @@ class WindowsCodeSigner(BaseCodeSigner): # one go (but only if this actually known to be much faster). num_files = len(files) for file_index, file in enumerate(files): + # Ignore file if it is not to be signed. + # Allows to manually construct ZIP of package and get it signed. + if not self.check_file_is_to_be_signed(file): + logger_server.info( + 'Ignoring file [%d/%d] %s', + file_index + 1, num_files, file.relative_filepath) + continue + command = self.get_sign_command_prefix() command.append(file.absolute_filepath) logger_server.info( @@ -71,5 +80,5 @@ class WindowsCodeSigner(BaseCodeSigner): file_index + 1, num_files, file.relative_filepath) # TODO(sergey): Check the status somehow. With a missing certificate # the command still exists with a zero code. - subprocess.run(command) + self.run_command_or_mock(command, util.Platform.WINDOWS) # TODO(sergey): Report number of signed and ignored files. diff --git a/build_files/buildbot/codesign_server_macos.py b/build_files/buildbot/codesign_server_macos.py new file mode 100755 index 00000000000..1bdb012fe67 --- /dev/null +++ b/build_files/buildbot/codesign_server_macos.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +# <pep8 compliant> + +import logging.config +from pathlib import Path +from typing import List + +from codesign.macos_code_signer import MacOSCodeSigner +import codesign.config_server + +if __name__ == "__main__": + entitlements_file = codesign.config_server.MACOS_ENTITLEMENTS_FILE + if not entitlements_file.exists(): + raise SystemExit( + 'Entitlements file {entitlements_file} does not exist.') + if not entitlements_file.is_file(): + raise SystemExit( + 'Entitlements file {entitlements_file} is not a file.') + + logging.config.dictConfig(codesign.config_server.LOGGING) + code_signer = MacOSCodeSigner(codesign.config_server) + code_signer.run_signing_server() diff --git a/build_files/buildbot/codesign_server_windows.py b/build_files/buildbot/codesign_server_windows.py index 2f7aab961f5..97ea4fd6756 100755 --- a/build_files/buildbot/codesign_server_windows.py +++ b/build_files/buildbot/codesign_server_windows.py @@ -30,15 +30,25 @@ import shutil from pathlib import Path from typing import List +import codesign.util as util + from codesign.windows_code_signer import WindowsCodeSigner import codesign.config_server if __name__ == "__main__": + logging.config.dictConfig(codesign.config_server.LOGGING) + + logger = logging.getLogger(__name__) + logger_server = logger.getChild('server') + # TODO(sergey): Consider moving such sanity checks into # CodeSigner.check_environment_or_die(). if not shutil.which('signtool.exe'): - raise SystemExit("signtool.exe is not found in %PATH%") + if util.get_current_platform() == util.Platform.WINDOWS: + raise SystemExit("signtool.exe is not found in %PATH%") + logger_server.info( + 'signtool.exe not found, ' + 'but will not be used on this foreign platform') - logging.config.dictConfig(codesign.config_server.LOGGING) code_signer = WindowsCodeSigner(codesign.config_server) code_signer.run_signing_server() diff --git a/build_files/buildbot/slave_bundle_dmg.py b/build_files/buildbot/slave_bundle_dmg.py new file mode 100755 index 00000000000..11d2c3cb602 --- /dev/null +++ b/build_files/buildbot/slave_bundle_dmg.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +import argparse +import re +import shutil +import subprocess +import sys +import time + +from pathlib import Path +from tempfile import TemporaryDirectory, NamedTemporaryFile +from typing import List + +BUILDBOT_DIRECTORY = Path(__file__).absolute().parent +CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'slave_codesign.py' +BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent +DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' + + +# Extra size which is added on top of actual files size when estimating size +# of destination DNG. +EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 + +################################################################################ +# Common utilities + + +def get_directory_size(root_directory: Path) -> int: + """ + Get size of directory on disk + """ + + total_size = 0 + for file in root_directory.glob('**/*'): + total_size += file.lstat().st_size + return total_size + + +################################################################################ +# DMG bundling specific logic + +def create_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'source_dir', + type=Path, + help='Source directory which points to either existing .app bundle' + 'or to a directory with .app bundles.') + parser.add_argument( + '--background-image', + type=Path, + help="Optional background picture which will be set on the DMG." + "If not provided default Blender's one is used.") + parser.add_argument( + '--volume-name', + type=str, + help='Optional name of a volume which will be used for DMG.') + parser.add_argument( + '--dmg', + type=Path, + help='Optional argument which points to a final DMG file name.') + parser.add_argument( + '--applescript', + type=Path, + help="Optional path to applescript to set up folder looks of DMG." + "If not provided default Blender's one is used.") + return parser + + +def collect_app_bundles(source_dir: Path) -> List[Path]: + """ + Collect all app bundles which are to be put into DMG + + If the source directory points to FOO.app it will be the only app bundle + packed. + + Otherwise all .app bundles from given directory are placed to a single + DMG. + """ + + if source_dir.name.endswith('.app'): + return [source_dir] + + app_bundles = [] + for filename in source_dir.glob('*'): + if not filename.is_dir(): + continue + if not filename.name.endswith('.app'): + continue + + app_bundles.append(filename) + + return app_bundles + + +def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: + app_bundles = collect_app_bundles(source_dir) + + if not app_bundles: + print('No app bundles found for packing') + return + + print(f'Found {len(app_bundles)} to pack:') + for app_bundle in app_bundles: + print(f'- {app_bundle}') + + return app_bundles + + +def estimate_dmg_size(app_bundles: List[Path]) -> int: + """ + Estimate size of DMG to hold requested app bundles + + The size is based on actual size of all files in all bundles plus some + space to compensate for different size-on-disk plus some space to hold + codesign signatures. + + Is better to be on a high side since the empty space is compressed, but + lack of space might cause silent failures later on. + """ + + app_bundles_size = 0 + for app_bundle in app_bundles: + app_bundles_size += get_directory_size(app_bundle) + + return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES + + +def copy_app_bundles_to_directory(app_bundles: List[Path], + directory: Path) -> None: + """ + Copy all bundles to a given directory + + This directory is what the DMG will be created from. + """ + for app_bundle in app_bundles: + print(f'Copying {app_bundle.name}...') + shutil.copytree(app_bundle, directory / app_bundle.name) + + +def get_main_app_bundle(app_bundles: List[Path]) -> Path: + """ + Get application bundle main for the installation + """ + return app_bundles[0] + + +def create_dmg_image(app_bundles: List[Path], + dmg_filepath: Path, + volume_name: str) -> None: + """ + Create DMG disk image and put app bundles in it + + No DMG configuration or codesigning is happening here. + """ + + if dmg_filepath.exists(): + print(f'Removing existing writable DMG {dmg_filepath}...') + dmg_filepath.unlink() + + print('Preparing directory with app bundles for the DMG...') + with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: + # Copy all bundles to a clean directory. + content_dir = Path(content_dir_str) + copy_app_bundles_to_directory(app_bundles, content_dir) + + # Estimate size of the DMG. + dmg_size = estimate_dmg_size(app_bundles) + print(f'Estimated DMG size: {dmg_size:,} bytes.') + + # Create the DMG. + print(f'Creating writable DMG {dmg_filepath}') + command = ('hdiutil', + 'create', + '-size', str(dmg_size), + '-fs', 'HFS+', + '-srcfolder', content_dir, + '-volname', volume_name, + '-format', 'UDRW', + dmg_filepath) + subprocess.run(command) + + +def get_writable_dmg_filepath(dmg_filepath: Path): + """ + Get file path for writable DMG image + """ + parent = dmg_filepath.parent + return parent / (dmg_filepath.stem + '-temp.dmg') + + +def mount_readwrite_dmg(dmg_filepath: Path) -> None: + """ + Mount writable DMG + + Mounting point would be /Volumes/<volume name> + """ + + print(f'Mounting read-write DMG ${dmg_filepath}') + command = ('hdiutil', + 'attach', '-readwrite', + '-noverify', + '-noautoopen', + dmg_filepath) + subprocess.run(command) + + +def get_mount_directory_for_volume_name(volume_name: str) -> Path: + """ + Get directory under which the volume will be mounted + """ + + return Path('/Volumes') / volume_name + + +def eject_volume(volume_name: str) -> None: + """ + Eject given volume, if mounted + """ + mount_directory = get_mount_directory_for_volume_name(volume_name) + if not mount_directory.exists(): + return + mount_directory_str = str(mount_directory) + + print(f'Ejecting volume {volume_name}') + + # Figure out which device to eject. + mount_output = subprocess.check_output(['mount']).decode() + device = '' + for line in mount_output.splitlines(): + if f'on {mount_directory_str} (' not in line: + continue + tokens = line.split(' ', 3) + if len(tokens) < 3: + continue + if tokens[1] != 'on': + continue + if device: + raise Exception( + f'Multiple devices found for mounting point {mount_directory}') + device = tokens[0] + + if not device: + raise Exception( + f'No device found for mounting point {mount_directory}') + + print(f'{mount_directory} is mounted as device {device}, ejecting...') + subprocess.run(['diskutil', 'eject', device]) + + +def copy_background_if_needed(background_image_filepath: Path, + mount_directory: Path) -> None: + """ + Copy background to the DMG + + If the background image is not specified it will not be copied. + """ + + if not background_image_filepath: + print('No background image provided.') + return + + print(f'Copying background image {background_image_filepath}') + + destination_dir = mount_directory / '.background' + destination_dir.mkdir(exist_ok=True) + + destination_filepath = destination_dir / background_image_filepath.name + shutil.copy(background_image_filepath, destination_filepath) + + +def create_applications_link(mount_directory: Path) -> None: + """ + Create link to /Applications in the given location + """ + + print('Creating link to /Applications') + + command = ('ln', '-s', '/Applications', mount_directory / ' ') + subprocess.run(command) + + +def run_applescript(applescript: Path, + volume_name: str, + app_bundles: List[Path], + background_image_filepath: Path) -> None: + """ + Run given applescript to adjust look and feel of the DMG + """ + + main_app_bundle = get_main_app_bundle(app_bundles) + + with NamedTemporaryFile( + mode='w', suffix='.applescript') as temp_applescript: + print('Adjusting applescript for volume name...') + # Adjust script to the specific volume name. + with open(applescript, mode='r') as input: + for line in input.readlines(): + stripped_line = line.strip() + if stripped_line.startswith('tell disk'): + line = re.sub('tell disk ".*"', + f'tell disk "{volume_name}"', + line) + elif stripped_line.startswith('set background picture'): + if not background_image_filepath: + continue + else: + background_image_short = \ + '.background:' + background_image_filepath.name + line = re.sub('to file ".*"', + f'to file "{background_image_short}"', + line) + line = line.replace('blender.app', main_app_bundle.name) + temp_applescript.write(line) + + temp_applescript.flush() + + print('Running applescript...') + command = ('osascript', temp_applescript.name) + subprocess.run(command) + + print('Waiting for applescript...') + + # NOTE: This is copied from bundle.sh. The exact reason for sleep is + # still remained a mystery. + time.sleep(5) + + +def codesign(subject: Path): + """ + Codesign file or directory + + NOTE: For DMG it will also notarize. + """ + + command = (CODESIGN_SCRIPT, subject) + subprocess.run(command) + + +def codesign_app_bundles_in_dmg(mount_directory: str) -> None: + """ + Code sign all binaries and bundles in the mounted directory + """ + + print(f'Codesigning all app bundles in {mount_directory}') + codesign(mount_directory) + + +def codesign_and_notarize_dmg(dmg_filepath: Path) -> None: + """ + Run codesign and notarization pipeline on the DMG + """ + + print(f'Codesigning and notarizing DMG {dmg_filepath}') + codesign(dmg_filepath) + + +def compress_dmg(writable_dmg_filepath: Path, + final_dmg_filepath: Path) -> None: + """ + Compress temporary read-write DMG + """ + command = ('hdiutil', 'convert', + writable_dmg_filepath, + '-format', 'UDZO', + '-o', final_dmg_filepath) + + if final_dmg_filepath.exists(): + print(f'Removing old compressed DMG {final_dmg_filepath}') + final_dmg_filepath.unlink() + + print('Compressing disk image...') + subprocess.run(command) + + +def create_final_dmg(app_bundles: List[Path], + dmg_filepath: Path, + background_image_filepath: Path, + volume_name: str, + applescript: Path) -> None: + """ + Create DMG with all app bundles + + Will take care configuring background, signing all binaries and app bundles + and notarizing the DMG. + """ + + print('Running all routines to create final DMG') + + writable_dmg_filepath = get_writable_dmg_filepath(dmg_filepath) + mount_directory = get_mount_directory_for_volume_name(volume_name) + + # Make sure volume is not mounted. + # If it is mounted it will prevent removing old DMG files and could make + # it so app bundles are copied to the wrong place. + eject_volume(volume_name) + + create_dmg_image(app_bundles, writable_dmg_filepath, volume_name) + + mount_readwrite_dmg(writable_dmg_filepath) + + # Run codesign first, prior to copying amything else. + # + # This allows to recurs into the content of bundles without worrying about + # possible interfereice of Application symlink. + codesign_app_bundles_in_dmg(mount_directory) + + copy_background_if_needed(background_image_filepath, mount_directory) + create_applications_link(mount_directory) + run_applescript(applescript, volume_name, app_bundles, + background_image_filepath) + + print('Ejecting read-write DMG image...') + eject_volume(volume_name) + + compress_dmg(writable_dmg_filepath, dmg_filepath) + writable_dmg_filepath.unlink() + + codesign_and_notarize_dmg(dmg_filepath) + + +def ensure_dmg_extension(filepath: Path) -> Path: + """ + Make sure given file have .dmg extension + """ + + if filepath.suffix != '.dmg': + return filepath.with_suffix(f'{filepath.suffix}.dmg') + return filepath + + +def get_dmg_filepath(requested_name: Path, app_bundles: List[Path]) -> Path: + """ + Get full file path for the final DMG image + + Will use the provided one when possible, otherwise will deduct it from + app bundles. + + If the name is deducted, the DMG is stored in the current directory. + """ + + if requested_name: + return ensure_dmg_extension(requested_name.absolute()) + + # TODO(sergey): This is not necessarily the main one. + main_bundle = app_bundles[0] + # Strip .app from the name + return Path(main_bundle.name[:-4] + '.dmg').absolute() + + +def get_background_image(requested_background_image: Path) -> Path: + """ + Get effective filepath for the background image + """ + + if requested_background_image: + return requested_background_image.absolute() + + return DARWIN_DIRECTORY / 'background.tif' + + +def get_applescript(requested_applescript: Path) -> Path: + """ + Get effective filepath for the applescript + """ + + if requested_applescript: + return requested_applescript.absolute() + + return DARWIN_DIRECTORY / 'blender.applescript' + + +def get_volume_name_from_dmg_filepath(dmg_filepath: Path) -> str: + """ + Deduct volume name from the DMG path + + Will use first part of the DMG file name prior to dash. + """ + + tokens = dmg_filepath.stem.split('-') + words = tokens[0].split() + + return ' '.join(word.capitalize() for word in words) + + +def get_volume_name(requested_volume_name: str, + dmg_filepath: Path) -> str: + """ + Get effective name for DMG volume + """ + + if requested_volume_name: + return requested_volume_name + + return get_volume_name_from_dmg_filepath(dmg_filepath) + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + + # Get normalized input parameters. + source_dir = args.source_dir.absolute() + background_image_filepath = get_background_image(args.background_image) + applescript = get_applescript(args.applescript) + + app_bundles = collect_and_log_app_bundles(source_dir) + if not app_bundles: + return + + dmg_filepath = get_dmg_filepath(args.dmg, app_bundles) + volume_name = get_volume_name(args.volume_name, dmg_filepath) + + print(f'Will produce DMG "{dmg_filepath.name}" (without quotes)') + + create_final_dmg(app_bundles, + dmg_filepath, + background_image_filepath, + volume_name, + applescript) + + +if __name__ == "__main__": + main() diff --git a/build_files/buildbot/slave_codesign.py b/build_files/buildbot/slave_codesign.py index 8dedf5ffcd3..a82ee98b1b5 100755 --- a/build_files/buildbot/slave_codesign.py +++ b/build_files/buildbot/slave_codesign.py @@ -45,7 +45,7 @@ def create_argument_parser(): def main(): parser = create_argument_parser() args = parser.parse_args() - path_to_sign = args.path_to_sign + path_to_sign = args.path_to_sign.absolute() if sys.platform == 'win32': # When WIX packed is used to generate .msi on Windows the CPack will diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py index f47cfe0347e..3cefe2d5ec6 100644 --- a/build_files/buildbot/slave_pack.py +++ b/build_files/buildbot/slave_pack.py @@ -109,14 +109,15 @@ def pack_mac(builder): package_filepath = os.path.join(builder.build_dir, package_filename) release_dir = os.path.join(builder.blender_dir, 'release', 'darwin') - bundle_sh = os.path.join(release_dir, 'bundle.sh') + buildbot_dir = os.path.join(builder.blender_dir, 'build_files', 'buildbot') + bundle_script = os.path.join(buildbot_dir, 'slave_bundle_dmg.py') - command = [bundle_sh] - command += ['--source', builder.install_dir] + command = [bundle_script] command += ['--dmg', package_filepath] if info.is_development_build: background_image = os.path.join(release_dir, 'buildbot', 'background.tif') command += ['--background-image', background_image] + command += [builder.install_dir] buildbot_utils.call(command) create_buildbot_upload_zip(builder, [(package_filepath, package_filename)]) diff --git a/source/blender/draw/engines/overlay/overlay_armature.c b/source/blender/draw/engines/overlay/overlay_armature.c index 416283e321b..97290abe4b7 100644 --- a/source/blender/draw/engines/overlay/overlay_armature.c +++ b/source/blender/draw/engines/overlay/overlay_armature.c @@ -185,10 +185,10 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata) DRWPass **p_armature_ps = &psl->armature_ps[i]; cb->custom_shapes_ghash = BLI_ghash_ptr_new(__func__); + cb->custom_shapes_transp_ghash = BLI_ghash_ptr_new(__func__); DRWState infront_state = (DRW_state_is_select() && (i == 1)) ? DRW_STATE_IN_FRONT_SELECT : 0; - state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_CULL_BACK | - (pd->armature.transparent ? DRW_STATE_BLEND_ALPHA : DRW_STATE_WRITE_DEPTH); + state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_WRITE_DEPTH; DRW_PASS_CREATE(*p_armature_ps, state | pd->clipping_state | infront_state); DRWPass *armature_ps = *p_armature_ps; @@ -202,32 +202,44 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata) sh = OVERLAY_shader_armature_sphere(false); grp = DRW_shgroup_create(sh, armature_ps); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); - DRW_shgroup_uniform_float_copy(grp, "alpha", pd->armature.transparent ? 0.4f : 1.0f); + DRW_shgroup_uniform_float_copy(grp, "alpha", 1.0f); cb->point_solid = BUF_INSTANCE(grp, format, DRW_cache_bone_point_get()); + grp = DRW_shgroup_create(sh, armature_ps); + DRW_shgroup_state_disable(grp, DRW_STATE_WRITE_DEPTH); + DRW_shgroup_state_enable(grp, DRW_STATE_BLEND_ALPHA); + DRW_shgroup_uniform_float_copy(grp, "alpha", 0.4f); + cb->point_transp = BUF_INSTANCE(grp, format, DRW_cache_bone_point_get()); + sh = OVERLAY_shader_armature_shape(false); - cb->custom_solid = grp = DRW_shgroup_create(sh, armature_ps); + grp = DRW_shgroup_create(sh, armature_ps); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); - DRW_shgroup_uniform_float_copy(grp, "alpha", pd->armature.transparent ? 0.6f : 1.0f); + DRW_shgroup_uniform_float_copy(grp, "alpha", 1.0f); + cb->custom_solid = grp; cb->box_solid = BUF_INSTANCE(grp, format, DRW_cache_bone_box_get()); cb->octa_solid = BUF_INSTANCE(grp, format, DRW_cache_bone_octahedral_get()); + grp = DRW_shgroup_create(sh, armature_ps); + DRW_shgroup_state_disable(grp, DRW_STATE_WRITE_DEPTH); + DRW_shgroup_state_enable(grp, DRW_STATE_BLEND_ALPHA); + DRW_shgroup_uniform_float_copy(grp, "alpha", 0.6f); + cb->custom_transp = grp; + cb->box_transp = BUF_INSTANCE(grp, format, DRW_cache_bone_box_get()); + cb->octa_transp = BUF_INSTANCE(grp, format, DRW_cache_bone_octahedral_get()); + sh = OVERLAY_shader_armature_sphere(true); grp = DRW_shgroup_create(sh, armature_ps); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); cb->point_outline = BUF_INSTANCE(grp, format, DRW_cache_bone_point_wire_outline_get()); sh = OVERLAY_shader_armature_shape(true); cb->custom_outline = grp = DRW_shgroup_create(sh, armature_ps); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); cb->box_outline = BUF_INSTANCE(grp, format, DRW_cache_bone_box_wire_get()); cb->octa_outline = BUF_INSTANCE(grp, format, DRW_cache_bone_octahedral_wire_get()); sh = OVERLAY_shader_armature_shape_wire(); cb->custom_wire = grp = DRW_shgroup_create(sh, armature_ps); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); } { @@ -236,7 +248,6 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata) sh = OVERLAY_shader_armature_degrees_of_freedom(); grp = DRW_shgroup_create(sh, armature_ps); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); cb->dof_lines = BUF_INSTANCE(grp, format, DRW_cache_bone_dof_lines_get()); grp = DRW_shgroup_create(sh, psl->armature_transp_ps); @@ -256,16 +267,22 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata) sh = OVERLAY_shader_armature_envelope(false); grp = DRW_shgroup_create(sh, armature_ps); + DRW_shgroup_state_enable(grp, DRW_STATE_CULL_BACK); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); DRW_shgroup_uniform_bool_copy(grp, "isDistance", false); - DRW_shgroup_uniform_float_copy(grp, "alpha", pd->armature.transparent ? 0.6f : 1.0f); + DRW_shgroup_uniform_float_copy(grp, "alpha", 1.0f); cb->envelope_solid = BUF_INSTANCE(grp, format, DRW_cache_bone_envelope_solid_get()); + grp = DRW_shgroup_create(sh, armature_ps); + DRW_shgroup_state_disable(grp, DRW_STATE_WRITE_DEPTH); + DRW_shgroup_state_enable(grp, DRW_STATE_BLEND_ALPHA | DRW_STATE_CULL_BACK); + DRW_shgroup_uniform_float_copy(grp, "alpha", 0.6f); + cb->envelope_transp = BUF_INSTANCE(grp, format, DRW_cache_bone_envelope_solid_get()); + format = formats->instance_bone_envelope_outline; sh = OVERLAY_shader_armature_envelope(true); grp = DRW_shgroup_create(sh, armature_ps); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); cb->envelope_outline = BUF_INSTANCE(grp, format, DRW_cache_bone_envelope_outline_get()); @@ -283,7 +300,6 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata) sh = OVERLAY_shader_armature_wire(); grp = DRW_shgroup_create(sh, armature_ps); - DRW_shgroup_state_disable(grp, DRW_STATE_BLEND_ALPHA); DRW_shgroup_uniform_block_persistent(grp, "globalsBlock", G_draw.block_ubo); cb->wire = BUF_LINE(grp, format); } @@ -2221,9 +2237,12 @@ static void armature_context_setup(ArmatureDrawContext *ctx, const bool is_pose_mode, float *const_color) { + const bool is_object_mode = !do_envelope_dist; const bool is_xray = (ob->dtx & OB_DRAWXRAY) != 0 || (pd->armature.do_pose_fade_geom && is_pose_mode); - const bool is_filled = !pd->armature.transparent || do_envelope_dist; + const bool draw_as_wire = (ob->dt < OB_SOLID); + const bool is_filled = (!pd->armature.transparent && !draw_as_wire) || !is_object_mode; + const bool is_transparent = pd->armature.transparent || (draw_as_wire && !is_object_mode); bArmature *arm = ob->data; OVERLAY_ArmatureCallBuffers *cb = &pd->armature_call_buffers[is_xray]; @@ -2232,7 +2251,9 @@ static void armature_context_setup(ArmatureDrawContext *ctx, switch (arm->drawtype) { case ARM_ENVELOPE: ctx->envelope_outline = cb->envelope_outline; - ctx->envelope_solid = (is_filled) ? cb->envelope_solid : NULL; + ctx->envelope_solid = (is_filled) ? + (is_transparent ? cb->envelope_transp : cb->envelope_solid) : + NULL; ctx->envelope_distance = (do_envelope_dist) ? cb->envelope_distance : NULL; break; case ARM_LINE: @@ -2243,31 +2264,31 @@ static void armature_context_setup(ArmatureDrawContext *ctx, break; case ARM_B_BONE: ctx->outline = cb->box_outline; - ctx->solid = (is_filled) ? cb->box_solid : NULL; + ctx->solid = (is_filled) ? (is_transparent ? cb->box_transp : cb->box_solid) : NULL; break; case ARM_OCTA: ctx->outline = cb->octa_outline; - ctx->solid = (is_filled) ? cb->octa_solid : NULL; + ctx->solid = (is_filled) ? (is_transparent ? cb->octa_transp : cb->octa_solid) : NULL; break; } ctx->ob = ob; ctx->extras = &pd->extra_call_buffers[is_xray]; ctx->dof_lines = cb->dof_lines; ctx->dof_sphere = cb->dof_sphere; - ctx->point_solid = (is_filled) ? cb->point_solid : NULL; + ctx->point_solid = (is_filled) ? (is_transparent ? cb->point_transp : cb->point_solid) : NULL; ctx->point_outline = cb->point_outline; - ctx->custom_solid = (is_filled) ? cb->custom_solid : NULL; + ctx->custom_solid = (is_filled) ? (is_transparent ? cb->custom_transp : cb->custom_solid) : NULL; ctx->custom_outline = cb->custom_outline; ctx->custom_wire = cb->custom_wire; - ctx->custom_shapes_ghash = cb->custom_shapes_ghash; - ctx->transparent = pd->armature.transparent; + ctx->custom_shapes_ghash = is_transparent ? cb->custom_shapes_transp_ghash : + cb->custom_shapes_ghash; ctx->show_relations = pd->armature.show_relations; ctx->do_relations = !DRW_state_is_select() && pd->armature.show_relations && (is_edit_mode | is_pose_mode); ctx->const_color = DRW_state_is_select() ? select_const_color : const_color; ctx->const_wire = (((ob->base_flag & BASE_SELECTED) || (arm->drawtype == ARM_WIRE)) ? 1.5f : - ((ctx->transparent) ? 1.0f : 0.0f)); + ((!is_filled || is_transparent) ? 1.0f : 0.0f)); /** See: 'set_pchan_color'*/ #define NO_ALPHA(c) (((c)[3] = 1.0f), (c)) @@ -2363,6 +2384,7 @@ void OVERLAY_armature_cache_finish(OVERLAY_Data *vedata) if (pd->armature_call_buffers[i].custom_shapes_ghash) { /* TODO(fclem): Do not free it for each frame but reuse it. Avoiding alloc cost. */ BLI_ghash_free(pd->armature_call_buffers[i].custom_shapes_ghash, NULL, NULL); + BLI_ghash_free(pd->armature_call_buffers[i].custom_shapes_transp_ghash, NULL, NULL); } } } diff --git a/source/blender/draw/engines/overlay/overlay_private.h b/source/blender/draw/engines/overlay/overlay_private.h index 996ee845799..afec20b477d 100644 --- a/source/blender/draw/engines/overlay/overlay_private.h +++ b/source/blender/draw/engines/overlay/overlay_private.h @@ -177,6 +177,7 @@ typedef struct OVERLAY_ExtraCallBuffers { typedef struct OVERLAY_ArmatureCallBuffers { DRWCallBuffer *box_outline; DRWCallBuffer *box_solid; + DRWCallBuffer *box_transp; DRWCallBuffer *dof_lines; DRWCallBuffer *dof_sphere; @@ -184,20 +185,25 @@ typedef struct OVERLAY_ArmatureCallBuffers { DRWCallBuffer *envelope_distance; DRWCallBuffer *envelope_outline; DRWCallBuffer *envelope_solid; + DRWCallBuffer *envelope_transp; DRWCallBuffer *octa_outline; DRWCallBuffer *octa_solid; + DRWCallBuffer *octa_transp; DRWCallBuffer *point_outline; DRWCallBuffer *point_solid; + DRWCallBuffer *point_transp; DRWCallBuffer *stick; DRWCallBuffer *wire; - DRWShadingGroup *custom_solid; DRWShadingGroup *custom_outline; + DRWShadingGroup *custom_solid; + DRWShadingGroup *custom_transp; DRWShadingGroup *custom_wire; + GHash *custom_shapes_transp_ghash; GHash *custom_shapes_ghash; } OVERLAY_ArmatureCallBuffers; diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl index ffc941149c7..6398426952c 100644 --- a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl +++ b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl @@ -7,6 +7,7 @@ in vec3 vPos[]; in vec2 ssPos[]; in vec2 ssNor[]; in vec4 vColSize[]; +in int inverted[]; flat out vec4 finalColor; flat out vec2 edgeStart; @@ -39,6 +40,7 @@ void main(void) } } + n0 = (inverted[0] == 1) ? -n0 : n0; /* Don't outline if concave edge. */ if (dot(n0, v13) > 0.0001) { return; @@ -68,7 +70,7 @@ void main(void) /* Offset away from the center to avoid overlap with solid shape. */ gl_Position.xy += (edge_dir - perp) * sizeViewportInv.xy * gl_Position.w; /* Improve AA bleeding inside bone silhouette. */ - gl_Position.z -= 1e-4; + gl_Position.z -= (is_persp) ? 1e-4 : 1e-6; edgeStart = edgePos = ((gl_Position.xy / gl_Position.w) * 0.5 + 0.5) * sizeViewport.xy; #ifdef USE_WORLD_CLIP_PLANES world_clip_planes_set_clip_distance(gl_in[1].gl_ClipDistance); @@ -79,7 +81,7 @@ void main(void) /* Offset away from the center to avoid overlap with solid shape. */ gl_Position.xy += (edge_dir + perp) * sizeViewportInv.xy * gl_Position.w; /* Improve AA bleeding inside bone silhouette. */ - gl_Position.z -= 1e-4; + gl_Position.z -= (is_persp) ? 1e-4 : 1e-6; edgeStart = edgePos = ((gl_Position.xy / gl_Position.w) * 0.5 + 0.5) * sizeViewport.xy; #ifdef USE_WORLD_CLIP_PLANES world_clip_planes_set_clip_distance(gl_in[2].gl_ClipDistance); diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl index cd9368a997a..b720be0c7d1 100644 --- a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl +++ b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl @@ -12,6 +12,7 @@ out vec3 vPos; out vec2 ssPos; out vec2 ssNor; out vec4 vColSize; +out int inverted; /* project to screen space */ vec2 proj(vec4 pos) @@ -30,6 +31,8 @@ void main() vPos = viewpos.xyz; pPos = ProjectionMatrix * viewpos; + inverted = int(dot(cross(model_mat[0].xyz, model_mat[1].xyz), model_mat[2].xyz) < 0.0); + /* This is slow and run per vertex, but it's still faster than * doing it per instance on CPU and sending it on via instance attribute. */ mat3 normal_mat = transpose(inverse(mat3(model_mat))); diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl index 54cd807edaa..17e8d0da5d9 100644 --- a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl +++ b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl @@ -2,12 +2,19 @@ uniform float alpha = 0.6; in vec4 finalColor; +flat in int inverted; layout(location = 0) out vec4 fragColor; layout(location = 1) out vec4 lineOutput; void main() { + /* Manual backface cullling.. Not ideal for performance + * but needed for view clarity in xray mode and support + * for inverted bone matrices. */ + if ((inverted == 1) == gl_FrontFacing) { + discard; + } fragColor = vec4(finalColor.rgb, alpha); lineOutput = vec4(0.0); } diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl index 8284bd43adc..a6af7214de1 100644 --- a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl +++ b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl @@ -7,6 +7,7 @@ in vec3 nor; in mat4 inst_obmat; out vec4 finalColor; +flat out int inverted; void main() { @@ -18,6 +19,8 @@ void main() mat3 normal_mat = transpose(inverse(mat3(model_mat))); vec3 normal = normalize(normal_world_to_view(normal_mat * nor)); + inverted = int(dot(cross(model_mat[0].xyz, model_mat[1].xyz), model_mat[2].xyz) < 0.0); + /* Do lighting at an angle to avoid flat shading on front facing bone. */ const vec3 light = vec3(0.1, 0.1, 0.8); float n = dot(normal, light); |