From b8479a70c936c5c59f199a847d679c80196e6c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 13 Apr 2021 17:47:48 +0200 Subject: Add `make source_archive_complete` target Add a `source_archive_complete` target for `make` that creates a source archive including the source packages of Blender's dependencies. This expands `make_source_archive.py` to include files from `${BUILD_DIR}/source_archive/packages/` as well. Reviewed By: dfelinto Maniphest Tasks: T86124 Differential Revision: https://developer.blender.org/D10727 --- build_files/build_environment/cmake/options.cmake | 10 +- build_files/utils/make_source_archive.py | 130 ++++++++++++++++++---- 2 files changed, 112 insertions(+), 28 deletions(-) (limited to 'build_files') diff --git a/build_files/build_environment/cmake/options.cmake b/build_files/build_environment/cmake/options.cmake index 15ceb693ae0..486b3d1a802 100644 --- a/build_files/build_environment/cmake/options.cmake +++ b/build_files/build_environment/cmake/options.cmake @@ -37,14 +37,8 @@ else(BUILD_MODE STREQUAL "Debug") endif() set(DOWNLOAD_DIR "${CMAKE_CURRENT_BINARY_DIR}/downloads" CACHE STRING "Path for downloaded files") -# look in blenders source folder for packages directory, if that exists -# it will our package folder, otherwise it will be in the build folder -if(EXISTS "${CMAKE_SOURCE_DIR}/../../packages") - set(PACKAGE_DIR_DEFAULT "${CMAKE_SOURCE_DIR}/../../packages") -else() - set(PACKAGE_DIR_DEFAULT "${CMAKE_CURRENT_BINARY_DIR}/packages") -endif() -set(PACKAGE_DIR ${PACKAGE_DIR_DEFAULT} CACHE STRING "Path for downloaded source files") +# This path must be hard-coded like this, so that the GNUmakefile knows where it is and can pass it to make_source_archive.py: +set(PACKAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/packages") option(PACKAGE_USE_UPSTREAM_SOURCES "Use soures upstream to download the package sources, when OFF the blender mirror will be used" ON) file(TO_CMAKE_PATH ${DOWNLOAD_DIR} DOWNLOAD_DIR) diff --git a/build_files/utils/make_source_archive.py b/build_files/utils/make_source_archive.py index 271ca358f7e..f9f3f59849f 100755 --- a/build_files/utils/make_source_archive.py +++ b/build_files/utils/make_source_archive.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 +import argparse import dataclasses import os import re import subprocess from pathlib import Path -from typing import Iterable, TextIO +from typing import Iterable, TextIO, Optional, Any # This script can run from any location, # output is created in the $CWD @@ -18,21 +19,43 @@ SKIP_NAMES = { ".gitignore", ".gitmodules", ".arcconfig", + ".svn", } def main() -> None: - output_dir = Path(".").absolute() blender_srcdir = Path(__file__).absolute().parent.parent.parent + + cli_parser = argparse.ArgumentParser( + description=f"Create a tarball of the Blender sources, optionally including sources of dependencies.", + epilog="This script is intended to be run by `make source_archive_complete`.", + ) + cli_parser.add_argument( + "-p", + "--include-packages", + type=Path, + default=None, + metavar="PACKAGE_PATH", + help="Include all source files from the given package directory as well.", + ) + + cli_args = cli_parser.parse_args() + print(f"Source dir: {blender_srcdir}") + curdir = blender_srcdir.parent + os.chdir(curdir) + blender_srcdir = blender_srcdir.relative_to(curdir) + + print(f"Output dir: {curdir}") + version = parse_blender_version(blender_srcdir) - manifest = output_dir / f"blender-{version}-manifest.txt" - tarball = output_dir / f"blender-{version}.tar.xz" + tarball = tarball_path(curdir, version, cli_args) + manifest = manifest_path(tarball) + packages_dir = packages_path(curdir, cli_args) - os.chdir(blender_srcdir) - create_manifest(version, manifest) - create_tarball(version, tarball, manifest) + create_manifest(version, manifest, blender_srcdir, packages_dir) + create_tarball(version, tarball, manifest, blender_srcdir, packages_dir) create_checksum_file(tarball) cleanup(manifest) print("Done!") @@ -84,43 +107,109 @@ def parse_blender_version(blender_srcdir: Path) -> BlenderVersion: ) +def tarball_path(output_dir: Path, version: BlenderVersion, cli_args: Any) -> Path: + extra = "" + if cli_args.include_packages: + extra = "-with-libraries" + + return output_dir / f"blender{extra}-{version}.tar.xz" + + +def manifest_path(tarball: Path) -> Path: + """Return the manifest path for the given tarball path. + + >>> from pathlib import Path + >>> tarball = Path("/home/sybren/workspace/blender-git/blender-test.tar.gz") + >>> manifest_path(tarball).as_posix() + '/home/sybren/workspace/blender-git/blender-test-manifest.txt' + """ + # ".tar.gz" is seen as two suffixes. + without_suffix = tarball.with_suffix("").with_suffix("") + name = without_suffix.name + return without_suffix.with_name(f"{name}-manifest.txt") + + +def packages_path(current_directory: Path, cli_args: Any) -> Optional[Path]: + if not cli_args.include_packages: + return None + + abspath = cli_args.include_packages.absolute() + + # os.path.relpath() can return paths like "../../packages", where + # Path.relative_to() will not go up directories (so its return value never + # has "../" in there). + relpath = os.path.relpath(abspath, current_directory) + + return Path(relpath) + + ### Manifest creation -def create_manifest(version: BlenderVersion, outpath: Path) -> None: +def create_manifest( + version: BlenderVersion, + outpath: Path, + blender_srcdir: Path, + packages_dir: Optional[Path], +) -> None: print(f'Building manifest of files: "{outpath}"...', end="", flush=True) with outpath.open("w", encoding="utf-8") as outfile: - main_files_to_manifest(outfile) - submodules_to_manifest(version, outfile) + main_files_to_manifest(blender_srcdir, outfile) + submodules_to_manifest(blender_srcdir, version, outfile) + + if packages_dir: + packages_to_manifest(outfile, packages_dir) print("OK") -def main_files_to_manifest(outfile: TextIO) -> None: - for path in git_ls_files(): +def main_files_to_manifest(blender_srcdir: Path, outfile: TextIO) -> None: + assert not blender_srcdir.is_absolute() + for path in git_ls_files(blender_srcdir): print(path, file=outfile) -def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None: +def submodules_to_manifest( + blender_srcdir: Path, version: BlenderVersion, outfile: TextIO +) -> None: skip_addon_contrib = version.is_release + assert not blender_srcdir.is_absolute() - for line in git_command("submodule"): + for line in git_command("-C", blender_srcdir, "submodule"): submodule = line.split()[1] # Don't use native slashes as GIT for MS-Windows outputs forward slashes. if skip_addon_contrib and submodule == "release/scripts/addons_contrib": continue - for path in git_ls_files(Path(submodule)): + for path in git_ls_files(blender_srcdir / submodule): print(path, file=outfile) -def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None: +def packages_to_manifest(outfile: TextIO, packages_dir: Path) -> None: + for path in packages_dir.glob("*"): + if not path.is_file(): + continue + if path.name in SKIP_NAMES: + continue + print(path, file=outfile) + + +### Higher-level functions + + +def create_tarball( + version: BlenderVersion, tarball: Path, manifest: Path, blender_srcdir: Path, packages_dir: Optional[Path] +) -> None: print(f'Creating archive: "{tarball}" ...', end="", flush=True) + command = ["tar"] + # Requires GNU `tar`, since `--transform` is used. - command = [ - "tar", + if packages_dir: + command += ["--transform", f"s,{packages_dir}/,packages/,g"] + + command += [ "--transform", - f"s,^,blender-{version}/,g", + f"s,^{blender_srcdir.name}/,blender-{version}/,g", "--use-compress-program=xz -9", "--create", f"--file={tarball}", @@ -130,7 +219,8 @@ def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> No "--owner=0", "--group=0", ] - subprocess.run(command, check=True, timeout=300) + + subprocess.run(command, check=True, timeout=3600) print("OK") -- cgit v1.2.3