From 8771f015f50ec72c3b0eaac1eb9498d277a4952b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Mon, 8 Mar 2021 18:10:26 +0100 Subject: Python version of `make_source_archive.sh` This is a Python version of the existing `make_source_archive.sh` script. IMO it's easier to read, and it'll also be easier to extend with the necessary functionality for D10598. The number of lines of code is larger than `make_source_archive.sh`, but it has considerably less invocations of `awk` ;-) And also the filtering is integrated, instead of forking out to Python to prevent certain files to be included in the tarball. Reviewed By: dfelinto, campbellbarton Differential Revision: https://developer.blender.org/D10629 --- build_files/utils/make_source_archive.py | 191 +++++++++++++++++++++++++++++++ build_files/utils/make_source_archive.sh | 82 ------------- 2 files changed, 191 insertions(+), 82 deletions(-) create mode 100755 build_files/utils/make_source_archive.py delete mode 100755 build_files/utils/make_source_archive.sh (limited to 'build_files') diff --git a/build_files/utils/make_source_archive.py b/build_files/utils/make_source_archive.py new file mode 100755 index 00000000000..24928742a2d --- /dev/null +++ b/build_files/utils/make_source_archive.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +import dataclasses +import os +import re +import subprocess +from pathlib import Path +from typing import Iterable, TextIO + +# This script can run from any location, +# output is created in the $CWD + +SKIP_NAMES = { + ".gitignore", + ".gitmodules", + ".arcconfig", +} + + +def main() -> None: + output_dir = Path(".").absolute() + blender_srcdir = Path(__file__).absolute().parent.parent.parent + print(f"Source dir: {blender_srcdir}") + + version = parse_blender_version(blender_srcdir) + manifest = output_dir / f"blender-{version}-manifest.txt" + tarball = output_dir / f"blender-{version}.tar.xz" + + os.chdir(blender_srcdir) + create_manifest(version, manifest) + create_tarball(version, tarball, manifest) + create_checksum_file(tarball) + cleanup(manifest) + print("Done!") + + +@dataclasses.dataclass +class BlenderVersion: + version: int # 293 for 2.93.1 + patch: int # 1 for 2.93.1 + cycle: str # 'alpha', 'beta', 'release', maybe others. + + @property + def is_release(self) -> bool: + return self.cycle == "release" + + def __str__(self) -> str: + """Convert to version string. + + >>> str(BlenderVersion(293, 1, "alpha")) + '2.93.1-alpha' + >>> str(BlenderVersion(327, 0, "release")) + '3.27.0' + """ + + as_string = f"{self.version/100:.2f}.{self.patch}" + if self.is_release: + return as_string + return f"{as_string}-{self.cycle}" + + +def parse_blender_version(blender_srcdir: Path) -> BlenderVersion: + version_path = blender_srcdir / "source/blender/blenkernel/BKE_blender_version.h" + + version_info = {} + line_re = re.compile(r"^#define (BLENDER_VERSION[A-Z_]*)\s+([0-9a-z]+)$") + + with version_path.open(encoding="utf-8") as version_file: + for line in version_file: + match = line_re.match(line.strip()) + if not match: + continue + version_info[match.group(1)] = match.group(2) + + return BlenderVersion( + int(version_info["BLENDER_VERSION"]), + int(version_info["BLENDER_VERSION_PATCH"]), + version_info["BLENDER_VERSION_CYCLE"], + ) + + +### Manifest creation + + +def create_manifest(version: BlenderVersion, outpath: Path) -> None: + print(f'Building manifest of files: "{outpath}"...', end="", flush=True) + with outpath.open("w", encoding="utf-8") as outfile: + main_files_to_manifest(outfile) + submodules_to_manifest(version, outfile) + print("OK") + + +def main_files_to_manifest(outfile: TextIO) -> None: + for path in git_ls_files(): + print(path, file=outfile) + + +def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None: + skip_addon_contrib = version.is_release + + for line in git_command("submodule"): + submodule = line.split()[1] + + if skip_addon_contrib and submodule == "release/scripts/addons_contrib": + continue + + for path in git_ls_files(Path(submodule)): + print(path, file=outfile) + + +def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None: + print(f'Creating archive: "{tarball}" ...', end="", flush=True) + command = [ + "tar", + "--transform", + f"s,^,blender-{version}/,g", + "--use-compress-program=xz -9", + "--create", + f"--file={tarball}", + f"--files-from={manifest}", + # Without owner/group args, extracting the files as root will + # use ownership from the tar archive: + "--owner=0", + "--group=0", + ] + subprocess.run(command, check=True, timeout=300) + print("OK") + + +def create_checksum_file(tarball: Path) -> None: + md5_path = tarball.with_name(tarball.name + ".md5sum") + print(f'Creating checksum: "{md5_path}" ...', end="", flush=True) + command = [ + "md5sum", + # The name is enough, as the tarball resides in the same dir as the MD5 + # file, and that's the current working directory. + tarball.name, + ] + md5_cmd = subprocess.run( + command, stdout=subprocess.PIPE, check=True, text=True, timeout=300 + ) + with md5_path.open("w") as outfile: + outfile.write(md5_cmd.stdout) + print("OK") + + +def cleanup(manifest: Path) -> None: + print("Cleaning up ...", end="", flush=True) + if manifest.exists(): + manifest.unlink() + print("OK") + + +## Low-level commands + + +def git_ls_files(directory: Path = Path(".")) -> Iterable[Path]: + """Generator, yields lines of output from 'git ls-files'. + + Only lines that are actually files (so no directories, sockets, etc.) are + returned, and never one from SKIP_NAMES. + """ + for line in git_command("-C", str(directory), "ls-files"): + path = directory / line + if not path.is_file() or path.name in SKIP_NAMES: + continue + yield path + + +def git_command(*cli_args) -> Iterable[str]: + """Generator, yields lines of output from a Git command.""" + command = ("git", *cli_args) + + # import shlex + # print(">", " ".join(shlex.quote(arg) for arg in command)) + + git = subprocess.run( + command, stdout=subprocess.PIPE, check=True, text=True, timeout=30 + ) + for line in git.stdout.split("\n"): + if line: + yield line + + +if __name__ == "__main__": + import doctest + + if doctest.testmod().failed: + raise SystemExit("ERROR: Self-test failed, refusing to run") + + main() diff --git a/build_files/utils/make_source_archive.sh b/build_files/utils/make_source_archive.sh deleted file mode 100755 index 5d9096f3235..00000000000 --- a/build_files/utils/make_source_archive.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh - -# This script can run from any location, -# output is created in the $CWD - -BASE_DIR="$PWD" - -blender_srcdir=$(dirname -- $0)/../.. -blender_version=$(grep "BLENDER_VERSION\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}') -blender_version_patch=$(grep "BLENDER_VERSION_PATCH\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}') -blender_version_cycle=$(grep "BLENDER_VERSION_CYCLE\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}') - -VERSION=$(expr $blender_version / 100).$(expr $blender_version % 100).$blender_version_patch -if [ "$blender_version_cycle" = "release" ] ; then - SUBMODULE_EXCLUDE="^\(release/scripts/addons_contrib\)$" -else - VERSION=$VERSION-$blender_version_cycle - SUBMODULE_EXCLUDE="^$" # dummy regex -fi - -MANIFEST="blender-$VERSION-manifest.txt" -TARBALL="blender-$VERSION.tar.xz" - -cd "$blender_srcdir" - -# not so nice, but works -FILTER_FILES_PY=\ -"import os, sys; "\ -"[print(l[:-1]) for l in sys.stdin.readlines() "\ -"if os.path.isfile(l[:-1]) "\ -"if os.path.basename(l[:-1]) not in {"\ -"'.gitignore', "\ -"'.gitmodules', "\ -"'.arcconfig', "\ -"}"\ -"]" - -# Build master list -echo -n "Building manifest of files: \"$BASE_DIR/$MANIFEST\" ..." -git ls-files | python3 -c "$FILTER_FILES_PY" > $BASE_DIR/$MANIFEST - -# Enumerate submodules -for lcv in $(git submodule | awk '{print $2}' | grep -v "$SUBMODULE_EXCLUDE"); do - cd "$BASE_DIR" - cd "$blender_srcdir/$lcv" - git ls-files | python3 -c "$FILTER_FILES_PY" | awk '$0="'"$lcv"/'"$0' >> $BASE_DIR/$MANIFEST - cd "$BASE_DIR" -done -echo "OK" - - -# Create the tarball -# -# Without owner/group args, extracting the files as root will -# use ownership from the tar archive. -cd "$blender_srcdir" -echo -n "Creating archive: \"$BASE_DIR/$TARBALL\" ..." -tar \ - --transform "s,^,blender-$VERSION/,g" \ - --use-compress-program="xz -9" \ - --create \ - --file="$BASE_DIR/$TARBALL" \ - --files-from="$BASE_DIR/$MANIFEST" \ - --owner=0 \ - --group=0 - -echo "OK" - - -# Create checksum file -cd "$BASE_DIR" -echo -n "Creating checksum: \"$BASE_DIR/$TARBALL.md5sum\" ..." -md5sum "$TARBALL" > "$TARBALL.md5sum" -echo "OK" - - -# Cleanup -echo -n "Cleaning up ..." -rm "$BASE_DIR/$MANIFEST" -echo "OK" - -echo "Done!" -- cgit v1.2.3