Python version of `make_source_archive.sh`

This is a Python version of the existing `make_source_archive.sh`
script. IMO it's easier to read, and it'll also be easier to extend with
the necessary functionality for D10598.

The number of lines of code is larger than `make_source_archive.sh`, but
it has considerably less invocations of `awk` ;-) And also the filtering
is integrated, instead of forking out to Python to prevent certain files
to be included in the tarball.

Reviewed By: dfelinto, campbellbarton

Differential Revision: https://developer.blender.org/D10629

2 files changed, 191 insertions, 82 deletions

diff --git a/build_files/utils/make_source_archive.py b/build_files/utils/make_source_archive.py
new file mode 100755
index 00000000000..24928742a2d
--- /dev/null
+++ b/build_files/utils/make_source_archive.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import dataclasses
+import os
+import re
+import subprocess
+from pathlib import Path
+from typing import Iterable, TextIO
+
+# This script can run from any location,
+# output is created in the $CWD
+
+SKIP_NAMES = {
+    ".gitignore",
+    ".gitmodules",
+    ".arcconfig",
+}
+
+
+def main() -> None:
+    output_dir = Path(".").absolute()
+    blender_srcdir = Path(__file__).absolute().parent.parent.parent
+    print(f"Source dir: {blender_srcdir}")
+
+    version = parse_blender_version(blender_srcdir)
+    manifest = output_dir / f"blender-{version}-manifest.txt"
+    tarball = output_dir / f"blender-{version}.tar.xz"
+
+    os.chdir(blender_srcdir)
+    create_manifest(version, manifest)
+    create_tarball(version, tarball, manifest)
+    create_checksum_file(tarball)
+    cleanup(manifest)
+    print("Done!")
+
+
+@dataclasses.dataclass
+class BlenderVersion:
+    version: int  # 293 for 2.93.1
+    patch: int  # 1 for 2.93.1
+    cycle: str  # 'alpha', 'beta', 'release', maybe others.
+
+    @property
+    def is_release(self) -> bool:
+        return self.cycle == "release"
+
+    def __str__(self) -> str:
+        """Convert to version string.
+
+        >>> str(BlenderVersion(293, 1, "alpha"))
+        '2.93.1-alpha'
+        >>> str(BlenderVersion(327, 0, "release"))
+        '3.27.0'
+        """
+
+        as_string = f"{self.version/100:.2f}.{self.patch}"
+        if self.is_release:
+            return as_string
+        return f"{as_string}-{self.cycle}"
+
+
+def parse_blender_version(blender_srcdir: Path) -> BlenderVersion:
+    version_path = blender_srcdir / "source/blender/blenkernel/BKE_blender_version.h"
+
+    version_info = {}
+    line_re = re.compile(r"^#define (BLENDER_VERSION[A-Z_]*)\s+([0-9a-z]+)$")
+
+    with version_path.open(encoding="utf-8") as version_file:
+        for line in version_file:
+            match = line_re.match(line.strip())
+            if not match:
+                continue
+            version_info[match.group(1)] = match.group(2)
+
+    return BlenderVersion(
+        int(version_info["BLENDER_VERSION"]),
+        int(version_info["BLENDER_VERSION_PATCH"]),
+        version_info["BLENDER_VERSION_CYCLE"],
+    )
+
+
+### Manifest creation
+
+
+def create_manifest(version: BlenderVersion, outpath: Path) -> None:
+    print(f'Building manifest of files:  "{outpath}"...', end="", flush=True)
+    with outpath.open("w", encoding="utf-8") as outfile:
+        main_files_to_manifest(outfile)
+        submodules_to_manifest(version, outfile)
+    print("OK")
+
+
+def main_files_to_manifest(outfile: TextIO) -> None:
+    for path in git_ls_files():
+        print(path, file=outfile)
+
+
+def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None:
+    skip_addon_contrib = version.is_release
+
+    for line in git_command("submodule"):
+        submodule = line.split()[1]
+
+        if skip_addon_contrib and submodule == "release/scripts/addons_contrib":
+            continue
+
+        for path in git_ls_files(Path(submodule)):
+            print(path, file=outfile)
+
+
+def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None:
+    print(f'Creating archive:            "{tarball}" ...', end="", flush=True)
+    command = [
+        "tar",
+        "--transform",
+        f"s,^,blender-{version}/,g",
+        "--use-compress-program=xz -9",
+        "--create",
+        f"--file={tarball}",
+        f"--files-from={manifest}",
+        # Without owner/group args, extracting the files as root will
+        # use ownership from the tar archive:
+        "--owner=0",
+        "--group=0",
+    ]
+    subprocess.run(command, check=True, timeout=300)
+    print("OK")
+
+
+def create_checksum_file(tarball: Path) -> None:
+    md5_path = tarball.with_name(tarball.name + ".md5sum")
+    print(f'Creating checksum:           "{md5_path}" ...', end="", flush=True)
+    command = [
+        "md5sum",
+        # The name is enough, as the tarball resides in the same dir as the MD5
+        # file, and that's the current working directory.
+        tarball.name,
+    ]
+    md5_cmd = subprocess.run(
+        command, stdout=subprocess.PIPE, check=True, text=True, timeout=300
+    )
+    with md5_path.open("w") as outfile:
+        outfile.write(md5_cmd.stdout)
+    print("OK")
+
+
+def cleanup(manifest: Path) -> None:
+    print("Cleaning up ...", end="", flush=True)
+    if manifest.exists():
+        manifest.unlink()
+    print("OK")
+
+
+## Low-level commands
+
+
+def git_ls_files(directory: Path = Path(".")) -> Iterable[Path]:
+    """Generator, yields lines of output from 'git ls-files'.
+
+    Only lines that are actually files (so no directories, sockets, etc.) are
+    returned, and never one from SKIP_NAMES.
+    """
+    for line in git_command("-C", str(directory), "ls-files"):
+        path = directory / line
+        if not path.is_file() or path.name in SKIP_NAMES:
+            continue
+        yield path
+
+
+def git_command(*cli_args) -> Iterable[str]:
+    """Generator, yields lines of output from a Git command."""
+    command = ("git", *cli_args)
+
+    # import shlex
+    # print(">", " ".join(shlex.quote(arg) for arg in command))
+
+    git = subprocess.run(
+        command, stdout=subprocess.PIPE, check=True, text=True, timeout=30
+    )
+    for line in git.stdout.split("\n"):
+        if line:
+            yield line
+
+
+if __name__ == "__main__":
+    import doctest
+
+    if doctest.testmod().failed:
+        raise SystemExit("ERROR: Self-test failed, refusing to run")
+
+    main()
diff --git a/build_files/utils/make_source_archive.sh b/build_files/utils/make_source_archive.sh
deleted file mode 100755
index 5d9096f3235..00000000000
--- a/build_files/utils/make_source_archive.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/sh
-
-# This script can run from any location,
-# output is created in the $CWD
-
-BASE_DIR="$PWD"
-
-blender_srcdir=$(dirname -- $0)/../..
-blender_version=$(grep "BLENDER_VERSION\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
-blender_version_patch=$(grep "BLENDER_VERSION_PATCH\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
-blender_version_cycle=$(grep "BLENDER_VERSION_CYCLE\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
-
-VERSION=$(expr $blender_version / 100).$(expr $blender_version % 100).$blender_version_patch
-if [ "$blender_version_cycle" = "release" ] ; then
-  SUBMODULE_EXCLUDE="^\(release/scripts/addons_contrib\)$"
-else
-  VERSION=$VERSION-$blender_version_cycle
-  SUBMODULE_EXCLUDE="^$"  # dummy regex
-fi
-
-MANIFEST="blender-$VERSION-manifest.txt"
-TARBALL="blender-$VERSION.tar.xz"
-
-cd "$blender_srcdir"
-
-# not so nice, but works
-FILTER_FILES_PY=\
-"import os, sys; "\
-"[print(l[:-1]) for l in sys.stdin.readlines() "\
-"if os.path.isfile(l[:-1]) "\
-"if os.path.basename(l[:-1]) not in {"\
-"'.gitignore', "\
-"'.gitmodules', "\
-"'.arcconfig', "\
-"}"\
-"]"
-
-# Build master list
-echo -n "Building manifest of files:  \"$BASE_DIR/$MANIFEST\" ..."
-git ls-files | python3 -c "$FILTER_FILES_PY" > $BASE_DIR/$MANIFEST
-
-# Enumerate submodules
-for lcv in $(git submodule | awk '{print $2}' | grep -v "$SUBMODULE_EXCLUDE"); do
-  cd "$BASE_DIR"
-  cd "$blender_srcdir/$lcv"
-  git ls-files | python3 -c "$FILTER_FILES_PY" | awk '$0="'"$lcv"/'"$0' >> $BASE_DIR/$MANIFEST
-  cd "$BASE_DIR"
-done
-echo "OK"
-
-
-# Create the tarball
-#
-# Without owner/group args, extracting the files as root will
-# use ownership from the tar archive.
-cd "$blender_srcdir"
-echo -n "Creating archive:            \"$BASE_DIR/$TARBALL\" ..."
-tar \
-  --transform "s,^,blender-$VERSION/,g" \
-  --use-compress-program="xz -9" \
-  --create \
-  --file="$BASE_DIR/$TARBALL" \
-  --files-from="$BASE_DIR/$MANIFEST" \
-  --owner=0 \
-  --group=0
-
-echo "OK"
-
-
-# Create checksum file
-cd "$BASE_DIR"
-echo -n "Creating checksum:           \"$BASE_DIR/$TARBALL.md5sum\" ..."
-md5sum "$TARBALL" > "$TARBALL.md5sum"
-echo "OK"
-
-
-# Cleanup
-echo -n "Cleaning up ..."
-rm "$BASE_DIR/$MANIFEST"
-echo "OK"
-
-echo "Done!"