From 9cbfcc4af500186ce8b30a915bef263a2cd1e1d5 Mon Sep 17 00:00:00 2001 From: Jeroen Bakker Date: Wed, 16 Dec 2020 11:11:15 +0100 Subject: Revert "Steam Release: Script creation of Steam build files" This reverts commit 1a375d6eceed4c17f92fd90699645856e0951530. --- release/steam/README.md | 70 ----- release/steam/blender_app_build.vdf.template | 17 -- release/steam/create_steam_builds.py | 397 --------------------------- release/steam/depot_build_linux.vdf.template | 31 --- release/steam/depot_build_macos.vdf.template | 30 -- release/steam/depot_build_win.vdf.template | 31 --- 6 files changed, 576 deletions(-) delete mode 100644 release/steam/README.md delete mode 100644 release/steam/blender_app_build.vdf.template delete mode 100644 release/steam/create_steam_builds.py delete mode 100644 release/steam/depot_build_linux.vdf.template delete mode 100644 release/steam/depot_build_macos.vdf.template delete mode 100644 release/steam/depot_build_win.vdf.template (limited to 'release/steam') diff --git a/release/steam/README.md b/release/steam/README.md deleted file mode 100644 index 05eda799c3f..00000000000 --- a/release/steam/README.md +++ /dev/null @@ -1,70 +0,0 @@ -Creating Steam builds for Blender -================================= - -This script automates creation of the Steam files: download of the archives, -extraction of the archives, preparation of the build scripts (VDF files), actual -building of the Steam game files. - -Requirements -============ - -* MacOS machine - Tested on Catalina 10.15.6. Extracting contents from the DMG - archive did not work Windows nor on Linux using 7-zip. All DMG archives tested - failed to be extracted. As such only MacOS is known to work. -* Steam SDK downloaded from SteamWorks - The `steamcmd` is used to generate the - Steam game files. The path to the `steamcmd` is what is actually needed. -* SteamWorks credentials - Needed to log in using `steamcmd`. -* Login to SteamWorks with the `steamcmd` from the command-line at least once - - Needded to ensure the user is properly logged in. On a new machine the user - will have to go through two-factor authentication. -* App ID and Depot IDs - Needed to create the VDF files. -* Python 3.x - 3.7 was tested. -* Base URL - for downloading the archives. - -Usage -===== - -```bash -$ export STEAMUSER=SteamUserName -$ export STEAMPW=SteamUserPW -$ export BASEURL=https://download.blender.org/release/Blender2.83/ -$ export VERSION=2.83.3 -$ export APPID=appidnr -$ export WINID=winidnr -$ export LINID=linuxidnr -$ export MACOSID=macosidnr - -# log in to SteamWorks from command-line at least once - -$ ../sdk/tools/ContentBuilder/builder_osx/steamcmd +login $STEAMUSER $STEAMPW - -# once that has been done we can now actually start our tool - -$ python3.7 create_steam_builds.py --baseurl $BASEURL --version $VERSION --appid $APPID --winid $WINID --linuxid $LINID --macosid $MACOSID --steamuser $STEAMUSER --steampw $STEAMPW --steamcmd ../sdk/tools/ContentBuilder/builder_osx/steamcmd -``` - -All arguments in the above example are required. - -At the start the tool will login using `steamcmd`. This is necessary to let the -Steam SDK update itself if necessary. - -There are a few optional arguments: - -* `--dryrun`: If set building the game files will not actually happen. A set of - log files and a preview manifest per depot will be created in the output folder. - This can be used to double-check everything works as expected. -* `--skipdl`: If set will skip downloading of the archives. The tool expects the - archives to already exist in the correct content location. -* `--skipextract`: If set will skip extraction of all archives. The tool expects - the archives to already have been correctly extracted in the content location. - -Run the tool with `-h` for detailed information on each argument. - -The content and output folders are generated through appending the version -without dots to the words `content` and `output` respectively, e.g. `content2833` -and `output2833`. These folders are created next to the tool. - -From all `.template` files the Steam build scripts will be generated also in the -same directory as the tool. The files will have the extension `.vdf`. - -In case of errors the tool will have a non-zero return code. \ No newline at end of file diff --git a/release/steam/blender_app_build.vdf.template b/release/steam/blender_app_build.vdf.template deleted file mode 100644 index 9e2d0625d72..00000000000 --- a/release/steam/blender_app_build.vdf.template +++ /dev/null @@ -1,17 +0,0 @@ -"appbuild" -{ - "appid" "[APPID]" - "desc" "Blender [VERSION]" // description for this build - "buildoutput" "./[OUTPUT]" // build output folder for .log, .csm & .csd files, relative to location of this file - "contentroot" "./[CONTENT]" // root content folder, relative to location of this file - "setlive" "" // branch to set live after successful build, non if empty - "preview" "[DRYRUN]" // 1 to enable preview builds, 0 to commit build to steampipe - "local" "" // set to flie path of local content server - - "depots" - { - "[WINID]" "depot_build_win.vdf" - "[LINUXID]" "depot_build_linux.vdf" - "[MACOSID]" "depot_build_macos.vdf" - } -} diff --git a/release/steam/create_steam_builds.py b/release/steam/create_steam_builds.py deleted file mode 100644 index 2ecd0c347f7..00000000000 --- a/release/steam/create_steam_builds.py +++ /dev/null @@ -1,397 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib -import requests -import shutil -import subprocess -from typing import Callable, Iterator, List, Tuple - -# supported archive and platform endings, used to create actual archive names -archive_endings = ["windows64.zip", "linux64.tar.xz", "macOS.dmg"] - - -def add_optional_argument(option: str, help: str) -> None: - global parser - """Add an optional argument - - Args: - option (str): Option to add - help (str): Help description for the argument - """ - parser.add_argument(option, help=help, action='store_const', const=1) - - -def blender_archives(version: str) -> Iterator[str]: - """Generator for Blender archives for version. - - Yields for items in archive_endings an archive name in the form of - blender-{version}-{ending}. - - Args: - version (str): Version string of the form 2.83.2 - - - Yields: - Iterator[str]: Name in the form of blender-{version}-{ending} - """ - global archive_endings - - for ending in archive_endings: - yield f"blender-{version}-{ending}" - - -def get_archive_type(archive_type: str, version: str) -> str: - """Return the archive of given type and version. - - Args: - archive_type (str): extension for archive type to check for - version (str): Version string in the form 2.83.2 - - Raises: - Exception: Execption when archive type isn't found - - Returns: - str: archive name for given type - """ - - for archive in blender_archives(version): - if archive.endswith(archive_type): - return archive - raise Exception("Unknown archive type") - - -def execute_command(cmd: List[str], name: str, errcode: int, cwd=".", capture_output=True) -> str: - """Execute the given command. - - Returns the process stdout upon success if any. - - On error print message the command with name that has failed. Print stdout - and stderr of the process if any, and then exit with given error code. - - Args: - cmd (List[str]): Command in list format, each argument as their own item - name (str): Name of command to use when printing to command-line - errcode (int): Error code to use in case of exit() - cwd (str, optional): Folder to use as current work directory for command - execution. Defaults to ".". - capture_output (bool, optional): Whether to capture command output or not. - Defaults to True. - - Returns: - str: stdout if any, or empty string - """ - cmd_process = subprocess.run( - cmd, capture_output=capture_output, encoding="UTF-8", cwd=cwd) - if cmd_process.returncode == 0: - if cmd_process.stdout: - return cmd_process.stdout - else: - return "" - else: - print(f"ERROR: {name} failed.") - if cmd_process.stdout: - print(cmd_process.stdout) - if cmd_process.stderr: - print(cmd_process.stderr) - exit(errcode) - return "" - - -def download_archives(base_url: str, archives: Callable[[str], Iterator[str]], version: str, dst_dir: pathlib.Path): - """Download archives from the given base_url. - - Archives is a generator for Blender archive names based on version. - - Archive names are appended to the base_url to load from, and appended to - dst_dir to save to. - - Args: - base_url (str): Base URL to load archives from - archives (Callable[[str], Iterator[str]]): Generator for Blender archive - names based on version - version (str): Version string in the form of 2.83.2 - dst_dir (pathlib.Path): Download destination - """ - - if base_url[-1] != '/': - base_url = base_url + '/' - - for archive in archives(version): - download_url = f"{base_url}{archive}" - target_file = dst_dir.joinpath(archive) - download_file(download_url, target_file) - - -def download_file(from_url: str, to_file: pathlib.Path) -> None: - """Download from_url as to_file. - - Actual downloading will be skipped if --skipdl is given on the command-line. - - Args: - from_url (str): Full URL to resource to download - to_file (pathlib.Path): Full path to save downloaded resource as - """ - global args - - if not args.skipdl or not to_file.exists(): - print(f"Downloading {from_url}") - with open(to_file, "wb") as download_zip: - response = requests.get(from_url) - if response.status_code != requests.codes.ok: - print(f"ERROR: failed to download {from_url} (status code: {response.status_code})") - exit(1313) - download_zip.write(response.content) - else: - print(f"Downloading {from_url} skipped") - print(" ... OK") - - -def copy_contents_from_dmg_to_path(dmg_file: pathlib.Path, dst: pathlib.Path) -> None: - """Copy the contents of the given DMG file to the destination folder. - - Args: - dmg_file (pathlib.Path): Full path to DMG archive to extract from - dst (pathlib.Path): Full path to destination to extract to - """ - hdiutil_attach = ["hdiutil", - "attach", - "-readonly", - f"{dmg_file}" - ] - attached = execute_command(hdiutil_attach, "hdiutil attach", 1) - - # Last line of output is what we want, it is of the form - # /dev/somedisk Apple_HFS /Volumes/Blender - # We want to retain the mount point, and the folder the mount is - # created on. The mounted disk we need for detaching, the folder we - # need to be able to copy the contents to where we can use them - attachment_items = attached.splitlines()[-1].split() - mounted_disk = attachment_items[0] - source_location = pathlib.Path(attachment_items[2], "Blender.app") - - print(f"{source_location} -> {dst}") - - shutil.copytree(source_location, dst) - - hdiutil_detach = ["hdiutil", - "detach", - f"{mounted_disk}" - ] - execute_command(hdiutil_detach, "hdiutil detach", 2) - - -def create_build_script(template_name: str, vars: List[Tuple[str, str]]) -> pathlib.Path: - """ - Create the Steam build script - - Use the given template and template variable tuple list. - - Returns pathlib.Path to the created script. - - Args: - template_name (str): [description] - vars (List[Tuple[str, str]]): [description] - - Returns: - pathlib.Path: Full path to the generated script - """ - build_script = pathlib.Path(".", template_name).read_text() - for var in vars: - build_script = build_script.replace(var[0], var[1]) - build_script_file = template_name.replace(".template", "") - build_script_path = pathlib.Path(".", build_script_file) - build_script_path.write_text(build_script) - return build_script_path - - -def clean_up() -> None: - """Remove intermediate files depending on given command-line arguments - """ - global content_location, args - - if not args.leavearch and not args.leaveextracted: - shutil.rmtree(content_location) - - if args.leavearch and not args.leaveextracted: - shutil.rmtree(content_location.joinpath(zip_extract_folder)) - shutil.rmtree(content_location.joinpath(tarxz_extract_folder)) - shutil.rmtree(content_location.joinpath(dmg_extract_folder)) - - if args.leaveextracted and not args.leavearch: - import os - os.remove(content_location.joinpath(zipped_blender)) - os.remove(content_location.joinpath(tarxz_blender)) - os.remove(content_location.joinpath(dmg_blender)) - - -def extract_archive(archive: str, extract_folder_name: str, - cmd: List[str], errcode: int) -> None: - """Extract all files from archive to given folder name. - - Will not extract if - target folder already exists, or if --skipextract was given on the - command-line. - - Args: - archive (str): Archive name to extract - extract_folder_name (str): Folder name to extract to - cmd (List[str]): Command with arguments to use - errcode (int): Error code to use for exit() - """ - global args, content_location - - extract_location = content_location.joinpath(extract_folder_name) - - pre_extract = set(content_location.glob("*")) - - if not args.skipextract or not extract_location.exists(): - print(f"Extracting files from {archive}...") - cmd.append(content_location.joinpath(archive)) - execute_command(cmd, cmd[0], errcode, cwd=content_location) - # in case we use a non-release archive the naming will be incorrect. - # simply rename to expected target name - post_extract = set(content_location.glob("*")) - diff_extract = post_extract - pre_extract - if not extract_location in diff_extract: - folder_to_rename = list(diff_extract)[0] - folder_to_rename.rename(extract_location) - print(" OK") - else: - print(f"Skipping extraction {archive}!") - -# ============================================================================== - - -parser = argparse.ArgumentParser() - -parser.add_argument("--baseurl", required=True, - help="The base URL for files to download, " - "i.e. https://download.blender.org/release/Blender2.83/") - -parser.add_argument("--version", required=True, - help="The Blender version to release, in the form 2.83.3") - -parser.add_argument("--appid", required=True, - help="The Blender App ID on Steam") -parser.add_argument("--winid", required=True, - help="The Windows depot ID") -parser.add_argument("--linuxid", required=True, - help="The Linux depot ID") -parser.add_argument("--macosid", required=True, - help="The MacOS depot ID") - -parser.add_argument("--steamcmd", required=True, - help="Path to the steamcmd") -parser.add_argument("--steamuser", required=True, - help="The login for the Steam builder user") -parser.add_argument("--steampw", required=True, - help="Login password for the Steam builder user") - -add_optional_argument("--dryrun", - "If set the Steam files will not be uploaded") -add_optional_argument("--leavearch", - help="If set don't clean up the downloaded archives") -add_optional_argument("--leaveextracted", - help="If set don't clean up the extraction folders") -add_optional_argument("--skipdl", - help="If set downloading the archives is skipped if it already exists locally.") -add_optional_argument("--skipextract", - help="If set skips extracting of archives. The tool assumes the archives" - "have already been extracted to their correct locations") - -args = parser.parse_args() - -VERSIONNODOTS = args.version.replace('.', '') -OUTPUT = f"output{VERSIONNODOTS}" -CONTENT = f"content{VERSIONNODOTS}" - -# ===== set up main locations - -content_location = pathlib.Path(".", CONTENT).absolute() -output_location = pathlib.Path(".", OUTPUT).absolute() - -content_location.mkdir(parents=True, exist_ok=True) -output_location.mkdir(parents=True, exist_ok=True) - -# ===== login - -# Logging into Steam once to ensure the SDK updates itself properly. If we don't -# do that the combined +login and +run_app_build_http at the end of the tool -# will fail. -steam_login = [args.steamcmd, - "+login", - args.steamuser, - args.steampw, - "+quit" - ] -print("Logging in to Steam...") -execute_command(steam_login, "Login to Steam", 10) -print(" OK") - -# ===== prepare Steam build scripts - -template_vars = [ - ("[APPID]", args.appid), - ("[OUTPUT]", OUTPUT), - ("[CONTENT]", CONTENT), - ("[VERSION]", args.version), - ("[WINID]", args.winid), - ("[LINUXID]", args.linuxid), - ("[MACOSID]", args.macosid), - ("[DRYRUN]", f"{args.dryrun}" if args.dryrun else "0") -] - -blender_app_build = create_build_script( - "blender_app_build.vdf.template", template_vars) -create_build_script("depot_build_win.vdf.template", template_vars) -create_build_script("depot_build_linux.vdf.template", template_vars) -create_build_script("depot_build_macos.vdf.template", template_vars) - -# ===== download archives - -download_archives(args.baseurl, blender_archives, - args.version, content_location) - -# ===== set up file and folder names - -zipped_blender = get_archive_type("zip", args.version) -zip_extract_folder = zipped_blender.replace(".zip", "") -tarxz_blender = get_archive_type("tar.xz", args.version) -tarxz_extract_folder = tarxz_blender.replace(".tar.xz", "") -dmg_blender = get_archive_type("dmg", args.version) -dmg_extract_folder = dmg_blender.replace(".dmg", "") - -# ===== extract - -unzip_cmd = ["unzip", "-q"] -extract_archive(zipped_blender, zip_extract_folder, unzip_cmd, 3) - -untarxz_cmd = ["tar", "-xf"] -extract_archive(tarxz_blender, tarxz_extract_folder, untarxz_cmd, 4) - -if not args.skipextract or not content_location.joinpath(dmg_extract_folder).exists(): - print("Extracting files from Blender MacOS archive...") - blender_dmg = content_location.joinpath(dmg_blender) - target_location = content_location.joinpath( - dmg_extract_folder, "Blender.app") - copy_contents_from_dmg_to_path(blender_dmg, target_location) - print(" OK") -else: - print("Skipping extraction of .dmg!") - -# ===== building - -print("Build Steam game files...") -steam_build = [args.steamcmd, - "+login", - args.steamuser, - args.steampw, - "+run_app_build_http", - blender_app_build.absolute(), - "+quit" - ] -execute_command(steam_build, "Build with steamcmd", 13) -print(" OK") - -clean_up() diff --git a/release/steam/depot_build_linux.vdf.template b/release/steam/depot_build_linux.vdf.template deleted file mode 100644 index 0f69008548e..00000000000 --- a/release/steam/depot_build_linux.vdf.template +++ /dev/null @@ -1,31 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[LINUXID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-linux64/" - - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} diff --git a/release/steam/depot_build_macos.vdf.template b/release/steam/depot_build_macos.vdf.template deleted file mode 100644 index 33dde860462..00000000000 --- a/release/steam/depot_build_macos.vdf.template +++ /dev/null @@ -1,30 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[MACOSID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-macOS/" - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} diff --git a/release/steam/depot_build_win.vdf.template b/release/steam/depot_build_win.vdf.template deleted file mode 100644 index 2c18a0f15dd..00000000000 --- a/release/steam/depot_build_win.vdf.template +++ /dev/null @@ -1,31 +0,0 @@ -"DepotBuildConfig" -{ - // Set your assigned depot ID here - "DepotID" "[WINID]" - - // Set a root for all content. - // All relative paths specified below (LocalPath in FileMapping entries, and FileExclusion paths) - // will be resolved relative to this root. - // If you don't define ContentRoot, then it will be assumed to be - // the location of this script file, which probably isn't what you want - "ContentRoot" "./blender-[VERSION]-windows64/" - - // include all files recursivley - "FileMapping" - { - // This can be a full path, or a path relative to ContentRoot - "LocalPath" "*" - - // This is a path relative to the install folder of your game - "DepotPath" "." - - // If LocalPath contains wildcards, setting this means that all - // matching files within subdirectories of LocalPath will also - // be included. - "recursive" "1" - } - - // but exclude all symbol files - // This can be a full path, or a path relative to ContentRoot - "FileExclusion" "*.pdb" -} -- cgit v1.2.3