From e218d8c24bfbd5384e040417e533ede96c4e831b Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Fri, 23 Aug 2019 19:35:02 +0200 Subject: Build: integrate make_update.py into Windows make.bat --- build_files/utils/make_update.py | 152 +++++++++++++++++++----------- build_files/windows/check_libraries.cmd | 9 ++ build_files/windows/find_dependencies.cmd | 8 +- build_files/windows/format.cmd | 5 +- build_files/windows/parse_arguments.cmd | 8 +- build_files/windows/update_sources.cmd | 25 ++--- 6 files changed, 125 insertions(+), 82 deletions(-) (limited to 'build_files') diff --git a/build_files/utils/make_update.py b/build_files/utils/make_update.py index b4125e0aeec..5d82075d562 100755 --- a/build_files/utils/make_update.py +++ b/build_files/utils/make_update.py @@ -1,12 +1,49 @@ #!/usr/bin/env python3 - +# +# "make update" for all platforms, updating svn libraries and tests and Blender +# git repository and submodules. +# +# For release branches, this will check out the appropriate branches of +# submodules and libraries. + +import argparse import os import re +import shutil import subprocess import sys +# Parse arguments + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--only-code", action="store_true") + parser.add_argument("--svn-command", default="svn") + parser.add_argument("--git-command", default="git") + return parser.parse_args() + +args = parse_arguments() +only_code = args.only_code +git_command = args.git_command +svn_command = args.svn_command + +if shutil.which(git_command) is None: + sys.stderr.write("git not found, can't update code\n") + sys.exit(1) + +if shutil.which(svn_command) is None: + sys.stderr.write("svn not found, can't update libraries\n") + sys.exit(1) + +# Utility functions + def call(cmd): print(" ".join(cmd)) + + # Flush to ensure correct order output on Windows. + sys.stdout.flush() + sys.stderr.flush() + retcode = subprocess.call(cmd) if retcode != 0: sys.exit(retcode) @@ -16,11 +53,11 @@ def print_stage(text): print(text) print("") -# Test if we are building a specific release version +# Test if we are building a specific release version. try: - branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + branch = subprocess.check_output([git_command, "rev-parse", "--abbrev-ref", "HEAD"]) except subprocess.CalledProcessError as e: - sys.stderr("Failed to get Blender git branch\n") + sys.stderr.write("Failed to get Blender git branch\n") sys.exit(1) branch = branch.strip().decode('utf8') @@ -29,63 +66,64 @@ if release_version: release_version = release_version.group(1) print("Using Release Blender v" + release_version) -# Setup for precompiled libraries and tests from svn -lib_dirpath = os.path.join('..', 'lib') - -if release_version: - svn_branch = "tags/blender-" + release_version + "-release" -else: - svn_branch = "trunk" -svn_url = "https://svn.blender.org/svnroot/bf-blender/" + svn_branch + "/lib/" - -# Checkout precompiled libraries -if sys.platform == 'darwin': - lib_platform = "darwin" -elif sys.platform == 'win32': - # Windows checkout is usually handled by bat scripts since python3 to run - # this script is bundled as part of the precompiled libraries. However it - # is used by the buildbot. - lib_platform = "win64_vc14" -else: - # No precompiled libraries for Linux. - lib_platform = None - -if lib_platform: - lib_platform_dirpath = os.path.join(lib_dirpath, lib_platform) - - if not os.path.exists(lib_platform_dirpath): - print_stage("Checking out Precompiled Libraries") - - svn_url_platform = svn_url + lib_platform - call(["svn", "checkout", svn_url_platform, lib_platform_dirpath]) - -# Update precompiled libraries and tests -print_stage("Updating Precompiled Libraries and Tests") - -if os.path.isdir(lib_dirpath): - for dirname in os.listdir(lib_dirpath): - if dirname == ".svn": - continue - - dirpath = os.path.join(lib_dirpath, dirname) - svn_dirpath = os.path.join(dirpath, ".svn") - svn_root_dirpath = os.path.join(lib_dirpath, ".svn") - - if os.path.isdir(dirpath) and \ - (os.path.exists(svn_dirpath) or os.path.exists(svn_root_dirpath)): - call(["svn", "cleanup", dirpath]) - call(["svn", "switch", svn_url + dirname, dirpath]) - call(["svn", "update", dirpath]) - -# Update blender repository and submodules +# Setup for precompiled libraries and tests from svn. +if not only_code: + lib_dirpath = os.path.join('..', 'lib') + + if release_version: + svn_branch = "tags/blender-" + release_version + "-release" + else: + svn_branch = "trunk" + svn_url = "https://svn.blender.org/svnroot/bf-blender/" + svn_branch + "/lib/" + + # Checkout precompiled libraries + if sys.platform == 'darwin': + lib_platform = "darwin" + elif sys.platform == 'win32': + # Windows checkout is usually handled by bat scripts since python3 to run + # this script is bundled as part of the precompiled libraries. However it + # is used by the buildbot. + lib_platform = "win64_vc14" + else: + # No precompiled libraries for Linux. + lib_platform = None + + if lib_platform: + lib_platform_dirpath = os.path.join(lib_dirpath, lib_platform) + + if not os.path.exists(lib_platform_dirpath): + print_stage("Checking out Precompiled Libraries") + + svn_url_platform = svn_url + lib_platform + call([svn_command, "checkout", svn_url_platform, lib_platform_dirpath]) + + # Update precompiled libraries and tests + print_stage("Updating Precompiled Libraries and Tests") + + if os.path.isdir(lib_dirpath): + for dirname in os.listdir(lib_dirpath): + if dirname == ".svn": + continue + + dirpath = os.path.join(lib_dirpath, dirname) + svn_dirpath = os.path.join(dirpath, ".svn") + svn_root_dirpath = os.path.join(lib_dirpath, ".svn") + + if os.path.isdir(dirpath) and \ + (os.path.exists(svn_dirpath) or os.path.exists(svn_root_dirpath)): + call([svn_command, "cleanup", dirpath]) + call([svn_command, "switch", svn_url + dirname, dirpath]) + call([svn_command, "update", dirpath]) + +# Update blender repository and submodules. print_stage("Updating Blender Git Repository and Submodules") -call(["git", "pull", "--rebase"]) -call(["git", "submodule", "update", "--init", "--recursive"]) +call([git_command, "pull", "--rebase"]) +call([git_command, "submodule", "update", "--init", "--recursive"]) if not release_version: # Update submodules to latest master if not building a specific release. # In that case submodules are set to a specific revision, which is checked # out by running "git submodule update". - call(["git", "submodule", "foreach", "git", "checkout", "master"]) - call(["git", "submodule", "foreach", "git", "pull", "--rebase", "origin", "master"]) + call([git_command, "submodule", "foreach", "git", "checkout", "master"]) + call([git_command, "submodule", "foreach", "git", "pull", "--rebase", "origin", "master"]) diff --git a/build_files/windows/check_libraries.cmd b/build_files/windows/check_libraries.cmd index fcae2c90657..b838c7d7d19 100644 --- a/build_files/windows/check_libraries.cmd +++ b/build_files/windows/check_libraries.cmd @@ -39,6 +39,15 @@ if NOT EXIST %BUILD_VS_LIBDIR% ( ) ) ) +) else ( + if NOT EXIST %PYTHON% ( + if not "%SVN%"=="" ( + echo. + echo Python not found in external libraries, updating to latest version + echo. + "%SVN%" update %BUILD_VS_LIBDIR% + ) + ) ) if NOT EXIST %BUILD_VS_LIBDIR% ( diff --git a/build_files/windows/find_dependencies.cmd b/build_files/windows/find_dependencies.cmd index a861cf1c98b..6a2233ecff7 100644 --- a/build_files/windows/find_dependencies.cmd +++ b/build_files/windows/find_dependencies.cmd @@ -2,10 +2,12 @@ REM find all dependencies and set the corresponding environment variables. for %%X in (svn.exe) do (set SVN=%%~$PATH:X) for %%X in (cmake.exe) do (set CMAKE=%%~$PATH:X) for %%X in (git.exe) do (set GIT=%%~$PATH:X) +set PYTHON=%BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe if NOT "%verbose%" == "" ( - echo svn : "%SVN%" - echo cmake : "%CMAKE%" - echo git : "%GIT%" + echo svn : "%SVN%" + echo cmake : "%CMAKE%" + echo git : "%GIT%" + echo python : "%PYTHON%" ) if "%CMAKE%" == "" ( echo Cmake not found in path, required for building, exiting... diff --git a/build_files/windows/format.cmd b/build_files/windows/format.cmd index f036257e220..43f8b61727d 100644 --- a/build_files/windows/format.cmd +++ b/build_files/windows/format.cmd @@ -9,7 +9,7 @@ exit /b 1 :detect_done echo found clang-format in %CF_PATH% -if EXIST %BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe ( +if EXIST %PYTHON% ( set PYTHON=%BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe goto detect_python_done ) @@ -25,6 +25,7 @@ set FORMAT_PATHS=%BLENDER_DIR%\source\tools\utils_maintenance\clang_format_paths REM The formatting script expects clang-format to be in the current PATH. set PATH=%CF_PATH%;%PATH% -%PYTHON% %FORMAT_PATHS% %FORMAT_ARGS% +REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts. +%PYTHON% -B %FORMAT_PATHS% %FORMAT_ARGS% :EOF diff --git a/build_files/windows/parse_arguments.cmd b/build_files/windows/parse_arguments.cmd index 3f40ef1f5ef..cb375b7fc9f 100644 --- a/build_files/windows/parse_arguments.cmd +++ b/build_files/windows/parse_arguments.cmd @@ -82,12 +82,10 @@ if NOT "%1" == "" ( REM Non-Build Commands ) else if "%1" == "update" ( SET BUILD_UPDATE=1 - set BUILD_UPDATE_SVN=1 - set BUILD_UPDATE_GIT=1 + set BUILD_UPDATE_ARGS= ) else if "%1" == "code_update" ( SET BUILD_UPDATE=1 - set BUILD_UPDATE_SVN=0 - set BUILD_UPDATE_GIT=1 + set BUILD_UPDATE_ARGS="--only-code" ) else if "%1" == "ninja" ( SET BUILD_WITH_NINJA=1 ) else if "%1" == "clean" ( @@ -108,4 +106,4 @@ if NOT "%1" == "" ( :EOF exit /b 0 :ERR -exit /b 1 \ No newline at end of file +exit /b 1 diff --git a/build_files/windows/update_sources.cmd b/build_files/windows/update_sources.cmd index 1f571eaf92d..f8fbd383090 100644 --- a/build_files/windows/update_sources.cmd +++ b/build_files/windows/update_sources.cmd @@ -1,18 +1,13 @@ -if "%BUILD_UPDATE_SVN%" == "1" ( - if "%SVN%" == "" ( - echo svn not found, cannot update libraries - goto UPDATE_GIT - ) - "%SVN%" up "%BLENDER_DIR%/../lib/*" +if EXIST %PYTHON% ( + goto detect_python_done ) -:UPDATE_GIT -if "%BUILD_UPDATE_GIT%" == "1" ( - if "%GIT%" == "" ( - echo Git not found, cannot update code - goto EOF - ) - "%GIT%" pull --rebase - "%GIT%" submodule foreach git pull --rebase origin master -) +echo python not found in lib folder +exit /b 1 + +:detect_python_done + +REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts. +%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" --svn-command "%SVN%" %BUILD_UPDATE_ARGS% + :EOF -- cgit v1.2.3