Build: integrate make_update.py into Windows make.bat

7 files changed, 133 insertions, 88 deletions

diff --git a/build_files/utils/make_update.py b/build_files/utils/make_update.py
index b4125e0aeec..5d82075d562 100755
--- a/build_files/utils/make_update.py
+++ b/build_files/utils/make_update.py
@@ -1,12 +1,49 @@
 #!/usr/bin/env python3
-
+#
+# "make update" for all platforms, updating svn libraries and tests and Blender
+# git repository and submodules.
+#
+# For release branches, this will check out the appropriate branches of
+# submodules and libraries.
+
+import argparse
 import os
 import re
+import shutil
 import subprocess
 import sys
 
+# Parse arguments
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--only-code", action="store_true")
+    parser.add_argument("--svn-command", default="svn")
+    parser.add_argument("--git-command", default="git")
+    return parser.parse_args()
+
+args = parse_arguments()
+only_code = args.only_code
+git_command = args.git_command
+svn_command = args.svn_command
+
+if shutil.which(git_command) is None:
+    sys.stderr.write("git not found, can't update code\n")
+    sys.exit(1)
+
+if shutil.which(svn_command) is None:
+    sys.stderr.write("svn not found, can't update libraries\n")
+    sys.exit(1)
+
+# Utility functions
+
 def call(cmd):
     print(" ".join(cmd))
+
+    # Flush to ensure correct order output on Windows.
+    sys.stdout.flush()
+    sys.stderr.flush()
+
     retcode = subprocess.call(cmd)
     if retcode != 0:
       sys.exit(retcode)
@@ -16,11 +53,11 @@ def print_stage(text):
     print(text)
     print("")
 
-# Test if we are building a specific release version
+# Test if we are building a specific release version.
 try:
-    branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
+    branch = subprocess.check_output([git_command, "rev-parse", "--abbrev-ref", "HEAD"])
 except subprocess.CalledProcessError as e:
-    sys.stderr("Failed to get Blender git branch\n")
+    sys.stderr.write("Failed to get Blender git branch\n")
     sys.exit(1)
 
 branch = branch.strip().decode('utf8')
@@ -29,63 +66,64 @@ if release_version:
     release_version = release_version.group(1)
     print("Using Release Blender v" + release_version)
 
-# Setup for precompiled libraries and tests from svn
-lib_dirpath = os.path.join('..', 'lib')
-
-if release_version:
-    svn_branch = "tags/blender-" + release_version + "-release"
-else:
-    svn_branch = "trunk"
-svn_url = "https://svn.blender.org/svnroot/bf-blender/" + svn_branch + "/lib/"
-
-# Checkout precompiled libraries
-if sys.platform == 'darwin':
-    lib_platform = "darwin"
-elif sys.platform == 'win32':
-    # Windows checkout is usually handled by bat scripts since python3 to run
-    # this script is bundled as part of the precompiled libraries. However it
-    # is used by the buildbot.
-    lib_platform = "win64_vc14"
-else:
-    # No precompiled libraries for Linux.
-    lib_platform = None
-
-if lib_platform:
-    lib_platform_dirpath = os.path.join(lib_dirpath, lib_platform)
-
-    if not os.path.exists(lib_platform_dirpath):
-        print_stage("Checking out Precompiled Libraries")
-
-        svn_url_platform = svn_url + lib_platform
-        call(["svn", "checkout", svn_url_platform, lib_platform_dirpath])
-
-# Update precompiled libraries and tests
-print_stage("Updating Precompiled Libraries and Tests")
-
-if os.path.isdir(lib_dirpath):
-  for dirname in os.listdir(lib_dirpath):
-    if dirname == ".svn":
-        continue
-
-    dirpath = os.path.join(lib_dirpath, dirname)
-    svn_dirpath = os.path.join(dirpath, ".svn")
-    svn_root_dirpath = os.path.join(lib_dirpath, ".svn")
-
-    if os.path.isdir(dirpath) and \
-       (os.path.exists(svn_dirpath) or os.path.exists(svn_root_dirpath)):
-        call(["svn", "cleanup", dirpath])
-        call(["svn", "switch", svn_url + dirname, dirpath])
-        call(["svn", "update", dirpath])
-
-# Update blender repository and submodules
+# Setup for precompiled libraries and tests from svn.
+if not only_code:
+    lib_dirpath = os.path.join('..', 'lib')
+
+    if release_version:
+        svn_branch = "tags/blender-" + release_version + "-release"
+    else:
+        svn_branch = "trunk"
+    svn_url = "https://svn.blender.org/svnroot/bf-blender/" + svn_branch + "/lib/"
+
+    # Checkout precompiled libraries
+    if sys.platform == 'darwin':
+        lib_platform = "darwin"
+    elif sys.platform == 'win32':
+        # Windows checkout is usually handled by bat scripts since python3 to run
+        # this script is bundled as part of the precompiled libraries. However it
+        # is used by the buildbot.
+        lib_platform = "win64_vc14"
+    else:
+        # No precompiled libraries for Linux.
+        lib_platform = None
+
+    if lib_platform:
+        lib_platform_dirpath = os.path.join(lib_dirpath, lib_platform)
+
+        if not os.path.exists(lib_platform_dirpath):
+            print_stage("Checking out Precompiled Libraries")
+
+            svn_url_platform = svn_url + lib_platform
+            call([svn_command, "checkout", svn_url_platform, lib_platform_dirpath])
+
+    # Update precompiled libraries and tests
+    print_stage("Updating Precompiled Libraries and Tests")
+
+    if os.path.isdir(lib_dirpath):
+      for dirname in os.listdir(lib_dirpath):
+        if dirname == ".svn":
+            continue
+
+        dirpath = os.path.join(lib_dirpath, dirname)
+        svn_dirpath = os.path.join(dirpath, ".svn")
+        svn_root_dirpath = os.path.join(lib_dirpath, ".svn")
+
+        if os.path.isdir(dirpath) and \
+           (os.path.exists(svn_dirpath) or os.path.exists(svn_root_dirpath)):
+            call([svn_command, "cleanup", dirpath])
+            call([svn_command, "switch", svn_url + dirname, dirpath])
+            call([svn_command, "update", dirpath])
+
+# Update blender repository and submodules.
 print_stage("Updating Blender Git Repository and Submodules")
 
-call(["git", "pull", "--rebase"])
-call(["git", "submodule", "update", "--init", "--recursive"])
+call([git_command, "pull", "--rebase"])
+call([git_command, "submodule", "update", "--init", "--recursive"])
 
 if not release_version:
     # Update submodules to latest master if not building a specific release.
     # In that case submodules are set to a specific revision, which is checked
     # out by running "git submodule update".
-    call(["git", "submodule", "foreach", "git", "checkout", "master"])
-    call(["git", "submodule", "foreach", "git", "pull", "--rebase", "origin", "master"])
+    call([git_command, "submodule", "foreach", "git", "checkout", "master"])
+    call([git_command, "submodule", "foreach", "git", "pull", "--rebase", "origin", "master"])
diff --git a/build_files/windows/check_libraries.cmd b/build_files/windows/check_libraries.cmd
index fcae2c90657..b838c7d7d19 100644
--- a/build_files/windows/check_libraries.cmd
+++ b/build_files/windows/check_libraries.cmd
@@ -39,6 +39,15 @@ if NOT EXIST %BUILD_VS_LIBDIR% (
 			)
 		)
 	)
+) else (
+	if NOT EXIST %PYTHON% (
+		if not "%SVN%"=="" (
+			echo.
+			echo Python not found in external libraries, updating to latest version
+			echo.
+			"%SVN%" update %BUILD_VS_LIBDIR%
+		)
+	)
 )
 
 if NOT EXIST %BUILD_VS_LIBDIR% (
diff --git a/build_files/windows/find_dependencies.cmd b/build_files/windows/find_dependencies.cmd
index a861cf1c98b..6a2233ecff7 100644
--- a/build_files/windows/find_dependencies.cmd
+++ b/build_files/windows/find_dependencies.cmd
@@ -2,10 +2,12 @@ REM find all dependencies and set the corresponding environment variables.
 for %%X in (svn.exe) do (set SVN=%%~$PATH:X)
 for %%X in (cmake.exe) do (set CMAKE=%%~$PATH:X)
 for %%X in (git.exe) do (set GIT=%%~$PATH:X)
+set PYTHON=%BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe
 if NOT "%verbose%" == "" (
-	echo svn   : "%SVN%"
-	echo cmake : "%CMAKE%"
-	echo git   : "%GIT%"
+	echo svn    : "%SVN%"
+	echo cmake  : "%CMAKE%"
+	echo git    : "%GIT%"
+	echo python : "%PYTHON%"
 )
 if "%CMAKE%" == "" (
 	echo Cmake not found in path, required for building, exiting...
diff --git a/build_files/windows/format.cmd b/build_files/windows/format.cmd
index f036257e220..43f8b61727d 100644
--- a/build_files/windows/format.cmd
+++ b/build_files/windows/format.cmd
@@ -9,7 +9,7 @@ exit /b 1
 :detect_done
 echo found clang-format in %CF_PATH%
 
-if EXIST %BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe (
+if EXIST %PYTHON% (
     set PYTHON=%BLENDER_DIR%\..\lib\win64_vc14\python\37\bin\python.exe
     goto detect_python_done
 )
@@ -25,6 +25,7 @@ set FORMAT_PATHS=%BLENDER_DIR%\source\tools\utils_maintenance\clang_format_paths
 REM The formatting script expects clang-format to be in the current PATH.
 set PATH=%CF_PATH%;%PATH%
 
-%PYTHON% %FORMAT_PATHS% %FORMAT_ARGS%
+REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
+%PYTHON% -B %FORMAT_PATHS% %FORMAT_ARGS%
 
 :EOF
diff --git a/build_files/windows/parse_arguments.cmd b/build_files/windows/parse_arguments.cmd
index 3f40ef1f5ef..cb375b7fc9f 100644
--- a/build_files/windows/parse_arguments.cmd
+++ b/build_files/windows/parse_arguments.cmd
@@ -82,12 +82,10 @@ if NOT "%1" == "" (
 	REM Non-Build Commands
 	) else if "%1" == "update" (
 		SET BUILD_UPDATE=1
-		set BUILD_UPDATE_SVN=1
-		set BUILD_UPDATE_GIT=1
+		set BUILD_UPDATE_ARGS=
 	) else if "%1" == "code_update" (
 		SET BUILD_UPDATE=1
-		set BUILD_UPDATE_SVN=0
-		set BUILD_UPDATE_GIT=1
+		set BUILD_UPDATE_ARGS="--only-code"
 	) else if "%1" == "ninja" (
 		SET BUILD_WITH_NINJA=1
 	) else if "%1" == "clean" (
@@ -108,4 +106,4 @@ if NOT "%1" == "" (
 :EOF
 exit /b 0
 :ERR
-exit /b 1
\ No newline at end of file
+exit /b 1
diff --git a/build_files/windows/update_sources.cmd b/build_files/windows/update_sources.cmd
index 1f571eaf92d..f8fbd383090 100644
--- a/build_files/windows/update_sources.cmd
+++ b/build_files/windows/update_sources.cmd
@@ -1,18 +1,13 @@
-if "%BUILD_UPDATE_SVN%" == "1" (
-	if "%SVN%" == "" (
-		echo svn not found, cannot update libraries
-		goto UPDATE_GIT
-	)
-	"%SVN%" up "%BLENDER_DIR%/../lib/*"
+if EXIST %PYTHON% (
+	goto detect_python_done
 )
-:UPDATE_GIT
 
-if "%BUILD_UPDATE_GIT%" == "1" (
-	if "%GIT%" == "" (
-		echo Git not found, cannot update code
-		goto EOF
-	)
-	"%GIT%" pull --rebase
-	"%GIT%" submodule foreach git pull --rebase origin master
-)
+echo python not found in lib folder
+exit /b 1
+
+:detect_python_done
+
+REM Use -B to avoid writing __pycache__ in lib directory and causing update conflicts.
+%PYTHON% -B %BLENDER_DIR%\build_files\utils\make_update.py --git-command "%GIT%" --svn-command "%SVN%" %BUILD_UPDATE_ARGS%
+
 :EOF
diff --git a/make.bat b/make.bat
index b0323cde1bb..3f09718ab4f 100644
--- a/make.bat
+++ b/make.bat
@@ -26,12 +26,6 @@ if "%SHOW_HELP%" == "1" (
 	goto EOF
 )
 
-if "%BUILD_UPDATE%" == "1" (
-	call "%BLENDER_DIR%\build_files\windows\update_sources.cmd"
-	goto EOF
-)
-
-
 if "%FORMAT%" == "1" (
 	call "%BLENDER_DIR%\build_files\windows\format.cmd"
 	goto EOF
@@ -54,6 +48,14 @@ if "%BUILD_VS_YEAR%" == "" (
 	)
 )
 
+if "%BUILD_UPDATE%" == "1" (
+	call "%BLENDER_DIR%\build_files\windows\check_libraries.cmd"
+	if errorlevel 1 goto EOF
+
+	call "%BLENDER_DIR%\build_files\windows\update_sources.cmd"
+	goto EOF
+)
+
 call "%BLENDER_DIR%\build_files\windows\set_build_dir.cmd"
 
 echo Building blender with VS%BUILD_VS_YEAR% for %BUILD_ARCH% in %BUILD_DIR%