Buildbot updates: working windows scons build, and run slave_*.py scripts

to compile/test/package directly from svn, easier to make changes this way.

5 files changed, 202 insertions, 131 deletions

diff --git a/build_files/buildbot/master.cfg b/build_files/buildbot/master.cfg
index c4c8d8108e1..aee6f1175eb 100644
--- a/build_files/buildbot/master.cfg
+++ b/build_files/buildbot/master.cfg
@@ -36,7 +36,7 @@ c['change_source'] = SVNPoller(
 # only take place on one slave.
 
 from buildbot.process.factory import BuildFactory
-from buildbot.steps.source import SVN
+from buildbot.steps.source import SVN 
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.shell import Compile
 from buildbot.steps.shell import Test
@@ -50,102 +50,76 @@ from buildbot.config import BuilderConfig
 c['builders'] = []
 buildernames = []
 
-
-def add_builder(c, name, factory):
+def add_builder(c, name, libdir, factory):
     slavenames = []
 
     for slave in master_private.slaves:
         if name in slave['builders']:
             slavenames.append(slave['name'])
 
-    f = factory(name)
-    c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
-    buildernames.append(name)
+    if len(slavenames) > 0:
+        f = factory(name, libdir)
+        c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
+        buildernames.append(name)
 
 # common steps
 
-
 def svn_step():
     return SVN(baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/blender', mode='update', defaultBranch='trunk', workdir='blender')
 
-
 def lib_svn_step(dir):
     return SVN(name='lib svn', baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, mode='update', defaultBranch='trunk', workdir='lib/' + dir)
 
-
-def cmake_compile_step():
-    return Compile(command=['make'], workdir='blender')
-
-
-def cmake_test_step():
-    return Test(workdir='blender')  # make test
-
-
 def scons_compile_step():
     return Compile(command=['python', 'scons/scons.py'], workdir='blender')
 
-
-class SlavePack(ShellCommand):
-    pack_script = 'slave_pack.py'
-
-    def start(self):
-        if self.getProperty('buildername').find('scons') >= 0:
-            self.setCommand(['python', pack_script, 'scons'])
-        else:
-            self.setCommand(['python', pack_script, 'cmake'])
-        ShellCommand.start(self)
-
-
-def file_upload(f, id):
+def scons_file_upload(f, id):
     filename = 'buildbot_upload_' + id + '.zip'
-    pack_script = 'slave_pack.py'
     unpack_script = 'master_unpack.py'
 
-    f.addStep(FileDownload(name='download', mastersrc=pack_script, slavedest=pack_script))
-    f.addStep(ShellCommand(name='package', command=['python', pack_script], description='packaging', descriptionDone='packaged'))
-    f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100 * 1024 * 1024))
+    f.addStep(Compile(name='package', command=['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave'], description='packaging', descriptionDone='packaged', workdir='blender'))
+    f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024, workdir='install'))
     f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
 
-# linux cmake
+# generic
 
+def generic_builder(id, libdir=''):
+    filename = 'buildbot_upload_' + id + '.zip'
+    compile_script = '../blender/build_files/buildbot/slave_compile.py'
+    test_script = '../blender/build_files/buildbot/slave_test.py'
+    pack_script = '../blender/build_files/buildbot/slave_pack.py'
+    unpack_script = 'master_unpack.py'
 
-def linux_cmake(id):
     f = BuildFactory()
     f.addStep(svn_step())
-    f.addStep(cmake_compile_step())
-    f.addStep(cmake_test_step())
-    file_upload(f, id)
-    return f
-
-add_builder(c, 'linux_x86_64_cmake', linux_cmake)
-
-# mac cmake
-
+    if libdir != '':
+        f.addStep(lib_svn_step(libdir))
 
-def mac_cmake(id):
-    f = BuildFactory()
-    f.addStep(svn_step())
-    f.addStep(lib_svn_step('darwin-9.x.universal'))
-    f.addStep(cmake_compile_step())
-    f.addStep(cmake_test_step())
-    file_upload(f, id)
+    f.addStep(Compile(command=['python', compile_script, id]))
+    f.addStep(Test(command=['python', test_script, id]))
+    f.addStep(ShellCommand(name='package', command=['python', pack_script, id], description='packaging', descriptionDone='packaged'))
+    f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024))
+    f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
     return f
 
-add_builder(c, 'mac_x86_64_cmake', mac_cmake)
+add_builder(c, 'mac_x86_64_cmake', 'darwin-9.x.universal', generic_builder)
+add_builder(c, 'mac_i386_cmake', 'darwin-9.x.universal', generic_builder)
+add_builder(c, 'mac_ppc_cmake', 'darwin-9.x.universal', generic_builder)
+add_builder(c, 'linux_x86_64_cmake', '', generic_builder)
+add_builder(c, 'linux_x86_64_scons', '', generic_builder)
 
 # win32 scons
 
-
 # TODO: add scons test target
-def win32_scons(id):
+def win32_scons(id, libdir):
     f = BuildFactory()
     f.addStep(svn_step())
-    f.addStep(lib_svn_step('windows'))
+    f.addStep(lib_svn_step(libdir))
     f.addStep(scons_compile_step())
-    file_upload(f, id)
+    scons_file_upload(f, id)
     return f
 
-add_builder(c, 'win32_scons', win32_scons)
+add_builder(c, 'win32_scons', 'windows', win32_scons)
 
 # SCHEDULERS
 #
@@ -178,16 +152,16 @@ c['status'] = []
 from buildbot.status import html
 from buildbot.status.web import auth, authz
 
-authz_cfg = authz.Authz(
+authz_cfg=authz.Authz(
     # change any of these to True to enable; see the manual for more
     # options
-    gracefulShutdown=False,
-    forceBuild=True,  # use this to test your slave once it is set up
-    forceAllBuilds=False,
-    pingBuilder=False,
-    stopBuild=False,
-    stopAllBuilds=False,
-    cancelPendingBuild=False,
+    gracefulShutdown = False,
+    forceBuild = True, # use this to test your slave once it is set up
+    forceAllBuilds = False,
+    pingBuilder = False,
+    stopBuild = False,
+    stopAllBuilds = False,
+    cancelPendingBuild = False,
 )
 
 c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
@@ -212,3 +186,4 @@ c['buildbotURL'] = "http://builder.blender.org/"
 # installations.
 
 c['db_url'] = "sqlite:///state.sqlite"
+
diff --git a/build_files/buildbot/master_unpack.py b/build_files/buildbot/master_unpack.py
index 8580fd3d6d6..2c2bc1898b2 100644
--- a/build_files/buildbot/master_unpack.py
+++ b/build_files/buildbot/master_unpack.py
@@ -18,69 +18,67 @@
 
 # Runs on Buildbot master, to unpack incoming unload.zip into latest
 # builds directory and remove older builds.
-
+ 
 import os
 import shutil
 import sys
 import zipfile
 
-
 # extension stripping
 def strip_extension(filename):
-    extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
+	extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
 
-    for ext in extensions:
-        if filename.endswith(ext):
-            filename = filename[:-len(ext)]
-
-    return filename
+	for ext in extensions:
+		if filename.endswith(ext):
+			filename = filename[:-len(ext)]
 
+	return filename
 
 # extract platform from package name
 def get_platform(filename):
-    # name is blender-version-platform.extension. we want to get the
-    # platform out, but there may be some variations, so we fiddle a
-    # bit to handle current and hopefully future names
-    filename = strip_extension(filename)
-    filename = strip_extension(filename)
+	# name is blender-version-platform.extension. we want to get the
+	# platform out, but there may be some variations, so we fiddle a
+	# bit to handle current and hopefully future names
+	filename = strip_extension(filename)
+	filename = strip_extension(filename)
 
-    tokens = filename.split("-")
-    platforms = ['osx', 'mac', 'bsd', 'windows', 'linux', 'source', 'irix', 'solaris']
-    platform_tokens = []
-    found = False
+	tokens = filename.split("-")
+	platforms = ['osx', 'mac', 'bsd', 'win', 'linux', 'source', 'irix', 'solaris']
+	platform_tokens = []
+	found = False
 
-    for i, token in enumerate(tokens):
-        if not found:
-            for platform in platforms:
-                if token.lower().find(platform) != -1:
-                    found = True
+	for i, token in enumerate(tokens):
+		if not found:
+			for platform in platforms:
+				if token.lower().find(platform) != -1:
+					found = True
 
-        if found:
-            platform_tokens += [token]
+		if found:
+			platform_tokens += [token]
 
-    return '-'.join(platform_tokens)
+	return '-'.join(platform_tokens)
 
 # get filename
 if len(sys.argv) < 2:
-    sys.stderr.write("Not enough arguments, expecting file to unpack\n")
-    sys.exit(1)
+	sys.stderr.write("Not enough arguments, expecting file to unpack\n")
+	sys.exit(1)
 
 filename = sys.argv[1]
 
 # open zip file
 if not os.path.exists(filename):
-    sys.stderr.write("File " + filename + " not found.\n")
-    sys.exit(1)
+	sys.stderr.write("File " + filename + " not found.\n")
+	sys.exit(1)
 
 try:
-    z = zipfile.ZipFile(filename, "r")
+	z = zipfile.ZipFile(filename, "r")
 except Exception, ex:
-    sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
-    sys.exit(1)
+	sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
+	sys.exit(1)
 
 if len(z.namelist()) != 1:
-    sys.stderr.write("Expected on file in " + filename + ".")
-    sys.exit(1)
+	sys.stderr.write("Expected on file in " + filename + ".")
+	sys.exit(1)
 
 package = z.namelist()[0]
 packagename = os.path.basename(package)
@@ -89,30 +87,31 @@ packagename = os.path.basename(package)
 platform = get_platform(packagename)
 
 if platform == '':
-    sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
-    sys.exit(1)
+	sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
+	sys.exit(1)
 
 # extract
-dir = 'public_html/latest_builds'
+dir = 'public_html/download'
 
 try:
-    zf = z.open(package)
-    f = file(os.path.join(dir, packagename), "wb")
+	zf = z.open(package)
+	f = file(os.path.join(dir, packagename), "wb")
 
-    shutil.copyfileobj(zf, f)
+	shutil.copyfileobj(zf, f)
 
-    zf.close()
-    z.close()
+	zf.close()
+	z.close()
 except Exception, ex:
-    sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
-    sys.exit(1)
+	sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
+	sys.exit(1)
 
 # remove other files from the same platform
 try:
-    for f in os.listdir(dir):
-        if f.lower().find(platform.lower()) != -1:
-            if f != packagename:
-                os.remove(os.path.join(dir, f))
+	for f in os.listdir(dir):
+		if f.lower().find(platform.lower()) != -1:
+			if f != packagename:
+				os.remove(os.path.join(dir, f))
 except Exception, ex:
-    sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
-    sys.exit(1)
+	sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
+	sys.exit(1)
+
diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py
new file mode 100644
index 00000000000..4892865bcea
--- /dev/null
+++ b/build_files/buildbot/slave_compile.py
@@ -0,0 +1,58 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+import os
+import subprocess
+import sys
+
+# get builder name
+if len(sys.argv) < 2:
+    sys.stderr.write("Not enough arguments, expecting builder name\n")
+    sys.exit(1)
+
+builder = sys.argv[1]
+
+# we run from build/ directory
+blender_dir = '../blender'
+
+if builder.find('cmake') != -1:
+    # cmake
+
+    # set build options
+    cmake_options = ['-DCMAKE_BUILD_TYPE:STRING=Release']
+
+    if builder == 'mac_x86_64_cmake':
+        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64')
+    elif builder == 'mac_i386_cmake':
+        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386')
+    elif builder == 'mac_ppc_cmake':
+        cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc')
+
+    # configure and make
+    retcode = subprocess.call(['cmake', blender_dir] + cmake_options)
+    if retcode != 0:
+        sys.exit(retcode)
+    retcode = subprocess.call(['make', '-s', '-j4', 'install'])
+    sys.exit(retcode)
+else:
+    # scons
+    os.chdir(blender_dir)
+
+    retcode = subprocess.call(['python', 'scons/scons.py'])
+    sys.exit(retcode)
+
diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py
index 6710591d0d7..12171cf4003 100644
--- a/build_files/buildbot/slave_pack.py
+++ b/build_files/buildbot/slave_pack.py
@@ -25,38 +25,36 @@ import subprocess
 import sys
 import zipfile
 
+# get builder name
+if len(sys.argv) < 2:
+    sys.stderr.write("Not enough arguments, expecting builder name\n")
+    sys.exit(1)
 
-# find release directory
-def find_release_directory():
-    for d in os.listdir('.'):
-        if os.path.isdir(d):
-            rd = os.path.join(d, 'release')
-            if os.path.exists(rd):
-                return rd
+builder = sys.argv[1]
 
-    return None
+# scons does own packaging
+if builder.find('scons') != -1:
+    os.chdir('../blender')
+    retcode = subprocess.call(['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave'])
+    sys.exit(retcode)
 
 # clean release directory if it already exists
-dir = find_release_directory()
+dir = 'release'
 
-if dir:
+if os.path.exists(dir):
     for f in os.listdir(dir):
         if os.path.isfile(os.path.join(dir, f)):
             os.remove(os.path.join(dir, f))
 
 # create release package
 try:
-    os.chdir('../blender')
     subprocess.call(['make', 'package_archive'])
-    os.chdir('../build')
 except Exception, ex:
     sys.stderr.write('Make package release failed' + str(ex) + '\n')
     sys.exit(1)
 
 # find release directory, must exist this time
-dir = find_release_directory()
-
-if not dir:
+if not os.path.exists(dir):
     sys.stderr.write("Failed to find release directory.\n")
     sys.exit(1)
 
@@ -85,3 +83,4 @@ try:
 except Exception, ex:
     sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
     sys.exit(1)
+
diff --git a/build_files/buildbot/slave_test.py b/build_files/buildbot/slave_test.py
new file mode 100644
index 00000000000..7e78f1ab5cd
--- /dev/null
+++ b/build_files/buildbot/slave_test.py
@@ -0,0 +1,40 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+import os
+import subprocess
+import sys
+
+# get builder name
+if len(sys.argv) < 2:
+    sys.stderr.write("Not enough arguments, expecting builder name\n")
+    sys.exit(1)
+
+builder = sys.argv[1]
+
+# we run from build/ directory
+blender_dir = '../blender'
+
+if builder.find('cmake') != -1:
+    # cmake
+    retcode = subprocess.call(['ctest', '.' '--output-on-failure'])
+    sys.exit(retcode)
+else:
+    # scons
+    pass
+