From 04a63cb3933dcf7d9a76805b9bf147c1c9dc6687 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Tue, 22 Mar 2011 13:05:05 +0000 Subject: Buildbot updates: working windows scons build, and run slave_*.py scripts to compile/test/package directly from svn, easier to make changes this way. --- build_files/buildbot/master.cfg | 107 +++++++++++++--------------------- build_files/buildbot/master_unpack.py | 99 ++++++++++++++++--------------- build_files/buildbot/slave_compile.py | 58 ++++++++++++++++++ build_files/buildbot/slave_pack.py | 29 +++++---- build_files/buildbot/slave_test.py | 40 +++++++++++++ 5 files changed, 202 insertions(+), 131 deletions(-) create mode 100644 build_files/buildbot/slave_compile.py create mode 100644 build_files/buildbot/slave_test.py (limited to 'build_files') diff --git a/build_files/buildbot/master.cfg b/build_files/buildbot/master.cfg index c4c8d8108e1..aee6f1175eb 100644 --- a/build_files/buildbot/master.cfg +++ b/build_files/buildbot/master.cfg @@ -36,7 +36,7 @@ c['change_source'] = SVNPoller( # only take place on one slave. from buildbot.process.factory import BuildFactory -from buildbot.steps.source import SVN +from buildbot.steps.source import SVN from buildbot.steps.shell import ShellCommand from buildbot.steps.shell import Compile from buildbot.steps.shell import Test @@ -50,102 +50,76 @@ from buildbot.config import BuilderConfig c['builders'] = [] buildernames = [] - -def add_builder(c, name, factory): +def add_builder(c, name, libdir, factory): slavenames = [] for slave in master_private.slaves: if name in slave['builders']: slavenames.append(slave['name']) - f = factory(name) - c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender')) - buildernames.append(name) + if len(slavenames) > 0: + f = factory(name, libdir) + c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender')) + buildernames.append(name) # common steps - def svn_step(): return SVN(baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/blender', mode='update', defaultBranch='trunk', workdir='blender') - def lib_svn_step(dir): return SVN(name='lib svn', baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, mode='update', defaultBranch='trunk', workdir='lib/' + dir) - -def cmake_compile_step(): - return Compile(command=['make'], workdir='blender') - - -def cmake_test_step(): - return Test(workdir='blender') # make test - - def scons_compile_step(): return Compile(command=['python', 'scons/scons.py'], workdir='blender') - -class SlavePack(ShellCommand): - pack_script = 'slave_pack.py' - - def start(self): - if self.getProperty('buildername').find('scons') >= 0: - self.setCommand(['python', pack_script, 'scons']) - else: - self.setCommand(['python', pack_script, 'cmake']) - ShellCommand.start(self) - - -def file_upload(f, id): +def scons_file_upload(f, id): filename = 'buildbot_upload_' + id + '.zip' - pack_script = 'slave_pack.py' unpack_script = 'master_unpack.py' - f.addStep(FileDownload(name='download', mastersrc=pack_script, slavedest=pack_script)) - f.addStep(ShellCommand(name='package', command=['python', pack_script], description='packaging', descriptionDone='packaged')) - f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100 * 1024 * 1024)) + f.addStep(Compile(name='package', command=['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave'], description='packaging', descriptionDone='packaged', workdir='blender')) + f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024, workdir='install')) f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked')) -# linux cmake +# generic +def generic_builder(id, libdir=''): + filename = 'buildbot_upload_' + id + '.zip' + compile_script = '../blender/build_files/buildbot/slave_compile.py' + test_script = '../blender/build_files/buildbot/slave_test.py' + pack_script = '../blender/build_files/buildbot/slave_pack.py' + unpack_script = 'master_unpack.py' -def linux_cmake(id): f = BuildFactory() f.addStep(svn_step()) - f.addStep(cmake_compile_step()) - f.addStep(cmake_test_step()) - file_upload(f, id) - return f - -add_builder(c, 'linux_x86_64_cmake', linux_cmake) - -# mac cmake - + if libdir != '': + f.addStep(lib_svn_step(libdir)) -def mac_cmake(id): - f = BuildFactory() - f.addStep(svn_step()) - f.addStep(lib_svn_step('darwin-9.x.universal')) - f.addStep(cmake_compile_step()) - f.addStep(cmake_test_step()) - file_upload(f, id) + f.addStep(Compile(command=['python', compile_script, id])) + f.addStep(Test(command=['python', test_script, id])) + f.addStep(ShellCommand(name='package', command=['python', pack_script, id], description='packaging', descriptionDone='packaged')) + f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024)) + f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked')) return f -add_builder(c, 'mac_x86_64_cmake', mac_cmake) +add_builder(c, 'mac_x86_64_cmake', 'darwin-9.x.universal', generic_builder) +add_builder(c, 'mac_i386_cmake', 'darwin-9.x.universal', generic_builder) +add_builder(c, 'mac_ppc_cmake', 'darwin-9.x.universal', generic_builder) +add_builder(c, 'linux_x86_64_cmake', '', generic_builder) +add_builder(c, 'linux_x86_64_scons', '', generic_builder) # win32 scons - # TODO: add scons test target -def win32_scons(id): +def win32_scons(id, libdir): f = BuildFactory() f.addStep(svn_step()) - f.addStep(lib_svn_step('windows')) + f.addStep(lib_svn_step(libdir)) f.addStep(scons_compile_step()) - file_upload(f, id) + scons_file_upload(f, id) return f -add_builder(c, 'win32_scons', win32_scons) +add_builder(c, 'win32_scons', 'windows', win32_scons) # SCHEDULERS # @@ -178,16 +152,16 @@ c['status'] = [] from buildbot.status import html from buildbot.status.web import auth, authz -authz_cfg = authz.Authz( +authz_cfg=authz.Authz( # change any of these to True to enable; see the manual for more # options - gracefulShutdown=False, - forceBuild=True, # use this to test your slave once it is set up - forceAllBuilds=False, - pingBuilder=False, - stopBuild=False, - stopAllBuilds=False, - cancelPendingBuild=False, + gracefulShutdown = False, + forceBuild = True, # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, ) c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) @@ -212,3 +186,4 @@ c['buildbotURL'] = "http://builder.blender.org/" # installations. c['db_url'] = "sqlite:///state.sqlite" + diff --git a/build_files/buildbot/master_unpack.py b/build_files/buildbot/master_unpack.py index 8580fd3d6d6..2c2bc1898b2 100644 --- a/build_files/buildbot/master_unpack.py +++ b/build_files/buildbot/master_unpack.py @@ -18,69 +18,67 @@ # Runs on Buildbot master, to unpack incoming unload.zip into latest # builds directory and remove older builds. - + import os import shutil import sys import zipfile - # extension stripping def strip_extension(filename): - extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'] + extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe'] - for ext in extensions: - if filename.endswith(ext): - filename = filename[:-len(ext)] - - return filename + for ext in extensions: + if filename.endswith(ext): + filename = filename[:-len(ext)] + return filename # extract platform from package name def get_platform(filename): - # name is blender-version-platform.extension. we want to get the - # platform out, but there may be some variations, so we fiddle a - # bit to handle current and hopefully future names - filename = strip_extension(filename) - filename = strip_extension(filename) + # name is blender-version-platform.extension. we want to get the + # platform out, but there may be some variations, so we fiddle a + # bit to handle current and hopefully future names + filename = strip_extension(filename) + filename = strip_extension(filename) - tokens = filename.split("-") - platforms = ['osx', 'mac', 'bsd', 'windows', 'linux', 'source', 'irix', 'solaris'] - platform_tokens = [] - found = False + tokens = filename.split("-") + platforms = ['osx', 'mac', 'bsd', 'win', 'linux', 'source', 'irix', 'solaris'] + platform_tokens = [] + found = False - for i, token in enumerate(tokens): - if not found: - for platform in platforms: - if token.lower().find(platform) != -1: - found = True + for i, token in enumerate(tokens): + if not found: + for platform in platforms: + if token.lower().find(platform) != -1: + found = True - if found: - platform_tokens += [token] + if found: + platform_tokens += [token] - return '-'.join(platform_tokens) + return '-'.join(platform_tokens) # get filename if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting file to unpack\n") - sys.exit(1) + sys.stderr.write("Not enough arguments, expecting file to unpack\n") + sys.exit(1) filename = sys.argv[1] # open zip file if not os.path.exists(filename): - sys.stderr.write("File " + filename + " not found.\n") - sys.exit(1) + sys.stderr.write("File " + filename + " not found.\n") + sys.exit(1) try: - z = zipfile.ZipFile(filename, "r") + z = zipfile.ZipFile(filename, "r") except Exception, ex: - sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n') - sys.exit(1) + sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n') + sys.exit(1) if len(z.namelist()) != 1: - sys.stderr.write("Expected on file in " + filename + ".") - sys.exit(1) + sys.stderr.write("Expected on file in " + filename + ".") + sys.exit(1) package = z.namelist()[0] packagename = os.path.basename(package) @@ -89,30 +87,31 @@ packagename = os.path.basename(package) platform = get_platform(packagename) if platform == '': - sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n') - sys.exit(1) + sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n') + sys.exit(1) # extract -dir = 'public_html/latest_builds' +dir = 'public_html/download' try: - zf = z.open(package) - f = file(os.path.join(dir, packagename), "wb") + zf = z.open(package) + f = file(os.path.join(dir, packagename), "wb") - shutil.copyfileobj(zf, f) + shutil.copyfileobj(zf, f) - zf.close() - z.close() + zf.close() + z.close() except Exception, ex: - sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n') - sys.exit(1) + sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n') + sys.exit(1) # remove other files from the same platform try: - for f in os.listdir(dir): - if f.lower().find(platform.lower()) != -1: - if f != packagename: - os.remove(os.path.join(dir, f)) + for f in os.listdir(dir): + if f.lower().find(platform.lower()) != -1: + if f != packagename: + os.remove(os.path.join(dir, f)) except Exception, ex: - sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n') - sys.exit(1) + sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n') + sys.exit(1) + diff --git a/build_files/buildbot/slave_compile.py b/build_files/buildbot/slave_compile.py new file mode 100644 index 00000000000..4892865bcea --- /dev/null +++ b/build_files/buildbot/slave_compile.py @@ -0,0 +1,58 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +import os +import subprocess +import sys + +# get builder name +if len(sys.argv) < 2: + sys.stderr.write("Not enough arguments, expecting builder name\n") + sys.exit(1) + +builder = sys.argv[1] + +# we run from build/ directory +blender_dir = '../blender' + +if builder.find('cmake') != -1: + # cmake + + # set build options + cmake_options = ['-DCMAKE_BUILD_TYPE:STRING=Release'] + + if builder == 'mac_x86_64_cmake': + cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=x86_64') + elif builder == 'mac_i386_cmake': + cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=i386') + elif builder == 'mac_ppc_cmake': + cmake_options.append('-DCMAKE_OSX_ARCHITECTURES:STRING=ppc') + + # configure and make + retcode = subprocess.call(['cmake', blender_dir] + cmake_options) + if retcode != 0: + sys.exit(retcode) + retcode = subprocess.call(['make', '-s', '-j4', 'install']) + sys.exit(retcode) +else: + # scons + os.chdir(blender_dir) + + retcode = subprocess.call(['python', 'scons/scons.py']) + sys.exit(retcode) + diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py index 6710591d0d7..12171cf4003 100644 --- a/build_files/buildbot/slave_pack.py +++ b/build_files/buildbot/slave_pack.py @@ -25,38 +25,36 @@ import subprocess import sys import zipfile +# get builder name +if len(sys.argv) < 2: + sys.stderr.write("Not enough arguments, expecting builder name\n") + sys.exit(1) -# find release directory -def find_release_directory(): - for d in os.listdir('.'): - if os.path.isdir(d): - rd = os.path.join(d, 'release') - if os.path.exists(rd): - return rd +builder = sys.argv[1] - return None +# scons does own packaging +if builder.find('scons') != -1: + os.chdir('../blender') + retcode = subprocess.call(['python', 'scons/scons.py', 'BF_QUICK=slnt', 'buildslave']) + sys.exit(retcode) # clean release directory if it already exists -dir = find_release_directory() +dir = 'release' -if dir: +if os.path.exists(dir): for f in os.listdir(dir): if os.path.isfile(os.path.join(dir, f)): os.remove(os.path.join(dir, f)) # create release package try: - os.chdir('../blender') subprocess.call(['make', 'package_archive']) - os.chdir('../build') except Exception, ex: sys.stderr.write('Make package release failed' + str(ex) + '\n') sys.exit(1) # find release directory, must exist this time -dir = find_release_directory() - -if not dir: +if not os.path.exists(dir): sys.stderr.write("Failed to find release directory.\n") sys.exit(1) @@ -85,3 +83,4 @@ try: except Exception, ex: sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n') sys.exit(1) + diff --git a/build_files/buildbot/slave_test.py b/build_files/buildbot/slave_test.py new file mode 100644 index 00000000000..7e78f1ab5cd --- /dev/null +++ b/build_files/buildbot/slave_test.py @@ -0,0 +1,40 @@ +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +import os +import subprocess +import sys + +# get builder name +if len(sys.argv) < 2: + sys.stderr.write("Not enough arguments, expecting builder name\n") + sys.exit(1) + +builder = sys.argv[1] + +# we run from build/ directory +blender_dir = '../blender' + +if builder.find('cmake') != -1: + # cmake + retcode = subprocess.call(['ctest', '.' '--output-on-failure']) + sys.exit(retcode) +else: + # scons + pass + -- cgit v1.2.3