Buildbot initial configuration files, for http://builder.blender.org

Maintained in svn to make it easier for others to contribute changes,
actually updating builder.blender.org requires manual update on the
server.

3 files changed, 407 insertions, 0 deletions

diff --git a/build_files/buildbot/master.cfg b/build_files/buildbot/master.cfg
new file mode 100644
index 00000000000..0b37e055584
--- /dev/null
+++ b/build_files/buildbot/master.cfg
@@ -0,0 +1,203 @@
+# -*- python -*-
+# ex: set syntax=python:
+
+# Dictionary that the buildmaster pays attention to.
+c = BuildmasterConfig = {}
+
+# BUILD SLAVES
+#
+# We load the slaves and their passwords from a separator file, so we can have
+# this one in SVN.
+
+from buildbot.buildslave import BuildSlave
+import master_private
+
+c['slaves'] = []
+
+for slave in master_private.slaves:
+	c['slaves'].append(BuildSlave(slave['name'], slave['password']))
+
+# TCP port through which slaves connect
+
+c['slavePortnum'] = 9989
+
+# CHANGE SOURCES
+
+from buildbot.changes.svnpoller import SVNPoller
+
+c['change_source'] = SVNPoller(
+       'https://svn.blender.org/svnroot/bf-blender/trunk/',
+       pollinterval=1200)
+
+# BUILDERS
+#
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which slaves can execute them.  Note that any particular build will
+# only take place on one slave.
+
+from buildbot.process.factory import BuildFactory
+from buildbot.steps.source import SVN 
+from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import Compile
+from buildbot.steps.shell import Test
+from buildbot.steps.transfer import FileUpload
+from buildbot.steps.transfer import FileDownload
+from buildbot.steps.master import MasterShellCommand
+from buildbot.config import BuilderConfig
+
+# add builder utility
+
+c['builders'] = []
+buildernames = []
+
+def add_builder(c, name, factory):
+	slavenames = []
+
+	for slave in master_private.slaves:
+		if name in slave['builders']:
+			slavenames.append(slave['name'])
+	
+	f = factory(name)
+	c['builders'].append(BuilderConfig(name=name, slavenames=slavenames, factory=f, category='blender'))
+	buildernames.append(name)
+
+# common steps
+
+def svn_step():
+	return SVN(baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/blender', mode='update', defaultBranch='trunk', workdir='blender')
+
+def lib_svn_step(dir):
+	return SVN(name='lib svn', baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, mode='update', defaultBranch='trunk', workdir='lib/' + dir)
+
+def cmake_compile_step():
+	return Compile(command=['make'], workdir='blender')
+
+def cmake_test_step():
+	return Test(workdir='blender') # make test
+
+def scons_compile_step():
+	return Compile(command=['python', 'scons/scons.py'], workdir='blender')
+
+class SlavePack(ShellCommand):
+	pack_script = 'slave_pack.py'
+	def start(self):
+		if self.getProperty('buildername').find('scons')>=0:
+			self.setCommand(['python', pack_script, 'scons'])
+		else:
+			self.setCommand(['python', pack_script, 'cmake'])
+		ShellCommand.start(self)
+
+def file_upload(f, id):
+	filename = 'buildbot_upload_' + id + '.zip'
+	pack_script = 'slave_pack.py'
+	unpack_script = 'master_unpack.py'
+
+	f.addStep(FileDownload(name='download', mastersrc=pack_script, slavedest=pack_script))
+	f.addStep(ShellCommand(name='package', command=['python', pack_script], description='packaging', descriptionDone='packaged'))
+	f.addStep(FileUpload(name='upload', slavesrc='buildbot_upload.zip', masterdest=filename, maxsize=100*1024*1024))
+	f.addStep(MasterShellCommand(name='unpack', command=['python', unpack_script, filename], description='unpacking', descriptionDone='unpacked'))
+
+# linux cmake
+
+def linux_cmake(id):
+	f = BuildFactory()
+	f.addStep(svn_step())
+	f.addStep(cmake_compile_step())
+	f.addStep(cmake_test_step())
+	file_upload(f, id)
+	return f
+
+add_builder(c, 'linux_x86_64_cmake', linux_cmake)
+
+# mac cmake
+
+def mac_cmake(id):
+	f = BuildFactory()
+	f.addStep(svn_step())
+	f.addStep(lib_svn_step('darwin-9.x.universal'))
+	f.addStep(cmake_compile_step())
+	f.addStep(cmake_test_step())
+	file_upload(f, id)
+	return f
+
+add_builder(c, 'mac_x86_64_cmake', mac_cmake)
+
+# win32 scons
+
+# TODO: add scons test target
+def win32_scons(id):
+	f = BuildFactory()
+	f.addStep(svn_step())
+	f.addStep(lib_svn_step('windows'))
+	f.addStep(scons_compile_step())
+	file_upload(f, id)
+	return f
+
+add_builder(c, 'win32_scons', win32_scons)
+
+# SCHEDULERS
+#
+# Decide how to react to incoming changes.
+
+from buildbot.scheduler import Scheduler
+from buildbot.schedulers import timed
+
+c['schedulers'] = []
+#c['schedulers'].append(Scheduler(name="all", branch=None,
+#                                 treeStableTimer=None,
+#                                 builderNames=[]))
+#c['schedulers'].append(timed.Periodic(name="nightly",
+#						builderNames=buildernames,
+#						periodicBuildTimer=24*60*60))
+
+c['schedulers'].append(timed.Nightly(name='nightly',
+    builderNames=buildernames,
+    hour=3,
+    minute=0))
+
+# STATUS TARGETS
+#
+# 'status' is a list of Status Targets. The results of each build will be
+# pushed to these targets. buildbot/status/*.py has a variety to choose from,
+# including web pages, email senders, and IRC bots.
+
+c['status'] = []
+
+from buildbot.status import html
+from buildbot.status.web import auth, authz
+
+authz_cfg=authz.Authz(
+    # change any of these to True to enable; see the manual for more
+    # options
+    gracefulShutdown = False,
+    forceBuild = True, # use this to test your slave once it is set up
+    forceAllBuilds = False,
+    pingBuilder = False,
+    stopBuild = False,
+    stopAllBuilds = False,
+    cancelPendingBuild = False,
+)
+
+c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+# PROJECT IDENTITY
+
+c['projectName'] = "Blender"
+c['projectURL'] = "http://www.blender.org"
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server (usually the html.WebStatus page) is visible. This
+# typically uses the port number set in the Waterfall 'status' entry, but
+# with an externally-visible host name which the buildbot cannot figure out
+# without some help.
+
+c['buildbotURL'] = "http://builder.blender.org/"
+
+# DB URL
+#
+# This specifies what database buildbot uses to store change and scheduler
+# state.  You can leave this at its default for all but the largest
+# installations.
+
+c['db_url'] = "sqlite:///state.sqlite"
+
diff --git a/build_files/buildbot/master_unpack.py b/build_files/buildbot/master_unpack.py
new file mode 100644
index 00000000000..c905b55a669
--- /dev/null
+++ b/build_files/buildbot/master_unpack.py
@@ -0,0 +1,117 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# Runs on Buildbot master, to unpack incoming unload.zip into latest
+# builds directory and remove older builds.
+ 
+import os
+import shutil
+import sys
+import zipfile
+
+# extension stripping
+def strip_extension(filename):
+	extensions = ['.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe']
+
+	for ext in extensions:
+		if filename.endswith(ext):
+			filename = filename[:-len(ext)]
+
+	return filename
+
+# extract platform from package name
+def get_platform(filename):
+	# name is blender-version-platform.extension. we want to get the
+	# platform out, but there may be some variations, so we fiddle a
+	# bit to handle current and hopefully future names
+	filename = strip_extension(filename)
+	filename = strip_extension(filename)
+
+	tokens = filename.split("-")
+	platforms = ['osx', 'mac', 'bsd', 'windows', 'linux', 'source', 'irix', 'solaris']
+	platform_tokens = []
+	found = False
+
+	for i, token in enumerate(tokens):
+		if not found:
+			for platform in platforms:
+				if token.lower().find(platform) != -1:
+					found = True
+
+		if found:
+			platform_tokens += [token]
+
+	return '-'.join(platform_tokens)
+
+# get filename
+if len(sys.argv) < 2:
+	sys.stderr.write("Not enough arguments, expecting file to unpack\n")
+	sys.exit(1)
+
+filename = sys.argv[1]
+
+# open zip file
+if not os.path.exists(filename):
+	sys.stderr.write("File " + filename + " not found.\n")
+	sys.exit(1)
+
+try:
+	z = zipfile.ZipFile(filename, "r")
+except Exception, ex:
+	sys.stderr.write('Failed to open zip file: ' + str(ex) + '\n')
+	sys.exit(1)
+
+if len(z.namelist()) != 1:
+	sys.stderr.write("Expected on file in " + filename + ".")
+	sys.exit(1)
+
+package = z.namelist()[0]
+packagename = os.path.basename(package)
+
+# detect platform
+platform = get_platform(packagename)
+
+if platform == '':
+	sys.stderr.write('Failed to detect platform from package: ' + packagename + '\n')
+	sys.exit(1)
+
+# extract
+dir = 'public_html/latest_builds'
+
+try:
+	zf = z.open(package)
+	f = file(os.path.join(dir, packagename), "wb")
+
+	shutil.copyfileobj(zf, f)
+
+	zf.close()
+	z.close()
+except Exception, ex:
+	sys.stderr.write('Failed to unzip package: ' + str(ex) + '\n')
+	sys.exit(1)
+
+# remove other files from the same platform
+try:
+	for f in os.listdir(dir):
+		if f.lower().find(platform.lower()) != -1:
+			if f != packagename:
+				os.remove(os.path.join(dir, f))
+except Exception, ex:
+	sys.stderr.write('Failed to remove old packages: ' + str(ex) + '\n')
+	sys.exit(1)
+
diff --git a/build_files/buildbot/slave_pack.py b/build_files/buildbot/slave_pack.py
new file mode 100644
index 00000000000..5fdcda51f5c
--- /dev/null
+++ b/build_files/buildbot/slave_pack.py
@@ -0,0 +1,87 @@
+# ##### BEGIN GPL LICENSE BLOCK #####
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software Foundation,
+#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+# ##### END GPL LICENSE BLOCK #####
+
+# Runs on buildbot slave, creating a release package using the build
+# system and zipping it into buildbot_upload.zip. This is then uploaded
+# to the master in the next buildbot step.
+
+import os
+import subprocess
+import sys
+import zipfile
+
+# find release directory
+def find_release_directory():
+    for d in os.listdir('.'):
+        if os.path.isdir(d):
+            rd = os.path.join(d, 'release')
+            if os.path.exists(rd):
+                return rd
+    
+    return None
+
+# clean release directory if it already exists
+dir = find_release_directory()
+
+if dir:
+    for f in os.listdir(dir):
+        if os.path.isfile(os.path.join(dir, f)):
+            os.remove(os.path.join(dir, f))
+
+# create release package
+try:
+    os.chdir('../blender')
+    subprocess.call(['make', 'package_archive'])
+    os.chdir('../build')
+except Exception, ex:
+    sys.stderr.write('Make package release failed' + str(ex) + '\n')
+    sys.exit(1)
+
+# find release directory, must exist this time
+dir = find_release_directory()
+
+if not dir:
+    sys.stderr.write("Failed to find release directory.\n")
+    sys.exit(1)
+
+# find release package
+file = None
+filepath = None
+
+for f in os.listdir(dir):
+    rf = os.path.join(dir, f)
+    if os.path.isfile(rf) and f.startswith('blender'):
+        file = f
+        filepath = rf
+
+if not file:
+    sys.stderr.write("Failed to find release package.\n")
+    sys.exit(1)
+
+# create zip file
+try:
+    upload_zip = "buildbot_upload.zip"
+    if os.path.exists(upload_zip):
+        os.remove(upload_zip)
+    z = zipfile.ZipFile(upload_zip, "w", compression=zipfile.ZIP_STORED)
+    z.write(filepath, arcname=file)
+    z.close()
+except Exception, ex:
+    sys.stderr.write('Create buildbot_upload.zip failed' + str(ex) + '\n')
+    sys.exit(1)
+