From e24dee435e263cfc7e6265eb3c8f3350a5d7f781 Mon Sep 17 00:00:00 2001 From: Sergey Sharybin Date: Fri, 23 Feb 2018 12:37:05 +0100 Subject: Buildbot: Remove master configuration files Those are stored in blender-buildbot repository now, so having them in Blender only causes extra work to keep files in sync. --- build_files/buildbot/master.cfg | 368 ---------------------------------- build_files/buildbot/master_unpack.py | 151 -------------- 2 files changed, 519 deletions(-) delete mode 100644 build_files/buildbot/master.cfg delete mode 100644 build_files/buildbot/master_unpack.py (limited to 'build_files') diff --git a/build_files/buildbot/master.cfg b/build_files/buildbot/master.cfg deleted file mode 100644 index e3101430e50..00000000000 --- a/build_files/buildbot/master.cfg +++ /dev/null @@ -1,368 +0,0 @@ -# -*- python -*- -# ex: set syntax=python: - -# - -# List of the branches being built automatically overnight -NIGHT_SCHEDULE_BRANCHES = [None, "blender2.8"] - -# List of the branches available for force build -FORCE_SCHEDULE_BRANCHES = ["master", "blender2.8", "experimental-build"] - -""" -Stock Twisted directory lister doesn't provide any information about last file -modification time, we hack the class a bit in order to have such functionaliity -:) -""" - -from buildbot.status.web.base import DirectoryLister - - -def get_files_and_directories(self, directory): - from twisted.web.static import (getTypeAndEncoding, - formatFileSize) - import urllib - import cgi - import time - import os - files = [] - dirs = [] - for path in directory: - url = urllib.quote(path, "/") - escapedPath = cgi.escape(path) - lastmodified = time.ctime(os.path.getmtime( - os.path.join(self.path, path))) - if os.path.isdir(os.path.join(self.path, path)): - url = url + '/' - dirs.append({'text': escapedPath + "/", 'href': url, - 'size': '', 'type': '[Directory]', - 'encoding': '', - 'lastmodified': lastmodified}) - else: - mimetype, encoding = getTypeAndEncoding(path, self.contentTypes, - self.contentEncodings, - self.defaultType) - try: - size = os.stat(os.path.join(self.path, path)).st_size - except OSError: - continue - files.append({ - 'text': escapedPath, "href": url, - 'type': '[%s]' % mimetype, - 'encoding': (encoding and '[%s]' % encoding or ''), - 'size': formatFileSize(size), - 'lastmodified': lastmodified}) - return dirs, files -DirectoryLister._getFilesAndDirectories = get_files_and_directories - -# Dictionary that the buildmaster pays attention to. -c = BuildmasterConfig = {} - -# BUILD SLAVES -# -# We load the slaves and their passwords from a separator file, so we can have -# this one in SVN. - -from buildbot.buildslave import BuildSlave -import master_private - -c['slaves'] = [] - -for slave in master_private.slaves: - c['slaves'].append(BuildSlave(slave['name'], slave['password'])) - -# TCP port through which slaves connect - -c['slavePortnum'] = 9989 - -# CHANGE SOURCES - -from buildbot.changes.svnpoller import SVNPoller -from buildbot.changes.gitpoller import GitPoller - -c['change_source'] = GitPoller( - 'git://git.blender.org/blender.git', - pollinterval=1200) - - -# CODEBASES -# -# Allow to control separately things like branches for each repo and submodules. - -all_repositories = { - r'git://git.blender.org/blender.git': 'blender', - r'git://git.blender.org/blender-translations.git': 'blender-translations', - r'git://git.blender.org/blender-addons.git': 'blender-addons', - r'git://git.blender.org/blender-addons-contrib.git': 'blender-addons-contrib', - r'git://git.blender.org/blender-dev-tools.git': 'blender-dev-tools', - r'https://svn.blender.org/svnroot/bf-blender/': 'lib svn', -} - - -def codebaseGenerator(chdict): - return all_repositories[chdict['repository']] - -c['codebaseGenerator'] = codebaseGenerator - - -# SCHEDULERS -# -# Decide how to react to incoming changes. - -# from buildbot.scheduler import Scheduler -from buildbot.schedulers import timed, forcesched - -c['schedulers'] = [] - - -def schedule_force_build(name): - c['schedulers'].append(forcesched.ForceScheduler(name='force ' + name, - builderNames=[name], - codebases=[forcesched.CodebaseParameter( - codebase="blender", - branch=forcesched.ChoiceStringParameter( - name="branch", choices=FORCE_SCHEDULE_BRANCHES, default="master"), - # Do not hide revision, can be handy! - repository=forcesched.FixedParameter(name="repository", default="", hide=True), - project=forcesched.FixedParameter(name="project", default="", hide=True)), - # For now, hide other codebases. - forcesched.CodebaseParameter(hide=True, codebase="blender-translations"), - forcesched.CodebaseParameter( - codebase="blender-addons", - branch=forcesched.ChoiceStringParameter( - name="branch", choices=["master", "blender2.8"], default="master"), - repository=forcesched.FixedParameter(name="repository", default="", hide=True), - project=forcesched.FixedParameter(name="project", default="", hide=True), - revision=forcesched.FixedParameter(name="revision", default="", hide=True), - ), - forcesched.CodebaseParameter(hide=True, codebase="blender-addons-contrib"), - forcesched.CodebaseParameter(hide=True, codebase="blender-dev-tools"), - forcesched.CodebaseParameter(hide=True, codebase="lib svn")], - properties=[])) - - -def schedule_build(name, hour, minute=0): - for current_branch in NIGHT_SCHEDULE_BRANCHES: - scheduler_name = "nightly " + name - if current_branch: - scheduler_name += ' ' + current_branch - # Use special addons submodule branch when building blender2.8 branch. - addons_branch = "master" - if current_branch == "blender2.8": - addons_branch = "blender2.8" - c['schedulers'].append(timed.Nightly(name=scheduler_name, - codebases={ - "blender": {"repository": ""}, - "blender-translations": {"repository": "", "branch": "master"}, - "blender-addons": {"repository": "", "branch": addons_branch}, - "blender-addons-contrib": {"repository": "", "branch": "master"}, - "blender-dev-tools": {"repository": "", "branch": "master"}, - "lib svn": {"repository": "", "branch": "trunk"}}, - branch=current_branch, - builderNames=[name], - hour=hour, - minute=minute)) - - -# BUILDERS -# -# The 'builders' list defines the Builders, which tell Buildbot how to -# perform a build: what steps, and which slaves can execute them. -# Note that any particular build will only take place on one slave. - -from buildbot.process.factory import BuildFactory -from buildbot.process.properties import Interpolate -from buildbot.steps.source import SVN -from buildbot.steps.source import Git -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shell import Compile -from buildbot.steps.shell import Test -from buildbot.steps.transfer import FileUpload -from buildbot.steps.master import MasterShellCommand -from buildbot.config import BuilderConfig - -# add builder utility - -c['builders'] = [] -buildernames = [] - - -def add_builder(c, name, libdir, factory, branch='', - rsync=False, hour=3, minute=0): - slavenames = [] - - for slave in master_private.slaves: - if name in slave['builders']: - slavenames.append(slave['name']) - - if len(slavenames) > 0: - f = factory(name, libdir, branch, rsync) - c['builders'].append(BuilderConfig(name=name, - slavenames=slavenames, - factory=f, - category='blender')) - buildernames.append(name) - - schedule_build(name, hour, minute) - schedule_force_build(name) - -# common steps - - -def git_submodule_step(submodule): - return Git(name=submodule + '.git', - repourl='git://git.blender.org/' + submodule + '.git', - mode='update', - codebase=submodule, - workdir=submodule + '.git') - - -def git_step(branch=''): - if branch: - return Git(name='blender.git', - repourl='git://git.blender.org/blender.git', - mode='update', - branch=branch, - codebase='blender', - workdir='blender.git', - submodules=True) - else: - return Git(name='blender.git', - repourl='git://git.blender.org/blender.git', - mode='update', - codebase='blender', - workdir='blender.git', - submodules=True) - - -def git_submodules_update(): - command = ['git', 'submodule', 'update', '--remote'] - return ShellCommand(name='Submodules Update', - command=command, - description='updating', - descriptionDone='up to date', - workdir='blender.git') - - -def lib_svn_step(dir): - return SVN(name='lib svn', - baseURL='https://svn.blender.org/svnroot/bf-blender/%%BRANCH%%/lib/' + dir, - codebase='lib svn', - mode='update', - defaultBranch='trunk', - workdir='lib/' + dir) - - -def rsync_step(id, branch, rsync_script): - return ShellCommand(name='rsync', - command=['python', rsync_script, id, branch], - description='uploading', - descriptionDone='uploaded', - workdir='install') - -# generic builder - - -def generic_builder(id, libdir='', branch='', rsync=False): - filename = 'uploaded/buildbot_upload_' + id + '.zip' - compile_script = '../blender.git/build_files/buildbot/slave_compile.py' - test_script = '../blender.git/build_files/buildbot/slave_test.py' - pack_script = '../blender.git/build_files/buildbot/slave_pack.py' - rsync_script = '../blender.git/build_files/buildbot/slave_rsync.py' - unpack_script = 'master_unpack.py' - - f = BuildFactory() - if libdir != '': - f.addStep(lib_svn_step(libdir)) - - for submodule in ('blender-translations', - 'blender-addons', - 'blender-addons-contrib', - 'blender-dev-tools'): - f.addStep(git_submodule_step(submodule)) - - f.addStep(git_step(branch)) - f.addStep(git_submodules_update()) - - f.addStep(Compile(command=['python', compile_script, id], timeout=3600)) - f.addStep(Test(command=['python', test_script, id])) - f.addStep(ShellCommand(name='package', - command=['python', pack_script, id, branch or Interpolate('%(src:blender:branch)s')], - description='packaging', - descriptionDone='packaged')) - if rsync: - f.addStep(rsync_step(id, branch, rsync_script)) - else: - f.addStep(FileUpload(name='upload', - slavesrc='buildbot_upload.zip', - masterdest=filename, - maxsize=180 * 1024 * 1024, - workdir='install')) - f.addStep(MasterShellCommand(name='unpack', - command=['python2.7', unpack_script, filename], - description='unpacking', - descriptionDone='unpacked')) - return f - -# Builders - -add_builder(c, 'mac_x86_64_10_9_cmake', 'darwin', generic_builder, hour=1) -add_builder(c, 'linux_glibc219_i686_cmake', '', generic_builder, hour=2) -add_builder(c, 'linux_glibc219_x86_64_cmake', '', generic_builder, hour=1) -add_builder(c, 'win32_cmake_vc2013', 'windows_vc12', generic_builder, hour=1) -add_builder(c, 'win64_cmake_vc2013', 'win64_vc12', generic_builder, hour=2) -add_builder(c, 'win32_cmake_vc2015', 'windows_vc14', generic_builder, hour=3) -add_builder(c, 'win64_cmake_vc2015', 'win64_vc14', generic_builder, hour=4) - -# STATUS TARGETS -# -# 'status' is a list of Status Targets. The results of each build will be -# pushed to these targets. buildbot/status/*.py has a variety to choose from, -# including web pages, email senders, and IRC bots. - -c['status'] = [] - -from buildbot.status import html -from buildbot.status.web import authz -from buildbot.status.web import auth - -users = [] -for slave in master_private.slaves: - users += [(slave['name'], slave['password'])] - -authz_cfg = authz.Authz( - auth=auth.BasicAuth(users), - # change any of these to True to enable; see the manual for more - # options - gracefulShutdown=False, - forceBuild=True, # use this to test your slave once it is set up - forceAllBuilds=False, - pingBuilder=False, - stopBuild=True, - stopAllBuilds=False, - cancelPendingBuild=True, -) - -c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) -#c['status'].append(html.WebStatus(http_port=8010)) - -# PROJECT IDENTITY - -c['projectName'] = "Blender" -c['projectURL'] = "http://www.blender.org" - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server (usually the html.WebStatus page) is visible. This -# typically uses the port number set in the Waterfall 'status' entry, but -# with an externally-visible host name which the buildbot cannot figure out -# without some help. - -c['buildbotURL'] = "http://builder.blender.org/" - -# DB URL -# -# This specifies what database buildbot uses to store change and scheduler -# state. You can leave this at its default for all but the largest -# installations. - -c['db_url'] = "sqlite:///state.sqlite" diff --git a/build_files/buildbot/master_unpack.py b/build_files/buildbot/master_unpack.py deleted file mode 100644 index 157e244e210..00000000000 --- a/build_files/buildbot/master_unpack.py +++ /dev/null @@ -1,151 +0,0 @@ -# ##### BEGIN GPL LICENSE BLOCK ##### -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# ##### END GPL LICENSE BLOCK ##### - -# Runs on Buildbot master, to unpack incoming unload.zip into latest -# builds directory and remove older builds. - -# - -import os -import shutil -import sys -import zipfile - - -# extension stripping -def strip_extension(filename): - extensions = '.zip', '.tar', '.bz2', '.gz', '.tgz', '.tbz', '.exe' - - for ext in extensions: - if filename.endswith(ext): - filename = filename[:-len(ext)] - - return filename - - -# extract platform from package name -def get_platform(filename): - # name is blender-version-platform.extension. we want to get the - # platform out, but there may be some variations, so we fiddle a - # bit to handle current and hopefully future names - filename = strip_extension(filename) - filename = strip_extension(filename) - - tokens = filename.split("-") - platforms = ('osx', 'mac', 'bsd', - 'win', 'linux', 'source', - 'irix', 'solaris') - platform_tokens = [] - found = False - - for i, token in enumerate(tokens): - if not found: - for platform in platforms: - if platform in token.lower(): - found = True - break - - if found: - platform_tokens += [token] - - return '-'.join(platform_tokens) - - -def get_branch(filename): - if filename.startswith("blender-2.8"): - return "blender2.8" - - tokens = filename.split("-") - branch = "" - - for token in tokens: - if token == "blender": - return branch - - if branch == "": - branch = token - else: - branch = branch + "-" + token - - return "" - -# get filename -if len(sys.argv) < 2: - sys.stderr.write("Not enough arguments, expecting file to unpack\n") - sys.exit(1) - -filename = sys.argv[1] - -# open zip file -if not os.path.exists(filename): - sys.stderr.write("File %r not found.\n" % filename) - sys.exit(1) - -try: - z = zipfile.ZipFile(filename, "r") -except Exception as ex: - sys.stderr.write('Failed to open zip file: %s\n' % str(ex)) - sys.exit(1) - -if len(z.namelist()) != 1: - sys.stderr.write("Expected one file in %r." % filename) - sys.exit(1) - -package = z.namelist()[0] -packagename = os.path.basename(package) - -# detect platform and branch -platform = get_platform(packagename) -branch = get_branch(packagename) - -if platform == '': - sys.stderr.write('Failed to detect platform ' + - 'from package: %r\n' % packagename) - sys.exit(1) - -# extract -if not branch or branch == 'master': - directory = 'public_html/download' -elif branch == 'experimental-build': - directory = 'public_html/download/experimental' -else: - directory = 'public_html/download' - -try: - filename = os.path.join(directory, packagename) - zf = z.open(package) - f = file(filename, "wb") - - shutil.copyfileobj(zf, f) - os.chmod(filename, 0644) - - zf.close() - z.close() -except Exception as ex: - sys.stderr.write('Failed to unzip package: %s\n' % str(ex)) - sys.exit(1) - -# remove other files from the same platform and branch -try: - for f in os.listdir(directory): - if get_platform(f) == platform and get_branch(f) == branch: - if f != packagename: - os.remove(os.path.join(directory, f)) -except Exception as ex: - sys.stderr.write('Failed to remove old packages: %s\n' % str(ex)) - sys.exit(1) -- cgit v1.2.3