diff options
author | Jeroen Bakker <jeroen@blender.org> | 2020-07-03 17:19:50 +0300 |
---|---|---|
committer | Jeroen Bakker <jeroen@blender.org> | 2020-07-03 17:19:50 +0300 |
commit | 2cbc328bdcf7a48243344b8b33914a43b497df23 (patch) | |
tree | ec640f4100f94069e687f077b1d4e7db7a65588f | |
parent | 314783f2e7701913fa3adb3a4704f9d57b2ac62e (diff) |
Build bot release pipelinebuildbot-lts
* downloads builds from builder.blender.org
* generates checksums (still need to cleanup filenames)
* create source archive
* upload to download.blender.org
Next steps
* Next step is to place them in the correct folder
* file permissions download.blender.org
* snap package
* steam packages
* release notes
TODO:
* add more generic functions in master.cfg
* use a separate folder per build
-rw-r--r-- | build_files/buildbot-lts/README.md | 19 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/Dockerfile | 36 | ||||
-rwxr-xr-x | build_files/buildbot-lts/docker/bin/create_checksum.sh | 4 | ||||
-rwxr-xr-x | build_files/buildbot-lts/docker/bin/upload_file.sh | 7 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/buildbot.tac | 20 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/docker-compose.yml | 36 | ||||
-rwxr-xr-x | build_files/buildbot-lts/docker/docker-entrypoint.sh | 18 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/master.cfg | 240 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/worker/Dockerfile | 27 | ||||
-rw-r--r-- | build_files/buildbot-lts/docker/worker/buildbot.tac | 39 |
10 files changed, 446 insertions, 0 deletions
diff --git a/build_files/buildbot-lts/README.md b/build_files/buildbot-lts/README.md index 73750be84f6..ec2a96c4652 100644 --- a/build_files/buildbot-lts/README.md +++ b/build_files/buildbot-lts/README.md @@ -13,3 +13,22 @@ private keys and the process needs to be controlled security wise. But of course the source and configurations are public available for anyone to check, develop and use. + +Setting up build-bot +-------------------- + +instructions from https://github.com/cjolowicz/docker-buildbot. + +Create custom buildbot worker containing packages we need for building (git, wget). + + cd docker + docker build -t buildbot --no-cache . + cd worker + docker build -t buildbot-worker --no-cache . + + docker network create net-buildbot + + docker rm lts-buildbot && docker run --init --name lts-buildbot --network net-buildbot --publish=127.0.0.1:8010:8010 -t -i buildbot + + docker rm lts-worker && docker run --init --name lts-worker --network net-buildbot --name lts-worker -e BUILDMASTER=lts-buildbot -e WORKERNAME=lts-worker -e WORKERPASS=secret -t -i buildbot-worker + diff --git a/build_files/buildbot-lts/docker/Dockerfile b/build_files/buildbot-lts/docker/Dockerfile new file mode 100644 index 00000000000..bf433f1afee --- /dev/null +++ b/build_files/buildbot-lts/docker/Dockerfile @@ -0,0 +1,36 @@ +FROM ubuntu:18.04 + +RUN apt update && apt upgrade -y +RUN apt install -y \ + gosu \ + wget \ + openssh-client \ + build-essential \ + libffi-dev \ + libssl-dev \ + python3-dev \ + python3-pip + + +ENV BUILDBOT_VERSION 2.5.0 +RUN pip3 --no-cache-dir install --upgrade pip && pip --no-cache-dir install \ + buildbot[bundle,tls]==$BUILDBOT_VERSION \ + buildbot-docker-swarm-worker + +COPY buildbot.tac /var/lib/buildbot/ +COPY docker-entrypoint.sh /usr/local/bin/ +COPY master.cfg /etc/buildbot/ +COPY bin/create_checksum.sh /var/lib/buildbot/bin/ +COPY bin/upload_file.sh /var/lib/buildbot/bin/ +COPY .ssh/id_rsa /var/lib/buildbot/.ssh/ + +RUN adduser --home /var/lib/buildbot --disabled-password --gecos '' buildbot +WORKDIR /var/lib/buildbot +RUN ln -s /etc/buildbot/master.cfg + +VOLUME /var/lib/buildbot +EXPOSE 8010 +EXPOSE 9989 + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/bin/create_checksum.sh b/build_files/buildbot-lts/docker/bin/create_checksum.sh new file mode 100755 index 00000000000..fe69e43e5db --- /dev/null +++ b/build_files/buildbot-lts/docker/bin/create_checksum.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +md5sum $1 > $2.md5 +sha256sum $1 > $2.sha256
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/bin/upload_file.sh b/build_files/buildbot-lts/docker/bin/upload_file.sh new file mode 100755 index 00000000000..95fb71388ec --- /dev/null +++ b/build_files/buildbot-lts/docker/bin/upload_file.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e +if [ ! -f /var/lib/buildbot/.ssh/known_hosts ] +then + ssh-keyscan download.blender.org > /var/lib/buildbot/.ssh/known_hosts +fi +scp $1 jeroen@download.blender.org:$2
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/buildbot.tac b/build_files/buildbot-lts/docker/buildbot.tac new file mode 100644 index 00000000000..930d392cc50 --- /dev/null +++ b/build_files/buildbot-lts/docker/buildbot.tac @@ -0,0 +1,20 @@ +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot.master import BuildMaster + +basedir = os.environ.get("BUILDBOT_BASEDIR", + os.path.abspath(os.path.dirname(__file__))) +configfile = 'master.cfg' + +# note: this line is matched against to check that this is a buildmaster +# directory; do not edit it. +application = service.Application('buildmaster') +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) + +m = BuildMaster(basedir, configfile, umask=None) +m.setServiceParent(application)
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/docker-compose.yml b/build_files/buildbot-lts/docker/docker-compose.yml new file mode 100644 index 00000000000..cf0db3f16c4 --- /dev/null +++ b/build_files/buildbot-lts/docker/docker-compose.yml @@ -0,0 +1,36 @@ +version: '1' +services: + buildbot: + image: buildbot/buildbot-master:master + env_file: + - db.env + environment: + - BUILDBOT_CONFIG_DIR=config + - BUILDBOT_CONFIG_URL=https://github.com/buildbot/buildbot-docker-example-config/archive/master.tar.gz + - BUILDBOT_WORKER_PORT=9989 + - BUILDBOT_WEB_URL=http://localhost:8010/ + - BUILDBOT_WEB_PORT=tcp:port=8010 + links: + - db + depends_on: + - db + ports: + - "8010:8010" + db: + env_file: + - db.env + image: "postgres:9.4" + expose: + - 5432 + + worker: + image: "buildbot/buildbot-worker:master" + environment: + BUILDMASTER: buildbot + BUILDMASTER_PORT: 9989 + WORKERNAME: lts-worker + WORKERPASS: pass + WORKER_ENVIRONMENT_BLACKLIST: DOCKER_BUILDBOT* BUILDBOT_ENV_* BUILDBOT_1* WORKER_ENVIRONMENT_BLACKLIST + + links: + - buildbot
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/docker-entrypoint.sh b/build_files/buildbot-lts/docker/docker-entrypoint.sh new file mode 100755 index 00000000000..b7298560662 --- /dev/null +++ b/build_files/buildbot-lts/docker/docker-entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +buildbot upgrade-master . + +chown buildbot /var/lib/buildbot/ +chown buildbot /var/lib/buildbot/.ssh/ +chown -R buildbot /var/lib/buildbot/* + +if [ -S /var/run/docker.sock ] +then + group=$(stat -c '%g' /var/run/docker.sock) +else + group=buildbot +fi + +gosu buildbot:$group "$@"
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/master.cfg b/build_files/buildbot-lts/docker/master.cfg new file mode 100644 index 00000000000..1dfa8bbdf7c --- /dev/null +++ b/build_files/buildbot-lts/docker/master.cfg @@ -0,0 +1,240 @@ +# -*- python -*- +# ex: set filetype=python: + +import os + +from buildbot.plugins import * + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. + +c['workers'] = [worker.Worker("lts-worker", 'secret')] + +if 'BUILDBOT_MQ_URL' in os.environ: + c['mq'] = { + 'type' : 'wamp', + 'router_url': os.environ['BUILDBOT_MQ_URL'], + 'realm': os.environ.get('BUILDBOT_MQ_REALM', 'buildbot').decode('utf-8'), + 'debug' : 'BUILDBOT_MQ_DEBUG' in os.environ, + 'debug_websockets' : 'BUILDBOT_MQ_DEBUG' in os.environ, + 'debug_lowlevel' : 'BUILDBOT_MQ_DEBUG' in os.environ, + } +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': os.environ.get("BUILDBOT_WORKER_PORT", 9989)}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. We don't build when sources change. +c['change_source'] = [] + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = [ + schedulers.ForceScheduler( + name="build", + builderNames=["release-blender-283"], + properties=[ + util.StringParameter( + name="blender_version", + label="Version of blender.", + default="2.83"), + util.StringParameter( + name="blender_version_full", + label="Version of blender (Full).", + default="2.83.1") + ] + ), + + schedulers.Triggerable( + name="deploy-source-archive", + builderNames=["deploy-source-archive"] + ), + + schedulers.Triggerable( + name="deploy-buildbot-packages", + builderNames=["deploy-buildbot-packages"] + ), +] + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + +deploy_source_archive_factory = util.BuildFactory() +deploy_source_archive_factory.addStep(steps.Git(name="Checkout Blender Source", repourl='https://git.blender.org/blender.git', branch=util.Interpolate('blender-v%(prop:blender_version)s-release'), mode='full')) +deploy_source_archive_factory.addStep(steps.ShellSequence(name="Make source archive", commands=[ + util.ShellArg(command=["make", "source_archive"], logfile='make_source_archive'), + # The next two shell commands are only needed during development + util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz')]), + util.ShellArg(command=["mv", "blender-2.83.2-beta.tar.xz.md5sum", util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum')]), +])) +deploy_source_archive_factory.addStep(steps.FileUpload( + name="Upload source archive", + workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz'), + masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz'))) +deploy_source_archive_factory.addStep(steps.FileUpload( + name="Upload source archive checksum", + workersrc=util.Interpolate('blender-v%(prop:blender_version_full)s.tar.xz.md5sum'), + masterdest=util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum'))) +deploy_source_archive_factory.addStep(steps.MasterShellCommand( + name="Upload source archive", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_source_archive_factory.addStep(steps.MasterShellCommand( + name="Upload windows64.zip build", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-v%(prop:blender_version_full)s.tar.xz.md5sum'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) + +deploy_buildbot_packages_factory = util.BuildFactory() +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Download linux64 build", + command=[ + 'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-linux64.tar.xz'), + "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Download windows64.msi build", + command=[ + 'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.msi'), + "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Download windows64.zip build", + command=[ + 'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-windows64.zip'), + "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Download macOS build", + command=[ + 'wget', util.Interpolate('https://builder.blender.org/download/blender-%(prop:blender_version_full)s-macOS.dmg'), + "-O", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Create checksum (md5/sha256)", + command=[ + 'sh', + '-C', + '/var/lib/buildbot/bin/create_checksum.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-*'), + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s'), +])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload linux64 build", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-linux64.tar.xz'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload windows64.msi build", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.msi'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload windows64.zip build", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-windows64.zip'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload macOS build", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s-macOS.dmg'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload md5 checksum", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.md5'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) +deploy_buildbot_packages_factory.addStep(steps.MasterShellCommand( + name="Upload sha256 checksum", + command=[ + 'sh', '-C', '/var/lib/buildbot/bin/upload_file.sh', + util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s/blender-%(prop:blender_version_full)s.sha256'), + util.Interpolate('/data/www/vhosts/download.blender.org/ftp/jeroen/')])) + +factory = util.BuildFactory() +factory.addStep(steps.MasterShellCommand(name="Init release folder", command=["mkdir", "-p", util.Interpolate('/var/lib/buildbot/builds//blender-v%(prop:blender_version_full)s')])) +factory.addStep(steps.Trigger( + name='Deploy', + schedulerNames=[ + 'deploy-source-archive', + 'deploy-buildbot-packages', + ], + waitForFinish=True, + set_properties={ + 'blender_version_full': util.Property('blender_version_full'), + 'blender_version': util.Property('blender_version'), + } +)) + +# Download builds from builder.blender.org +c['builders'] = [ + util.BuilderConfig(name="release-blender-283", + workernames=["lts-worker"], + factory=factory), + util.BuilderConfig(name="deploy-source-archive", + workernames=["lts-worker"], + factory=deploy_source_archive_factory), + util.BuilderConfig(name="deploy-buildbot-packages", + workernames=["lts-worker"], + factory=deploy_buildbot_packages_factory), +] + +####### STATUS TARGETS + +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# like IRC bots. + +# c['status'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = "Blender Release LTS" +c['titleURL'] = "https://www.blender.org/download/lts/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = os.environ.get("BUILDBOT_WEB_URL", "http://localhost:8010/") + +# minimalistic config to activate new web UI +c['www'] = dict(port=os.environ.get("BUILDBOT_WEB_PORT", 8010), + plugins=dict(waterfall_view={}, console_view={})) + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + 'db_url' : os.environ.get("BUILDBOT_DB_URL", "sqlite://").format(**os.environ), +}
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/worker/Dockerfile b/build_files/buildbot-lts/docker/worker/Dockerfile new file mode 100644 index 00000000000..dbecf4ea9ba --- /dev/null +++ b/build_files/buildbot-lts/docker/worker/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:18.04 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + build-essential \ + git \ + wget \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + ; \ + rm -rf /var/lib/apt/lists/* + +ENV BUILDBOT_VERSION 2.5.0 +RUN pip3 install --upgrade pip && \ + pip --no-cache-dir install twisted[tls] && \ + pip --no-cache-dir install buildbot_worker==$BUILDBOT_VERSION + +RUN useradd --create-home --home-dir=/var/lib/buildbot buildbot +WORKDIR /var/lib/buildbot +USER buildbot + +COPY buildbot.tac . + +CMD ["twistd", "--pidfile=", "--nodaemon", "--python=buildbot.tac"]
\ No newline at end of file diff --git a/build_files/buildbot-lts/docker/worker/buildbot.tac b/build_files/buildbot-lts/docker/worker/buildbot.tac new file mode 100644 index 00000000000..c32355a4ca4 --- /dev/null +++ b/build_files/buildbot-lts/docker/worker/buildbot.tac @@ -0,0 +1,39 @@ +import fnmatch +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot_worker.bot import Worker + +# setup worker +basedir = os.path.abspath(os.path.dirname(__file__)) +application = service.Application('buildbot-worker') + + +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) +# and worker on the same process! +buildmaster_host = os.environ.get("BUILDMASTER", 'localhost') +port = int(os.environ.get("BUILDMASTER_PORT", 9989)) +workername = os.environ.get("WORKERNAME", 'docker') +passwd = os.environ.get("WORKERPASS") + +# delete the password from the environ so that it is not leaked in the log +blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split() +for name in list(os.environ.keys()): + for toremove in blacklist: + if fnmatch.fnmatch(name, toremove): + del os.environ[name] + +keepalive = 600 +umask = None +maxdelay = 300 +allow_shutdown = None +maxretries = 10 + +s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown, maxRetries=maxretries) +s.setServiceParent(application)
\ No newline at end of file |