#!/usr/bin/python -u -OO import os from optparse import OptionParser from bockbuild.util.util import * from bockbuild.util.csproj import * from bockbuild.environment import Environment from bockbuild.package import * from bockbuild.profile import Profile import collections import hashlib import itertools import traceback from collections import namedtuple ProfileDesc = namedtuple ('Profile', 'name description path modes') global active_profile, bockbuild active_profile = None bockbuild = None def find_profiles (base_path): assert Profile.loaded == None search_path = first_existing(['%s/bockbuild' % base_path, '%s/packaging' % base_path]) sys.path.append(search_path) profiles = [] resolved_names = [] while True: progress_made = False for path in iterate_dir (search_path, with_dirs=True): file = '%s/profile.py' % path if os.path.isdir (path) and os.path.isfile (file): name = os.path.basename (path) if name in resolved_names: continue fail = None profile = None try: execfile(file, globals()) if not Profile.loaded: fail = 'No profile loaded' profile = Profile.loaded except Exception as e: fail = e finally: Profile.loaded = None if not fail: profile = Profile.loaded Profile.loaded = None progress_made = True description = "" if hasattr(profile.__class__, 'description'): description = profile.__class__.description profiles.append (ProfileDesc (name = name, description = description, path = path, modes = "")) resolved_names.append(name) else: warn(fail) if not progress_made: break assert Profile.loaded == None return profiles class Bockbuild: def run(self): self.name = 'bockbuild' self.root = os.path.dirname (os.path.abspath(__file__)) # Bockbuild system root self.execution_root = os.getcwd() self.resources = set([os.path.realpath( os.path.join(self.root, 'packages'))]) # list of paths on where to look for packages, patches, etc. config.state_root = self.root # root path for all storage; artifacts, build I/O, cache, storage and output config.protected_git_repos.append (self.root) config.absolute_root = os.path.commonprefix([self.root, self.execution_root]) self.build_root = os.path.join(config.state_root, 'builds') self.staged_prefix = os.path.join(config.state_root, 'stage') self.toolchain_root = os.path.join(config.state_root, 'toolchain') self.artifact_root = os.path.join(config.state_root, 'artifacts') self.package_root = os.path.join(config.state_root, 'distribution') self.scratch = os.path.join(config.state_root, 'scratch') self.logs = os.path.join(config.state_root, 'logs') self.env_file = os.path.join(config.state_root, 'last-successful-build.env') self.source_cache = os.getenv('BOCKBUILD_SOURCE_CACHE') or os.path.realpath( os.path.join(config.state_root, 'cache')) self.cpu_count = get_cpu_count() self.host = get_host() self.uname = backtick('uname -a') self.full_rebuild = False self.toolchain = [] find_git(self) self.bockbuild_rev = git_shortid(self, self.root) self.profile_root = git_rootdir (self, self.execution_root) self.profiles = find_profiles (self.profile_root) for profile in self.profiles: self.resources.add(profile.path) loginit('bockbuild (%s)' % (self.bockbuild_rev)) info('cmd: %s' % ' '.join(sys.argv)) if len (sys.argv) < 2: info ('Profiles in %s --' % self.git ('config --get remote.origin.url', self.profile_root)[0]) info(map (lambda x: '\t%s: %s' % (x.name, x.description), self.profiles)) finish (exit_codes.FAILURE) global active_profile Package.profile = active_profile = self.load_profile (sys.argv[1]) self.parser = self.init_parser() self.cmd_options, self.cmd_args = self.parser.parse_args(sys.argv[2:]) self.packages_to_build = self.cmd_args or active_profile.packages active_profile.setup() self.verbose = self.cmd_options.verbose config.verbose = self.cmd_options.verbose self.arch = self.cmd_options.arch self.unsafe = self.cmd_options.unsafe config.trace = self.cmd_options.trace self.tracked_env = [] ensure_dir(self.source_cache, purge=False) ensure_dir(self.artifact_root, purge=False) ensure_dir(self.build_root, purge=False) ensure_dir(self.scratch, purge=True) ensure_dir(self.logs, purge=False) self.build() def init_parser(self): parser = OptionParser( usage='usage: %prog [options] [package_names...]') parser.add_option('--build', action='store_true', dest='do_build', default=True, help='build the profile') parser.add_option('--package', action='store_true', dest='do_package', default=False, help='package the profile') parser.add_option('--verbose', action='store_true', dest='verbose', default=False, help='show all build output (e.g. configure, make)') parser.add_option('-d', '--debug', default=False, action='store_true', dest='debug', help='Build with debug flags enabled') parser.add_option('-e', '--environment', default=False, action='store_true', dest='dump_environment', help='Dump the profile environment as a shell-sourceable list of exports ') parser.add_option('-r', '--release', default=False, action='store_true', dest='release_build', help='Whether or not this build is a release build') parser.add_option('', '--csproj-env', default=False, action='store_true', dest='dump_environment_csproj', help='Dump the profile environment xml formarted for use in .csproj files') parser.add_option('', '--csproj-insert', default=None, action='store', dest='csproj_file', help='Inserts the profile environment variables into VS/MonoDevelop .csproj files') parser.add_option('', '--arch', default='default', action='store', dest='arch', help='Select the target architecture(s) for the package') parser.add_option('', '--shell', default=False, action='store_true', dest='shell', help='Get an shell with the package environment') parser.add_option('', '--unsafe', default=False, action='store_true', dest='unsafe', help='Prevents full rebuilds when a build environment change is detected. Useful for debugging.') parser.add_option('', '--trace', default=False, action='store_true', dest='trace', help='Enable tracing (for diagnosing bockbuild problems') return parser def build_distribution(self, packages, dest, stage, arch): # TODO: full relocation means that we shouldn't need dest at this stage build_list = [] stage_invalidated = False #if anything is dirty we flush the stageination path and fill it again if self.full_rebuild: ensure_dir (stage, purge = True) progress('Fetching packages') for package in packages.values(): package.build_artifact = os.path.join( self.artifact_root, '%s-%s' % (package.name, arch)) package.buildstring_file = package.build_artifact + '.buildstring' package.log = os.path.join(self.logs, package.name + '.log') package.source_dir_name = expand_macros(package.source_dir_name, package) workspace_path = os.path.join(self.build_root, package.source_dir_name) package.fetch(workspace_path) if self.full_rebuild: package.request_build('Full rebuild') elif not os.path.exists(package.build_artifact): package.request_build('No artifact') elif is_expired(package.build_artifact, config.artifact_lifespan_days): package.request_build('Artifact expired (older than %d days)' % config.artifact_lifespan_days) elif is_changed(package.buildstring, package.buildstring_file): package.request_build('Updated') if package.needs_build: build_list.append(package) stage_invalidated = True verbose('%d packages need building:' % len(build_list)) verbose(['%s (%s)' % (x.name, x.needs_build) for x in build_list]) if stage_invalidated: ensure_dir (stage, purge = True) for package in packages.values(): package.deploy_requests.append (stage) for package in packages.values(): if os.path.exists(package.log): delete(package.log) package.start_build(arch, dest, stage) # make artifact in scratch # delete artifact + buildstring with open(package.buildstring_file, 'w') as output: output.write('\n'.join(package.buildstring)) def build(self): profile = active_profile env = profile.env if self.cmd_options.dump_environment: env.compile() env.dump() sys.exit(0) if self.cmd_options.dump_environment_csproj: # specify to use our GAC, else MonoDevelop would # use its own env.set('MONO_GAC_PREFIX', self.staged_prefix) env.compile() env.dump_csproj() sys.exit(0) if self.cmd_options.csproj_file is not None: env.set('MONO_GAC_PREFIX', self.staged_prefix) env.compile() env.write_csproj(self.cmd_options.csproj_file) sys.exit(0) profile.toolchain_packages = collections.OrderedDict() for source in self.toolchain: package = self.load_package(source) profile.toolchain_packages[package.name] = package profile.release_packages = collections.OrderedDict() for source in self.packages_to_build: package = self.load_package(source) profile.release_packages[package.name] = package profile.setup_release() if self.track_env(): if self.unsafe: warn('Build environment changed, but overriding full rebuild!') else: info('Build environment changed, full rebuild triggered') self.full_rebuild = True ensure_dir(self.build_root, purge=True) if self.cmd_options.shell: title('Shell') self.shell() if self.cmd_options.do_build: title('Building toolchain') self.build_distribution( profile.toolchain_packages, self.toolchain_root, self.toolchain_root, arch='toolchain') title('Building release') self.build_distribution( profile.release_packages, profile.prefix, self.staged_prefix, arch=self.arch) # update env with open(self.env_file, 'w') as output: output.write('\n'.join(self.tracked_env)) if self.cmd_options.do_package: title('Packaging') protect_dir(self.staged_prefix) ensure_dir(self.package_root, True) run_shell('rsync -aPq %s/* %s' % (self.staged_prefix, self.package_root), False) unprotect_dir(self.package_root) profile.process_release(self.package_root) profile.package() finish(exit_codes.SUCCESS) def track_env(self): env = active_profile.env env.compile() env.export() self.env_script = os.path.join( self.root, self.profile_name) + '_env.sh' env.write_source_script(self.env_script) self.tracked_env.extend(env.serialize()) return is_changed(self.tracked_env, self.env_file) def load_package(self, source): if isinstance(source, Package): # package can already be loaded in the source list return source fullpath = None for i in self.resources: candidate_fullpath = os.path.join(i, source + '.py') if os.path.exists(candidate_fullpath): if fullpath is not None: error ('Package "%s" resolved in multiple locations (search paths: %s' % (source, self.resources)) fullpath = candidate_fullpath if not fullpath: error("Package '%s' not found ('search paths: %s')" % (source, self.resources)) Package.last_instance = None trace(fullpath) execfile(fullpath, globals()) if Package.last_instance is None: error('%s does not provide a valid package.' % source) new_package = Package.last_instance new_package._path = fullpath return new_package def load_profile(self, source): if Profile.loaded: error ('A profile is already loaded: %s' % Profile.loaded) path = None for profile in self.profiles: if profile.name == source: path = profile.path if path == None: if isinstance(source, Profile): # package can already be loaded in the source list Profile.loaded = source else: error("Profile '%s' not found" % source) fullpath = os.path.join(path, 'profile.py') if not os.path.exists(fullpath): error("Profile '%s' not found" % source) sys.path.append (path) self.resources.add (path) execfile(fullpath, globals()) Profile.loaded.attach (self) if Profile.loaded is None: error('%s does not provide a valid profile (developers: ensure Profile.attach() is called.)' % source) if Profile.loaded.bockbuild is None: error ('Profile init is invalid: Failed to attach to bockbuild object') new_profile = Profile.loaded new_profile._path = fullpath new_profile.directory = path new_profile.git_root = git_rootdir (self, os.path.dirname (path)) config.protected_git_repos.append (new_profile.git_root) self.profile_name = source return new_profile if __name__ == "__main__": try: bockbuild = Bockbuild() bockbuild.run() except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() error('%s (%s)' % (e, exc_type.__name__), more_output=True) error(('%s:%s @%s\t\t"%s"' % p for p in traceback.extract_tb( exc_traceback)[-5:])) except KeyboardInterrupt: error('Interrupted.') finally: if config.exit_code == exit_codes.NOTSET: print 'spurious sys.exit() call' if config.exit_code == exit_codes.SUCCESS: logprint('\n** %s **\n' % 'Goodbye!', bcolors.BOLD) sys.exit (config.exit_code)