From acc46d974b9d5aedb93f984e4bf3b8ecb6d41d9d Mon Sep 17 00:00:00 2001 From: Jon Turney Date: Mon, 6 Feb 2023 15:49:48 +0000 Subject: Always remove files from stagingdir We want different treatment for files which are manually uploaded (into homedir) and files which arrive from the build service deployer (into stagingdir) Broken manual uploads were left in place so they can be inspected and manually fixed by someone. Currently, broken build deploys will also hang around forever, but they are expected to be "correct", and the initator of the build has no access to the stagingdir to fix things. Instead make them get automatically removed. --- calm/calm.py | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/calm/calm.py b/calm/calm.py index 5d1753b..03dae45 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -161,7 +161,7 @@ def process_uploads(args, state): with logfilters.AttrFilter(maint=m.name): process_maintainer_uploads(args, state, all_packages, m, args.homedir, 'upload') - process_maintainer_uploads(args, state, all_packages, m, args.stagingdir, 'staging') + process_maintainer_uploads(args, state, all_packages, m, args.stagingdir, 'staging', scrub=True) # record updated reminder times for maintainers maintainers.update_reminder_times(mlist) @@ -169,14 +169,12 @@ def process_uploads(args, state): return state.packages -def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): - name = m.name - +def process_maintainer_uploads(args, state, all_packages, m, basedir, desc, scrub=False): # for each arch and noarch scan_result = {} - skip_maintainer = False + success = True for arch in common_constants.ARCHES + ['noarch', 'src'] + common_constants.ARCHIVED_ARCHES: - logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name)) + logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, m.name)) # read uploads scan_result[arch] = uploads.scan(basedir, m, all_packages, arch, args) @@ -185,18 +183,31 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): uploads.remove(args, scan_result[arch].remove_always) if scan_result[arch].error: - logging.error("error while reading uploaded arch %s packages from maintainer %s" % (arch, name)) - skip_maintainer = True + logging.error("error while reading uploaded arch %s packages from maintainer %s" % (arch, m.name)) + success = False continue + if success: + success = _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc) + + # remove upload files on success in homedir, always in stagingdir + for arch in common_constants.ARCHES + ['noarch', 'src']: + if scrub or success: + uploads.remove(args, scan_result[arch].remove_success) + + # clean up any empty directories + if not args.dryrun: + utils.rmemptysubdirs(os.path.join(basedir, m.name)) + + +def _process_maintainer_uploads(scan_result, args, state, all_packages, m, basedir, desc): + name = m.name + # if there are no added or removed files for this maintainer, we # don't have anything to do if not any([scan_result[a].to_relarea or scan_result[a].to_vault for a in scan_result]): logging.debug("nothing to do for maintainer %s" % (name)) - skip_maintainer = True - - if skip_maintainer: - return + return True # for each arch merged_packages = {} @@ -225,7 +236,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): # if an error occurred ... if not valid: # ... discard move list and merged_packages - return + return False # check for packages which are stale as a result of this upload, # which we will want in the same report @@ -236,7 +247,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): if not stale_to_vault: # ... discard move list and merged_packages logging.error("error while evaluating stale packages for %s" % (name)) - return + return False # check for conflicting movelists conflicts = False @@ -249,7 +260,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): if conflicts: # ... discard move list and merged_packages logging.error("error while validating movelists for %s" % (name)) - return + return False # for each arch and noarch for arch in common_constants.ARCHES + ['noarch', 'src']: @@ -259,10 +270,11 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): if scan_result[arch].to_vault: logging.info("vaulting %d package(s) for arch %s, by request" % (len(scan_result[arch].to_vault), arch)) scan_result[arch].to_vault.move_to_vault(args) - uploads.remove(args, scan_result[arch].remove_success) + if scan_result[arch].to_relarea: logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch)) scan_result[arch].to_relarea.move_to_relarea(m, args, desc) + # XXX: Note that there seems to be a separate process, not run # from cygwin-admin's crontab, which changes the ownership of # files in the release area to cyguser:cygwin @@ -279,10 +291,6 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): # use merged package list state.packages[arch] = merged_packages[arch] - # clean up any empty directories - if not args.dryrun: - utils.rmemptysubdirs(os.path.join(basedir, m.name)) - # report what we've done added = [] for arch in common_constants.ARCHES + ['noarch', 'src']: @@ -291,6 +299,8 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): logging.debug(msg) irk.irk("calm %s" % msg) + return True + # # -- cgit v1.2.3