diff options
author | Jon Turney <jon.turney@dronecode.org.uk> | 2021-05-05 16:08:46 +0300 |
---|---|---|
committer | Jon Turney <jon.turney@dronecode.org.uk> | 2022-07-01 14:52:51 +0300 |
commit | eca3a88d174516051883b5c7404f6985939d4b4e (patch) | |
tree | 9df03a2af7702155c0ccdbfd941bf4fb6947e753 | |
parent | 385eeb6a094e30aaaf224b9c5b4a025a09002726 (diff) |
Improve handling of old-style obsoletion packages
Identify old-style obsoletion packages (where the package is empty, has
a category of _obsolete, and requires: it's replacement), and upgrade
them to the new style (where the obsoleting package has an obsoletes:).
This is done per-arch, simply because lots of these historic obsoletion
packages only exist for x86, and it's redundant to obsolete: them on
x86_64.
Treating this similarly to past_mistakes.missing_obsoletes means that a
warning occurs when new version of package is missing that expected
obsolete:
Initially only apply to packages over a certain age, so we can observe
the effect on a small number of packages to make checking it's doing the
right thing easier.
-rwxr-xr-x | calm/calm.py | 12 | ||||
-rwxr-xr-x | calm/package.py | 92 | ||||
-rw-r--r-- | calm/past_mistakes.py | 36 |
3 files changed, 124 insertions, 16 deletions
diff --git a/calm/calm.py b/calm/calm.py index 49bdcfd..04db081 100755 --- a/calm/calm.py +++ b/calm/calm.py @@ -89,6 +89,7 @@ class CalmState(object): self.subject = '' self.packages = {} self.valid_provides = set() + self.missing_obsolete = {} # @@ -104,10 +105,13 @@ def process_relarea(args, state): logging.debug("reading existing packages for arch %s" % (arch)) packages[arch] = package.read_packages(args.rel_area, arch) - # validate the package set for each arch state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: - if not package.validate_packages(args, packages[arch], state.valid_provides): + state.missing_obsolete[arch] = package.upgrade_oldstyle_obsoletes(packages[arch]) + + # validate the package set for each arch + for arch in common_constants.ARCHES: + if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete[arch]): logging.error("existing %s package set has errors" % (arch)) error = True @@ -208,7 +212,7 @@ def process_maintainer_uploads(args, state, all_packages, m, basedir, desc): state.valid_provides = db.update_package_names(args, merged_packages) for arch in common_constants.ARCHES: logging.debug("validating merged %s package set for maintainer %s" % (arch, name)) - if not package.validate_packages(args, merged_packages[arch], state.valid_provides): + if not package.validate_packages(args, merged_packages[arch], state.valid_provides, state.missing_obsolete): logging.error("error while validating merged %s packages for %s" % (arch, name)) valid = False @@ -329,7 +333,7 @@ def remove_stale_packages(args, packages, state): error = False state.valid_provides = db.update_package_names(args, packages) for arch in common_constants.ARCHES: - if not package.validate_packages(args, packages[arch], state.valid_provides): + if not package.validate_packages(args, packages[arch], state.valid_provides, state.missing_obsolete): logging.error("%s package set has errors after removing stale packages" % arch) error = True diff --git a/calm/package.py b/calm/package.py index 482d042..6370351 100755 --- a/calm/package.py +++ b/calm/package.py @@ -497,14 +497,80 @@ def sort_key(k): # +# generate missing_obsolete data for upgrading old-style obsoletion install +# packages: +# +# they are empty, have the '_obsolete' category, and requires: exactly 1 +# package, their replacement. +# +# generate a record to add an obsoletes: header to the replacement package. +# + +OBSOLETE_AGE_THRESHOLD_YEARS = 20 + + +def upgrade_oldstyle_obsoletes(packages): + missing_obsolete = {} + certain_age = time.time() - (OBSOLETE_AGE_THRESHOLD_YEARS * 365.25 * 24 * 60 * 60) + logging.debug("cut-off date for _obsolete package to be considered for conversion is %s" % (time.strftime("%F %T %Z", time.localtime(certain_age)))) + + for p in sorted(packages): + if packages[p].kind == Kind.binary: + for vr in packages[p].versions(): + # initially apply to a subset over a certain age, to gradually + # introduce this change + mtime = packages[p].tar(vr).mtime + if mtime > certain_age: + continue + logging.debug("_obsolete package '%s' version '%s' mtime '%s' is over cut-off age" % (p, vr, time.strftime("%F %T %Z", time.localtime(mtime)))) + + if packages[p].tar(vr).is_empty: + if '_obsolete' in packages[p].version_hints[vr]['category']: + requires = packages[p].version_hints[vr].get('requires', '').split() + + if p in past_mistakes.old_style_obsolete_by: + o = past_mistakes.old_style_obsolete_by[p] + + # empty replacement means "ignore" + if not o: + continue + + logging.debug('%s is hardcoded as obsoleted by %s ' % (p, o)) + + else: + if len(requires) == 0: + # obsolete but has no replacement + logging.warning('%s is obsolete, but has no replacement' % (p)) + continue + elif len(requires) == 1: + o = requires[0] + elif len(requires) >= 2: + # obsolete with multiple replacements (pick one?) + logging.warning('%s %s is obsoleted by %d packages (%s)' % (p, vr, len(requires), requires)) + continue + + if o in packages: + if o not in missing_obsolete: + missing_obsolete[o] = set() + + missing_obsolete[o].add(p) + logging.info("converting from empty, _obsolete category package '%s' to 'obsoletes: %s' in package '%s'" % (p, p, o)) + + return missing_obsolete + + +# # validate the package database # -def validate_packages(args, packages, valid_requires_extra=None): +def validate_packages(args, packages, valid_requires_extra=None, missing_obsolete_extra=None): error = False if packages is None: return False + if missing_obsolete_extra is None: + missing_obsolete_extra = {} + # build the set of valid things to requires: etc. valid_requires = set() @@ -573,17 +639,19 @@ def validate_packages(args, packages, valid_requires_extra=None): # some old packages are missing needed obsoletes:, add them where # needed, and make sure the uploader is warned if/when package is # updated - if p in past_mistakes.missing_obsolete: - obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') - obsoletes = [o.strip() for o in obsoletes] - obsoletes = [o for o in obsoletes if o] - - needed = past_mistakes.missing_obsolete[p] - for n in needed: - if n not in obsoletes: - obsoletes.append(n) - packages[p].version_hints[v]['obsoletes'] = ','.join(obsoletes) - logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) + for mo in [past_mistakes.missing_obsolete, missing_obsolete_extra]: + if p in mo: + obsoletes = packages[p].version_hints[v].get('obsoletes', '').split(',') + obsoletes = [o.strip() for o in obsoletes] + obsoletes = [o for o in obsoletes if o] + + # XXX: this needs to recurse so we don't drop transitive missing obsoletes? + needed = mo[p] + for n in sorted(needed): + if n not in obsoletes: + obsoletes.append(n) + packages[p].version_hints[v]['obsoletes'] = ', '.join(obsoletes) + logging.info("added 'obsoletes: %s' to package '%s' version '%s'" % (n, p, v)) # if external-source is used, the package must exist if 'external-source' in hints: diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py index af6d650..c0e724f 100644 --- a/calm/past_mistakes.py +++ b/calm/past_mistakes.py @@ -220,3 +220,39 @@ empty_source = { 'pinentry-qt3-src': ['0.7.6-3'], # obsoleted by pinentry-qt 'xerces-c-devel-src': ['2.8.0-1'], # obsoleted by libxerces-c-devel } + +# additional data for the heuristic for upgrading old-style obsoletion packages +old_style_obsolete_by = { + 'at-spi2-atk': 'libatk-bridge2.0_0', + 'qt-gstreamer': 'libQtGStreamer1_0_0', + 'lighttpd-mod_trigger_b4_dl': 'lighttpd', + # these require: both epoch1 and epoch2 replacements, but epoch1 contains + # the same version + 'gcc-tools-autoconf': 'gcc-tools-epoch1-autoconf', + 'gcc-tools-automake': 'gcc-tools-epoch1-automake', + # these are odd and only exist to record an optional dependency on the + # language runtime (dynamically loaded at runtime), which is also noted in + # build-requires: + 'vim-lua': 'vim', + 'vim-perl': 'vim', + 'vim-python': 'vim', + 'vim-python3': 'vim', + 'vim-ruby': 'vim', + # (An empty replacement means "don't apply this heuristic". We use that to + # not bother with some x86-only packages which have complex replacements, + # since they will be going away relatively soon anyhow...) + 'SuiteSparse': '', + 'libSuiteSparse-devel': '', + 'libksba': '', + 'libstdc++6-devel': '', + 'octave-forge': '', + 'plotutils-devel': '', + 'rpm-doc': '', + 'tetex-base': '', + 'tetex-bin': '', + 'tetex-extra': '', + 'tetex-tiny': '', + 'tetex-x11': '', + 'texlive-collection-langtibetan': '', + 'texlive-collection-texinfo': '', +} |