diff options
| author | Jon Turney <jon.turney@dronecode.org.uk> | 2016-05-08 15:37:08 +0300 |
|---|---|---|
| committer | Jon Turney <jon.turney@dronecode.org.uk> | 2016-05-09 16:48:38 +0300 |
| commit | 7dc8fae9002cec3048c14b9e351a1b289afc0d53 (patch) | |
| tree | a9d9d85251f06d3f0c24725f2622a7553fda03bf | |
| parent | d8b998ded7d572db2c11f97f558510ce28751a1c (diff) | |
Add noarch support
- Unfold process() as we need to validate both x86 and x86_64 before we can
move uploaded noarch packages to relarea
- Add arch as an argument to upload.scan(), package.write_setup_ini(),
pkg2html.update_package_listing() etc. rather than passing it around in args
- Use paths relative to relarea, rather than relarea/arch
- Generalize package merge to merge more than 2 sets of packages
- Remove uploads-scan script used in working-up
- Move the !reminder-timestamp file up to maintainer home dir
- Add a noarch package to testdata
- Update tests
27 files changed, 272 insertions, 244 deletions
@@ -26,22 +26,27 @@ # # -# read packages from release area +# for each arch +# - read and validate packages from release area +# - stop if there are errors +# otherwise, # for each maintainer # - read and validate any package uploads # - build a list of files to move and remove -# - merge package sets -# - remove from the package set files which are to be removed -# - validate merged package set -# - process remove list +# - for each arch +# -- merge package sets +# -- remove from the package set files which are to be removed +# -- validate merged package set +# -- process remove list # - on failure # -- mail maintainer with errors # -- empty move list -# -- discard merged package set +# -- discard merged package sets # - on success # -- process move list # -- mail maintainer with movelist -# -- continue with merged package set +# -- continue with merged package sets +# write package listings # write setup.ini file # @@ -68,22 +73,26 @@ import uploads # # -def process_arch(args): - subject = 'calm: cygwin package upload report from %s' % (os.uname()[1]) - details = '%s%s' % (args.arch, ',dry-run' if args.dryrun else '') +def process(args): + subject = 'calm%s: cygwin package upload report from %s' % (' [dry-run]' if args.dryrun else '', os.uname()[1]) # send one email per run to leads, if any errors occurred - with mail_logs(args.email, toaddrs=args.email, subject='%s [%s]' % (subject, details), thresholdLevel=logging.ERROR) as leads_email: + with mail_logs(args.email, toaddrs=args.email, subject='%s' % (subject), thresholdLevel=logging.ERROR) as leads_email: if args.dryrun: logging.warning("--dry-run is in effect, nothing will really be done") - # build package list - packages = package.read_packages(args.rel_area, args.arch) + # for each arch + packages = {} + for arch in common_constants.ARCHES: + logging.debug("reading existing packages for arch %s" % (arch)) - # validate the package set - if not package.validate_packages(args, packages): - logging.error("existing package set has errors, not processing uploads or writing setup.ini") - return False + # build package list + packages[arch] = package.read_packages(args.rel_area, arch) + + # validate the package set + if not package.validate_packages(args, packages[arch]): + logging.error("existing %s package set has errors", arch) + return None # read maintainer list mlist = maintainers.Maintainer.read(args) @@ -96,57 +105,83 @@ def process_arch(args): m = mlist[name] # also send a mail to each maintainer about their packages - with mail_logs(args.email, toaddrs=m.email, subject='%s for %s [%s]' % (subject, name, details), thresholdLevel=logging.INFO) as maint_email: + with mail_logs(args.email, toaddrs=m.email, subject='%s for %s' % (subject, name), thresholdLevel=logging.INFO) as maint_email: - scan_result = uploads.scan(m, all_packages, args) + # for each arch and noarch + scan_result = {} + skip_maintainer = False + for arch in common_constants.ARCHES + ['noarch']: + logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name)) - uploads.remove(args, scan_result.remove_always) + # read uploads + scan_result[arch] = uploads.scan(m, all_packages, arch, args) - if scan_result.error: - logging.error("error while reading uploaded packages for %s" % (name)) - continue + # remove triggers + uploads.remove(args, scan_result[arch].remove_always) + + if scan_result[arch].error: + logging.error("error while reading uploaded arch %s packages from maintainer %s" % (arch, name)) + skip_maintainer = True + continue + + # queue for source package validator + queue.add(args, scan_result[arch].to_relarea, os.path.join(m.homedir())) # if there are no uploaded or removed packages for this # maintainer, we don't have anything to do - if not scan_result.packages and not scan_result.to_vault: + if not any([scan_result[a].packages or scan_result[a].to_vault for a in scan_result]): logging.debug("nothing to do for maintainer %s" % (name)) + skip_maintainer = True + + if skip_maintainer: continue - # queue for source package validator - queue.add(args, scan_result.to_relarea, os.path.join(m.homedir(), args.arch)) - - # merge package set - merged_packages = package.merge(packages, scan_result.packages) - - # remove file which are to be removed - # - # XXX: this doesn't properly account for removing setup.hint - # files - for p in scan_result.to_vault: - for f in scan_result.to_vault[p]: - package.delete(merged_packages, p, f) - - # validate the package set - if package.validate_packages(args, merged_packages): - # process the move list - uploads.move_to_vault(args, scan_result.to_vault) - uploads.remove(args, scan_result.remove_success) - uploads.move_to_relarea(m, args, scan_result.to_relarea) - # use merged package list - packages = merged_packages - logging.debug("added %d packages from maintainer %s" % (len(scan_result.packages), name)) - else: - # otherwise we discard move list and merged_packages - logging.error("error while merging uploaded packages for %s" % (name)) + # for each arch + merged_packages = {} + valid = True + for arch in common_constants.ARCHES: + logging.debug("merging %s package set with uploads from maintainer %s" % (arch, name)) + + # merge package sets + merged_packages[arch] = package.merge(packages[arch], scan_result[arch].packages, scan_result['noarch'].packages) + if not merged_packages[arch]: + valid = False + break + + # remove files which are to be removed + # + # XXX: this doesn't properly account for removing setup.hint + # files + for p in scan_result[arch].to_vault: + for f in scan_result[arch].to_vault[p]: + package.delete(merged_packages[arch], p, f) + + # validate the package set + logging.debug("validating merged %s package set for maintainer %s" % (arch, name)) + if not package.validate_packages(args, merged_packages[arch]): + valid = False + + if not valid: + # discard move list and merged_packages + logging.error("error while merging uploaded %s packages for %s" % (arch, name)) + continue - # write setup.ini - package.write_setup_ini(args, packages) + # for each arch and noarch + for arch in common_constants.ARCHES + ['noarch']: + logging.debug("moving %s packages for maintainer %s" % (arch, name)) - # update packages listings - # XXX: perhaps we need a --[no]listing command line option to disable this from being run? - pkg2html.update_package_listings(args, packages) + # process the move lists + uploads.move_to_vault(args, scan_result[arch].to_vault) + uploads.remove(args, scan_result[arch].remove_success) + uploads.move_to_relarea(m, args, scan_result[arch].to_relarea) - return True + # for each arch + for arch in common_constants.ARCHES: + # use merged package list + packages[arch] = merged_packages[arch] + logging.debug("added %d + %d packages from maintainer %s" % (len(scan_result[arch].packages), len(scan_result['noarch'].packages), name)) + + return packages # @@ -154,9 +189,22 @@ def process_arch(args): # def main(args): + # read package set and process uploads + packages = process(args) + + if not packages: + logging.error("not processing uploads or writing setup.ini") + return + + # for each arch + for arch in common_constants.ARCHES: + # update packages listings + # XXX: perhaps we need a --[no]listing command line option to disable this from being run? + pkg2html.update_package_listings(args, packages[arch], arch) + # for each arch for arch in common_constants.ARCHES: - logging.debug("processing arch %s" % (arch)) + logging.debug("writing setup.ini for arch %s" % (arch)) args.arch = arch args.setup_version = setup_exe.extract_version(os.path.join(args.setupdir, 'setup-' + args.arch + '.exe')) @@ -170,10 +218,11 @@ def main(args): args.inifile = tmpfile.name changed = False - if not process_arch(args): - # generating setup.ini failed - pass - elif not os.path.exists(inifile): + + # write setup.ini + package.write_setup_ini(args, packages[arch], arch) + + if not os.path.exists(inifile): # if the setup.ini file doesn't exist yet logging.warning('no existing %s' % (inifile)) changed = True @@ -63,7 +63,7 @@ def main(args): return # write setup.ini - package.write_setup_ini(args, packages) + package.write_setup_ini(args, packages, args.arch) if args.stats: stats(packages) @@ -74,13 +74,15 @@ class Tar(object): def read_packages(rel_area, arch): packages = defaultdict(Package) - releasedir = os.path.join(rel_area, arch) - logging.debug('reading packages from %s' % releasedir) + # both noarch/ and <arch>/ directories are considered + for root in ['noarch', arch]: + releasedir = os.path.join(rel_area, root) + logging.debug('reading packages from %s' % releasedir) - for (dirpath, subdirs, files) in os.walk(releasedir): - read_package(packages, releasedir, dirpath, files) + for (dirpath, subdirs, files) in os.walk(releasedir): + read_package(packages, rel_area, dirpath, files) - logging.debug("%d packages read" % len(packages)) + logging.debug("%d packages read" % len(packages)) return packages @@ -223,7 +225,7 @@ def read_package(packages, basedir, dirpath, files, strict=False): logging.log(strict_lvl, "package '%s' sdesc starts with '%s'; this is redundant as the UI will show both the package name and sdesc" % (p, ''.join(colon.group(1, 2)))) warnings = True - elif (len(files) > 0) and (relpath.count(os.path.sep) > 0): + elif (len(files) > 0) and (relpath.count(os.path.sep) > 1): logging.warning("no setup.hint in %s but has files: %s" % (dirpath, ', '.join(files))) if strict: @@ -494,7 +496,7 @@ def validate_package_maintainers(args, packages): # # write setup.ini # -def write_setup_ini(args, packages): +def write_setup_ini(args, packages, arch): logging.debug('writing %s' % (args.inifile)) @@ -510,7 +512,7 @@ def write_setup_ini(args, packages): if args.release: print("release: %s" % args.release, file=f) - print("arch: %s" % args.arch, file=f) + print("arch: %s" % arch, file=f) print("setup-timestamp: %d" % time.time(), file=f) if args.setup_version: print("setup-version: %s" % args.setup_version, file=f) @@ -550,18 +552,18 @@ def write_setup_ini(args, packages): if 'install' in packages[p].vermap[version]: t = packages[p].vermap[version]['install'] - tar_line('install', args.arch, packages[p], t, f) + tar_line('install', packages[p], t, f) # look for corresponding source in this package first if 'source' in packages[p].vermap[version]: t = packages[p].vermap[version]['source'] - tar_line('source', args.arch, packages[p], t, f) + tar_line('source', packages[p], t, f) # if that doesn't exist, follow external-source elif 'external-source' in packages[p].hints: s = packages[p].hints['external-source'] if 'source' in packages[s].vermap[version]: t = packages[s].vermap[version]['source'] - tar_line('source', args.arch, packages[s], t, f) + tar_line('source', packages[s], t, f) else: logging.warning("package '%s' version '%s' has no source in external-source '%s'" % (p, version, s)) @@ -570,8 +572,8 @@ def write_setup_ini(args, packages): # helper function to output details for a particular tar file -def tar_line(category, arch, p, t, f): - fn = os.path.join(arch, p.path, t) +def tar_line(category, p, t, f): + fn = os.path.join(p.path, t) sha512 = p.tars[t].sha512 size = p.tars[t].size print("%s: %s %d %s" % (category, fn, size, sha512), file=f) @@ -584,42 +586,45 @@ def upper_first_character(s): # -# merge two sets of packages +# merge sets of packages # # for each package which exist in both a and b: -# - they must exist at the same relative path, or the package from a is used -# - we combine the list of tarfiles, duplicates are not expected +# - they must exist at the same relative path +# - we combine the list of tarfiles, duplicates are not permitted # - we use the hints from b, and warn if they are different to the hints for a # -def merge(a, b): +def merge(a, *l): # start with a copy of a c = copy.deepcopy(a) - for p in b: - # if the package is in b but not in a, add it to the copy - if p not in a: - c[p] = b[p] - # else, if the package is both in a and b, we have to do a merge - else: - # package must exist at same relative path - if a[p].path != b[p].path: - logging.error("package '%s' is at paths %s and %s" % (p, a[p].path, b[p].path)) + for b in l: + for p in b: + # if the package is in b but not in a, add it to the copy + if p not in a: + c[p] = b[p] + # else, if the package is both in a and b, we have to do a merge else: - for t in b[p].tars: - if t in c[p].tars: - logging.error("package '%s' has duplicate tarfile %s" % (p, t)) - else: - c[p].tars[t] = b[p].tars[t] + # package must exist at same relative path + if a[p].path != b[p].path: + logging.error("package '%s' is at paths %s and %s" % (p, a[p].path, b[p].path)) + return None + else: + for t in b[p].tars: + if t in c[p].tars: + logging.error("package '%s' has duplicate tarfile %s" % (p, t)) + return None + else: + c[p].tars[t] = b[p].tars[t] - # use hints from b, but warn if they have changed - if a[p].hints != b[p].hints: - c[p].hints = b[p].hints + # use hints from b, but warn if they have changed + if a[p].hints != b[p].hints: + c[p].hints = b[p].hints - diff = '\n'.join(difflib.ndiff( - pprint.pformat(a[p].hints).splitlines(), - pprint.pformat(b[p].hints).splitlines())) + diff = '\n'.join(difflib.ndiff( + pprint.pformat(a[p].hints).splitlines(), + pprint.pformat(b[p].hints).splitlines())) - logging.warning("package '%s' hints changed\n%s" % (p, diff)) + logging.warning("package '%s' hints changed\n%s" % (p, diff)) return c diff --git a/pkg2html.py b/pkg2html.py index 171d618..2e16076 100755 --- a/pkg2html.py +++ b/pkg2html.py @@ -34,6 +34,9 @@ # - remove any listing files for which there was no package # - remove any empty directories (TBD) # +# note that the directory hierarchy of (noarch|arch)/package/subpackages is +# flattened in the package listing to just the package name +# from collections import defaultdict import argparse @@ -54,8 +57,8 @@ import package # # -def update_package_listings(args, packages): - base = os.path.join(args.htdocs, args.arch) +def update_package_listings(args, packages, arch): + base = os.path.join(args.htdocs, arch) if not args.dryrun: try: os.makedirs(base, exist_ok=True) @@ -77,7 +80,7 @@ def update_package_listings(args, packages): if not args.dryrun: with open(htaccess, 'w') as f: - print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (args.arch), + print('Redirect temp /packages/%s/index.html https://cygwin.com/packages/package_list.html' % (arch), file=f) toremove = glob.glob(os.path.join(base, '*', '*')) @@ -145,7 +148,7 @@ def update_package_listings(args, packages): <h1>%s</h1> <tt><pre>''' % (header)), file=f) - tf = os.path.join(args.rel_area, args.arch, packages[p].path, t) + tf = os.path.join(args.rel_area, packages[p].path, t) if not os.path.exists(tf): # XXX: this shouldn't happen with a full mirror... print('tarfile %s not found' % tf, file=f) @@ -190,7 +193,7 @@ def update_package_listings(args, packages): <b class="rbottom"><b class="r4"></b><b class="r3"></b><b class="r2"></b><b class="r1"></b></b> </div> <br> - <table class="pkglist">''') % (args.arch, args.arch), file=index) + <table class="pkglist">''') % (arch, arch), file=index) for p in sorted(packages.keys(), key=package.sort_key): # don't write anything if 'skip' @@ -199,7 +202,7 @@ def update_package_listings(args, packages): header = packages[p].hints['sdesc'].replace('"', '') - print('<tr><td><a href="' + args.arch + '/' + p + '">' + p + '</a></td><td>' + header + '</td></tr>', file=index) + print('<tr><td><a href="' + arch + '/' + p + '">' + p + '</a></td><td>' + header + '</td></tr>', file=index) print(textwrap.dedent('''\ </table> @@ -233,4 +236,4 @@ if __name__ == "__main__": logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s') packages = package.read_packages(args.rel_area, args.arch) - update_package_listings(args, packages) + update_package_listings(args, packages, args.arch) @@ -53,7 +53,7 @@ def add(args, movelist, fromdir): for p in movelist: for f in movelist[p]: if re.search(r'-src.tar.(bz2|gz|lzma|xz)$', f): - srcpkgs.append(os.path.join(args.arch, p, f)) + srcpkgs.append(os.path.join(p, f)) # if so... # @@ -61,7 +61,7 @@ def add(args, movelist, fromdir): # source file if len(srcpkgs) >= 1: # keep all the files for comparison - uploads.copy(args, movelist, fromdir, os.path.join(upload_root, args.arch)) + uploads.copy(args, movelist, fromdir, upload_root) # queue any srcpkgs for p in srcpkgs: diff --git a/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected b/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected new file mode 100644 index 0000000..9516e05 --- /dev/null +++ b/testdata/hints/noarch/release/perl-Net-SMTP-SSL/expected @@ -0,0 +1,5 @@ +{'category': 'Perl', + 'requires': '', + 'sdesc': '"Perl distribution Net-SMTP-SSL"', + 'ldesc': '"Implements the same API as Net::SMTP, but uses IO::Socket::SSL for\n' + 'its network operations in order to support encrypted connections."'} diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz Binary files differnew file mode 100644 index 0000000..9e7f93d --- /dev/null +++ b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2-src.tar.xz diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz Binary files differnew file mode 100644 index 0000000..d986060 --- /dev/null +++ b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-2.tar.xz diff --git a/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint new file mode 100644 index 0000000..f91a0e1 --- /dev/null +++ b/testdata/homes/Blooey McFooey/noarch/release/perl-Net-SMTP-SSL/setup.hint @@ -0,0 +1,5 @@ +category: Perl +requires: +sdesc: "Perl distribution Net-SMTP-SSL" +ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for +its network operations in order to support encrypted connections." diff --git a/testdata/htdocs.expected/x86/packages.inc b/testdata/htdocs.expected/x86/packages.inc index 8f95160..289a07e 100755 --- a/testdata/htdocs.expected/x86/packages.inc +++ b/testdata/htdocs.expected/x86/packages.inc @@ -16,6 +16,7 @@ <tr><td><a href="x86/libdns_sd1">libdns_sd1</a></td><td>Bonjour Zeroconf implementation</td></tr> <tr><td><a href="x86/mDNSResponder">mDNSResponder</a></td><td>Bonjour Zeroconf implementation</td></tr> <tr><td><a href="x86/openssh">openssh</a></td><td>The OpenSSH server and client programs</td></tr> +<tr><td><a href="x86/perl-Net-SMTP-SSL">perl-Net-SMTP-SSL</a></td><td>Perl distribution Net-SMTP-SSL</td></tr> <tr><td><a href="x86/rpm-doc">rpm-doc</a></td><td>Obsolete package for RPM package management system manual pages</td></tr> <tr><td><a href="x86/testpackage">testpackage</a></td><td>A test package</td></tr> </table> diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess new file mode 100644 index 0000000..3196d64 --- /dev/null +++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/.htaccess @@ -0,0 +1,3 @@ +Options Indexes +IndexOptions -FancyIndexing +AddType text/html 1 2 3 4 5 6 7 8 9 diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1 b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1 new file mode 100644 index 0000000..0f6212c --- /dev/null +++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1 @@ -0,0 +1,21 @@ +<html> +<h1>perl-Net-SMTP-SSL: Perl distribution Net-SMTP-SSL (installed binaries and support files)</h1> +<tt><pre> + 2015-06-26 19:10 0 usr/ + 2015-06-26 19:10 0 usr/lib/ + 2015-06-26 19:10 0 usr/lib/perl5/ + 2015-06-26 19:10 0 usr/lib/perl5/vendor_perl/ + 2015-06-26 19:10 0 usr/lib/perl5/vendor_perl/5.22/ + 2015-06-26 19:10 0 usr/lib/perl5/vendor_perl/5.22/Net/ + 2015-06-26 19:10 0 usr/lib/perl5/vendor_perl/5.22/Net/SMTP/ + 2015-06-21 01:58 1462 usr/lib/perl5/vendor_perl/5.22/Net/SMTP/SSL.pm + 2015-06-26 19:10 0 usr/share/ + 2015-06-26 19:10 0 usr/share/doc/ + 2015-06-26 19:10 0 usr/share/doc/perl-Net-SMTP-SSL/ + 2015-06-26 19:10 199 usr/share/doc/perl-Net-SMTP-SSL/Changes + 2015-06-26 19:10 867 usr/share/doc/perl-Net-SMTP-SSL/README + 2015-06-26 19:10 0 usr/share/man/ + 2015-06-26 19:10 0 usr/share/man/man3/ + 2015-06-26 19:10 1619 usr/share/man/man3/Net.SMTP.SSL.3pm.gz +</pre></tt> +</html> diff --git a/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src new file mode 100644 index 0000000..76e9a50 --- /dev/null +++ b/testdata/htdocs.expected/x86/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src @@ -0,0 +1,8 @@ +<html> +<h1>perl-Net-SMTP-SSL: Perl distribution Net-SMTP-SSL (source code)</h1> +<tt><pre> + 2015-06-26 19:10 0 perl-Net-SMTP-SSL-1.03-1.src/ + 2015-06-26 19:10 2271 perl-Net-SMTP-SSL-1.03-1.src/Net-SMTP-SSL-1.03.tar.gz + 2015-06-26 19:10 296 perl-Net-SMTP-SSL-1.03-1.src/perl-Net-SMTP-SSL.cygport +</pre></tt> +</html> diff --git a/testdata/inifile/setup.ini.expected b/testdata/inifile/setup.ini.expected index 6c09d84..c41865a 100644 --- a/testdata/inifile/setup.ini.expected +++ b/testdata/inifile/setup.ini.expected @@ -182,6 +182,17 @@ 'source: x86/release/openssh/openssh-7.2p2-1-src.tar.xz 228 ' 'e675b0ac4bc2c3e1c4971bc56d77b0cd53a9bdf5632873a235d7582e29dfd3e8a7bb04b28f6cdee3e6b3d14c25ed39392538e3f628a9bfda6c905646ebc3c225\n' '\n' + '@ perl-Net-SMTP-SSL\n' + 'sdesc: "Perl distribution Net-SMTP-SSL"\n' + 'ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for\n' + 'its network operations in order to support encrypted connections."\n' + 'category: Perl\n' + 'version: 1.03-1\n' + 'install: noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz 3180 ' + 'aac6428f56fed431da1430242a327f36f1ec1ca6106366acb6752dd87f1b9adb87767709be9279b2f9435d16cee003f119a5ed4519c7365c8411404555618e66\n' + 'source: noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz 2752 ' + '4de528554acb0b63f2a964bc8a0421118f96ce05cb4c9d111983aaea1a4a93025b31627fd66875fa09c2dfaecc11a89f2a294bdd3c77055bfaa93a27fdfc1f07\n' + '\n' '@ rpm-doc\n' 'sdesc: "Obsolete package for RPM package management system manual pages"\n' 'category: _obsolete\n' diff --git a/testdata/pkglist/cygwin-pkg-maint b/testdata/pkglist/cygwin-pkg-maint index 41b4175..c157e17 100644 --- a/testdata/pkglist/cygwin-pkg-maint +++ b/testdata/pkglist/cygwin-pkg-maint @@ -1606,7 +1606,7 @@ perl-Net-DNS Achim Gratz perl-Net-DNS-SEC Achim Gratz perl-Net-HTTP Achim Gratz/Yaakov Selkowitz perl-Net-IP Achim Gratz -perl-Net-SMTP-SSL Yaakov Selkowitz +perl-Net-SMTP-SSL Yaakov Selkowitz/Blooey McFooey perl-Net-SSLeay Achim Gratz/Yaakov Selkowitz perl-Number-Compare Achim Gratz/Ken Brown perl-OpenGL Yaakov Selkowitz diff --git a/testdata/pkglist/expected b/testdata/pkglist/expected index b340aa2..c9cd92c 100644 --- a/testdata/pkglist/expected +++ b/testdata/pkglist/expected @@ -3,7 +3,7 @@ 'Adam Dinwoodie': maintainers.Maintainer('Adam Dinwoodie', [], ['git']), 'Alexey Sokolov': maintainers.Maintainer('Alexey Sokolov', [], ['znc']), 'Andrew Schulman': maintainers.Maintainer('Andrew Schulman', [], ['atool', 'autossh', 'bc', 'discus', 'fish', 'lftp', 'libargp', 'nosleep', 'orpie', 'pinfo', 'ploticus', 'ploticus-doc', 'screen', 'sitecopy', 'sng', 'socat', 'stow', 'stunnel', 'time', 'unison2.27', 'unison2.32', 'unison2.40', 'unison2.45', 'unison2.48']), - 'Blooey McFooey': maintainers.Maintainer('Blooey McFooey', [], ['testpackage']), + 'Blooey McFooey': maintainers.Maintainer('Blooey McFooey', [], ['perl-Net-SMTP-SSL', 'testpackage']), 'Bob Heckel': maintainers.Maintainer('Bob Heckel', [], ['libgc', 'w3m']), 'Chris J. Breisch': maintainers.Maintainer('Chris J. Breisch', [], ['man-db']), 'Chris LeBlanc': maintainers.Maintainer('Chris LeBlanc', [], ['python-h5py', 'python3-h5py']), diff --git a/testdata/process_arch/homedir.expected b/testdata/process_arch/homedir.expected index 8dcf175..1c52158 100644 --- a/testdata/process_arch/homedir.expected +++ b/testdata/process_arch/homedir.expected @@ -1,5 +1,8 @@ {'.': ['an_unexpected_file'], 'Blooey McFooey': [], + 'Blooey McFooey/noarch': [], + 'Blooey McFooey/noarch/release': [], + 'Blooey McFooey/noarch/release/perl-Net-SMTP-SSL': [], 'Blooey McFooey/x86': [], 'Blooey McFooey/x86/release': [], 'Blooey McFooey/x86/release/after-ready': ['after-ready-1.0-1.tar.bz2', 'setup.hint'], diff --git a/testdata/process_arch/htdocs.expected b/testdata/process_arch/htdocs.expected index 6554900..4670972 100644 --- a/testdata/process_arch/htdocs.expected +++ b/testdata/process_arch/htdocs.expected @@ -19,6 +19,11 @@ 'x86/libdns_sd1': ['.htaccess', 'libdns_sd1-379.32.1-1'], 'x86/mDNSResponder': ['.htaccess', 'mDNSResponder-379.32.1-1', 'mDNSResponder-379.32.1-1-src'], 'x86/openssh': ['.htaccess', 'openssh-7.2p2-1', 'openssh-7.2p2-1-src'], + 'x86/perl-Net-SMTP-SSL': ['.htaccess', + 'perl-Net-SMTP-SSL-1.03-1', + 'perl-Net-SMTP-SSL-1.03-1-src', + 'perl-Net-SMTP-SSL-1.03-2', + 'perl-Net-SMTP-SSL-1.03-2-src'], 'x86/rpm-doc': ['.htaccess', 'rpm-doc-4.1-2', 'rpm-doc-4.1-2-src', 'rpm-doc-999-1'], 'x86/testpackage': ['.htaccess', 'testpackage-1.0-1', 'testpackage-1.0-1-src'], 'x86/testpackage-subpackage': ['.htaccess', 'testpackage-subpackage-1.0-1']} diff --git a/testdata/process_arch/rel_area.expected b/testdata/process_arch/rel_area.expected index f081c02..44161bf 100644 --- a/testdata/process_arch/rel_area.expected +++ b/testdata/process_arch/rel_area.expected @@ -1,4 +1,12 @@ {'.': ['setup.ini'], + 'noarch': ['sha512.sum'], + 'noarch/release': ['sha512.sum'], + 'noarch/release/perl-Net-SMTP-SSL': ['perl-Net-SMTP-SSL-1.03-1-src.tar.xz', + 'perl-Net-SMTP-SSL-1.03-1.tar.xz', + 'perl-Net-SMTP-SSL-1.03-2-src.tar.xz', + 'perl-Net-SMTP-SSL-1.03-2.tar.xz', + 'setup.hint', + 'sha512.sum'], 'x86': ['sha512.sum'], 'x86/release': ['sha512.sum'], 'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.tar.bz2', 'setup.hint', 'sha512.sum'], diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz Binary files differnew file mode 100644 index 0000000..9e7f93d --- /dev/null +++ b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1-src.tar.xz diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz Binary files differnew file mode 100644 index 0000000..d986060 --- /dev/null +++ b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/perl-Net-SMTP-SSL-1.03-1.tar.xz diff --git a/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint new file mode 100644 index 0000000..f91a0e1 --- /dev/null +++ b/testdata/relarea/noarch/release/perl-Net-SMTP-SSL/setup.hint @@ -0,0 +1,5 @@ +category: Perl +requires: +sdesc: "Perl distribution Net-SMTP-SSL" +ldesc: "Implements the same API as Net::SMTP, but uses IO::Socket::SSL for +its network operations in order to support encrypted connections." diff --git a/testdata/uploads/move.expected b/testdata/uploads/move.expected index b0caaf3..47cb0af 100644 --- a/testdata/uploads/move.expected +++ b/testdata/uploads/move.expected @@ -1,3 +1,3 @@ -{'release/testpackage': ['setup.hint', 'testpackage-1.0-1-src.tar.bz2', 'testpackage-1.0-1.tar.bz2'], - 'release/testpackage/testpackage-subpackage': ['setup.hint', 'testpackage-subpackage-1.0-1.tar.bz2'], - 'release/testpackage2/testpackage2-subpackage': ['setup.hint', 'testpackage2-subpackage-1.0-1.tar.bz2']} +{'x86/release/testpackage': ['setup.hint', 'testpackage-1.0-1-src.tar.bz2', 'testpackage-1.0-1.tar.bz2'], + 'x86/release/testpackage/testpackage-subpackage': ['setup.hint', 'testpackage-subpackage-1.0-1.tar.bz2'], + 'x86/release/testpackage2/testpackage2-subpackage': ['setup.hint', 'testpackage2-subpackage-1.0-1.tar.bz2']} diff --git a/testdata/uploads/pkglist.expected b/testdata/uploads/pkglist.expected index f98adb3..bbe52b0 100644 --- a/testdata/uploads/pkglist.expected +++ b/testdata/uploads/pkglist.expected @@ -1,14 +1,14 @@ -{'testpackage': Package('release/testpackage', {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False), +{'testpackage': Package('x86/release/testpackage', {'testpackage-1.0-1-src.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False), 'testpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test package"', 'ldesc': '"A test package\n' "It's description might contains some unicode gibberish\n" 'Like it’s you’re Markup Language™ Nokogiri’s tool―that Bézier."', 'category': 'Devel', 'requires': 'cygwin'}), - 'testpackage-subpackage': Package('release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage"', + 'testpackage-subpackage': Package('x86/release/testpackage/testpackage-subpackage', {'testpackage-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage"', 'ldesc': '"A test subpackage"', 'category': 'Devel', 'external-source': 'testpackage'}), - 'testpackage2-subpackage': Package('release/testpackage2/testpackage2-subpackage', {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage 2"', + 'testpackage2-subpackage': Package('x86/release/testpackage2/testpackage2-subpackage', {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('aff488008bee3486e25b539fe6ccd1397bd3c5c0ba2ee2cf34af279554baa195af7493ee51d6f8510735c9a2ea54436d776a71e768165716762aec286abbbf83', 195, False)}, {'sdesc': '"A test subpackage 2"', 'ldesc': '"A test subpackage 2"', 'category': 'Devel'})} @@ -123,7 +123,7 @@ class TestMain(unittest.TestCase): packages = package.read_packages(args.rel_area, args.arch) package.validate_packages(args, packages) - pkg2html.update_package_listings(args, packages) + pkg2html.update_package_listings(args, packages, args.arch) # compare the output files with expected for (dirpath, subdirs, files) in os.walk(htdocs): @@ -200,10 +200,11 @@ class TestMain(unittest.TestCase): for (f, t) in ready_fns: os.system('touch %s "%s"' % (t, f)) - scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args) + scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args.arch, args) self.assertEqual(scan_result.error, False) compare_with_expected_file(self, 'testdata/uploads', scan_result.to_relarea, 'move') + self.assertCountEqual(scan_result.to_vault, {'x86/release/testpackage': ['x86/release/testpackage/testpackage-0.1-1.tar.bz2']}) self.assertCountEqual(scan_result.remove_always, [f for (f, t) in ready_fns]) self.assertEqual(scan_result.remove_success, ['testdata/homes/Blooey McFooey/x86/release/testpackage/-testpackage-0.1-1.tar.bz2']) compare_with_expected_file(self, 'testdata/uploads', scan_result.packages, 'pkglist') @@ -222,9 +223,9 @@ class TestMain(unittest.TestCase): setattr(args, 'setup_version', '4.321') packages = package.read_packages(args.rel_area, args.arch) - package.delete(packages, 'release/nonexistent', 'nosuchfile-1.0.0.tar.xz') + package.delete(packages, 'x86/release/nonexistent', 'nosuchfile-1.0.0.tar.xz') self.assertEqual(package.validate_packages(args, packages), True) - package.write_setup_ini(args, packages) + package.write_setup_ini(args, packages, args.arch) with open(args.inifile) as inifile: results = inifile.read() # fix the timestamp to match expected @@ -233,7 +234,7 @@ class TestMain(unittest.TestCase): # XXX: delete a needed package, and check validate fails - def test_process_arch(self): + def test_process(self): self.maxDiff = None args = types.SimpleNamespace() @@ -242,7 +243,6 @@ class TestMain(unittest.TestCase): setattr(args, d, tempfile.mktemp()) logging.info('%s = %s', d, getattr(args, d)) - setattr(args, 'arch', 'x86') setattr(args, 'dryrun', False) setattr(args, 'email', None) setattr(args, 'force', False) @@ -258,11 +258,16 @@ class TestMain(unittest.TestCase): m_homedir = os.path.join(getattr(args, 'homedir'), 'Blooey McFooey') ready_fns = [(os.path.join(m_homedir, 'x86', 'release', 'testpackage', '!ready'), ''), (os.path.join(m_homedir, 'x86', 'release', 'testpackage2', 'testpackage2-subpackage', '!ready'), ''), - (os.path.join(m_homedir, 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700')] + (os.path.join(m_homedir, 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700'), + (os.path.join(m_homedir, 'noarch', 'release', 'perl-Net-SMTP-SSL', '!ready'), '')] for (f, t) in ready_fns: os.system('touch %s "%s"' % (t, f)) - self.assertEqual(calm.process_arch(args), True) + packages = calm.process(args) + self.assertTrue(packages) + + pkg2html.update_package_listings(args, packages['x86'], 'x86') + package.write_setup_ini(args, packages['x86'], 'x86') for d in ['rel_area', 'homedir', 'htdocs', 'vault']: with self.subTest(directory=d): @@ -272,7 +277,7 @@ class TestMain(unittest.TestCase): if __name__ == '__main__': # ensure sha512.sum files exist - os.system("find testdata/relarea/x86 -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;") + os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;") # should remove a sha512.sum file so that we test functioning when it's absent os.unlink('testdata/relarea/x86/release/naim/sha512.sum') # remove !ready files diff --git a/upload-scan b/upload-scan deleted file mode 100755 index 08ab677..0000000 --- a/upload-scan +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2015 Jon Turney -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -# -# check maintainer upload directories for files to move to release area -# -# - only process packages for which we are listed as a maintainer -# - only move things with an mtime earlier than the mtime of the !ready file -# - validate the syntax of the setup.ini file before moving anything -# - note if package file already exists in release area with identical contents -# - error if package file already exist in release area with different contents -# - remove !ready file -# - send a report on what we did to the maintainer and project leads -# - -import argparse -import logging -import os -import sys - -from buffering_smtp_handler import mail_logs -import common_constants -import maintainers -import uploads - - -# -# -# - -def main(args): - # create maintainer list - mlist = maintainers.Maintainer.read(args) - - # make the list of all packages - all_packages = maintainers.Maintainer.all_packages(mlist) - - for arch in common_constants.ARCHES: - args.arch = arch - - for name in sorted(mlist.keys()): - m = mlist[name] - - with mail_logs(args.email, toaddrs=(args.email or []) + m.email, subject='upset messages') as maint_email: - - # search for and validate uploaded packages - (error, packages, to_relarea, to_vault, remove_always, remove_success) = uploads.scan(m, all_packages, args) - - # always remove all the !ready files - uploads.remove(args, remove_always) - - # but only move something if there were no errors - if not error: - uploads.move_to_vault(args, to_vault) - uploads.remove(args, remove_success) - uploads.move_to_relarea(m, args, to_relarea) - - -# -# -# - -if __name__ == "__main__": - homedir_default = common_constants.HOMEDIR - orphanmaint_default = common_constants.ORPHANMAINT - pkglist_default = common_constants.PKGMAINT - relarea_default = common_constants.FTP - vault_default = common_constants.VAULT - - parser = argparse.ArgumentParser(description='Move files from maintainer upload directories to release area') - parser.add_argument('--email', action='store', dest='email', nargs='?', const=common_constants.EMAILS, help='email output to maintainer and ADDRS (default: ' + common_constants.EMAILS + ')', metavar='ADDRS') - parser.add_argument('--homedir', action='store', metavar='DIR', help="maintainer home directory (default: " + homedir_default + ")", default=homedir_default) - parser.add_argument('--orphanmaint', action='store', metavar='NAMES', help="orphan package maintainers (default: '" + orphanmaint_default + "')", default=orphanmaint_default) - parser.add_argument('--pkglist', action='store', metavar='FILE', help="package maintainer list (default: " + pkglist_default + ")", default=pkglist_default) - parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area') - parser.add_argument('--vault', action='store', metavar='DIR', help="vault directory (default: " + vault_default + ")", default=vault_default, dest='vault') - parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', help="don't do anything") - parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output') - (args) = parser.parse_args() - - if args.verbose: - logging.getLogger().setLevel(logging.INFO) - - logging.basicConfig(format=os.path.basename(sys.argv[0])+': %(message)s') - - if args.email: - args.email = args.email.split(',') - - main(args) @@ -46,9 +46,9 @@ ScanResult = namedtuple('ScanResult', 'error,packages,to_relarea,to_vault,remove # # -def scan(m, all_packages, args): - basedir = os.path.join(m.homedir(), args.arch) - releasedir = os.path.join(args.rel_area, args.arch) +def scan(m, all_packages, arch, args): + basedir = os.path.join(m.homedir(), arch) + releasedir = os.path.join(args.rel_area, arch) packages = defaultdict(package.Package) move = defaultdict(list) @@ -70,7 +70,7 @@ def scan(m, all_packages, args): # the mtime of this file indicates when 'ignoring as there is no !ready' # warnings were last emitted - reminder_file = os.path.join(basedir, '!reminder-timestamp') + reminder_file = os.path.join(m.homedir(), '!reminder-timestamp') if os.path.exists(reminder_file): reminder_time = os.path.getmtime(reminder_file) else: @@ -80,10 +80,10 @@ def scan(m, all_packages, args): # scan package directories for (dirpath, subdirs, files) in os.walk(os.path.join(basedir, 'release')): - relpath = os.path.relpath(dirpath, basedir) + relpath = os.path.relpath(dirpath, m.homedir()) # skip uninteresting directories - if (not files) or (relpath == 'release'): + if (not files) or (relpath == os.path.join(arch, 'release')): continue logging.debug('reading uploads from %s' % dirpath) @@ -190,7 +190,7 @@ def scan(m, all_packages, args): # read and validate package if files: # strict means we consider warnings as fatal for upload - if package.read_package(packages, basedir, dirpath, files, strict=True): + if package.read_package(packages, m.homedir(), dirpath, files, strict=True): error = True # if we didn't need to check the reminder timestamp, it can be reset @@ -246,14 +246,14 @@ def move(args, movelist, fromdir, todir): def move_to_relarea(m, args, movelist): - move(args, movelist, os.path.join(m.homedir(), args.arch), os.path.join(args.rel_area, args.arch)) + move(args, movelist, m.homedir(), args.rel_area) # XXX: Note that there seems to be a separate process, not run from # cygwin-admin's crontab, which changes the ownership of files in the # release area to cyguser:cygwin def move_to_vault(args, movelist): - move(args, movelist, os.path.join(args.rel_area, args.arch), os.path.join(args.vault, args.arch)) + move(args, movelist, args.rel_area, args.vault) # |