Use a separate staging dir for automated build uploads

Look in a different directory for packages being deployed by scallywag.

Future work: this will allow disabling the uploading of local builds (on
a per-package basis), whilst still allowing automated build uploads.

Update tests appropriately.

7 files changed, 36 insertions, 20 deletions

diff --git a/calm/calm.py b/calm/calm.py
index c0d756a..49bdcfd 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -150,7 +150,8 @@ def process_uploads(args, state):
         m = mlist[name]
 
         with logfilters.AttrFilter(maint=m.name):
-            process_maintainer_uploads(args, state, all_packages, m)
+            process_maintainer_uploads(args, state, all_packages, m, args.homedir, 'upload')
+            process_maintainer_uploads(args, state, all_packages, m, args.stagingdir, 'staging')
 
     # record updated reminder times for maintainers
     maintainers.update_reminder_times(mlist)
@@ -158,7 +159,7 @@ def process_uploads(args, state):
     return state.packages
 
 
-def process_maintainer_uploads(args, state, all_packages, m):
+def process_maintainer_uploads(args, state, all_packages, m, basedir, desc):
     name = m.name
 
     # for each arch and noarch
@@ -168,7 +169,7 @@ def process_maintainer_uploads(args, state, all_packages, m):
         logging.debug("reading uploaded arch %s packages from maintainer %s" % (arch, name))
 
         # read uploads
-        scan_result[arch] = uploads.scan(m, all_packages, arch, args)
+        scan_result[arch] = uploads.scan(basedir, m, all_packages, arch, args)
 
         # remove triggers
         uploads.remove(args, scan_result[arch].remove_always)
@@ -251,7 +252,7 @@ def process_maintainer_uploads(args, state, all_packages, m):
         uploads.remove(args, scan_result[arch].remove_success)
         if scan_result[arch].to_relarea:
             logging.info("adding %d package(s) for arch %s" % (len(scan_result[arch].to_relarea), arch))
-        scan_result[arch].to_relarea.move_to_relarea(m, args)
+        scan_result[arch].to_relarea.move_to_relarea(m, args, desc)
         # XXX: Note that there seems to be a separate process, not run
         # from cygwin-admin's crontab, which changes the ownership of
         # files in the release area to cyguser:cygwin
@@ -270,7 +271,7 @@ def process_maintainer_uploads(args, state, all_packages, m):
 
     # clean up any empty directories
     if not args.dryrun:
-        utils.rmemptysubdirs(m.homedir())
+        utils.rmemptysubdirs(os.path.join(basedir, m.name))
 
     # report what we've done
     added = []
@@ -705,6 +706,7 @@ def logging_setup(args):
 def main():
     htdocs_default = os.path.join(common_constants.HTDOCS, 'packages')
     homedir_default = common_constants.HOMEDIR
+    stagingdir_default = common_constants.STAGINGDIR
     orphanmaint_default = common_constants.ORPHANMAINT
     pidfile_default = '/sourceware/cygwin-staging/calm.pid'
     pkglist_default = common_constants.PKGMAINT
@@ -726,6 +728,7 @@ def main():
     parser.add_argument('--release', action='store', help='value for setup-release key (default: cygwin)', default='cygwin')
     parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='rel_area')
     parser.add_argument('--setupdir', action='store', metavar='DIR', help="setup executable directory (default: " + setupdir_default + ")", default=setupdir_default)
+    parser.add_argument('--stagingdir', action='store', metavar='DIR', help="automated build staging directory (default: " + stagingdir_default + ")", default=stagingdir_default)
     parser.add_argument('--no-stale', action='store_false', dest='stale', help="don't vault stale packages")
     parser.set_defaults(stale=True)
     parser.add_argument('--reports', action='store_true', dest='reports', help="don't produce reports", default=None)
diff --git a/calm/common_constants.py b/calm/common_constants.py
index 1887681..906a56a 100644
--- a/calm/common_constants.py
+++ b/calm/common_constants.py
@@ -32,6 +32,9 @@ import os
 # base directory for maintainer upload directories
 HOMEDIR = '/sourceware/cygwin-staging/home'
 
+# base directory for automated build staging directories
+STAGINGDIR = '/sourceware/cygwin-staging/staging'
+
 # the 'release area', contains all released files, which are rsync'ed to mirrors
 FTP = '/var/ftp/pub/cygwin'
 
diff --git a/calm/movelist.py b/calm/movelist.py
index 07b69d7..6101271 100644
--- a/calm/movelist.py
+++ b/calm/movelist.py
@@ -34,10 +34,13 @@ from . import utils
 #
 
 class MoveList(object):
-    def __init__(self):
+    def __init__(self, basedir=None):
         # a movelist is a dict with relative directory paths for keys and a list
         # of filenames for each value
         self.movelist = defaultdict(list)
+        # the directory the paths are relative to (by default the 'relarea')
+        if basedir:
+            self.basedir = basedir
 
     def __len__(self):
         return len(self.movelist)
@@ -69,10 +72,10 @@ class MoveList(object):
                     else:
                         logging.error("can't %s %s, as it doesn't exist" % (verb, f))
 
-    def move_to_relarea(self, m, args):
+    def move_to_relarea(self, m, args, desc):
         if self.movelist:
-            logging.info("move from %s's upload area to release area:" % (m.name))
-        self._move(args, m.homedir(), args.rel_area, 'deploy')
+            logging.info("move from %s's %s area to release area:" % (m.name, desc))
+        self._move(args, self.basedir, args.rel_area, 'deploy')
 
     def move_to_vault(self, args):
         if self.movelist:
diff --git a/calm/uploads.py b/calm/uploads.py
index 961b397..3bb8faa 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -52,11 +52,12 @@ ScanResult = namedtuple('ScanResult', 'error,packages,to_relarea,to_vault,remove
 #
 #
 
-def scan(m, all_packages, arch, args):
-    basedir = os.path.join(m.homedir(), arch)
+def scan(scandir, m, all_packages, arch, args):
+    homedir = os.path.join(scandir, m.name)
+    basedir = os.path.join(homedir, arch)
 
     packages = defaultdict(package.Package)
-    move = MoveList()
+    move = MoveList(homedir)
     vault = MoveList()
     remove = []
     remove_success = []
@@ -80,7 +81,7 @@ def scan(m, all_packages, arch, args):
 
     # scan package directories
     for (dirpath, _subdirs, files) in os.walk(os.path.join(basedir, 'release')):
-        relpath = os.path.relpath(dirpath, m.homedir())
+        relpath = os.path.relpath(dirpath, homedir)
         removed_files = []
 
         # filter out files we don't need to consider
@@ -277,7 +278,7 @@ def scan(m, all_packages, arch, args):
 
         # read and validate package
         if files:
-            if package.read_package_dir(packages, m.homedir(), dirpath, files, remove=removed_files, upload=True):
+            if package.read_package_dir(packages, homedir, dirpath, files, remove=removed_files, upload=True):
                 error = True
 
     # always consider timestamp as checked during a dry-run, so it is never
diff --git a/test/test_calm.py b/test/test_calm.py
index fe05f9a..e34ac99 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -48,6 +48,8 @@ import calm.package as package
 import calm.pkg2html as pkg2html
 import calm.uploads as uploads
 
+ARGDIRS = ['rel_area', 'homedir', 'htdocs', 'stagingdir', 'vault']
+
 
 #
 # helper functions
@@ -351,7 +353,7 @@ class CalmTest(unittest.TestCase):
         for (f, t) in ready_fns:
             os.system('touch %s "%s"' % (t, f))
 
-        scan_result = uploads.scan(m, pkglist + ['not-on-maintainer-list'], args.arch, args)
+        scan_result = uploads.scan('testdata/homes', m, pkglist + ['not-on-maintainer-list'], args.arch, args)
 
         os.chdir(oldcwd)
         shutil.rmtree(test_root)
@@ -395,12 +397,13 @@ class CalmTest(unittest.TestCase):
 
         args = types.SimpleNamespace()
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
         shutil.copytree('testdata/relarea', args.rel_area)
         shutil.copytree('testdata/homes.conflict', args.homedir)
+        os.mkdir(args.stagingdir)
 
         args.dryrun = False
         args.email = None
@@ -417,7 +420,7 @@ class CalmTest(unittest.TestCase):
         state.packages = calm.calm.process_uploads(args, state)
         self.assertTrue(state.packages)
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             with self.subTest(directory=d):
                 dirlist = capture_dirtree(getattr(args, d))
                 compare_with_expected_file(self, 'testdata/conflict', dirlist, d)
@@ -428,7 +431,7 @@ class CalmTest(unittest.TestCase):
 
         args = types.SimpleNamespace()
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             setattr(args, d, tempfile.mktemp())
             logging.info('%s = %s', d, getattr(args, d))
 
@@ -446,6 +449,7 @@ class CalmTest(unittest.TestCase):
 
         shutil.copytree('testdata/relarea', args.rel_area)
         shutil.copytree('testdata/homes', args.homedir)
+        os.mkdir(args.stagingdir)
 
         # set appropriate !readys
         m_homedir = os.path.join(args.homedir, 'Blooey McFooey')
@@ -472,7 +476,7 @@ class CalmTest(unittest.TestCase):
             results = re.sub('generated at .*', 'generated at 2016-09-13 21:04:40 BST', results, 1)
             compare_with_expected_file(self, 'testdata/process_arch', (results,), 'setup.ini')
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             with self.subTest(directory=d):
                 dirlist = capture_dirtree(getattr(args, d))
                 compare_with_expected_file(self, 'testdata/process_arch', dirlist, d)
@@ -483,7 +487,7 @@ class CalmTest(unittest.TestCase):
             del j['timestamp']
             compare_with_expected_file(self, 'testdata/process_arch', json.dumps(j, sort_keys=True, indent=4), 'packages.json')
 
-        for d in ['rel_area', 'homedir', 'htdocs', 'vault']:
+        for d in ARGDIRS:
             shutil.rmtree(getattr(args, d))
 
     @classmethod
diff --git a/test/testdata/conflict/stagingdir.expected b/test/testdata/conflict/stagingdir.expected
new file mode 100644
index 0000000..e82d3c8
--- /dev/null
+++ b/test/testdata/conflict/stagingdir.expected
@@ -0,0 +1 @@
+{'.': []}
diff --git a/test/testdata/process_arch/stagingdir.expected b/test/testdata/process_arch/stagingdir.expected
new file mode 100644
index 0000000..e82d3c8
--- /dev/null
+++ b/test/testdata/process_arch/stagingdir.expected
@@ -0,0 +1 @@
+{'.': []}