Use separate .hint for source and install packages

This makes keeping track of stale hints much simpler and more reliable

Add a tool to add missing -src.hint to the relarea, by copying (if
install package also exists) or moving (if it doesn't) the corresponding
.hint

Fix up uploads which don't have a -src.hint, by copying or moving the
corresponding .hint likewise.

3 files changed, 95 insertions, 22 deletions

diff --git a/calm/fix-missing-src-hint.py b/calm/fix-missing-src-hint.py
new file mode 100644
index 0000000..46d2f19
--- /dev/null
+++ b/calm/fix-missing-src-hint.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2020 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import logging
+import os
+import re
+import shutil
+import sys
+
+from . import common_constants
+
+#
+#
+#
+
+
+def fix_hints(relarea):
+    for (dirpath, subdirs, files) in os.walk(relarea):
+        for f in files:
+            match = re.match(r'^(.*)-src\.tar\.(bz2|gz|lzma|xz)$', f)
+            if match:
+                pvr = match.group(1)
+                old = pvr + '.hint'
+                new = pvr + '-src.hint'
+                if (old in files) and (new not in files):
+                    logging.info("copying '%s' to '%s'" % (old, new))
+                    shutil.copy2(os.path.join(dirpath, old), os.path.join(dirpath, new))
+                    if f.replace('-src', '') not in files:
+                        logging.info("removing '%s'" % (old))
+                        os.rename(os.path.join(dirpath, old), os.path.join(dirpath, old + '.bak'))
+
+
+#
+#
+#
+
+
+if __name__ == "__main__":
+    relarea_default = common_constants.FTP
+
+    parser = argparse.ArgumentParser(description='src hint creator')
+    parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output', default=0)
+    parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='relarea')
+    (args) = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+
+    logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s')
+
+    fix_hints(args.relarea)
diff --git a/calm/package.py b/calm/package.py
index dca89b2..99a76e5 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -225,7 +225,10 @@ def read_package_dir(packages, basedir, dirpath, files, remove=[], upload=False)
             fl['all'].append(f)
             files.remove(f)
         elif re.match(r'^' + re.escape(p) + r'.*\.hint$', f):
-            fl['all'].append(f)
+            if f.endswith('-src.hint'):
+                fl[Kind.source].append(f)
+            else:
+                fl[Kind.binary].append(f)
             files.remove(f)
         elif re.match(r'^' + re.escape(p) + r'.*\.tar\.(bz2|gz|lzma|xz)$', f):
             if '-src.tar' in f:
@@ -372,7 +375,7 @@ def read_one_package(packages, p, relpath, dirpath, files, remove, kind):
     hints = {}
     actual_tars = {}
     for vr in vr_list:
-        hint_fn = '%s-%s.hint' % (p, vr)
+        hint_fn = '%s-%s%s.hint' % (p, vr, '-src' if kind == Kind.source else '')
         if hint_fn in files:
             # is there a PVR.hint file?
             pvr_hint = read_hints(p, os.path.join(dirpath, hint_fn), hint.pvr)
@@ -1403,26 +1406,8 @@ def stale_packages(packages):
 
         for v in po.hints:
             # if there's a pvr.hint without a fresh source or install of the
-            # same version, move it as well (this is complicated as the hint may
-            # be used by both a binary and source package; give the source
-            # package ownership, if it exists)
-
-            if po.kind == Kind.source:
-                if ((po.orig_name in packages) and
-                    (v in packages[po.orig_name].vermap) and
-                    ('install' in packages[po.orig_name].vermap[v])):
-                    sourceless = packages[po.orig_name].tar(v, 'install').sourceless
-                else:
-                    sourceless = False
-            else:
-                if 'install' in po.vermap.get(v, {}):
-                    sourceless = po.tar(v, 'install').sourceless
-                else:
-                    sourceless = False
-
-            binary_owns_hint = ('external-source' in po.hints[v].hints) or sourceless
-
-            if all_stale.get(v, True) and ((po.kind == Kind.binary) == binary_owns_hint):
+            # same version, move it as well
+            if all_stale.get(v, True):
                 stale.add(po.hints[v].path, po.hints[v].fn)
                 logging.debug("package '%s' version '%s' hint is stale" % (pn, v))
 
diff --git a/calm/uploads.py b/calm/uploads.py
index 747b0dc..9ea62c5 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -30,6 +30,7 @@ import filecmp
 import os
 import logging
 import re
+import shutil
 import tarfile
 import time
 
@@ -169,6 +170,21 @@ def scan(m, all_packages, arch, args):
                 files.remove(old)
                 files.append(new)
 
+        # see if we can fix-up missing -src.hint file
+        for f in sorted(files):
+            match = re.match(r'^(.*)-src\.tar\.(bz2|gz|lzma|xz)$', f)
+            if match:
+                pvr = match.group(1)
+                old = pvr + '.hint'
+                new = pvr + '-src.hint'
+                if (old in files) and (new not in files):
+                    logging.warning("copying '%s' to '%s'" % (old, new))
+                    shutil.copy2(os.path.join(dirpath, old), os.path.join(dirpath, new))
+                    files.append(new)
+                    if f.replace('-src', '') not in files:
+                        logging.info("ignoring '%s'" % (old))
+                        files.remove(old)
+
         # filter out files we don't need to consider
         for f in sorted(files):
             fn = os.path.join(dirpath, f)