Various fixes and improvements to dedup tool

3 files changed, 78 insertions, 9 deletions

diff --git a/calm/dedupsrc.py b/calm/dedupsrc.py
index 8462fa7..d8a8e84 100755..100644
--- a/calm/dedupsrc.py
+++ b/calm/dedupsrc.py
@@ -24,6 +24,7 @@
 #
 # Move a given source archive to src/ (assuming it is indentical in x86/ and
 # x86_64/) and adjust hints appropriately.
+# (XXX: could probably be extended to move to noarch/ if not source, as well)
 #
 
 import argparse
@@ -35,6 +36,8 @@ import sys
 from . import common_constants
 from . import hint
 
+binary_only_hints = ['requires', 'depends', 'obsoletes', 'external-source']
+
 #
 #
 #
@@ -45,6 +48,38 @@ def hint_file_write(fn, hints):
         for k, v in hints.items():
             print("%s: %s" % (k, v), file=f)
 
+
+#
+#
+#
+
+def invent_sdesc(path, vr):
+    for (dirpath, subdirs, files) in os.walk(path):
+        # debuginfo packages never have a good sdesc
+        if 'debuginfo' in dirpath:
+            continue
+
+        # but just pick the sdesc from first sub-package which has one ...
+        for f in files:
+            if re.match('^.*-' + re.escape(vr) + '.hint$', f):
+                hints = hint.hint_file_parse(os.path.join(dirpath, f), hint.pvr)
+                if 'sdesc' in hints:
+                    sdesc = hints['sdesc']
+
+                    # ... which doesn't contain 'Obsoleted'
+                    if 'Obsoleted' in sdesc:
+                        continue
+
+                    # remove anything inside parentheses at the end of quoted
+                    # sdesc
+                    sdesc = re.sub(r'"(.*)"', r'\1', sdesc)
+                    sdesc = re.sub(r'(\(.*?\))$', '', sdesc)
+                    sdesc = sdesc.strip()
+                    sdesc = '"' + sdesc + '"'
+
+                    return sdesc
+
+    return None
 #
 #
 #
@@ -81,21 +116,37 @@ def dedup(archive, relarea):
 
         hints[arch] = hint.hint_file_parse(hint_pathname, hint.pvr)
 
+        # remove hints which only have meaning for binary packages
+        #
+        # (requires: tends to have libgcc1 more often on x86, so otherwise this
+        # would cause spurious differences between hints to be reported)
+        for h in binary_only_hints:
+            if h in hints[arch]:
+                del hints[arch][h]
+
     if hints['x86'] != hints['x86_64']:
         print('hints for %s-%s differ between arches' % (p, vr))
         return 1
 
+    if ('skip' in hints['x86']) and (len(hints['x86']) == 1):
+        print('hints for %s-%s is skip: only' % (p, vr))
+        hints['x86']['category'] = ''
+        # if hint only contains skip:, try to come up with a plausible sdesc
+        sdesc = invent_sdesc(os.path.join(relarea, 'x86', path), vr)
+        if sdesc:
+            print('suggested sdesc is %s' % (sdesc))
+            hints['x86']['sdesc'] = sdesc
+
+    if 'sdesc' not in hints['x86']:
+        print('hints for %s-%s has no sdesc:' % (p, vr))
+        return 1
+
     # ensure target directory exists
     try:
         os.makedirs(os.path.join(relarea, 'src', path, p + '-src'))
     except FileExistsError:
         pass
 
-    # move the src files to src/
-    for arch in ['x86', 'x86_64']:
-        print('%s -> %s' % (os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename)))
-        os.rename(os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename))
-
     # write .hint file for new -src package
     src_hints = copy.copy(hints['x86'])
 
@@ -104,16 +155,21 @@ def dedup(archive, relarea):
         sdesc += ' (source code)'
         src_hints['sdesc'] = '"' + sdesc + '"'
 
-    if 'requires' in src_hints:
-        del src_hints['requires']
+    if 'Source' not in src_hints['category']:
+        src_hints['category'] = src_hints['category'] + ' Source'
 
-    if 'external-source' in src_hints:
-        del src_hints['external-source']
+    if 'parse-warnings' in src_hints:
+        del src_hints['parse-warnings']
 
     to_hint_pathname = os.path.join(relarea, 'src', path, p + '-src', to_hint_filename)
     print('writing %s' % (to_hint_pathname))
     hint_file_write(to_hint_pathname, src_hints)
 
+    # move the src files to src/
+    for arch in ['x86', 'x86_64']:
+        print('%s -> %s' % (os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename)))
+        os.rename(os.path.join(relarea, arch, path, filename), os.path.join(relarea, 'src', path, p + '-src', to_filename))
+
     # adjust external-source in .hint for all subpackages
     for arch in ['x86', 'x86_64']:
         for (dirpath, subdirs, files) in os.walk(os.path.join(relarea, arch, path)):
@@ -122,6 +178,8 @@ def dedup(archive, relarea):
             if filename in files:
                 hint_pathname = os.path.join(dirpath, filename)
                 hints = hint.hint_file_parse(hint_pathname, hint.pvr)
+                if 'parse-warnings' in hints:
+                    del hints['parse-warnings']
                 if ('skip' in hints):
                     # p was source only, so no package remains
                     print('removing %s' % (hint_pathname))
diff --git a/calm/hint.py b/calm/hint.py
index 0c9fd50..f454fa8 100755
--- a/calm/hint.py
+++ b/calm/hint.py
@@ -124,6 +124,7 @@ categories = ['accessibility',
               'science',
               'security',
               'shells',
+              'source',  # added to all source packages created by deduplicator to ensure they have a category
               'sugar',
               'system',
               'tcl',
diff --git a/calm/package.py b/calm/package.py
index 2d00299..6a92692 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -671,6 +671,16 @@ def validate_packages(args, packages):
                         packages[es_p].is_used_by.add(p)
                         continue
 
+                # this is a bodge to follow external-source: which hasn't been
+                # updated following a source package de-duplication
+                es_p = es_p + '-src'
+                if es_p in packages:
+                    if 'source' in packages[es_p].vermap[v]:
+                        logging.warning("package '%s' version '%s' external-source: should be %s" % (p, v, es_p))
+                        packages[es_p].tar(v, 'source').is_used = True
+                        packages[es_p].is_used_by.add(p)
+                        continue
+
             # unless this package is marked as 'self-source'
             if p in past_mistakes.self_source:
                 continue