Add a tool to fix hint files with invalid hints

Add a tool to fix .hint files containing keys which are no longer valid
for that type of package (source or install).

Also fix invalid keys in -src.hint created at upload.

5 files changed, 150 insertions, 32 deletions

diff --git a/calm/fix-invalid-key-hint.py b/calm/fix-invalid-key-hint.py
new file mode 100644
index 0000000..5f3340c
--- /dev/null
+++ b/calm/fix-invalid-key-hint.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021 Jon Turney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import argparse
+import logging
+import os
+import sys
+
+from . import common_constants
+from . import fixes
+
+
+#
+#
+#
+
+def fix_hints(relarea, packages):
+    for (dirpath, _subdirs, files) in os.walk(relarea):
+
+        # only apply to listed packages, if specified
+        if packages:
+            relpath = os.path.relpath(dirpath, relarea)
+            relpath = relpath.split(os.path.sep)
+            if (len(relpath) < 3) or (relpath[2] not in packages):
+                continue
+
+        for f in files:
+            if f.endswith('.hint') and f != 'override.hint':
+                if fixes.fix_hint(dirpath, f, '', ['invalid_keys']):
+                    logging.warning('fixed hints %s' % f)
+
+
+#
+#
+#
+
+def main():
+    relarea_default = common_constants.FTP
+
+    parser = argparse.ArgumentParser(description='fix invalid keys in hint')
+
+    parser.add_argument('package', nargs='*', metavar='PACKAGE')
+    parser.add_argument('-v', '--verbose', action='count', dest='verbose', help='verbose output', default=0)
+    parser.add_argument('--releasearea', action='store', metavar='DIR', help="release directory (default: " + relarea_default + ")", default=relarea_default, dest='relarea')
+    # XXX: should take an argument listing fixes to apply
+
+    (args) = parser.parse_args()
+    if args.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+
+    logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s')
+
+    fix_hints(args.relarea, args.package)
+
+
+#
+#
+#
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/calm/fix-missing-src-hint-info.py b/calm/fix-missing-src-hint-info.py
index c1c9d28..88142a6 100644
--- a/calm/fix-missing-src-hint-info.py
+++ b/calm/fix-missing-src-hint-info.py
@@ -50,7 +50,7 @@ def fix_hints(relarea, packages):
                     logging.error('hint %s missing' % hf)
                     continue
 
-                fixes.fix_homepage_src_hint(dirpath, hf, f)
+                fixes.fix_hint(dirpath, hf, f, ['homepage'])
 
 
 #
diff --git a/calm/fixes.py b/calm/fixes.py
index e7a9839..613e361 100644
--- a/calm/fixes.py
+++ b/calm/fixes.py
@@ -84,16 +84,7 @@ def follow_redirect(homepage):
     return homepage
 
 
-def fix_homepage_src_hint(dirpath, hf, tf):
-    pn = os.path.basename(dirpath)
-    hintfile = os.path.join(dirpath, hf)
-    hints = hint.hint_file_parse(hintfile, hint.spvr)
-
-    hints.pop('parse-warnings', None)
-    if 'parse-errors' in hints:
-        logging.error('invalid hints %s' % hf)
-        return
-
+def _fix_homepage_src_hint(hints, dirpath, _hf, tf):
     # already present?
     if 'homepage' in hints:
         homepage = hints['homepage']
@@ -109,6 +100,7 @@ def fix_homepage_src_hint(dirpath, hf, tf):
                 if match:
                     if homepage:
                         logging.warning('multiple HOMEPAGE lines in .cygport in srcpkg %s', tf)
+                    pn = os.path.basename(dirpath)
                     homepage = match.group(2)
                     homepage = re.sub(r'\$({|)(PN|ORIG_PN|NAME)(}|)', pn, homepage)
 
@@ -118,10 +110,11 @@ def fix_homepage_src_hint(dirpath, hf, tf):
 
         if not homepage:
             logging.info('cannot determine homepage: from srcpkg %s' % tf)
-            return
+            return False
 
         logging.info('adding homepage:%s to hints for srcpkg %s' % (homepage, tf))
 
+    # check if redirect?
     redirect_homepage = follow_redirect(homepage)
 
     # trivial URL transformations aren't interesting
@@ -138,8 +131,49 @@ def fix_homepage_src_hint(dirpath, hf, tf):
             if not redirect_homepage.startswith(homepage):
                 logging.warning('homepage:%s permanently redirects to %s' % (homepage, redirect_homepage))
 
-    # write updated hints
+    # changed?
     if homepage != hints.get('homepage', None):
         hints['homepage'] = homepage
+        return True
+
+    return False
+
+
+def _fix_invalid_keys_hint(hints, _dirpath, hf, _tf):
+    # eliminate keys that aren't appropriate to the package type
+    if hf.endswith('-src.hint'):
+        valid_keys = hint.hintkeys[hint.spvr]
+    else:
+        valid_keys = hint.hintkeys[hint.pvr]
+
+    changed = False
+    for k in list(hints.keys()):
+        if k not in valid_keys:
+            logging.debug("discarding invalid key '%s:' from hint '%s'" % (k, hf))
+            hints.pop(k, None)
+            changed = True
+
+    return changed
+
+
+def fix_hint(dirpath, hf, tf, problems):
+    hintfile = os.path.join(dirpath, hf)
+    hints = hint.hint_file_parse(hintfile, None)
+
+    hints.pop('parse-warnings', None)
+    if 'parse-errors' in hints:
+        logging.error('invalid hints %s' % hf)
+        return
+
+    changed = False
+    if 'homepage' in problems:
+        changed = _fix_homepage_src_hint(hints, dirpath, hf, tf)
+    if 'invalid_keys' in problems:
+        changed = _fix_invalid_keys_hint(hints, dirpath, hf, tf) or changed
+
+    # write updated hints
+    if changed:
         shutil.copy2(hintfile, hintfile + '.bak')
         hint.hint_file_write(hintfile, hints)
+
+    return changed
diff --git a/calm/hint.py b/calm/hint.py
index 2c31714..83dd35c 100755
--- a/calm/hint.py
+++ b/calm/hint.py
@@ -197,7 +197,7 @@ def hint_file_parse(fn, kind):
     errors = []
     warnings = []
 
-    assert(kind in hintkeys)
+    assert((kind in hintkeys) or (kind is None))
 
     with open(fn, 'rb') as f:
         c = f.read()
@@ -221,21 +221,26 @@ def hint_file_parse(fn, kind):
                     key = match.group(1)
                     value = match.group(2)
 
-                    if key not in hintkeys[kind]:
-                        errors.append('unknown key %s at line %d' % (key, i))
-                        continue
-                    valtype = hintkeys[kind][key]
+                    if kind is not None:
+                        if key not in hintkeys[kind]:
+                            errors.append('unknown key %s at line %d' % (key, i))
+                            continue
+                        valtype = hintkeys[kind][key]
 
-                    # check if the key occurs more than once
-                    if key in hints:
-                        errors.append('duplicate key %s' % (key))
+                        # check if the key occurs more than once
+                        if key in hints:
+                            errors.append('duplicate key %s' % (key))
 
-                    # check the value meets any key-specific constraints
-                    if (valtype == 'val') and (len(value) == 0):
-                        errors.append('%s has empty value' % (key))
+                        # check the value meets any key-specific constraints
+                        if (valtype == 'val') and (len(value) == 0):
+                            errors.append('%s has empty value' % (key))
 
-                    if (valtype == 'noval') and (len(value) != 0):
-                        errors.append("%s has non-empty value '%s'" % (key, value))
+                        if (valtype == 'noval') and (len(value) != 0):
+                            errors.append("%s has non-empty value '%s'" % (key, value))
+
+                        # only 'ldesc' and 'message' are allowed a multi-line value
+                        if (valtype != 'multilineval') and (len(value.splitlines()) > 1):
+                            errors.append("key %s has multi-line value" % (key))
 
                     # validate all categories are in the category list (case-insensitively)
                     if key == 'category':
@@ -266,10 +271,6 @@ def hint_file_parse(fn, kind):
                             warnings.append("sdesc contains '  '")
                             value = value.replace('  ', ' ')
 
-                    # only 'ldesc' and 'message' are allowed a multi-line value
-                    if (valtype != 'multilineval') and (len(value.splitlines()) > 1):
-                        errors.append("key %s has multi-line value" % (key))
-
                     # message must have an id and some text
                     if key == 'message':
                         if not re.match(r'(\S+)\s+(\S.*)', value):
@@ -289,7 +290,7 @@ def hint_file_parse(fn, kind):
 
             # for the pvr kind, 'category' and 'sdesc' must be present
             # XXX: genini also requires 'requires' but that seems wrong
-            if kind != override:
+            if (kind == pvr) or (kind == spvr):
                 mandatory = ['category', 'sdesc']
                 for k in mandatory:
                     if k not in hints:
diff --git a/calm/uploads.py b/calm/uploads.py
index 9998e46..961b397 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -193,8 +193,10 @@ def scan(m, all_packages, arch, args):
                         remove.append(os.path.join(dirpath, old))
 
                 # see if we can fix-up missing homepage: in -src.hint file
+                # check homepage: for liveliness and redirection
+                # discard any keys which are invalid in a -src.hint
                 if (new in files):
-                    fixes.fix_homepage_src_hint(dirpath, new, f)
+                    fixes.fix_hint(dirpath, new, f, ['homepage', 'invalid_keys'])
 
         # filter out files we don't need to consider
         for f in sorted(files):