Add zstd to permitted archive compressions

Use the xtarfile module to handle zstd compressed archive.

Note this isn't quite as straightforward as 'import xtarfile as
tarfile', because it isn't precisely a drop-in replacement for tarfile,
because:

(i) the 'mode' argument to xtarfile.open() is mandatory, and
(ii) it doesn't re-export tarfile exception classes such as tarfile.ReadError

16 files changed, 47 insertions, 10 deletions

diff --git a/calm/common_constants.py b/calm/common_constants.py
index 09c96df..05473f5 100644
--- a/calm/common_constants.py
+++ b/calm/common_constants.py
@@ -83,5 +83,5 @@ if os.uname()[1] == 'tambora':
     MAILHOST = 'allegra'
 
 # package compressions
-PACKAGE_COMPRESSIONS = ['bz2', 'gz', 'lzma', 'xz']
+PACKAGE_COMPRESSIONS = ['bz2', 'gz', 'lzma', 'xz', 'zst']
 PACKAGE_COMPRESSIONS_RE = r'\.(' + '|'.join(PACKAGE_COMPRESSIONS) + r')'
diff --git a/calm/find-duplicates.py b/calm/find-duplicates.py
index 9c5b94e..9e1aabb 100644
--- a/calm/find-duplicates.py
+++ b/calm/find-duplicates.py
@@ -27,6 +27,7 @@ import re
 import os
 import sys
 import tarfile
+import xtarfile
 
 from . import common_constants
 
@@ -65,7 +66,7 @@ def read_tar(f):
     result = {}
 
     try:
-        with tarfile.open(f) as t:
+        with xtarfile.open(f, mode='r') as t:
             for m in t:
                 if m.isfile():
                     f = t.extractfile(m)
diff --git a/calm/fixes.py b/calm/fixes.py
index 063ebcd..e7a9839 100644
--- a/calm/fixes.py
+++ b/calm/fixes.py
@@ -30,13 +30,14 @@ import socket
 import tarfile
 import urllib.request
 import urllib.error
+import xtarfile
 
 from . import hint
 
 
 def read_cygport(dirpath, tf):
     try:
-        with tarfile.open(os.path.join(dirpath, tf)) as a:
+        with xtarfile.open(os.path.join(dirpath, tf), mode='r') as a:
             cygports = [m for m in a.getmembers() if m.name.endswith('.cygport')]
 
             if len(cygports) != 1:
diff --git a/calm/package.py b/calm/package.py
index 16ec04a..b28907a 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -35,9 +35,9 @@ import logging
 import os
 import pprint
 import re
-import tarfile
 import textwrap
 import time
+import xtarfile
 
 from .version import SetupVersion
 from .movelist import MoveList
@@ -455,7 +455,7 @@ def tarfile_is_empty(tf):
 
     # if it's really a tar file, does it contain zero files?
     try:
-        with tarfile.open(tf) as a:
+        with xtarfile.open(tf, mode='r') as a:
             if any(a) == 0:
                 return True
     except Exception as e:
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 32709bb..ffd16b0 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -49,9 +49,9 @@ import os
 import re
 import string
 import sys
-import tarfile
 import textwrap
 import time
+import xtarfile
 
 from .version import SetupVersion
 from . import common_constants
@@ -482,7 +482,7 @@ def write_arch_listing(args, packages, arch):
                             pass
                         else:
                             try:
-                                with tarfile.open(tf) as a:
+                                with xtarfile.open(tf, mode='r') as a:
                                     for i in a:
                                         print('    %-16s%12d %s' % (time.strftime('%Y-%m-%d %H:%M', time.gmtime(i.mtime)), i.size, i.name), file=f, end='')
                                         if i.isdir():
diff --git a/calm/uploads.py b/calm/uploads.py
index 376753d..9998e46 100644
--- a/calm/uploads.py
+++ b/calm/uploads.py
@@ -31,8 +31,8 @@ import os
 import logging
 import re
 import shutil
-import tarfile
 import time
+import xtarfile
 
 from .movelist import MoveList
 from . import common_constants
@@ -238,8 +238,9 @@ def scan(m, all_packages, arch, args):
                 try:
                     # we need to extract all of an archive contents to validate
                     # it
-                    with tarfile.open(fn) as a:
+                    with xtarfile.open(fn, mode='r') as a:
                         a.getmembers()
+
                 except Exception as e:
                     valid = False
                     logging.error("exception %s while reading %s" % (type(e).__name__, fn))
diff --git a/requirements.txt b/requirements.txt
index 3e80396..bc5496c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ flake8-import-order == 0.14.1
 lockfile
 pycodestyle
 python-daemon
+xtarfile[zstd]
diff --git a/setup.py b/setup.py
index 31f5eff..291be8b 100644
--- a/setup.py
+++ b/setup.py
@@ -19,4 +19,7 @@ setup(
     },
     url='https://cygwin.com/git/?p=cygwin-apps/calm.git',
     test_suite='tests',
+    install_requires=[
+        'xtarfile[zstd]',
+    ],
 )
diff --git a/test/test_calm.py b/test/test_calm.py
index fa48097..4c24313 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -326,7 +326,7 @@ class CalmTest(unittest.TestCase):
         oldcwd = os.getcwd()
         os.chdir(test_root)
 
-        pkglist = ['after-ready', 'not-ready', 'testpackage', 'testpackage2']
+        pkglist = ['after-ready', 'not-ready', 'testpackage', 'testpackage2', 'testpackage-zstd']
 
         mlist = {}
         mlist = maintainers.add_directories(mlist, 'testdata/homes')
@@ -335,6 +335,7 @@ class CalmTest(unittest.TestCase):
 
         ready_fns = [(os.path.join(m.homedir(), 'x86', 'release', 'testpackage', '!ready'), ''),
                      (os.path.join(m.homedir(), 'x86', 'release', 'testpackage2', 'testpackage2-subpackage', '!ready'), ''),
+                     (os.path.join(m.homedir(), 'x86', 'release', 'testpackage-zstd', '!ready'), ''),
                      (os.path.join(m.homedir(), 'x86', 'release', 'after-ready', '!ready'), '-t 198709011700'),
                      (os.path.join(m.homedir(), 'x86', 'release', 'corrupt', '!ready'), '')]
         for (f, t) in ready_fns:
diff --git a/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.hint b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.hint
new file mode 100644
index 0000000..386cdb9
--- /dev/null
+++ b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.hint
@@ -0,0 +1,6 @@
+category: Base
+build-depends: cygport 
+sdesc: "test package (zstd compressed)"
+ldesc: "test package (zstd compressed)"
+skip:
+
diff --git a/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.tar.zst b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.tar.zst
new file mode 100644
index 0000000..61c3fe0
--- /dev/null
+++ b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1-src.tar.zst
diff --git a/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.hint b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.hint
new file mode 100644
index 0000000..e5d07d2
--- /dev/null
+++ b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.hint
@@ -0,0 +1,5 @@
+category: Base
+requires:   
+sdesc: "test package (zstd compressed)"
+ldesc: "test package (zstd compressed)"
+
diff --git a/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.tar.zst b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.tar.zst
new file mode 100644
index 0000000..e21ce6d
--- /dev/null
+++ b/test/testdata/homes/Blooey McFooey/x86/release/testpackage-zstd/testpackage-zstd-1.0-1.tar.zst
diff --git a/test/testdata/process_arch/homedir.expected b/test/testdata/process_arch/homedir.expected
index 42e9902..7c07d4f 100644
--- a/test/testdata/process_arch/homedir.expected
+++ b/test/testdata/process_arch/homedir.expected
@@ -13,6 +13,10 @@
  'Blooey McFooey/x86/release/per-version': [],
  'Blooey McFooey/x86/release/per-version-replacement-hint-only': [],
  'Blooey McFooey/x86/release/testpackage': ['testpackage-1.0-1-src.hint.bak'],
+ 'Blooey McFooey/x86/release/testpackage-zstd': ['testpackage-zstd-1.0-1-src.hint',
+                                                 'testpackage-zstd-1.0-1-src.tar.zst',
+                                                 'testpackage-zstd-1.0-1.hint',
+                                                 'testpackage-zstd-1.0-1.tar.zst'],
  'Blooey McFooey/x86/release/testpackage/testpackage-subpackage': [],
  'Blooey McFooey/x86/release/testpackage2': ['setup.hint', 'testpackage2-1.0-1.tar.bz2'],
  'Blooey McFooey/x86/release/testpackage2/testpackage2-subpackage': ['inprogress.SftpXFR.1234',
diff --git a/test/testdata/uploads/move.expected b/test/testdata/uploads/move.expected
index bfb7728..c0112d2 100644
--- a/test/testdata/uploads/move.expected
+++ b/test/testdata/uploads/move.expected
@@ -2,6 +2,10 @@
                              'testpackage-1.0-1-src.tar.bz2',
                              'testpackage-1.0-1.hint',
                              'testpackage-1.0-1.tar.bz2'],
+ 'x86/release/testpackage-zstd': ['testpackage-zstd-1.0-1-src.hint',
+                                  'testpackage-zstd-1.0-1-src.tar.zst',
+                                  'testpackage-zstd-1.0-1.hint',
+                                  'testpackage-zstd-1.0-1.tar.zst'],
  'x86/release/testpackage/testpackage-subpackage': ['testpackage-subpackage-1.0-1.hint',
                                                     'testpackage-subpackage-1.0-1.tar.bz2'],
  'x86/release/testpackage2/testpackage2-subpackage': ['testpackage2-subpackage-1.0-1.hint',
diff --git a/test/testdata/uploads/pkglist.expected b/test/testdata/uploads/pkglist.expected
index 4691268..e5cbbbf 100644
--- a/test/testdata/uploads/pkglist.expected
+++ b/test/testdata/uploads/pkglist.expected
@@ -19,6 +19,16 @@
            'ldesc': '"A test subpackage"',
            'category': 'Devel',
            'external-source': 'testpackage-src'}}, {}, False),
+ 'testpackage-zstd': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1.tar.zst': Tar('testpackage-zstd-1.0-1.tar.zst', 'x86/release/testpackage-zstd', '044066c54c036190f9b0496ccf31f74748d209cce961352e19631876d5abd79ef6d2b34edfb955b8d1a7a781294ee0636bb1305afe410b34562367a2cb77988d', 98, False)}}, {'1.0-1': {'category': 'Base',
+           'requires': '',
+           'sdesc': '"test package (zstd compressed)"',
+           'ldesc': '"test package (zstd compressed)"',
+           'depends': ''}}, {}, False),
+ 'testpackage-zstd-src': Package('testpackage-zstd', {'1.0-1': {'testpackage-zstd-1.0-1-src.tar.zst': Tar('testpackage-zstd-1.0-1-src.tar.zst', 'x86/release/testpackage-zstd', '90561ec4dad76268773856cbdda891b0e7b53f26492777f1ff76757844cb47124396feb76f1e30bc1baa680f1d788de21d89e612faeb30b5039b210ca9186434', 313, False)}}, {'1.0-1': {'category': 'Base',
+           'build-depends': 'cygport',
+           'sdesc': '"test package (zstd compressed)"',
+           'ldesc': '"test package (zstd compressed)"',
+           'skip': ''}}, {}, True),
  'testpackage2-subpackage': Package('testpackage2/testpackage2-subpackage', {'1.0-1': {'testpackage2-subpackage-1.0-1.tar.bz2': Tar('testpackage2-subpackage-1.0-1.tar.bz2', 'x86/release/testpackage2/testpackage2-subpackage', 'c4bf8e28d71b532e2b741e2931906dec0f0a70d4d051c0503476f864a5228f43765ae3342aafcebfd5a1738073537726b2bfbbd89c6da939a5f46d95aca3feaf', 46, True)}}, {'1.0-1': {'sdesc': '"A test subpackage 2"',
            'ldesc': '"A test subpackage 2"',
            'category': 'Devel'}}, {}, False)}