Refactor tests out of packaged module for dns plugins (#7599)

* Refactor tests out of module for certbot-dns-cloudflare

* Refactor tests out of module for certbot-dns-cloudxns

* Refactor tests out of module for certbot-dns-digitalocean

* Refactor tests out of module for certbot-dns-dnsimple

* Refactor tests out of module for certbot-dns-dnsmadeeasy

* Refactor tests out of module for certbot-dns-gehirn

* Refactor tests out of module for certbot-dns-google

* Refactor tests out of module for certbot-dns-linode

* Refactor tests out of module for certbot-dns-luadns

* Refactor tests out of module for certbot-dns-nsone

* Refactor tests out of module for certbot-dns-ovh

* Refactor tests out of module for certbot-dns-rfc2136

* Refactor tests out of module for certbot-dns-sakuracloud

* Refactor tests out of module for certbot-dns-route53

* Move certbot-dns-google testdata/ under tests/

* Use pytest for dns plugins

* Exclude pycache and .py[cod]

43 files changed, 294 insertions, 1 deletions

diff --git a/certbot-dns-cloudflare/MANIFEST.in b/certbot-dns-cloudflare/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-cloudflare/MANIFEST.in
+++ b/certbot-dns-cloudflare/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py
index c059a1ed5..05a85bb66 100644
--- a/certbot-dns-cloudflare/setup.py
+++ b/certbot-dns-cloudflare/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-cloudflare',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-cloudflare = certbot_dns_cloudflare._internal.dns_cloudflare:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_cloudflare',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py
index b24628b0d..b24628b0d 100644
--- a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py
+++ b/certbot-dns-cloudflare/tests/dns_cloudflare_test.py
diff --git a/certbot-dns-cloudxns/MANIFEST.in b/certbot-dns-cloudxns/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-cloudxns/MANIFEST.in
+++ b/certbot-dns-cloudxns/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py
index 399692ae5..189af0a55 100644
--- a/certbot-dns-cloudxns/setup.py
+++ b/certbot-dns-cloudxns/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-cloudxns',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-cloudxns = certbot_dns_cloudxns._internal.dns_cloudxns:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_cloudxns',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py
index 7b8d0944d..7b8d0944d 100644
--- a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py
+++ b/certbot-dns-cloudxns/tests/dns_cloudxns_test.py
diff --git a/certbot-dns-digitalocean/MANIFEST.in b/certbot-dns-digitalocean/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-digitalocean/MANIFEST.in
+++ b/certbot-dns-digitalocean/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py
index d626da66f..d5e500be0 100644
--- a/certbot-dns-digitalocean/setup.py
+++ b/certbot-dns-digitalocean/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -21,6 +23,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-digitalocean',
     version=version,
@@ -64,5 +80,7 @@ setup(
             'dns-digitalocean = certbot_dns_digitalocean._internal.dns_digitalocean:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_digitalocean',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py
index 71301a47c..71301a47c 100644
--- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py
+++ b/certbot-dns-digitalocean/tests/dns_digitalocean_test.py
diff --git a/certbot-dns-dnsimple/MANIFEST.in b/certbot-dns-dnsimple/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-dnsimple/MANIFEST.in
+++ b/certbot-dns-dnsimple/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py
index 3359fc578..09c90ff0d 100644
--- a/certbot-dns-dnsimple/setup.py
+++ b/certbot-dns-dnsimple/setup.py
@@ -1,6 +1,8 @@
 import os
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -32,6 +34,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-dnsimple',
     version=version,
@@ -75,5 +91,7 @@ setup(
             'dns-dnsimple = certbot_dns_dnsimple._internal.dns_dnsimple:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_dnsimple',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py
index ca5eb4f36..ca5eb4f36 100644
--- a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py
+++ b/certbot-dns-dnsimple/tests/dns_dnsimple_test.py
diff --git a/certbot-dns-dnsmadeeasy/MANIFEST.in b/certbot-dns-dnsmadeeasy/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-dnsmadeeasy/MANIFEST.in
+++ b/certbot-dns-dnsmadeeasy/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-dnsmadeeasy/setup.py b/certbot-dns-dnsmadeeasy/setup.py
index 5812765e4..8c9c73319 100644
--- a/certbot-dns-dnsmadeeasy/setup.py
+++ b/certbot-dns-dnsmadeeasy/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-dnsmadeeasy',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-dnsmadeeasy = certbot_dns_dnsmadeeasy._internal.dns_dnsmadeeasy:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_dnsmadeeasy',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy_test.py b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py
index b94cc7d05..b94cc7d05 100644
--- a/certbot-dns-dnsmadeeasy/certbot_dns_dnsmadeeasy/dns_dnsmadeeasy_test.py
+++ b/certbot-dns-dnsmadeeasy/tests/dns_dnsmadeeasy_test.py
diff --git a/certbot-dns-gehirn/MANIFEST.in b/certbot-dns-gehirn/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-gehirn/MANIFEST.in
+++ b/certbot-dns-gehirn/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-gehirn/setup.py b/certbot-dns-gehirn/setup.py
index 5fb46576b..cb7768a29 100644
--- a/certbot-dns-gehirn/setup.py
+++ b/certbot-dns-gehirn/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -19,6 +21,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-gehirn',
     version=version,
@@ -62,5 +78,7 @@ setup(
             'dns-gehirn = certbot_dns_gehirn._internal.dns_gehirn:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_gehirn',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py b/certbot-dns-gehirn/tests/dns_gehirn_test.py
index f5b95b6c3..f5b95b6c3 100644
--- a/certbot-dns-gehirn/certbot_dns_gehirn/dns_gehirn_test.py
+++ b/certbot-dns-gehirn/tests/dns_gehirn_test.py
diff --git a/certbot-dns-google/MANIFEST.in b/certbot-dns-google/MANIFEST.in
index c91330e38..a7301ee7f 100644
--- a/certbot-dns-google/MANIFEST.in
+++ b/certbot-dns-google/MANIFEST.in
@@ -2,3 +2,6 @@ include LICENSE.txt
 include README.rst
 recursive-include docs *
 recursive-include certbot_dns_google/testdata *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py
index d53862610..21ea2eb38 100644
--- a/certbot-dns-google/setup.py
+++ b/certbot-dns-google/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -23,6 +25,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-google',
     version=version,
@@ -66,5 +82,7 @@ setup(
             'dns-google = certbot_dns_google._internal.dns_google:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_google',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/tests/dns_google_test.py
index e91db58ab..e91db58ab 100644
--- a/certbot-dns-google/certbot_dns_google/dns_google_test.py
+++ b/certbot-dns-google/tests/dns_google_test.py
diff --git a/certbot-dns-google/certbot_dns_google/testdata/discovery.json b/certbot-dns-google/tests/testdata/discovery.json
index 79a406645..79a406645 100644
--- a/certbot-dns-google/certbot_dns_google/testdata/discovery.json
+++ b/certbot-dns-google/tests/testdata/discovery.json
diff --git a/certbot-dns-linode/MANIFEST.in b/certbot-dns-linode/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-linode/MANIFEST.in
+++ b/certbot-dns-linode/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-linode/setup.py b/certbot-dns-linode/setup.py
index ce2c91078..b6e20ec74 100644
--- a/certbot-dns-linode/setup.py
+++ b/certbot-dns-linode/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 version = '1.0.0.dev0'
 
@@ -18,6 +20,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-linode',
     version=version,
@@ -61,5 +77,7 @@ setup(
             'dns-linode = certbot_dns_linode._internal.dns_linode:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_linode',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py b/certbot-dns-linode/tests/dns_linode_test.py
index 3cf615486..3cf615486 100644
--- a/certbot-dns-linode/certbot_dns_linode/dns_linode_test.py
+++ b/certbot-dns-linode/tests/dns_linode_test.py
diff --git a/certbot-dns-luadns/MANIFEST.in b/certbot-dns-luadns/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-luadns/MANIFEST.in
+++ b/certbot-dns-luadns/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-luadns/setup.py b/certbot-dns-luadns/setup.py
index eb0d5b69b..2350638b5 100644
--- a/certbot-dns-luadns/setup.py
+++ b/certbot-dns-luadns/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-luadns',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-luadns = certbot_dns_luadns._internal.dns_luadns:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_luadns',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns_test.py b/certbot-dns-luadns/tests/dns_luadns_test.py
index 934d3e103..934d3e103 100644
--- a/certbot-dns-luadns/certbot_dns_luadns/dns_luadns_test.py
+++ b/certbot-dns-luadns/tests/dns_luadns_test.py
diff --git a/certbot-dns-nsone/MANIFEST.in b/certbot-dns-nsone/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-nsone/MANIFEST.in
+++ b/certbot-dns-nsone/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-nsone/setup.py b/certbot-dns-nsone/setup.py
index f3abb4d9d..09a5b7a93 100644
--- a/certbot-dns-nsone/setup.py
+++ b/certbot-dns-nsone/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-nsone',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-nsone = certbot_dns_nsone._internal.dns_nsone:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_nsone',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone_test.py b/certbot-dns-nsone/tests/dns_nsone_test.py
index dd6168f08..dd6168f08 100644
--- a/certbot-dns-nsone/certbot_dns_nsone/dns_nsone_test.py
+++ b/certbot-dns-nsone/tests/dns_nsone_test.py
diff --git a/certbot-dns-ovh/MANIFEST.in b/certbot-dns-ovh/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-ovh/MANIFEST.in
+++ b/certbot-dns-ovh/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-ovh/setup.py b/certbot-dns-ovh/setup.py
index 308229ade..99b6e365b 100644
--- a/certbot-dns-ovh/setup.py
+++ b/certbot-dns-ovh/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-ovh',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-ovh = certbot_dns_ovh._internal.dns_ovh:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_ovh',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py b/certbot-dns-ovh/tests/dns_ovh_test.py
index a420239ab..a420239ab 100644
--- a/certbot-dns-ovh/certbot_dns_ovh/dns_ovh_test.py
+++ b/certbot-dns-ovh/tests/dns_ovh_test.py
diff --git a/certbot-dns-rfc2136/MANIFEST.in b/certbot-dns-rfc2136/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-rfc2136/MANIFEST.in
+++ b/certbot-dns-rfc2136/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-rfc2136/setup.py b/certbot-dns-rfc2136/setup.py
index 37b0e600c..d767caa1f 100644
--- a/certbot-dns-rfc2136/setup.py
+++ b/certbot-dns-rfc2136/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 
 version = '1.0.0.dev0'
@@ -20,6 +22,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-rfc2136',
     version=version,
@@ -63,5 +79,7 @@ setup(
             'dns-rfc2136 = certbot_dns_rfc2136._internal.dns_rfc2136:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_rfc2136',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py
index c767dba23..c767dba23 100644
--- a/certbot-dns-rfc2136/certbot_dns_rfc2136/dns_rfc2136_test.py
+++ b/certbot-dns-rfc2136/tests/dns_rfc2136_test.py
diff --git a/certbot-dns-route53/MANIFEST.in b/certbot-dns-route53/MANIFEST.in
index ca37a7baf..fc62028b0 100644
--- a/certbot-dns-route53/MANIFEST.in
+++ b/certbot-dns-route53/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-route53/setup.py b/certbot-dns-route53/setup.py
index e766bf684..eb66a2d43 100644
--- a/certbot-dns-route53/setup.py
+++ b/certbot-dns-route53/setup.py
@@ -1,5 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
+from setuptools.command.test import test as TestCommand
+import sys
 
 version = '1.0.0.dev0'
 
@@ -14,6 +16,20 @@ install_requires = [
     'zope.interface',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-route53',
     version=version,
@@ -55,5 +71,7 @@ setup(
             'certbot-route53:auth = certbot_dns_route53.authenticator:Authenticator'
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_route53',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py b/certbot-dns-route53/tests/dns_route53_test.py
index 180ebdf6b..180ebdf6b 100644
--- a/certbot-dns-route53/certbot_dns_route53/dns_route53_test.py
+++ b/certbot-dns-route53/tests/dns_route53_test.py
diff --git a/certbot-dns-sakuracloud/MANIFEST.in b/certbot-dns-sakuracloud/MANIFEST.in
index 18f018c08..5a661cef6 100644
--- a/certbot-dns-sakuracloud/MANIFEST.in
+++ b/certbot-dns-sakuracloud/MANIFEST.in
@@ -1,3 +1,6 @@
 include LICENSE.txt
 include README.rst
 recursive-include docs *
+recursive-include tests *
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/certbot-dns-sakuracloud/setup.py b/certbot-dns-sakuracloud/setup.py
index 779d7a9d8..b9584234b 100644
--- a/certbot-dns-sakuracloud/setup.py
+++ b/certbot-dns-sakuracloud/setup.py
@@ -1,6 +1,7 @@
 from setuptools import setup
 from setuptools import find_packages
-
+from setuptools.command.test import test as TestCommand
+import sys
 
 version = '1.0.0.dev0'
 
@@ -19,6 +20,20 @@ docs_extras = [
     'sphinx_rtd_theme',
 ]
 
+class PyTest(TestCommand):
+    user_options = []
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def run_tests(self):
+        import shlex
+        # import here, cause outside the eggs aren't loaded
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
+
 setup(
     name='certbot-dns-sakuracloud',
     version=version,
@@ -62,5 +77,7 @@ setup(
             'dns-sakuracloud = certbot_dns_sakuracloud._internal.dns_sakuracloud:Authenticator',
         ],
     },
+    tests_require=["pytest"],
     test_suite='certbot_dns_sakuracloud',
+    cmdclass={"test": PyTest},
 )
diff --git a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py
index 16890b5a9..16890b5a9 100644
--- a/certbot-dns-sakuracloud/certbot_dns_sakuracloud/dns_sakuracloud_test.py
+++ b/certbot-dns-sakuracloud/tests/dns_sakuracloud_test.py