diff options
author | Adrien Ferrand <adferrand@users.noreply.github.com> | 2019-03-02 00:18:06 +0300 |
---|---|---|
committer | ohemorange <ebportnoy@gmail.com> | 2019-03-02 00:18:06 +0300 |
commit | 841f8efd0aa7dc2ba249b303e6be664d0a6647e3 (patch) | |
tree | 6d1408e31198fb6d743eb6caa8b3a3ee8775fc98 | |
parent | efc8d49806b14a31d88cfc0f1b6daca1dd373d8d (diff) |
[Unix] Create a framework for certbot integration tests: PART 1 (#6578)
* First part
* Several optimizations about the docker env setup
* Documentation
* Various corrections and documentation. Add acme and certbot explicitly as dependencies of certbot-ci.
* Correct a variable misinterpreted as a pytest hook
* Correct strict parsing option on pebble
* Refactor acme setup to be executed from pytest hooks.
* Pass TRAVIS env variable to trigger specific xdist logic
* Retrigger build.
* Work in progress
* Config operational
* Propagate to xdist
* Corrections on acme and misc
* Correct subnet for pebble
* Remove gobetween, as tls-sni challenges are not tested anymore.
* Improve pebble setup. Reduce LOC.
* Update acme.py
* Optimize acme ca setup, with less temporary assets
* Silent setup
* Clean code
* Remove unused workspace
* Use default network driver
* Remove bridge
* Update package documentation
* Remove rerun capability for integration tests, not needed.
* Add documentation
* Variable for all ports and subnets used by the stack
* Update certbot-ci/certbot_integration_tests/conftest.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/utils/acme.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/utils/misc.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update tox.ini
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/utils/misc.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/utils/acme.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/utils/acme.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Update certbot-ci/certbot_integration_tests/conftest.py
Co-Authored-By: adferrand <adferrand@users.noreply.github.com>
* Rename to acme_server
* Add comment
* Refactor in a unique context fixture
* Remove the need of CERTBOT_ACME_XDIST environment variable
* Remove nonstrict/strict options in pebble
* Clean dependencies
* Clean tox
* Change function name
* Add comment about coveragerc specificities
* Change a comment.
* Update setup.py
* Update conftest.py
* Use the production-ready docker-compose.yml file for Pebble
* New style class
* Tune pebble to have a stable test environment
* Pin a dependency
-rw-r--r-- | certbot-ci/certbot_integration_tests/.coveragerc | 8 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/__init__.py | 1 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/certbot_tests/__init__.py | 0 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/certbot_tests/context.py | 16 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/certbot_tests/test_main.py | 40 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/conftest.py | 92 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/nginx_tests/__init__.py | 0 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/nginx_tests/context.py | 5 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/nginx_tests/test_main.py | 17 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/utils/__init__.py | 0 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/utils/acme_server.py | 194 | ||||
-rw-r--r-- | certbot-ci/certbot_integration_tests/utils/misc.py | 45 | ||||
-rw-r--r-- | certbot-ci/setup.py | 45 | ||||
-rw-r--r-- | tools/dev_constraints.txt | 1 | ||||
-rw-r--r-- | tox.ini | 12 |
15 files changed, 476 insertions, 0 deletions
diff --git a/certbot-ci/certbot_integration_tests/.coveragerc b/certbot-ci/certbot_integration_tests/.coveragerc new file mode 100644 index 000000000..de36d4e02 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/.coveragerc @@ -0,0 +1,8 @@ +[run] +# Avoid false warnings because certbot packages are not installed in the thread that executes +# the coverage: indeed, certbot is launched as a CLI from a subprocess. +disable_warnings = module-not-imported,no-data-collected + +[report] +# Exclude unit tests in coverage during integration tests. +omit = **/*_test.py,**/tests/*,**/certbot_nginx/parser_obj.py diff --git a/certbot-ci/certbot_integration_tests/__init__.py b/certbot-ci/certbot_integration_tests/__init__.py new file mode 100644 index 000000000..434a85a23 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/__init__.py @@ -0,0 +1 @@ +"""Package certbot_integration_test is for tests that require a live acme ca server instance""" diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/__init__.py diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/context.py b/certbot-ci/certbot_integration_tests/certbot_tests/context.py new file mode 100644 index 000000000..9045cd37d --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/context.py @@ -0,0 +1,16 @@ +class IntegrationTestsContext(object): + """General fixture describing a certbot integration tests context""" + def __init__(self, request): + self.request = request + if hasattr(request.config, 'slaveinput'): # Worker node + self.worker_id = request.config.slaveinput['slaveid'] + self.acme_xdist = request.config.slaveinput['acme_xdist'] + else: # Primary node + self.worker_id = 'primary' + self.acme_xdist = request.config.acme_xdist + self.directory_url = self.acme_xdist['directory_url'] + self.tls_alpn_01_port = self.acme_xdist['https_port'][self.worker_id] + self.http_01_port = self.acme_xdist['http_port'][self.worker_id] + + def cleanup(self): + pass diff --git a/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py new file mode 100644 index 000000000..5b0981b36 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/certbot_tests/test_main.py @@ -0,0 +1,40 @@ +import requests +import urllib3 + +import pytest + +from certbot_integration_tests.certbot_tests import context as certbot_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = certbot_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello_1(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass + + +def test_hello_2(context): + assert context.http_01_port + assert context.tls_alpn_01_port + try: + response = requests.get(context.directory_url, verify=False) + response.raise_for_status() + assert response.json() + response.close() + except urllib3.exceptions.InsecureRequestWarning: + pass diff --git a/certbot-ci/certbot_integration_tests/conftest.py b/certbot-ci/certbot_integration_tests/conftest.py new file mode 100644 index 000000000..892c16266 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/conftest.py @@ -0,0 +1,92 @@ +""" +General conftest for pytest execution of all integration tests lying +in the certbot_integration tests package. +As stated by pytest documentation, conftest module is used to set on +for a directory a specific configuration using built-in pytest hooks. + +See https://docs.pytest.org/en/latest/reference.html#hook-reference +""" +import contextlib +import sys +import subprocess + +from certbot_integration_tests.utils import acme_server as acme_lib + + +def pytest_addoption(parser): + """ + Standard pytest hook to add options to the pytest parser. + :param parser: current pytest parser that will be used on the CLI + """ + parser.addoption('--acme-server', default='pebble', + choices=['boulder-v1', 'boulder-v2', 'pebble'], + help='select the ACME server to use (boulder-v1, boulder-v2, ' + 'pebble), defaulting to pebble') + + +def pytest_configure(config): + """ + Standard pytest hook used to add a configuration logic for each node of a pytest run. + :param config: the current pytest configuration + """ + if not hasattr(config, 'slaveinput'): # If true, this is the primary node + with _print_on_err(): + config.acme_xdist = _setup_primary_node(config) + + +def pytest_configure_node(node): + """ + Standard pytest-xdist hook used to configure a worker node. + :param node: current worker node + """ + node.slaveinput['acme_xdist'] = node.config.acme_xdist + + +@contextlib.contextmanager +def _print_on_err(): + """ + During pytest-xdist setup, stdout is used for nodes communication, so print is useless. + However, stderr is still available. This context manager transfers stdout to stderr + for the duration of the context, allowing to display prints to the user. + """ + old_stdout = sys.stdout + sys.stdout = sys.stderr + try: + yield + finally: + sys.stdout = old_stdout + + +def _setup_primary_node(config): + """ + Setup the environment for integration tests. + Will: + - check runtime compatiblity (Docker, docker-compose, Nginx) + - create a temporary workspace and the persistent GIT repositories space + - configure and start paralleled ACME CA servers using Docker + - transfer ACME CA servers configurations to pytest nodes using env variables + :param config: Configuration of the pytest primary node + """ + # Check for runtime compatibility: some tools are required to be available in PATH + try: + subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + try: + subprocess.check_output(['docker-compose', '-v'], stderr=subprocess.STDOUT) + except (subprocess.CalledProcessError, OSError): + raise ValueError('Error: docker-compose is required in PATH to launch the integration tests, ' + 'but is not installed or not available for current user.') + + # Parameter numprocesses is added to option by pytest-xdist + workers = ['primary'] if not config.option.numprocesses\ + else ['gw{0}'.format(i) for i in range(config.option.numprocesses)] + + # By calling setup_acme_server we ensure that all necessary acme server instances will be + # fully started. This runtime is reflected by the acme_xdist returned. + acme_xdist = acme_lib.setup_acme_server(config.option.acme_server, workers) + print('ACME xdist config:\n{0}'.format(acme_xdist)) + + return acme_xdist diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py b/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/__init__.py diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/context.py b/certbot-ci/certbot_integration_tests/nginx_tests/context.py new file mode 100644 index 000000000..6d7d6012b --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/context.py @@ -0,0 +1,5 @@ +from certbot_integration_tests.certbot_tests import context as certbot_context + + +class IntegrationTestsContext(certbot_context.IntegrationTestsContext): + """General fixture describing a certbot-nginx integration tests context""" diff --git a/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py new file mode 100644 index 000000000..472e5e7b7 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/nginx_tests/test_main.py @@ -0,0 +1,17 @@ +import pytest + +from certbot_integration_tests.nginx_tests import context as nginx_context + + +@pytest.fixture() +def context(request): + # Fixture request is a built-in pytest fixture describing current test request. + integration_test_context = nginx_context.IntegrationTestsContext(request) + try: + yield integration_test_context + finally: + integration_test_context.cleanup() + + +def test_hello(context): + print(context.directory_url) diff --git a/certbot-ci/certbot_integration_tests/utils/__init__.py b/certbot-ci/certbot_integration_tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/__init__.py diff --git a/certbot-ci/certbot_integration_tests/utils/acme_server.py b/certbot-ci/certbot_integration_tests/utils/acme_server.py new file mode 100644 index 000000000..33ef05194 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/acme_server.py @@ -0,0 +1,194 @@ +"""Module to setup an ACME CA server environment able to run multiple tests in parallel""" +from __future__ import print_function +import tempfile +import atexit +import os +import subprocess +import shutil +import sys +from os.path import join + +import requests +import json +import yaml + +from certbot_integration_tests.utils import misc + +# These ports are set implicitly in the docker-compose.yml files of Boulder/Pebble. +CHALLTESTSRV_PORT = 8055 +HTTP_01_PORT = 5002 + + +def setup_acme_server(acme_server, nodes): + """ + This method will setup an ACME CA server and an HTTP reverse proxy instance, to allow parallel + execution of integration tests against the unique http-01 port expected by the ACME CA server. + Instances are properly closed and cleaned when the Python process exits using atexit. + Typically all pytest integration tests will be executed in this context. + This method returns an object describing ports and directory url to use for each pytest node + with the relevant pytest xdist node. + :param str acme_server: the type of acme server used (boulder-v1, boulder-v2 or pebble) + :param str[] nodes: list of node names that will be setup by pytest xdist + :return: a dict describing the challenge ports that have been setup for the nodes + :rtype: dict + """ + acme_type = 'pebble' if acme_server == 'pebble' else 'boulder' + acme_xdist = _construct_acme_xdist(acme_server, nodes) + workspace = _construct_workspace(acme_type) + + _prepare_traefik_proxy(workspace, acme_xdist) + _prepare_acme_server(workspace, acme_type, acme_xdist) + + return acme_xdist + + +def _construct_acme_xdist(acme_server, nodes): + """Generate and return the acme_xdist dict""" + acme_xdist = {'acme_server': acme_server, 'challtestsrv_port': CHALLTESTSRV_PORT} + + # Directory and ACME port are set implicitly in the docker-compose.yml files of Boulder/Pebble. + if acme_server == 'pebble': + acme_xdist['directory_url'] = 'https://localhost:14000/dir' + else: # boulder + port = 4001 if acme_server == 'boulder-v2' else 4000 + acme_xdist['directory_url'] = 'http://localhost:{0}/directory'.format(port) + + acme_xdist['http_port'] = {node: port for (node, port) + in zip(nodes, range(5200, 5200 + len(nodes)))} + acme_xdist['https_port'] = {node: port for (node, port) + in zip(nodes, range(5100, 5100 + len(nodes)))} + + return acme_xdist + + +def _construct_workspace(acme_type): + """Create a temporary workspace for integration tests stack""" + workspace = tempfile.mkdtemp() + + def cleanup(): + """Cleanup function to call that will teardown relevant dockers and their configuration.""" + for instance in [acme_type, 'traefik']: + print('=> Tear down the {0} instance...'.format(instance)) + instance_path = join(workspace, instance) + try: + if os.path.isfile(join(instance_path, 'docker-compose.yml')): + _launch_command(['docker-compose', 'down'], cwd=instance_path) + except subprocess.CalledProcessError: + pass + print('=> Finished tear down of {0} instance.'.format(acme_type)) + + shutil.rmtree(workspace) + + # Here with atexit we ensure that clean function is called no matter what. + atexit.register(cleanup) + + return workspace + + +def _prepare_acme_server(workspace, acme_type, acme_xdist): + """Configure and launch the ACME server, Boulder or Pebble""" + print('=> Starting {0} instance deployment...'.format(acme_type)) + instance_path = join(workspace, acme_type) + try: + # Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production. + _launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type), + '--single-branch', '--depth=1', instance_path]) + if acme_type == 'boulder': + # Allow Boulder to ignore usual limit rate policies, useful for tests. + os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'), + join(instance_path, 'test/rate-limit-policies.yml')) + if acme_type == 'pebble': + # Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid + # nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment. + with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler: + config = yaml.load(file_handler.read()) + + config['services']['pebble'].setdefault('environment', [])\ + .extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0']) + with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler: + file_handler.write(yaml.dump(config)) + + # Launch the ACME CA server. + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + # Wait for the ACME CA server to be up. + print('=> Waiting for {0} instance to respond...'.format(acme_type)) + misc.check_until_timeout(acme_xdist['directory_url']) + + # Configure challtestsrv to answer any A record request with ip of the docker host. + acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50' + response = requests.post('http://localhost:{0}/set-default-ipv4' + .format(acme_xdist['challtestsrv_port']), + json={'ip': '{0}.1'.format(acme_subnet)}) + response.raise_for_status() + + print('=> Finished {0} instance deployment.'.format(acme_type)) + except BaseException: + print('Error while setting up {0} instance.'.format(acme_type)) + raise + + +def _prepare_traefik_proxy(workspace, acme_xdist): + """Configure and launch Traefik, the HTTP reverse proxy""" + print('=> Starting traefik instance deployment...') + instance_path = join(workspace, 'traefik') + traefik_subnet = '10.33.33' + traefik_api_port = 8056 + try: + os.mkdir(instance_path) + + with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h: + file_h.write('''\ +version: '3' +services: + traefik: + image: traefik + command: --api --rest + ports: + - {http_01_port}:80 + - {traefik_api_port}:8080 + networks: + traefiknet: + ipv4_address: {traefik_subnet}.2 +networks: + traefiknet: + ipam: + config: + - subnet: {traefik_subnet}.0/24 +'''.format(traefik_subnet=traefik_subnet, + traefik_api_port=traefik_api_port, + http_01_port=HTTP_01_PORT)) + + _launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path) + + misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port)) + config = { + 'backends': { + node: { + 'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}} + } for node, port in acme_xdist['http_port'].items() + }, + 'frontends': { + node: { + 'backend': node, 'passHostHeader': True, + 'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}} + } for node in acme_xdist['http_port'].keys() + } + } + response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port), + data=json.dumps(config)) + response.raise_for_status() + + print('=> Finished traefik instance deployment.') + except BaseException: + print('Error while setting up traefik instance.') + raise + + +def _launch_command(command, cwd=os.getcwd()): + """Launch silently an OS command, output will be displayed in case of failure""" + try: + subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=cwd, universal_newlines=True) + except subprocess.CalledProcessError as e: + sys.stderr.write(e.output) + raise diff --git a/certbot-ci/certbot_integration_tests/utils/misc.py b/certbot-ci/certbot_integration_tests/utils/misc.py new file mode 100644 index 000000000..a3b134788 --- /dev/null +++ b/certbot-ci/certbot_integration_tests/utils/misc.py @@ -0,0 +1,45 @@ +""" +Misc module contains stateless functions that could be used during pytest execution, +or outside during setup/teardown of the integration tests environment. +""" +import os +import time +import contextlib + +import requests + + +def check_until_timeout(url): + """ + Wait and block until given url responds with status 200, or raise an exception + after 150 attempts. + :param str url: the URL to test + :raise ValueError: exception raised after 150 unsuccessful attempts to reach the URL + """ + import urllib3 + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + for _ in range(0, 150): + time.sleep(1) + try: + if requests.get(url, verify=False).status_code == 200: + return + except requests.exceptions.ConnectionError: + pass + + raise ValueError('Error, url did not respond after 150 attempts: {0}'.format(url)) + + +@contextlib.contextmanager +def execute_in_given_cwd(cwd): + """ + Context manager that will execute any command in the given cwd after entering context, + and restore current cwd when context is destroyed. + :param str cwd: the path to use as the temporary current workspace for python execution + """ + current_cwd = os.getcwd() + try: + os.chdir(cwd) + yield + finally: + os.chdir(current_cwd) diff --git a/certbot-ci/setup.py b/certbot-ci/setup.py new file mode 100644 index 000000000..595bba69e --- /dev/null +++ b/certbot-ci/setup.py @@ -0,0 +1,45 @@ +from setuptools import setup +from setuptools import find_packages + + +version = '0.32.0.dev0' + +install_requires = [ + 'pytest', + 'pytest-cov', + 'pytest-xdist', + 'pytest-sugar', + 'coverage', + 'requests', + 'pyyaml', +] + +setup( + name='certbot-ci', + version=version, + description="Certbot continuous integration framework", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, +) diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..f8c3f4461 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -51,6 +51,7 @@ pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.22.5 +pytest-sugar==0.9.2 python-dateutil==2.6.1 python-digitalocean==1.11 PyYAML==3.13 @@ -250,3 +250,15 @@ commands = whitelist_externals = docker-compose passenv = DOCKER_* + +[testenv:integration] +commands = + {[base]pip_install} acme . certbot-nginx certbot-ci + pytest {toxinidir}/certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} \ + --cov=acme --cov=certbot --cov=certbot_nginx --cov-report= \ + --cov-config={toxinidir}/certbot-ci/certbot_integration_tests/.coveragerc \ + -W 'ignore:Unverified HTTPS request' + coverage report --fail-under=65 --show-missing +passenv = + DOCKER_* |