diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/_changelog_top.txt | 6 | ||||
| -rwxr-xr-x | tools/_release.sh | 78 | ||||
| -rw-r--r--[-rwxr-xr-x] | tools/_venv_common.py | 95 | ||||
| -rw-r--r-- | tools/deactivate.py | 2 | ||||
| -rwxr-xr-x | tools/deps.sh | 15 | ||||
| -rw-r--r-- | tools/dev_constraints.txt | 103 | ||||
| -rwxr-xr-x | tools/docker-warning.sh | 4 | ||||
| -rwxr-xr-x | tools/extract_changelog.py | 41 | ||||
| -rwxr-xr-x | tools/install_and_test.py | 25 | ||||
| -rwxr-xr-x | tools/merge_requirements.py | 52 | ||||
| -rw-r--r-- | tools/oldest_constraints.txt | 7 | ||||
| -rwxr-xr-x | tools/pip_install.py | 46 | ||||
| -rwxr-xr-x | tools/pip_install_editable.py | 1 | ||||
| -rwxr-xr-x | tools/readlink.py | 1 | ||||
| -rwxr-xr-x | tools/simple_http_server.py | 10 | ||||
| -rwxr-xr-x | tools/sphinx-quickstart.sh | 2 | ||||
| -rwxr-xr-x | tools/strip_hashes.py | 50 | ||||
| -rwxr-xr-x | tools/venv.py | 54 | ||||
| -rwxr-xr-x | tools/venv3.py | 54 |
19 files changed, 404 insertions, 242 deletions
diff --git a/tools/_changelog_top.txt b/tools/_changelog_top.txt index 6983b3a43..a0f9b6553 100644 --- a/tools/_changelog_top.txt +++ b/tools/_changelog_top.txt @@ -12,10 +12,4 @@ * -Despite us having broken lockstep, we are continuing to release new versions of -all Certbot components during releases for the time being, however, the only -package with changes other than its version number was: - -* - More details about these changes can be found on our GitHub repo. diff --git a/tools/_release.sh b/tools/_release.sh index d75a0f487..1819adad2 100755 --- a/tools/_release.sh +++ b/tools/_release.sh @@ -7,6 +7,24 @@ if [ "$RELEASE_DIR" = "" ]; then exit 1 fi +ExitWarning() { + exit_status="$?" + if [ "$exit_status" != 0 ]; then + # Don't print each command before executing it because it will disrupt + # the desired output. + set +x + echo '******************************' + echo '* *' + echo '* THE RELEASE SCRIPT FAILED! *' + echo '* *' + echo '******************************' + set -x + fi + exit "$exit_status" +} + +trap ExitWarning EXIT + version="$1" echo Releasing production version "$version"... nextversion="$2" @@ -30,7 +48,6 @@ SUBPKGS_NOT_IN_AUTO="certbot-dns-cloudflare certbot-dns-cloudxns certbot-dns-dig SUBPKGS_IN_AUTO="certbot $SUBPKGS_IN_AUTO_NO_CERTBOT" SUBPKGS_NO_CERTBOT="$SUBPKGS_IN_AUTO_NO_CERTBOT $SUBPKGS_NOT_IN_AUTO" SUBPKGS="$SUBPKGS_IN_AUTO $SUBPKGS_NOT_IN_AUTO" -subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs # - it causes problems when running pytest - the latter tries to @@ -66,17 +83,23 @@ fi git checkout "$RELEASE_BRANCH" # Update changelog -sed -i "s/master/$(date +'%Y-%m-%d')/" CHANGELOG.md -git add CHANGELOG.md -git diff --cached +sed -i "s/master/$(date +'%Y-%m-%d')/" certbot/CHANGELOG.md +git add certbot/CHANGELOG.md git commit -m "Update changelog for $version release" -for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test . +for pkg_dir in $SUBPKGS certbot-compatibility-test do sed -i 's/\.dev0//' "$pkg_dir/setup.py" git add "$pkg_dir/setup.py" -done + if [ -f "$pkg_dir/local-oldest-requirements.txt" ]; then + sed -i "s/-e acme\[dev\]/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e acme/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e certbot\[dev\]/certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + sed -i "s/-e certbot/certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" + git add "$pkg_dir/local-oldest-requirements.txt" + fi +done SetVersion() { ver="$1" @@ -90,7 +113,7 @@ SetVersion() { fi sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done - init_file="certbot/__init__.py" + init_file="certbot/certbot/__init__.py" if [ $(grep -c '^__version' "$init_file") != 1 ]; then echo "Unexpected count of __version variables in $init_file" exit 1 @@ -102,8 +125,11 @@ SetVersion() { SetVersion "$version" +# Unset CERTBOT_OLDEST to prevent wheels from being built improperly due to +# conditionals like the one found in certbot-dns-dnsimple's setup.py file. +unset CERTBOT_OLDEST echo "Preparing sdists and wheels" -for pkg_dir in . $SUBPKGS_NO_CERTBOT +for pkg_dir in $SUBPKGS do cd $pkg_dir @@ -123,8 +149,7 @@ done mkdir "dist.$version" -mv dist "dist.$version/certbot" -for pkg_dir in $SUBPKGS_NO_CERTBOT +for pkg_dir in $SUBPKGS do mv $pkg_dir/dist "dist.$version/$pkg_dir/" done @@ -153,7 +178,7 @@ cd ~- # get a snapshot of the CLI help for the docs # We set CERTBOT_DOCS to use dummy values in example user-agent string. -CERTBOT_DOCS=1 certbot --help all > docs/cli-help.txt +CERTBOT_DOCS=1 certbot --help all > certbot/docs/cli-help.txt jws --help > acme/docs/jws-help.txt cd .. @@ -167,12 +192,12 @@ mkdir kgs kgs="kgs/$version" pip freeze | tee $kgs python ../tools/pip_install.py pytest -for module in $subpkgs_modules ; do +cd ~- +for module in $SUBPKGS ; do echo testing $module # use an empty configuration file rather than the one in the repo root - pytest -c <(echo '') --pyargs $module + pytest -c <(echo '') $module done -cd ~- # pin pip hashes of the things we just built for pkg in $SUBPKGS_IN_AUTO ; do @@ -221,8 +246,7 @@ mv letsencrypt-auto-source/letsencrypt-auto.asc letsencrypt-auto-source/certbot- cp -p letsencrypt-auto-source/letsencrypt-auto certbot-auto cp -p letsencrypt-auto-source/letsencrypt-auto letsencrypt-auto -git add certbot-auto letsencrypt-auto letsencrypt-auto-source docs/cli-help.txt -git diff --cached +git add certbot-auto letsencrypt-auto letsencrypt-auto-source certbot/docs/cli-help.txt while ! git commit --gpg-sign="$RELEASE_GPG_KEY" -m "Release $version"; do echo "Unable to sign the release commit using git." echo "You may have to configure git to use gpg2 by running:" @@ -241,17 +265,16 @@ echo gpg2 -U $RELEASE_GPG_KEY --detach-sign --armor $name.$rev.tar.xz cd ~- # Add master section to CHANGELOG.md -header=$(head -n 4 CHANGELOG.md) +header=$(head -n 4 certbot/CHANGELOG.md) body=$(sed s/nextversion/$nextversion/ tools/_changelog_top.txt) -footer=$(tail -n +5 CHANGELOG.md) +footer=$(tail -n +5 certbot/CHANGELOG.md) echo "$header $body -$footer" > CHANGELOG.md -git add CHANGELOG.md -git diff --cached -git commit -m "Add contents to CHANGELOG.md for next version" +$footer" > certbot/CHANGELOG.md +git add certbot/CHANGELOG.md +git commit -m "Add contents to certbot/CHANGELOG.md for next version" echo "New root: $root" echo "Test commands (in the letstest repo):" @@ -265,16 +288,5 @@ if [ "$RELEASE_BRANCH" = candidate-"$version" ] ; then SetVersion "$nextversion".dev0 letsencrypt-auto-source/build.py git add letsencrypt-auto-source/letsencrypt-auto - for pkg_dir in $SUBPKGS_NO_CERTBOT . - do - if [ -f "$pkg_dir/local-oldest-requirements.txt" ]; then - sed -i "s/-e acme\[dev\]/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" - sed -i "s/-e acme/acme[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" - sed -i "s/-e \.\[dev\]/certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" - sed -i "s/-e \./certbot[dev]==$version/" "$pkg_dir/local-oldest-requirements.txt" - git add "$pkg_dir/local-oldest-requirements.txt" - fi - done - git diff git commit -m "Bump version to $nextversion" fi diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 540842773..c61385054 100755..100644 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -12,14 +12,37 @@ VENV_NAME. from __future__ import print_function +import glob import os +import re import shutil -import glob -import time import subprocess import sys -import re -import shlex +import time + +REQUIREMENTS = [ + '-e acme[dev]', + '-e certbot[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e letshelp-certbot', + '-e certbot-compatibility-test', + '-e certbot-ci', +] VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') @@ -107,29 +130,35 @@ def subprocess_with_print(cmd, env=os.environ, shell=False): subprocess.check_call(cmd, env=env, shell=shell) -def get_venv_bin_path(venv_path): +def get_venv_python_path(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): - return os.path.abspath(os.path.dirname(python_linux)) + return os.path.abspath(python_linux) python_windows = os.path.join(venv_path, 'Scripts\\python.exe') if os.path.isfile(python_windows): - return os.path.abspath(os.path.dirname(python_windows)) + return os.path.abspath(python_windows) raise ValueError(( 'Error, could not find python executable in venv path {0}: is it a valid venv ?' .format(venv_path))) -def main(venv_name, venv_args, args): - """Creates a virtual environment and installs packages. +def prepare_venv_path(venv_name): + """Determines the venv path and prepares it for use. + + This function cleans up any Python eggs in the current working directory + and ensures the venv path is available for use. The path used is the + VENV_NAME environment variable if it is set and venv_name otherwise. If + there is already a directory at the desired path, the existing directory is + renamed by appending a timestamp to the directory name. :param str venv_name: The name or path at where the virtual - environment should be created. - :param str venv_args: Command line arguments for virtualenv - :param str args: Command line arguments that should be given to pip - to install packages - """ + environment should be created if VENV_NAME isn't set. + + :returns: path where the virtual environment should be created + :rtype: str + """ for path in glob.glob('*.egg-info'): if os.path.isdir(path): shutil.rmtree(path) @@ -145,21 +174,25 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - command = [sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', venv_name] - command.extend(shlex.split(venv_args)) - subprocess_with_print(command) + return venv_name + - # We execute the following commands in the context of the virtual environment, to install - # the packages in it. To do so, we append the venv binary to the PATH that will be used for - # these commands. With this trick, correct python executable will be selected. - new_environ = os.environ.copy() - new_environ['PATH'] = os.pathsep.join([get_venv_bin_path(venv_name), new_environ['PATH']]) - subprocess_with_print('python {0}'.format('./letsencrypt-auto-source/pieces/pipstrap.py'), - env=new_environ, shell=True) - subprocess_with_print('python -m pip install --upgrade "setuptools>=30.3"', - env=new_environ, shell=True) - subprocess_with_print('python {0} {1}'.format('./tools/pip_install.py', ' '.join(args)), - env=new_environ, shell=True) +def install_packages(venv_name, pip_args): + """Installs packages in the given venv. + + :param str venv_name: The name or path at where the virtual + environment should be created. + :param pip_args: Command line arguments that should be given to + pip to install packages + :type pip_args: `list` of `str` + + """ + # Using the python executable from venv, we ensure to execute following commands in this venv. + py_venv = get_venv_python_path(venv_name) + subprocess_with_print([py_venv, os.path.abspath('letsencrypt-auto-source/pieces/pipstrap.py')]) + command = [py_venv, os.path.abspath('tools/pip_install.py')] + command.extend(pip_args) + subprocess_with_print(command) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -176,9 +209,3 @@ def main(venv_name, venv_args, args): print('---------------------------------------------------------------------------') else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) - - -if __name__ == '__main__': - main('venv', - '', - sys.argv[1:]) diff --git a/tools/deactivate.py b/tools/deactivate.py index d43b84552..10c9ecd35 100644 --- a/tools/deactivate.py +++ b/tools/deactivate.py @@ -16,8 +16,8 @@ import os import sys from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa import josepy as jose from acme import client as acme_client diff --git a/tools/deps.sh b/tools/deps.sh deleted file mode 100755 index e12f201a5..000000000 --- a/tools/deps.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# -# Find all Python imports. -# -# ./tools/deps.sh certbot -# ./tools/deps.sh acme -# ./tools/deps.sh certbot-apache -# ... -# -# Manually compare the output with deps in setup.py. - -git grep -h -E '^(import|from.*import)' $1/ | \ - awk '{print $2}' | \ - grep -vE "^$1" | \ - sort -u diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 88340cb00..265d967d8 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,77 +1,114 @@ -# Specifies Python package versions for development. +# Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. # Some dev package versions specified here may be overridden by higher level constraints # files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 +appnope==0.1.0 asn1crypto==0.22.0 -astroid==1.3.5 +astroid==2.3.3 attrs==17.3.0 Babel==2.5.1 +backports.functools-lru-cache==1.5 backports.shutil-get-terminal-size==1.0.0 -boto3==1.9.36 -botocore==1.12.36 -cloudflare==1.5.1 -coverage==4.4.2 -decorator==4.1.2 -dns-lexicon==3.0.8 +backports.ssl-match-hostname==3.7.0.1 +bcrypt==3.1.6 +boto3==1.11.7 +botocore==1.14.7 +cached-property==1.5.1 +cloudflare==2.3.1 +codecov==2.0.15 +configparser==3.7.4 +contextlib2==0.6.0.post1 +coverage==4.5.4 +decorator==4.4.1 +dns-lexicon==3.3.17 dnspython==1.15.0 -docutils==0.12 +docker==3.7.2 +docker-compose==1.25.0 +docker-pycreds==0.4.0 +dockerpty==0.4.1 +docopt==0.6.2 +docutils==0.15.2 execnet==1.5.0 +functools32==3.2.3.post2 future==0.16.0 -futures==3.1.1 -google-api-python-client==1.5 +futures==3.3.0 +filelock==3.0.12 +google-api-python-client==1.5.5 httplib2==0.10.3 imagesize==0.7.1 -ipdb==0.10.2 -ipython==5.5.0 +importlib-metadata==0.23 +ipdb==0.12.3 +ipython==5.8.0 ipython-genutils==0.2.0 +isort==4.3.21 Jinja2==2.9.6 -jmespath==0.9.3 +jmespath==0.9.4 josepy==1.1.0 +jsonschema==2.6.0 +lazy-object-proxy==1.4.3 logger==1.4 logilab-common==1.4.1 MarkupSafe==1.0 -mypy==0.600 +mccabe==0.6.1 +more-itertools==5.0.0 +mypy==0.710 +mypy-extensions==0.4.3 ndg-httpsclient==0.3.2 -oauth2client==2.0.0 -pathlib2==2.3.0 -pexpect==4.2.1 -pickleshare==0.7.4 +oauth2client==4.0.0 +packaging==19.2 +paramiko==2.4.2 +pathlib2==2.3.5 +pexpect==4.7.0 +pickleshare==0.7.5 pkginfo==1.4.2 -pluggy==0.5.2 -prompt-toolkit==1.0.15 -ptyprocess==0.5.2 -py==1.4.34 +pluggy==0.13.0 +prompt-toolkit==1.0.18 +ptyprocess==0.6.0 +py==1.8.0 pyasn1==0.1.9 pyasn1-modules==0.0.10 Pygments==2.2.0 -pylint==1.4.2 +pylint==2.4.3 +# If pynsist version is upgraded, our NSIS template windows-installer/template.nsi +# must be upgraded if necessary using the new built-in one from pynsist. +pynacl==1.3.0 +pynsist==2.4 pytest==3.2.5 pytest-cov==2.5.1 pytest-forked==0.2 pytest-xdist==1.22.5 -python-dateutil==2.6.1 +pytest-sugar==0.9.2 +pytest-rerunfailures==4.2 +python-dateutil==2.8.1 python-digitalocean==1.11 +pywin32==227 PyYAML==3.13 repoze.sphinx.autointerface==0.8 requests-file==1.4.2 requests-toolbelt==0.8.0 rsa==3.4.2 -s3transfer==0.1.11 -scandir==1.6 +s3transfer==0.3.1 +scandir==1.10.0 simplegeneric==0.8.1 +singledispatch==3.4.0.3 snowballstemmer==1.2.1 Sphinx==1.7.5 sphinx-rtd-theme==0.2.4 sphinxcontrib-websupport==1.0.1 +texttable==0.9.1 tldextract==2.2.0 -tox==2.9.1 +toml==0.10.0 +tox==3.14.0 tqdm==4.19.4 -traitlets==4.3.2 +traitlets==4.3.3 twine==1.11.0 -typed-ast==1.1.0 +typed-ast==1.4.0 typing==3.6.4 -uritemplate==0.6 -virtualenv==15.1.0 -wcwidth==0.1.7 +uritemplate==3.0.0 +virtualenv==16.6.2 +wcwidth==0.1.8 +websocket-client==0.56.0 +wrapt==1.11.2 +zipp==0.6.0 diff --git a/tools/docker-warning.sh b/tools/docker-warning.sh deleted file mode 100755 index e4f5f40ee..000000000 --- a/tools/docker-warning.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -e -echo "Warning: This Docker image will soon be switching to Alpine Linux." >&2 -echo "You can switch now using the certbot/certbot repo on Docker Hub." >&2 -exec /opt/certbot/venv/bin/certbot $@ diff --git a/tools/extract_changelog.py b/tools/extract_changelog.py new file mode 100755 index 000000000..fb0b849aa --- /dev/null +++ b/tools/extract_changelog.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +from __future__ import print_function + +import os +import re +import sys + +CERTBOT_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +NEW_SECTION_PATTERN = re.compile(r'^##\s*[\d.]+\s*-\s*[\d-]+$') + + +def main(): + version = sys.argv[1] + + section_pattern = re.compile(r'^##\s*{0}\s*-\s*[\d-]+$' + .format(version.replace('.', '\\.'))) + + with open(os.path.join(CERTBOT_ROOT, 'certbot', 'CHANGELOG.md')) as file_h: + lines = file_h.read().splitlines() + + changelog = [] + + i = 0 + while i < len(lines): + if section_pattern.match(lines[i]): + i = i + 1 + while i < len(lines): + if NEW_SECTION_PATTERN.match(lines[i]): + break + changelog.append(lines[i]) + i = i + 1 + i = i + 1 + + changelog = [entry for entry in changelog if entry] + + print('\n'.join(changelog)) + + +if __name__ == '__main__': + main() diff --git a/tools/install_and_test.py b/tools/install_and_test.py index b15c8eca5..192708957 100755 --- a/tools/install_and_test.py +++ b/tools/install_and_test.py @@ -8,19 +8,16 @@ from __future__ import print_function import os -import sys -import tempfile -import shutil -import subprocess import re +import subprocess +import sys -SKIP_PROJECTS_ON_WINDOWS = [ - 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] +SKIP_PROJECTS_ON_WINDOWS = ['certbot-apache', 'letshelp-certbot'] -def call_with_print(command, cwd=None): +def call_with_print(command): print(command) - subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True) def main(args): @@ -42,16 +39,8 @@ def main(args): call_with_print(' '.join(current_command)) pkg = re.sub(r'\[\w+\]', '', requirement) - if pkg == '.': - pkg = 'certbot' - - temp_cwd = tempfile.mkdtemp() - shutil.copy2("pytest.ini", temp_cwd) - try: - call_with_print(' '.join([ - sys.executable, '-m', 'pytest', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) - finally: - shutil.rmtree(temp_cwd) + call_with_print(' '.join([ + sys.executable, '-m', 'pytest', pkg])) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index 4205e6bcf..0d41d12c4 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,27 +10,36 @@ from __future__ import print_function import sys -def read_file(file_path): - """Reads in a Python requirements file. +def process_entries(entries): + """ Ignore empty lines, comments and editable requirements - :param str file_path: path to requirements file + :param list entries: List of entries :returns: mapping from a project to its pinned version :rtype: dict - """ data = {} - with open(file_path) as file_h: - for line in file_h: - line = line.strip() - if line and not line.startswith('#') and not line.startswith('-e'): - project, version = line.split('==') - if not version: - raise ValueError("Unexpected syntax '{0}'".format(line)) - data[project] = version + for e in entries: + e = e.strip() + if e and not e.startswith('#') and not e.startswith('-e'): + project, version = e.split('==') + if not version: + raise ValueError("Unexpected syntax '{0}'".format(e)) + data[project] = version return data +def read_file(file_path): + """Reads in a Python requirements file. + + :param str file_path: path to requirements file + + :returns: list of entries in the file + :rtype: list + + """ + with open(file_path) as file_h: + return file_h.readlines() def output_requirements(requirements): """Prepare print requirements to stdout. @@ -46,14 +55,25 @@ def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier - files. + files. Files are read from file paths passed from the command line arguments. - :param tuple paths: paths to requirements files + If no command line arguments are defined, data is read from stdin instead. + + :param tuple paths: paths to requirements files provided on command line """ data = {} - for path in paths: - data.update(read_file(path)) + if paths: + for path in paths: + data.update(process_entries(read_file(path))) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + data.update(process_entries(stdin_data)) + return output_requirements(data) diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index e48d6b13c..c5a5c5aa0 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -16,6 +16,7 @@ pyOpenSSL==0.13.1 pyparsing==1.5.6 pyRFC3339==1.0 python-augeas==0.5.0 +oauth2client==4.0.0 six==1.9.0 # setuptools 0.9.8 is the actual version packaged, but some other dependencies # in this file require setuptools>=1.0 and there are no relevant changes for us @@ -35,11 +36,12 @@ idna==2.0 pbr==1.8.0 pytz==2012rc0 +# Debian Buster constraints +google-api-python-client==1.5.5 + # Our setup.py constraints cloudflare==1.5.1 cryptography==1.2.3 -google-api-python-client==1.5 -oauth2client==2.0 parsedatetime==1.3 pyparsing==1.5.5 python-digitalocean==1.11 @@ -51,6 +53,7 @@ funcsigs==0.4 zope.hookable==4.0.4 # Ubuntu Bionic constraints. +distro==1.0.1 # Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins # using their local-oldest-requirements.txt dns-lexicon==2.2.1 diff --git a/tools/pip_install.py b/tools/pip_install.py index dd6302b48..0a3961384 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -8,17 +8,19 @@ # CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and # no other arguments. -from __future__ import print_function, absolute_import +from __future__ import absolute_import +from __future__ import print_function -import subprocess import os -import sys import re import shutil +import subprocess +import sys import tempfile import merge_requirements as merge_module import readlink +import strip_hashes def find_tools_path(): @@ -47,10 +49,8 @@ def certbot_normal_processing(tools_path, test_constraints): with open(certbot_requirements, 'r') as fd: data = fd.readlines() with open(test_constraints, 'w') as fd: - for line in data: - search = re.search(r'^(\S*==\S*).*$', line) - if search: - fd.write('{0}{1}'.format(search.group(1), os.linesep)) + data = "\n".join(strip_hashes.process_entries(data)) + fd.write(data) def merge_requirements(tools_path, requirements, test_constraints, all_constraints): @@ -70,9 +70,15 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain fd.write(merged_requirements) -def call_with_print(command, cwd=None): +def call_with_print(command): print(command) - subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + subprocess.check_call(command, shell=True) + + +def pip_install_with_print(args_str): + command = '"{0}" -m pip install --disable-pip-version-check {1}'.format(sys.executable, + args_str) + call_with_print(command) def main(args): @@ -90,8 +96,7 @@ def main(args): if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint - call_with_print('"{0}" -m pip install {1}' - .format(sys.executable, ' '.join(args))) + pip_install_with_print(' '.join(args)) else: # Otherwise, we merge requirements to build the constraints and pin dependencies requirements = None @@ -101,12 +106,19 @@ def main(args): certbot_normal_processing(tools_path, test_constraints) merge_requirements(tools_path, requirements, test_constraints, all_constraints) - if requirements: - call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' - .format(sys.executable, all_constraints, requirements)) - - call_with_print('"{0}" -m pip install --constraint "{1}" {2}' - .format(sys.executable, all_constraints, ' '.join(args))) + if requirements: # This branch is executed during the oldest tests + # First step, install the transitive dependencies of oldest requirements + # in respect with oldest constraints. + pip_install_with_print('--constraint "{0}" --requirement "{1}"' + .format(all_constraints, requirements)) + # Second step, ensure that oldest requirements themselves are effectively + # installed using --force-reinstall, and avoid corner cases like the one described + # in https://github.com/certbot/certbot/issues/7014. + pip_install_with_print('--force-reinstall --no-deps --requirement "{0}"' + .format(requirements)) + + pip_install_with_print('--constraint "{0}" {1}'.format( + all_constraints, ' '.join(args))) finally: if os.environ.get('TRAVIS'): print('travis_fold:end:install_certbot_deps') diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py index 8eaf3a9fa..3f7c02ba9 100755 --- a/tools/pip_install_editable.py +++ b/tools/pip_install_editable.py @@ -8,6 +8,7 @@ import sys import pip_install + def main(args): new_args = [] for arg in args: diff --git a/tools/readlink.py b/tools/readlink.py index 0199ce184..446c8ebdc 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -11,6 +11,7 @@ from __future__ import print_function import os import sys + def main(link): return os.path.realpath(link) diff --git a/tools/simple_http_server.py b/tools/simple_http_server.py index 14ac9a3d3..24c55962d 100755 --- a/tools/simple_http_server.py +++ b/tools/simple_http_server.py @@ -1,9 +1,13 @@ #!/usr/bin/env python -"""A version of Python 2.x's SimpleHTTPServer that flushes its output.""" -from BaseHTTPServer import HTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler +"""A version of Python's SimpleHTTPServer that flushes its output.""" import sys +try: + from http.server import HTTPServer, SimpleHTTPRequestHandler +except ImportError: + from BaseHTTPServer import HTTPServer + from SimpleHTTPServer import SimpleHTTPRequestHandler + def serve_forever(port=0): """Spins up an HTTP server on all interfaces and the given port. diff --git a/tools/sphinx-quickstart.sh b/tools/sphinx-quickstart.sh index 72dc9e200..35a7f7fad 100755 --- a/tools/sphinx-quickstart.sh +++ b/tools/sphinx-quickstart.sh @@ -14,7 +14,7 @@ sed -i -e "s|\# import os|import os|" conf.py sed -i -e "s|\# needs_sphinx = '1.0'|needs_sphinx = '1.0'|" conf.py sed -i -e "s|intersphinx_mapping = {'https://docs.python.org/': None}|intersphinx_mapping = {\n 'python': ('https://docs.python.org/', None),\n 'acme': ('https://acme-python.readthedocs.org/en/latest/', None),\n 'certbot': ('https://certbot.eff.org/docs/', None),\n}|" conf.py sed -i -e "s|html_theme = 'alabaster'|\n# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs\n# on_rtd is whether we are on readthedocs.org\non_rtd = os.environ.get('READTHEDOCS', None) == 'True'\nif not on_rtd: # only import and set the theme if we're building docs locally\n import sphinx_rtd_theme\n html_theme = 'sphinx_rtd_theme'\n html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n# otherwise, readthedocs.org uses their theme by default, so no need to specify it|" conf.py -sed -i -e "s|# Add any paths that contain templates here, relative to this directory.|autodoc_member_order = 'bysource'\nautodoc_default_flags = ['show-inheritance', 'private-members']\n\n# Add any paths that contain templates here, relative to this directory.|" conf.py +sed -i -e "s|# Add any paths that contain templates here, relative to this directory.|autodoc_member_order = 'bysource'\nautodoc_default_flags = ['show-inheritance']\n\n# Add any paths that contain templates here, relative to this directory.|" conf.py sed -i -e "s|# The name of the Pygments (syntax highlighting) style to use.|default_role = 'py:obj'\n\n# The name of the Pygments (syntax highlighting) style to use.|" conf.py echo "/_build/" >> .gitignore echo "================= diff --git a/tools/strip_hashes.py b/tools/strip_hashes.py new file mode 100755 index 000000000..988e72eb8 --- /dev/null +++ b/tools/strip_hashes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +"""Removes hash information from requirement files passed to it as file path +arguments or simply piped to stdin.""" + +import re +import sys + + +def process_entries(entries): + """Strips off hash strings from dependencies. + + :param list entries: List of entries + + :returns: list of dependencies without hashes + :rtype: list + """ + out_lines = [] + for e in entries: + e = e.strip() + search = re.search(r'^(\S*==\S*).*$', e) + if search: + out_lines.append(search.group(1)) + return out_lines + +def main(*paths): + """ + Reads dependency definitions from a (list of) file(s) provided on the + command line. If no command line arguments are present, data is read from + stdin instead. + + Hashes are removed from returned entries. + """ + + deps = [] + if paths: + for path in paths: + with open(path) as file_h: + deps += process_entries(file_h.readlines()) + else: + # Need to check if interactive to avoid blocking if nothing is piped + if not sys.stdin.isatty(): + stdin_data = [] + for line in sys.stdin: + stdin_data.append(line) + deps += process_entries(stdin_data) + + return "\n".join(deps) + +if __name__ == '__main__': + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/venv.py b/tools/venv.py index 93b012e76..f99386eff 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -1,41 +1,37 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client import os +import sys import _venv_common -REQUIREMENTS = [ - '-e acme[dev]', - '-e .[dev,docs]', - '-e certbot-apache', - '-e certbot-dns-cloudflare', - '-e certbot-dns-cloudxns', - '-e certbot-dns-digitalocean', - '-e certbot-dns-dnsimple', - '-e certbot-dns-dnsmadeeasy', - '-e certbot-dns-gehirn', - '-e certbot-dns-google', - '-e certbot-dns-linode', - '-e certbot-dns-luadns', - '-e certbot-dns-nsone', - '-e certbot-dns-ovh', - '-e certbot-dns-rfc2136', - '-e certbot-dns-route53', - '-e certbot-dns-sakuracloud', - '-e certbot-nginx', - '-e certbot-postfix', - '-e letshelp-certbot', - '-e certbot-compatibility-test', -] - - -def main(): + +def create_venv(venv_path): + """Create a Python 2 virtual environment at venv_path. + + :param str venv_path: path where the venv should be created + + """ + python2 = _venv_common.find_python_executable(2) + command = [sys.executable, '-m', 'virtualenv', '--python', python2, venv_path] + + environ = os.environ.copy() + environ['VIRTUALENV_NO_DOWNLOAD'] = '1' + _venv_common.subprocess_with_print(command, environ) + + +def main(pip_args=None): if os.name == 'nt': raise ValueError('Certbot for Windows is not supported on Python 2.x.') - venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(2)) - _venv_common.main('venv', venv_args, REQUIREMENTS) + venv_path = _venv_common.prepare_venv_path('venv') + create_venv(venv_path) + + if not pip_args: + pip_args = _venv_common.REQUIREMENTS + + _venv_common.install_packages(venv_path, pip_args) if __name__ == '__main__': - main() + main(sys.argv[1:]) diff --git a/tools/venv3.py b/tools/venv3.py index c2374ba5a..7ead82bd5 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -1,36 +1,30 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Developer virtualenv setup for Certbot client +import sys + import _venv_common -REQUIREMENTS = [ - '-e acme[dev]', - '-e .[dev,docs]', - '-e certbot-apache', - '-e certbot-dns-cloudflare', - '-e certbot-dns-cloudxns', - '-e certbot-dns-digitalocean', - '-e certbot-dns-dnsimple', - '-e certbot-dns-dnsmadeeasy', - '-e certbot-dns-gehirn', - '-e certbot-dns-google', - '-e certbot-dns-linode', - '-e certbot-dns-luadns', - '-e certbot-dns-nsone', - '-e certbot-dns-ovh', - '-e certbot-dns-rfc2136', - '-e certbot-dns-route53', - '-e certbot-dns-sakuracloud', - '-e certbot-nginx', - '-e certbot-postfix', - '-e letshelp-certbot', - '-e certbot-compatibility-test', -] - - -def main(): - venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(3)) - _venv_common.main('venv3', venv_args, REQUIREMENTS) + +def create_venv(venv_path): + """Create a Python 3 virtual environment at venv_path. + + :param str venv_path: path where the venv should be created + + """ + python3 = _venv_common.find_python_executable(3) + command = [python3, '-m', 'venv', venv_path] + _venv_common.subprocess_with_print(command) + + +def main(pip_args=None): + venv_path = _venv_common.prepare_venv_path('venv3') + create_venv(venv_path) + + if not pip_args: + pip_args = _venv_common.REQUIREMENTS + ['-e certbot[dev3]'] + + _venv_common.install_packages(venv_path, pip_args) if __name__ == '__main__': - main() + main(sys.argv[1:]) |