diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/_venv_common.py | 73 | ||||
-rwxr-xr-x | tools/_venv_common.sh | 26 | ||||
-rw-r--r-- | tools/dev_constraints.txt | 2 | ||||
-rwxr-xr-x | tools/install_and_test.py | 62 | ||||
-rwxr-xr-x | tools/install_and_test.sh | 29 | ||||
-rwxr-xr-x | tools/merge_requirements.py | 16 | ||||
-rwxr-xr-x | tools/pip_install.py | 95 | ||||
-rwxr-xr-x | tools/pip_install.sh | 44 | ||||
-rwxr-xr-x | tools/pip_install_editable.py | 20 | ||||
-rwxr-xr-x | tools/pip_install_editable.sh | 10 | ||||
-rwxr-xr-x | tools/readlink.py | 7 | ||||
-rwxr-xr-x | tools/venv.py | 59 | ||||
-rwxr-xr-x | tools/venv.sh | 34 | ||||
-rwxr-xr-x | tools/venv3.py | 54 | ||||
-rwxr-xr-x | tools/venv3.sh | 33 |
15 files changed, 378 insertions, 186 deletions
diff --git a/tools/_venv_common.py b/tools/_venv_common.py new file mode 100755 index 000000000..b180518f9 --- /dev/null +++ b/tools/_venv_common.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import shutil +import glob +import time +import subprocess +import sys + +def subprocess_with_print(command): + print(command) + return subprocess.call(command, shell=True) + +def get_venv_python(venv_path): + python_linux = os.path.join(venv_path, 'bin/python') + python_windows = os.path.join(venv_path, 'Scripts\\python.exe') + if os.path.isfile(python_linux): + return python_linux + if os.path.isfile(python_windows): + return python_windows + + raise ValueError(( + 'Error, could not find python executable in venv path {0}: is it a valid venv ?' + .format(venv_path))) + +def main(venv_name, venv_args, args): + for path in glob.glob('*.egg-info'): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + + if os.path.isdir(venv_name): + os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) + + exit_code = 0 + + exit_code = subprocess_with_print(' '.join([ + sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', + venv_name, venv_args])) or exit_code + + python_executable = get_venv_python(venv_name) + + exit_code = subprocess_with_print(' '.join([ + python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) + command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code + command.extend(args) + exit_code = subprocess_with_print(' '.join(command)) or exit_code + + if os.path.isdir(os.path.join(venv_name, 'bin')): + # Linux/OSX specific + print('-------------------------------------------------------------------') + print('Please run the following command to activate developer environment:') + print('source {0}/bin/activate'.format(venv_name)) + print('-------------------------------------------------------------------') + elif os.path.isdir(os.path.join(venv_name, 'Scripts')): + # Windows specific + print('---------------------------------------------------------------------------') + print('Please run one of the following commands to activate developer environment:') + print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) + print('---------------------------------------------------------------------------') + else: + raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + + return exit_code + +if __name__ == '__main__': + sys.exit(main(os.environ.get('VENV_NAME', 'venv'), + os.environ.get('VENV_ARGS', ''), + sys.argv[1:])) diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh deleted file mode 100755 index 0f0ff7e28..000000000 --- a/tools/_venv_common.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -xe - -VENV_NAME=${VENV_NAME:-venv} - -# .egg-info directories tend to cause bizarre problems (e.g. `pip -e -# .` might unexpectedly install letshelp-certbot only, in case -# `python letshelp-certbot/setup.py build` has been called -# earlier) -rm -rf *.egg-info - -# virtualenv setup is NOT idempotent: shutil.Error: -# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and -# `venv/bin/python2` are the same file -mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true -virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS -. ./$VENV_NAME/bin/activate - -# Use pipstrap to update Python packaging tools to only update to a well tested -# version and to work around https://github.com/pypa/pip/issues/4817 on older -# systems. -python letsencrypt-auto-source/pieces/pipstrap.py -./tools/pip_install.sh "$@" - -set +x -echo "Please run the following command to activate developer environment:" -echo "source $VENV_NAME/bin/activate" diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 00ecee03e..380d49cb3 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -12,7 +12,7 @@ botocore==1.7.41 cloudflare==1.5.1 coverage==4.4.2 decorator==4.1.2 -dns-lexicon==2.7.3 +dns-lexicon==2.7.14 dnspython==1.15.0 docutils==0.14 execnet==1.5.0 diff --git a/tools/install_and_test.py b/tools/install_and_test.py new file mode 100755 index 000000000..149ffc776 --- /dev/null +++ b/tools/install_and_test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# pip installs the requested packages in editable mode and runs unit tests on +# them. Each package is installed and tested in the order they are provided +# before the script moves on to the next package. If CERTBOT_NO_PIN is set not +# set to 1, packages are installed using pinned versions of all of our +# dependencies. See pip_install.py for more information on the versions pinned +# to. +from __future__ import print_function + +import os +import sys +import tempfile +import shutil +import subprocess +import re + +SKIP_PROJECTS_ON_WINDOWS = [ + 'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot'] + +def call_with_print(command, cwd=None): + print(command) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + if os.environ.get('CERTBOT_NO_PIN') == '1': + command = [sys.executable, '-m', 'pip', '-q', '-e'] + else: + script_dir = os.path.dirname(os.path.abspath(__file__)) + command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')] + + new_args = [] + for arg in args: + if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS: + print(( + 'Info: currently {0} is not supported on Windows and will not be tested.' + .format(arg))) + else: + new_args.append(arg) + + exit_code = 0 + + for requirement in new_args: + current_command = command[:] + current_command.append(requirement) + exit_code = call_with_print(' '.join(current_command)) or exit_code + pkg = re.sub(r'\[\w+\]', '', requirement) + + if pkg == '.': + pkg = 'certbot' + + temp_cwd = tempfile.mkdtemp() + try: + exit_code = call_with_print(' '.join([ + sys.executable, '-m', 'pytest', '--numprocesses', 'auto', + '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code + finally: + shutil.rmtree(temp_cwd) + + return exit_code + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh deleted file mode 100755 index 819f683aa..000000000 --- a/tools/install_and_test.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -e -# pip installs the requested packages in editable mode and runs unit tests on -# them. Each package is installed and tested in the order they are provided -# before the script moves on to the next package. If CERTBOT_NO_PIN is set not -# set to 1, packages are installed using pinned versions of all of our -# dependencies. See pip_install.sh for more information on the versions pinned -# to. - -if [ "$CERTBOT_NO_PIN" = 1 ]; then - pip_install="pip install -q -e" -else - pip_install="$(dirname $0)/pip_install_editable.sh" -fi - -temp_cwd=$(mktemp -d) -trap "rm -rf $temp_cwd" EXIT - -set -x -for requirement in "$@" ; do - $pip_install $requirement - pkg=$(echo $requirement | cut -f1 -d\[) # remove any extras such as [dev] - pkg=$(echo "$pkg" | tr - _ ) # convert package names to Python import names - if [ $pkg = "." ]; then - pkg="certbot" - fi - cd "$temp_cwd" - pytest --numprocesses auto --quiet --pyargs $pkg - cd - -done diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index c8fb95351..ad44a55d0 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys - def read_file(file_path): """Reads in a Python requirements file. @@ -32,17 +31,17 @@ def read_file(file_path): return d -def print_requirements(requirements): - """Prints requirements to stdout. +def output_requirements(requirements): + """Prepare print requirements to stdout. :param dict requirements: mapping from a project to its pinned version """ - print('\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items()))) + return '\n'.join('{0}=={1}'.format(k, v) + for k, v in sorted(requirements.items())) -def merge_requirements_files(*files): +def main(*files): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier @@ -54,8 +53,9 @@ def merge_requirements_files(*files): d = {} for f in files: d.update(read_file(f)) - print_requirements(d) + return output_requirements(d) if __name__ == '__main__': - merge_requirements_files(*sys.argv[1:]) + merged_requirements = main(*sys.argv[1:]) + print(merged_requirements) diff --git a/tools/pip_install.py b/tools/pip_install.py new file mode 100755 index 000000000..273ce5ec2 --- /dev/null +++ b/tools/pip_install.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set +# to 1, a combination of tools/oldest_constraints.txt, +# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# top level of the package's directory is used, otherwise, a combination of +# certbot-auto's requirements file and tools/dev_constraints.txt is used. The +# other file always takes precedence over tools/dev_constraints.txt. If +# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and +# no other arguments. + +from __future__ import print_function, absolute_import + +import subprocess +import os +import sys +import re +import shutil +import tempfile + +import merge_requirements as merge_module +import readlink + +def find_tools_path(): + return os.path.dirname(readlink.main(__file__)) + +def certbot_oldest_processing(tools_path, args, test_constraints): + if args[0] != '-e' or len(args) != 2: + raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' + 'with a single -e <path> argument.') + # remove any extras such as [dev] + pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + # packages like acme don't have any local oldest requirements + if not os.path.isfile(requirements): + requirements = None + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + + return requirements + +def certbot_normal_processing(tools_path, test_constraints): + repo_path = os.path.dirname(tools_path) + certbot_requirements = os.path.normpath(os.path.join( + repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt')) + with open(certbot_requirements, 'r') as fd: + data = fd.readlines() + with open(test_constraints, 'w') as fd: + for line in data: + search = re.search(r'^(\S*==\S*).*$', line) + if search: + fd.write('{0}{1}'.format(search.group(1), os.linesep)) + +def merge_requirements(tools_path, test_constraints, all_constraints): + merged_requirements = merge_module.main( + os.path.join(tools_path, 'dev_constraints.txt'), + test_constraints + ) + with open(all_constraints, 'w') as fd: + fd.write(merged_requirements) + +def call_with_print(command, cwd=None): + print(command) + return subprocess.call(command, shell=True, cwd=cwd or os.getcwd()) + +def main(args): + tools_path = find_tools_path() + working_dir = tempfile.mkdtemp() + + exit_code = 0 + + try: + test_constraints = os.path.join(working_dir, 'test_constraints.txt') + all_constraints = os.path.join(working_dir, 'all_constraints.txt') + + requirements = None + if os.environ.get('CERTBOT_OLDEST') == '1': + requirements = certbot_oldest_processing(tools_path, args, test_constraints) + else: + certbot_normal_processing(tools_path, test_constraints) + + merge_requirements(tools_path, test_constraints, all_constraints) + if requirements: + exit_code = call_with_print(' '.join([ + sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, + '--requirement', requirements])) or exit_code + + command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] + command.extend(args) + exit_code = call_with_print(' '.join(command)) or exit_code + finally: + shutil.rmtree(working_dir) + + return exit_code + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install.sh b/tools/pip_install.sh deleted file mode 100755 index 78e2afa17..000000000 --- a/tools/pip_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -e -# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set -# to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the -# top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If -# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and -# no other arguments. - -# get the root of the Certbot repo -tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0)) -all_constraints=$(mktemp) -test_constraints=$(mktemp) -trap "rm -f $all_constraints $test_constraints" EXIT - -if [ "$CERTBOT_OLDEST" = 1 ]; then - if [ "$1" != "-e" -o "$#" -ne "2" ]; then - echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument." - exit 1 - fi - pkg_dir=$(echo $2 | cut -f1 -d\[) # remove any extras such as [dev] - requirements="$pkg_dir/local-oldest-requirements.txt" - # packages like acme don't have any local oldest requirements - if [ ! -f "$requirements" ]; then - unset requirements - fi - cp "$tools_dir/oldest_constraints.txt" "$test_constraints" -else - repo_root=$(dirname "$tools_dir") - certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" - sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints" -fi - -"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \ - "$test_constraints" > "$all_constraints" - -set -x - -# install the requested packages using the pinned requirements as constraints -if [ -n "$requirements" ]; then - pip install -q --constraint "$all_constraints" --requirement "$requirements" -fi -pip install -q --constraint "$all_constraints" "$@" diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py new file mode 100755 index 000000000..35cc2264d --- /dev/null +++ b/tools/pip_install_editable.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# pip installs packages in editable mode using certbot-auto's requirements file +# as constraints + +from __future__ import absolute_import + +import sys + +import pip_install + +def main(args): + new_args = [] + for arg in args: + new_args.append('-e') + new_args.append(arg) + + return pip_install.main(new_args) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh deleted file mode 100755 index 6130bf6e7..000000000 --- a/tools/pip_install_editable.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -e -# pip installs packages in editable mode using certbot-auto's requirements file -# as constraints - -args="" -for requirement in "$@" ; do - args="$args -e $requirement" -done - -"$(dirname $0)/pip_install.sh" $args diff --git a/tools/readlink.py b/tools/readlink.py index 02c74c48d..0199ce184 100755 --- a/tools/readlink.py +++ b/tools/readlink.py @@ -7,7 +7,12 @@ platforms. """ from __future__ import print_function + import os import sys -print(os.path.realpath(sys.argv[1])) +def main(link): + return os.path.realpath(link) + +if __name__ == '__main__': + print(main(sys.argv[1])) diff --git a/tools/venv.py b/tools/venv.py new file mode 100755 index 000000000..2cc43251d --- /dev/null +++ b/tools/venv.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess +import sys + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + command_python2_st_code = subprocess.call( + 'command -v python2', shell=True, stdout=fnull, stderr=fnull) + if not command_python2_st_code: + return '--python python2' + + command_python27_st_code = subprocess.call( + 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) + if not command_python27_st_code: + return '--python python2.7' + + raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) + +def main(): + if os.name == 'nt': + raise ValueError('Certbot for Windows is not supported on Python 2.x.') + + venv_args = get_venv_args() + + return _venv_common.main('venv', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/venv.sh b/tools/venv.sh deleted file mode 100755 index 5692f9ebf..000000000 --- a/tools/venv.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -xe -# Developer virtualenv setup for Certbot client - -if command -v python2; then - export VENV_ARGS="--python python2" -elif command -v python2.7; then - export VENV_ARGS="--python python2.7" -else - echo "Couldn't find python2 or python2.7 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test diff --git a/tools/venv3.py b/tools/venv3.py new file mode 100755 index 000000000..1bacc9c9a --- /dev/null +++ b/tools/venv3.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# Developer virtualenv setup for Certbot client + +from __future__ import absolute_import + +import os +import subprocess +import sys + +import _venv_common + +REQUIREMENTS = [ + '-e acme[dev]', + '-e .[dev,docs]', + '-e certbot-apache', + '-e certbot-dns-cloudflare', + '-e certbot-dns-cloudxns', + '-e certbot-dns-digitalocean', + '-e certbot-dns-dnsimple', + '-e certbot-dns-dnsmadeeasy', + '-e certbot-dns-gehirn', + '-e certbot-dns-google', + '-e certbot-dns-linode', + '-e certbot-dns-luadns', + '-e certbot-dns-nsone', + '-e certbot-dns-ovh', + '-e certbot-dns-rfc2136', + '-e certbot-dns-route53', + '-e certbot-dns-sakuracloud', + '-e certbot-nginx', + '-e certbot-postfix', + '-e letshelp-certbot', + '-e certbot-compatibility-test', +] + +def get_venv_args(): + with open(os.devnull, 'w') as fnull: + where_python3_st_code = subprocess.call( + 'where python3', shell=True, stdout=fnull, stderr=fnull) + command_python3_st_code = subprocess.call( + 'command -v python3', shell=True, stdout=fnull, stderr=fnull) + + if not where_python3_st_code or not command_python3_st_code: + return '--python python3' + + raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) + +def main(): + venv_args = get_venv_args() + + return _venv_common.main('venv3', venv_args, REQUIREMENTS) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/venv3.sh b/tools/venv3.sh deleted file mode 100755 index 07512f370..000000000 --- a/tools/venv3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -xe -# Developer Python3 virtualenv setup for Certbot - -if command -v python3; then - export VENV_NAME="${VENV_NAME:-venv3}" - export VENV_ARGS="--python python3" -else - echo "Couldn't find python3 in $PATH" - exit 1 -fi - -./tools/_venv_common.sh \ - -e acme[dev] \ - -e .[dev,docs] \ - -e certbot-apache \ - -e certbot-dns-cloudflare \ - -e certbot-dns-cloudxns \ - -e certbot-dns-digitalocean \ - -e certbot-dns-dnsimple \ - -e certbot-dns-dnsmadeeasy \ - -e certbot-dns-gehirn \ - -e certbot-dns-google \ - -e certbot-dns-linode \ - -e certbot-dns-luadns \ - -e certbot-dns-nsone \ - -e certbot-dns-ovh \ - -e certbot-dns-rfc2136 \ - -e certbot-dns-route53 \ - -e certbot-dns-sakuracloud \ - -e certbot-nginx \ - -e certbot-postfix \ - -e letshelp-certbot \ - -e certbot-compatibility-test |