15 files changed, 378 insertions, 186 deletions
diff --git a/tools/_venv_common.py b/tools/_venv_common.py
new file mode 100755
index 000000000..b180518f9
--- /dev/null
+++ b/tools/_venv_common.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import shutil
+import glob
+import time
+import subprocess
+import sys
+
+def subprocess_with_print(command):
+    print(command)
+    return subprocess.call(command, shell=True)
+
+def get_venv_python(venv_path):
+    python_linux = os.path.join(venv_path, 'bin/python')
+    python_windows = os.path.join(venv_path, 'Scripts\\python.exe')
+    if os.path.isfile(python_linux):
+        return python_linux
+    if os.path.isfile(python_windows):
+        return python_windows
+
+    raise ValueError((
+        'Error, could not find python executable in venv path {0}: is it a valid venv ?'
+        .format(venv_path)))
+
+def main(venv_name, venv_args, args):
+    for path in glob.glob('*.egg-info'):
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+    if os.path.isdir(venv_name):
+        os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time())))
+
+    exit_code = 0
+
+    exit_code = subprocess_with_print(' '.join([
+        sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools',
+        venv_name, venv_args])) or exit_code
+
+    python_executable = get_venv_python(venv_name)
+
+    exit_code = subprocess_with_print(' '.join([
+        python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')]))
+    command = [python_executable, os.path.normpath('./tools/pip_install.py')] or exit_code
+    command.extend(args)
+    exit_code = subprocess_with_print(' '.join(command)) or exit_code
+
+    if os.path.isdir(os.path.join(venv_name, 'bin')):
+        # Linux/OSX specific
+        print('-------------------------------------------------------------------')
+        print('Please run the following command to activate developer environment:')
+        print('source {0}/bin/activate'.format(venv_name))
+        print('-------------------------------------------------------------------')
+    elif os.path.isdir(os.path.join(venv_name, 'Scripts')):
+        # Windows specific
+        print('---------------------------------------------------------------------------')
+        print('Please run one of the following commands to activate developer environment:')
+        print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name))
+        print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name))
+        print('---------------------------------------------------------------------------')
+    else:
+        raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name))
+
+    return exit_code
+
+if __name__ == '__main__':
+    sys.exit(main(os.environ.get('VENV_NAME', 'venv'),
+                  os.environ.get('VENV_ARGS', ''), 
+                  sys.argv[1:]))
diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh
deleted file mode 100755
index 0f0ff7e28..000000000
--- a/tools/_venv_common.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/sh -xe
-
-VENV_NAME=${VENV_NAME:-venv}
-
-# .egg-info directories tend to cause bizarre problems (e.g. `pip -e
-# .` might unexpectedly install letshelp-certbot only, in case
-# `python letshelp-certbot/setup.py build` has been called
-# earlier)
-rm -rf *.egg-info
-
-# virtualenv setup is NOT idempotent: shutil.Error:
-# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and
-# `venv/bin/python2` are the same file
-mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
-virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS
-. ./$VENV_NAME/bin/activate
-
-# Use pipstrap to update Python packaging tools to only update to a well tested
-# version and to work around https://github.com/pypa/pip/issues/4817 on older
-# systems.
-python letsencrypt-auto-source/pieces/pipstrap.py
-./tools/pip_install.sh "$@"
-
-set +x
-echo "Please run the following command to activate developer environment:"
-echo "source $VENV_NAME/bin/activate"
diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt
index 00ecee03e..380d49cb3 100644
--- a/tools/dev_constraints.txt
+++ b/tools/dev_constraints.txt
@@ -12,7 +12,7 @@ botocore==1.7.41
 cloudflare==1.5.1
 coverage==4.4.2
 decorator==4.1.2
-dns-lexicon==2.7.3
+dns-lexicon==2.7.14
 dnspython==1.15.0
 docutils==0.14
 execnet==1.5.0
diff --git a/tools/install_and_test.py b/tools/install_and_test.py
new file mode 100755
index 000000000..149ffc776
--- /dev/null
+++ b/tools/install_and_test.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# pip installs the requested packages in editable mode and runs unit tests on
+# them. Each package is installed and tested in the order they are provided
+# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
+# set to 1, packages are installed using pinned versions of all of our
+# dependencies. See pip_install.py for more information on the versions pinned
+# to.
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+import shutil
+import subprocess
+import re
+
+SKIP_PROJECTS_ON_WINDOWS = [
+    'certbot-apache', 'certbot-nginx', 'certbot-postfix', 'letshelp-certbot']
+
+def call_with_print(command, cwd=None):
+    print(command)
+    return subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    if os.environ.get('CERTBOT_NO_PIN') == '1':
+        command = [sys.executable, '-m', 'pip', '-q', '-e']
+    else:
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        command = [sys.executable, os.path.join(script_dir, 'pip_install_editable.py')]
+
+    new_args = []
+    for arg in args:
+        if os.name == 'nt' and arg in SKIP_PROJECTS_ON_WINDOWS:
+            print((
+                'Info: currently {0} is not supported on Windows and will not be tested.'
+                .format(arg)))
+        else:
+            new_args.append(arg)
+
+    exit_code = 0
+
+    for requirement in new_args:
+        current_command = command[:]
+        current_command.append(requirement)
+        exit_code = call_with_print(' '.join(current_command)) or exit_code
+        pkg = re.sub(r'\[\w+\]', '', requirement)
+
+        if pkg == '.':
+            pkg = 'certbot'
+
+        temp_cwd = tempfile.mkdtemp()
+        try:
+            exit_code = call_with_print(' '.join([
+                sys.executable, '-m', 'pytest', '--numprocesses', 'auto',
+                '--quiet', '--pyargs', pkg.replace('-', '_')]), cwd=temp_cwd) or exit_code
+        finally:
+            shutil.rmtree(temp_cwd)
+
+    return exit_code
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/tools/install_and_test.sh b/tools/install_and_test.sh
deleted file mode 100755
index 819f683aa..000000000
--- a/tools/install_and_test.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh -e
-# pip installs the requested packages in editable mode and runs unit tests on
-# them. Each package is installed and tested in the order they are provided
-# before the script moves on to the next package. If CERTBOT_NO_PIN is set not
-# set to 1, packages are installed using pinned versions of all of our
-# dependencies. See pip_install.sh for more information on the versions pinned
-# to.
-
-if [ "$CERTBOT_NO_PIN" = 1 ]; then
-  pip_install="pip install -q -e"
-else
-  pip_install="$(dirname $0)/pip_install_editable.sh"
-fi
-
-temp_cwd=$(mktemp -d)
-trap "rm -rf $temp_cwd" EXIT
-
-set -x
-for requirement in "$@" ; do
-  $pip_install $requirement
-  pkg=$(echo $requirement | cut -f1 -d\[)  # remove any extras such as [dev]
-  pkg=$(echo "$pkg" | tr - _ )  # convert package names to Python import names
-  if [ $pkg = "." ]; then
-    pkg="certbot"
-  fi
-  cd "$temp_cwd"
-  pytest --numprocesses auto --quiet --pyargs $pkg
-  cd -
-done
diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py
index c8fb95351..ad44a55d0 100755
--- a/tools/merge_requirements.py
+++ b/tools/merge_requirements.py
@@ -10,7 +10,6 @@ from __future__ import print_function
 
 import sys
 
-
 def read_file(file_path):
     """Reads in a Python requirements file.
 
@@ -32,17 +31,17 @@ def read_file(file_path):
     return d
 
 
-def print_requirements(requirements):
-    """Prints requirements to stdout.
+def output_requirements(requirements):
+    """Prepare print requirements to stdout.
 
     :param dict requirements: mapping from a project to its pinned version
 
     """
-    print('\n'.join('{0}=={1}'.format(k, v)
-          for k, v in sorted(requirements.items())))
+    return '\n'.join('{0}=={1}'.format(k, v)
+          for k, v in sorted(requirements.items()))
 
 
-def merge_requirements_files(*files):
+def main(*files):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
@@ -54,8 +53,9 @@ def merge_requirements_files(*files):
     d = {}
     for f in files:
         d.update(read_file(f))
-    print_requirements(d)
+    return output_requirements(d)
 
 
 if __name__ == '__main__':
-    merge_requirements_files(*sys.argv[1:])
+    merged_requirements = main(*sys.argv[1:])
+    print(merged_requirements)
diff --git a/tools/pip_install.py b/tools/pip_install.py
new file mode 100755
index 000000000..273ce5ec2
--- /dev/null
+++ b/tools/pip_install.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
+# to 1, a combination of tools/oldest_constraints.txt,
+# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# top level of the package's directory is used, otherwise, a combination of
+# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
+# other file always takes precedence over tools/dev_constraints.txt. If
+# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
+# no other arguments.
+
+from __future__ import print_function, absolute_import
+
+import subprocess
+import os
+import sys
+import re
+import shutil
+import tempfile
+
+import merge_requirements as merge_module
+import readlink
+
+def find_tools_path():
+    return os.path.dirname(readlink.main(__file__))
+
+def certbot_oldest_processing(tools_path, args, test_constraints):
+    if args[0] != '-e' or len(args) != 2:
+        raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
+                         'with a single -e <path> argument.')
+    # remove any extras such as [dev]
+    pkg_dir = re.sub(r'\[\w+\]', '', args[1])
+    requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
+    # packages like acme don't have any local oldest requirements
+    if not os.path.isfile(requirements):
+        requirements = None
+    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
+
+    return requirements
+
+def certbot_normal_processing(tools_path, test_constraints):
+    repo_path = os.path.dirname(tools_path)
+    certbot_requirements = os.path.normpath(os.path.join(
+        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
+    with open(certbot_requirements, 'r') as fd:
+        data = fd.readlines()
+    with open(test_constraints, 'w') as fd:
+        for line in data:
+            search = re.search(r'^(\S*==\S*).*$', line)
+            if search:
+                fd.write('{0}{1}'.format(search.group(1), os.linesep))
+
+def merge_requirements(tools_path, test_constraints, all_constraints):
+    merged_requirements = merge_module.main(
+        os.path.join(tools_path, 'dev_constraints.txt'),
+        test_constraints
+    )
+    with open(all_constraints, 'w') as fd:
+        fd.write(merged_requirements)
+
+def call_with_print(command, cwd=None):
+    print(command)
+    return subprocess.call(command, shell=True, cwd=cwd or os.getcwd())
+
+def main(args):
+    tools_path = find_tools_path()
+    working_dir = tempfile.mkdtemp()
+
+    exit_code = 0
+
+    try:
+        test_constraints = os.path.join(working_dir, 'test_constraints.txt')
+        all_constraints = os.path.join(working_dir, 'all_constraints.txt')
+
+        requirements = None
+        if os.environ.get('CERTBOT_OLDEST') == '1':
+            requirements = certbot_oldest_processing(tools_path, args, test_constraints)
+        else:
+            certbot_normal_processing(tools_path, test_constraints)
+
+        merge_requirements(tools_path, test_constraints, all_constraints)
+        if requirements:
+            exit_code = call_with_print(' '.join([
+                sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints,
+                '--requirement', requirements])) or exit_code
+
+        command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints]
+        command.extend(args)
+        exit_code = call_with_print(' '.join(command)) or exit_code
+    finally:
+        shutil.rmtree(working_dir)
+
+    return exit_code
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/tools/pip_install.sh b/tools/pip_install.sh
deleted file mode 100755
index 78e2afa17..000000000
--- a/tools/pip_install.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/sh -e
-# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
-# to 1, a combination of tools/oldest_constraints.txt,
-# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
-# top level of the package's directory is used, otherwise, a combination of
-# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
-# other file always takes precedence over tools/dev_constraints.txt. If
-# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
-# no other arguments.
-
-# get the root of the Certbot repo
-tools_dir=$(dirname $("$(dirname $0)/readlink.py" $0))
-all_constraints=$(mktemp)
-test_constraints=$(mktemp)
-trap "rm -f $all_constraints $test_constraints" EXIT
-
-if [ "$CERTBOT_OLDEST" = 1 ]; then
-    if [ "$1" != "-e" -o "$#" -ne "2" ]; then
-        echo "When CERTBOT_OLDEST is set, this script must be run with a single -e <path> argument."
-        exit 1
-    fi
-    pkg_dir=$(echo $2 | cut -f1 -d\[)  # remove any extras such as [dev]
-    requirements="$pkg_dir/local-oldest-requirements.txt"
-    # packages like acme don't have any local oldest requirements
-    if [ ! -f "$requirements" ]; then
-        unset requirements
-    fi
-    cp "$tools_dir/oldest_constraints.txt" "$test_constraints"
-else
-    repo_root=$(dirname "$tools_dir")
-    certbot_requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt"
-    sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' "$certbot_requirements" > "$test_constraints"
-fi
-
-"$tools_dir/merge_requirements.py" "$tools_dir/dev_constraints.txt" \
-                                   "$test_constraints" > "$all_constraints"
-
-set -x
-
-# install the requested packages using the pinned requirements as constraints
-if [ -n "$requirements" ]; then
-    pip install -q --constraint "$all_constraints" --requirement "$requirements"
-fi
-pip install -q --constraint "$all_constraints" "$@"
diff --git a/tools/pip_install_editable.py b/tools/pip_install_editable.py
new file mode 100755
index 000000000..35cc2264d
--- /dev/null
+++ b/tools/pip_install_editable.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# pip installs packages in editable mode using certbot-auto's requirements file
+# as constraints
+
+from __future__ import absolute_import
+
+import sys
+
+import pip_install
+
+def main(args):
+    new_args = []
+    for arg in args:
+        new_args.append('-e')
+        new_args.append(arg)
+
+    return pip_install.main(new_args)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
diff --git a/tools/pip_install_editable.sh b/tools/pip_install_editable.sh
deleted file mode 100755
index 6130bf6e7..000000000
--- a/tools/pip_install_editable.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh -e
-# pip installs packages in editable mode using certbot-auto's requirements file
-# as constraints
-
-args=""
-for requirement in "$@" ; do
-  args="$args -e $requirement"
-done
-
-"$(dirname $0)/pip_install.sh" $args
diff --git a/tools/readlink.py b/tools/readlink.py
index 02c74c48d..0199ce184 100755
--- a/tools/readlink.py
+++ b/tools/readlink.py
@@ -7,7 +7,12 @@ platforms.
 
 """
 from __future__ import print_function
+
 import os
 import sys
 
-print(os.path.realpath(sys.argv[1]))
+def main(link):
+    return os.path.realpath(link)
+
+if __name__ == '__main__':
+    print(main(sys.argv[1]))
diff --git a/tools/venv.py b/tools/venv.py
new file mode 100755
index 000000000..2cc43251d
--- /dev/null
+++ b/tools/venv.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+import sys
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        command_python2_st_code = subprocess.call(
+            'command -v python2', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python2_st_code:
+            return '--python python2'
+
+        command_python27_st_code = subprocess.call(
+            'command -v python2.7', shell=True, stdout=fnull, stderr=fnull)
+        if not command_python27_st_code:
+            return '--python python2.7'
+
+    raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    if os.name == 'nt':
+        raise ValueError('Certbot for Windows is not supported on Python 2.x.')
+
+    venv_args = get_venv_args()
+
+    return _venv_common.main('venv', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/tools/venv.sh b/tools/venv.sh
deleted file mode 100755
index 5692f9ebf..000000000
--- a/tools/venv.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh -xe
-# Developer virtualenv setup for Certbot client
-
-if command -v python2; then
-    export VENV_ARGS="--python python2"
-elif command -v python2.7; then
-    export VENV_ARGS="--python python2.7"
-else
-    echo "Couldn't find python2 or python2.7 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test
diff --git a/tools/venv3.py b/tools/venv3.py
new file mode 100755
index 000000000..1bacc9c9a
--- /dev/null
+++ b/tools/venv3.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Developer virtualenv setup for Certbot client
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+import sys
+
+import _venv_common
+
+REQUIREMENTS = [
+    '-e acme[dev]',
+    '-e .[dev,docs]',
+    '-e certbot-apache',
+    '-e certbot-dns-cloudflare',
+    '-e certbot-dns-cloudxns',
+    '-e certbot-dns-digitalocean',
+    '-e certbot-dns-dnsimple',
+    '-e certbot-dns-dnsmadeeasy',
+    '-e certbot-dns-gehirn',
+    '-e certbot-dns-google',
+    '-e certbot-dns-linode',
+    '-e certbot-dns-luadns',
+    '-e certbot-dns-nsone',
+    '-e certbot-dns-ovh',
+    '-e certbot-dns-rfc2136',
+    '-e certbot-dns-route53',
+    '-e certbot-dns-sakuracloud',
+    '-e certbot-nginx',
+    '-e certbot-postfix',
+    '-e letshelp-certbot',
+    '-e certbot-compatibility-test',
+]
+
+def get_venv_args():
+    with open(os.devnull, 'w') as fnull:
+        where_python3_st_code = subprocess.call(
+            'where python3', shell=True, stdout=fnull, stderr=fnull)
+        command_python3_st_code = subprocess.call(
+            'command -v python3', shell=True, stdout=fnull, stderr=fnull)
+
+    if not where_python3_st_code or not command_python3_st_code:
+        return '--python python3'
+
+    raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH')))
+
+def main():
+    venv_args = get_venv_args()
+
+    return _venv_common.main('venv3', venv_args, REQUIREMENTS)
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/tools/venv3.sh b/tools/venv3.sh
deleted file mode 100755
index 07512f370..000000000
--- a/tools/venv3.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/sh -xe
-# Developer Python3 virtualenv setup for Certbot
-
-if command -v python3; then
-    export VENV_NAME="${VENV_NAME:-venv3}"
-    export VENV_ARGS="--python python3"
-else
-    echo "Couldn't find python3 in $PATH"
-    exit 1
-fi
-
-./tools/_venv_common.sh \
-  -e acme[dev] \
-  -e .[dev,docs] \
-  -e certbot-apache \
-  -e certbot-dns-cloudflare \
-  -e certbot-dns-cloudxns \
-  -e certbot-dns-digitalocean \
-  -e certbot-dns-dnsimple \
-  -e certbot-dns-dnsmadeeasy \
-  -e certbot-dns-gehirn \
-  -e certbot-dns-google \
-  -e certbot-dns-linode \
-  -e certbot-dns-luadns \
-  -e certbot-dns-nsone \
-  -e certbot-dns-ovh \
-  -e certbot-dns-rfc2136 \
-  -e certbot-dns-route53 \
-  -e certbot-dns-sakuracloud \
-  -e certbot-nginx \
-  -e certbot-postfix \
-  -e letshelp-certbot \
-  -e certbot-compatibility-test