Cleanup test farm tests and add test_sdists (#4089)

* add get_certbot_version.sh

* Use get_certbot_version.sh in build.py

* make test_leauto_upgrades.sh more robust

* auto upgrades break tests

* OCSP experimental is not needed anymore

* Add test_sdists.sh

* Use LE_AUTO_VERSION, not repo version

* install OS deps earlier

* use readlink not realpath

* undo changes to build.py

* Factor out version code from build.py

* Use version.py in test_sdists

* Remove get_certbot_version

* workaround setuptools breakage

4 files changed, 39 insertions, 41 deletions

diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index 08247fe8d..f7a74821b 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -29,7 +29,8 @@ unset PIP_INDEX_URL
 export PIP_EXTRA_INDEX_URL="$SAVE"
 
 git checkout -f "$BRANCH"
-if ! ./letsencrypt-auto -v --debug --version | grep 0.9.0 ; then
+EXPECTED_VERSION=$(grep -m1 LE_AUTO_VERSION letsencrypt-auto | cut -d\" -f2)
+if ! ./letsencrypt-auto -v --debug --version --no-self-upgrade | grep $EXPECTED_VERSION ; then
     echo upgrade appeared to fail
     exit 1
 fi
diff --git a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
index 815522d38..7a86f8d9d 100755
--- a/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
+++ b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
@@ -23,7 +23,7 @@ cd tests/letstest/testdata/
 
 # 2. refer to the config with the same level of relativity that it itself
 # contains :/
-OUT=`letsencrypt-auto certificates --config-dir sample-config -v`
+OUT=`letsencrypt-auto certificates --config-dir sample-config -v --no-self-upgrade`
 TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
 REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
 
diff --git a/tests/letstest/scripts/test_ocsp_experimental.sh b/tests/letstest/scripts/test_ocsp_experimental.sh
deleted file mode 100755
index 91298385c..000000000
--- a/tests/letstest/scripts/test_ocsp_experimental.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash -x
-
-# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
-
-# with curl, instance metadata available from EC2 metadata service:
-#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
-
-cd letsencrypt
-export PATH="$PWD/letsencrypt-auto-source:$PATH"
-letsencrypt-auto-source/letsencrypt-auto --os-packages-only --debug --version
-tools/venv.sh
-sudo venv/bin/certbot certonly --no-self-upgrade -v --standalone --debug \
-                   --text --agree-dev-preview --agree-tos \
-                   --renew-by-default --redirect \
-                   --register-unsafely-without-email \
-                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
-
-# we have to jump through some hoops to cope with relative paths in renewal
-# conf files ...
-# 1. be in the right directory
-cd tests/letstest/testdata/
-
-# 2. refer to the config with the same level of relativity that it itself
-# contains :/
-OUT=`sudo ../../../venv/bin/certbot certificates  -v --config-dir sample-config`
-TEST_CERTS=`echo "$OUT" | grep TEST_CERT | wc -l`
-REVOKED=`echo "$OUT" | grep REVOKED | wc -l`
-
-if [ "$TEST_CERTS" != 2 ] ; then
-    echo "Did not find two test certs as expected ($TEST_CERTS)"
-    exit 1
-fi
-
-if [ "$REVOKED" != 1 ] ; then
-    echo "Did not find one revoked cert as expected ($REVOKED)"
-    exit 1
-fi
diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh
new file mode 100755
index 000000000..e4ebd2e14
--- /dev/null
+++ b/tests/letstest/scripts/test_sdists.sh
@@ -0,0 +1,36 @@
+#!/bin/sh -xe
+
+cd letsencrypt
+./certbot-auto --os-packages-only -n --debug
+
+PLUGINS="certbot-apache certbot-nginx"
+PYTHON=$(command -v python2.7 || command -v python27 || command -v python2 || command -v python)
+TEMP_DIR=$(mktemp -d)
+VERSION=$(letsencrypt-auto-source/version.py)
+
+# setup venv
+virtualenv --no-site-packages -p $PYTHON --setuptools venv
+. ./venv/bin/activate
+pip install -U pip
+pip install -U setuptools
+
+# build sdists
+for pkg_dir in acme . $PLUGINS; do
+    cd $pkg_dir
+    python setup.py clean
+    rm -rf build dist
+    python setup.py sdist
+    mv dist/* $TEMP_DIR
+    cd -
+done
+
+# test sdists
+cd $TEMP_DIR
+for pkg in acme certbot $PLUGINS; do
+    tar -xvf "$pkg-$VERSION.tar.gz"
+    cd "$pkg-$VERSION"
+    python setup.py build
+    python setup.py test
+    python setup.py install
+    cd -
+done