Cleanup scripts and switch to pyenv (#9214)

I think test_apache2.sh still has value as it allows us to test our Apache plugin with the Apache layouts found on different OSes. Unfortunately, many of the OSes we're currently testing against don't have Python 3.7+ packaged yet we still support these OSes through things like snap where we bundle our own version of Python.

To allow us to continue testing on these OSes, I switched to installing Python through pyenv. I also took the opportunity to clean up the scripts, removing a lot of code, failing more quickly, and simplifying failure logic in test_apache2.sh.

2 files changed, 32 insertions, 148 deletions

diff --git a/letstest/scripts/bootstrap_os_packages.sh b/letstest/scripts/bootstrap_os_packages.sh
index 26d6f6ddb..fbdca71e5 100755
--- a/letstest/scripts/bootstrap_os_packages.sh
+++ b/letstest/scripts/bootstrap_os_packages.sh
@@ -8,134 +8,22 @@ error() {
     echo "$@"
 }
 
-if command -v command > /dev/null 2>&1 ; then
-  export EXISTS="command -v"
-elif which which > /dev/null 2>&1 ; then
-  export EXISTS="which"
-else
-  error "Cannot find command nor which... please install one!"
-  exit 1
-fi
-
-# Sets LE_PYTHON to Python version string and PYVER to the first two
-# digits of the python version.
-DeterminePythonVersion() {
-  # If no Python is found, PYVER is set to 0.
-  for LE_PYTHON in python3 python2.7 python27 python2 python; do
-    # Break (while keeping the LE_PYTHON value) if found.
-    $EXISTS "$LE_PYTHON" > /dev/null && break
-  done
-  if [ "$?" != "0" ]; then
-    PYVER=0
-    return 0
-  fi
-
-  PYVER=$("$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//')
-}
-
-BootstrapDebCommon() {
+if [ -f /etc/debian_version ]; then
   sudo apt-get update || error apt-get update hit problems but continuing anyway...
 
-  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    python3 \
-    python3-dev \
-    python3-venv \
-    gcc \
-    libaugeas0 \
-    libssl-dev \
-    openssl \
-    libffi-dev \
-    ca-certificates \
-    build-essential \
-    curl \
-    make # needed on debian 9 arm64 which doesn't have a python3 pynacl wheel
-
-  # make sure rust isn't installed by the package manager
-  if ! sudo apt-get remove -y rustc; then
-    error "Could not remove existing rust. Aborting bootstrap!"
-    exit 1
-  fi
-
-  # Install rust for cryptography (needed on Debian)
-  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-  . $HOME/.cargo/env
-}
-
-# Sets TOOL to the name of the package manager
-InitializeRPMCommonBase() {
-  if type dnf 2>/dev/null
-  then
-    TOOL=dnf
-  elif type yum 2>/dev/null
-  then
-    TOOL=yum
-
-  else
-    error "Neither yum nor dnf found. Aborting bootstrap!"
-    exit 1
-  fi
-
-}
-
-BootstrapRpmCommonBase() {
-  # Arguments: whitespace-delimited python packages to install
-
-  InitializeRPMCommonBase
-
-  pkgs="
-    gcc
-    augeas-libs
-    openssl
-    openssl-devel
-    libffi-devel
-    redhat-rpm-config
-    ca-certificates
-    cargo
-  "
-
-  # Add the python packages
-  pkgs="$pkgs
-    $1
-  "
-
-  if $TOOL list installed "httpd" >/dev/null 2>&1; then
-    pkgs="$pkgs
-      mod_ssl
-    "
-  fi
-
-  if ! sudo $TOOL install -y $pkgs; then
-    error "Could not install OS dependencies. Aborting bootstrap!"
-    exit 1
-  fi
-}
-
-BootstrapRpmPython3() {
-  InitializeRPMCommonBase
-
-  python_pkgs="python3
-    python3-devel
-  "
+  PYENV_DEPS="make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
+              wget curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev \
+              liblzma-dev git"
+  ALL_DEPS="libaugeas0 $PYENV_DEPS"
+  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y $ALL_DEPS
+elif [ -f /etc/redhat-release ]; then
+  PYENV_DEPS="gcc zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel \
+              tk-devel libffi-devel xz-devel git"
+  ALL_DEPS="augeas-libs $PYENV_DEPS"
 
-  # We only expect this branch to be taken on RHEL 7.
-  if ! sudo $TOOL list 'python3*-devel' >/dev/null 2>&1; then
-    sudo yum-config-manager --enable rhel-7-server-rhui-extras-rpms rhel-7-server-rhui-optional-rpms
+  if yum list installed "httpd" >/dev/null 2>&1; then
+    ALL_DEPS="mod_ssl $ALL_DEPS"
   fi
 
-  BootstrapRpmCommonBase "$python_pkgs"
-}
-
-# Set Bootstrap to the function that installs OS dependencies on this system.
-if [ -f /etc/debian_version ]; then
-  Bootstrap() {
-    BootstrapDebCommon
-  }
-elif [ -f /etc/redhat-release ]; then
-  DeterminePythonVersion
-  Bootstrap() {
-    BootstrapRpmPython3
-  }
-
+  sudo yum install -y $ALL_DEPS
 fi
-
-Bootstrap
diff --git a/letstest/scripts/test_apache2.sh b/letstest/scripts/test_apache2.sh
index f7338b3b3..5a8f97042 100755
--- a/letstest/scripts/test_apache2.sh
+++ b/letstest/scripts/test_apache2.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash -ex
 
 # $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
 # are dynamically set at execution
@@ -8,7 +8,6 @@ then
     CONFFILE=/etc/apache2/sites-available/000-default.conf
     sudo apt-get update
     sudo DEBIAN_FRONTEND=noninteractive apt-get -y --no-upgrade install apache2 curl
-    sudo DEBIAN_FRONTEND=noninteractive apt-get -y install realpath # needed for test-apache-conf
     # For apache 2.4, set up ServerName
     sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE
     sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE
@@ -41,9 +40,19 @@ cd letsencrypt
 
 echo "Bootstrapping dependencies..."
 sudo letstest/scripts/bootstrap_os_packages.sh
-if [ $? -ne 0 ] ; then
-    exit 1
-fi
+
+# Install pyenv
+curl https://pyenv.run | bash
+export PYENV_ROOT="$HOME/.pyenv"
+export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init --path)"
+eval "$(pyenv init -)"
+
+# Install and configure Python
+# Python<=3.9 must be used because Python 3.10 requires too new of a version of
+# OpenSSL.
+pyenv install 3.9.10
+pyenv shell 3.9.10
 
 tools/venv.py -e acme -e certbot -e certbot-apache -e certbot-ci tox
 PEBBLE_LOGS="acme_server.log"
@@ -56,12 +65,15 @@ PEBBLE_URL="https://localhost:14000/dir"
 #   existing virtual host for the port used for http-01 validation.
 venv/bin/run_acme_server --http-01-port 80 > "${PEBBLE_LOGS}" 2>&1 &
 
-DumpPebbleLogs() {
-    if [ -f "${PEBBLE_LOGS}" ] ; then
+DumpPebbleLogsOnFailure() {
+    exit_status="$?"
+    if [ "$exit_status" != 0 ] && [ -f "${PEBBLE_LOGS}" ] ; then
         echo "Pebble's logs were:"
         cat "${PEBBLE_LOGS}"
     fi
+    exit "$exit_status"
 }
+trap DumpPebbleLogsOnFailure EXIT
 
 for n in $(seq 1 150) ; do
     if curl --insecure "${PEBBLE_URL}" 2>/dev/null; then
@@ -80,9 +92,6 @@ fi
 sudo "venv/bin/certbot" -v --debug --text --agree-tos --no-verify-ssl \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain "${PUBLIC_HOSTNAME}" --server "${PEBBLE_URL}"
-if [ $? -ne 0 ] ; then
-    FAIL=1
-fi
 
 # Check that ssl_module detection is working on various systems
 if [ "$OS_TYPE" = "ubuntu" ] ; then
@@ -95,9 +104,6 @@ fi
 OPENSSL_VERSION=$(strings "$MOD_SSL_LOCATION" | egrep -o -m1 '^OpenSSL ([0-9]\.[^ ]+) ' | tail -c +9)
 APACHE_VERSION=$(sudo $APACHE_NAME -v | egrep -o 'Apache/([0-9]\.[^ ]+)' | tail -c +8)
 "venv/bin/python" letstest/scripts/test_openssl_version.py "$OPENSSL_VERSION" "$APACHE_VERSION"
-if [ $? -ne 0 ] ; then
-    FAIL=1
-fi
 
 
 if [ "$OS_TYPE" = "ubuntu" ] ; then
@@ -106,13 +112,3 @@ if [ "$OS_TYPE" = "ubuntu" ] ; then
 else
     echo Not running hackish apache tests on $OS_TYPE
 fi
-
-if [ $? -ne 0 ] ; then
-    FAIL=1
-fi
-
-# return error if any of the subtests failed
-if [ "$FAIL" = 1 ] ; then
-    DumpPebbleLogs
-    exit 1
-fi