Start disabling certbot-auto upgrades (#8623)

* add amazon linux to auto targets

* disable updates outside of debian and rhel

* test certbot-auto with disabled upgrades

* try new approach to testing

* remove bad space

* tweak error text

* add changelog entry

* fix bad certbot-auto commit

* test new error text

* update changelog

* update error text

2 files changed, 55 insertions, 1 deletions

diff --git a/tests/letstest/auto_targets.yaml b/tests/letstest/auto_targets.yaml
index 01d410227..164580e86 100644
--- a/tests/letstest/auto_targets.yaml
+++ b/tests/letstest/auto_targets.yaml
@@ -57,3 +57,10 @@ targets:
     type: centos
     virt: hvm
     user: centos
+  #-----------------------------------------------------------------------------
+  # Amazon Linux
+  - ami: ami-0ff8a91507f77f867
+    name: amazon
+    type: centos
+    virt: hvm
+    user: ec2-user
diff --git a/tests/letstest/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
index c599623cb..d24258a22 100755
--- a/tests/letstest/scripts/test_leauto_upgrades.sh
+++ b/tests/letstest/scripts/test_leauto_upgrades.sh
@@ -43,9 +43,11 @@ fi
 # directory to be served.
 MY_TEMP_DIR=$(mktemp -d)
 PORT_FILE="$MY_TEMP_DIR/port"
+LOG_FILE="$MY_TEMP_DIR/log"
 SERVER_PATH=$("$PYTHON_NAME" tools/readlink.py tools/simple_http_server.py)
 cd "$MY_TEMP_DIR"
-"$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE &
+# We set PYTHONUNBUFFERED to disable buffering of output to LOG_FILE
+PYTHONUNBUFFERED=1 "$PYTHON_NAME" "$SERVER_PATH" 0 > $PORT_FILE 2> "$LOG_FILE" &
 SERVER_PID=$!
 trap 'kill "$SERVER_PID" && rm -rf "$MY_TEMP_DIR"' EXIT
 cd ~-
@@ -119,3 +121,48 @@ if ! diff letsencrypt-auto letsencrypt-auto-source/letsencrypt-auto ; then
     echo letsencrypt-auto and letsencrypt-auto-source/letsencrypt-auto differ
     exit 1
 fi
+
+# Now let's test if letsencrypt-auto still tries to upgrade to a new version.
+# Regardless of the OS, versions of the script with development version numbers
+# ending in .dev0 will not upgrade. See
+# https://github.com/certbot/certbot/blob/bdfb9f19c4086a60ef010d2431768850c26d838a/certbot-auto#L1947-L1948.
+# In order to test the process of different OSes setting NO_SELF_UPGRADE as
+# part of the script's deprecation, we make use of the fact that
+# letsencrypt-auto should still attempt to fetch the version number from PyPI
+# even if it has a development version number unless NO_SELF_UPGRADE is set in
+# which case all of that logic should be skipped.
+#
+# First we make a copy of the current server logs.
+PREVIOUS_LOG_FILE="$MY_TEMP_DIR/previous-log"
+cp "$LOG_FILE" "$PREVIOUS_LOG_FILE"
+
+# Next we run letsencrypt-auto and make sure there were no problems checking
+# for updates, the Certbot install still works, the version number is what
+# we expect, and it prints a message about not receiving updates.
+if ./letsencrypt-auto -v --debug --version | grep "WARNING: couldn't find Python" ; then
+    echo "Had problems checking for updates!"
+    exit 1
+fi
+if ! ./letsencrypt-auto -v --debug --version 2>&1 | tail -n1 | grep "^certbot $EXPECTED_VERSION$" ; then
+    echo unexpected certbot version found
+    exit 1
+fi
+if ! ./letsencrypt-auto -v --debug --version 2>&1 | grep "will no longer receive updates" ; then
+    echo script did not print warning about not receiving updates!
+    exit 1
+fi
+
+# Finally, we check if our local server received more requests. Over time,
+# we'll move more and more OSes into this case until it this is the expected
+# behavior on all systems.
+if [ -f /etc/issue ] && grep -iq "Amazon Linux" /etc/issue; then
+    if ! diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then
+        echo our local server received unexpected requests
+        exit 1
+    fi
+else
+    if diff "$LOG_FILE" "$PREVIOUS_LOG_FILE" ; then
+        echo our local server did not receive the requests we expected
+        exit 1
+    fi
+fi