Retry Snap upload in pipeline (#8300)

* add set -e to all bash instances in deploy-stage.yml

* retry uploading snap if we fail

* Add the rest of the set -e calls for bash in azure while we're here

* use retry based on travis_retry

* add set -e to the script: sections that run on macOS/Linux

* actually don't fail on result

* reset result before running command because bash short circuits or conditionals

* remove inapplicable comment

5 files changed, 37 insertions, 5 deletions

diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml
index 2d659aef5..8da30b1f5 100644
--- a/.azure-pipelines/templates/jobs/packaging-jobs.yml
+++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml
@@ -13,17 +13,18 @@ jobs:
           arm64v8:
             DOCKER_ARCH: arm64v8
     steps:
-      - bash: tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
+      - bash: set -e && tools/docker/build.sh $(dockerTag) $DOCKER_ARCH
         displayName: Build the Docker images
       # We don't filter for the Docker Hub organization to continue to allow
       # easy testing of these scripts on forks.
       - bash: |
+          set -e
           DOCKER_IMAGES=$(docker images --filter reference='*/certbot' --filter reference='*/dns-*' --format '{{.Repository}}')
           docker save --output images.tar $DOCKER_IMAGES
         displayName: Save the Docker images
       # If the name of the tar file or artifact changes, the deploy stage will
       # also need to be updated.
-      - bash: mv images.tar $(Build.ArtifactStagingDirectory)
+      - bash: set -e && mv images.tar $(Build.ArtifactStagingDirectory)
         displayName: Prepare Docker artifact
       - task: PublishPipelineArtifact@1
         inputs:
@@ -109,6 +110,7 @@ jobs:
         ARCHS: amd64
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
           sudo snap install --classic snapcraft
@@ -122,6 +124,7 @@ jobs:
         inputs:
           secureFile: launchpad-credentials
       - script: |
+          set -e
           git config --global user.email "$(Build.RequestedForEmail)"
           git config --global user.name "$(Build.RequestedFor)"
           mkdir -p ~/.local/share/snapcraft/provider/launchpad
@@ -129,6 +132,7 @@ jobs:
           python3 tools/snap/build_remote.py ALL --archs ${ARCHS}
         displayName: Build snaps
       - script: |
+          set -e
           mv *.snap $(Build.ArtifactStagingDirectory)
           mv certbot-dns-*/*.snap $(Build.ArtifactStagingDirectory)
         displayName: Prepare artifacts
@@ -147,6 +151,7 @@ jobs:
           versionSpec: 3.8
           addToPath: true
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
           python3 -m venv venv
@@ -159,9 +164,11 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
         displayName: Install Certbot snap
       - script: |
+          set -e
           venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
@@ -170,6 +177,7 @@ jobs:
       vmImage: ubuntu-18.04
     steps:
       - script: |
+          set -e
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends snapd
         displayName: Install dependencies
@@ -183,10 +191,12 @@ jobs:
           path: $(Build.SourcesDirectory)/snap
         displayName: Retrieve Certbot snaps
       - script: |
+          set -e
           python3 -m venv venv
           venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
+          set -e
           sudo -E venv/bin/pytest certbot-ci/snap_integration_tests/dns_tests --allow-persistent-changes --snap-folder $(Build.SourcesDirectory)/snap --snap-arch amd64
         displayName: Test DNS plugins snaps
diff --git a/.azure-pipelines/templates/stages/changelog-stage.yml b/.azure-pipelines/templates/stages/changelog-stage.yml
index 64d9e2161..7d089f8d4 100644
--- a/.azure-pipelines/templates/stages/changelog-stage.yml
+++ b/.azure-pipelines/templates/stages/changelog-stage.yml
@@ -7,6 +7,7 @@ stages:
         steps:
           # If we change the output filename from `release_notes.md`, it should also be changed in tools/create_github_release.py
           - bash: |
+              set -e
               CERTBOT_VERSION="$(cd certbot && python -c "import certbot; print(certbot.__version__)" && cd ~-)"
               "${BUILD_REPOSITORY_LOCALPATH}\tools\extract_changelog.py" "${CERTBOT_VERSION}" >> "${BUILD_ARTIFACTSTAGINGDIRECTORY}/release_notes.md"
             displayName: Prepare changelog
diff --git a/.azure-pipelines/templates/stages/deploy-stage.yml b/.azure-pipelines/templates/stages/deploy-stage.yml
index e6cb6a0ba..17cb824d5 100644
--- a/.azure-pipelines/templates/stages/deploy-stage.yml
+++ b/.azure-pipelines/templates/stages/deploy-stage.yml
@@ -39,6 +39,7 @@ stages:
           - group: certbot-common
         steps:
           - bash: |
+              set -e
               sudo apt-get update
               sudo apt-get install -y --no-install-recommends snapd
               sudo snap install --classic snapcraft
@@ -53,10 +54,25 @@ stages:
             inputs:
               secureFile: snapcraft.cfg
           - bash: |
+              set -e
+              retry_command() {
+                # based on travis_retry.bash https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash
+                local result=0
+                local count=1
+                while [[ "${count}" -le 3 ]]; do
+                  result=0
+                  "${@}" || result="${?}"
+                  if [[ $result -eq 0 ]]; then break; fi
+                  count="$((count + 1))"
+                  sleep 1
+                done
+
+                return "${result}"
+              }
               mkdir -p .snapcraft
               ln -s $(snapcraftCfg.secureFilePath) .snapcraft/snapcraft.cfg
               for SNAP_FILE in snap/*.snap; do
-                snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
+                retry_command eval snapcraft upload --release=${{ parameters.snapReleaseChannel }} "${SNAP_FILE}"
               done
             displayName: Publish to Snap store
       - job: publish_docker
@@ -76,7 +92,7 @@ stages:
               artifact: docker_$(DOCKER_ARCH)
               path: $(Build.SourcesDirectory)
             displayName: Retrieve Docker images
-          - bash: docker load --input $(Build.SourcesDirectory)/images.tar
+          - bash: set -e && docker load --input $(Build.SourcesDirectory)/images.tar
             displayName: Load Docker images
           - task: Docker@2
             inputs:
@@ -93,5 +109,5 @@ stages:
               # Certbot organization on Docker Hub.
               containerRegistry: docker-hub
             displayName: Login to Docker Hub
-          - bash: tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
+          - bash: set -e && tools/docker/deploy.sh $(dockerTag) $DOCKER_ARCH
             displayName: Deploy the Docker images
diff --git a/.azure-pipelines/templates/stages/notify-failure-stage.yml b/.azure-pipelines/templates/stages/notify-failure-stage.yml
index 9a7f8b1fc..1542f5ebc 100644
--- a/.azure-pipelines/templates/stages/notify-failure-stage.yml
+++ b/.azure-pipelines/templates/stages/notify-failure-stage.yml
@@ -8,6 +8,7 @@ stages:
           vmImage: ubuntu-latest
         steps:
           - bash: |
+              set -e
               MESSAGE="\
               ---\n\
               ##### Azure Pipeline
diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml
index 7f6c3150f..14b27b08f 100644
--- a/.azure-pipelines/templates/steps/tox-steps.yml
+++ b/.azure-pipelines/templates/steps/tox-steps.yml
@@ -1,9 +1,11 @@
 steps:
   - bash: |
+      set -e
       brew install augeas
     condition: startswith(variables['IMAGE_NAME'], 'macOS')
     displayName: Install MacOS dependencies
   - bash: |
+      set -e
       sudo apt-get update
       sudo apt-get install -y --no-install-recommends \
         python-dev \
@@ -29,6 +31,7 @@ steps:
     # set, pip updates dependencies it thinks are already satisfied to avoid some
     # problems with its lack of real dependency resolution.
   - bash: |
+      set -e
       python letsencrypt-auto-source/pieces/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
@@ -38,6 +41,7 @@ steps:
       secureFile: azure-test-farm.pem
     condition: contains(variables['TOXENV'], 'test-farm')
   - bash: |
+      set -e
       export TARGET_BRANCH="`echo "${BUILD_SOURCEBRANCH}" | sed -E 's!refs/(heads|tags)/!!g'`"
       [ -z "${SYSTEM_PULLREQUEST_TARGETBRANCH}" ] || export TARGET_BRANCH="${SYSTEM_PULLREQUEST_TARGETBRANCH}"
       env