Invoke pipstrap in tox and during the CI (#8316)

Partial fix for #8256

This PR makes tox calls pipstrap before any commands is executed, and Azure Pipelines calls pipstrap when appropriate (when an actual call to pip is done). 

* Invoke pipstrap in tox and during the CI

* Set default value for PYTHON_VERSION and always set python interpreter

* Set Python for snaps_build also

* Fix the build for Windows installer

* Add a warning comment for pinned versions in pipstrap

* Rebuild letsencrypt-auto

* Same version than the installer build

* Let's update to latest pip for installer tests

4 files changed, 21 insertions, 3 deletions

diff --git a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml
index 0e1a98861..67fa34880 100644
--- a/.azure-pipelines/templates/jobs/extended-tests-jobs.yml
+++ b/.azure-pipelines/templates/jobs/extended-tests-jobs.yml
@@ -3,6 +3,8 @@ jobs:
     variables:
       - name: IMAGE_NAME
         value: ubuntu-18.04
+      - name: PYTHON_VERSION
+        value: 3.8
       - group: certbot-common
     strategy:
       matrix:
diff --git a/.azure-pipelines/templates/jobs/packaging-jobs.yml b/.azure-pipelines/templates/jobs/packaging-jobs.yml
index b0c7998cb..2d659aef5 100644
--- a/.azure-pipelines/templates/jobs/packaging-jobs.yml
+++ b/.azure-pipelines/templates/jobs/packaging-jobs.yml
@@ -78,9 +78,16 @@ jobs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
+      # pip 9.0 provided by pipstrap is not able to resolve properly the pywin32 dependency
+      # required by certbot-ci: as a temporary workaround until pipstrap is updated, we install
+      # a recent version of pip, but we also to disable the isolated feature as described in
+      # https://github.com/certbot/certbot/issues/8256
       - script: |
           py -3 -m venv venv
+          venv\Scripts\python -m pip install pip==20.2.3 setuptools==50.3.0 wheel==0.35.1
           venv\Scripts\python tools\pip_install.py -e certbot-ci
+        env:
+          PIP_NO_BUILD_ISOLATION: no
         displayName: Prepare Certbot-CI
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
@@ -135,10 +142,16 @@ jobs:
     pool:
       vmImage: ubuntu-18.04
     steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: 3.8
+          addToPath: true
       - script: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends nginx-light snapd
-          python tools/pip_install.py -U tox
+          python3 -m venv venv
+          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
+          venv/bin/python tools/pip_install.py -U tox
         displayName: Install dependencies
       - task: DownloadPipelineArtifact@2
         inputs:
@@ -149,7 +162,7 @@ jobs:
           sudo snap install --dangerous --classic snap/certbot_*_amd64.snap
         displayName: Install Certbot snap
       - script: |
-          python -m tox -e integration-external,apacheconftest-external-with-pebble
+          venv/bin/python -m tox -e integration-external,apacheconftest-external-with-pebble
         displayName: Run tox
   - job: snap_dns_run
     dependsOn: snaps_build
@@ -171,6 +184,7 @@ jobs:
         displayName: Retrieve Certbot snaps
       - script: |
           python3 -m venv venv
+          venv/bin/python letsencrypt-auto-source/pieces/pipstrap.py
           venv/bin/python tools/pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
diff --git a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml
index 3bb73b67e..d5b3a0a16 100644
--- a/.azure-pipelines/templates/jobs/standard-tests-jobs.yml
+++ b/.azure-pipelines/templates/jobs/standard-tests-jobs.yml
@@ -1,5 +1,7 @@
 jobs:
   - job: test
+    variables:
+      PYTHON_VERSION: 3.8
     strategy:
       matrix:
         macos-py27:
diff --git a/.azure-pipelines/templates/steps/tox-steps.yml b/.azure-pipelines/templates/steps/tox-steps.yml
index 828552e43..7f6c3150f 100644
--- a/.azure-pipelines/templates/steps/tox-steps.yml
+++ b/.azure-pipelines/templates/steps/tox-steps.yml
@@ -21,7 +21,6 @@ steps:
     inputs:
       versionSpec: $(PYTHON_VERSION)
       addToPath: true
-    condition: ne(variables['PYTHON_VERSION'], '')
     # tools/pip_install.py is used to pin packages to a known working version
     # except in tests where the environment variable CERTBOT_NO_PIN is set.
     # virtualenv is listed here explicitly to make sure it is upgraded when
@@ -30,6 +29,7 @@ steps:
     # set, pip updates dependencies it thinks are already satisfied to avoid some
     # problems with its lack of real dependency resolution.
   - bash: |
+      python letsencrypt-auto-source/pieces/pipstrap.py
       python tools/pip_install.py -I tox virtualenv
     displayName: Install runtime dependencies
   - task: DownloadSecureFile@1