Windows installer integration tests (#7724)

As discussed in #7539, we need proper tests of the Windows installer itself in order to variety that all the logic contained in a production-grade runtime of Certbot on Windows is correctly setup by each version of the installer, and so for a variety of Windows OSes. 

This PR handles this requirement. The new `windows_installer_integration_tests` module in `certbot-ci` will:
* run the given Windows installer
* check that Certbot is properly installed and working
* check that the scheduled renew task is set up
* check that the scheduled task actually launch the Certbot renew logic

The Windows nightly tests are updated accordingly, in order to have the tests run on Windows Server 2012R2, 2016 and 2019.

These tests will evolve as we add more logic on the installer. 

* Configure an integration test testing the windows installer

* Write the test module

* Configurable installer path, prepare azure pipelines

* Fix option

* Update test_main.py

* Add confirmation for this destructive test

* Use regex to validate certbot --version output

* Explicit dependency on a log output

* Use an exception to ask confirmation

* Use --allow-persistent-changes

5 files changed, 111 insertions, 7 deletions

diff --git a/.azure-pipelines/templates/installer-tests.yml b/.azure-pipelines/templates/installer-tests.yml
index f1ccd92ed..6d5672339 100644
--- a/.azure-pipelines/templates/installer-tests.yml
+++ b/.azure-pipelines/templates/installer-tests.yml
@@ -33,22 +33,24 @@ jobs:
     pool:
       vmImage: $(imageName)
     steps:
+      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
+        displayName: Get Python
+      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
+        displayName: Install Python
       - task: DownloadPipelineArtifact@2
         inputs:
           artifact: windows-installer
           path: $(Build.SourcesDirectory)/bin
         displayName: Retrieve Windows installer
-      - script: $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe /S
-        displayName: Install Certbot
-      - powershell: Invoke-WebRequest https://www.python.org/ftp/python/3.8.1/python-3.8.1-amd64-webinstall.exe -OutFile C:\py3-setup.exe
-        displayName: Get Python
-      - script: C:\py3-setup.exe /quiet PrependPath=1 InstallAllUsers=1 Include_launcher=1 InstallLauncherAllUsers=1 Include_test=0 Include_doc=0 Include_dev=1 Include_debug=0 Include_tcltk=0 TargetDir=C:\py3
-        displayName: Install Python
       - script: |
           py -3 -m venv venv
           venv\Scripts\python tools\pip_install.py -e certbot-ci
         displayName: Prepare Certbot-CI
       - script: |
           set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
+          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win32.exe
+        displayName: Run windows installer integration tests
+      - script: |
+          set PATH=%ProgramFiles(x86)%\Certbot\bin;%PATH%
           venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run integration tests
+        displayName: Run certbot integration tests
diff --git a/certbot-ci/windows_installer_integration_tests/__init__.py b/certbot-ci/windows_installer_integration_tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/certbot-ci/windows_installer_integration_tests/__init__.py
diff --git a/certbot-ci/windows_installer_integration_tests/conftest.py b/certbot-ci/windows_installer_integration_tests/conftest.py
new file mode 100644
index 000000000..e36654f90
--- /dev/null
+++ b/certbot-ci/windows_installer_integration_tests/conftest.py
@@ -0,0 +1,38 @@
+"""
+General conftest for pytest execution of all integration tests lying
+in the window_installer_integration tests package.
+As stated by pytest documentation, conftest module is used to set on
+for a directory a specific configuration using built-in pytest hooks.
+
+See https://docs.pytest.org/en/latest/reference.html#hook-reference
+"""
+from __future__ import print_function
+import os
+
+import pytest
+
+ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+
+
+def pytest_addoption(parser):
+    """
+    Standard pytest hook to add options to the pytest parser.
+    :param parser: current pytest parser that will be used on the CLI
+    """
+    parser.addoption('--installer-path',
+                     default=os.path.join(ROOT_PATH, 'windows-installer', 'build',
+                                          'nsis', 'certbot-beta-installer-win32.exe'),
+                     help='set the path of the windows installer to use, default to '
+                          'CERTBOT_ROOT_PATH\\windows-installer\\build\\nsis\\certbot-beta-installer-win32.exe')
+    parser.addoption('--allow-persistent-changes', action='store_true',
+                     help='needs to be set, and confirm that the test will make persistent changes on this machine')
+
+
+def pytest_configure(config):
+    """
+    Standard pytest hook used to add a configuration logic for each node of a pytest run.
+    :param config: the current pytest configuration
+    """
+    if not config.option.allow_persistent_changes:
+        raise RuntimeError('This integration test would install Certbot on your machine. '
+                           'Please run it again with the `--allow-persistent-changes` flag set to acknowledge.')
diff --git a/certbot-ci/windows_installer_integration_tests/test_main.py b/certbot-ci/windows_installer_integration_tests/test_main.py
new file mode 100644
index 000000000..c8c347aa8
--- /dev/null
+++ b/certbot-ci/windows_installer_integration_tests/test_main.py
@@ -0,0 +1,61 @@
+import os
+import time
+import unittest
+import subprocess
+import re
+
+
+@unittest.skipIf(os.name != 'nt', reason='Windows installer tests must be run on Windows.')
+def test_it(request):
+    try:
+        subprocess.check_call(['certbot', '--version'])
+    except (subprocess.CalledProcessError, OSError):
+        pass
+    else:
+        raise AssertionError('Expect certbot to not be available in the PATH.')
+
+    try:
+        # Install certbot
+        subprocess.check_call([request.config.option.installer_path, '/S'])
+
+        # Assert certbot is installed and runnable
+        output = subprocess.check_output(['certbot', '--version'], universal_newlines=True)
+        assert re.match(r'^certbot \d+\.\d+\.\d+.*$', output), 'Flag --version does not output a version.'
+
+        # Assert renew task is installed and ready
+        output = _ps('(Get-ScheduledTask -TaskName "Certbot Renew Task").State', capture_stdout=True)
+        assert output.strip() == 'Ready'
+
+        # Assert renew task is working
+        now = time.time()
+        _ps('Start-ScheduledTask -TaskName "Certbot Renew Task"')
+
+        status = 'Running'
+        while status != 'Ready':
+            status = _ps('(Get-ScheduledTask -TaskName "Certbot Renew Task").State', capture_stdout=True).strip()
+            time.sleep(1)
+
+        log_path = os.path.join('C:\\', 'Certbot', 'log', 'letsencrypt.log')
+
+        modification_time = os.path.getmtime(log_path)
+        assert now < modification_time, 'Certbot log file has not been modified by the renew task.'
+
+        with open(log_path) as file_h:
+            data = file_h.read()
+        assert 'no renewal failures' in data, 'Renew task did not execute properly.'
+
+    finally:
+        # Sadly this command cannot work in non interactive mode: uninstaller will ask explicitly permission in an UAC prompt
+        # print('Uninstalling Certbot ...')
+        # uninstall_path = _ps('(gci "HKLM:\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall"'
+        #                      ' | foreach { gp $_.PSPath }'
+        #                      ' | ? { $_ -match "Certbot" }'
+        #                      ' | select UninstallString)'
+        #                      '.UninstallString', capture_stdout=True)
+        # subprocess.check_call([uninstall_path, '/S'])
+        pass
+
+
+def _ps(powershell_str, capture_stdout=False):
+    fn = subprocess.check_output if capture_stdout else subprocess.check_call
+    return fn(['powershell.exe', '-c', powershell_str], universal_newlines=True)
diff --git a/certbot/certbot/_internal/renewal.py b/certbot/certbot/_internal/renewal.py
index 930f6c1a9..bf30404f5 100644
--- a/certbot/certbot/_internal/renewal.py
+++ b/certbot/certbot/_internal/renewal.py
@@ -471,4 +471,7 @@ def handle_renewal_request(config):
     if renew_failures or parse_failures:
         raise errors.Error("{0} renew failure(s), {1} parse failure(s)".format(
             len(renew_failures), len(parse_failures)))
+
+    # Windows installer integration tests rely on handle_renewal_request behavior here.
+    # If the text below changes, these tests will need to be updated accordingly.
     logger.debug("no renewal failures")