Migrate the CI pipeline from Travis to Azure Pipeline (#8098)

Fixes #8071 and fixes https://github.com/certbot/certbot/issues/8110.

This PR migrates every job from Travis in Azure Pipeline.

This PR essentially converts the Travis jobs into Azure Pipeline with a complete iso-fonctionality (or I made a mistake). The jobs are added in the relevant existing pipelines (`main`, `nightly`, `advanced-test`, `release`). A global refactoring thanks to the templating system is done to reduce greatly the verbosity of the pipeline descriptions.

A specific feature (not present in Travis) is added: the stage `On_Failure`. Using directly the Mattermost API, it allows to notify pipeline failure in a Mattermost channel with a link to the failed pipelines without the need to authenticate to Microsoft.

See https://github.com/certbot/certbot/pull/8098#issuecomment-649873641 for the post merge actions to do at the end of this work.

5 files changed, 16 insertions, 30 deletions

diff --git a/tests/letstest/multitester.py b/tests/letstest/multitester.py
index 09821e7dd..5ed5ae549 100644
--- a/tests/letstest/multitester.py
+++ b/tests/letstest/multitester.py
@@ -63,10 +63,6 @@ parser.add_argument('aws_profile',
 parser.add_argument('test_script',
                     default='test_letsencrypt_auto_certonly_standalone.sh',
                     help='path of bash script in to deploy and run')
-#parser.add_argument('--script_args',
-#                    nargs='+',
-#                    help='space-delimited list of arguments to pass to the bash test script',
-#                    required=False)
 parser.add_argument('--repo',
                     default='https://github.com/letsencrypt/letsencrypt.git',
                     help='certbot git repo to use')
@@ -238,20 +234,20 @@ def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20
 # Fabric Routines
 #-------------------------------------------------------------------------------
 def local_git_clone(local_cxn, repo_url):
-    "clones master of repo_url"
+    """clones master of repo_url"""
     local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
     local_cxn.local('cd %s && git clone %s letsencrypt'% (LOGDIR, repo_url))
     local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt'% LOGDIR)
 
 def local_git_branch(local_cxn, repo_url, branch_name):
-    "clones branch <branch_name> of repo_url"
+    """clones branch <branch_name> of repo_url"""
     local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
     local_cxn.local('cd %s && git clone %s letsencrypt --branch %s --single-branch'%
         (LOGDIR, repo_url, branch_name))
     local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR)
 
 def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True):
-    "clones specified pull request from repo_url and optionally merges into master"
+    """clones specified pull request from repo_url and optionally merges into master"""
     local_cxn.local('cd %s && if [ -d letsencrypt ]; then rm -rf letsencrypt; fi' % LOGDIR)
     local_cxn.local('cd %s && git clone %s letsencrypt' % (LOGDIR, repo_url))
     local_cxn.local('cd %s && cd letsencrypt && '
@@ -264,20 +260,20 @@ def local_git_PR(local_cxn, repo_url, PRnumstr, merge_master=True):
     local_cxn.local('cd %s && tar czf le.tar.gz letsencrypt' % LOGDIR)
 
 def local_repo_to_remote(cxn):
-    "copies local tarball of repo to remote"
+    """copies local tarball of repo to remote"""
     filename = 'le.tar.gz'
     local_path = os.path.join(LOGDIR, filename)
     cxn.put(local=local_path, remote='')
     cxn.run('tar xzf %s' % filename)
 
 def local_repo_clean(local_cxn):
-    "delete tarball"
+    """delete tarball"""
     filename = 'le.tar.gz'
     local_path = os.path.join(LOGDIR, filename)
     local_cxn.local('rm %s' % local_path)
 
 def deploy_script(cxn, scriptpath, *args):
-    "copies to remote and executes local script"
+    """copies to remote and executes local script"""
     cxn.put(local=scriptpath, remote='', preserve_mode=True)
     scriptfile = os.path.split(scriptpath)[1]
     args_str = ' '.join(args)
@@ -425,23 +421,23 @@ def main():
     print("Making local dir for test repo and logs: %s"%LOGDIR)
     local_cxn.local('mkdir %s'%LOGDIR)
 
-    # figure out what git object to test and locally create it in LOGDIR
-    print("Making local git repo")
     try:
+        # figure out what git object to test and locally create it in LOGDIR
+        print("Making local git repo")
         if cl_args.pull_request != '~':
-            print('Testing PR %s '%cl_args.pull_request,
+            print('Testing PR %s ' % cl_args.pull_request,
                   "MERGING into master" if cl_args.merge_master else "")
             local_git_PR(local_cxn, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
         elif cl_args.branch != '~':
-            print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
+            print('Testing branch %s of %s' % (cl_args.branch, cl_args.repo))
             local_git_branch(local_cxn, cl_args.repo, cl_args.branch)
         else:
-            print('Testing master of %s'%cl_args.repo)
+            print('Testing current branch of %s' % cl_args.repo)
             local_git_clone(local_cxn, cl_args.repo)
     except BaseException:
         print("FAIL: trouble with git repo")
         traceback.print_exc()
-        exit()
+        exit(1)
 
 
     # Set up EC2 instances
diff --git a/tests/letstest/travis-setup.sh b/tests/letstest/travis-setup.sh
deleted file mode 100755
index 261a1504f..000000000
--- a/tests/letstest/travis-setup.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash -ex
-#
-# Preps the test farm tests to be run in Travis.
-
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
-    echo This script must be run in Travis on a non-pull request build
-    exit 1
-fi
-
-openssl aes-256-cbc -K "${encrypted_9a387195a62e_key}" -iv "${encrypted_9a387195a62e_iv}" -in travis-test-farm.pem.enc -out travis-test-farm.pem -d
diff --git a/tests/letstest/travis-test-farm.pem.enc b/tests/letstest/travis-test-farm.pem.enc
deleted file mode 100644
index f8b1d576c..000000000
--- a/tests/letstest/travis-test-farm.pem.enc
+++ /dev/null
diff --git a/tests/lock_test.py b/tests/lock_test.py
index 29a77ae17..56399c874 100644
--- a/tests/lock_test.py
+++ b/tests/lock_test.py
@@ -59,9 +59,9 @@ def set_up():
     command = set_up_command(config_dir, logs_dir, work_dir, nginx_dir)
 
     dirs = [logs_dir, config_dir, work_dir]
-    # Travis and Circle CI set CI to true so we
-    # will always test Nginx's lock during CI
-    if os.environ.get('CI') == 'true' or util.exe_exists('nginx'):
+    # If Nginx is installed, do the test, otherwise skip it.
+    # Issue https://github.com/certbot/certbot/issues/8121 tracks the work to remove this control.
+    if util.exe_exists('nginx'):
         dirs.append(nginx_dir)
     else:
         logger.warning('Skipping Nginx lock tests')
diff --git a/tests/modification-check.py b/tests/modification-check.py
index 811f369d4..7a69fb1db 100755
--- a/tests/modification-check.py
+++ b/tests/modification-check.py
@@ -57,7 +57,7 @@ def validate_scripts_content(repo_path, temp_cwd):
             'letsencrypt-auto-source/pieces/fetch.py')), temp_cwd)
 
         # Compare file against current version in the target branch
-        branch = os.environ.get('TRAVIS_BRANCH', 'master')
+        branch = os.environ.get('TARGET_BRANCH', 'master')
         url = (
             'https://raw.githubusercontent.com/certbot/certbot/{0}/certbot-auto'
             .format(branch))