Enable mypy strict mode (#8766)test-master

Built on top of #8748, this PR reenables mypy strict mode and adds the appropriate corrections to pass the types checks.

* Upgrade mypy

* First step for acme

* Cast for the rescue

* Fixing types for certbot

* Fix typing for certbot-nginx

* Finalize type fixes, configure no optional strict check for mypy in tox

* Align requirements

* Isort

* Pylint

* Protocol for python 3.6

* Use Python 3.9 for mypy, make code compatible with Python 3.8<

* Pylint and mypy

* Pragma no cover

* Pythonic NotImplemented constant

* More type definitions

* Add comments

* Simplify typing logic

* Use vararg tuple

* Relax constraints on mypy

* Add more type

* Do not silence error if target is not defined

* Conditionally import Protocol for type checking only

* Clean up imports

* Add comments

* Align python version linting with mypy and coverage

* Just ignore types in an unused module

* Add comments

* Fix lint

* Work in progress

* Finish type control

* Isort

* Fix pylint

* Fix imports

* Fix cli subparser

* Some fixes

* Coverage

* Remove --no-strict-optional (obviously...)

* Update certbot-apache/certbot_apache/_internal/configurator.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* Update certbot/certbot/_internal/display/completer.py

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* Cleanup dns_google

* Improve lock controls and fix subparser

* Use the expected interfaces

* Fix code

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

4 files changed, 42 insertions, 29 deletions

diff --git a/certbot-apache/certbot_apache/_internal/configurator.py b/certbot-apache/certbot_apache/_internal/configurator.py
index c24a646db..28f73b32d 100644
--- a/certbot-apache/certbot_apache/_internal/configurator.py
+++ b/certbot-apache/certbot_apache/_internal/configurator.py
@@ -12,6 +12,7 @@ from typing import cast
 from typing import DefaultDict
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Set
 from typing import Union
 
@@ -36,6 +37,9 @@ from certbot_apache._internal import dualparser
 from certbot_apache._internal import http_01
 from certbot_apache._internal import obj
 from certbot_apache._internal import parser
+from certbot_apache._internal.dualparser import DualBlockNode
+from certbot_apache._internal.obj import VirtualHost
+from certbot_apache._internal.parser import ApacheParser
 
 try:
     import apacheconfig
@@ -232,11 +236,11 @@ class ApacheConfigurator(common.Installer):
         self.parsed_paths: List[str] = []
         # These will be set in the prepare function
         self._prepared = False
-        self.parser = None
-        self.parser_root = None
+        self.parser: ApacheParser
+        self.parser_root: Optional[DualBlockNode] = None
         self.version = version
         self._openssl_version = openssl_version
-        self.vhosts = None
+        self.vhosts: List[VirtualHost]
         self.options = copy.deepcopy(self.OS_DEFAULTS)
         self._enhance_func = {"redirect": self._enable_redirect,
                               "ensure-http-header": self._set_http_header,
@@ -345,8 +349,9 @@ class ApacheConfigurator(common.Installer):
                    "augeaspath": self.parser.get_root_augpath(),
                    "ac_ast": None}
         if self.USE_PARSERNODE:
-            self.parser_root = self.get_parsernode_root(pn_meta)
-            self.parsed_paths = self.parser_root.parsed_paths()
+            parser_root = self.get_parsernode_root(pn_meta)
+            self.parser_root = parser_root
+            self.parsed_paths = parser_root.parsed_paths()
 
         # Check for errors in parsing files with Augeas
         self.parser.check_parsing_errors("httpd.aug")
@@ -408,7 +413,7 @@ class ApacheConfigurator(common.Installer):
         super(ApacheConfigurator, self).recovery_routine()
         # Reload configuration after these changes take effect if needed
         # ie. ApacheParser has been initialized.
-        if self.parser:
+        if hasattr(self, "parser"):
             # TODO: wrap into non-implementation specific  parser interface
             self.parser.aug.load()
 
@@ -1051,6 +1056,9 @@ class ApacheConfigurator(common.Installer):
         :rtype: list
         """
 
+        if not self.parser_root:
+            raise errors.Error("This ApacheConfigurator instance is not"  # pragma: no cover
+                               " configured to use a node parser.")
         vhs = []
         vhosts = self.parser_root.find_blocks("VirtualHost", exclude=False)
         for vhblock in vhosts:
diff --git a/certbot-apache/certbot_apache/_internal/override_centos.py b/certbot-apache/certbot_apache/_internal/override_centos.py
index fc77aaafc..bd9a47b52 100644
--- a/certbot-apache/certbot_apache/_internal/override_centos.py
+++ b/certbot-apache/certbot_apache/_internal/override_centos.py
@@ -119,8 +119,9 @@ class CentOSConfigurator(configurator.ApacheConfigurator):
             else:
                 loadmod_args = path_args
 
-            if self.parser.not_modssl_ifmodule(noarg_path):  # pylint: disable=no-member
-                if self.parser.loc["default"] in noarg_path:
+            centos_parser: CentOSParser = cast(CentOSParser, self.parser)
+            if centos_parser.not_modssl_ifmodule(noarg_path):
+                if centos_parser.loc["default"] in noarg_path:
                     # LoadModule already in the main configuration file
                     if ("ifmodule/" in noarg_path.lower() or
                         "ifmodule[1]" in noarg_path.lower()):
diff --git a/certbot-apache/certbot_apache/_internal/parser.py b/certbot-apache/certbot_apache/_internal/parser.py
index 1a6af4c4b..ff7e90f3b 100644
--- a/certbot-apache/certbot_apache/_internal/parser.py
+++ b/certbot-apache/certbot_apache/_internal/parser.py
@@ -5,12 +5,18 @@ import logging
 import re
 from typing import Dict
 from typing import List
+from typing import Optional
 
 from certbot import errors
 from certbot.compat import os
 from certbot_apache._internal import apache_util
 from certbot_apache._internal import constants
 
+try:
+    from augeas import Augeas
+except ImportError:  # pragma: no cover
+    Augeas = None  # type: ignore
+
 logger = logging.getLogger(__name__)
 
 
@@ -39,8 +45,7 @@ class ApacheParser:
         self.configurator = configurator
 
         # Initialize augeas
-        self.aug = None
-        self.init_augeas()
+        self.aug = init_augeas()
 
         if not self.check_aug_version():
             raise errors.NotSupportedError(
@@ -48,7 +53,7 @@ class ApacheParser:
                 "version 1.2.0 or higher, please make sure you have you have "
                 "those installed.")
 
-        self.modules: Dict[str, str] = {}
+        self.modules: Dict[str, Optional[str]] = {}
         self.parser_paths: Dict[str, List[str]] = {}
         self.variables: Dict[str, str] = {}
 
@@ -83,23 +88,6 @@ class ApacheParser:
             if self.find_dir("Define", exclude=False):
                 raise errors.PluginError("Error parsing runtime variables")
 
-    def init_augeas(self):
-        """ Initialize the actual Augeas instance """
-
-        try:
-            import augeas
-        except ImportError:  # pragma: no cover
-            raise errors.NoInstallationError("Problem in Augeas installation")
-
-        self.aug = augeas.Augeas(
-            # specify a directory to load our preferred lens from
-            loadpath=constants.AUGEAS_LENS_DIR,
-            # Do not save backup (we do it ourselves), do not load
-            # anything by default
-            flags=(augeas.Augeas.NONE |
-                   augeas.Augeas.NO_MODL_AUTOLOAD |
-                   augeas.Augeas.ENABLE_SPAN))
-
     def check_parsing_errors(self, lens):
         """Verify Augeas can parse all of the lens files.
 
@@ -949,3 +937,19 @@ def get_aug_path(file_path):
 
     """
     return "/files%s" % file_path
+
+
+def init_augeas() -> Augeas:
+    """ Initialize the actual Augeas instance """
+
+    if not Augeas:  # pragma: no cover
+        raise errors.NoInstallationError("Problem in Augeas installation")
+
+    return Augeas(
+        # specify a directory to load our preferred lens from
+        loadpath=constants.AUGEAS_LENS_DIR,
+        # Do not save backup (we do it ourselves), do not load
+        # anything by default
+        flags=(Augeas.NONE |
+               Augeas.NO_MODL_AUTOLOAD |
+               Augeas.ENABLE_SPAN))
diff --git a/certbot-apache/tests/parser_test.py b/certbot-apache/tests/parser_test.py
index 7aedec31d..2eff9a9dd 100644
--- a/certbot-apache/tests/parser_test.py
+++ b/certbot-apache/tests/parser_test.py
@@ -339,7 +339,7 @@ class ParserInitTest(util.ApacheTest):
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
-    @mock.patch("certbot_apache._internal.parser.ApacheParser.init_augeas")
+    @mock.patch("certbot_apache._internal.parser.init_augeas")
     def test_prepare_no_augeas(self, mock_init_augeas):
         from certbot_apache._internal.parser import ApacheParser
         mock_init_augeas.side_effect = errors.NoInstallationError