Split Ansible docs from core docs (#73616)

* excludes scenario guides from core docs, splits porting guides and roadmaps, symlinks indices to create index.html pages, and adds .gitignore entries for conf.py and the toplevel index.rst files generated by the docs build

This solution builds three types of docs:
* ansible-2.10 and earlier: all the docs.  Handle this via `make webdocs
  ANSIBLE_VERSION=2.10`
* ansible-3 and later: a subset of the docs for the ansible package.
  Handle this via `make webdocs ANSIBLE_VERSION=3` (change the
  ANSIBLE_VERSION to match the version being built for.
* ansible-core: a subset of the docs for the ansible-core package.
  Handle this via `make coredocs`.

* `make webdocs` now always builds all the collection docs
*  Use `make coredocs` to limit it to core plugins only
*  The user specifies the desired version. If no ANSIBLE_VERSION is specified, build plugins for the latest release of ansible
 
Co-authored-by: Toshio Kuratomi <a.badger@gmail.com>
Co-authored-by: Matt Clay <matt@mystile.com>

1 files changed, 85 insertions, 27 deletions

diff --git a/hacking/build_library/build_ansible/command_plugins/docs_build.py b/hacking/build_library/build_ansible/command_plugins/docs_build.py
index 21b09b03963..e38ef2010c0 100644
--- a/hacking/build_library/build_ansible/command_plugins/docs_build.py
+++ b/hacking/build_library/build_ansible/command_plugins/docs_build.py
@@ -19,6 +19,7 @@ from ansible.release import __version__ as ansible_base__version__
 # Pylint doesn't understand Python3 namespace modules.
 # pylint: disable=relative-beyond-top-level
 from ..commands import Command
+from ..errors import InvalidUserInput, MissingUserInput
 # pylint: enable=relative-beyond-top-level
 
 
@@ -29,6 +30,67 @@ DEFAULT_TOP_DIR = pathlib.Path(__file__).parents[4]
 DEFAULT_OUTPUT_DIR = pathlib.Path(__file__).parents[4] / 'docs/docsite'
 
 
+class NoSuchFile(Exception):
+    """An expected file was not found."""
+
+
+#
+# Helpers
+#
+
+def find_latest_ansible_dir(build_data_working):
+    """Find the most recent ansible major version."""
+    # imports here so that they don't cause unnecessary deps for all of the plugins
+    from packaging.version import InvalidVersion, Version
+
+    ansible_directories = glob.glob(os.path.join(build_data_working, '[0-9.]*'))
+
+    # Find the latest ansible version directory
+    latest = None
+    latest_ver = Version('0')
+    for directory_name in (d for d in ansible_directories if os.path.isdir(d)):
+        try:
+            new_version = Version(os.path.basename(directory_name))
+        except InvalidVersion:
+            continue
+
+        if new_version > latest_ver:
+            latest_ver = new_version
+            latest = directory_name
+
+    if latest is None:
+        raise NoSuchFile('Could not find an ansible data directory in {0}'.format(build_data_working))
+
+    return latest
+
+
+def find_latest_deps_file(build_data_working, ansible_version):
+    """Find the most recent ansible deps file for the given ansible major version."""
+    # imports here so that they don't cause unnecessary deps for all of the plugins
+    from packaging.version import Version
+
+    data_dir = os.path.join(build_data_working, ansible_version)
+    deps_files = glob.glob(os.path.join(data_dir, '*.deps'))
+    if not deps_files:
+        raise Exception('No deps files exist for version {0}'.format(ansible_version))
+
+    # Find the latest version of the deps file for this major version
+    latest = None
+    latest_ver = Version('0')
+    for filename in deps_files:
+        with open(filename, 'r') as f:
+            deps_data = yaml.safe_load(f.read())
+        new_version = Version(deps_data['_ansible_version'])
+        if new_version > latest_ver:
+            latest_ver = new_version
+            latest = filename
+
+    if latest is None:
+        raise NoSuchFile('Could not find an ansible deps file in {0}'.format(data_dir))
+
+    return latest
+
+
 #
 # Subcommand base
 #
@@ -70,36 +132,23 @@ def generate_full_docs(args):
     # imports here so that they don't cause unnecessary deps for all of the plugins
     import sh
     from antsibull.cli import antsibull_docs
-    from packaging.version import Version
-
-    ansible_base_ver = Version(ansible_base__version__)
-    ansible_base_major_ver = '{0}.{1}'.format(ansible_base_ver.major, ansible_base_ver.minor)
 
     with TemporaryDirectory() as tmp_dir:
         sh.git(['clone', 'https://github.com/ansible-community/ansible-build-data'], _cwd=tmp_dir)
-        # This is wrong.  Once ansible and ansible-base major.minor versions get out of sync this
-        # will stop working.  We probably need to walk all subdirectories in reverse version order
-        # looking for the latest ansible version which uses something compatible with
-        # ansible_base_major_ver.
-        deps_files = glob.glob(os.path.join(tmp_dir, 'ansible-build-data',
-                                            ansible_base_major_ver, '*.deps'))
-        if not deps_files:
-            raise Exception('No deps files exist for version {0}'.format(ansible_base_major_ver))
-
-        # Find the latest version of the deps file for this version
-        latest = None
-        latest_ver = Version('0')
-        for filename in deps_files:
-            with open(filename, 'r') as f:
-                deps_data = yaml.safe_load(f.read())
-            new_version = Version(deps_data['_ansible_version'])
-            if new_version > latest_ver:
-                latest_ver = new_version
-                latest = filename
-
-        # Make a copy of the deps file so that we can set the ansible-base version to use
+        # If we want to validate that the ansible version and ansible-base branch version match,
+        # this would be the place to do it.
+
+        build_data_working = os.path.join(tmp_dir, 'ansible-build-data')
+
+        ansible_version = args.ansible_version
+        if ansible_version is None:
+            ansible_version = find_latest_ansible_dir(build_data_working)
+
+        latest_filename = find_latest_deps_file(build_data_working, ansible_version)
+
+        # Make a copy of the deps file so that we can set the ansible-base version we'll use
         modified_deps_file = os.path.join(tmp_dir, 'ansible.deps')
-        shutil.copyfile(latest, modified_deps_file)
+        shutil.copyfile(latest_filename, modified_deps_file)
 
         # Put our version of ansible-base into the deps file
         with open(modified_deps_file, 'r') as f:
@@ -136,6 +185,8 @@ class CollectionPluginDocs(Command):
                             ' documentation location that says the module is in a collection and'
                             ' point to generated plugin documentation under the collections/'
                             ' hierarchy.')
+        # I think we should make the actions a subparser but need to look in git history and see if
+        # we tried that and changed it for some reason.
         parser.add_argument('action', action='store', choices=('full', 'base', 'named'),
                             default='full', help=cls._ACTION_HELP)
         parser.add_argument("-o", "--output-dir", action="store", dest="output_dir",
@@ -149,10 +200,17 @@ class CollectionPluginDocs(Command):
                             dest="limit_to", default=None,
                             help="Limit building module documentation to comma-separated list of"
                             " plugins. Specify non-existing plugin name for no plugins.")
+        parser.add_argument('--ansible-version', action='store',
+                            dest='ansible_version', default=None,
+                            help='The version of the ansible package to make documentation for.'
+                            '  This only makes sense when used with full.')
 
     @staticmethod
     def main(args):
-        # normalize CLI args
+        # normalize and validate CLI args
+
+        if args.ansible_version and args.action != 'full':
+            raise InvalidUserInput('--ansible-version is only for use with "full".')
 
         if not args.output_dir:
             args.output_dir = os.path.abspath(str(DEFAULT_OUTPUT_DIR))