Revert "Revert "TEST COMMIT: API doc generation changes.""

This reverts commit 502089f275ded113732c24cad2a96e2a899ecd5c.

Enable again temporarily the new test code for API doc generation.

2 files changed, 273 insertions, 136 deletions

diff --git a/doc/python_api/sphinx_changelog_gen.py b/doc/python_api/sphinx_changelog_gen.py
index a782c6483b6..6a5ebc1cfb8 100644
--- a/doc/python_api/sphinx_changelog_gen.py
+++ b/doc/python_api/sphinx_changelog_gen.py
@@ -1,59 +1,112 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 """
-Dump the python API into a text file so we can generate changelogs.
+---------------
 
-output from this tool should be added into "doc/python_api/rst/change_log.rst"
+Dump the python API into a JSON file, or generate changelogs from those JSON API dumps.
 
-# dump api blender_version.py in CWD
-blender --background --python doc/python_api/sphinx_changelog_gen.py -- --dump
+Typically, changelog output from this tool should be added into "doc/python_api/rst/change_log.rst"
 
-# create changelog
-blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \
-        --api_from blender_2_63_0.py \
-        --api_to   blender_2_64_0.py \
-        --api_out changes.rst
+API dump files are saved together with the generated API doc on the server, with a general index file.
+This way the changelog generation simply needs to re-download the previous version's dump for the diffing process.
+
+---------------
 
+# Dump api blender_version.json in CWD:
+blender --background  --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \
+        --indexpath="path/to/api/docs/api_dump_index.json" \
+        dump --filepath-out="path/to/api/docs/<version>/api_dump.json"
+
+# Create changelog:
+blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \
+        --indexpath="path/to/api/docs/api_dump_index.json" \
+        changelog --filepath-out doc/python_api/rst/change_log.rst
 
-# Api comparison can also run without blender
+# Api comparison can also run without blender,
+# will by default generate changeloig between the last two available versions listed in the index,
+# unless input files are provided explicitely:
 python doc/python_api/sphinx_changelog_gen.py -- \
-        --api_from blender_api_2_63_0.py \
-        --api_to   blender_api_2_64_0.py \
-        --api_out changes.rst
+        --indexpath="path/to/api/docs/api_dump_index.json" \
+        changelog --filepath-in-from blender_api_2_63_0.json \
+                  --filepath-in-to   blender_api_2_64_0.json \
+                  --filepath-out changes.rst
 
-# Save the latest API dump in this folder, renaming it with its revision.
-# This way the next person updating it doesn't need to build an old Blender only for that
+--------------
 
-"""
+API dump index format:
 
-# format
-'''
-{"module.name":
-    {"parent.class":
-        {"basic_type", "member_name":
-            ("Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types)}, ...
-    }, ...
+{[version_main, version_sub]: "<version>/api_dump.json", ...
 }
-'''
 
-api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types"
+API dump format:
+
+[
+    [version_main, vserion_sub, version_path],
+    {"module.name":
+        {"parent.class":
+            {"basic_type", "member_name":
+                ["Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types]}, ...
+        }, ...
+    }
+]
 
+"""
+
+import json
+import os
+
+
+api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types"
 API_BASIC_TYPE = 0
 API_F_ARGS = 7
 
 
-def api_dunp_fname():
-    import bpy
-    return "blender_api_%s.py" % "_".join([str(i) for i in bpy.app.version])
+def api_version():
+    try:
+        import bpy
+    except:
+        return None, None
+    version = tuple(bpy.app.version[:2])
+    version_key = "%d.%d" % (version[0], version[1])
+    return version, version_key
+
+
+def api_version_previous_in_index(index, version):
+    print("Searching for previous version to %s in %r" % (version, index))
+    version_prev = (version[0], version[1])
+    while True:
+        version_prev = (version_prev[0], version_prev[1] - 1)
+        if version_prev[1] < 0:
+            version_prev = (version_prev[0] - 1, 99)
+        if version_prev[0] < 0:
+            return None, None
+        version_prev_key = "%d.%d" % (version_prev[0], version_prev[1])
+        print("Checking for previous version %s" % (version_prev,))
+        if version_prev_key in index:
+            print("Found previous version %s: %r" % (version_prev, index[version_prev_key]))
+            return version_prev, version_prev_key
+
+
+class JSONEncoderAPIDump(json.JSONEncoder):
+    def default(self, o):
+        if o is ...:
+            return "..."
+        if isinstance(o, set):
+            return tuple(o)
+        return json.JSONEncoder.default(self, o)
+
+
+def api_dump(args):
+    import rna_info
+    import inspect
 
+    version, version_key = api_version()
+    if version is None:
+        raise(ValueError("API dumps can only be generated from within Blender."))
 
-def api_dump():
     dump = {}
     dump_module = dump["bpy.types"] = {}
 
-    import rna_info
-    import inspect
-
     struct = rna_info.BuildRNAInfo()[0]
     for struct_id, struct_info in sorted(struct.items()):
 
@@ -155,17 +208,24 @@ def api_dump():
             )
         del funcs
 
-    import pprint
+    filepath_out = args.filepath_out
+    with open(filepath_out, 'w', encoding='utf-8') as file_handle:
+        json.dump((version, dump), file_handle, cls=JSONEncoderAPIDump)
 
-    filename = api_dunp_fname()
-    filehandle = open(filename, 'w', encoding='utf-8')
-    tot = filehandle.write(pprint.pformat(dump, width=1))
-    filehandle.close()
-    print("%s, %d bytes written" % (filename, tot))
+    indexpath = args.indexpath
+    if os.path.exists(indexpath):
+        with open(indexpath, 'r', encoding='utf-8') as file_handle:
+            index = json.load(file_handle)
+    else:
+        index = {}
+    index[version_key] = filepath_out
+    with open(indexpath, 'w', encoding='utf-8') as file_handle:
+        json.dump(index, file_handle)
 
+    print("API version %s dumped into %r, and index %r has been updated" % (version_key, filepath_out, indexpath))
 
-def compare_props(a, b, fuzz=0.75):
 
+def compare_props(a, b, fuzz=0.75):
     # must be same basic_type, function != property
     if a[0] != b[0]:
         return False
@@ -180,15 +240,45 @@ def compare_props(a, b, fuzz=0.75):
     return ((tot / totlen) >= fuzz)
 
 
-def api_changelog(api_from, api_to, api_out):
+def api_changelog(args):
+    indexpath = args.indexpath
+    filepath_in_from = args.filepath_in_from
+    filepath_in_to = args.filepath_in_to
+    filepath_out = args.filepath_out
+
+    rootpath = os.path.dirname(indexpath)
+
+    version, version_key = api_version()
+    if version is None and (filepath_in_from is None or filepath_in_to is None):
+        raise(ValueError("API dumps files must be given when ran outside of Blender."))
+
+    with open(indexpath, 'r', encoding='utf-8') as file_handle:
+        index = json.load(file_handle)
+
+    if filepath_in_to == None:
+        filepath_in_to = index.get(version_key, None)
+    if filepath_in_to == None:
+        raise(ValueError("Cannot find API dump file for Blender version " + str(version) + " in index file."))
+
+    print("Found to file: %r" % filepath_in_to)
 
-    file_handle = open(api_from, 'r', encoding='utf-8')
-    dict_from = eval(file_handle.read())
-    file_handle.close()
+    if filepath_in_from == None:
+        version_from, version_from_key = api_version_previous_in_index(index, version)
+        if version_from is None:
+            raise(ValueError("No previous version of Blender could be found in the index."))
+        filepath_in_from = index.get(version_from_key, None)
+    if filepath_in_from is None:
+        raise(ValueError("Cannot find API dump file for previous Blender version " + str(version_from) + " in index file."))
 
-    file_handle = open(api_to, 'r', encoding='utf-8')
-    dict_to = eval(file_handle.read())
-    file_handle.close()
+    print("Found from file: %r" % filepath_in_from)
+
+
+    with open(os.path.join(rootpath, filepath_in_from), 'r', encoding='utf-8') as file_handle:
+        _, dict_from = json.load(file_handle)
+
+    with open(os.path.join(rootpath, filepath_in_to), 'r', encoding='utf-8') as file_handle:
+        dump_version, dict_to = json.load(file_handle)
+        assert(tuple(dump_version) == version)
 
     api_changes = []
 
@@ -249,63 +339,69 @@ def api_changelog(api_from, api_to, api_out):
 
     # also document function argument changes
 
-    fout = open(api_out, 'w', encoding='utf-8')
-    fw = fout.write
-    # print(api_changes)
-
-    # :class:`bpy_struct.id_data`
-
-    def write_title(title, title_char):
-        fw("%s\n%s\n\n" % (title, title_char * len(title)))
-
-    for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes:
-        class_name = class_id.split(".")[-1]
-        title = mod_id + "." + class_name
-        write_title(title, "-")
-
-        if props_new:
-            write_title("Added", "^")
-            for prop_id in props_new:
-                fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id))
-            fw("\n")
-
-        if props_old:
-            write_title("Removed", "^")
-            for prop_id in props_old:
-                fw("* **%s**\n" % prop_id)  # can't link to removed docs
-            fw("\n")
-
-        if props_moved:
-            write_title("Renamed", "^")
-            for prop_id_old, prop_id in props_moved:
-                fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id))
-            fw("\n")
-
-        if func_args:
-            write_title("Function Arguments", "^")
-            for func_id, args_old, args_new in func_args:
-                args_new = ", ".join(args_new)
-                args_old = ", ".join(args_old)
-                fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old))
-            fw("\n")
-
-    fout.close()
-
-    print("Written: %r" % api_out)
-
-
-def main():
+    with open(filepath_out, 'w', encoding='utf-8') as fout:
+        fw = fout.write
+        # print(api_changes)
+
+        # :class:`bpy_struct.id_data`
+
+        # Write header.
+        fw(""
+           ":tocdepth: 2\n"
+           "\n"
+           "Blender API Change Log\n"
+           "**********************\n"
+           "\n"
+           ".. note, this document is auto generated by sphinx_changelog_gen.py\n"
+           "\n"
+           "\n"
+           "%s to %s\n"
+           "============\n"
+           "\n" % (version_from_key, version_key))
+
+        def write_title(title, title_char):
+            fw("%s\n%s\n\n" % (title, title_char * len(title)))
+
+        for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes:
+            class_name = class_id.split(".")[-1]
+            title = mod_id + "." + class_name
+            write_title(title, "-")
+
+            if props_new:
+                write_title("Added", "^")
+                for prop_id in props_new:
+                    fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id))
+                fw("\n")
+
+            if props_old:
+                write_title("Removed", "^")
+                for prop_id in props_old:
+                    fw("* **%s**\n" % prop_id)  # can't link to removed docs
+                fw("\n")
+
+            if props_moved:
+                write_title("Renamed", "^")
+                for prop_id_old, prop_id in props_moved:
+                    fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id))
+                fw("\n")
+
+            if func_args:
+                write_title("Function Arguments", "^")
+                for func_id, args_old, args_new in func_args:
+                    args_new = ", ".join(args_new)
+                    args_old = ", ".join(args_old)
+                    fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old))
+                fw("\n")
+
+    print("Written: %r" % filepath_out)
+
+
+def main(argv=None):
     import sys
-    import os
-
-    try:
-        import argparse
-    except ImportError:
-        print("Old Blender, just dumping")
-        api_dump()
-        return
+    import argparse
 
-    argv = sys.argv
+    if argv is None:
+        argv = sys.argv
 
     if "--" not in argv:
         argv = []  # as if no args are passed
@@ -316,42 +412,39 @@ def main():
     usage_text = "Run blender in background mode with this script: "
     "blender --background --factory-startup --python %s -- [options]" % os.path.basename(__file__)
 
-    epilog = "Run this before releases"
-
-    parser = argparse.ArgumentParser(description=usage_text, epilog=epilog)
-
-    parser.add_argument(
-        "--dump", dest="dump", action='store_true',
-        help="When set the api will be dumped into blender_version.py")
-
+    parser = argparse.ArgumentParser(description=usage_text,
+                                     epilog=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument(
-        "--api_from", dest="api_from", metavar='FILE',
-        help="File to compare from (previous version)")
-    parser.add_argument(
-        "--api_to", dest="api_to", metavar='FILE',
-        help="File to compare from (current)")
-    parser.add_argument(
-        "--api_out", dest="api_out", metavar='FILE',
-        help="Output sphinx changelog")
-
-    args = parser.parse_args(argv)  # In this example we won't use the args
-
-    if not argv:
-        print("No args given!")
-        parser.print_help()
-        return
-
-    if args.dump:
-        api_dump()
-    else:
-        if args.api_from and args.api_to and args.api_out:
-            api_changelog(args.api_from, args.api_to, args.api_out)
-        else:
-            print("Error: --api_from/api_to/api_out args needed")
-            parser.print_help()
-            return
-
-    print("batch job finished, exiting")
+        "--indexpath", dest="indexpath", metavar='FILE', required=True,
+        help="Path of the JSON file containing the index of all available API dumps.")
+
+    parser_commands = parser.add_subparsers(required=True)
+
+    parser_dump = parser_commands.add_parser('dump', help="Dump the current Blender Python API into a JSON file.")
+    parser_dump.add_argument(
+        "--filepath-out", dest="filepath_out", metavar='FILE', required=True,
+        help="Path of the JSON file containing the dump of the API.")
+    parser_dump.set_defaults(func=api_dump)
+
+    parser_changelog = parser_commands.add_parser('changelog', help="Generate the RST changelog page based on two Blender Python API JSON dumps.")
+
+    parser_changelog.add_argument(
+        "--filepath-in-from", dest="filepath_in_from", metavar='FILE', default=None,
+        help="JSON dump file to compare from (typically, previous version). "
+             "If not given, will be automatically determined from current Blender version and index file.")
+    parser_changelog.add_argument(
+        "--filepath-in-to", dest="filepath_in_to", metavar='FILE', default=None,
+        help="JSON dump file to compare to (typically, current version). "
+             "If not given, will be automatically determined from current Blender version and index file.")
+    parser_changelog.add_argument(
+        "--filepath-out", dest="filepath_out", metavar='FILE', required=True,
+        help="Output sphinx changelog RST file.")
+    parser_changelog.set_defaults(func=api_changelog)
+
+    args = parser.parse_args(argv)
+
+    args.func(args)
 
 
 if __name__ == "__main__":
diff --git a/doc/python_api/sphinx_doc_gen.py b/doc/python_api/sphinx_doc_gen.py
index 8d4320917fc..13dc710994c 100644
--- a/doc/python_api/sphinx_doc_gen.py
+++ b/doc/python_api/sphinx_doc_gen.py
@@ -142,6 +142,26 @@ def handle_args():
     )
 
     parser.add_argument(
+        "--api-changelog-generate",
+        dest="changelog",
+        default=False,
+        action='store_true',
+        help="Generate the API changelog RST file "
+        "(default=False, requires `--api-dump-index-path` parameter)",
+        required=False,
+    )
+
+    parser.add_argument(
+        "--api-dump-index-path",
+        dest="api_dump_index_path",
+        metavar='FILE',
+        default=None,
+        help="Path to the API dump index JSON file "
+        "(required when `--api-changelog-generate` is True)",
+        required=False,
+    )
+
+    parser.add_argument(
         "-o", "--output",
         dest="output_dir",
         type=str,
@@ -514,6 +534,27 @@ if ARGS.sphinx_build_pdf:
         sphinx_make_pdf_log = os.path.join(ARGS.output_dir, ".latex_make.log")
         SPHINX_MAKE_PDF_STDOUT = open(sphinx_make_pdf_log, "w", encoding="utf-8")
 
+
+# --------------------------------CHANGELOG GENERATION--------------------------------------
+
+API_DUMP_INDEX_FILEPATH = ARGS.api_dump_index_path
+API_DUMP_ROOT = os.path.dirname(API_DUMP_INDEX_FILEPATH)
+API_DUMP_FILEPATH = os.path.abspath(os.path.join(API_DUMP_ROOT, BLENDER_VERSION_DOTS, "api_dump.json"))
+
+API_CHANGELOG_FILEPATH = os.path.abspath(os.path.join(SPHINX_IN_TMP, "change_log.rst"))
+
+def generate_changelog():
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("sphinx_changelog_gen",
+                                                  os.path.abspath(os.path.join(SCRIPT_DIR, "sphinx_changelog_gen.py")))
+    sphinx_changelog_gen = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(sphinx_changelog_gen)
+
+    sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "dump", "--filepath-out", API_DUMP_FILEPATH))
+
+    sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "changelog", "--filepath-out", API_CHANGELOG_FILEPATH))
+
+
 # --------------------------------API DUMP--------------------------------------
 
 # Lame, python won't give some access.
@@ -2473,6 +2514,9 @@ def main():
 
     rna2sphinx(SPHINX_IN_TMP)
 
+    if ARGS.changelog:
+        generate_changelog()
+
     if ARGS.full_rebuild:
         # Only for full updates.
         shutil.rmtree(SPHINX_IN, True)