From 5a30fe29ef2e1f424df0403284b3ebba5644403f Mon Sep 17 00:00:00 2001 From: Bastien Montagne Date: Thu, 16 Jun 2022 16:36:11 +0200 Subject: Revert "Revert "TEST COMMIT: API doc generation changes."" This reverts commit 502089f275ded113732c24cad2a96e2a899ecd5c. Enable again temporarily the new test code for API doc generation. --- doc/python_api/sphinx_changelog_gen.py | 365 +++++++++++++++++++++------------ doc/python_api/sphinx_doc_gen.py | 44 ++++ 2 files changed, 273 insertions(+), 136 deletions(-) (limited to 'doc') diff --git a/doc/python_api/sphinx_changelog_gen.py b/doc/python_api/sphinx_changelog_gen.py index a782c6483b6..6a5ebc1cfb8 100644 --- a/doc/python_api/sphinx_changelog_gen.py +++ b/doc/python_api/sphinx_changelog_gen.py @@ -1,59 +1,112 @@ # SPDX-License-Identifier: GPL-2.0-or-later """ -Dump the python API into a text file so we can generate changelogs. +--------------- -output from this tool should be added into "doc/python_api/rst/change_log.rst" +Dump the python API into a JSON file, or generate changelogs from those JSON API dumps. -# dump api blender_version.py in CWD -blender --background --python doc/python_api/sphinx_changelog_gen.py -- --dump +Typically, changelog output from this tool should be added into "doc/python_api/rst/change_log.rst" -# create changelog -blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \ - --api_from blender_2_63_0.py \ - --api_to blender_2_64_0.py \ - --api_out changes.rst +API dump files are saved together with the generated API doc on the server, with a general index file. +This way the changelog generation simply needs to re-download the previous version's dump for the diffing process. + +--------------- +# Dump api blender_version.json in CWD: +blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \ + --indexpath="path/to/api/docs/api_dump_index.json" \ + dump --filepath-out="path/to/api/docs//api_dump.json" + +# Create changelog: +blender --background --factory-startup --python doc/python_api/sphinx_changelog_gen.py -- \ + --indexpath="path/to/api/docs/api_dump_index.json" \ + changelog --filepath-out doc/python_api/rst/change_log.rst -# Api comparison can also run without blender +# Api comparison can also run without blender, +# will by default generate changeloig between the last two available versions listed in the index, +# unless input files are provided explicitely: python doc/python_api/sphinx_changelog_gen.py -- \ - --api_from blender_api_2_63_0.py \ - --api_to blender_api_2_64_0.py \ - --api_out changes.rst + --indexpath="path/to/api/docs/api_dump_index.json" \ + changelog --filepath-in-from blender_api_2_63_0.json \ + --filepath-in-to blender_api_2_64_0.json \ + --filepath-out changes.rst -# Save the latest API dump in this folder, renaming it with its revision. -# This way the next person updating it doesn't need to build an old Blender only for that +-------------- -""" +API dump index format: -# format -''' -{"module.name": - {"parent.class": - {"basic_type", "member_name": - ("Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types)}, ... - }, ... +{[version_main, version_sub]: "/api_dump.json", ... } -''' -api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types" +API dump format: + +[ + [version_main, vserion_sub, version_path], + {"module.name": + {"parent.class": + {"basic_type", "member_name": + ["Name", type, range, length, default, descr, f_args, f_arg_types, f_ret_types]}, ... + }, ... + } +] +""" + +import json +import os + + +api_names = "basic_type" "name", "type", "range", "length", "default", "descr", "f_args", "f_arg_types", "f_ret_types" API_BASIC_TYPE = 0 API_F_ARGS = 7 -def api_dunp_fname(): - import bpy - return "blender_api_%s.py" % "_".join([str(i) for i in bpy.app.version]) +def api_version(): + try: + import bpy + except: + return None, None + version = tuple(bpy.app.version[:2]) + version_key = "%d.%d" % (version[0], version[1]) + return version, version_key + + +def api_version_previous_in_index(index, version): + print("Searching for previous version to %s in %r" % (version, index)) + version_prev = (version[0], version[1]) + while True: + version_prev = (version_prev[0], version_prev[1] - 1) + if version_prev[1] < 0: + version_prev = (version_prev[0] - 1, 99) + if version_prev[0] < 0: + return None, None + version_prev_key = "%d.%d" % (version_prev[0], version_prev[1]) + print("Checking for previous version %s" % (version_prev,)) + if version_prev_key in index: + print("Found previous version %s: %r" % (version_prev, index[version_prev_key])) + return version_prev, version_prev_key + + +class JSONEncoderAPIDump(json.JSONEncoder): + def default(self, o): + if o is ...: + return "..." + if isinstance(o, set): + return tuple(o) + return json.JSONEncoder.default(self, o) + + +def api_dump(args): + import rna_info + import inspect + version, version_key = api_version() + if version is None: + raise(ValueError("API dumps can only be generated from within Blender.")) -def api_dump(): dump = {} dump_module = dump["bpy.types"] = {} - import rna_info - import inspect - struct = rna_info.BuildRNAInfo()[0] for struct_id, struct_info in sorted(struct.items()): @@ -155,17 +208,24 @@ def api_dump(): ) del funcs - import pprint + filepath_out = args.filepath_out + with open(filepath_out, 'w', encoding='utf-8') as file_handle: + json.dump((version, dump), file_handle, cls=JSONEncoderAPIDump) - filename = api_dunp_fname() - filehandle = open(filename, 'w', encoding='utf-8') - tot = filehandle.write(pprint.pformat(dump, width=1)) - filehandle.close() - print("%s, %d bytes written" % (filename, tot)) + indexpath = args.indexpath + if os.path.exists(indexpath): + with open(indexpath, 'r', encoding='utf-8') as file_handle: + index = json.load(file_handle) + else: + index = {} + index[version_key] = filepath_out + with open(indexpath, 'w', encoding='utf-8') as file_handle: + json.dump(index, file_handle) + print("API version %s dumped into %r, and index %r has been updated" % (version_key, filepath_out, indexpath)) -def compare_props(a, b, fuzz=0.75): +def compare_props(a, b, fuzz=0.75): # must be same basic_type, function != property if a[0] != b[0]: return False @@ -180,15 +240,45 @@ def compare_props(a, b, fuzz=0.75): return ((tot / totlen) >= fuzz) -def api_changelog(api_from, api_to, api_out): +def api_changelog(args): + indexpath = args.indexpath + filepath_in_from = args.filepath_in_from + filepath_in_to = args.filepath_in_to + filepath_out = args.filepath_out + + rootpath = os.path.dirname(indexpath) + + version, version_key = api_version() + if version is None and (filepath_in_from is None or filepath_in_to is None): + raise(ValueError("API dumps files must be given when ran outside of Blender.")) + + with open(indexpath, 'r', encoding='utf-8') as file_handle: + index = json.load(file_handle) + + if filepath_in_to == None: + filepath_in_to = index.get(version_key, None) + if filepath_in_to == None: + raise(ValueError("Cannot find API dump file for Blender version " + str(version) + " in index file.")) + + print("Found to file: %r" % filepath_in_to) - file_handle = open(api_from, 'r', encoding='utf-8') - dict_from = eval(file_handle.read()) - file_handle.close() + if filepath_in_from == None: + version_from, version_from_key = api_version_previous_in_index(index, version) + if version_from is None: + raise(ValueError("No previous version of Blender could be found in the index.")) + filepath_in_from = index.get(version_from_key, None) + if filepath_in_from is None: + raise(ValueError("Cannot find API dump file for previous Blender version " + str(version_from) + " in index file.")) - file_handle = open(api_to, 'r', encoding='utf-8') - dict_to = eval(file_handle.read()) - file_handle.close() + print("Found from file: %r" % filepath_in_from) + + + with open(os.path.join(rootpath, filepath_in_from), 'r', encoding='utf-8') as file_handle: + _, dict_from = json.load(file_handle) + + with open(os.path.join(rootpath, filepath_in_to), 'r', encoding='utf-8') as file_handle: + dump_version, dict_to = json.load(file_handle) + assert(tuple(dump_version) == version) api_changes = [] @@ -249,63 +339,69 @@ def api_changelog(api_from, api_to, api_out): # also document function argument changes - fout = open(api_out, 'w', encoding='utf-8') - fw = fout.write - # print(api_changes) - - # :class:`bpy_struct.id_data` - - def write_title(title, title_char): - fw("%s\n%s\n\n" % (title, title_char * len(title))) - - for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes: - class_name = class_id.split(".")[-1] - title = mod_id + "." + class_name - write_title(title, "-") - - if props_new: - write_title("Added", "^") - for prop_id in props_new: - fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id)) - fw("\n") - - if props_old: - write_title("Removed", "^") - for prop_id in props_old: - fw("* **%s**\n" % prop_id) # can't link to removed docs - fw("\n") - - if props_moved: - write_title("Renamed", "^") - for prop_id_old, prop_id in props_moved: - fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id)) - fw("\n") - - if func_args: - write_title("Function Arguments", "^") - for func_id, args_old, args_new in func_args: - args_new = ", ".join(args_new) - args_old = ", ".join(args_old) - fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old)) - fw("\n") - - fout.close() - - print("Written: %r" % api_out) - - -def main(): + with open(filepath_out, 'w', encoding='utf-8') as fout: + fw = fout.write + # print(api_changes) + + # :class:`bpy_struct.id_data` + + # Write header. + fw("" + ":tocdepth: 2\n" + "\n" + "Blender API Change Log\n" + "**********************\n" + "\n" + ".. note, this document is auto generated by sphinx_changelog_gen.py\n" + "\n" + "\n" + "%s to %s\n" + "============\n" + "\n" % (version_from_key, version_key)) + + def write_title(title, title_char): + fw("%s\n%s\n\n" % (title, title_char * len(title))) + + for mod_id, class_id, props_moved, props_new, props_old, func_args in api_changes: + class_name = class_id.split(".")[-1] + title = mod_id + "." + class_name + write_title(title, "-") + + if props_new: + write_title("Added", "^") + for prop_id in props_new: + fw("* :class:`%s.%s.%s`\n" % (mod_id, class_name, prop_id)) + fw("\n") + + if props_old: + write_title("Removed", "^") + for prop_id in props_old: + fw("* **%s**\n" % prop_id) # can't link to removed docs + fw("\n") + + if props_moved: + write_title("Renamed", "^") + for prop_id_old, prop_id in props_moved: + fw("* **%s** -> :class:`%s.%s.%s`\n" % (prop_id_old, mod_id, class_name, prop_id)) + fw("\n") + + if func_args: + write_title("Function Arguments", "^") + for func_id, args_old, args_new in func_args: + args_new = ", ".join(args_new) + args_old = ", ".join(args_old) + fw("* :class:`%s.%s.%s` (%s), *was (%s)*\n" % (mod_id, class_name, func_id, args_new, args_old)) + fw("\n") + + print("Written: %r" % filepath_out) + + +def main(argv=None): import sys - import os - - try: - import argparse - except ImportError: - print("Old Blender, just dumping") - api_dump() - return + import argparse - argv = sys.argv + if argv is None: + argv = sys.argv if "--" not in argv: argv = [] # as if no args are passed @@ -316,42 +412,39 @@ def main(): usage_text = "Run blender in background mode with this script: " "blender --background --factory-startup --python %s -- [options]" % os.path.basename(__file__) - epilog = "Run this before releases" - - parser = argparse.ArgumentParser(description=usage_text, epilog=epilog) - - parser.add_argument( - "--dump", dest="dump", action='store_true', - help="When set the api will be dumped into blender_version.py") - + parser = argparse.ArgumentParser(description=usage_text, + epilog=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( - "--api_from", dest="api_from", metavar='FILE', - help="File to compare from (previous version)") - parser.add_argument( - "--api_to", dest="api_to", metavar='FILE', - help="File to compare from (current)") - parser.add_argument( - "--api_out", dest="api_out", metavar='FILE', - help="Output sphinx changelog") - - args = parser.parse_args(argv) # In this example we won't use the args - - if not argv: - print("No args given!") - parser.print_help() - return - - if args.dump: - api_dump() - else: - if args.api_from and args.api_to and args.api_out: - api_changelog(args.api_from, args.api_to, args.api_out) - else: - print("Error: --api_from/api_to/api_out args needed") - parser.print_help() - return - - print("batch job finished, exiting") + "--indexpath", dest="indexpath", metavar='FILE', required=True, + help="Path of the JSON file containing the index of all available API dumps.") + + parser_commands = parser.add_subparsers(required=True) + + parser_dump = parser_commands.add_parser('dump', help="Dump the current Blender Python API into a JSON file.") + parser_dump.add_argument( + "--filepath-out", dest="filepath_out", metavar='FILE', required=True, + help="Path of the JSON file containing the dump of the API.") + parser_dump.set_defaults(func=api_dump) + + parser_changelog = parser_commands.add_parser('changelog', help="Generate the RST changelog page based on two Blender Python API JSON dumps.") + + parser_changelog.add_argument( + "--filepath-in-from", dest="filepath_in_from", metavar='FILE', default=None, + help="JSON dump file to compare from (typically, previous version). " + "If not given, will be automatically determined from current Blender version and index file.") + parser_changelog.add_argument( + "--filepath-in-to", dest="filepath_in_to", metavar='FILE', default=None, + help="JSON dump file to compare to (typically, current version). " + "If not given, will be automatically determined from current Blender version and index file.") + parser_changelog.add_argument( + "--filepath-out", dest="filepath_out", metavar='FILE', required=True, + help="Output sphinx changelog RST file.") + parser_changelog.set_defaults(func=api_changelog) + + args = parser.parse_args(argv) + + args.func(args) if __name__ == "__main__": diff --git a/doc/python_api/sphinx_doc_gen.py b/doc/python_api/sphinx_doc_gen.py index 8d4320917fc..13dc710994c 100644 --- a/doc/python_api/sphinx_doc_gen.py +++ b/doc/python_api/sphinx_doc_gen.py @@ -141,6 +141,26 @@ def handle_args(): required=False, ) + parser.add_argument( + "--api-changelog-generate", + dest="changelog", + default=False, + action='store_true', + help="Generate the API changelog RST file " + "(default=False, requires `--api-dump-index-path` parameter)", + required=False, + ) + + parser.add_argument( + "--api-dump-index-path", + dest="api_dump_index_path", + metavar='FILE', + default=None, + help="Path to the API dump index JSON file " + "(required when `--api-changelog-generate` is True)", + required=False, + ) + parser.add_argument( "-o", "--output", dest="output_dir", @@ -514,6 +534,27 @@ if ARGS.sphinx_build_pdf: sphinx_make_pdf_log = os.path.join(ARGS.output_dir, ".latex_make.log") SPHINX_MAKE_PDF_STDOUT = open(sphinx_make_pdf_log, "w", encoding="utf-8") + +# --------------------------------CHANGELOG GENERATION-------------------------------------- + +API_DUMP_INDEX_FILEPATH = ARGS.api_dump_index_path +API_DUMP_ROOT = os.path.dirname(API_DUMP_INDEX_FILEPATH) +API_DUMP_FILEPATH = os.path.abspath(os.path.join(API_DUMP_ROOT, BLENDER_VERSION_DOTS, "api_dump.json")) + +API_CHANGELOG_FILEPATH = os.path.abspath(os.path.join(SPHINX_IN_TMP, "change_log.rst")) + +def generate_changelog(): + import importlib.util + spec = importlib.util.spec_from_file_location("sphinx_changelog_gen", + os.path.abspath(os.path.join(SCRIPT_DIR, "sphinx_changelog_gen.py"))) + sphinx_changelog_gen = importlib.util.module_from_spec(spec) + spec.loader.exec_module(sphinx_changelog_gen) + + sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "dump", "--filepath-out", API_DUMP_FILEPATH)) + + sphinx_changelog_gen.main(("--", "--indexpath", API_DUMP_INDEX_FILEPATH, "changelog", "--filepath-out", API_CHANGELOG_FILEPATH)) + + # --------------------------------API DUMP-------------------------------------- # Lame, python won't give some access. @@ -2473,6 +2514,9 @@ def main(): rna2sphinx(SPHINX_IN_TMP) + if ARGS.changelog: + generate_changelog() + if ARGS.full_rebuild: # Only for full updates. shutil.rmtree(SPHINX_IN, True) -- cgit v1.2.3