LibLink/Append: Port `bpy.data.libraries.load` to new `BKE_blendfile_link_append` module.

Note that this fully replaces the 'PyCapsule' storage of linked/appended items
in the python API code by the generic storage of items in the
`BlendfileLinkAppendContext` data.

Maniphest Tasks: T91414

Differential Revision: https://developer.blender.org/D13331

1 files changed, 134 insertions, 115 deletions

diff --git a/source/blender/python/intern/bpy_library_load.c b/source/blender/python/intern/bpy_library_load.c
index 059d692a9ba..fe279c0b940 100644
--- a/source/blender/python/intern/bpy_library_load.c
+++ b/source/blender/python/intern/bpy_library_load.c
@@ -34,6 +34,7 @@
 #include "BLI_string.h"
 #include "BLI_utildefines.h"
 
+#include "BKE_blendfile_link_append.h"
 #include "BKE_context.h"
 #include "BKE_idtype.h"
 #include "BKE_lib_id.h"
@@ -346,11 +347,63 @@ static void bpy_lib_exit_warn_type(BPy_Library *self, PyObject *item)
   PyErr_Restore(exc, val, tb);
 }
 
+struct LibExitLappContextItemsIterData {
+  short idcode;
+  BPy_Library *py_library;
+  PyObject *py_list;
+  Py_ssize_t py_list_size;
+};
+
+static bool bpy_lib_exit_lapp_context_items_cb(BlendfileLinkAppendContext *lapp_context,
+                                               BlendfileLinkAppendContextItem *item,
+                                               void *userdata)
+{
+  struct LibExitLappContextItemsIterData *data = userdata;
+
+  /* Since `bpy_lib_exit` loops over all ID types, all items in `lapp_context` end up being looped
+   * over for each ID type, so when it does not match the item can simply be skipped: it either has
+   * already been processed, or will be processed in a later loop. */
+  if (BKE_blendfile_link_append_context_item_idcode_get(lapp_context, item) != data->idcode) {
+    return true;
+  }
+
+  const int py_list_index = POINTER_AS_INT(
+      BKE_blendfile_link_append_context_item_userdata_get(lapp_context, item));
+  ID *new_id = BKE_blendfile_link_append_context_item_newid_get(lapp_context, item);
+
+  BLI_assert(py_list_index < data->py_list_size);
+
+  /* Fully invalid items (which got set to `Py_None` already in first loop of `bpy_lib_exit`)
+   * should never be accessed here, since their index should never be set to any item in
+   * `lapp_context`. */
+  PyObject *item_src = PyList_GET_ITEM(data->py_list, py_list_index);
+  BLI_assert(item_src != Py_None);
+
+  PyObject *py_item;
+  if (new_id != NULL) {
+    PointerRNA newid_ptr;
+    RNA_id_pointer_create(new_id, &newid_ptr);
+    py_item = pyrna_struct_CreatePyObject(&newid_ptr);
+  }
+  else {
+    const char *item_idname = PyUnicode_AsUTF8(item_src);
+    const char *idcode_name_plural = BKE_idtype_idcode_to_name_plural(data->idcode);
+
+    bpy_lib_exit_warn_idname(data->py_library, idcode_name_plural, item_idname);
+
+    py_item = Py_INCREF_RET(Py_None);
+  }
+
+  PyList_SET_ITEM(data->py_list, py_list_index, py_item);
+
+  Py_DECREF(item_src);
+
+  return true;
+}
+
 static PyObject *bpy_lib_exit(BPy_Library *self, PyObject *UNUSED(args))
 {
   Main *bmain = self->bmain;
-  Main *mainl = NULL;
-  const int err = 0;
   const bool do_append = ((self->flag & FILE_LINK) == 0);
 
   BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, true);
@@ -360,134 +413,100 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject *UNUSED(args))
   struct LibraryLink_Params liblink_params;
   BLO_library_link_params_init(&liblink_params, bmain, self->flag, id_tag_extra);
 
-  mainl = BLO_library_link_begin(&(self->blo_handle), self->relpath, &liblink_params);
-
-  {
-    int idcode_step = 0, idcode;
-    while ((idcode = BKE_idtype_idcode_iter_step(&idcode_step))) {
-      if (BKE_idtype_idcode_is_linkable(idcode) && (idcode != ID_WS || do_append)) {
-        const char *name_plural = BKE_idtype_idcode_to_name_plural(idcode);
-        PyObject *ls = PyDict_GetItemString(self->dict, name_plural);
-        // printf("lib: %s\n", name_plural);
-        if (ls && PyList_Check(ls)) {
-          /* loop */
-          const Py_ssize_t size = PyList_GET_SIZE(ls);
-          Py_ssize_t i;
-
-          for (i = 0; i < size; i++) {
-            PyObject *item_src = PyList_GET_ITEM(ls, i);
-            PyObject *item_dst; /* must be set below */
-            const char *item_idname = PyUnicode_AsUTF8(item_src);
-
-            // printf("  %s\n", item_idname);
-
-            if (item_idname) {
-              ID *id = BLO_library_link_named_part(
-                  mainl, &(self->blo_handle), idcode, item_idname, &liblink_params);
-              if (id) {
-
-                if (self->bmain_is_temp) {
-                  /* If this fails, #LibraryLink_Params.id_tag_extra is not being applied. */
-                  BLI_assert(id->tag & LIB_TAG_TEMP_MAIN);
-                }
+  BlendfileLinkAppendContext *lapp_context = BKE_blendfile_link_append_context_new(
+      &liblink_params);
+  BKE_blendfile_link_append_context_library_add(lapp_context, self->abspath, self->blo_handle);
 
-#ifdef USE_RNA_DATABLOCKS
-                /* swap name for pointer to the id */
-                item_dst = PyCapsule_New((void *)id, NULL, NULL);
-#else
-                /* leave as is */
-                continue;
-#endif
-              }
-              else {
-                bpy_lib_exit_warn_idname(self, name_plural, item_idname);
-                /* just warn for now */
-                /* err = -1; */
-                item_dst = Py_INCREF_RET(Py_None);
-              }
-
-              /* ID or None */
-            }
-            else {
-              /* XXX, could complain about this */
-              bpy_lib_exit_warn_type(self, item_src);
-              PyErr_Clear();
-              item_dst = Py_INCREF_RET(Py_None);
-            }
-
-            /* item_dst must be new or already incref'd */
-            Py_DECREF(item_src);
-            PyList_SET_ITEM(ls, i, item_dst);
-          }
-        }
-      }
+  int idcode_step = 0;
+  short idcode;
+  while ((idcode = BKE_idtype_idcode_iter_step(&idcode_step))) {
+    if (!BKE_idtype_idcode_is_linkable(idcode) || (idcode == ID_WS && !do_append)) {
+      continue;
     }
-  }
 
-  if (err == -1) {
-    /* exception raised above, XXX, this leaks some memory */
-    BLO_blendhandle_close(self->blo_handle);
-    self->blo_handle = NULL;
-    BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, false);
-    return NULL;
-  }
+    const char *name_plural = BKE_idtype_idcode_to_name_plural(idcode);
+    PyObject *ls = PyDict_GetItemString(self->dict, name_plural);
+    // printf("lib: %s\n", name_plural);
+    if (ls == NULL || !PyList_Check(ls)) {
+      continue;
+    }
 
-  Library *lib = mainl->curlib; /* newly added lib, assign before append end */
-  BLO_library_link_end(mainl, &(self->blo_handle), &liblink_params);
-  BLO_blendhandle_close(self->blo_handle);
-  self->blo_handle = NULL;
+    const Py_ssize_t size = PyList_GET_SIZE(ls);
+    if (size == 0) {
+      continue;
+    }
 
-  GHash *old_to_new_ids = BLI_ghash_ptr_new(__func__);
+    /* loop */
+    for (Py_ssize_t i = 0; i < size; i++) {
+      PyObject *item_src = PyList_GET_ITEM(ls, i);
+      const char *item_idname = PyUnicode_AsUTF8(item_src);
 
-  /* copied from wm_operator.c */
-  {
-    /* mark all library linked objects to be updated */
-    BKE_main_lib_objects_recalc_all(bmain);
+      // printf("  %s\n", item_idname);
 
-    /* append, rather than linking */
-    if (do_append) {
-      BKE_library_make_local(bmain, lib, old_to_new_ids, true, false);
+      /* NOTE: index of item in py list is stored in userdata pointer, so that it can be found
+       * later on to replace the ID name by the actual ID pointer. */
+      if (item_idname != NULL) {
+        BlendfileLinkAppendContextItem *item = BKE_blendfile_link_append_context_item_add(
+            lapp_context, item_idname, idcode, POINTER_FROM_INT(i));
+        BKE_blendfile_link_append_context_item_library_index_enable(lapp_context, item, 0);
+      }
+      else {
+        /* XXX, could complain about this */
+        bpy_lib_exit_warn_type(self, item_src);
+        PyErr_Clear();
+
+#ifdef USE_RNA_DATABLOCKS
+        /* We can replace the item immediately with `None`. */
+        PyObject *py_item = Py_INCREF_RET(Py_None);
+        PyList_SET_ITEM(ls, i, py_item);
+        Py_DECREF(item_src);
+#endif
+      }
     }
   }
 
-  BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, false);
+  BKE_blendfile_link(lapp_context, NULL);
+  if (do_append) {
+    BKE_blendfile_append(lapp_context, NULL);
+  }
 
-  /* finally swap the capsules for real bpy objects
-   * important since BLO_library_append_end initializes NodeTree types used by srna->refine */
+  /* If enabled, replace named items in given lists by the final matching new ID pointer. */
 #ifdef USE_RNA_DATABLOCKS
-  {
-    int idcode_step = 0, idcode;
-    while ((idcode = BKE_idtype_idcode_iter_step(&idcode_step))) {
-      if (BKE_idtype_idcode_is_linkable(idcode) && (idcode != ID_WS || do_append)) {
-        const char *name_plural = BKE_idtype_idcode_to_name_plural(idcode);
-        PyObject *ls = PyDict_GetItemString(self->dict, name_plural);
-        if (ls && PyList_Check(ls)) {
-          const Py_ssize_t size = PyList_GET_SIZE(ls);
-          Py_ssize_t i;
-          PyObject *item;
-
-          for (i = 0; i < size; i++) {
-            item = PyList_GET_ITEM(ls, i);
-            if (PyCapsule_CheckExact(item)) {
-              PointerRNA id_ptr;
-              ID *id;
-
-              id = PyCapsule_GetPointer(item, NULL);
-              id = BLI_ghash_lookup_default(old_to_new_ids, id, id);
-              Py_DECREF(item);
-
-              RNA_id_pointer_create(id, &id_ptr);
-              item = pyrna_struct_CreatePyObject(&id_ptr);
-              PyList_SET_ITEM(ls, i, item);
-            }
-          }
-        }
-      }
+  idcode_step = 0;
+  while ((idcode = BKE_idtype_idcode_iter_step(&idcode_step))) {
+    if (!BKE_idtype_idcode_is_linkable(idcode) || (idcode == ID_WS && !do_append)) {
+      continue;
+    }
+    const char *name_plural = BKE_idtype_idcode_to_name_plural(idcode);
+    PyObject *ls = PyDict_GetItemString(self->dict, name_plural);
+    // printf("lib: %s\n", name_plural);
+    if (ls == NULL || !PyList_Check(ls)) {
+      continue;
     }
+
+    const Py_ssize_t size = PyList_GET_SIZE(ls);
+    if (size == 0) {
+      continue;
+    }
+
+    /* Loop over linked items in `lapp_context` to find matching python one in the list, and
+     * replace them with proper ID pointer. */
+    struct LibExitLappContextItemsIterData iter_data = {
+        .idcode = idcode, .py_library = self, .py_list = ls, .py_list_size = size};
+    BKE_blendfile_link_append_context_item_foreach(
+        lapp_context,
+        bpy_lib_exit_lapp_context_items_cb,
+        BKE_BLENDFILE_LINK_APPEND_FOREACH_ITEM_FLAG_DO_DIRECT,
+        &iter_data);
   }
-#endif /* USE_RNA_DATABLOCKS */
+#endif  // USE_RNA_DATABLOCKS
+
+  BLO_blendhandle_close(self->blo_handle);
+  self->blo_handle = NULL;
+
+  BKE_blendfile_link_append_context_free(lapp_context);
+  BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, false);
 
-  BLI_ghash_free(old_to_new_ids, NULL, NULL);
   Py_RETURN_NONE;
 }