PyAPI: add bpy.types.BlendFile.temp_data for temporary library loading

This adds support for creating a `BlendFile` (internally called `Main`),
which is limited to a context.

Temporary data can now be created which can then use
`.libraries.load()` the same as with `bpy.data`.

To prevent errors caused by mixing the temporary ID's with data in
`bpy.data` they are tagged as temporary so they can't be assigned
to properties, however they can be passed as arguments to functions.

Reviewed By: mont29, sybren

Ref D10612

11 files changed, 306 insertions, 7 deletions

diff --git a/source/blender/blenkernel/intern/lib_id.c b/source/blender/blenkernel/intern/lib_id.c
index a511c1f9c4c..f3a1c01ad26 100644
--- a/source/blender/blenkernel/intern/lib_id.c
+++ b/source/blender/blenkernel/intern/lib_id.c
@@ -1139,6 +1139,7 @@ static uint global_session_uuid = 0;
 void BKE_lib_libblock_session_uuid_ensure(ID *id)
 {
   if (id->session_uuid == MAIN_ID_SESSION_UUID_UNSET) {
+    BLI_assert((id->tag & LIB_TAG_TEMP_MAIN) == 0); /* Caller must ensure this. */
     id->session_uuid = atomic_add_and_fetch_uint32(&global_session_uuid, 1);
     /* In case overflow happens, still assign a valid ID. This way opening files many times works
      * correctly. */
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index aa3362aa211..de7353d827a 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -2424,7 +2424,9 @@ static void direct_link_id_common(
     id->session_uuid = MAIN_ID_SESSION_UUID_UNSET;
   }
 
-  BKE_lib_libblock_session_uuid_ensure(id);
+  if ((tag & LIB_TAG_TEMP_MAIN) == 0) {
+    BKE_lib_libblock_session_uuid_ensure(id);
+  }
 
   id->lib = current_library;
   id->us = ID_FAKE_USERS(id);
@@ -3168,7 +3170,9 @@ static ID *create_placeholder(Main *mainvar, const short idcode, const char *idn
   BLI_addtail(lb, ph_id);
   id_sort_by_name(lb, ph_id, NULL);
 
-  BKE_lib_libblock_session_uuid_ensure(ph_id);
+  if ((tag & LIB_TAG_TEMP_MAIN) == 0) {
+    BKE_lib_libblock_session_uuid_ensure(ph_id);
+  }
 
   return ph_id;
 }
@@ -4998,6 +5002,11 @@ static Main *library_link_begin(
 {
   Main *mainl;
 
+  /* Only allow specific tags to be set as extra,
+   * otherwise this could conflict with library loading logic.
+   * Other flags can be added here, as long as they are safe. */
+  BLI_assert((id_tag_extra & ~LIB_TAG_TEMP_MAIN) == 0);
+
   (*fd)->id_tag_extra = id_tag_extra;
 
   (*fd)->mainlist = MEM_callocN(sizeof(ListBase), "FileData.mainlist");
diff --git a/source/blender/blenloader/intern/versioning_250.c b/source/blender/blenloader/intern/versioning_250.c
index 983fdce15f1..467fd8b0399 100644
--- a/source/blender/blenloader/intern/versioning_250.c
+++ b/source/blender/blenloader/intern/versioning_250.c
@@ -449,7 +449,9 @@ static void versions_gpencil_add_main(ListBase *lb, ID *id, const char *name)
   BKE_id_new_name_validate(lb, id, name);
   /* alphabetic insertion: is in BKE_id_new_name_validate */
 
-  BKE_lib_libblock_session_uuid_ensure(id);
+  if ((id->tag & LIB_TAG_TEMP_MAIN) == 0) {
+    BKE_lib_libblock_session_uuid_ensure(id);
+  }
 
   if (G.debug & G_DEBUG) {
     printf("Converted GPencil to ID: %s\n", id->name + 2);
diff --git a/source/blender/makesdna/DNA_ID.h b/source/blender/makesdna/DNA_ID.h
index 14f0fef5270..c708219cfe8 100644
--- a/source/blender/makesdna/DNA_ID.h
+++ b/source/blender/makesdna/DNA_ID.h
@@ -566,6 +566,12 @@ enum {
   /* RESET_AFTER_USE Used by undo system to tag unchanged IDs re-used from old Main (instead of
    * read from memfile). */
   LIB_TAG_UNDO_OLD_ID_REUSED = 1 << 19,
+
+  /* This ID is part of a temporary #Main which is expected to be freed in a short time-frame.
+   * Don't allow assigning this to non-temporary members (since it's likely to cause errors).
+   * When set #ID.session_uuid isn't initialized, since the data isn't part of the session. */
+  LIB_TAG_TEMP_MAIN = 1 << 20,
+
 };
 
 /* Tag given ID for an update in all the dependency graphs. */
diff --git a/source/blender/python/intern/CMakeLists.txt b/source/blender/python/intern/CMakeLists.txt
index 5d8330e368d..56ef5c8187a 100644
--- a/source/blender/python/intern/CMakeLists.txt
+++ b/source/blender/python/intern/CMakeLists.txt
@@ -75,6 +75,7 @@ set(SRC
   bpy_rna_anim.c
   bpy_rna_array.c
   bpy_rna_callback.c
+  bpy_rna_data.c
   bpy_rna_driver.c
   bpy_rna_gizmo.c
   bpy_rna_id_collection.c
@@ -113,6 +114,7 @@ set(SRC
   bpy_rna.h
   bpy_rna_anim.h
   bpy_rna_callback.h
+  bpy_rna_data.h
   bpy_rna_driver.h
   bpy_rna_gizmo.h
   bpy_rna_id_collection.h
diff --git a/source/blender/python/intern/bpy.c b/source/blender/python/intern/bpy.c
index 74fc8bcfec9..547cf2ad38f 100644
--- a/source/blender/python/intern/bpy.c
+++ b/source/blender/python/intern/bpy.c
@@ -44,6 +44,7 @@
 #include "bpy_operator.h"
 #include "bpy_props.h"
 #include "bpy_rna.h"
+#include "bpy_rna_data.h"
 #include "bpy_rna_gizmo.h"
 #include "bpy_rna_id_collection.h"
 #include "bpy_rna_types_capi.h"
@@ -425,6 +426,8 @@ void BPy_init_modules(struct bContext *C)
   /* needs to be first so bpy_types can run */
   BPY_library_load_type_ready();
 
+  BPY_rna_data_context_type_ready();
+
   BPY_rna_gizmo_module(mod);
 
   bpy_import_test("bpy_types");
diff --git a/source/blender/python/intern/bpy_library_load.c b/source/blender/python/intern/bpy_library_load.c
index 5d9adb08f3d..1ee14df24cf 100644
--- a/source/blender/python/intern/bpy_library_load.c
+++ b/source/blender/python/intern/bpy_library_load.c
@@ -34,6 +34,7 @@
 #include "BLI_string.h"
 #include "BLI_utildefines.h"
 
+#include "BKE_context.h"
 #include "BKE_idtype.h"
 #include "BKE_lib_id.h"
 #include "BKE_main.h"
@@ -67,8 +68,10 @@ typedef struct {
   BlendHandle *blo_handle;
   int flag;
   PyObject *dict;
-  /* Borrowed reference to the `bmain`, taken from the RNA instance of #RNA_BlendDataLibraries. */
+  /* Borrowed reference to the `bmain`, taken from the RNA instance of #RNA_BlendDataLibraries.
+   * Defaults to #G.main, Otherwise use a temporary #Main when `bmain_is_temp` is true. */
   Main *bmain;
+  bool bmain_is_temp;
 } BPy_Library;
 
 static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *kwds);
@@ -185,6 +188,7 @@ PyDoc_STRVAR(
     "   :type assets_only: bool\n");
 static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *kw)
 {
+  Main *bmain_base = CTX_data_main(BPY_context_get());
   Main *bmain = self->ptr.data; /* Typically #G_MAIN */
   BPy_Library *ret;
   const char *filename = NULL;
@@ -212,6 +216,7 @@ static PyObject *bpy_lib_load(BPy_PropertyRNA *self, PyObject *args, PyObject *k
   BLI_path_abs(ret->abspath, BKE_main_blendfile_path(bmain));
 
   ret->bmain = bmain;
+  ret->bmain_is_temp = (bmain != bmain_base);
 
   ret->blo_handle = NULL;
   ret->flag = ((is_link ? FILE_LINK : 0) | (is_rel ? FILE_RELPATH : 0) |
@@ -344,8 +349,9 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject *UNUSED(args))
   BKE_main_id_tag_all(bmain, LIB_TAG_PRE_EXISTING, true);
 
   /* here appending/linking starts */
+  const int id_tag_extra = self->bmain_is_temp ? LIB_TAG_TEMP_MAIN : 0;
   struct LibraryLink_Params liblink_params;
-  BLO_library_link_params_init(&liblink_params, bmain, self->flag, 0);
+  BLO_library_link_params_init(&liblink_params, bmain, self->flag, id_tag_extra);
 
   mainl = BLO_library_link_begin(&(self->blo_handle), self->relpath, &liblink_params);
 
@@ -372,6 +378,12 @@ static PyObject *bpy_lib_exit(BPy_Library *self, PyObject *UNUSED(args))
               ID *id = BLO_library_link_named_part(
                   mainl, &(self->blo_handle), idcode, item_idname, &liblink_params);
               if (id) {
+
+                if (self->bmain_is_temp) {
+                  /* If this fails, #LibraryLink_Params.id_tag_extra is not being applied. */
+                  BLI_assert(id->tag & LIB_TAG_TEMP_MAIN);
+                }
+
 #ifdef USE_RNA_DATABLOCKS
                 /* swap name for pointer to the id */
                 item_dst = PyCapsule_New((void *)id, NULL, NULL);
diff --git a/source/blender/python/intern/bpy_rna.c b/source/blender/python/intern/bpy_rna.c
index ecaa5791e38..7a43c9cb997 100644
--- a/source/blender/python/intern/bpy_rna.c
+++ b/source/blender/python/intern/bpy_rna.c
@@ -2061,6 +2061,19 @@ static int pyrna_py_to_prop(
               Py_XDECREF(value_new);
               return -1;
             }
+
+            if (value_owner_id->tag & LIB_TAG_TEMP_MAIN) {
+              /* Allow passing temporary ID's to functions, but not attribute assignment. */
+              if (ptr->type != &RNA_Function) {
+                PyErr_Format(PyExc_TypeError,
+                             "%.200s %.200s.%.200s ID type assignment is temporary, can't assign",
+                             error_prefix,
+                             RNA_struct_identifier(ptr->type),
+                             RNA_property_identifier(prop));
+                Py_XDECREF(value_new);
+                return -1;
+              }
+            }
           }
         }
 
diff --git a/source/blender/python/intern/bpy_rna_data.c b/source/blender/python/intern/bpy_rna_data.c
new file mode 100644
index 00000000000..3771cc05490
--- /dev/null
+++ b/source/blender/python/intern/bpy_rna_data.c
@@ -0,0 +1,219 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup pythonintern
+ *
+ * This file defines the API to support temporarily creating #Main data.
+ * The only use case for this is currently to support temporarily loading data-blocks
+ * which can be freed, without them polluting the current #G_MAIN.
+ *
+ * This is exposed via a context manager `bpy.types.BlendData.temp_data(...)`
+ * which returns a new `bpy.types.BlendData` that is freed once the context manager exits.
+ */
+
+#include <Python.h>
+#include <stddef.h>
+
+#include "BLI_string.h"
+#include "BLI_utildefines.h"
+
+#include "BKE_global.h"
+#include "BKE_main.h"
+
+#include "RNA_access.h"
+
+#include "bpy_rna.h"
+#include "bpy_rna_data.h"
+
+typedef struct {
+  PyObject_HEAD /* required python macro */
+      BPy_StructRNA *data_rna;
+  char filepath[1024];
+} BPy_DataContext;
+
+static PyObject *bpy_rna_data_temp_data(PyObject *self, PyObject *args, PyObject *kwds);
+static PyObject *bpy_rna_data_context_enter(BPy_DataContext *self);
+static PyObject *bpy_rna_data_context_exit(BPy_DataContext *self, PyObject *args);
+
+static PyMethodDef bpy_rna_data_context_methods[] = {
+    {"__enter__", (PyCFunction)bpy_rna_data_context_enter, METH_NOARGS},
+    {"__exit__", (PyCFunction)bpy_rna_data_context_exit, METH_VARARGS},
+    {NULL} /* sentinel */
+};
+
+static int bpy_rna_data_context_traverse(BPy_DataContext *self, visitproc visit, void *arg)
+{
+  Py_VISIT(self->data_rna);
+  return 0;
+}
+
+static int bpy_rna_data_context_clear(BPy_DataContext *self)
+{
+  Py_CLEAR(self->data_rna);
+  return 0;
+}
+
+static void bpy_rna_data_context_dealloc(BPy_DataContext *self)
+{
+  PyObject_GC_UnTrack(self);
+  Py_CLEAR(self->data_rna);
+  PyObject_GC_Del(self);
+}
+
+static PyTypeObject bpy_rna_data_context_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0) "bpy_rna_data_context", /* tp_name */
+    sizeof(BPy_DataContext),                               /* tp_basicsize */
+    0,                                                     /* tp_itemsize */
+    /* methods */
+    (destructor)bpy_rna_data_context_dealloc, /* tp_dealloc */
+    0,                                        /* tp_vectorcall_offset */
+    NULL,                                     /* getattrfunc tp_getattr; */
+    NULL,                                     /* setattrfunc tp_setattr; */
+    NULL,
+    /* tp_compare */ /* DEPRECATED in python 3.0! */
+    NULL,            /* tp_repr */
+
+    /* Method suites for standard classes */
+
+    NULL, /* PyNumberMethods *tp_as_number; */
+    NULL, /* PySequenceMethods *tp_as_sequence; */
+    NULL, /* PyMappingMethods *tp_as_mapping; */
+
+    /* More standard operations (here for binary compatibility) */
+
+    NULL, /* hashfunc tp_hash; */
+    NULL, /* ternaryfunc tp_call; */
+    NULL, /* reprfunc tp_str; */
+
+    /* will only use these if this is a subtype of a py class */
+    NULL /*PyObject_GenericGetAttr is assigned later */, /* getattrofunc tp_getattro; */
+    NULL,                                                /* setattrofunc tp_setattro; */
+
+    /* Functions to access object as input/output buffer */
+    NULL, /* PyBufferProcs *tp_as_buffer; */
+
+    /*** Flags to define presence of optional/expanded features ***/
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* long tp_flags; */
+
+    NULL, /*  char *tp_doc;  Documentation string */
+    /*** Assigned meaning in release 2.0 ***/
+    /* call function for all accessible objects */
+    (traverseproc)bpy_rna_data_context_traverse, /* traverseproc tp_traverse; */
+
+    /* delete references to contained objects */
+    (inquiry)bpy_rna_data_context_clear, /* inquiry tp_clear; */
+
+    /***  Assigned meaning in release 2.1 ***/
+    /*** rich comparisons (subclassed) ***/
+    NULL, /* richcmpfunc tp_richcompare; */
+
+    /***  weak reference enabler ***/
+    0,
+    /*** Added in release 2.2 ***/
+    /*   Iterators */
+    NULL, /* getiterfunc tp_iter; */
+    NULL, /* iternextfunc tp_iternext; */
+
+    /*** Attribute descriptor and subclassing stuff ***/
+    bpy_rna_data_context_methods, /* struct PyMethodDef *tp_methods; */
+    NULL,                         /* struct PyMemberDef *tp_members; */
+    NULL,                         /* struct PyGetSetDef *tp_getset; */
+    NULL,                         /* struct _typeobject *tp_base; */
+    NULL,                         /* PyObject *tp_dict; */
+    NULL,                         /* descrgetfunc tp_descr_get; */
+    NULL,                         /* descrsetfunc tp_descr_set; */
+    0,                            /* long tp_dictoffset; */
+    NULL,                         /* initproc tp_init; */
+    NULL,                         /* allocfunc tp_alloc; */
+    NULL,                         /* newfunc tp_new; */
+    /*  Low-level free-memory routine */
+    NULL, /* freefunc tp_free;  */
+    /* For PyObject_IS_GC */
+    NULL, /* inquiry tp_is_gc;  */
+    NULL, /* PyObject *tp_bases; */
+    /* method resolution order */
+    NULL, /* PyObject *tp_mro;  */
+    NULL, /* PyObject *tp_cache; */
+    NULL, /* PyObject *tp_subclasses; */
+    NULL, /* PyObject *tp_weaklist; */
+    NULL,
+};
+
+PyDoc_STRVAR(bpy_rna_data_context_load_doc,
+             ".. method:: temp_data(filepath=None)\n"
+             "\n"
+             "   A context manager that temporarily creates blender file data.\n"
+             "\n"
+             "   :arg filepath: The file path for the newly temporary data. "
+             "When None, the path of the currently open file is used.\n"
+             "   :type filepath: str or NoneType\n"
+             "\n"
+             "   :return: Blend file data which is freed once the context exists.\n"
+             "   :rtype: :class:`bpy.types.BlendData`\n");
+
+static PyObject *bpy_rna_data_temp_data(PyObject *UNUSED(self), PyObject *args, PyObject *kw)
+{
+  BPy_DataContext *ret;
+  const char *filepath = NULL;
+  static const char *_keywords[] = {"filepath", NULL};
+  static _PyArg_Parser _parser = {"|$z:temp_data", _keywords, 0};
+  if (!_PyArg_ParseTupleAndKeywordsFast(args, kw, &_parser, &filepath)) {
+    return NULL;
+  }
+
+  ret = PyObject_GC_New(BPy_DataContext, &bpy_rna_data_context_Type);
+
+  STRNCPY(ret->filepath, filepath ? filepath : G_MAIN->name);
+
+  return (PyObject *)ret;
+}
+
+static PyObject *bpy_rna_data_context_enter(BPy_DataContext *self)
+{
+  Main *bmain_temp = BKE_main_new();
+  PointerRNA ptr;
+  RNA_pointer_create(NULL, &RNA_BlendData, bmain_temp, &ptr);
+
+  self->data_rna = (BPy_StructRNA *)pyrna_struct_CreatePyObject(&ptr);
+
+  PyObject_GC_Track(self);
+
+  return (PyObject *)self->data_rna;
+}
+
+static PyObject *bpy_rna_data_context_exit(BPy_DataContext *self, PyObject *UNUSED(args))
+{
+  BKE_main_free(self->data_rna->ptr.data);
+  RNA_POINTER_INVALIDATE(&self->data_rna->ptr);
+  Py_RETURN_NONE;
+}
+
+PyMethodDef BPY_rna_data_context_method_def = {
+    "temp_data",
+    (PyCFunction)bpy_rna_data_temp_data,
+    METH_STATIC | METH_VARARGS | METH_KEYWORDS,
+    bpy_rna_data_context_load_doc,
+};
+
+int BPY_rna_data_context_type_ready(void)
+{
+  if (PyType_Ready(&bpy_rna_data_context_Type) < 0) {
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/source/blender/python/intern/bpy_rna_data.h b/source/blender/python/intern/bpy_rna_data.h
new file mode 100644
index 00000000000..b1d226d9dc4
--- /dev/null
+++ b/source/blender/python/intern/bpy_rna_data.h
@@ -0,0 +1,29 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup pythonintern
+ */
+
+#pragma once
+
+int BPY_rna_data_context_type_ready(void);
+
+extern PyMethodDef BPY_rna_data_context_method_def;
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/source/blender/python/intern/bpy_rna_types_capi.c b/source/blender/python/intern/bpy_rna_types_capi.c
index 042f7b6fd67..9b15e84663d 100644
--- a/source/blender/python/intern/bpy_rna_types_capi.c
+++ b/source/blender/python/intern/bpy_rna_types_capi.c
@@ -36,6 +36,7 @@
 #include "bpy_library.h"
 #include "bpy_rna.h"
 #include "bpy_rna_callback.h"
+#include "bpy_rna_data.h"
 #include "bpy_rna_id_collection.h"
 #include "bpy_rna_types_capi.h"
 #include "bpy_rna_ui.h"
@@ -56,6 +57,7 @@ static struct PyMethodDef pyrna_blenddata_methods[] = {
     {NULL, NULL, 0, NULL}, /* #BPY_rna_id_collection_user_map_method_def */
     {NULL, NULL, 0, NULL}, /* #BPY_rna_id_collection_batch_remove_method_def */
     {NULL, NULL, 0, NULL}, /* #BPY_rna_id_collection_orphans_purge_method_def */
+    {NULL, NULL, 0, NULL}, /* #BPY_rna_data_context_method_def */
     {NULL, NULL, 0, NULL},
 };
 
@@ -207,8 +209,9 @@ void BPY_rna_types_extend_capi(void)
   ARRAY_SET_ITEMS(pyrna_blenddata_methods,
                   BPY_rna_id_collection_user_map_method_def,
                   BPY_rna_id_collection_batch_remove_method_def,
-                  BPY_rna_id_collection_orphans_purge_method_def);
-  BLI_assert(ARRAY_SIZE(pyrna_blenddata_methods) == 4);
+                  BPY_rna_id_collection_orphans_purge_method_def,
+                  BPY_rna_data_context_method_def);
+  BLI_assert(ARRAY_SIZE(pyrna_blenddata_methods) == 5);
   pyrna_struct_type_extend_capi(&RNA_BlendData, pyrna_blenddata_methods, NULL);
 
   /* BlendDataLibraries */