LibOverride: move most of 'complete override creation' from ED_object to BKE_lib_override.

This code is fairly complex and can be used in more places, better not
duplicate that logic and just have it in BKE area.

3 files changed, 221 insertions, 167 deletions

diff --git a/source/blender/blenkernel/BKE_lib_override.h b/source/blender/blenkernel/BKE_lib_override.h
index 8f6603e28ae..ba8f2eb3399 100644
--- a/source/blender/blenkernel/BKE_lib_override.h
+++ b/source/blender/blenkernel/BKE_lib_override.h
@@ -50,6 +50,8 @@ struct IDOverrideLibraryPropertyOperation;
 struct Main;
 struct PointerRNA;
 struct PropertyRNA;
+struct Scene;
+struct ViewLayer;
 
 struct IDOverrideLibrary *BKE_lib_override_library_init(struct ID *local_id,
                                                         struct ID *reference_id);
@@ -67,6 +69,11 @@ void BKE_lib_override_library_dependencies_tag(struct Main *bmain,
                                                struct ID *id_root,
                                                const uint tag,
                                                const bool do_create_main_relashionships);
+bool BKE_lib_override_library_create(struct Main *bmain,
+                                     struct Scene *scene,
+                                     struct ViewLayer *view_layer,
+                                     struct ID *id_root,
+                                     struct ID *id_reference);
 
 struct IDOverrideLibraryProperty *BKE_lib_override_library_property_find(
     struct IDOverrideLibrary *override, const char *rna_path);
diff --git a/source/blender/blenkernel/intern/lib_override.c b/source/blender/blenkernel/intern/lib_override.c
index 58856729f24..02f5ac66c64 100644
--- a/source/blender/blenkernel/intern/lib_override.c
+++ b/source/blender/blenkernel/intern/lib_override.c
@@ -27,20 +27,25 @@
 #include "MEM_guardedalloc.h"
 
 #include "DNA_ID.h"
+#include "DNA_collection_types.h"
 #include "DNA_key_types.h"
 #include "DNA_object_types.h"
+#include "DNA_scene_types.h"
 
 #include "DEG_depsgraph.h"
 #include "DEG_depsgraph_build.h"
 
 #include "BKE_armature.h"
+#include "BKE_collection.h"
 #include "BKE_idtype.h"
 #include "BKE_key.h"
+#include "BKE_layer.h"
 #include "BKE_lib_id.h"
 #include "BKE_lib_override.h"
 #include "BKE_lib_query.h"
 #include "BKE_lib_remap.h"
 #include "BKE_main.h"
+#include "BKE_scene.h"
 
 #include "BLI_ghash.h"
 #include "BLI_listbase.h"
@@ -355,7 +360,7 @@ bool BKE_lib_override_library_create_from_tag(Main *bmain)
   return success;
 }
 
-static bool lib_override_hierarchy_recursive_tag(Main *bmain, ID *id)
+static bool lib_override_hierarchy_recursive_tag(Main *bmain, ID *id, const uint tag)
 {
   MainIDRelationsEntry *entry = BLI_ghash_lookup(bmain->relations->id_user_to_used, id);
 
@@ -368,13 +373,13 @@ static bool lib_override_hierarchy_recursive_tag(Main *bmain, ID *id)
   for (; entry != NULL; entry = entry->next) {
     /* We only consider IDs from the same library. */
     if (entry->id_pointer != NULL && (*entry->id_pointer)->lib == id->lib) {
-      if (lib_override_hierarchy_recursive_tag(bmain, *entry->id_pointer)) {
-        id->tag |= LIB_TAG_DOIT;
+      if (lib_override_hierarchy_recursive_tag(bmain, *entry->id_pointer, tag)) {
+        id->tag |= tag;
       }
     }
   }
 
-  return (id->tag & LIB_TAG_DOIT) != 0;
+  return (id->tag & tag) != 0;
 }
 
 /**
@@ -391,8 +396,6 @@ void BKE_lib_override_library_dependencies_tag(struct Main *bmain,
                                                const uint tag,
                                                const bool do_create_main_relashionships)
 {
-  id_root->tag |= tag;
-
   if (do_create_main_relashionships) {
     BKE_main_relations_create(bmain, 0);
   }
@@ -400,11 +403,207 @@ void BKE_lib_override_library_dependencies_tag(struct Main *bmain,
   /* Then we tag all intermediary data-blocks in-between two overridden ones (e.g. if a shapekey
    * has a driver using an armature object's bone, we need to override the shapekey/obdata, the
    * objects using them, etc.) */
-  lib_override_hierarchy_recursive_tag(bmain, id_root);
+  lib_override_hierarchy_recursive_tag(bmain, id_root, tag);
 
   BKE_main_relations_free(bmain);
 }
 
+static int lib_override_library_make_tag_ids_cb(LibraryIDLinkCallbackData *cb_data)
+{
+  if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_LOOPBACK)) {
+    return IDWALK_RET_STOP_RECURSION;
+  }
+
+  ID *id_root = cb_data->user_data;
+  Library *library_root = id_root->lib;
+  ID *id = *cb_data->id_pointer;
+  ID *id_owner = cb_data->id_owner;
+
+  BLI_assert(id_owner == cb_data->id_self);
+
+  if (ELEM(id, NULL, id_owner)) {
+    return IDWALK_RET_NOP;
+  }
+
+  BLI_assert(id->lib != NULL);
+  BLI_assert(id_owner->lib == library_root);
+
+  if (id->tag & LIB_TAG_DOIT) {
+    /* Already processed, but maybe not with the same chain of dependency, so we need to check that
+     * one nonetheless. */
+    return IDWALK_RET_STOP_RECURSION;
+  }
+
+  if (id->lib != library_root) {
+    /* We do not override data-blocks from other libraries, nor do we process them. */
+    return IDWALK_RET_STOP_RECURSION;
+  }
+
+  /* We tag all collections and objects for override. And we also tag all other data-blocks which
+   * would user one of those. */
+  if (ELEM(GS(id->name), ID_OB, ID_GR)) {
+    id->tag |= LIB_TAG_DOIT;
+  }
+
+  return IDWALK_RET_NOP;
+}
+
+/**
+ * Advanced 'smart' function to create fully functionnal overrides.
+ *
+ * \note Currently it only does special things if given \a id_root is an object of collection, more
+ * specific behaviors may be added in the future for other ID types.
+ *
+ * \note It will overrides all IDs tagged with \a LIB_TAG_DOIT, and it does not clear that tag at
+ * its beginning, so caller code can add extra data-blocks to be overridden as well.
+ *
+ * \note In the future that same function may be extended to support 'refresh' of overrides
+ * (rebuilding overrides from linked data, trying to preserve local overrides already defined).
+ *
+ * \param id_root The root ID to create an override from.
+ * \param id_reference some reference ID used to do some post-processing after overrides have been
+ *                     created, may be NULL. Typically, the Empty object instantiating the linked
+ * collection we override, currently. \return true if override was successfully created.
+ */
+bool BKE_lib_override_library_create(
+    Main *bmain, Scene *scene, ViewLayer *view_layer, ID *id_root, ID *id_reference)
+{
+  /* Tag all collections and objects, as well as other IDs using them. */
+  id_root->tag |= LIB_TAG_DOIT;
+
+  BKE_main_relations_create(bmain, 0);
+
+  if (ELEM(GS(id_root->name), ID_OB, ID_GR)) {
+    BKE_library_foreach_ID_link(bmain,
+                                id_root,
+                                lib_override_library_make_tag_ids_cb,
+                                id_root,
+                                IDWALK_READONLY | IDWALK_RECURSE);
+
+    /* Then, we remove (untag) bone shape objects, you shall never want to override those
+     * (hopefully)... */
+    LISTBASE_FOREACH (Object *, ob, &bmain->objects) {
+      if (ob->type == OB_ARMATURE && ob->pose != NULL && (ob->id.tag & LIB_TAG_DOIT)) {
+        for (bPoseChannel *pchan = ob->pose->chanbase.first; pchan != NULL; pchan = pchan->next) {
+          if (pchan->custom != NULL) {
+            pchan->custom->id.tag &= ~LIB_TAG_DOIT;
+          }
+        }
+      }
+    }
+  }
+
+  /* Note that this call will also free the main relations data we created above. */
+  BKE_lib_override_library_dependencies_tag(bmain, id_root, LIB_TAG_DOIT, false);
+
+  const bool success = BKE_lib_override_library_create_from_tag(bmain);
+
+  if (success) {
+    BKE_main_collection_sync(bmain);
+
+    switch (GS(id_root->name)) {
+      case ID_GR: {
+        Object *ob_reference = id_reference != NULL && GS(id_reference->name) == ID_OB ?
+                                   (Object *)id_reference :
+                                   NULL;
+        Collection *collection_new = ((Collection *)id_root->newid);
+        if (ob_reference != NULL) {
+          BKE_collection_add_from_object(bmain, scene, ob_reference, collection_new);
+        }
+        else {
+          BKE_collection_add_from_collection(
+              bmain, scene, ((Collection *)id_root), collection_new);
+        }
+
+        FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN (collection_new, ob_new) {
+          if (ob_new != NULL && ob_new->id.override_library != NULL) {
+            if (ob_reference != NULL) {
+              Base *base;
+              if ((base = BKE_view_layer_base_find(view_layer, ob_new)) == NULL) {
+                BKE_collection_object_add_from(bmain, scene, ob_reference, ob_new);
+                base = BKE_view_layer_base_find(view_layer, ob_new);
+                DEG_id_tag_update_ex(
+                    bmain, &ob_new->id, ID_RECALC_TRANSFORM | ID_RECALC_BASE_FLAGS);
+              }
+
+              if (ob_new == (Object *)ob_reference->id.newid) {
+                /* TODO: is setting active needed? */
+                BKE_view_layer_base_select_and_set_active(view_layer, base);
+              }
+            }
+            else if (BKE_view_layer_base_find(view_layer, ob_new) == NULL) {
+              BKE_collection_object_add(bmain, collection_new, ob_new);
+              DEG_id_tag_update_ex(bmain, &ob_new->id, ID_RECALC_TRANSFORM | ID_RECALC_BASE_FLAGS);
+            }
+          }
+        }
+        FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
+        break;
+      }
+      case ID_OB: {
+        BKE_collection_object_add_from(
+            bmain, scene, (Object *)id_root, ((Object *)id_root->newid));
+        break;
+      }
+      default:
+        break;
+    }
+
+    /* We need to ensure all new overrides of objects are properly instantiated. */
+    LISTBASE_FOREACH (Object *, ob, &bmain->objects) {
+      Object *ob_new = (Object *)ob->id.newid;
+      if (ob_new != NULL) {
+        BLI_assert(ob_new->id.override_library != NULL &&
+                   ob_new->id.override_library->reference == &ob->id);
+
+        Collection *default_instantiating_collection = NULL;
+        if (BKE_view_layer_base_find(view_layer, ob_new) == NULL) {
+          if (default_instantiating_collection == NULL) {
+            switch (GS(id_root->name)) {
+              case ID_GR: {
+                default_instantiating_collection = BKE_collection_add(
+                    bmain, (Collection *)id_root, "OVERRIDE_HIDDEN");
+                break;
+              }
+              case ID_OB: {
+                /* Add the new container collection to one of the collections instantiating the
+                 * root object, or scene's master collection if none found. */
+                Object *ob_root = (Object *)id_root;
+                LISTBASE_FOREACH (Collection *, collection, &bmain->collections) {
+                  if (BKE_collection_has_object(collection, ob_root) &&
+                      BKE_view_layer_has_collection(view_layer, collection)) {
+                    default_instantiating_collection = BKE_collection_add(
+                        bmain, collection, "OVERRIDE_HIDDEN");
+                  }
+                }
+                if (default_instantiating_collection == NULL) {
+                  default_instantiating_collection = BKE_collection_add(
+                      bmain, scene->master_collection, "OVERRIDE_HIDDEN");
+                }
+                break;
+              }
+              default:
+                BLI_assert(0);
+            }
+            /* Hide the collection from viewport and render. */
+            default_instantiating_collection->flag |= COLLECTION_RESTRICT_VIEWPORT |
+                                                      COLLECTION_RESTRICT_RENDER;
+          }
+
+          BKE_collection_object_add(bmain, default_instantiating_collection, ob_new);
+          DEG_id_tag_update_ex(bmain, &ob_new->id, ID_RECALC_TRANSFORM | ID_RECALC_BASE_FLAGS);
+        }
+      }
+    }
+  }
+
+  /* Cleanup. */
+  BKE_main_id_clear_newpoins(bmain);
+  BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
+
+  return success;
+}
+
 /* We only build override GHash on request. */
 BLI_INLINE IDOverrideLibraryRuntime *override_library_rna_path_mapping_ensure(
     IDOverrideLibrary *override)
diff --git a/source/blender/editors/object/object_relations.c b/source/blender/editors/object/object_relations.c
index 94f85641a73..d8c8bdc3968 100644
--- a/source/blender/editors/object/object_relations.c
+++ b/source/blender/editors/object/object_relations.c
@@ -2242,46 +2242,6 @@ void OBJECT_OT_make_local(wmOperatorType *ot)
 /** \name Make Library Override Operator
  * \{ */
 
-static int make_override_tag_ids_cb(LibraryIDLinkCallbackData *cb_data)
-{
-  if (cb_data->cb_flag & (IDWALK_CB_EMBEDDED | IDWALK_CB_LOOPBACK)) {
-    return IDWALK_RET_STOP_RECURSION;
-  }
-
-  ID *id_root = cb_data->user_data;
-  Library *library_root = id_root->lib;
-  ID *id = *cb_data->id_pointer;
-  ID *id_owner = cb_data->id_owner;
-
-  BLI_assert(id_owner == cb_data->id_self);
-
-  if (ELEM(id, NULL, id_owner)) {
-    return IDWALK_RET_NOP;
-  }
-
-  BLI_assert(id->lib != NULL);
-  BLI_assert(id_owner->lib == library_root);
-
-  if (id->tag & LIB_TAG_DOIT) {
-    /* Already processed, but maybe not with the same chain of dependency, so we need to check that
-     * one nonetheless. */
-    return IDWALK_RET_STOP_RECURSION;
-  }
-
-  if (id->lib != library_root) {
-    /* We do not override data-blocks from other libraries, nor do we process them. */
-    return IDWALK_RET_STOP_RECURSION;
-  }
-
-  /* We tag all collections and objects for override. And we also tag all other data-blocks which
-   * would user one of those. */
-  if (ELEM(GS(id->name), ID_OB, ID_GR)) {
-    id->tag |= LIB_TAG_DOIT;
-  }
-
-  return IDWALK_RET_NOP;
-}
-
 /* Set the object to override. */
 static int make_override_library_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
@@ -2326,11 +2286,11 @@ static int make_override_library_invoke(bContext *C, wmOperator *op, const wmEve
 static int make_override_library_exec(bContext *C, wmOperator *op)
 {
   Main *bmain = CTX_data_main(C);
+  Scene *scene = CTX_data_scene(C);
+  ViewLayer *view_layer = CTX_data_view_layer(C);
   Object *obact = CTX_data_active_object(C);
   ID *id_root = NULL;
 
-  bool success = false;
-
   if (!ID_IS_LINKED(obact) && obact->instance_collection != NULL &&
       ID_IS_LINKED(obact->instance_collection)) {
     if (!ID_IS_OVERRIDABLE_LIBRARY(obact->instance_collection)) {
@@ -2374,127 +2334,15 @@ static int make_override_library_exec(bContext *C, wmOperator *op)
 
   BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
 
-  /* Tag all collections and objects, as well as other IDs using them. */
-  id_root->tag |= LIB_TAG_DOIT;
-
-  BKE_main_relations_create(bmain, 0);
-
-  BKE_library_foreach_ID_link(
-      bmain, id_root, make_override_tag_ids_cb, id_root, IDWALK_READONLY | IDWALK_RECURSE);
+  const bool success = BKE_lib_override_library_create(
+      bmain, scene, view_layer, id_root, &obact->id);
 
-  /* Then, we remove (untag) bone shape objects, you shall never want to override those
-   * (hopefully)... */
-  LISTBASE_FOREACH (Object *, ob, &bmain->objects) {
-    if (ob->type == OB_ARMATURE && ob->pose != NULL && (ob->id.tag & LIB_TAG_DOIT)) {
-      for (bPoseChannel *pchan = ob->pose->chanbase.first; pchan != NULL; pchan = pchan->next) {
-        if (pchan->custom != NULL) {
-          pchan->custom->id.tag &= ~LIB_TAG_DOIT;
-        }
-      }
-    }
+  /* Remove the instance empty from this scene, the items now have an overridden collection
+   * instead. */
+  if (success && id_root != &obact->id) {
+    ED_object_base_free_and_unlink(bmain, scene, obact);
   }
 
-  /* Note that this call will also free the main relations data we created above. */
-  BKE_lib_override_library_dependencies_tag(bmain, id_root, LIB_TAG_DOIT, false);
-
-  success = BKE_lib_override_library_create_from_tag(bmain);
-
-  if (success) {
-    Scene *scene = CTX_data_scene(C);
-    ViewLayer *view_layer = CTX_data_view_layer(C);
-
-    BKE_main_collection_sync(bmain);
-
-    switch (GS(id_root->name)) {
-      case ID_GR: {
-        Collection *collection_new = ((Collection *)id_root->newid);
-        BKE_collection_add_from_object(bmain, scene, obact, collection_new);
-
-        FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN (collection_new, ob_new) {
-          if (ob_new != NULL && ob_new->id.override_library != NULL) {
-            Base *base;
-            if ((base = BKE_view_layer_base_find(view_layer, ob_new)) == NULL) {
-              BKE_collection_object_add_from(bmain, scene, obact, ob_new);
-              base = BKE_view_layer_base_find(view_layer, ob_new);
-              DEG_id_tag_update_ex(bmain, &ob_new->id, ID_RECALC_TRANSFORM | ID_RECALC_BASE_FLAGS);
-            }
-
-            if (ob_new == (Object *)obact->id.newid) {
-              /* TODO: is setting active needed? */
-              BKE_view_layer_base_select_and_set_active(view_layer, base);
-            }
-          }
-        }
-        FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
-        break;
-      }
-      case ID_OB: {
-        BKE_collection_object_add_from(bmain, scene, obact, ((Object *)id_root->newid));
-        break;
-      }
-      default:
-        BLI_assert(0);
-    }
-
-    /* We need to ensure all new overrides of objects are properly instantiated. */
-    LISTBASE_FOREACH (Object *, ob, &bmain->objects) {
-      Object *ob_new = (Object *)ob->id.newid;
-      if (ob_new != NULL) {
-        BLI_assert(ob_new->id.override_library != NULL &&
-                   ob_new->id.override_library->reference == &ob->id);
-
-        Collection *default_instantiating_collection = NULL;
-        Base *base;
-        if ((base = BKE_view_layer_base_find(view_layer, ob_new)) == NULL) {
-          if (default_instantiating_collection == NULL) {
-            switch (GS(id_root->name)) {
-              case ID_GR: {
-                default_instantiating_collection = BKE_collection_add(
-                    bmain, (Collection *)id_root, "OVERRIDE_HIDDEN");
-                break;
-              }
-              case ID_OB: {
-                /* Add the new container collection to one of the collections instantiating the
-                 * root object, or scene's master collection if none found. */
-                Object *ob_root = (Object *)id_root;
-                LISTBASE_FOREACH (Collection *, collection, &bmain->collections) {
-                  if (BKE_collection_has_object(collection, ob_root) &&
-                      BKE_view_layer_has_collection(view_layer, collection)) {
-                    default_instantiating_collection = BKE_collection_add(
-                        bmain, collection, "OVERRIDE_HIDDEN");
-                  }
-                }
-                if (default_instantiating_collection == NULL) {
-                  default_instantiating_collection = BKE_collection_add(
-                      bmain, scene->master_collection, "OVERRIDE_HIDDEN");
-                }
-                break;
-              }
-              default:
-                BLI_assert(0);
-            }
-            /* Hide the collection from viewport and render. */
-            default_instantiating_collection->flag |= COLLECTION_RESTRICT_VIEWPORT |
-                                                      COLLECTION_RESTRICT_RENDER;
-          }
-
-          BKE_collection_object_add(bmain, default_instantiating_collection, ob_new);
-          DEG_id_tag_update_ex(bmain, &ob_new->id, ID_RECALC_TRANSFORM | ID_RECALC_BASE_FLAGS);
-        }
-      }
-    }
-
-    /* Remove the instance empty from this scene, the items now have an overridden collection
-     * instead. */
-    if (id_root != &obact->id) {
-      ED_object_base_free_and_unlink(bmain, scene, obact);
-    }
-  }
-
-  /* Cleanup. */
-  BKE_main_id_clear_newpoins(bmain);
-  BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
-
   DEG_id_tag_update(&CTX_data_scene(C)->id, ID_RECALC_BASE_FLAGS | ID_RECALC_COPY_ON_WRITE);
   WM_event_add_notifier(C, NC_WINDOW, NULL);