Fix T68971: Copy As New Driver from Material node creates a bad reference.

NodeTree structures of materials and some other data blocks are
effectively node group datablock objects that are contained inside
the parent block. Thus, direct references to them are only valid
while blender is running, and are lost on save.

Fix Copy As New Driver to create a reference that goes through
the owner datablock, by adding a new ID flag to mark private
pseudo-datablocks.

Also fix functions that return full paths to structures and
properties, e.g. used in python tooltips. Functions for paths
from ID to struct or property can't be changed because of
Animation Data related code.

Reviewers: mont29

Differential Revision: https://developer.blender.org/D5559

13 files changed, 189 insertions, 36 deletions

diff --git a/source/blender/blenkernel/BKE_blender_version.h b/source/blender/blenkernel/BKE_blender_version.h
index 09900651dad..e0f89024420 100644
--- a/source/blender/blenkernel/BKE_blender_version.h
+++ b/source/blender/blenkernel/BKE_blender_version.h
@@ -27,7 +27,7 @@
  * \note Use #STRINGIFY() rather than defining with quotes.
  */
 #define BLENDER_VERSION 281
-#define BLENDER_SUBVERSION 3
+#define BLENDER_SUBVERSION 4
 /** Several breakages with 280, e.g. collections vs layers. */
 #define BLENDER_MINVERSION 280
 #define BLENDER_MINSUBVERSION 0
diff --git a/source/blender/blenkernel/BKE_node.h b/source/blender/blenkernel/BKE_node.h
index efc4e4fdc12..cbfc8ffc7d0 100644
--- a/source/blender/blenkernel/BKE_node.h
+++ b/source/blender/blenkernel/BKE_node.h
@@ -384,6 +384,7 @@ void ntreeUserIncrefID(struct bNodeTree *ntree);
 void ntreeUserDecrefID(struct bNodeTree *ntree);
 
 struct bNodeTree *ntreeFromID(const struct ID *id);
+struct ID *BKE_node_tree_find_owner_ID(struct Main *bmain, struct bNodeTree *ntree);
 
 void ntreeMakeLocal(struct Main *bmain,
                     struct bNodeTree *ntree,
diff --git a/source/blender/blenkernel/intern/library.c b/source/blender/blenkernel/intern/library.c
index 2dbca3b4db1..7c9f5c0ae77 100644
--- a/source/blender/blenkernel/intern/library.c
+++ b/source/blender/blenkernel/intern/library.c
@@ -1417,6 +1417,9 @@ void BKE_libblock_copy_ex(Main *bmain, const ID *id, ID **r_newid, const int fla
   const bool use_nodetree_alloc_exception = ((GS(id->name) == ID_NT) && (bmain != NULL) &&
                                              (BLI_findindex(&bmain->nodetrees, id) < 0));
 
+  /* The id->flag bits to copy over. */
+  const int copy_flag_mask = LIB_PRIVATE_DATA;
+
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) != 0 || bmain != NULL);
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) != 0 || (flag & LIB_ID_CREATE_NO_ALLOCATE) == 0);
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) == 0 || (flag & LIB_ID_CREATE_NO_USER_REFCOUNT) != 0);
@@ -1448,6 +1451,8 @@ void BKE_libblock_copy_ex(Main *bmain, const ID *id, ID **r_newid, const int fla
     memcpy(cpn + id_offset, cp + id_offset, id_len - id_offset);
   }
 
+  new_id->flag = (new_id->flag & ~copy_flag_mask) | (id->flag & copy_flag_mask);
+
   if (id->properties) {
     new_id->properties = IDP_CopyProperty_ex(id->properties, flag);
   }
diff --git a/source/blender/blenkernel/intern/node.c b/source/blender/blenkernel/intern/node.c
index 206c59c110a..2eba71fa6bd 100644
--- a/source/blender/blenkernel/intern/node.c
+++ b/source/blender/blenkernel/intern/node.c
@@ -1406,6 +1406,7 @@ bNodeTree *ntreeAddTree(Main *bmain, const char *name, const char *idname)
   }
   else {
     ntree = MEM_callocN(sizeof(bNodeTree), "new node tree");
+    ntree->id.flag |= LIB_PRIVATE_DATA;
     *((short *)ntree->id.name) = ID_NT;
     BLI_strncpy(ntree->id.name + 2, name, sizeof(ntree->id.name));
   }
@@ -2172,6 +2173,7 @@ void ntreeSetOutput(bNodeTree *ntree)
    * might be different for editor or for "real" use... */
 }
 
+/* Returns the private NodeTree object of the datablock, if it has one. */
 bNodeTree *ntreeFromID(const ID *id)
 {
   switch (GS(id->name)) {
@@ -2192,6 +2194,28 @@ bNodeTree *ntreeFromID(const ID *id)
   }
 }
 
+/* Finds and returns the datablock that privately owns the given tree, or NULL. */
+ID *BKE_node_tree_find_owner_ID(Main *bmain, struct bNodeTree *ntree)
+{
+  ListBase *lists[] = {&bmain->materials,
+                       &bmain->lights,
+                       &bmain->worlds,
+                       &bmain->textures,
+                       &bmain->scenes,
+                       &bmain->linestyles,
+                       NULL};
+
+  for (int i = 0; lists[i] != NULL; i++) {
+    LISTBASE_FOREACH (ID *, id, lists[i]) {
+      if (ntreeFromID(id) == ntree) {
+        return id;
+      }
+    }
+  }
+
+  return NULL;
+}
+
 void ntreeMakeLocal(Main *bmain, bNodeTree *ntree, bool id_in_mainlist, const bool lib_local)
 {
   BKE_id_make_local_generic(bmain, &ntree->id, id_in_mainlist, lib_local);
diff --git a/source/blender/blenloader/intern/versioning_280.c b/source/blender/blenloader/intern/versioning_280.c
index d2017864192..a66b9336632 100644
--- a/source/blender/blenloader/intern/versioning_280.c
+++ b/source/blender/blenloader/intern/versioning_280.c
@@ -3693,6 +3693,17 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain)
     }
   }
 
+  if (!MAIN_VERSION_ATLEAST(bmain, 281, 4)) {
+    ID *id;
+    FOREACH_MAIN_ID_BEGIN (bmain, id) {
+      bNodeTree *ntree = ntreeFromID(id);
+      if (ntree) {
+        ntree->id.flag |= LIB_PRIVATE_DATA;
+      }
+    }
+    FOREACH_MAIN_ID_END;
+  }
+
   {
     /* Versioning code until next subversion bump goes here. */
   }
diff --git a/source/blender/editors/interface/interface_ops.c b/source/blender/editors/interface/interface_ops.c
index 0afd67e5e66..68d21e88211 100644
--- a/source/blender/editors/interface/interface_ops.c
+++ b/source/blender/editors/interface/interface_ops.c
@@ -100,10 +100,12 @@ static bool copy_data_path_button_poll(bContext *C)
 
 static int copy_data_path_button_exec(bContext *C, wmOperator *op)
 {
+  Main *bmain = CTX_data_main(C);
   PointerRNA ptr;
   PropertyRNA *prop;
   char *path;
   int index;
+  ID *id;
 
   const bool full_path = RNA_boolean_get(op->ptr, "full_path");
 
@@ -111,18 +113,20 @@ static int copy_data_path_button_exec(bContext *C, wmOperator *op)
   UI_context_active_but_prop_get(C, &ptr, &prop, &index);
 
   if (ptr.owner_id != NULL) {
-
     if (full_path) {
-
       if (prop) {
-        path = RNA_path_full_property_py_ex(&ptr, prop, index, true);
+        path = RNA_path_full_property_py_ex(bmain, &ptr, prop, index, true);
       }
       else {
-        path = RNA_path_full_struct_py(&ptr);
+        path = RNA_path_full_struct_py(bmain, &ptr);
       }
     }
     else {
-      path = RNA_path_from_ID_to_property(&ptr, prop);
+      path = RNA_path_from_real_ID_to_property_index(bmain, &ptr, prop, 0, -1, &id);
+
+      if (!path) {
+        path = RNA_path_from_ID_to_property(&ptr, prop);
+      }
     }
 
     if (path) {
@@ -185,8 +189,9 @@ static bool copy_as_driver_button_poll(bContext *C)
   return 0;
 }
 
-static int copy_as_driver_button_exec(bContext *C, wmOperator *UNUSED(op))
+static int copy_as_driver_button_exec(bContext *C, wmOperator *op)
 {
+  Main *bmain = CTX_data_main(C);
   PointerRNA ptr;
   PropertyRNA *prop;
   int index;
@@ -195,14 +200,19 @@ static int copy_as_driver_button_exec(bContext *C, wmOperator *UNUSED(op))
   UI_context_active_but_prop_get(C, &ptr, &prop, &index);
 
   if (ptr.owner_id && ptr.data && prop) {
+    ID *id;
     int dim = RNA_property_array_dimension(&ptr, prop, NULL);
-    char *path = RNA_path_from_ID_to_property_index(&ptr, prop, dim, index);
+    char *path = RNA_path_from_real_ID_to_property_index(bmain, &ptr, prop, dim, index, &id);
 
     if (path) {
-      ANIM_copy_as_driver(ptr.owner_id, path, RNA_property_identifier(prop));
+      ANIM_copy_as_driver(id, path, RNA_property_identifier(prop));
       MEM_freeN(path);
       return OPERATOR_FINISHED;
     }
+    else {
+      BKE_reportf(op->reports, RPT_ERROR, "Could not compute a valid data path");
+      return OPERATOR_CANCELLED;
+    }
   }
 
   return OPERATOR_CANCELLED;
diff --git a/source/blender/editors/interface/interface_region_tooltip.c b/source/blender/editors/interface/interface_region_tooltip.c
index f2ca9cebf7b..7387fe5eb1c 100644
--- a/source/blender/editors/interface/interface_region_tooltip.c
+++ b/source/blender/editors/interface/interface_region_tooltip.c
@@ -860,10 +860,10 @@ static uiTooltipData *ui_tooltip_data_from_button(bContext *C, uiBut *but)
       /* move ownership (no need for re-alloc) */
       if (but->rnaprop) {
         field->text = RNA_path_full_property_py_ex(
-            &but->rnapoin, but->rnaprop, but->rnaindex, true);
+            CTX_data_main(C), &but->rnapoin, but->rnaprop, but->rnaindex, true);
       }
       else {
-        field->text = RNA_path_full_struct_py(&but->rnapoin);
+        field->text = RNA_path_full_struct_py(CTX_data_main(C), &but->rnapoin);
       }
     }
   }
diff --git a/source/blender/editors/space_console/space_console.c b/source/blender/editors/space_console/space_console.c
index 999255aef88..f5c02dbd724 100644
--- a/source/blender/editors/space_console/space_console.c
+++ b/source/blender/editors/space_console/space_console.c
@@ -26,6 +26,7 @@
 #include "BLI_blenlib.h"
 #include "BLI_utildefines.h"
 
+#include "BKE_global.h"
 #include "BKE_context.h"
 #include "BKE_screen.h"
 
@@ -173,7 +174,7 @@ static void id_drop_copy(wmDrag *drag, wmDropBox *drop)
   ID *id = WM_drag_ID(drag, 0);
 
   /* copy drag path to properties */
-  char *text = RNA_path_full_ID_py(id);
+  char *text = RNA_path_full_ID_py(G_MAIN, id);
   RNA_string_set(drop->ptr, "text", text);
   MEM_freeN(text);
 }
diff --git a/source/blender/editors/space_text/space_text.c b/source/blender/editors/space_text/space_text.c
index c1a3c79b0d8..9f39313b9ab 100644
--- a/source/blender/editors/space_text/space_text.c
+++ b/source/blender/editors/space_text/space_text.c
@@ -29,6 +29,7 @@
 
 #include "BLI_blenlib.h"
 
+#include "BKE_global.h"
 #include "BKE_context.h"
 #include "BKE_library.h"
 #include "BKE_screen.h"
@@ -356,7 +357,7 @@ static void text_drop_paste(wmDrag *drag, wmDropBox *drop)
   ID *id = WM_drag_ID(drag, 0);
 
   /* copy drag path to properties */
-  text = RNA_path_full_ID_py(id);
+  text = RNA_path_full_ID_py(G_MAIN, id);
   RNA_string_set(drop->ptr, "text", text);
   MEM_freeN(text);
 }
diff --git a/source/blender/makesdna/DNA_ID.h b/source/blender/makesdna/DNA_ID.h
index ef9069acb78..ed7416e06c9 100644
--- a/source/blender/makesdna/DNA_ID.h
+++ b/source/blender/makesdna/DNA_ID.h
@@ -467,7 +467,11 @@ typedef enum ID_Type {
 
 /* id->flag (persitent). */
 enum {
+  /* Don't delete the datablock even if unused. */
   LIB_FAKEUSER = 1 << 9,
+  /* The datablock structure is a sub-object of a different one.
+   * Direct persistent references are not allowed. */
+  LIB_PRIVATE_DATA = 1 << 10,
 };
 
 /**
diff --git a/source/blender/makesrna/RNA_access.h b/source/blender/makesrna/RNA_access.h
index f03aacf8dbd..c1199ac94b6 100644
--- a/source/blender/makesrna/RNA_access.h
+++ b/source/blender/makesrna/RNA_access.h
@@ -1151,24 +1151,37 @@ struct PropertyElemRNA {
 };
 bool RNA_path_resolve_elements(PointerRNA *ptr, const char *path, struct ListBase *r_elements);
 
+struct ID *RNA_find_real_ID_and_path(struct Main *bmain, struct ID *id, const char **r_path);
+
 char *RNA_path_from_ID_to_struct(PointerRNA *ptr);
+
+char *RNA_path_from_real_ID_to_struct(struct Main *bmain, PointerRNA *ptr, struct ID **r_real);
+
 char *RNA_path_from_ID_to_property(PointerRNA *ptr, PropertyRNA *prop);
 char *RNA_path_from_ID_to_property_index(PointerRNA *ptr,
                                          PropertyRNA *prop,
                                          int array_dim,
                                          int index);
 
+char *RNA_path_from_real_ID_to_property_index(struct Main *bmain,
+                                              PointerRNA *ptr,
+                                              PropertyRNA *prop,
+                                              int array_dim,
+                                              int index,
+                                              struct ID **r_real);
+
 char *RNA_path_resolve_from_type_to_property(struct PointerRNA *ptr,
                                              struct PropertyRNA *prop,
                                              const struct StructRNA *type);
 
-char *RNA_path_full_ID_py(struct ID *id);
-char *RNA_path_full_struct_py(struct PointerRNA *ptr);
-char *RNA_path_full_property_py_ex(PointerRNA *ptr,
-                                   PropertyRNA *prop,
-                                   int index,
-                                   bool use_fallback);
-char *RNA_path_full_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
+char *RNA_path_full_ID_py(struct Main *bmain, struct ID *id);
+char *RNA_path_full_struct_py(struct Main *bmain, struct PointerRNA *ptr);
+char *RNA_path_full_property_py_ex(
+    struct Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index, bool use_fallback);
+char *RNA_path_full_property_py(struct Main *bmain,
+                                struct PointerRNA *ptr,
+                                struct PropertyRNA *prop,
+                                int index);
 char *RNA_path_struct_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
 char *RNA_path_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
 
diff --git a/source/blender/makesrna/intern/rna_access.c b/source/blender/makesrna/intern/rna_access.c
index 8327456f460..bc015a378cc 100644
--- a/source/blender/makesrna/intern/rna_access.c
+++ b/source/blender/makesrna/intern/rna_access.c
@@ -47,6 +47,7 @@
 #include "BKE_fcurve.h"
 #include "BKE_main.h"
 #include "BKE_report.h"
+#include "BKE_node.h"
 
 #include "DEG_depsgraph.h"
 
@@ -5757,6 +5758,61 @@ static char *rna_path_from_ID_to_idpgroup(PointerRNA *ptr)
   }
 }
 
+/**
+ * Find the actual ID pointer and path from it to the given ID.
+ *
+ * \param id: ID reference to search the global owner for.
+ * \param[out] r_path: Path from the real ID to the initial ID.
+ * \return The ID pointer, or NULL in case of failure.
+ */
+ID *RNA_find_real_ID_and_path(Main *bmain, ID *id, const char **r_path)
+{
+  if (r_path) {
+    *r_path = "";
+  }
+
+  if ((id != NULL) && (id->flag & LIB_PRIVATE_DATA)) {
+    switch (GS(id->name)) {
+      case ID_NT:
+        if (r_path) {
+          *r_path = "node_tree";
+        }
+        return BKE_node_tree_find_owner_ID(bmain, (bNodeTree *)id);
+
+      default:
+        return NULL;
+    }
+  }
+  else {
+    return id;
+  }
+}
+
+static char *rna_prepend_real_ID_path(Main *bmain, ID *id, char *path, ID **r_real)
+{
+  if (path) {
+    const char *prefix;
+    char *new_path = NULL;
+
+    *r_real = RNA_find_real_ID_and_path(bmain, id, &prefix);
+
+    if (*r_real) {
+      if (prefix[0]) {
+        new_path = BLI_sprintfN("%s%s%s", prefix, path[0] == '[' ? "" : ".", path);
+      }
+      else {
+        return path;
+      }
+    }
+
+    MEM_freeN(path);
+    return new_path;
+  }
+  else {
+    return NULL;
+  }
+}
+
 char *RNA_path_from_ID_to_struct(PointerRNA *ptr)
 {
   char *ptrpath = NULL;
@@ -5799,6 +5855,13 @@ char *RNA_path_from_ID_to_struct(PointerRNA *ptr)
   return ptrpath;
 }
 
+char *RNA_path_from_real_ID_to_struct(Main *bmain, PointerRNA *ptr, struct ID **r_real)
+{
+  char *path = RNA_path_from_ID_to_struct(ptr);
+
+  return rna_prepend_real_ID_path(bmain, ptr->owner_id, path, r_real);
+}
+
 static void rna_path_array_multi_from_flat_index(const int dimsize[RNA_MAX_ARRAY_LENGTH],
                                                  const int totdims,
                                                  const int index_dim,
@@ -5905,6 +5968,14 @@ char *RNA_path_from_ID_to_property(PointerRNA *ptr, PropertyRNA *prop)
   return RNA_path_from_ID_to_property_index(ptr, prop, 0, -1);
 }
 
+char *RNA_path_from_real_ID_to_property_index(
+    Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index_dim, int index, ID **r_real)
+{
+  char *path = RNA_path_from_ID_to_property_index(ptr, prop, index_dim, index);
+
+  return rna_prepend_real_ID_path(bmain, ptr->owner_id, path, r_real);
+}
+
 /**
  * \return the path to given ptr/prop from the closest ancestor of given type,
  * if any (else return NULL).
@@ -5951,20 +6022,34 @@ char *RNA_path_resolve_from_type_to_property(PointerRNA *ptr,
  * Get the ID as a python representation, eg:
  *   bpy.data.foo["bar"]
  */
-char *RNA_path_full_ID_py(ID *id)
+char *RNA_path_full_ID_py(Main *bmain, ID *id)
 {
+  const char *path;
+  ID *id_real = RNA_find_real_ID_and_path(bmain, id, &path);
+
+  if (id_real) {
+    id = id_real;
+  }
+  else {
+    path = "";
+  }
+
   char id_esc[(sizeof(id->name) - 2) * 2];
 
   BLI_strescape(id_esc, id->name + 2, sizeof(id_esc));
 
-  return BLI_sprintfN("bpy.data.%s[\"%s\"]", BKE_idcode_to_name_plural(GS(id->name)), id_esc);
+  return BLI_sprintfN("bpy.data.%s[\"%s\"]%s%s",
+                      BKE_idcode_to_name_plural(GS(id->name)),
+                      id_esc,
+                      path[0] ? "." : "",
+                      path);
 }
 
 /**
  * Get the ID.struct as a python representation, eg:
  *   bpy.data.foo["bar"].some_struct
  */
-char *RNA_path_full_struct_py(struct PointerRNA *ptr)
+char *RNA_path_full_struct_py(Main *bmain, struct PointerRNA *ptr)
 {
   char *id_path;
   char *data_path;
@@ -5976,7 +6061,7 @@ char *RNA_path_full_struct_py(struct PointerRNA *ptr)
   }
 
   /* never fails */
-  id_path = RNA_path_full_ID_py(ptr->owner_id);
+  id_path = RNA_path_full_ID_py(bmain, ptr->owner_id);
 
   data_path = RNA_path_from_ID_to_struct(ptr);
 
@@ -5996,10 +6081,8 @@ char *RNA_path_full_struct_py(struct PointerRNA *ptr)
  * Get the ID.struct.property as a python representation, eg:
  *   bpy.data.foo["bar"].some_struct.some_prop[10]
  */
-char *RNA_path_full_property_py_ex(PointerRNA *ptr,
-                                   PropertyRNA *prop,
-                                   int index,
-                                   bool use_fallback)
+char *RNA_path_full_property_py_ex(
+    Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index, bool use_fallback)
 {
   char *id_path;
   const char *data_delim;
@@ -6013,7 +6096,7 @@ char *RNA_path_full_property_py_ex(PointerRNA *ptr,
   }
 
   /* never fails */
-  id_path = RNA_path_full_ID_py(ptr->owner_id);
+  id_path = RNA_path_full_ID_py(bmain, ptr->owner_id);
 
   data_path = RNA_path_from_ID_to_property(ptr, prop);
   if (data_path) {
@@ -6046,9 +6129,9 @@ char *RNA_path_full_property_py_ex(PointerRNA *ptr,
   return ret;
 }
 
-char *RNA_path_full_property_py(PointerRNA *ptr, PropertyRNA *prop, int index)
+char *RNA_path_full_property_py(Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index)
 {
-  return RNA_path_full_property_py_ex(ptr, prop, index, false);
+  return RNA_path_full_property_py_ex(bmain, ptr, prop, index, false);
 }
 
 /**
@@ -6646,16 +6729,16 @@ char *RNA_pointer_as_string_id(bContext *C, PointerRNA *ptr)
   return cstring;
 }
 
-static char *rna_pointer_as_string__bldata(PointerRNA *ptr)
+static char *rna_pointer_as_string__bldata(Main *bmain, PointerRNA *ptr)
 {
   if (ptr->type == NULL || ptr->owner_id == NULL) {
     return BLI_strdup("None");
   }
   else if (RNA_struct_is_ID(ptr->type)) {
-    return RNA_path_full_ID_py(ptr->owner_id);
+    return RNA_path_full_ID_py(bmain, ptr->owner_id);
   }
   else {
-    return RNA_path_full_struct_py(ptr);
+    return RNA_path_full_struct_py(bmain, ptr);
   }
 }
 
@@ -6672,7 +6755,7 @@ char *RNA_pointer_as_string(bContext *C,
     return RNA_pointer_as_string_id(C, ptr_prop);
   }
   else {
-    return rna_pointer_as_string__bldata(ptr_prop);
+    return rna_pointer_as_string__bldata(CTX_data_main(C), ptr_prop);
   }
 }
 
diff --git a/source/blender/windowmanager/intern/wm_operators.c b/source/blender/windowmanager/intern/wm_operators.c
index b33b4184424..9657347a1c4 100644
--- a/source/blender/windowmanager/intern/wm_operators.c
+++ b/source/blender/windowmanager/intern/wm_operators.c
@@ -539,7 +539,7 @@ char *WM_prop_pystring_assign(bContext *C, PointerRNA *ptr, PropertyRNA *prop, i
 
   if (lhs == NULL) {
     /* fallback to bpy.data.foo[id] if we dont find in the context */
-    lhs = RNA_path_full_property_py(ptr, prop, index);
+    lhs = RNA_path_full_property_py(CTX_data_main(C), ptr, prop, index);
   }
 
   if (!lhs) {