LibOverride: Fix broken Hook modifier case.

Hook modifier uses same kind of parent inverse matrix as regular object
parenting, so we need to bypass that matrix re-computation when setting
the overridding object here as well.

Also cleanup some minor glitches in object parents' override_apply func.

2 files changed, 58 insertions, 16 deletions

diff --git a/source/blender/makesrna/intern/rna_modifier.c b/source/blender/makesrna/intern/rna_modifier.c
index 26bb83fe6e8..d333d8eddaf 100644
--- a/source/blender/makesrna/intern/rna_modifier.c
+++ b/source/blender/makesrna/intern/rna_modifier.c
@@ -755,6 +755,47 @@ static void rna_HookModifier_object_set(PointerRNA *ptr,
   BKE_object_modifier_hook_reset(owner, hmd);
 }
 
+static bool rna_HookModifier_object_override_apply(Main *UNUSED(bmain),
+                                                   PointerRNA *ptr_dst,
+                                                   PointerRNA *ptr_src,
+                                                   PointerRNA *ptr_storage,
+                                                   PropertyRNA *prop_dst,
+                                                   PropertyRNA *prop_src,
+                                                   PropertyRNA *UNUSED(prop_storage),
+                                                   const int len_dst,
+                                                   const int len_src,
+                                                   const int len_storage,
+                                                   PointerRNA *UNUSED(ptr_item_dst),
+                                                   PointerRNA *UNUSED(ptr_item_src),
+                                                   PointerRNA *UNUSED(ptr_item_storage),
+                                                   IDOverrideLibraryPropertyOperation *opop)
+{
+  BLI_assert(len_dst == len_src && (!ptr_storage || len_dst == len_storage) && len_dst == 0);
+  BLI_assert(opop->operation == IDOVERRIDE_LIBRARY_OP_REPLACE &&
+             "Unsupported RNA override operation on Hook modifier target objet pointer");
+  UNUSED_VARS_NDEBUG(ptr_storage, len_dst, len_src, len_storage, opop);
+
+  /* We need a special handling here because setting hook target resets invert parent matrix,
+   * which is evil in our case. */
+  HookModifierData *hmd = ptr_dst->data;
+  Object *owner = (Object *)ptr_dst->owner_id;
+  Object *target_dst = RNA_property_pointer_get(ptr_dst, prop_dst).data;
+  Object *target_src = RNA_property_pointer_get(ptr_src, prop_src).data;
+
+  BLI_assert(target_dst == hmd->object);
+
+  if (target_src == target_dst) {
+    return false;
+  }
+
+  hmd->object = target_src;
+  if (target_src == NULL) {
+    /* The only case where we do want default behavior (with matrix reset). */
+    BKE_object_modifier_hook_reset(owner, hmd);
+  }
+  return true;
+}
+
 static void rna_HookModifier_subtarget_set(PointerRNA *ptr, const char *value)
 {
   Object *owner = (Object *)ptr->owner_id;
@@ -2289,6 +2330,7 @@ static void rna_def_modifier_hook(BlenderRNA *brna)
       prop, "Object", "Parent Object for hook, also recalculates and clears offset");
   RNA_def_property_flag(prop, PROP_EDITABLE | PROP_ID_SELF_CHECK);
   RNA_def_property_override_flag(prop, PROPOVERRIDE_OVERRIDABLE_LIBRARY);
+  RNA_def_property_override_funcs(prop, NULL, NULL, "rna_HookModifier_object_override_apply");
   RNA_def_property_pointer_funcs(prop, NULL, "rna_HookModifier_object_set", NULL, NULL);
   RNA_def_property_update(prop, 0, "rna_Modifier_dependency_update");
 
diff --git a/source/blender/makesrna/intern/rna_object.c b/source/blender/makesrna/intern/rna_object.c
index d605aa6b4f7..3c0fd282b17 100644
--- a/source/blender/makesrna/intern/rna_object.c
+++ b/source/blender/makesrna/intern/rna_object.c
@@ -546,27 +546,27 @@ static void rna_Object_parent_set(PointerRNA *ptr,
   }
 }
 
-bool rna_Object_parent_override_apply(Main *UNUSED(bmain),
-                                      PointerRNA *ptr_dst,
-                                      PointerRNA *ptr_src,
-                                      PointerRNA *ptr_storage,
-                                      PropertyRNA *prop_dst,
-                                      PropertyRNA *prop_src,
-                                      PropertyRNA *UNUSED(prop_storage),
-                                      const int len_dst,
-                                      const int len_src,
-                                      const int len_storage,
-                                      PointerRNA *UNUSED(ptr_item_dst),
-                                      PointerRNA *UNUSED(ptr_item_src),
-                                      PointerRNA *UNUSED(ptr_item_storage),
-                                      IDOverrideLibraryPropertyOperation *opop)
+static bool rna_Object_parent_override_apply(Main *UNUSED(bmain),
+                                             PointerRNA *ptr_dst,
+                                             PointerRNA *ptr_src,
+                                             PointerRNA *ptr_storage,
+                                             PropertyRNA *prop_dst,
+                                             PropertyRNA *prop_src,
+                                             PropertyRNA *UNUSED(prop_storage),
+                                             const int len_dst,
+                                             const int len_src,
+                                             const int len_storage,
+                                             PointerRNA *UNUSED(ptr_item_dst),
+                                             PointerRNA *UNUSED(ptr_item_src),
+                                             PointerRNA *UNUSED(ptr_item_storage),
+                                             IDOverrideLibraryPropertyOperation *opop)
 {
   BLI_assert(len_dst == len_src && (!ptr_storage || len_dst == len_storage) && len_dst == 0);
   BLI_assert(opop->operation == IDOVERRIDE_LIBRARY_OP_REPLACE &&
-             "Unsupported RNA override operation on animdata pointer");
+             "Unsupported RNA override operation on object parent pointer");
   UNUSED_VARS_NDEBUG(ptr_storage, len_dst, len_src, len_storage, opop);
 
-  /* We need a special handling here because setting parent resets pinvert parent matrix,
+  /* We need a special handling here because setting parent resets invert parent matrix,
    * which is evil in our case. */
   Object *ob = (Object *)ptr_dst->data;
   Object *parent_dst = RNA_property_pointer_get(ptr_dst, prop_dst).data;