Fix T40230: Recursion check when adding objects to groups is incorrect.

rB568f0c7 added a recursion check that is supposed to prevent cyclic
cases where a group includes itself via dupli instancing.

The check function was descending into all groups nested inside the
target group - which works for single level recursion like in the test
case, but does not handle generic recursion. Basically it asked:
"is object X in the group already or in any instanced dupligroup?"

The new check instead asks:
"is group G dupli'd by X or any instanced subgroup thereof?"
which is what we really need to know.

1 files changed, 14 insertions, 21 deletions

diff --git a/source/blender/editors/object/object_group.c b/source/blender/editors/object/object_group.c
index 69bd64542f4..09b8b09eb60 100644
--- a/source/blender/editors/object/object_group.c
+++ b/source/blender/editors/object/object_group.c
@@ -74,33 +74,26 @@ static bool group_link_early_exit_check(Group *group, Object *object)
 	return false;
 }
 
-static bool check_group_contains_object_recursive(Group *group, Object *object)
+static bool check_object_instances_group_recursive(Object *object, Group *group)
 {
-	GroupObject *group_object;
-
-	if ((group->id.flag & LIB_DOIT) == 0) {
+	if ((object->id.flag & LIB_DOIT) == 0) {
 		/* Cycle already exists in groups, let's prevent further crappyness */
 		return true;
 	}
-
-	group->id.flag &= ~LIB_DOIT;
-
-	for (group_object = group->gobject.first; group_object; group_object = group_object->next) {
-		Object *current_object = group_object->ob;
-
-		if (current_object == object) {
+	object->id.flag &= ~LIB_DOIT;
+	
+	if (object->dup_group) {
+		if (object->dup_group == group)
 			return true;
-		}
-
-		if (current_object->dup_group) {
-			if (check_group_contains_object_recursive(current_object->dup_group, object)) {
-				return true;
+		else {
+			GroupObject *gob;
+			for (gob = object->dup_group->gobject.first; gob; gob = gob->next) {
+				if (check_object_instances_group_recursive(gob->ob, group))
+					return true;
 			}
 		}
 	}
-
-	group->id.flag |= LIB_DOIT;
-
+	
 	return false;
 }
 
@@ -195,7 +188,7 @@ static int objects_add_active_exec(bContext *C, wmOperator *op)
 			if (group_link_early_exit_check(group, base->object))
 				continue;
 
-			if (base->object->dup_group != group && !check_group_contains_object_recursive(group, base->object)) {
+			if (!check_object_instances_group_recursive(base->object, group)) {
 				BKE_group_object_add(group, base->object, scene, base);
 				updated = true;
 			}
@@ -498,7 +491,7 @@ static int group_link_exec(bContext *C, wmOperator *op)
 	 * contains our current object.
 	 */
 	BKE_main_id_tag_listbase(&bmain->group, true);
-	if (ob->dup_group == group || check_group_contains_object_recursive(group, ob)) {
+	if (check_object_instances_group_recursive(ob, group)) {
 		BKE_report(op->reports, RPT_ERROR, "Could not add the group because of dependency cycle detected");
 		return OPERATOR_CANCELLED;
 	}