Fix #35704: Simplify on scene with dupli recursion crashes

Couple of issues here:

- User shouldn't be able to run into dupligroup recursion.
  It was checking already when setting a group for dupli.
  Added check to operator which adds object to group.

- It's still possible files with recursion are hanging around,
  so made simplify function robust to such kind of crap.

2 files changed, 69 insertions, 0 deletions

diff --git a/source/blender/editors/object/object_group.c b/source/blender/editors/object/object_group.c
index 8afd3049b84..f48e4ff5056 100644
--- a/source/blender/editors/object/object_group.c
+++ b/source/blender/editors/object/object_group.c
@@ -42,6 +42,7 @@
 #include "BKE_context.h"
 #include "BKE_depsgraph.h"
 #include "BKE_group.h"
+#include "BKE_library.h"
 #include "BKE_main.h"
 #include "BKE_report.h"
 #include "BKE_object.h"
@@ -377,8 +378,50 @@ void OBJECT_OT_group_add(wmOperatorType *ot)
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
+static bool group_link_early_exit_check(Group *group, Object *object)
+{
+	GroupObject *group_object;
+
+	for (group_object = group->gobject.first; group_object; group_object = group_object->next) {
+		if (group_object->ob == object) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static bool check_group_contains_object_recursive(Group *group, Object *object)
+{
+	GroupObject *group_object;
+
+	if ((group->id.flag & LIB_DOIT) == 0) {
+		/* Cycle already exists in groups, let's prevent further crappyness */
+		return true;
+	}
+
+	group->id.flag &= ~LIB_DOIT;
+
+	for (group_object = group->gobject.first; group_object; group_object = group_object->next) {
+		Object *current_object = group_object->ob;
+
+		if (current_object == object) {
+			return true;
+		}
+
+		if (current_object->dup_group) {
+			if (check_group_contains_object_recursive(current_object->dup_group, object)) {
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
 static int group_link_exec(bContext *C, wmOperator *op)
 {
+	Main *bmain = CTX_data_main(C);
 	Scene *scene = CTX_data_scene(C);
 	Object *ob = ED_object_context(C);
 	Group *group = BLI_findlink(&CTX_data_main(C)->group, RNA_enum_get(op->ptr, "group"));
@@ -386,6 +429,25 @@ static int group_link_exec(bContext *C, wmOperator *op)
 	if (ELEM(NULL, ob, group))
 		return OPERATOR_CANCELLED;
 
+	/* Early return check, if the object is already in group
+	 * we could sckip all the dependency check and just consider
+	 * operator is finished.
+	 */
+	if (group_link_early_exit_check(group, ob)) {
+		return OPERATOR_FINISHED;
+	}
+
+	/* Adding object to group which is used as dupligroup for self is bad idea.
+	 *
+	 * It is also  bad idea to add object to group which is in group which
+	 * contains our current object.
+	 */
+	tag_main_lb(&bmain->group, TRUE);
+	if (ob->dup_group == group || check_group_contains_object_recursive(group, ob)) {
+		BKE_report(op->reports, RPT_ERROR, "Could not add the group because of dependency cycle detected");
+		return OPERATOR_CANCELLED;
+	}
+
 	BKE_group_object_add(group, ob, scene, NULL);
 
 	WM_event_add_notifier(C, NC_OBJECT | ND_DRAW, ob);
diff --git a/source/blender/makesrna/intern/rna_scene.c b/source/blender/makesrna/intern/rna_scene.c
index ec672522a76..0218c188062 100644
--- a/source/blender/makesrna/intern/rna_scene.c
+++ b/source/blender/makesrna/intern/rna_scene.c
@@ -1255,6 +1255,12 @@ static void object_simplify_update(Object *ob)
 	ModifierData *md;
 	ParticleSystem *psys;
 
+	if ((ob->id.flag & LIB_DOIT) == 0) {
+		return;
+	}
+
+	ob->id.flag &= ~LIB_DOIT;
+
 	for (md = ob->modifiers.first; md; md = md->next) {
 		if (ELEM3(md->type, eModifierType_Subsurf, eModifierType_Multires, eModifierType_ParticleSystem)) {
 			ob->recalc |= PSYS_RECALC_CHILD;
@@ -1279,6 +1285,7 @@ static void rna_Scene_use_simplify_update(Main *bmain, Scene *UNUSED(scene), Poi
 	Scene *sce_iter;
 	Base *base;
 
+	tag_main_lb(&bmain->object, TRUE);
 	for (SETLOOPER(sce, sce_iter, base))
 		object_simplify_update(base->object);