fix [#29616] Crash/infinite loop from missing cyclic check in group/dupligroups (unlikely/intentional)

calling BKE_ptcache_ids_from_object() could recursively call BKE_object_handle_update(), when there is no reason to update object transforms at all. Add option not to do any updates and just return an object list.

3 files changed, 48 insertions, 22 deletions

diff --git a/source/blender/blenkernel/BKE_anim.h b/source/blender/blenkernel/BKE_anim.h
index d1908fe7a3a..f506c67a36c 100644
--- a/source/blender/blenkernel/BKE_anim.h
+++ b/source/blender/blenkernel/BKE_anim.h
@@ -65,6 +65,7 @@ int where_on_path(struct Object *ob, float ctime, float vec[4], float dir[3], fl
 /* ---------------------------------------------------- */
 /* Dupli-Geometry */
 
+struct ListBase *object_duplilist_ex(struct Scene *sce, struct Object *ob, int update);
 struct ListBase *object_duplilist(struct Scene *sce, struct Object *ob);
 void free_object_duplilist(struct ListBase *lb);
 int count_duplilist(struct Object *ob);
diff --git a/source/blender/blenkernel/intern/anim.c b/source/blender/blenkernel/intern/anim.c
index 4f68ab82329..1b301ba43b3 100644
--- a/source/blender/blenkernel/intern/anim.c
+++ b/source/blender/blenkernel/intern/anim.c
@@ -74,7 +74,8 @@
 /* --------------------- */
 /* forward declarations */
 
-static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBase *duplilist, float par_space_mat[][4], int par_index, int level, int animated);
+static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBase *duplilist, float par_space_mat[][4], int par_index,
+                                       int level, short animated, short update);
 
 /* ******************************************************************** */
 /* Animation Visualization */
@@ -699,7 +700,7 @@ int where_on_path(Object *ob, float ctime, float vec[4], float dir[3], float qua
 /* ******************************************************************** */
 /* Dupli-Geometry */
 
-static DupliObject *new_dupli_object(ListBase *lb, Object *ob, float mat[][4], int lay, int index, int par_index, int type, int animated)
+static DupliObject *new_dupli_object(ListBase *lb, Object *ob, float mat[][4], int lay, int index, int par_index, int type, short animated)
 {
 	DupliObject *dob = MEM_callocN(sizeof(DupliObject), "dupliobject");
 	
@@ -717,7 +718,8 @@ static DupliObject *new_dupli_object(ListBase *lb, Object *ob, float mat[][4], i
 	return dob;
 }
 
-static void group_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_index, int level, int animated)
+static void group_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_index,
+                            int level, short animated, short update)
 {
 	DupliObject *dob;
 	Group *group;
@@ -731,8 +733,14 @@ static void group_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_inde
 	if (level > MAX_DUPLI_RECUR) return;
 	
 	/* handles animated groups, and */
+
 	/* we need to check update for objects that are not in scene... */
-	group_handle_recalc_and_update(scene, ob, group);
+	if (update) {
+		/* note: update is optional because we don't always need object
+		 * transformations to be correct. Also fixes bug [#29616]. */
+		group_handle_recalc_and_update(scene, ob, group);
+	}
+
 	animated = animated || group_is_animated(ob, group);
 	
 	for (go = group->gobject.first; go; go = go->next) {
@@ -764,14 +772,14 @@ static void group_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_inde
 
 			if (go->ob->transflag & OB_DUPLI) {
 				copy_m4_m4(dob->ob->obmat, dob->mat);
-				object_duplilist_recursive(&group->id, scene, go->ob, lb, ob->obmat, par_index, level + 1, animated);
+				object_duplilist_recursive(&group->id, scene, go->ob, lb, ob->obmat, par_index, level + 1, animated, update);
 				copy_m4_m4(dob->ob->obmat, dob->omat);
 			}
 		}
 	}
 }
 
-static void frames_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_index, int level, int animated)
+static void frames_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_index, int level, short animated)
 {
 	extern int enable_cu_speed; /* object.c */
 	Object copyob;
@@ -843,7 +851,8 @@ static void frames_duplilist(ListBase *lb, Scene *scene, Object *ob, int par_ind
 typedef struct VertexDupliData {
 	ID *id; /* scene or group, for recursive loops */
 	int level;
-	int animated;
+	short animated;
+	short update;
 	ListBase *lb;
 	float pmat[4][4];
 	float obmat[4][4]; /* Only used for dupliverts inside dupligroups, where the ob->obmat is modified */
@@ -899,12 +908,13 @@ static void vertex_dupli__mapFunc(void *userData, int index, const float co[3],
 		float tmpmat[4][4];
 		copy_m4_m4(tmpmat, vdd->ob->obmat);
 		copy_m4_m4(vdd->ob->obmat, obmat); /* pretend we are really this mat */
-		object_duplilist_recursive((ID *)vdd->id, vdd->scene, vdd->ob, vdd->lb, obmat, vdd->par_index, vdd->level + 1, vdd->animated);
+		object_duplilist_recursive((ID *)vdd->id, vdd->scene, vdd->ob, vdd->lb, obmat, vdd->par_index, vdd->level + 1, vdd->animated, vdd->update);
 		copy_m4_m4(vdd->ob->obmat, tmpmat);
 	}
 }
 
-static void vertex_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int par_index, int level, int animated)
+static void vertex_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int par_index,
+                             int level, short animated, short update)
 {
 	Object *ob, *ob_iter;
 	Mesh *me = par->data;
@@ -983,6 +993,7 @@ static void vertex_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, fl
 					vdd.id = id;
 					vdd.level = level;
 					vdd.animated = animated;
+					vdd.update = update;
 					vdd.lb = lb;
 					vdd.ob = ob;
 					vdd.scene = scene;
@@ -1027,7 +1038,8 @@ static void vertex_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, fl
 	dm->release(dm);
 }
 
-static void face_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int par_index, int level, int animated)
+static void face_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int par_index,
+                           int level, short animated, short update)
 {
 	Object *ob, *ob_iter;
 	Base *base = NULL;
@@ -1197,7 +1209,7 @@ static void face_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, floa
 							float tmpmat[4][4];
 							copy_m4_m4(tmpmat, ob->obmat);
 							copy_m4_m4(ob->obmat, obmat); /* pretend we are really this mat */
-							object_duplilist_recursive((ID *)id, scene, ob, lb, ob->obmat, par_index, level + 1, animated);
+							object_duplilist_recursive((ID *)id, scene, ob, lb, ob->obmat, par_index, level + 1, animated, update);
 							copy_m4_m4(ob->obmat, tmpmat);
 						}
 					}
@@ -1217,7 +1229,8 @@ static void face_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, floa
 	dm->release(dm);
 }
 
-static void new_particle_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int UNUSED(par_index), ParticleSystem *psys, int level, int animated)
+static void new_particle_duplilist(ListBase *lb, ID *id, Scene *scene, Object *par, float par_space_mat[][4], int UNUSED(par_index), ParticleSystem *psys,
+                                   int level, short animated, short update)
 {
 	GroupObject *go;
 	Object *ob = NULL, **oblist = NULL, obcopy, *obcopylist = NULL;
@@ -1300,7 +1313,9 @@ static void new_particle_duplilist(ListBase *lb, ID *id, Scene *scene, Object *p
 
 		/* gather list of objects or single object */
 		if (part->ren_as == PART_DRAW_GR) {
-			group_handle_recalc_and_update(scene, par, part->dup_group);
+			if (update) {
+				group_handle_recalc_and_update(scene, par, part->dup_group);
+			}
 
 			if (part->draw & PART_DRAW_COUNT_GR) {
 				for (dw = part->dupliweights.first; dw; dw = dw->next)
@@ -1555,7 +1570,7 @@ static Object *find_family_object(Object **obar, char *family, char ch)
 }
 
 
-static void font_duplilist(ListBase *lb, Scene *scene, Object *par, int par_index, int level, int animated)
+static void font_duplilist(ListBase *lb, Scene *scene, Object *par, int par_index, int level, short animated)
 {
 	Object *ob, *obar[256] = {NULL};
 	Curve *cu;
@@ -1603,7 +1618,8 @@ static void font_duplilist(ListBase *lb, Scene *scene, Object *par, int par_inde
 
 /* ------------- */
 
-static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBase *duplilist, float par_space_mat[][4], int par_index, int level, int animated)
+static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBase *duplilist, float par_space_mat[][4], int par_index,
+                                       int level, short animated, short update)
 {	
 	if ((ob->transflag & OB_DUPLI) == 0)
 		return;
@@ -1623,11 +1639,11 @@ static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBas
 	if (ob->transflag & OB_DUPLIPARTS) {
 		ParticleSystem *psys = ob->particlesystem.first;
 		for (; psys; psys = psys->next)
-			new_particle_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, psys, level + 1, animated);
+			new_particle_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, psys, level + 1, animated, update);
 	}
 	else if (ob->transflag & OB_DUPLIVERTS) {
 		if (ob->type == OB_MESH) {
-			vertex_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, level + 1, animated);
+			vertex_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, level + 1, animated, update);
 		}
 		else if (ob->type == OB_FONT) {
 			if (GS(id->name) == ID_SCE) { /* TODO - support dupligroups */
@@ -1637,7 +1653,7 @@ static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBas
 	}
 	else if (ob->transflag & OB_DUPLIFACES) {
 		if (ob->type == OB_MESH)
-			face_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, level + 1, animated);
+			face_duplilist(duplilist, id, scene, ob, par_space_mat, par_index, level + 1, animated, update);
 	}
 	else if (ob->transflag & OB_DUPLIFRAMES) {
 		if (GS(id->name) == ID_SCE) { /* TODO - support dupligroups */
@@ -1647,7 +1663,7 @@ static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBas
 	else if (ob->transflag & OB_DUPLIGROUP) {
 		DupliObject *dob;
 		
-		group_duplilist(duplilist, scene, ob, par_index, level + 1, animated); /* now recursive */
+		group_duplilist(duplilist, scene, ob, par_index, level + 1, animated, update); /* now recursive */
 
 		if (level == 0) {
 			for (dob = duplilist->first; dob; dob = dob->next)
@@ -1659,14 +1675,22 @@ static void object_duplilist_recursive(ID *id, Scene *scene, Object *ob, ListBas
 
 /* Returns a list of DupliObject
  * note; group dupli's already set transform matrix. see note in group_duplilist() */
-ListBase *object_duplilist(Scene *sce, Object *ob)
+ListBase *object_duplilist_ex(Scene *sce, Object *ob, int update)
 {
 	ListBase *duplilist = MEM_mallocN(sizeof(ListBase), "duplilist");
 	duplilist->first = duplilist->last = NULL;
-	object_duplilist_recursive((ID *)sce, sce, ob, duplilist, NULL, 0, 0, 0);
+	object_duplilist_recursive((ID *)sce, sce, ob, duplilist, NULL, 0, 0, 0, update);
 	return duplilist;
 }
 
+/* note: previously updating was always done, this is why it defaults to be on
+ * but there are likely places it can be called without updating */
+ListBase *object_duplilist(Scene *sce, Object *ob)
+{
+	return object_duplilist_ex(sce, ob, TRUE);
+}
+
+
 void free_object_duplilist(ListBase *lb)
 {
 	DupliObject *dob;
diff --git a/source/blender/blenkernel/intern/pointcache.c b/source/blender/blenkernel/intern/pointcache.c
index 5e12b15a658..780528f4a0d 100644
--- a/source/blender/blenkernel/intern/pointcache.c
+++ b/source/blender/blenkernel/intern/pointcache.c
@@ -1029,7 +1029,8 @@ void BKE_ptcache_ids_from_object(ListBase *lb, Object *ob, Scene *scene, int dup
 	if (scene && (duplis-- > 0) && (ob->transflag & OB_DUPLI)) {
 		ListBase *lb_dupli_ob;
 
-		if ((lb_dupli_ob=object_duplilist(scene, ob))) {
+		/* don't update the dupli groups, we only wan't their pid's */
+		if ((lb_dupli_ob = object_duplilist_ex(scene, ob, FALSE))) {
 			DupliObject *dob;
 			for (dob= lb_dupli_ob->first; dob; dob= dob->next) {
 				if (dob->ob != ob) { /* avoids recursive loops with dupliframes: bug 22988 */