Fix T39942: Displacement of group instance objects when switching to textured viewport shading

Usual dupli object issue, sometimes it's needed that all the object in
dupli group have modified obmat.

Made it an utility function now, which is used by convertblender and
dupli draw code now.

4 files changed, 74 insertions, 42 deletions

diff --git a/source/blender/blenkernel/BKE_anim.h b/source/blender/blenkernel/BKE_anim.h
index e87fef1c864..e49fe98aa14 100644
--- a/source/blender/blenkernel/BKE_anim.h
+++ b/source/blender/blenkernel/BKE_anim.h
@@ -71,5 +71,17 @@ struct ListBase *object_duplilist(struct EvaluationContext *eval_ctx, struct Sce
 void free_object_duplilist(struct ListBase *lb);
 int count_duplilist(struct Object *ob);
 
-#endif
+typedef struct DupliExtraData {
+	float obmat[4][4];
+} DupliExtraData;
+
+typedef struct DupliApplyData {
+	int num_objects;
+	DupliExtraData *extra;
+} DupliApplyData;
 
+DupliApplyData *duplilist_apply_matrix(struct ListBase *duplilist);
+void duplilist_restore_matrix(struct ListBase *duplilist, DupliApplyData *apply_data);
+void duplilist_free_apply_data(DupliApplyData *apply_data);
+
+#endif
diff --git a/source/blender/blenkernel/intern/object_dupli.c b/source/blender/blenkernel/intern/object_dupli.c
index d246a77c0f7..30e2cc253a4 100644
--- a/source/blender/blenkernel/intern/object_dupli.c
+++ b/source/blender/blenkernel/intern/object_dupli.c
@@ -1245,3 +1245,42 @@ int count_duplilist(Object *ob)
 	}
 	return 1;
 }
+
+DupliApplyData *duplilist_apply_matrix(ListBase *duplilist)
+{
+	DupliApplyData *apply_data = NULL;
+	int num_objects = BLI_countlist(duplilist);
+	if (num_objects > 0) {
+		DupliObject *dob;
+		int i;
+		apply_data = MEM_mallocN(sizeof(DupliApplyData), "DupliObject apply data");
+		apply_data->num_objects = num_objects;
+		apply_data->extra = MEM_mallocN(sizeof(DupliExtraData) * (size_t) num_objects,
+		                                "DupliObject apply extra data");
+
+		for (dob = duplilist->first, i = 0; dob; dob = dob->next, ++i) {
+			copy_m4_m4(apply_data->extra[i].obmat, dob->ob->obmat);
+			copy_m4_m4(dob->ob->obmat, dob->mat);
+		}
+	}
+	return apply_data;
+}
+
+void duplilist_restore_matrix(ListBase *duplilist, DupliApplyData *apply_data)
+{
+	DupliObject *dob;
+	int i;
+	/* Restore object matrices.
+	 * NOTE: this has to happen in reverse order, since nested
+	 * dupli objects can repeatedly override the obmat.
+	 */
+	for (dob = duplilist->last, i = apply_data->num_objects - 1; dob; dob = dob->prev, --i) {
+		copy_m4_m4(dob->ob->obmat, apply_data->extra[i].obmat);
+	}
+}
+
+void duplilist_free_apply_data(DupliApplyData *apply_data)
+{
+	MEM_freeN(apply_data->extra);
+	MEM_freeN(apply_data);
+}
diff --git a/source/blender/editors/space_view3d/view3d_draw.c b/source/blender/editors/space_view3d/view3d_draw.c
index b00cec7b0f7..bcde630e270 100644
--- a/source/blender/editors/space_view3d/view3d_draw.c
+++ b/source/blender/editors/space_view3d/view3d_draw.c
@@ -1966,7 +1966,6 @@ static void draw_dupli_objects_color(Scene *scene, ARegion *ar, View3D *v3d, Bas
 	RegionView3D *rv3d = ar->regiondata;
 	ListBase *lb;
 	LodLevel *savedlod;
-	float savedobmat[4][4];
 	DupliObject *dob_prev = NULL, *dob, *dob_next = NULL;
 	Base tbase = {NULL};
 	BoundBox bb, *bb_tmp; /* use a copy because draw_object, calls clear_mesh_caches */
@@ -1974,13 +1973,16 @@ static void draw_dupli_objects_color(Scene *scene, ARegion *ar, View3D *v3d, Bas
 	short transflag, use_displist = -1;  /* -1 is initialize */
 	char dt;
 	short dtx;
-	
+	DupliApplyData *apply_data;
+
 	if (base->object->restrictflag & OB_RESTRICT_VIEW) return;
 	
 	tbase.flag = OB_FROMDUPLI | base->flag;
 	lb = object_duplilist(G.main->eval_ctx, scene, base->object);
 	// BLI_sortlist(lb, dupli_ob_sort); /* might be nice to have if we have a dupli list with mixed objects. */
 
+	apply_data = duplilist_apply_matrix(lb);
+
 	dob = dupli_step(lb->first);
 	if (dob) dob_next = dupli_step(dob->next);
 
@@ -1989,8 +1991,6 @@ static void draw_dupli_objects_color(Scene *scene, ARegion *ar, View3D *v3d, Bas
 
 		/* Make sure lod is updated from dupli's position */
 
-		copy_m4_m4(savedobmat, dob->ob->obmat);
-		copy_m4_m4(dob->ob->obmat, dob->mat);
 		savedlod = dob->ob->currentlod;
 
 #ifdef WITH_GAMEENGINE
@@ -2083,11 +2083,13 @@ static void draw_dupli_objects_color(Scene *scene, ARegion *ar, View3D *v3d, Bas
 		tbase.object->dtx = dtx;
 		tbase.object->transflag = transflag;
 		tbase.object->currentlod = savedlod;
-		copy_m4_m4(tbase.object->obmat, savedobmat);
 	}
-	
-	/* Transp afterdraw disabled, afterdraw only stores base pointers, and duplis can be same obj */
-	
+
+	if (apply_data) {
+		duplilist_restore_matrix(lb, apply_data);
+		duplilist_free_apply_data(apply_data);
+	}
+
 	free_object_duplilist(lb);
 	
 	if (use_displist)
diff --git a/source/blender/render/intern/source/convertblender.c b/source/blender/render/intern/source/convertblender.c
index 7c8a6e030d5..37ef16d7105 100644
--- a/source/blender/render/intern/source/convertblender.c
+++ b/source/blender/render/intern/source/convertblender.c
@@ -4935,13 +4935,6 @@ static void add_group_render_dupli_obs(Render *re, Group *group, int nolamps, in
 	}
 }
 
-/* additional data for dupli objects outside
- * of the main dupli list
- */
-typedef struct DupliObjectExtra {
-	float omat[4][4];
-} DupliObjectExtra;
-
 static void database_init_objects(Render *re, unsigned int renderlay, int nolamps, int onlyselected, Object *actob, int timeoffset)
 {
 	Base *base;
@@ -5001,26 +4994,18 @@ static void database_init_objects(Render *re, unsigned int renderlay, int nolamp
 			if ((ob->transflag & OB_DUPLI) && (ob->type!=OB_MBALL)) {
 				DupliObject *dob;
 				ListBase *duplilist;
-				DupliObjectExtra *duplilist_extra = NULL;
-				int totdob, i;
+				DupliApplyData *duplilist_apply_data = NULL;
+				int i;
 
 				/* create list of duplis generated by this object, particle
 				 * system need to have render settings set for dupli particles */
 				dupli_render_particle_set(re, ob, timeoffset, 0, 1);
 				duplilist = object_duplilist(re->eval_ctx, re->scene, ob);
-				totdob = BLI_countlist(duplilist);
-				if (totdob > 0)
-					duplilist_extra = MEM_mallocN(sizeof(DupliObjectExtra) * totdob, "DupliObject extra data");
+				duplilist_apply_data = duplilist_apply_matrix(duplilist);
 				dupli_render_particle_set(re, ob, timeoffset, 0, 0);
 
-				/* set dupli obmats */
-				for (dob= duplilist->first, i = 0; dob; dob= dob->next, ++i) {
-					copy_m4_m4(duplilist_extra[i].omat, dob->ob->obmat);
-					copy_m4_m4(dob->ob->obmat, dob->mat);
-				}
-
 				for (dob= duplilist->first, i = 0; dob; dob= dob->next, ++i) {
-					DupliObjectExtra *dob_extra = &duplilist_extra[i];
+					DupliExtraData *dob_extra = &duplilist_apply_data->extra[i];
 					Object *obd= dob->ob;
 
 					/* group duplis need to set ob matrices correct, for deform. so no_draw is part handled */
@@ -5055,7 +5040,7 @@ static void database_init_objects(Render *re, unsigned int renderlay, int nolamp
 							obi= RE_addRenderInstance(re, NULL, obd, ob, dob->persistent_id[0], 0, mat, ob->lay); 
 
 							/* fill in instance variables for texturing */
-							set_dupli_tex_mat(re, obi, dob, dob_extra->omat);
+							set_dupli_tex_mat(re, obi, dob, dob_extra->obmat);
 							if (dob->type != OB_DUPLIGROUP) {
 								copy_v3_v3(obi->dupliorco, dob->orco);
 								obi->dupliuv[0]= dob->uv[0];
@@ -5081,7 +5066,7 @@ static void database_init_objects(Render *re, unsigned int renderlay, int nolamp
 									mul_m4_m4m4(mat, re->viewmat, dob->mat);
 								obi= RE_addRenderInstance(re, NULL, obd, ob, dob->persistent_id[0], psysindex++, mat, obd->lay);
 
-								set_dupli_tex_mat(re, obi, dob, dob_extra->omat);
+								set_dupli_tex_mat(re, obi, dob, dob_extra->obmat);
 								if (dob->type != OB_DUPLIGROUP) {
 									copy_v3_v3(obi->dupliorco, dob->orco);
 									obi->dupliuv[0]= dob->uv[0];
@@ -5097,7 +5082,7 @@ static void database_init_objects(Render *re, unsigned int renderlay, int nolamp
 
 						if (obi==NULL)
 							/* can't instance, just create the object */
-							init_render_object(re, obd, ob, dob, dob_extra->omat, timeoffset);
+							init_render_object(re, obd, ob, dob, dob_extra->obmat, timeoffset);
 						
 						if (dob->type != OB_DUPLIGROUP) {
 							obd->flag |= OB_DONE;
@@ -5105,22 +5090,16 @@ static void database_init_objects(Render *re, unsigned int renderlay, int nolamp
 						}
 					}
 					else
-						init_render_object(re, obd, ob, dob, dob_extra->omat, timeoffset);
+						init_render_object(re, obd, ob, dob, dob_extra->obmat, timeoffset);
 					
 					if (re->test_break(re->tbh)) break;
 				}
-				
-				/* restore obmats
-				 * NOTE: this has to happen in reverse order, since nested
-				 * dupli objects can repeatedly override the obmat
-				 */
-				for (dob= duplilist->last, i = totdob - 1; dob; dob= dob->prev, --i) {
-					copy_m4_m4(dob->ob->obmat, duplilist_extra[i].omat);
+
+				if (duplilist_apply_data) {
+					duplilist_restore_matrix(duplilist, duplilist_apply_data);
+					duplilist_free_apply_data(duplilist_apply_data);
 				}
-				
 				free_object_duplilist(duplilist);
-				if (duplilist_extra)
-					MEM_freeN(duplilist_extra);
 
 				if (allow_render_object(re, ob, nolamps, onlyselected, actob))
 					init_render_object(re, ob, NULL, NULL, NULL, timeoffset);