Fix T47038: Particles in Particle Edit Mode get added in completely wrong location.

It also fixes another issue (crash) related to symmetric editing.

Quite involved, we (try to!) fix complete broken logic of parts of particle code, which would use poly index
as tessface one (or vice-versa). Issue most probably goes back to BMesh integration time...

This patch mostly fixes particle editing mode:
  - Adding/removing particles when using generative modifiers (like subsurf) should now work.
  - Adding/removing particles with a non-tessellated mesh (i.e. one having ngons) should also mostly work.
  - X-axis-mirror-editing particles over ngons does not really work, not sure why currently.
  - All this in both 'modes' (with or without using modifier stack for particles).

Tech side:
  - Store a deformed-only DM in particle modifier data.
  - Rename existing DM to make it clear it's a final one.
  - Use deformed-only DM's tessface2poly mapping to 'solve' poly/tessface mismatches.
  - Make (part of) mirror-editing code able to use a DM instead of raw mesh, so that we can mirror based on final DM
    when editing particles using modifier stack (mandatory, since there is no way currently to find orig tessface
    from an final DM tessface index).

Note that this patch is not really nice and clean (current particles are beyond hope on this side anyway),
it's more like some urgency bandage. Whole crap needs complete rewrite anyway,
BMesh's polygons make it really hard to work with current system (and looptri would not help much here).

Also, did not test everything possibly affected by those changes, so it needs some users' testing & validation too.

Reviewers: psy-fi

Subscribers: dfelinto, eyecandy

Maniphest Tasks: T47038

Differential Revision: https://developer.blender.org/D1685

1 files changed, 71 insertions, 60 deletions

diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index 64ad3a100cc..deba3bbca32 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -659,7 +659,7 @@ static void foreach_mouse_hit_key(PEData *data, ForKeyMatFunc func, int selected
 				if (selected==0 || key->flag & PEK_SELECT) {
 					if (key_inside_circle(data, data->rad, KEY_WCO, &data->dist)) {
 						if (edit->psys && !(edit->psys->flag & PSYS_GLOBAL_HAIR)) {
-							psys_mat_hair_to_global(data->ob, psmd->dm, psys->part->from, psys->particles + p, mat);
+							psys_mat_hair_to_global(data->ob, psmd->dm_final, psys->part->from, psys->particles + p, mat);
 							invert_m4_m4(imat, mat);
 						}
 
@@ -674,7 +674,7 @@ static void foreach_mouse_hit_key(PEData *data, ForKeyMatFunc func, int selected
 				if (selected==0 || key->flag & PEK_SELECT) {
 					if (key_inside_circle(data, data->rad, KEY_WCO, &data->dist)) {
 						if (edit->psys && !(edit->psys->flag & PSYS_GLOBAL_HAIR)) {
-							psys_mat_hair_to_global(data->ob, psmd->dm, psys->part->from, psys->particles + p, mat);
+							psys_mat_hair_to_global(data->ob, psmd->dm_final, psys->part->from, psys->particles + p, mat);
 							invert_m4_m4(imat, mat);
 						}
 
@@ -761,7 +761,7 @@ static void PE_update_mirror_cache(Object *ob, ParticleSystem *psys)
 	psmd= psys_get_modifier(ob, psys);
 	totpart= psys->totpart;
 
-	if (!psmd->dm)
+	if (!psmd->dm_final)
 		return;
 
 	tree= BLI_kdtree_new(totpart);
@@ -769,7 +769,7 @@ static void PE_update_mirror_cache(Object *ob, ParticleSystem *psys)
 	/* insert particles into kd tree */
 	LOOP_PARTICLES {
 		key = pa->hair;
-		psys_mat_hair_to_orco(ob, psmd->dm, psys->part->from, pa, mat);
+		psys_mat_hair_to_orco(ob, psmd->dm_final, psys->part->from, pa, mat);
 		copy_v3_v3(co, key->co);
 		mul_m4_v3(mat, co);
 		BLI_kdtree_insert(tree, p, co);
@@ -783,7 +783,7 @@ static void PE_update_mirror_cache(Object *ob, ParticleSystem *psys)
 	
 	LOOP_PARTICLES {
 		key = pa->hair;
-		psys_mat_hair_to_orco(ob, psmd->dm, psys->part->from, pa, mat);
+		psys_mat_hair_to_orco(ob, psmd->dm_final, psys->part->from, pa, mat);
 		copy_v3_v3(co, key->co);
 		mul_m4_v3(mat, co);
 		co[0] = -co[0];
@@ -898,7 +898,7 @@ static void PE_apply_mirror(Object *ob, ParticleSystem *psys)
 	edit= psys->edit;
 	psmd= psys_get_modifier(ob, psys);
 
-	if (!psmd->dm)
+	if (!psmd->dm_final)
 		return;
 
 	if (!edit->mirror_cache)
@@ -911,7 +911,7 @@ static void PE_apply_mirror(Object *ob, ParticleSystem *psys)
 	 * to avoid doing mirror twice */
 	LOOP_POINTS {
 		if (point->flag & PEP_EDIT_RECALC) {
-			PE_mirror_particle(ob, psmd->dm, psys, psys->particles + p, NULL);
+			PE_mirror_particle(ob, psmd->dm_final, psys, psys->particles + p, NULL);
 
 			if (edit->mirror_cache[p] != -1)
 				edit->points[edit->mirror_cache[p]].flag &= ~PEP_EDIT_RECALC;
@@ -946,11 +946,11 @@ static void pe_deflect_emitter(Scene *scene, Object *ob, PTCacheEdit *edit)
 	psys = edit->psys;
 	psmd = psys_get_modifier(ob, psys);
 
-	if (!psmd->dm)
+	if (!psmd->dm_final)
 		return;
 
 	LOOP_EDITED_POINTS {
-		psys_mat_hair_to_object(ob, psmd->dm, psys->part->from, psys->particles + p, hairmat);
+		psys_mat_hair_to_object(ob, psmd->dm_final, psys->part->from, psys->particles + p, hairmat);
 	
 		LOOP_KEYS {
 			mul_m4_v3(hairmat, key->co);
@@ -1095,7 +1095,7 @@ void recalc_lengths(PTCacheEdit *edit)
 /* calculate a tree for finding nearest emitter's vertice */
 void recalc_emitter_field(Object *ob, ParticleSystem *psys)
 {
-	DerivedMesh *dm=psys_get_modifier(ob, psys)->dm;
+	DerivedMesh *dm=psys_get_modifier(ob, psys)->dm_final;
 	PTCacheEdit *edit= psys->edit;
 	float *vec, *nor;
 	int i, totface /*, totvert*/;
@@ -1188,12 +1188,12 @@ void update_world_cos(Object *ob, PTCacheEdit *edit)
 	POINT_P; KEY_K;
 	float hairmat[4][4];
 
-	if (psys==0 || psys->edit==0 || psmd->dm==NULL)
+	if (psys==0 || psys->edit==0 || psmd->dm_final==NULL)
 		return;
 
 	LOOP_POINTS {
 		if (!(psys->flag & PSYS_GLOBAL_HAIR))
-			psys_mat_hair_to_global(ob, psmd->dm, psys->part->from, psys->particles+p, hairmat);
+			psys_mat_hair_to_global(ob, psmd->dm_final, psys->part->from, psys->particles+p, hairmat);
 
 		LOOP_KEYS {
 			copy_v3_v3(key->world_co, key->co);
@@ -1839,7 +1839,7 @@ int PE_lasso_select(bContext *C, const int mcords[][2], const short moves, bool
 
 	LOOP_VISIBLE_POINTS {
 		if (edit->psys && !(psys->flag & PSYS_GLOBAL_HAIR))
-			psys_mat_hair_to_global(ob, psmd->dm, psys->part->from, psys->particles + p, mat);
+			psys_mat_hair_to_global(ob, psmd->dm_final, psys->part->from, psys->particles + p, mat);
 
 		if (pset->selectmode==SCE_SELECT_POINT) {
 			LOOP_KEYS {
@@ -2273,7 +2273,7 @@ static int remove_tagged_particles(Object *ob, ParticleSystem *psys, int mirror)
 		psmd= psys_get_modifier(ob, psys);
 
 		LOOP_TAGGED_POINTS {
-			PE_mirror_particle(ob, psmd->dm, psys, psys->particles + p, NULL);
+			PE_mirror_particle(ob, psmd->dm_final, psys, psys->particles + p, NULL);
 		}
 	}
 
@@ -2353,7 +2353,7 @@ static void remove_tagged_keys(Object *ob, ParticleSystem *psys)
 
 		LOOP_POINTS {
 			LOOP_TAGGED_KEYS {
-				PE_mirror_particle(ob, psmd->dm, psys, psys->particles + p, NULL);
+				PE_mirror_particle(ob, psmd->dm_final, psys, psys->particles + p, NULL);
 				break;
 			}
 		}
@@ -2567,7 +2567,7 @@ static int remove_doubles_exec(bContext *C, wmOperator *op)
 			
 		/* insert particles into kd tree */
 		LOOP_SELECTED_POINTS {
-			psys_mat_hair_to_object(ob, psmd->dm, psys->part->from, psys->particles+p, mat);
+			psys_mat_hair_to_object(ob, psmd->dm_final, psys->part->from, psys->particles+p, mat);
 			copy_v3_v3(co, point->keys->co);
 			mul_m4_v3(mat, co);
 			BLI_kdtree_insert(tree, p, co);
@@ -2577,7 +2577,7 @@ static int remove_doubles_exec(bContext *C, wmOperator *op)
 
 		/* tag particles to be removed */
 		LOOP_SELECTED_POINTS {
-			psys_mat_hair_to_object(ob, psmd->dm, psys->part->from, psys->particles+p, mat);
+			psys_mat_hair_to_object(ob, psmd->dm_final, psys->part->from, psys->particles+p, mat);
 			copy_v3_v3(co, point->keys->co);
 			mul_m4_v3(mat, co);
 
@@ -2808,13 +2808,17 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 		return;
 
 	psmd= psys_get_modifier(ob, psys);
-	if (!psmd->dm)
+	if (!psmd->dm_final)
 		return;
 
+	const bool use_dm_final_indices = (psys->part->use_modifier_stack && !psmd->dm_final->deformedOnly);
+
 	/* NOTE: this is not nice to use tessfaces but hard to avoid since pa->num uses tessfaces */
 	BKE_mesh_tessface_ensure(me);
 
-	mirrorfaces= mesh_get_x_mirror_faces(ob, NULL);
+	/* Note: In case psys uses DM tessface indices, we mirror final DM iteslef, not orig mesh. Avoids an (impossible)
+	 *       dm -> orig -> dm tessface indices conversion... */
+	mirrorfaces = mesh_get_x_mirror_faces(ob, NULL, use_dm_final_indices ? psmd->dm_final : NULL);
 
 	if (!edit->mirror_cache)
 		PE_update_mirror_cache(ob, psys);
@@ -2823,11 +2827,12 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 	newtotpart= psys->totpart;
 	LOOP_VISIBLE_POINTS {
 		pa = psys->particles + p;
+
 		if (!tagged) {
 			if (point_is_selected(point)) {
 				if (edit->mirror_cache[p] != -1) {
 					/* already has a mirror, don't need to duplicate */
-					PE_mirror_particle(ob, psmd->dm, psys, pa, NULL);
+					PE_mirror_particle(ob, psmd->dm_final, psys, pa, NULL);
 					continue;
 				}
 				else
@@ -2840,6 +2845,8 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 	}
 
 	if (newtotpart != psys->totpart) {
+		MFace *mtessface = use_dm_final_indices ? psmd->dm_final->getTessFaceArray(psmd->dm_final) : me->mface;
+
 		/* allocate new arrays and copy existing */
 		new_pars= MEM_callocN(newtotpart*sizeof(ParticleData), "ParticleData new");
 		new_points= MEM_callocN(newtotpart*sizeof(PTCacheEditPoint), "PTCacheEditPoint new");
@@ -2869,10 +2876,12 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 
 		for (p=0, point=edit->points; p<totpart; p++, point++) {
 			pa = psys->particles + p;
+			const int pa_num = pa->num;
 
 			if (point->flag & PEP_HIDE)
 				continue;
-			if (!(point->flag & PEP_TAG) || mirrorfaces[pa->num*2] == -1)
+
+			if (!(point->flag & PEP_TAG) || mirrorfaces[pa_num * 2] == -1)
 				continue;
 
 			/* duplicate */
@@ -2882,27 +2891,31 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 			if (point->keys) newpoint->keys= MEM_dupallocN(point->keys);
 
 			/* rotate weights according to vertex index rotation */
-			rotation= mirrorfaces[pa->num*2+1];
+			rotation= mirrorfaces[pa_num * 2 + 1];
 			newpa->fuv[0] = pa->fuv[2];
 			newpa->fuv[1] = pa->fuv[1];
 			newpa->fuv[2] = pa->fuv[0];
 			newpa->fuv[3] = pa->fuv[3];
-			while (rotation-- > 0)
-				if (me->mface[pa->num].v4) {
+			while (rotation--) {
+				if (mtessface[pa_num].v4) {
 					SHIFT4(float, newpa->fuv[0], newpa->fuv[1], newpa->fuv[2], newpa->fuv[3]);
 				}
 				else {
 					SHIFT3(float, newpa->fuv[0], newpa->fuv[1], newpa->fuv[2]);
 				}
+			}
 
-			/* assign face inddex */
-			newpa->num= mirrorfaces[pa->num*2];
-			newpa->num_dmcache= psys_particle_dm_face_lookup(ob, psmd->dm, newpa->num, newpa->fuv, NULL);
+			/* assign face index */
+			/* NOTE: mesh_get_x_mirror_faces generates -1 for non-found mirror, same as DMCACHE_NOTFOUND... */
+			newpa->num = mirrorfaces[pa_num * 2];
 
-			if ((newpa->num_dmcache != DMCACHE_NOTFOUND) && psys->part->use_modifier_stack && !psmd->dm->deformedOnly) {
-				newpa->num = newpa->num_dmcache;
+			if (use_dm_final_indices) {
 				newpa->num_dmcache = DMCACHE_ISCHILD;
 			}
+			else {
+				newpa->num_dmcache = psys_particle_dm_face_lookup(
+				                         psmd->dm_final, psmd->dm_deformed, newpa->num, newpa->fuv, NULL);
+			}
 
 			/* update edit key pointers */
 			key= newpoint->keys;
@@ -2912,7 +2925,7 @@ static void PE_mirror_x(Scene *scene, Object *ob, int tagged)
 			}
 
 			/* map key positions as mirror over x axis */
-			PE_mirror_particle(ob, psmd->dm, psys, pa, newpa);
+			PE_mirror_particle(ob, psmd->dm_final, psys, pa, newpa);
 
 			newpa++;
 			newpoint++;
@@ -3360,7 +3373,7 @@ static int particle_intersect_dm(Scene *scene, Object *ob, DerivedMesh *dm,
 	totface=dm->getNumTessFaces(dm);
 	mface=dm->getTessFaceDataArray(dm, CD_MFACE);
 	mvert=dm->getVertDataArray(dm, CD_MVERT);
-	
+
 	/* lets intersect the faces */
 	for (i=0; i<totface; i++, mface++) {
 		if (vert_cos) {
@@ -3452,6 +3465,7 @@ static int brush_add(PEData *data, short number)
 {
 	Scene *scene= data->scene;
 	Object *ob= data->ob;
+	DerivedMesh *dm;
 	PTCacheEdit *edit = data->edit;
 	ParticleSystem *psys= edit->psys;
 	ParticleData *add_pars;
@@ -3465,11 +3479,7 @@ static int brush_add(PEData *data, short number)
 	float framestep, timestep;
 	short size= pset->brush[PE_BRUSH_ADD].size;
 	short size2= size*size;
-	DerivedMesh *dm=0;
 	RNG *rng;
-	const int *index_mf_to_mpoly;
-	const int *index_mp_to_orig;
-	bool release_dm = false;
 
 	invert_m4_m4(imat, ob->obmat);
 
@@ -3487,15 +3497,13 @@ static int brush_add(PEData *data, short number)
 
 	timestep= psys_get_timestep(&sim);
 
-	if (psmd->dm->deformedOnly || psys->part->use_modifier_stack)
-		dm = psmd->dm;
+	if (psys->part->use_modifier_stack || psmd->dm_final->deformedOnly) {
+		dm = psmd->dm_final;
+	}
 	else {
-		dm = mesh_get_derived_deform(scene, ob, CD_MASK_BAREMESH | CD_MASK_MFACE);
-		release_dm = true;
+		dm = psmd->dm_deformed;
 	}
-
-	index_mf_to_mpoly = dm->getTessFaceDataArray(dm, CD_ORIGINDEX);
-	index_mp_to_orig  = dm->getPolyDataArray(dm, CD_ORIGINDEX);
+	BLI_assert(dm);
 
 	for (i=0; i<number; i++) {
 		if (number>1) {
@@ -3523,16 +3531,22 @@ static int brush_add(PEData *data, short number)
 		
 		/* warning, returns the derived mesh face */
 		if (particle_intersect_dm(scene, ob, dm, 0, co1, co2, &min_d, &add_pars[n].num_dmcache, add_pars[n].fuv, 0, 0, 0, 0)) {
-			if (index_mf_to_mpoly && index_mp_to_orig)
-				add_pars[n].num = DM_origindex_mface_mpoly(index_mf_to_mpoly, index_mp_to_orig, add_pars[n].num_dmcache);
-			else
+			if (psys->part->use_modifier_stack && !psmd->dm_final->deformedOnly) {
+				add_pars[n].num = add_pars[n].num_dmcache;
+				add_pars[n].num_dmcache = DMCACHE_ISCHILD;
+			}
+			else if (dm == psmd->dm_deformed) {
+				/* Final DM is not same topology as orig mesh, we have to map num_dmcache to real final dm. */
+				add_pars[n].num = add_pars[n].num_dmcache;
+				add_pars[n].num_dmcache = psys_particle_dm_face_lookup(
+				                              psmd->dm_final, psmd->dm_deformed,
+				                              add_pars[n].num, add_pars[n].fuv, NULL);
+			}
+			else {
 				add_pars[n].num = add_pars[n].num_dmcache;
+			}
 
-			if (psys_particle_dm_face_lookup(ob, psmd->dm, add_pars[n].num_dmcache, add_pars[n].fuv, NULL) != DMCACHE_NOTFOUND) {
-				if (psys->part->use_modifier_stack && !psmd->dm->deformedOnly) {
-					add_pars[n].num = add_pars[n].num_dmcache;
-					add_pars[n].num_dmcache = DMCACHE_ISCHILD;
-				}
+			if (add_pars[n].num != DMCACHE_NOTFOUND) {
 				n++;
 			}
 		}
@@ -3567,7 +3581,7 @@ static int brush_add(PEData *data, short number)
 			tree=BLI_kdtree_new(psys->totpart);
 			
 			for (i=0, pa=psys->particles; i<totpart; i++, pa++) {
-				psys_particle_on_dm(psmd->dm, psys->part->from, pa->num, pa->num_dmcache, pa->fuv, pa->foffset, cur_co, 0, 0, 0, 0, 0);
+				psys_particle_on_dm(psmd->dm_final, psys->part->from, pa->num, pa->num_dmcache, pa->fuv, pa->foffset, cur_co, 0, 0, 0, 0, 0);
 				BLI_kdtree_insert(tree, i, cur_co);
 			}
 
@@ -3611,7 +3625,7 @@ static int brush_add(PEData *data, short number)
 				int w, maxw;
 				float maxd, totw=0.0, weight[3];
 
-				psys_particle_on_dm(psmd->dm, psys->part->from, pa->num, pa->num_dmcache, pa->fuv, pa->foffset, co1, 0, 0, 0, 0, 0);
+				psys_particle_on_dm(psmd->dm_final, psys->part->from, pa->num, pa->num_dmcache, pa->fuv, pa->foffset, co1, 0, 0, 0, 0, 0);
 				maxw = BLI_kdtree_find_nearest_n(tree, co1, ptn, 3);
 
 				maxd= ptn[maxw-1].dist;
@@ -3676,7 +3690,7 @@ static int brush_add(PEData *data, short number)
 				}
 			}
 			for (k=0, hkey=pa->hair; k<pset->totaddkey; k++, hkey++) {
-				psys_mat_hair_to_global(ob, psmd->dm, psys->part->from, pa, hairmat);
+				psys_mat_hair_to_global(ob, psmd->dm_final, psys->part->from, pa, hairmat);
 				invert_m4_m4(imat, hairmat);
 				mul_m4_v3(imat, hkey->co);
 			}
@@ -3688,9 +3702,6 @@ static int brush_add(PEData *data, short number)
 
 	MEM_freeN(add_pars);
 
-	if (release_dm)
-		dm->release(dm);
-
 	BLI_rng_free(rng);
 	
 	return n;
@@ -3866,7 +3877,7 @@ static void brush_edit_apply(bContext *C, wmOperator *op, PointerRNA *itemptr)
 				case PE_BRUSH_PUFF:
 				{
 					if (edit->psys) {
-						data.dm= psmd->dm;
+						data.dm= psmd->dm_final;
 						data.mval= mval;
 						data.rad= pe_brush_size_get(scene, brush);
 						data.select= selected;
@@ -3922,7 +3933,7 @@ static void brush_edit_apply(bContext *C, wmOperator *op, PointerRNA *itemptr)
 				case PE_BRUSH_WEIGHT:
 				{
 					if (edit->psys) {
-						data.dm= psmd->dm;
+						data.dm= psmd->dm_final;
 						data.mval= mval;
 						data.rad= pe_brush_size_get(scene, brush);
 
@@ -4545,7 +4556,7 @@ int PE_minmax(Scene *scene, float min[3], float max[3])
 
 	LOOP_VISIBLE_POINTS {
 		if (psys)
-			psys_mat_hair_to_global(ob, psmd->dm, psys->part->from, psys->particles+p, mat);
+			psys_mat_hair_to_global(ob, psmd->dm_final, psys->part->from, psys->particles+p, mat);
 
 		LOOP_SELECTED_KEYS {
 			copy_v3_v3(co, key->co);
@@ -4576,7 +4587,7 @@ void PE_create_particle_edit(Scene *scene, Object *ob, PointCache *cache, Partic
 	int totpoint;
 
 	/* no psmd->dm happens in case particle system modifier is not enabled */
-	if (!(psys && psmd && psmd->dm) && !cache)
+	if (!(psys && psmd && psmd->dm_final) && !cache)
 		return;
 
 	if (cache && cache->flag & PTCACHE_DISK_CACHE)