fix [#31235] Limited Dissolve problems

this is in fact 2 bugs.
- unselected edges between 2 faces that were joined didnt get removed.
- in face mode, edges and verts at the boundary of the selection would get incorrectly dissolved.

also quiet float/double promotion warning.

2 files changed, 67 insertions, 15 deletions

diff --git a/source/blender/bmesh/operators/bmo_dissolve.c b/source/blender/bmesh/operators/bmo_dissolve.c
index 8e7723fefdd..ae1773af05e 100644
--- a/source/blender/bmesh/operators/bmo_dissolve.c
+++ b/source/blender/bmesh/operators/bmo_dissolve.c
@@ -478,8 +478,8 @@ void dummy_exec(BMesh *bm, BMOperator *op)
 
 /* Limited Dissolve */
 
-#define UNIT_TO_ANGLE DEG2RADF(90.0)
-#define ANGLE_TO_UNIT (1.0 / UNIT_TO_ANGLE)
+#define UNIT_TO_ANGLE DEG2RADF(90.0f)
+#define ANGLE_TO_UNIT (1.0f / UNIT_TO_ANGLE)
 
 /* multiply vertex edge angle by face angle
  * this means we are not left with sharp corners between _almost_ planer faces
@@ -523,8 +523,17 @@ void bmo_dissolve_limit_exec(BMesh *bm, BMOperator *op)
 	                                                 sizeof(DissolveElemWeight), __func__);
 	int i, tot_found;
 
+	BMIter iter;
+	BMEdge *e_iter;
+	BMEdge **earray;
+
 	/* --- first edges --- */
 
+	/* wire -> tag */
+	BM_ITER_MESH(e_iter, &iter, bm, BM_EDGES_OF_MESH) {
+		BM_elem_flag_set(e_iter, BM_ELEM_TAG, BM_edge_is_wire(e_iter));
+	}
+
 	/* go through and split edge */
 	for (i = 0, tot_found = 0; i < einput->len; i++) {
 		BMEdge *e = ((BMEdge **)einput->data.p)[i];
@@ -562,18 +571,6 @@ void bmo_dissolve_limit_exec(BMesh *bm, BMOperator *op)
 				}
 			}
 		}
-
-		/* remove all edges/verts left behind from dissolving */
-		for (i = 0; i < einput->len; i++) {
-			BMEdge *e = (BMEdge *)weight_elems[i].ele;
-			if (BM_edge_is_wire(e)) {
-				BMVert *v1 = e->v1;
-				BMVert *v2 = e->v2;
-				BM_edge_kill(bm, e);
-				if (v1->e == NULL) BM_vert_kill(bm, v1);
-				if (v2->e == NULL) BM_vert_kill(bm, v2);
-			}
-		}
 	}
 
 	/* --- second verts --- */
@@ -612,4 +609,25 @@ void bmo_dissolve_limit_exec(BMesh *bm, BMOperator *op)
 	}
 
 	MEM_freeN(weight_elems);
+
+	/* --- cleanup --- */
+	earray = MEM_mallocN(sizeof(BMEdge *) * bm->totedge, __func__);
+	BM_ITER_MESH_INDEX(e_iter, &iter, bm, BM_EDGES_OF_MESH, i) {
+		earray[i] = e_iter;
+	}
+	/* remove all edges/verts left behind from dissolving */
+	for (i = bm->totedge - 1; i != -1; i--) {
+		e_iter = earray[i];
+
+		if (BM_edge_is_wire(e_iter) && (BM_elem_flag_test(e_iter, BM_ELEM_TAG) == FALSE)) {
+			/* edge has become wire */
+			BMVert *v1 = e_iter->v1;
+			BMVert *v2 = e_iter->v2;
+			BM_edge_kill(bm, e_iter);
+			if (v1->e == NULL) BM_vert_kill(bm, v1);
+			if (v2->e == NULL) BM_vert_kill(bm, v2);
+		}
+	}
+
+	MEM_freeN(earray);
 }
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 524ee029339..96cfd95b96a 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -3205,11 +3205,45 @@ static int edbm_dissolve_limited_exec(bContext *C, wmOperator *op)
 {
 	Object *obedit = CTX_data_edit_object(C);
 	BMEditMesh *em = BMEdit_FromObject(obedit);
+	BMesh *bm = em->bm;
 	float angle_limit = RNA_float_get(op->ptr, "angle_limit");
 
+	char dissolve_flag;
+
+	if (em->selectmode == SCE_SELECT_FACE) {
+		/* flush selection to tags and untag edges/verts with partially selected faces */
+		BMIter iter;
+		BMIter liter;
+
+		BMElem *ele;
+		BMFace *f;
+		BMLoop *l;
+
+		BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
+			BM_elem_flag_set(ele, BM_ELEM_TAG, BM_elem_flag_test(ele, BM_ELEM_SELECT));
+		}
+		BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
+			BM_elem_flag_set(ele, BM_ELEM_TAG, BM_elem_flag_test(ele, BM_ELEM_SELECT));
+		}
+
+		BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
+			if (!BM_elem_flag_test(f, BM_ELEM_SELECT)) {
+				BM_ITER_ELEM (l, &liter, f, BM_LOOPS_OF_FACE) {
+					BM_elem_flag_disable(l->v, BM_ELEM_TAG);
+					BM_elem_flag_disable(l->e, BM_ELEM_TAG);
+				}
+			}
+		}
+
+		dissolve_flag = BM_ELEM_TAG;
+	}
+	else {
+		dissolve_flag = BM_ELEM_SELECT;
+	}
+
 	if (!EDBM_op_callf(em, op,
 	                   "dissolve_limit edges=%he verts=%hv angle_limit=%f",
-	                   BM_ELEM_SELECT, BM_ELEM_SELECT, angle_limit))
+	                   dissolve_flag, dissolve_flag, angle_limit))
 	{
 		return OPERATOR_CANCELLED;
 	}