Fix T87259: Un-Subdivide creates duplicate faces

Add argument to BM_vert_collapse_faces to remove any faces that become
duplicate as result of the collapse.

9 files changed, 54 insertions, 25 deletions

diff --git a/source/blender/bmesh/intern/bmesh_core.c b/source/blender/bmesh/intern/bmesh_core.c
index cf907862120..e72c689ddfb 100644
--- a/source/blender/bmesh/intern/bmesh_core.c
+++ b/source/blender/bmesh/intern/bmesh_core.c
@@ -1805,7 +1805,8 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
                                          BMVert *v_kill,
                                          const bool do_del,
                                          const bool check_edge_exists,
-                                         const bool kill_degenerate_faces)
+                                         const bool kill_degenerate_faces,
+                                         const bool kill_duplicate_faces)
 {
   BMEdge *e_old;
   BMVert *v_old, *v_target;
@@ -1840,6 +1841,9 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
     BLI_SMALLSTACK_DECLARE(faces_degenerate, BMFace *);
     BMLoop *l_kill_next;
 
+    /* Candidates for being duplicate. */
+    BLI_SMALLSTACK_DECLARE(faces_duplicate_candidate, BMFace *);
+
 #ifndef NDEBUG
     /* For verification later, count valence of 'v_old' and 'v_target' */
     valence1 = bmesh_disk_count(v_old);
@@ -1877,9 +1881,14 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
 
         /* fix len attribute of face */
         l_kill->f->len--;
-        if (kill_degenerate_faces) {
-          if (l_kill->f->len < 3) {
-            BLI_SMALLSTACK_PUSH(faces_degenerate, l_kill->f);
+        if (kill_degenerate_faces && (l_kill->f->len < 3)) {
+          BLI_SMALLSTACK_PUSH(faces_degenerate, l_kill->f);
+        }
+        else {
+          /* The duplicate test isn't reliable at this point as `e_splice` might be set,
+           * so the duplicate test needs to run once the edge has been spliced. */
+          if (kill_duplicate_faces) {
+            BLI_SMALLSTACK_PUSH(faces_duplicate_candidate, l_kill->f);
           }
         }
         l_kill_next = l_kill->radial_next;
@@ -1940,6 +1949,15 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
       }
     }
 
+    if (kill_duplicate_faces) {
+      BMFace *f_kill;
+      while ((f_kill = BLI_SMALLSTACK_POP(faces_duplicate_candidate))) {
+        if (BM_face_find_double(f_kill)) {
+          BM_face_kill(bm, f_kill);
+        }
+      }
+    }
+
     BM_CHECK_ELEMENT(v_old);
     BM_CHECK_ELEMENT(v_target);
     BM_CHECK_ELEMENT(e_old);
diff --git a/source/blender/bmesh/intern/bmesh_core.h b/source/blender/bmesh/intern/bmesh_core.h
index df73984e6cf..8f7580714ae 100644
--- a/source/blender/bmesh/intern/bmesh_core.h
+++ b/source/blender/bmesh/intern/bmesh_core.h
@@ -115,7 +115,8 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
                                          BMVert *v_kill,
                                          const bool do_del,
                                          const bool check_edge_exists,
-                                         const bool kill_degenerate_faces);
+                                         const bool kill_degenerate_faces,
+                                         const bool kill_duplicate_faces);
 BMVert *bmesh_kernel_join_vert_kill_edge(BMesh *bm,
                                          BMEdge *e_kill,
                                          BMVert *v_kill,
diff --git a/source/blender/bmesh/intern/bmesh_mods.c b/source/blender/bmesh/intern/bmesh_mods.c
index 8e5ed9c3bf0..76e32667804 100644
--- a/source/blender/bmesh/intern/bmesh_mods.c
+++ b/source/blender/bmesh/intern/bmesh_mods.c
@@ -72,7 +72,7 @@ bool BM_vert_dissolve(BMesh *bm, BMVert *v)
     }
     if (!v->e->l) {
       if (len == 2) {
-        return (BM_vert_collapse_edge(bm, v->e, v, true, true) != NULL);
+        return (BM_vert_collapse_edge(bm, v->e, v, true, true, true) != NULL);
       }
       /* used to kill the vertex here, but it may be connected to faces.
        * so better do nothing */
@@ -82,7 +82,7 @@ bool BM_vert_dissolve(BMesh *bm, BMVert *v)
   }
   if (len == 2 && BM_vert_face_count_is_equal(v, 1)) {
     /* boundary vertex on a face */
-    return (BM_vert_collapse_edge(bm, v->e, v, true, true) != NULL);
+    return (BM_vert_collapse_edge(bm, v->e, v, true, true, true) != NULL);
   }
   return BM_disk_dissolve(bm, v);
 }
@@ -133,7 +133,7 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
     if (UNLIKELY(!BM_faces_join_pair(bm, e->l, e->l->radial_next, true))) {
       return false;
     }
-    if (UNLIKELY(!BM_vert_collapse_faces(bm, v->e, v, 1.0, true, false, true))) {
+    if (UNLIKELY(!BM_vert_collapse_faces(bm, v->e, v, 1.0, true, false, true, true))) {
       return false;
     }
 #endif
@@ -141,7 +141,7 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
   }
   if (keepedge == NULL && len == 2) {
     /* collapse the vertex */
-    e = BM_vert_collapse_faces(bm, v->e, v, 1.0, true, true, true);
+    e = BM_vert_collapse_faces(bm, v->e, v, 1.0, true, true, true, true);
 
     if (!e) {
       return false;
@@ -184,7 +184,8 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
 
     /* collapse the vertex */
     /* note, the baseedge can be a boundary of manifold, use this as join_faces arg */
-    e = BM_vert_collapse_faces(bm, baseedge, v, 1.0, true, !BM_edge_is_boundary(baseedge), true);
+    e = BM_vert_collapse_faces(
+        bm, baseedge, v, 1.0, true, !BM_edge_is_boundary(baseedge), true, true);
 
     if (!e) {
       return false;
@@ -432,7 +433,8 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
                                float fac,
                                const bool do_del,
                                const bool join_faces,
-                               const bool kill_degenerate_faces)
+                               const bool kill_degenerate_faces,
+                               const bool kill_duplicate_faces)
 {
   BMEdge *e_new = NULL;
   BMVert *tv = BM_edge_other_vert(e_kill, v_kill);
@@ -503,7 +505,8 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
     /* same as BM_vert_collapse_edge() however we already
      * have vars to perform this operation so don't call. */
     e_new = bmesh_kernel_join_edge_kill_vert(
-        bm, e_kill, v_kill, do_del, true, kill_degenerate_faces);
+        bm, e_kill, v_kill, do_del, true, kill_degenerate_faces, kill_duplicate_faces);
+
     /* e_new = BM_edge_exists(tv, tv2); */ /* same as return above */
   }
 
@@ -517,8 +520,12 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
  *
  * \return The New Edge
  */
-BMEdge *BM_vert_collapse_edge(
-    BMesh *bm, BMEdge *e_kill, BMVert *v_kill, const bool do_del, const bool kill_degenerate_faces)
+BMEdge *BM_vert_collapse_edge(BMesh *bm,
+                              BMEdge *e_kill,
+                              BMVert *v_kill,
+                              const bool do_del,
+                              const bool kill_degenerate_faces,
+                              const bool kill_duplicate_faces)
 {
   /* nice example implementation but we want loops to have their customdata
    * accounted for */
@@ -546,7 +553,8 @@ BMEdge *BM_vert_collapse_edge(
 #else
   /* with these args faces are never joined, same as above
    * but account for loop customdata */
-  return BM_vert_collapse_faces(bm, e_kill, v_kill, 1.0f, do_del, false, kill_degenerate_faces);
+  return BM_vert_collapse_faces(
+      bm, e_kill, v_kill, 1.0f, do_del, false, kill_degenerate_faces, kill_duplicate_faces);
 #endif
 }
 
diff --git a/source/blender/bmesh/intern/bmesh_mods.h b/source/blender/bmesh/intern/bmesh_mods.h
index 7491c309754..4328187b95e 100644
--- a/source/blender/bmesh/intern/bmesh_mods.h
+++ b/source/blender/bmesh/intern/bmesh_mods.h
@@ -51,12 +51,14 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
                                float fac,
                                const bool do_del,
                                const bool join_faces,
-                               const bool kill_degenerate_faces);
+                               const bool kill_degenerate_faces,
+                               const bool kill_duplicate_faces);
 BMEdge *BM_vert_collapse_edge(BMesh *bm,
                               BMEdge *e_kill,
                               BMVert *v_kill,
                               const bool do_del,
-                              const bool kill_degenerate_faces);
+                              const bool kill_degenerate_faces,
+                              const bool kill_duplicate_faces);
 
 BMVert *BM_edge_collapse(BMesh *bm,
                          BMEdge *e_kill,
diff --git a/source/blender/bmesh/operators/bmo_dissolve.c b/source/blender/bmesh/operators/bmo_dissolve.c
index da2603ad8cd..6723c0f7cb0 100644
--- a/source/blender/bmesh/operators/bmo_dissolve.c
+++ b/source/blender/bmesh/operators/bmo_dissolve.c
@@ -242,7 +242,7 @@ void bmo_dissolve_faces_exec(BMesh *bm, BMOperator *op)
     BM_ITER_MESH_MUTABLE (v, v_next, &viter, bm, BM_VERTS_OF_MESH) {
       if (BMO_vert_flag_test(bm, v, VERT_MARK)) {
         if (BM_vert_is_edge_pair(v)) {
-          BM_vert_collapse_edge(bm, v->e, v, true, true);
+          BM_vert_collapse_edge(bm, v->e, v, true, true, true);
         }
       }
     }
@@ -355,7 +355,7 @@ void bmo_dissolve_edges_exec(BMesh *bm, BMOperator *op)
     BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
       if (BMO_vert_flag_test(bm, v, VERT_MARK)) {
         if (BM_vert_is_edge_pair(v)) {
-          BM_vert_collapse_edge(bm, v->e, v, true, true);
+          BM_vert_collapse_edge(bm, v->e, v, true, true, true);
         }
       }
     }
@@ -462,7 +462,7 @@ void bmo_dissolve_verts_exec(BMesh *bm, BMOperator *op)
   /* final cleanup */
   BMO_ITER (v, &oiter, op->slots_in, "verts", BM_VERT) {
     if (BM_vert_is_edge_pair(v)) {
-      BM_vert_collapse_edge(bm, v->e, v, false, true);
+      BM_vert_collapse_edge(bm, v->e, v, false, true, true);
     }
   }
 
diff --git a/source/blender/bmesh/operators/bmo_offset_edgeloops.c b/source/blender/bmesh/operators/bmo_offset_edgeloops.c
index d723e128bf1..2c7e478b549 100644
--- a/source/blender/bmesh/operators/bmo_offset_edgeloops.c
+++ b/source/blender/bmesh/operators/bmo_offset_edgeloops.c
@@ -263,7 +263,7 @@ void bmo_offset_edgeloops_exec(BMesh *bm, BMOperator *op)
       }
 
       while ((v = STACK_POP(varr))) {
-        bmesh_kernel_join_edge_kill_vert(bm, v->e, v, true, false, false);
+        bmesh_kernel_join_edge_kill_vert(bm, v->e, v, true, false, false, true);
       }
     }
   }
diff --git a/source/blender/bmesh/tools/bmesh_decimate_dissolve.c b/source/blender/bmesh/tools/bmesh_decimate_dissolve.c
index 4a024f745ed..8b4a9bb26ac 100644
--- a/source/blender/bmesh/tools/bmesh_decimate_dissolve.c
+++ b/source/blender/bmesh/tools/bmesh_decimate_dissolve.c
@@ -439,7 +439,7 @@ void BM_mesh_decimate_dissolve_ex(BMesh *bm,
     for (i = 0; i < vinput_len; i++) {
       BMVert *v = vinput_arr[i];
       if (LIKELY(v != NULL) && BM_vert_is_edge_pair(v)) {
-        BM_vert_collapse_edge(bm, v->e, v, true, true); /* join edges */
+        BM_vert_collapse_edge(bm, v->e, v, true, true, true); /* join edges */
       }
     }
   }
@@ -482,7 +482,7 @@ void BM_mesh_decimate_dissolve_ex(BMesh *bm,
           BM_vert_is_edge_pair(v)
 #endif
       ) {
-        e_new = BM_vert_collapse_edge(bm, v->e, v, true, true); /* join edges */
+        e_new = BM_vert_collapse_edge(bm, v->e, v, true, true, true); /* join edges */
 
         if (e_new) {
 
diff --git a/source/blender/bmesh/tools/bmesh_decimate_unsubdivide.c b/source/blender/bmesh/tools/bmesh_decimate_unsubdivide.c
index 0a512fdd592..c96a7be1adf 100644
--- a/source/blender/bmesh/tools/bmesh_decimate_unsubdivide.c
+++ b/source/blender/bmesh/tools/bmesh_decimate_unsubdivide.c
@@ -110,7 +110,7 @@ static bool bm_vert_dissolve_fan(BMesh *bm, BMVert *v)
   if (tot_edge == 2) {
     /* check for 2 wire verts only */
     if (tot_edge_wire == 2) {
-      return (BM_vert_collapse_edge(bm, v->e, v, true, true) != NULL);
+      return (BM_vert_collapse_edge(bm, v->e, v, true, true, true) != NULL);
     }
   }
   else if (tot_edge == 4) {
diff --git a/source/blender/bmesh/tools/bmesh_intersect.c b/source/blender/bmesh/tools/bmesh_intersect.c
index 81b016e9601..c176210426b 100644
--- a/source/blender/bmesh/tools/bmesh_intersect.c
+++ b/source/blender/bmesh/tools/bmesh_intersect.c
@@ -1622,7 +1622,7 @@ bool BM_mesh_intersect(BMesh *bm,
           }
 
           if (ok) {
-            BM_vert_collapse_edge(bm, v->e, v, true, false);
+            BM_vert_collapse_edge(bm, v->e, v, true, false, false);
           }
         }
       }