Fix T87259: Un-Subdivide creates duplicate faces

Add argument to BM_vert_collapse_faces to remove any faces that become
duplicate as result of the collapse.

4 files changed, 46 insertions, 17 deletions

diff --git a/source/blender/bmesh/intern/bmesh_core.c b/source/blender/bmesh/intern/bmesh_core.c
index cf907862120..e72c689ddfb 100644
--- a/source/blender/bmesh/intern/bmesh_core.c
+++ b/source/blender/bmesh/intern/bmesh_core.c
@@ -1805,7 +1805,8 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
                                          BMVert *v_kill,
                                          const bool do_del,
                                          const bool check_edge_exists,
-                                         const bool kill_degenerate_faces)
+                                         const bool kill_degenerate_faces,
+                                         const bool kill_duplicate_faces)
 {
   BMEdge *e_old;
   BMVert *v_old, *v_target;
@@ -1840,6 +1841,9 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
     BLI_SMALLSTACK_DECLARE(faces_degenerate, BMFace *);
     BMLoop *l_kill_next;
 
+    /* Candidates for being duplicate. */
+    BLI_SMALLSTACK_DECLARE(faces_duplicate_candidate, BMFace *);
+
 #ifndef NDEBUG
     /* For verification later, count valence of 'v_old' and 'v_target' */
     valence1 = bmesh_disk_count(v_old);
@@ -1877,9 +1881,14 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
 
         /* fix len attribute of face */
         l_kill->f->len--;
-        if (kill_degenerate_faces) {
-          if (l_kill->f->len < 3) {
-            BLI_SMALLSTACK_PUSH(faces_degenerate, l_kill->f);
+        if (kill_degenerate_faces && (l_kill->f->len < 3)) {
+          BLI_SMALLSTACK_PUSH(faces_degenerate, l_kill->f);
+        }
+        else {
+          /* The duplicate test isn't reliable at this point as `e_splice` might be set,
+           * so the duplicate test needs to run once the edge has been spliced. */
+          if (kill_duplicate_faces) {
+            BLI_SMALLSTACK_PUSH(faces_duplicate_candidate, l_kill->f);
           }
         }
         l_kill_next = l_kill->radial_next;
@@ -1940,6 +1949,15 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
       }
     }
 
+    if (kill_duplicate_faces) {
+      BMFace *f_kill;
+      while ((f_kill = BLI_SMALLSTACK_POP(faces_duplicate_candidate))) {
+        if (BM_face_find_double(f_kill)) {
+          BM_face_kill(bm, f_kill);
+        }
+      }
+    }
+
     BM_CHECK_ELEMENT(v_old);
     BM_CHECK_ELEMENT(v_target);
     BM_CHECK_ELEMENT(e_old);
diff --git a/source/blender/bmesh/intern/bmesh_core.h b/source/blender/bmesh/intern/bmesh_core.h
index df73984e6cf..8f7580714ae 100644
--- a/source/blender/bmesh/intern/bmesh_core.h
+++ b/source/blender/bmesh/intern/bmesh_core.h
@@ -115,7 +115,8 @@ BMEdge *bmesh_kernel_join_edge_kill_vert(BMesh *bm,
                                          BMVert *v_kill,
                                          const bool do_del,
                                          const bool check_edge_exists,
-                                         const bool kill_degenerate_faces);
+                                         const bool kill_degenerate_faces,
+                                         const bool kill_duplicate_faces);
 BMVert *bmesh_kernel_join_vert_kill_edge(BMesh *bm,
                                          BMEdge *e_kill,
                                          BMVert *v_kill,
diff --git a/source/blender/bmesh/intern/bmesh_mods.c b/source/blender/bmesh/intern/bmesh_mods.c
index 8e5ed9c3bf0..76e32667804 100644
--- a/source/blender/bmesh/intern/bmesh_mods.c
+++ b/source/blender/bmesh/intern/bmesh_mods.c
@@ -72,7 +72,7 @@ bool BM_vert_dissolve(BMesh *bm, BMVert *v)
     }
     if (!v->e->l) {
       if (len == 2) {
-        return (BM_vert_collapse_edge(bm, v->e, v, true, true) != NULL);
+        return (BM_vert_collapse_edge(bm, v->e, v, true, true, true) != NULL);
       }
       /* used to kill the vertex here, but it may be connected to faces.
        * so better do nothing */
@@ -82,7 +82,7 @@ bool BM_vert_dissolve(BMesh *bm, BMVert *v)
   }
   if (len == 2 && BM_vert_face_count_is_equal(v, 1)) {
     /* boundary vertex on a face */
-    return (BM_vert_collapse_edge(bm, v->e, v, true, true) != NULL);
+    return (BM_vert_collapse_edge(bm, v->e, v, true, true, true) != NULL);
   }
   return BM_disk_dissolve(bm, v);
 }
@@ -133,7 +133,7 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
     if (UNLIKELY(!BM_faces_join_pair(bm, e->l, e->l->radial_next, true))) {
       return false;
     }
-    if (UNLIKELY(!BM_vert_collapse_faces(bm, v->e, v, 1.0, true, false, true))) {
+    if (UNLIKELY(!BM_vert_collapse_faces(bm, v->e, v, 1.0, true, false, true, true))) {
       return false;
     }
 #endif
@@ -141,7 +141,7 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
   }
   if (keepedge == NULL && len == 2) {
     /* collapse the vertex */
-    e = BM_vert_collapse_faces(bm, v->e, v, 1.0, true, true, true);
+    e = BM_vert_collapse_faces(bm, v->e, v, 1.0, true, true, true, true);
 
     if (!e) {
       return false;
@@ -184,7 +184,8 @@ bool BM_disk_dissolve(BMesh *bm, BMVert *v)
 
     /* collapse the vertex */
     /* note, the baseedge can be a boundary of manifold, use this as join_faces arg */
-    e = BM_vert_collapse_faces(bm, baseedge, v, 1.0, true, !BM_edge_is_boundary(baseedge), true);
+    e = BM_vert_collapse_faces(
+        bm, baseedge, v, 1.0, true, !BM_edge_is_boundary(baseedge), true, true);
 
     if (!e) {
       return false;
@@ -432,7 +433,8 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
                                float fac,
                                const bool do_del,
                                const bool join_faces,
-                               const bool kill_degenerate_faces)
+                               const bool kill_degenerate_faces,
+                               const bool kill_duplicate_faces)
 {
   BMEdge *e_new = NULL;
   BMVert *tv = BM_edge_other_vert(e_kill, v_kill);
@@ -503,7 +505,8 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
     /* same as BM_vert_collapse_edge() however we already
      * have vars to perform this operation so don't call. */
     e_new = bmesh_kernel_join_edge_kill_vert(
-        bm, e_kill, v_kill, do_del, true, kill_degenerate_faces);
+        bm, e_kill, v_kill, do_del, true, kill_degenerate_faces, kill_duplicate_faces);
+
     /* e_new = BM_edge_exists(tv, tv2); */ /* same as return above */
   }
 
@@ -517,8 +520,12 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
  *
  * \return The New Edge
  */
-BMEdge *BM_vert_collapse_edge(
-    BMesh *bm, BMEdge *e_kill, BMVert *v_kill, const bool do_del, const bool kill_degenerate_faces)
+BMEdge *BM_vert_collapse_edge(BMesh *bm,
+                              BMEdge *e_kill,
+                              BMVert *v_kill,
+                              const bool do_del,
+                              const bool kill_degenerate_faces,
+                              const bool kill_duplicate_faces)
 {
   /* nice example implementation but we want loops to have their customdata
    * accounted for */
@@ -546,7 +553,8 @@ BMEdge *BM_vert_collapse_edge(
 #else
   /* with these args faces are never joined, same as above
    * but account for loop customdata */
-  return BM_vert_collapse_faces(bm, e_kill, v_kill, 1.0f, do_del, false, kill_degenerate_faces);
+  return BM_vert_collapse_faces(
+      bm, e_kill, v_kill, 1.0f, do_del, false, kill_degenerate_faces, kill_duplicate_faces);
 #endif
 }
 
diff --git a/source/blender/bmesh/intern/bmesh_mods.h b/source/blender/bmesh/intern/bmesh_mods.h
index 7491c309754..4328187b95e 100644
--- a/source/blender/bmesh/intern/bmesh_mods.h
+++ b/source/blender/bmesh/intern/bmesh_mods.h
@@ -51,12 +51,14 @@ BMEdge *BM_vert_collapse_faces(BMesh *bm,
                                float fac,
                                const bool do_del,
                                const bool join_faces,
-                               const bool kill_degenerate_faces);
+                               const bool kill_degenerate_faces,
+                               const bool kill_duplicate_faces);
 BMEdge *BM_vert_collapse_edge(BMesh *bm,
                               BMEdge *e_kill,
                               BMVert *v_kill,
                               const bool do_del,
-                              const bool kill_degenerate_faces);
+                              const bool kill_degenerate_faces,
+                              const bool kill_duplicate_faces);
 
 BMVert *BM_edge_collapse(BMesh *bm,
                          BMEdge *e_kill,