BMesh: remove redundant mesh-backups from EDBM_op_* API

Using BMesh operators through the edit-mesh API created a full copy
of the mesh so it was possible to restore the mesh in case
one of the operators raised an error.

Remove support for automatic backup/restore from the EDBM_op_* API's
as it adds significant overhead and was rarely used.

Operators that need this can use the BMBackup API to backup & restore
the mesh in case of failure.

Add warning levels to BMO_error_raise so operators can report problems
without it being interpreted as a request to cancel the operation.

For high-poly meshes creating and freeing a full copy is an expensive
operation, removing this gives a speedup of ~1.77x for most operators
except for "connect_verts" / "connect_vert_pair"
which still uses this functionality.

2 files changed, 64 insertions, 65 deletions

diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index e1f192bea77..0917d677691 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -1264,9 +1264,12 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
     }
   }
   if (checks_succeded) {
+    BMBackup em_backup = EDBM_redo_state_store(em);
+
     BM_custom_loop_normals_to_vector_layer(bm);
 
     BMO_op_exec(bm, &bmop);
+    const bool failure = BMO_error_occurred_at_level(bm, BMO_ERROR_FATAL);
     len = BMO_slot_get(bmop.slots_out, "edges.out")->len;
 
     if (len && is_pair) {
@@ -1275,8 +1278,14 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
           em->bm, bmop.slots_out, "edges.out", BM_EDGE, BM_ELEM_SELECT, true);
     }
 
-    if (!EDBM_op_finish(em, &bmop, op, true)) {
+    bool em_backup_free = true;
+    if (!EDBM_op_finish(em, &bmop, op, false)) {
+      len = 0;
+    }
+    else if (failure) {
       len = 0;
+      EDBM_redo_state_free(&em_backup, em, true);
+      em_backup_free = false;
     }
     else {
       /* so newly created edges get the selection state from the vertex */
@@ -1291,6 +1300,10 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
                       .is_destructive = true,
                   });
     }
+
+    if (em_backup_free) {
+      EDBM_redo_state_free(&em_backup, NULL, false);
+    }
   }
   MEM_freeN(verts);
 
@@ -7304,7 +7317,7 @@ static int edbm_bridge_edge_loops_for_single_editmesh(wmOperator *op,
 
   BMO_op_exec(em->bm, &bmop);
 
-  if (!BMO_error_occurred(em->bm)) {
+  if (!BMO_error_occurred_at_level(em->bm, BMO_ERROR_CANCEL)) {
     /* when merge is used the edges are joined and remain selected */
     if (use_merge == false) {
       EDBM_flag_disable_all(em, BM_ELEM_SELECT);
@@ -7670,7 +7683,7 @@ static int edbm_convex_hull_exec(bContext *C, wmOperator *op)
     BMO_op_exec(em->bm, &bmop);
 
     /* Hull fails if input is coplanar */
-    if (BMO_error_occurred(em->bm)) {
+    if (BMO_error_occurred_at_level(em->bm, BMO_ERROR_CANCEL)) {
       EDBM_op_finish(em, &bmop, op, true);
       continue;
     }
diff --git a/source/blender/editors/mesh/editmesh_utils.c b/source/blender/editors/mesh/editmesh_utils.c
index b384845513a..12d85c88084 100644
--- a/source/blender/editors/mesh/editmesh_utils.c
+++ b/source/blender/editors/mesh/editmesh_utils.c
@@ -116,45 +116,6 @@ void EDBM_redo_state_free(BMBackup *backup, BMEditMesh *em, int recalctess)
 /** \} */
 
 /* -------------------------------------------------------------------- */
-/** \name Edit-Mesh Copy API (Internal)
- * \{ */
-
-static void edbm_op_emcopy_incref_and_ensure(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  if (em->emcopy == NULL) {
-    em->emcopy = BKE_editmesh_copy(em);
-  }
-  em->emcopyusers++;
-}
-
-static void edbm_op_emcopy_decref(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  em->emcopyusers--;
-  if (em->emcopyusers < 0) {
-    printf("warning: em->emcopyusers was less than zero.\n");
-  }
-  if (em->emcopyusers <= 0) {
-    BKE_editmesh_free(em->emcopy);
-    MEM_freeN(em->emcopy);
-    em->emcopy = NULL;
-  }
-}
-
-static void edbm_op_emcopy_restore_and_clear(BMEditMesh *em, const BMOperator *UNUSED(bmop))
-{
-  BLI_assert(em->emcopy != NULL);
-  BMEditMesh *emcopy = em->emcopy;
-  EDBM_mesh_free(em);
-  *em = *emcopy;
-
-  MEM_freeN(emcopy);
-  em->emcopyusers = 0;
-  em->emcopy = NULL;
-}
-
-/** \} */
-
-/* -------------------------------------------------------------------- */
 /** \name BMesh Operator (BMO) API Wrapper
  * \{ */
 
@@ -171,8 +132,6 @@ bool EDBM_op_init(BMEditMesh *em, BMOperator *bmop, wmOperator *op, const char *
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, bmop);
-
   va_end(list);
 
   return true;
@@ -187,29 +146,62 @@ bool EDBM_op_finish(BMEditMesh *em, BMOperator *bmop, wmOperator *op, const bool
 {
   const char *errmsg;
 
-  BMO_op_finish(em->bm, bmop);
+#ifndef NDEBUG
+  struct {
+    int verts_len, edges_len, loops_len, faces_len;
+  } em_state_prev = {
+      .verts_len = em->bm->totvert,
+      .edges_len = em->bm->totedge,
+      .loops_len = em->bm->totloop,
+      .faces_len = em->bm->totface,
+  };
+#endif
 
-  if (BMO_error_get(em->bm, &errmsg, NULL)) {
-    BLI_assert(em->emcopy != NULL);
+  BMO_op_finish(em->bm, bmop);
 
-    if (do_report) {
-      BKE_report(op->reports, RPT_ERROR, errmsg);
+  bool changed = false;
+  bool changed_was_set = false;
+
+  eBMOpErrorLevel level;
+  while (BMO_error_pop(em->bm, &errmsg, NULL, &level)) {
+    ReportType type = RPT_INFO;
+    switch (level) {
+      case BMO_ERROR_CANCEL: {
+        changed_was_set = true;
+        break;
+      }
+      case BMO_ERROR_WARN: {
+        type = RPT_WARNING;
+        changed_was_set = true;
+        changed = true;
+        break;
+      }
+      case BMO_ERROR_FATAL: {
+        type = RPT_ERROR;
+        changed_was_set = true;
+        changed = true;
+        break;
+      }
     }
 
-    edbm_op_emcopy_restore_and_clear(em, bmop);
-
-    /* when copying, tessellation isn't to for faster copying,
-     * but means we need to re-tessellate here */
-    if (em->looptris == NULL) {
-      BKE_editmesh_looptri_calc(em);
+    if (do_report) {
+      BKE_report(op->reports, type, errmsg);
     }
-
-    return false;
+  }
+  if (changed_was_set == false) {
+    changed = true;
   }
 
-  edbm_op_emcopy_decref(em, bmop);
+#ifndef NDEBUG
+  if (changed == false) {
+    BLI_assert((em_state_prev.verts_len == em->bm->totvert) &&
+               (em_state_prev.edges_len == em->bm->totedge) &&
+               (em_state_prev.loops_len == em->bm->totloop) &&
+               (em_state_prev.faces_len == em->bm->totface));
+  }
+#endif
 
-  return true;
+  return changed;
 }
 
 bool EDBM_op_callf(BMEditMesh *em, wmOperator *op, const char *fmt, ...)
@@ -226,8 +218,6 @@ bool EDBM_op_callf(BMEditMesh *em, wmOperator *op, const char *fmt, ...)
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   va_end(list);
@@ -255,8 +245,6 @@ bool EDBM_op_call_and_selectf(BMEditMesh *em,
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   slot_select_out = BMO_slot_get(bmop.slots_out, select_slot_out);
@@ -287,8 +275,6 @@ bool EDBM_op_call_silentf(BMEditMesh *em, const char *fmt, ...)
     return false;
   }
 
-  edbm_op_emcopy_incref_and_ensure(em, &bmop);
-
   BMO_op_exec(bm, &bmop);
 
   va_end(list);