Fix depsgraphs sharing IDs via evaluated edit mesh

The evaluated mesh is a result of evaluated modifiers, and referencing
other evaluated IDs such as materials.
It can not be stored in the EditMesh structure which is intended to be
re-used by many areas. Such sharing was causing ownership errors causing
bugs like

  T93855: Cycles crash with edit mode and simultaneous viewport and final render

The proposed solution is to store the evaluated edit mesh and its cage in
the object's runtime field. The motivation goes as following:

- It allows to avoid ownership problems like the ones in the linked report.
- Object level is chosen over mesh level is because the evaluated mesh
  is affected by modifiers, which are on the object level.

This patch allows to have modifier stack of an object which shares mesh with
an object which is in edit mode to be properly taken into account (before
the change the modifier stack from the active object will be used for all
objects which share the mesh).

There is a change in the way how copy-on-write is handled in the edit mode to
allow proper state update when changing active scene (or having two windows
with different scenes). Previously, the copt-on-write would have been ignored
by skipping tagging CoW component. Now it is ignored from within the CoW
operation callback. This allows to update edit pointers for objects which are
not from the current depsgraph and where the edit_mesh was never assigned in
the case when the depsgraph was evaluated prior the active depsgraph.

There is no user level changes changes expected with the CoW handling changes:
should not affect on neither performance, nor memory consumption.

Tested scenarios:

- Various modifiers configurations of objects sharing mesh and be part of the
  same scene.

- Steps from the reports: T93855, T82952, T77359

This also fixes T76609, T72733 and perhaps other reports.

Differential Revision: https://developer.blender.org/D13824

1 files changed, 13 insertions, 26 deletions

diff --git a/source/blender/blenkernel/intern/editmesh.c b/source/blender/blenkernel/intern/editmesh.c
index 6ef811c46c7..0774a1a3d88 100644
--- a/source/blender/blenkernel/intern/editmesh.c
+++ b/source/blender/blenkernel/intern/editmesh.c
@@ -39,6 +39,8 @@
 #include "BKE_mesh_wrapper.h"
 #include "BKE_object.h"
 
+#include "DEG_depsgraph_query.h"
+
 BMEditMesh *BKE_editmesh_create(BMesh *bm)
 {
   BMEditMesh *em = MEM_callocN(sizeof(BMEditMesh), __func__);
@@ -51,9 +53,6 @@ BMEditMesh *BKE_editmesh_copy(BMEditMesh *em)
   BMEditMesh *em_copy = MEM_callocN(sizeof(BMEditMesh), __func__);
   *em_copy = *em;
 
-  em_copy->mesh_eval_cage = em_copy->mesh_eval_final = NULL;
-  em_copy->bb_cage = NULL;
-
   em_copy->bm = BM_mesh_copy(em->bm);
 
   /* The tessellation is NOT calculated on the copy here,
@@ -194,22 +193,8 @@ void BKE_editmesh_looptri_and_normals_calc_with_partial(BMEditMesh *em,
                                          });
 }
 
-void BKE_editmesh_free_derived_caches(BMEditMesh *em)
-{
-  if (em->mesh_eval_cage) {
-    BKE_id_free(NULL, em->mesh_eval_cage);
-  }
-  if (em->mesh_eval_final && em->mesh_eval_final != em->mesh_eval_cage) {
-    BKE_id_free(NULL, em->mesh_eval_final);
-  }
-  em->mesh_eval_cage = em->mesh_eval_final = NULL;
-
-  MEM_SAFE_FREE(em->bb_cage);
-}
-
 void BKE_editmesh_free_data(BMEditMesh *em)
 {
-  BKE_editmesh_free_derived_caches(em);
 
   if (em->looptris) {
     MEM_freeN(em->looptris);
@@ -283,13 +268,15 @@ const float (*BKE_editmesh_vert_coords_when_deformed(struct Depsgraph *depsgraph
   *r_is_alloc = false;
 
   Mesh *me = ob->data;
+  Object *object_eval = DEG_get_evaluated_object(depsgraph, ob);
+  Mesh *editmesh_eval_final = BKE_object_get_editmesh_eval_final(object_eval);
 
   if ((me->runtime.edit_data != NULL) && (me->runtime.edit_data->vertexCos != NULL)) {
     /* Deformed, and we have deformed coords already. */
     coords = me->runtime.edit_data->vertexCos;
   }
-  else if ((em->mesh_eval_final != NULL) &&
-           (em->mesh_eval_final->runtime.wrapper_type == ME_WRAPPER_TYPE_BMESH)) {
+  else if ((editmesh_eval_final != NULL) &&
+           (editmesh_eval_final->runtime.wrapper_type == ME_WRAPPER_TYPE_BMESH)) {
     /* If this is an edit-mesh type, leave NULL as we can use the vertex coords. */
   }
   else {
@@ -334,18 +321,18 @@ void BKE_editmesh_ensure_autosmooth(BMEditMesh *em, Mesh *me)
   }
 }
 
-BoundBox *BKE_editmesh_cage_boundbox_get(BMEditMesh *em)
+BoundBox *BKE_editmesh_cage_boundbox_get(struct Object *object, BMEditMesh *UNUSED(em))
 {
-  if (em->bb_cage == NULL) {
+  if (object->runtime.editmesh_bb_cage == NULL) {
     float min[3], max[3];
     INIT_MINMAX(min, max);
-    if (em->mesh_eval_cage) {
-      BKE_mesh_wrapper_minmax(em->mesh_eval_cage, min, max);
+    if (object->runtime.editmesh_eval_cage) {
+      BKE_mesh_wrapper_minmax(object->runtime.editmesh_eval_cage, min, max);
     }
 
-    em->bb_cage = MEM_callocN(sizeof(BoundBox), "BMEditMesh.bb_cage");
-    BKE_boundbox_init_from_minmax(em->bb_cage, min, max);
+    object->runtime.editmesh_bb_cage = MEM_callocN(sizeof(BoundBox), "BMEditMesh.bb_cage");
+    BKE_boundbox_init_from_minmax(object->runtime.editmesh_bb_cage, min, max);
   }
 
-  return em->bb_cage;
+  return object->runtime.editmesh_bb_cage;
 }