Fix T98813: crash with GPU subdiv in edit mode and instanced geometry

Instancing with geometry nodes uses just the evaluated Mesh, and ignores the
Object that it came from. That meant that it would try to look up the subsurf
modifier on the instancer object which does not have the subsurf modifier.

Instead of storing a session UUID and looking up the modifier data, store a
point to the subsurf modifier runtime data. Unlike the modifier data, this
runtime data is preserved across depsgraph CoW. It must be for the subdiv
descriptor contained in it to stay valid along with the draw cache.

As a bonus, this moves various Mesh_Runtime variables into the subsurf runtime
data, reducing memory usage for meshes not using subdivision surfaces.

Also fixes T98693, issues with subdivision level >= 8 due to integer overflow.

Differential Revision: https://developer.blender.org/D15184

1 files changed, 20 insertions, 13 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.cc b/source/blender/blenkernel/intern/DerivedMesh.cc
index 8496692ceb3..b3a9d894944 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.cc
+++ b/source/blender/blenkernel/intern/DerivedMesh.cc
@@ -52,6 +52,7 @@
 #include "BKE_object.h"
 #include "BKE_object_deform.h"
 #include "BKE_paint.h"
+#include "BKE_subdiv_modifier.h"
 
 #include "BLI_sys_types.h" /* for intptr_t support */
 
@@ -598,10 +599,10 @@ static bool mesh_has_modifier_final_normals(const Mesh *mesh_input,
   /* Test if mesh has the required loop normals, in case an additional modifier
    * evaluation from another instance or from an operator requests it but the
    * initial normals were not loop normals. */
-  const bool do_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
-  return (!do_loop_normals || CustomData_has_layer(&mesh_final->ldata, CD_NORMAL));
+  return (!calc_loop_normals || CustomData_has_layer(&mesh_final->ldata, CD_NORMAL));
 }
 
 static void mesh_calc_modifier_final_normals(const Mesh *mesh_input,
@@ -610,16 +611,19 @@ static void mesh_calc_modifier_final_normals(const Mesh *mesh_input,
                                              Mesh *mesh_final)
 {
   /* Compute normals. */
-  const bool do_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
   /* Needed as `final_datamask` is not preserved outside modifier stack evaluation. */
-  mesh_final->runtime.subsurf_do_loop_normals = do_loop_normals;
+  SubsurfRuntimeData *subsurf_runtime_data = mesh_final->runtime.subsurf_runtime_data;
+  if (subsurf_runtime_data) {
+    subsurf_runtime_data->calc_loop_normals = calc_loop_normals;
+  }
 
-  if (do_loop_normals) {
+  if (calc_loop_normals) {
     /* Compute loop normals (NOTE: will compute poly and vert normals as well, if needed!). In case
      * of deferred CPU subdivision, this will be computed when the wrapper is generated. */
-    if (mesh_final->runtime.subsurf_resolution == 0) {
+    if (!subsurf_runtime_data || subsurf_runtime_data->resolution == 0) {
       BKE_mesh_calc_normals_split(mesh_final);
     }
   }
@@ -1266,15 +1270,18 @@ static void editbmesh_calc_modifier_final_normals(Mesh *mesh_final,
     return;
   }
 
-  const bool do_loop_normals = ((mesh_final->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_final->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
-  mesh_final->runtime.subsurf_do_loop_normals = do_loop_normals;
+  SubsurfRuntimeData *subsurf_runtime_data = mesh_final->runtime.subsurf_runtime_data;
+  if (subsurf_runtime_data) {
+    subsurf_runtime_data->calc_loop_normals = calc_loop_normals;
+  }
 
-  if (do_loop_normals) {
+  if (calc_loop_normals) {
     /* Compute loop normals. In case of deferred CPU subdivision, this will be computed when the
      * wrapper is generated. */
-    if (mesh_final->runtime.subsurf_resolution == 0) {
+    if (!subsurf_runtime_data || subsurf_runtime_data->resolution == 0) {
       BKE_mesh_calc_normals_split(mesh_final);
     }
   }