Fix T98813: crash with GPU subdiv in edit mode and instanced geometry

Instancing with geometry nodes uses just the evaluated Mesh, and ignores the
Object that it came from. That meant that it would try to look up the subsurf
modifier on the instancer object which does not have the subsurf modifier.

Instead of storing a session UUID and looking up the modifier data, store a
point to the subsurf modifier runtime data. Unlike the modifier data, this
runtime data is preserved across depsgraph CoW. It must be for the subdiv
descriptor contained in it to stay valid along with the draw cache.

As a bonus, this moves various Mesh_Runtime variables into the subsurf runtime
data, reducing memory usage for meshes not using subdivision surfaces.

Also fixes T98693, issues with subdivision level >= 8 due to integer overflow.

Differential Revision: https://developer.blender.org/D15184

1 files changed, 9 insertions, 14 deletions

diff --git a/source/blender/blenkernel/intern/mesh_wrapper.cc b/source/blender/blenkernel/intern/mesh_wrapper.cc
index c505a74724b..fdebf1d6a26 100644
--- a/source/blender/blenkernel/intern/mesh_wrapper.cc
+++ b/source/blender/blenkernel/intern/mesh_wrapper.cc
@@ -314,28 +314,23 @@ static Mesh *mesh_wrapper_ensure_subdivision(const Object *ob, Mesh *me)
     return me;
   }
 
+  SubsurfRuntimeData *runtime_data = (SubsurfRuntimeData *)smd->modifier.runtime;
+  if (runtime_data == nullptr || runtime_data->settings.level == 0) {
+    return me;
+  }
+
   /* Initialize the settings before ensuring the descriptor as this is checked to decide whether
    * subdivision is needed at all, and checking the descriptor status might involve checking if the
    * data is out-of-date, which is a very expensive operation. */
   SubdivToMeshSettings mesh_settings;
-  mesh_settings.resolution = me->runtime.subsurf_resolution;
-  mesh_settings.use_optimal_display = me->runtime.subsurf_use_optimal_display;
+  mesh_settings.resolution = runtime_data->resolution;
+  mesh_settings.use_optimal_display = runtime_data->use_optimal_display;
 
   if (mesh_settings.resolution < 3) {
     return me;
   }
 
-  const bool apply_render = me->runtime.subsurf_apply_render;
-
-  SubdivSettings subdiv_settings;
-  BKE_subsurf_modifier_subdiv_settings_init(&subdiv_settings, smd, apply_render);
-  if (subdiv_settings.level == 0) {
-    return me;
-  }
-
-  SubsurfRuntimeData *runtime_data = BKE_subsurf_modifier_ensure_runtime(smd);
-
-  Subdiv *subdiv = BKE_subsurf_modifier_subdiv_descriptor_ensure(smd, &subdiv_settings, me, false);
+  Subdiv *subdiv = BKE_subsurf_modifier_subdiv_descriptor_ensure(runtime_data, me, false);
   if (subdiv == nullptr) {
     /* Happens on bad topology, but also on empty input mesh. */
     return me;
@@ -358,7 +353,7 @@ static Mesh *mesh_wrapper_ensure_subdivision(const Object *ob, Mesh *me)
     CustomData_set_layer_flag(&me->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
     CustomData_set_layer_flag(&subdiv_mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
   }
-  else if (me->runtime.subsurf_do_loop_normals) {
+  else if (runtime_data->calc_loop_normals) {
     BKE_mesh_calc_normals_split(subdiv_mesh);
   }