Fix T98813: crash with GPU subdiv in edit mode and instanced geometry

Instancing with geometry nodes uses just the evaluated Mesh, and ignores the
Object that it came from. That meant that it would try to look up the subsurf
modifier on the instancer object which does not have the subsurf modifier.

Instead of storing a session UUID and looking up the modifier data, store a
point to the subsurf modifier runtime data. Unlike the modifier data, this
runtime data is preserved across depsgraph CoW. It must be for the subdiv
descriptor contained in it to stay valid along with the draw cache.

As a bonus, this moves various Mesh_Runtime variables into the subsurf runtime
data, reducing memory usage for meshes not using subdivision surfaces.

Also fixes T98693, issues with subdivision level >= 8 due to integer overflow.

Differential Revision: https://developer.blender.org/D15184

4 files changed, 90 insertions, 64 deletions

diff --git a/source/blender/blenkernel/BKE_subdiv_modifier.h b/source/blender/blenkernel/BKE_subdiv_modifier.h
index 4ad17610207..271026d4253 100644
--- a/source/blender/blenkernel/BKE_subdiv_modifier.h
+++ b/source/blender/blenkernel/BKE_subdiv_modifier.h
@@ -7,6 +7,8 @@
 
 #pragma once
 
+#include "BKE_subdiv.h"
+
 #include "BLI_sys_types.h"
 
 #ifdef __cplusplus
@@ -24,9 +26,30 @@ struct Subdiv;
 struct SubdivSettings;
 struct SubsurfModifierData;
 
-void BKE_subsurf_modifier_subdiv_settings_init(struct SubdivSettings *settings,
-                                               const struct SubsurfModifierData *smd,
-                                               bool use_render_params);
+/* Runtime subsurf modifier data, cached in modifier on evaluated meshes. */
+typedef struct SubsurfRuntimeData {
+  /* Subdivision settings, exists before descriptor or mesh wrapper is created. */
+  SubdivSettings settings;
+
+  /* Cached subdivision surface descriptor, with topology and settings. */
+  struct Subdiv *subdiv;
+  bool set_by_draw_code;
+
+  /* Cached mesh wrapper data, to be used for GPU subdiv or lazy evaluation on CPU. */
+  bool has_gpu_subdiv;
+  int resolution;
+  bool use_optimal_display;
+  bool calc_loop_normals;
+  bool use_loop_normals;
+
+  /* Cached from the draw code for stats display. */
+  int stats_totvert;
+  int stats_totedge;
+  int stats_totpoly;
+  int stats_totloop;
+} SubsurfRuntimeData;
+
+bool BKE_subsurf_modifier_runtime_init(struct SubsurfModifierData *smd, bool use_render_params);
 
 bool BKE_subsurf_modifier_use_custom_loop_normals(const struct SubsurfModifierData *smd,
                                                   const struct Mesh *mesh);
@@ -57,12 +80,7 @@ extern void (*BKE_subsurf_modifier_free_gpu_cache_cb)(struct Subdiv *subdiv);
  * which matches settings and topology.
  */
 struct Subdiv *BKE_subsurf_modifier_subdiv_descriptor_ensure(
-    const struct SubsurfModifierData *smd,
-    const struct SubdivSettings *subdiv_settings,
-    const struct Mesh *mesh,
-    bool for_draw_code);
-
-struct SubsurfRuntimeData *BKE_subsurf_modifier_ensure_runtime(struct SubsurfModifierData *smd);
+    struct SubsurfRuntimeData *runtime_data, const struct Mesh *mesh, bool for_draw_code);
 
 /**
  * Return the #ModifierMode required for the evaluation of the subsurf modifier,
diff --git a/source/blender/blenkernel/intern/DerivedMesh.cc b/source/blender/blenkernel/intern/DerivedMesh.cc
index 8496692ceb3..b3a9d894944 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.cc
+++ b/source/blender/blenkernel/intern/DerivedMesh.cc
@@ -52,6 +52,7 @@
 #include "BKE_object.h"
 #include "BKE_object_deform.h"
 #include "BKE_paint.h"
+#include "BKE_subdiv_modifier.h"
 
 #include "BLI_sys_types.h" /* for intptr_t support */
 
@@ -598,10 +599,10 @@ static bool mesh_has_modifier_final_normals(const Mesh *mesh_input,
   /* Test if mesh has the required loop normals, in case an additional modifier
    * evaluation from another instance or from an operator requests it but the
    * initial normals were not loop normals. */
-  const bool do_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
-  return (!do_loop_normals || CustomData_has_layer(&mesh_final->ldata, CD_NORMAL));
+  return (!calc_loop_normals || CustomData_has_layer(&mesh_final->ldata, CD_NORMAL));
 }
 
 static void mesh_calc_modifier_final_normals(const Mesh *mesh_input,
@@ -610,16 +611,19 @@ static void mesh_calc_modifier_final_normals(const Mesh *mesh_input,
                                              Mesh *mesh_final)
 {
   /* Compute normals. */
-  const bool do_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_input->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
   /* Needed as `final_datamask` is not preserved outside modifier stack evaluation. */
-  mesh_final->runtime.subsurf_do_loop_normals = do_loop_normals;
+  SubsurfRuntimeData *subsurf_runtime_data = mesh_final->runtime.subsurf_runtime_data;
+  if (subsurf_runtime_data) {
+    subsurf_runtime_data->calc_loop_normals = calc_loop_normals;
+  }
 
-  if (do_loop_normals) {
+  if (calc_loop_normals) {
     /* Compute loop normals (NOTE: will compute poly and vert normals as well, if needed!). In case
      * of deferred CPU subdivision, this will be computed when the wrapper is generated. */
-    if (mesh_final->runtime.subsurf_resolution == 0) {
+    if (!subsurf_runtime_data || subsurf_runtime_data->resolution == 0) {
       BKE_mesh_calc_normals_split(mesh_final);
     }
   }
@@ -1266,15 +1270,18 @@ static void editbmesh_calc_modifier_final_normals(Mesh *mesh_final,
     return;
   }
 
-  const bool do_loop_normals = ((mesh_final->flag & ME_AUTOSMOOTH) != 0 ||
-                                (final_datamask->lmask & CD_MASK_NORMAL) != 0);
+  const bool calc_loop_normals = ((mesh_final->flag & ME_AUTOSMOOTH) != 0 ||
+                                  (final_datamask->lmask & CD_MASK_NORMAL) != 0);
 
-  mesh_final->runtime.subsurf_do_loop_normals = do_loop_normals;
+  SubsurfRuntimeData *subsurf_runtime_data = mesh_final->runtime.subsurf_runtime_data;
+  if (subsurf_runtime_data) {
+    subsurf_runtime_data->calc_loop_normals = calc_loop_normals;
+  }
 
-  if (do_loop_normals) {
+  if (calc_loop_normals) {
     /* Compute loop normals. In case of deferred CPU subdivision, this will be computed when the
      * wrapper is generated. */
-    if (mesh_final->runtime.subsurf_resolution == 0) {
+    if (!subsurf_runtime_data || subsurf_runtime_data->resolution == 0) {
       BKE_mesh_calc_normals_split(mesh_final);
     }
   }
diff --git a/source/blender/blenkernel/intern/mesh_wrapper.cc b/source/blender/blenkernel/intern/mesh_wrapper.cc
index c505a74724b..fdebf1d6a26 100644
--- a/source/blender/blenkernel/intern/mesh_wrapper.cc
+++ b/source/blender/blenkernel/intern/mesh_wrapper.cc
@@ -314,28 +314,23 @@ static Mesh *mesh_wrapper_ensure_subdivision(const Object *ob, Mesh *me)
     return me;
   }
 
+  SubsurfRuntimeData *runtime_data = (SubsurfRuntimeData *)smd->modifier.runtime;
+  if (runtime_data == nullptr || runtime_data->settings.level == 0) {
+    return me;
+  }
+
   /* Initialize the settings before ensuring the descriptor as this is checked to decide whether
    * subdivision is needed at all, and checking the descriptor status might involve checking if the
    * data is out-of-date, which is a very expensive operation. */
   SubdivToMeshSettings mesh_settings;
-  mesh_settings.resolution = me->runtime.subsurf_resolution;
-  mesh_settings.use_optimal_display = me->runtime.subsurf_use_optimal_display;
+  mesh_settings.resolution = runtime_data->resolution;
+  mesh_settings.use_optimal_display = runtime_data->use_optimal_display;
 
   if (mesh_settings.resolution < 3) {
     return me;
   }
 
-  const bool apply_render = me->runtime.subsurf_apply_render;
-
-  SubdivSettings subdiv_settings;
-  BKE_subsurf_modifier_subdiv_settings_init(&subdiv_settings, smd, apply_render);
-  if (subdiv_settings.level == 0) {
-    return me;
-  }
-
-  SubsurfRuntimeData *runtime_data = BKE_subsurf_modifier_ensure_runtime(smd);
-
-  Subdiv *subdiv = BKE_subsurf_modifier_subdiv_descriptor_ensure(smd, &subdiv_settings, me, false);
+  Subdiv *subdiv = BKE_subsurf_modifier_subdiv_descriptor_ensure(runtime_data, me, false);
   if (subdiv == nullptr) {
     /* Happens on bad topology, but also on empty input mesh. */
     return me;
@@ -358,7 +353,7 @@ static Mesh *mesh_wrapper_ensure_subdivision(const Object *ob, Mesh *me)
     CustomData_set_layer_flag(&me->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
     CustomData_set_layer_flag(&subdiv_mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
   }
-  else if (me->runtime.subsurf_do_loop_normals) {
+  else if (runtime_data->calc_loop_normals) {
     BKE_mesh_calc_normals_split(subdiv_mesh);
   }
 
diff --git a/source/blender/blenkernel/intern/subdiv_modifier.c b/source/blender/blenkernel/intern/subdiv_modifier.c
index e43da956ce5..f5423dccc0f 100644
--- a/source/blender/blenkernel/intern/subdiv_modifier.c
+++ b/source/blender/blenkernel/intern/subdiv_modifier.c
@@ -3,8 +3,6 @@
 
 #include "BKE_subdiv_modifier.h"
 
-#include "BLI_session_uuid.h"
-
 #include "MEM_guardedalloc.h"
 
 #include "DNA_mesh_types.h"
@@ -21,22 +19,40 @@
 
 #include "opensubdiv_capi.h"
 
-void BKE_subsurf_modifier_subdiv_settings_init(SubdivSettings *settings,
-                                               const SubsurfModifierData *smd,
-                                               const bool use_render_params)
+bool BKE_subsurf_modifier_runtime_init(SubsurfModifierData *smd, const bool use_render_params)
 {
   const int requested_levels = (use_render_params) ? smd->renderLevels : smd->levels;
 
-  settings->is_simple = (smd->subdivType == SUBSURF_TYPE_SIMPLE);
-  settings->is_adaptive = !(smd->flags & eSubsurfModifierFlag_UseRecursiveSubdivision);
-  settings->level = settings->is_simple ?
-                        1 :
-                        (settings->is_adaptive ? smd->quality : requested_levels);
-  settings->use_creases = (smd->flags & eSubsurfModifierFlag_UseCrease);
-  settings->vtx_boundary_interpolation = BKE_subdiv_vtx_boundary_interpolation_from_subsurf(
+  SubdivSettings settings;
+  settings.is_simple = (smd->subdivType == SUBSURF_TYPE_SIMPLE);
+  settings.is_adaptive = !(smd->flags & eSubsurfModifierFlag_UseRecursiveSubdivision);
+  settings.level = settings.is_simple ? 1 :
+                                        (settings.is_adaptive ? smd->quality : requested_levels);
+  settings.use_creases = (smd->flags & eSubsurfModifierFlag_UseCrease);
+  settings.vtx_boundary_interpolation = BKE_subdiv_vtx_boundary_interpolation_from_subsurf(
       smd->boundary_smooth);
-  settings->fvar_linear_interpolation = BKE_subdiv_fvar_interpolation_from_uv_smooth(
+  settings.fvar_linear_interpolation = BKE_subdiv_fvar_interpolation_from_uv_smooth(
       smd->uv_smooth);
+
+  SubsurfRuntimeData *runtime_data = (SubsurfRuntimeData *)smd->modifier.runtime;
+  if (settings.level == 0) {
+    /* Modifier is effectively disabled, but still update settings if runtime data
+     * was already allocated. */
+    if (runtime_data) {
+      runtime_data->settings = settings;
+    }
+
+    return false;
+  }
+  else {
+    /* Allocate runtime data if it did not exist yet. */
+    if (runtime_data == NULL) {
+      runtime_data = MEM_callocN(sizeof(*runtime_data), "subsurf runtime");
+      smd->modifier.runtime = runtime_data;
+    }
+    runtime_data->settings = settings;
+    return true;
+  }
 }
 
 static ModifierData *modifier_get_last_enabled_for_mode(const Scene *scene,
@@ -133,37 +149,27 @@ bool BKE_subsurf_modifier_can_do_gpu_subdiv(const Scene *scene,
 
 bool BKE_subsurf_modifier_has_gpu_subdiv(const Mesh *mesh)
 {
-  return BLI_session_uuid_is_generated(&mesh->runtime.subsurf_session_uuid);
+  SubsurfRuntimeData *runtime_data = mesh->runtime.subsurf_runtime_data;
+  return runtime_data && runtime_data->has_gpu_subdiv;
 }
 
 void (*BKE_subsurf_modifier_free_gpu_cache_cb)(Subdiv *subdiv) = NULL;
 
-Subdiv *BKE_subsurf_modifier_subdiv_descriptor_ensure(const SubsurfModifierData *smd,
-                                                      const SubdivSettings *subdiv_settings,
+Subdiv *BKE_subsurf_modifier_subdiv_descriptor_ensure(SubsurfRuntimeData *runtime_data,
                                                       const Mesh *mesh,
                                                       const bool for_draw_code)
 {
-  SubsurfRuntimeData *runtime_data = (SubsurfRuntimeData *)smd->modifier.runtime;
   if (runtime_data->subdiv && runtime_data->set_by_draw_code != for_draw_code) {
     BKE_subdiv_free(runtime_data->subdiv);
     runtime_data->subdiv = NULL;
   }
-  Subdiv *subdiv = BKE_subdiv_update_from_mesh(runtime_data->subdiv, subdiv_settings, mesh);
+  Subdiv *subdiv = BKE_subdiv_update_from_mesh(
+      runtime_data->subdiv, &runtime_data->settings, mesh);
   runtime_data->subdiv = subdiv;
   runtime_data->set_by_draw_code = for_draw_code;
   return subdiv;
 }
 
-SubsurfRuntimeData *BKE_subsurf_modifier_ensure_runtime(SubsurfModifierData *smd)
-{
-  SubsurfRuntimeData *runtime_data = (SubsurfRuntimeData *)smd->modifier.runtime;
-  if (runtime_data == NULL) {
-    runtime_data = MEM_callocN(sizeof(*runtime_data), "subsurf runtime");
-    smd->modifier.runtime = runtime_data;
-  }
-  return runtime_data;
-}
-
 int BKE_subsurf_modifier_eval_required_mode(bool is_final_render, bool is_edit_mode)
 {
   if (is_final_render) {