Disable GPU subdivision if autosmooth or split normals are used

These features are complicated to support on GPU and hardly compatible
with subdivision in the first place. In the future, with T68891 and
T68893, subdivision and custom smooth shading will be separate workflows.
For now, and to better prepare for this future (although long term
plan), we should discourage workflows mixing subdivision and custom
smooth normals, and as such, this disables GPU subdivision when
autosmoothing or custom split normals are used.

This also adds a message in the modifier's UI to indicate that GPU
subdivision will be disabled if autosmooth or custom split normals are
used on the mesh.

Differential Revision: https://developer.blender.org/D14194

4 files changed, 79 insertions, 20 deletions

diff --git a/source/blender/blenkernel/BKE_subdiv_modifier.h b/source/blender/blenkernel/BKE_subdiv_modifier.h
index 8a179206dd7..5447f9fda7c 100644
--- a/source/blender/blenkernel/BKE_subdiv_modifier.h
+++ b/source/blender/blenkernel/BKE_subdiv_modifier.h
@@ -40,18 +40,30 @@ void BKE_subsurf_modifier_subdiv_settings_init(struct SubdivSettings *settings,
                                                const struct SubsurfModifierData *smd,
                                                bool use_render_params);
 
+bool BKE_subsurf_modifier_use_custom_loop_normals(const struct SubsurfModifierData *smd,
+                                                  const struct Mesh *mesh);
+
+/**
+ * Return true if GPU subdivision evaluation is disabled by force due to incompatible mesh or
+ * modifier settings. This will only return true if GPU subdivision is enabled in the preferences
+ * and supported by the GPU. It is mainly useful for showing UI messages.
+ */
+bool BKE_subsurf_modifier_force_disable_gpu_evaluation_for_mesh(
+    const struct SubsurfModifierData *smd, const struct Mesh *mesh);
 /**
  * \param skip_check_is_last: When true, we assume that the modifier passed is the last enabled
  * modifier in the stack.
  */
 bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const struct Scene *scene,
                                                const struct Object *ob,
+                                               const struct Mesh *mesh,
                                                const struct SubsurfModifierData *smd,
                                                int required_mode,
                                                bool skip_check_is_last);
 
 bool BKE_subsurf_modifier_can_do_gpu_subdiv(const struct Scene *scene,
                                             const struct Object *ob,
+                                            const struct Mesh *mesh,
                                             int required_mode);
 
 extern void (*BKE_subsurf_modifier_free_gpu_cache_cb)(struct Subdiv *subdiv);
diff --git a/source/blender/blenkernel/intern/subdiv_modifier.c b/source/blender/blenkernel/intern/subdiv_modifier.c
index 525c4837bc4..df798ccd490 100644
--- a/source/blender/blenkernel/intern/subdiv_modifier.c
+++ b/source/blender/blenkernel/intern/subdiv_modifier.c
@@ -70,23 +70,20 @@ static ModifierData *modifier_get_last_enabled_for_mode(const Scene *scene,
   return md;
 }
 
-bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
-                                               const Object *ob,
-                                               const SubsurfModifierData *smd,
-                                               int required_mode,
-                                               bool skip_check_is_last)
+bool BKE_subsurf_modifier_use_custom_loop_normals(const SubsurfModifierData *smd, const Mesh *mesh)
 {
-  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
-    return false;
-  }
+  return (smd->flags & eSubsurfModifierFlag_UseCustomNormals) && (mesh->flag & ME_AUTOSMOOTH) &&
+         CustomData_has_layer(&mesh->ldata, CD_CUSTOMLOOPNORMAL);
+}
 
-  if (!skip_check_is_last) {
-    ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
-    if (md != (const ModifierData *)smd) {
-      return false;
-    }
-  }
+bool subsurf_modifier_use_autosmooth_or_split_normals(const SubsurfModifierData *smd,
+                                                      const Mesh *mesh)
+{
+  return (mesh->flag & ME_AUTOSMOOTH) || BKE_subsurf_modifier_use_custom_loop_normals(smd, mesh);
+}
 
+static bool is_subdivision_evaluation_possible_on_gpu(void)
+{
   /* Only OpenGL is supported for OpenSubdiv evaluation for now. */
   if (GPU_backend_get_type() != GPU_BACKEND_OPENGL) {
     return false;
@@ -104,8 +101,52 @@ bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
   return true;
 }
 
+bool BKE_subsurf_modifier_force_disable_gpu_evaluation_for_mesh(const SubsurfModifierData *smd,
+                                                                const Mesh *mesh)
+{
+  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
+    /* GPU subdivision is explicitely disabled, so we don't force it. */
+    return false;
+  }
+
+  if (!is_subdivision_evaluation_possible_on_gpu()) {
+    /* The GPU type is not compatible with the subdivision. */
+    return false;
+  }
+
+  return subsurf_modifier_use_autosmooth_or_split_normals(smd, mesh);
+}
+
+bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
+                                               const Object *ob,
+                                               const Mesh *mesh,
+                                               const SubsurfModifierData *smd,
+                                               int required_mode,
+                                               bool skip_check_is_last)
+{
+  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
+    return false;
+  }
+
+  /* Deactivate GPU subdivision if autosmooth or custom split normals are used as those are
+   * complicated to support on GPU, and should really be separate workflows. */
+  if (subsurf_modifier_use_autosmooth_or_split_normals(smd, mesh)) {
+    return false;
+  }
+
+  if (!skip_check_is_last) {
+    ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
+    if (md != (const ModifierData *)smd) {
+      return false;
+    }
+  }
+
+  return is_subdivision_evaluation_possible_on_gpu();
+}
+
 bool BKE_subsurf_modifier_can_do_gpu_subdiv(const Scene *scene,
                                             const Object *ob,
+                                            const Mesh *mesh,
                                             int required_mode)
 {
   ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
@@ -119,7 +160,7 @@ bool BKE_subsurf_modifier_can_do_gpu_subdiv(const Scene *scene,
   }
 
   return BKE_subsurf_modifier_can_do_gpu_subdiv_ex(
-      scene, ob, (SubsurfModifierData *)md, required_mode, true);
+      scene, ob, mesh, (SubsurfModifierData *)md, required_mode, true);
 }
 
 void (*BKE_subsurf_modifier_free_gpu_cache_cb)(Subdiv *subdiv) = NULL;
diff --git a/source/blender/draw/intern/draw_cache_impl_mesh.c b/source/blender/draw/intern/draw_cache_impl_mesh.c
index db4c95baedb..71f13fe9423 100644
--- a/source/blender/draw/intern/draw_cache_impl_mesh.c
+++ b/source/blender/draw/intern/draw_cache_impl_mesh.c
@@ -1726,7 +1726,7 @@ void DRW_mesh_batch_cache_create_requested(struct TaskGraph *task_graph,
 
   const int required_mode = BKE_subsurf_modifier_eval_required_mode(DRW_state_is_scene_render(),
                                                                     is_editmode);
-  const bool do_subdivision = BKE_subsurf_modifier_can_do_gpu_subdiv(scene, ob, required_mode);
+  const bool do_subdivision = BKE_subsurf_modifier_can_do_gpu_subdiv(scene, ob, me, required_mode);
 
   MeshBufferList *mbuflist = &cache->final.buff;
 
diff --git a/source/blender/modifiers/intern/MOD_subsurf.c b/source/blender/modifiers/intern/MOD_subsurf.c
index a8c6687193b..2006ca1b1b7 100644
--- a/source/blender/modifiers/intern/MOD_subsurf.c
+++ b/source/blender/modifiers/intern/MOD_subsurf.c
@@ -250,7 +250,8 @@ static Mesh *modifyMesh(ModifierData *md, const ModifierEvalContext *ctx, Mesh *
      * assigned at this stage of modifier stack evaluation. */
     const bool is_editmode = (mesh->edit_mesh != NULL);
     const int required_mode = BKE_subsurf_modifier_eval_required_mode(is_render_mode, is_editmode);
-    if (BKE_subsurf_modifier_can_do_gpu_subdiv_ex(scene, ctx->object, smd, required_mode, false)) {
+    if (BKE_subsurf_modifier_can_do_gpu_subdiv_ex(
+            scene, ctx->object, mesh, smd, required_mode, false)) {
       subdiv_cache_cpu_evaluation_settings(ctx, mesh, smd);
       return result;
     }
@@ -262,9 +263,7 @@ static Mesh *modifyMesh(ModifierData *md, const ModifierEvalContext *ctx, Mesh *
     /* Happens on bad topology, but also on empty input mesh. */
     return result;
   }
-  const bool use_clnors = (smd->flags & eSubsurfModifierFlag_UseCustomNormals) &&
-                          (mesh->flag & ME_AUTOSMOOTH) &&
-                          CustomData_has_layer(&mesh->ldata, CD_CUSTOMLOOPNORMAL);
+  const bool use_clnors = BKE_subsurf_modifier_use_custom_loop_normals(smd, mesh);
   if (use_clnors) {
     /* If custom normals are present and the option is turned on calculate the split
      * normals and clear flag so the normals get interpolated to the result mesh. */
@@ -428,6 +427,13 @@ static void panel_draw(const bContext *C, Panel *panel)
 
   uiItemR(layout, ptr, "show_only_control_edges", 0, NULL, ICON_NONE);
 
+  SubsurfModifierData *smd = ptr->data;
+  Object *ob = ob_ptr.data;
+  Mesh *mesh = ob->data;
+  if (BKE_subsurf_modifier_force_disable_gpu_evaluation_for_mesh(smd, mesh)) {
+    uiItemL(layout, "Autosmooth or custom normals detected, disabling GPU subdivision", ICON_INFO);
+  }
+
   modifier_panel_end(layout, ptr);
 }