Disable GPU subdivision if autosmooth or split normals are used

These features are complicated to support on GPU and hardly compatible
with subdivision in the first place. In the future, with T68891 and
T68893, subdivision and custom smooth shading will be separate workflows.
For now, and to better prepare for this future (although long term
plan), we should discourage workflows mixing subdivision and custom
smooth normals, and as such, this disables GPU subdivision when
autosmoothing or custom split normals are used.

This also adds a message in the modifier's UI to indicate that GPU
subdivision will be disabled if autosmooth or custom split normals are
used on the mesh.

Differential Revision: https://developer.blender.org/D14194

1 files changed, 56 insertions, 15 deletions

diff --git a/source/blender/blenkernel/intern/subdiv_modifier.c b/source/blender/blenkernel/intern/subdiv_modifier.c
index 525c4837bc4..df798ccd490 100644
--- a/source/blender/blenkernel/intern/subdiv_modifier.c
+++ b/source/blender/blenkernel/intern/subdiv_modifier.c
@@ -70,23 +70,20 @@ static ModifierData *modifier_get_last_enabled_for_mode(const Scene *scene,
   return md;
 }
 
-bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
-                                               const Object *ob,
-                                               const SubsurfModifierData *smd,
-                                               int required_mode,
-                                               bool skip_check_is_last)
+bool BKE_subsurf_modifier_use_custom_loop_normals(const SubsurfModifierData *smd, const Mesh *mesh)
 {
-  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
-    return false;
-  }
+  return (smd->flags & eSubsurfModifierFlag_UseCustomNormals) && (mesh->flag & ME_AUTOSMOOTH) &&
+         CustomData_has_layer(&mesh->ldata, CD_CUSTOMLOOPNORMAL);
+}
 
-  if (!skip_check_is_last) {
-    ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
-    if (md != (const ModifierData *)smd) {
-      return false;
-    }
-  }
+bool subsurf_modifier_use_autosmooth_or_split_normals(const SubsurfModifierData *smd,
+                                                      const Mesh *mesh)
+{
+  return (mesh->flag & ME_AUTOSMOOTH) || BKE_subsurf_modifier_use_custom_loop_normals(smd, mesh);
+}
 
+static bool is_subdivision_evaluation_possible_on_gpu(void)
+{
   /* Only OpenGL is supported for OpenSubdiv evaluation for now. */
   if (GPU_backend_get_type() != GPU_BACKEND_OPENGL) {
     return false;
@@ -104,8 +101,52 @@ bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
   return true;
 }
 
+bool BKE_subsurf_modifier_force_disable_gpu_evaluation_for_mesh(const SubsurfModifierData *smd,
+                                                                const Mesh *mesh)
+{
+  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
+    /* GPU subdivision is explicitely disabled, so we don't force it. */
+    return false;
+  }
+
+  if (!is_subdivision_evaluation_possible_on_gpu()) {
+    /* The GPU type is not compatible with the subdivision. */
+    return false;
+  }
+
+  return subsurf_modifier_use_autosmooth_or_split_normals(smd, mesh);
+}
+
+bool BKE_subsurf_modifier_can_do_gpu_subdiv_ex(const Scene *scene,
+                                               const Object *ob,
+                                               const Mesh *mesh,
+                                               const SubsurfModifierData *smd,
+                                               int required_mode,
+                                               bool skip_check_is_last)
+{
+  if ((U.gpu_flag & USER_GPU_FLAG_SUBDIVISION_EVALUATION) == 0) {
+    return false;
+  }
+
+  /* Deactivate GPU subdivision if autosmooth or custom split normals are used as those are
+   * complicated to support on GPU, and should really be separate workflows. */
+  if (subsurf_modifier_use_autosmooth_or_split_normals(smd, mesh)) {
+    return false;
+  }
+
+  if (!skip_check_is_last) {
+    ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
+    if (md != (const ModifierData *)smd) {
+      return false;
+    }
+  }
+
+  return is_subdivision_evaluation_possible_on_gpu();
+}
+
 bool BKE_subsurf_modifier_can_do_gpu_subdiv(const Scene *scene,
                                             const Object *ob,
+                                            const Mesh *mesh,
                                             int required_mode)
 {
   ModifierData *md = modifier_get_last_enabled_for_mode(scene, ob, required_mode);
@@ -119,7 +160,7 @@ bool BKE_subsurf_modifier_can_do_gpu_subdiv(const Scene *scene,
   }
 
   return BKE_subsurf_modifier_can_do_gpu_subdiv_ex(
-      scene, ob, (SubsurfModifierData *)md, required_mode, true);
+      scene, ob, mesh, (SubsurfModifierData *)md, required_mode, true);
 }
 
 void (*BKE_subsurf_modifier_free_gpu_cache_cb)(Subdiv *subdiv) = NULL;