Fix T70476: Sculpting with Subsurf on top produces artifacts

The issue was caused by crazy space distortion orientation happening
for subsurf modifier.

Solved by making it so subsurf only deforms the surface but keeps
matrices as-is. This is not fully mathematically correct, but is better
that the fall-back solution which was doing wrong matrices anyway.
Also, this is closer to have subsurf was handled prior to the
related changes.

Reviewed By: brecht, pablodp606

Differential Revision: https://developer.blender.org/D5991

7 files changed, 34 insertions, 18 deletions

diff --git a/release/datafiles/locale b/release/datafiles/locale
-Subproject 1f6ec7f7a145c36808231090c8666febb49db22
+Subproject 88497d7507e76281fbdd0cc1f8451f81ae7e378
diff --git a/release/scripts/addons b/release/scripts/addons
-Subproject eb9bab0e7153fda8de113af9e3c54eca74c986e
+Subproject d39c2b61470059dd31cf688441c817a875b928c
diff --git a/release/scripts/addons_contrib b/release/scripts/addons_contrib
-Subproject d9ed9d4d064c74c86e2767cd4be32d602a0ee31
+Subproject 69bcc72f1ec4df73265ce35851658ef184b9d0f
diff --git a/source/blender/blenkernel/intern/crazyspace.c b/source/blender/blenkernel/intern/crazyspace.c
index 6740fc985e9..fe834658689 100644
--- a/source/blender/blenkernel/intern/crazyspace.c
+++ b/source/blender/blenkernel/intern/crazyspace.c
@@ -355,7 +355,7 @@ static void crazyspace_init_verts_and_matrices(const Mesh *mesh,
   BLI_assert(num_verts == mesh->totvert);
 }
 
-static bool crazyspace_modifier_supports_deform(ModifierData *md)
+static bool crazyspace_modifier_supports_deform_matrices(ModifierData *md)
 {
   if (ELEM(md->type, eModifierType_Subsurf, eModifierType_Multires)) {
     return true;
@@ -364,6 +364,12 @@ static bool crazyspace_modifier_supports_deform(ModifierData *md)
   return (mti->type == eModifierTypeType_OnlyDeform);
 }
 
+static bool crazyspace_modifier_supports_deform(ModifierData *md)
+{
+  const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
+  return (mti->type == eModifierTypeType_OnlyDeform);
+}
+
 int BKE_sculpt_get_first_deform_matrices(struct Depsgraph *depsgraph,
                                          Scene *scene,
                                          Object *object,
@@ -391,13 +397,12 @@ int BKE_sculpt_get_first_deform_matrices(struct Depsgraph *depsgraph,
   md = modifiers_getVirtualModifierList(&object_eval, &virtualModifierData);
 
   for (; md; md = md->next) {
-    const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
-
     if (!modifier_isEnabled(scene, md, eModifierMode_Realtime)) {
       continue;
     }
 
-    if (mti->type == eModifierTypeType_OnlyDeform) {
+    if (crazyspace_modifier_supports_deform_matrices(md)) {
+      const ModifierTypeInfo *mti = modifierType_getInfo(md->type);
       if (defmats == NULL) {
         /* NOTE: Evaluated object si re-set to its original undeformed
          * state. */
diff --git a/source/blender/modifiers/intern/MOD_multires.c b/source/blender/modifiers/intern/MOD_multires.c
index 53bb579128a..883c17aa1e1 100644
--- a/source/blender/modifiers/intern/MOD_multires.c
+++ b/source/blender/modifiers/intern/MOD_multires.c
@@ -228,16 +228,22 @@ static Mesh *applyModifier(ModifierData *md, const ModifierEvalContext *ctx, Mes
   return result;
 }
 
-static void deformVerts(ModifierData *md,
-                        const ModifierEvalContext *UNUSED(ctx),
-                        Mesh *mesh,
-                        float (*vertex_cos)[3],
-                        int num_verts)
+static void deformMatrices(ModifierData *md,
+                           const ModifierEvalContext *UNUSED(ctx),
+                           Mesh *mesh,
+                           float (*vertex_cos)[3],
+                           float (*deform_matrices)[3][3],
+                           int num_verts)
+
 {
 #if !defined(WITH_OPENSUBDIV)
   modifier_setError(md, "Disabled, built without OpenSubdiv");
   return;
 #endif
+
+  /* Subsurf does not require extra space mapping, keep matrices as is. */
+  (void)deform_matrices;
+
   MultiresModifierData *mmd = (MultiresModifierData *)md;
   SubdivSettings subdiv_settings;
   BKE_multires_subdiv_settings_init(&subdiv_settings, mmd);
@@ -268,8 +274,8 @@ ModifierTypeInfo modifierType_Multires = {
 
     /* copyData */ copyData,
 
-    /* deformVerts */ deformVerts,
-    /* deformMatrices */ NULL,
+    /* deformVerts */ NULL,
+    /* deformMatrices */ deformMatrices,
     /* deformVertsEM */ NULL,
     /* deformMatricesEM */ NULL,
     /* applyModifier */ applyModifier,
diff --git a/source/blender/modifiers/intern/MOD_subsurf.c b/source/blender/modifiers/intern/MOD_subsurf.c
index 55df1b5ddda..4a5887c3122 100644
--- a/source/blender/modifiers/intern/MOD_subsurf.c
+++ b/source/blender/modifiers/intern/MOD_subsurf.c
@@ -241,16 +241,21 @@ static Mesh *applyModifier(ModifierData *md, const ModifierEvalContext *ctx, Mes
   return result;
 }
 
-static void deformVerts(ModifierData *md,
-                        const ModifierEvalContext *UNUSED(ctx),
-                        Mesh *mesh,
-                        float (*vertex_cos)[3],
-                        int num_verts)
+static void deformMatrices(ModifierData *md,
+                           const ModifierEvalContext *UNUSED(ctx),
+                           Mesh *mesh,
+                           float (*vertex_cos)[3],
+                           float (*deform_matrices)[3][3],
+                           int num_verts)
 {
 #if !defined(WITH_OPENSUBDIV)
   modifier_setError(md, "Disabled, built without OpenSubdiv");
   return;
 #endif
+
+  /* Subsurf does not require extra space mapping, keep matrices as is. */
+  (void)deform_matrices;
+
   SubsurfModifierData *smd = (SubsurfModifierData *)md;
   SubdivSettings subdiv_settings;
   subdiv_settings_init(&subdiv_settings, smd);
@@ -281,8 +286,8 @@ ModifierTypeInfo modifierType_Subsurf = {
 
     /* copyData */ copyData,
 
-    /* deformVerts */ deformVerts,
-    /* deformMatrices */ NULL,
+    /* deformVerts */ NULL,
+    /* deformMatrices */ deformMatrices,
     /* deformVertsEM */ NULL,
     /* deformMatricesEM */ NULL,
     /* applyModifier */ applyModifier,
diff --git a/source/tools b/source/tools
-Subproject 7b740545cd039ddcadbfb9bffcac842476eee27
+Subproject 4ad446dbddd1b23acc185b8c056474d27b69278