Fix T84364: Sculpt symmetrize fails with shape keys

Use the BMesh symmetrize operator instead of using the modifier code.

While we could support shape-keys with the existing code used by the
mirror modifier, we'd need to add code-paths for evaluated mesh & bmesh
conversion to handle shape-keys differently just for this one case,
since we want to avoid copying & processing shape-keys layers for
evaluated meshes in general.

5 files changed, 77 insertions, 70 deletions

diff --git a/source/blender/blenkernel/BKE_mesh_mirror.h b/source/blender/blenkernel/BKE_mesh_mirror.h
index 2c6920a18bf..a91f0787e68 100644
--- a/source/blender/blenkernel/BKE_mesh_mirror.h
+++ b/source/blender/blenkernel/BKE_mesh_mirror.h
@@ -23,27 +23,30 @@
  * \ingroup bke
  */
 
-#include "BLI_utildefines.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct Main;
+struct Mesh;
 struct MirrorModifierData;
-struct ModifierEvalContext;
 struct Object;
 
-struct Mesh *BKE_mesh_mirror_bisect_on_mirror_plane(struct MirrorModifierData *mmd,
-                                                    const struct Mesh *mesh,
-                                                    int axis,
-                                                    const float plane_co[3],
-                                                    float plane_no[3]);
-
-struct Mesh *BKE_mesh_mirror_apply_mirror_on_axis(struct MirrorModifierData *mmd,
-                                                  const struct ModifierEvalContext *UNUSED(ctx),
-                                                  struct Object *ob,
-                                                  const struct Mesh *mesh,
-                                                  int axis);
+struct Mesh *BKE_mesh_mirror_bisect_on_mirror_plane_for_modifier(struct MirrorModifierData *mmd,
+                                                                 const struct Mesh *mesh,
+                                                                 int axis,
+                                                                 const float plane_co[3],
+                                                                 float plane_no[3]);
+
+void BKE_mesh_mirror_apply_mirror_on_axis(struct Main *bmain,
+                                          struct Mesh *mesh,
+                                          const int axis,
+                                          const float dist);
+
+struct Mesh *BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(struct MirrorModifierData *mmd,
+                                                               struct Object *ob,
+                                                               const struct Mesh *mesh,
+                                                               int axis);
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/blenkernel/intern/mesh_mirror.c b/source/blender/blenkernel/intern/mesh_mirror.c
index 46764a56e60..a22b52d68d5 100644
--- a/source/blender/blenkernel/intern/mesh_mirror.c
+++ b/source/blender/blenkernel/intern/mesh_mirror.c
@@ -41,11 +41,11 @@
 
 #include "MOD_modifiertypes.h"
 
-Mesh *BKE_mesh_mirror_bisect_on_mirror_plane(MirrorModifierData *mmd,
-                                             const Mesh *mesh,
-                                             int axis,
-                                             const float plane_co[3],
-                                             float plane_no[3])
+Mesh *BKE_mesh_mirror_bisect_on_mirror_plane_for_modifier(MirrorModifierData *mmd,
+                                                          const Mesh *mesh,
+                                                          int axis,
+                                                          const float plane_co[3],
+                                                          float plane_no[3])
 {
   bool do_bisect_flip_axis = ((axis == 0 && mmd->flag & MOD_MIR_BISECT_FLIP_AXIS_X) ||
                               (axis == 1 && mmd->flag & MOD_MIR_BISECT_FLIP_AXIS_Y) ||
@@ -97,11 +97,47 @@ Mesh *BKE_mesh_mirror_bisect_on_mirror_plane(MirrorModifierData *mmd,
   return result;
 }
 
-Mesh *BKE_mesh_mirror_apply_mirror_on_axis(MirrorModifierData *mmd,
-                                           const ModifierEvalContext *UNUSED(ctx),
-                                           Object *ob,
-                                           const Mesh *mesh,
-                                           int axis)
+void BKE_mesh_mirror_apply_mirror_on_axis(struct Main *bmain,
+                                          Mesh *mesh,
+                                          const int axis,
+                                          const float dist)
+{
+  BMesh *bm = BKE_mesh_to_bmesh_ex(mesh,
+                                   &(struct BMeshCreateParams){
+                                       .use_toolflags = 1,
+                                   },
+                                   &(struct BMeshFromMeshParams){
+                                       .calc_face_normal = true,
+                                       .cd_mask_extra =
+                                           {
+                                               .vmask = CD_MASK_SHAPEKEY,
+                                           },
+                                   });
+  BMO_op_callf(bm,
+               (BMO_FLAG_DEFAULTS & ~BMO_FLAG_RESPECT_HIDE),
+               "symmetrize input=%avef direction=%i dist=%f use_shapekey=%b",
+               axis,
+               dist,
+               true);
+
+  BM_mesh_bm_to_me(bmain,
+                   bm,
+                   mesh,
+                   (&(struct BMeshToMeshParams){
+                       .calc_object_remap = true,
+
+                   }));
+  BM_mesh_free(bm);
+}
+
+/**
+ * \warning This should _not_ be used to modify original meshes since
+ * it doesn't handle shape-keys, use #BKE_mesh_mirror_apply_mirror_on_axis instead.
+ */
+Mesh *BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(MirrorModifierData *mmd,
+                                                        Object *ob,
+                                                        const Mesh *mesh,
+                                                        const int axis)
 {
   const float tolerance_sq = mmd->tolerance * mmd->tolerance;
   const bool do_vtargetmap = (mmd->flag & MOD_MIR_NO_MERGE) == 0;
@@ -157,7 +193,8 @@ Mesh *BKE_mesh_mirror_apply_mirror_on_axis(MirrorModifierData *mmd,
 
   Mesh *mesh_bisect = NULL;
   if (do_bisect) {
-    mesh_bisect = BKE_mesh_mirror_bisect_on_mirror_plane(mmd, mesh, axis, plane_co, plane_no);
+    mesh_bisect = BKE_mesh_mirror_bisect_on_mirror_plane_for_modifier(
+        mmd, mesh, axis, plane_co, plane_no);
     mesh = mesh_bisect;
   }
 
diff --git a/source/blender/editors/object/object_remesh.c b/source/blender/editors/object/object_remesh.c
index 11be71623d8..058c34fb977 100644
--- a/source/blender/editors/object/object_remesh.c
+++ b/source/blender/editors/object/object_remesh.c
@@ -773,7 +773,7 @@ static Mesh *remesh_symmetry_bisect(Mesh *mesh, eSymmetryAxes symmetry_axes)
       zero_v3(plane_no);
       plane_no[axis] = -1.0f;
       mesh_bisect_temp = mesh_bisect;
-      mesh_bisect = BKE_mesh_mirror_bisect_on_mirror_plane(
+      mesh_bisect = BKE_mesh_mirror_bisect_on_mirror_plane_for_modifier(
           &mmd, mesh_bisect, axis, plane_co, plane_no);
       if (mesh_bisect_temp != mesh_bisect) {
         BKE_id_free(NULL, mesh_bisect_temp);
@@ -803,7 +803,7 @@ static Mesh *remesh_symmetry_mirror(Object *ob, Mesh *mesh, eSymmetryAxes symmet
       mmd.flag = 0;
       mmd.flag &= MOD_MIR_AXIS_X << i;
       mesh_mirror_temp = mesh_mirror;
-      mesh_mirror = BKE_mesh_mirror_apply_mirror_on_axis(&mmd, NULL, ob, mesh_mirror, axis);
+      mesh_mirror = BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(&mmd, ob, mesh_mirror, axis);
       if (mesh_mirror_temp != mesh_mirror) {
         BKE_id_free(NULL, mesh_mirror_temp);
       }
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index f574f21b39d..5a75f4e9cb7 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -8166,10 +8166,12 @@ static bool sculpt_no_multires_poll(bContext *C)
 
 static int sculpt_symmetrize_exec(bContext *C, wmOperator *op)
 {
+  Main *bmain = CTX_data_main(C);
   Object *ob = CTX_data_active_object(C);
   const Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
   SculptSession *ss = ob->sculpt;
   PBVH *pbvh = ss->pbvh;
+  const float dist = RNA_float_get(op->ptr, "merge_tolerance");
 
   if (!pbvh) {
     return OPERATOR_CANCELLED;
@@ -8212,41 +8214,9 @@ static int sculpt_symmetrize_exec(bContext *C, wmOperator *op)
       /* Mesh Symmetrize. */
       ED_sculpt_undo_geometry_begin(ob, "mesh symmetrize");
       Mesh *mesh = ob->data;
-      Mesh *mesh_mirror;
-      MirrorModifierData mmd = {{0}};
-      int axis = 0;
-      mmd.flag = 0;
-      mmd.tolerance = RNA_float_get(op->ptr, "merge_tolerance");
-      switch (sd->symmetrize_direction) {
-        case BMO_SYMMETRIZE_NEGATIVE_X:
-          axis = 0;
-          mmd.flag |= MOD_MIR_AXIS_X | MOD_MIR_BISECT_AXIS_X | MOD_MIR_BISECT_FLIP_AXIS_X;
-          break;
-        case BMO_SYMMETRIZE_NEGATIVE_Y:
-          axis = 1;
-          mmd.flag |= MOD_MIR_AXIS_Y | MOD_MIR_BISECT_AXIS_Y | MOD_MIR_BISECT_FLIP_AXIS_Y;
-          break;
-        case BMO_SYMMETRIZE_NEGATIVE_Z:
-          axis = 2;
-          mmd.flag |= MOD_MIR_AXIS_Z | MOD_MIR_BISECT_AXIS_Z | MOD_MIR_BISECT_FLIP_AXIS_Z;
-          break;
-        case BMO_SYMMETRIZE_POSITIVE_X:
-          axis = 0;
-          mmd.flag |= MOD_MIR_AXIS_X | MOD_MIR_BISECT_AXIS_X;
-          break;
-        case BMO_SYMMETRIZE_POSITIVE_Y:
-          axis = 1;
-          mmd.flag |= MOD_MIR_AXIS_Y | MOD_MIR_BISECT_AXIS_Y;
-          break;
-        case BMO_SYMMETRIZE_POSITIVE_Z:
-          axis = 2;
-          mmd.flag |= MOD_MIR_AXIS_Z | MOD_MIR_BISECT_AXIS_Z;
-          break;
-      }
-      mesh_mirror = BKE_mesh_mirror_apply_mirror_on_axis(&mmd, NULL, ob, mesh, axis);
-      if (mesh_mirror) {
-        BKE_mesh_nomain_to_mesh(mesh_mirror, mesh, ob, &CD_MASK_MESH, true);
-      }
+
+      BKE_mesh_mirror_apply_mirror_on_axis(bmain, mesh, sd->symmetrize_direction, dist);
+
       ED_sculpt_undo_geometry_end(ob);
       BKE_mesh_calc_normals(ob->data);
       BKE_mesh_batch_cache_dirty_tag(ob->data, BKE_MESH_BATCH_DIRTY_ALL);
diff --git a/source/blender/modifiers/intern/MOD_mirror.c b/source/blender/modifiers/intern/MOD_mirror.c
index 7f34c6581ad..afe94d8dead 100644
--- a/source/blender/modifiers/intern/MOD_mirror.c
+++ b/source/blender/modifiers/intern/MOD_mirror.c
@@ -81,20 +81,17 @@ static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphConte
   }
 }
 
-static Mesh *mirrorModifier__doMirror(MirrorModifierData *mmd,
-                                      const ModifierEvalContext *ctx,
-                                      Object *ob,
-                                      Mesh *mesh)
+static Mesh *mirrorModifier__doMirror(MirrorModifierData *mmd, Object *ob, Mesh *mesh)
 {
   Mesh *result = mesh;
 
   /* check which axes have been toggled and mirror accordingly */
   if (mmd->flag & MOD_MIR_AXIS_X) {
-    result = BKE_mesh_mirror_apply_mirror_on_axis(mmd, ctx, ob, result, 0);
+    result = BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(mmd, ob, result, 0);
   }
   if (mmd->flag & MOD_MIR_AXIS_Y) {
     Mesh *tmp = result;
-    result = BKE_mesh_mirror_apply_mirror_on_axis(mmd, ctx, ob, result, 1);
+    result = BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(mmd, ob, result, 1);
     if (tmp != mesh) {
       /* free intermediate results */
       BKE_id_free(NULL, tmp);
@@ -102,7 +99,7 @@ static Mesh *mirrorModifier__doMirror(MirrorModifierData *mmd,
   }
   if (mmd->flag & MOD_MIR_AXIS_Z) {
     Mesh *tmp = result;
-    result = BKE_mesh_mirror_apply_mirror_on_axis(mmd, ctx, ob, result, 2);
+    result = BKE_mesh_mirror_apply_mirror_on_axis_for_modifier(mmd, ob, result, 2);
     if (tmp != mesh) {
       /* free intermediate results */
       BKE_id_free(NULL, tmp);
@@ -117,7 +114,7 @@ static Mesh *modifyMesh(ModifierData *md, const ModifierEvalContext *ctx, Mesh *
   Mesh *result;
   MirrorModifierData *mmd = (MirrorModifierData *)md;
 
-  result = mirrorModifier__doMirror(mmd, ctx, ctx->object, mesh);
+  result = mirrorModifier__doMirror(mmd, ctx->object, mesh);
 
   if (result != mesh) {
     result->runtime.cd_dirty_vert |= CD_MASK_NORMAL;