Curves: fix transforms in Add brush

Symmetry should be applied in the space of the curves object,
not in the space of the surface object.

3 files changed, 71 insertions, 31 deletions

diff --git a/source/blender/editors/sculpt_paint/curves_sculpt_add.cc b/source/blender/editors/sculpt_paint/curves_sculpt_add.cc
index 5f168bd4e05..f013fa05f4c 100644
--- a/source/blender/editors/sculpt_paint/curves_sculpt_add.cc
+++ b/source/blender/editors/sculpt_paint/curves_sculpt_add.cc
@@ -168,7 +168,7 @@ struct AddOperationExecutor {
     world_to_surface_mat_ = surface_to_world_mat_.inverted();
     surface_to_curves_mat_ = world_to_curves_mat_ * surface_to_world_mat_;
     surface_to_curves_normal_mat_ = surface_to_curves_mat_.inverted().transposed();
-    curves_to_surface_mat_ = curves_to_world_mat_ * world_to_surface_mat_;
+    curves_to_surface_mat_ = world_to_surface_mat_ * curves_to_world_mat_;
 
     if (!CustomData_has_layer(&surface_->ldata, CD_NORMAL)) {
       BKE_mesh_calc_normals_split(surface_);
@@ -269,16 +269,6 @@ struct AddOperationExecutor {
     ED_region_tag_redraw(ctx_.region);
   }
 
-  float3 get_bary_coords(const Mesh &mesh, const MLoopTri &looptri, const float3 position) const
-  {
-    const float3 &v0 = mesh.mvert[mesh.mloop[looptri.tri[0]].v].co;
-    const float3 &v1 = mesh.mvert[mesh.mloop[looptri.tri[1]].v].co;
-    const float3 &v2 = mesh.mvert[mesh.mloop[looptri.tri[2]].v].co;
-    float3 bary_coords;
-    interp_weights_tri_v3(bary_coords, v0, v1, v2, position);
-    return bary_coords;
-  }
-
   /**
    * Sample a single point exactly at the mouse position.
    */
@@ -287,15 +277,15 @@ struct AddOperationExecutor {
     float3 ray_start_wo, ray_end_wo;
     ED_view3d_win_to_segment_clipped(
         ctx_.depsgraph, ctx_.region, ctx_.v3d, brush_pos_re_, ray_start_wo, ray_end_wo, true);
-    const float3 ray_start_su = world_to_surface_mat_ * ray_start_wo;
-    const float3 ray_end_su = world_to_surface_mat_ * ray_end_wo;
+    const float3 ray_start_cu = world_to_curves_mat_ * ray_start_wo;
+    const float3 ray_end_cu = world_to_curves_mat_ * ray_end_wo;
 
     const Vector<float4x4> symmetry_brush_transforms = get_symmetry_brush_transforms(
         eCurvesSymmetryType(curves_id_->symmetry));
 
     for (const float4x4 &brush_transform : symmetry_brush_transforms) {
-      this->sample_in_center(
-          r_added_points, brush_transform * ray_start_su, brush_transform * ray_end_su);
+      const float4x4 transform = curves_to_surface_mat_ * brush_transform;
+      this->sample_in_center(r_added_points, transform * ray_start_cu, transform * ray_end_cu);
     }
   }
 
@@ -322,7 +312,7 @@ struct AddOperationExecutor {
 
     const int looptri_index = ray_hit.index;
     const float3 brush_pos_su = ray_hit.co;
-    const float3 bary_coords = this->get_bary_coords(
+    const float3 bary_coords = compute_bary_coord_in_triangle(
         *surface_, surface_looptris_[looptri_index], brush_pos_su);
 
     const float3 brush_pos_cu = surface_to_curves_mat_ * brush_pos_su;
@@ -363,8 +353,11 @@ struct AddOperationExecutor {
       float3 ray_start_wo, ray_end_wo;
       ED_view3d_win_to_segment_clipped(
           ctx_.depsgraph, ctx_.region, ctx_.v3d, pos_re, ray_start_wo, ray_end_wo, true);
-      const float3 ray_start_su = brush_transform * (world_to_surface_mat_ * ray_start_wo);
-      const float3 ray_end_su = brush_transform * (world_to_surface_mat_ * ray_end_wo);
+      const float3 ray_start_cu = brush_transform * (world_to_curves_mat_ * ray_start_wo);
+      const float3 ray_end_cu = brush_transform * (world_to_curves_mat_ * ray_end_wo);
+
+      const float3 ray_start_su = curves_to_surface_mat_ * ray_start_cu;
+      const float3 ray_end_su = curves_to_surface_mat_ * ray_end_cu;
       const float3 ray_direction_su = math::normalize(ray_end_su - ray_start_su);
 
       BVHTreeRayHit ray_hit;
@@ -392,7 +385,7 @@ struct AddOperationExecutor {
       const int looptri_index = ray_hit.index;
       const float3 pos_su = ray_hit.co;
 
-      const float3 bary_coords = this->get_bary_coords(
+      const float3 bary_coords = compute_bary_coord_in_triangle(
           *surface_, surface_looptris_[looptri_index], pos_su);
 
       const float3 pos_cu = surface_to_curves_mat_ * pos_su;
@@ -417,8 +410,8 @@ struct AddOperationExecutor {
                                      brush_ray_start_wo,
                                      brush_ray_end_wo,
                                      true);
-    const float3 brush_ray_start_su = world_to_surface_mat_ * brush_ray_start_wo;
-    const float3 brush_ray_end_su = world_to_surface_mat_ * brush_ray_end_wo;
+    const float3 brush_ray_start_cu = world_to_curves_mat_ * brush_ray_start_wo;
+    const float3 brush_ray_end_cu = world_to_curves_mat_ * brush_ray_end_wo;
 
     /* Find ray that starts on the boundary of the brush. That is used to compute the brush radius
      * in 3D. */
@@ -430,18 +423,19 @@ struct AddOperationExecutor {
                                      brush_radius_ray_start_wo,
                                      brush_radius_ray_end_wo,
                                      true);
-    const float3 brush_radius_ray_start_su = world_to_surface_mat_ * brush_radius_ray_start_wo;
-    const float3 brush_radius_ray_end_su = world_to_surface_mat_ * brush_radius_ray_end_wo;
+    const float3 brush_radius_ray_start_cu = world_to_curves_mat_ * brush_radius_ray_start_wo;
+    const float3 brush_radius_ray_end_cu = world_to_curves_mat_ * brush_radius_ray_end_wo;
 
     const Vector<float4x4> symmetry_brush_transforms = get_symmetry_brush_transforms(
         eCurvesSymmetryType(curves_id_->symmetry));
     for (const float4x4 &brush_transform : symmetry_brush_transforms) {
+      const float4x4 transform = curves_to_surface_mat_ * brush_transform;
       this->sample_spherical(rng,
                              r_added_points,
-                             brush_transform * brush_ray_start_su,
-                             brush_transform * brush_ray_end_su,
-                             brush_transform * brush_radius_ray_start_su,
-                             brush_transform * brush_radius_ray_end_su);
+                             transform * brush_ray_start_cu,
+                             transform * brush_ray_end_cu,
+                             transform * brush_radius_ray_start_cu,
+                             transform * brush_radius_ray_end_cu);
     }
   }
 
@@ -782,7 +776,8 @@ struct AddOperationExecutor {
       for (const int i : range) {
         const int looptri_index = added_points.looptri_indices[i];
         const float3 &bary_coord = added_points.bary_coords[i];
-        normals_su[i] = this->compute_point_normal_su(looptri_index, bary_coord);
+        normals_su[i] = compute_surface_point_normal(
+            surface_looptris_[looptri_index], bary_coord, corner_normals_su_);
       }
     });
     return normals_su;
@@ -877,11 +872,13 @@ struct AddOperationExecutor {
               const int neighbor_looptri_index = nearest.index;
               const MLoopTri &neighbor_looptri = surface_looptris_[neighbor_looptri_index];
 
-              const float3 neighbor_bary_coord = this->get_bary_coords(
+              const float3 neighbor_bary_coord = compute_bary_coord_in_triangle(
                   *surface_, neighbor_looptri, nearest.co);
 
-              const float3 neighbor_normal_su = this->compute_point_normal_su(
-                  neighbor_looptri_index, neighbor_bary_coord);
+              const float3 neighbor_normal_su = compute_surface_point_normal(
+                  surface_looptris_[neighbor_looptri_index],
+                  neighbor_bary_coord,
+                  corner_normals_su_);
               const float3 neighbor_normal_cu = math::normalize(surface_to_curves_normal_mat_ *
                                                                 neighbor_normal_su);
 
diff --git a/source/blender/editors/sculpt_paint/curves_sculpt_brush.cc b/source/blender/editors/sculpt_paint/curves_sculpt_brush.cc
index 11d3548a082..7e583773512 100644
--- a/source/blender/editors/sculpt_paint/curves_sculpt_brush.cc
+++ b/source/blender/editors/sculpt_paint/curves_sculpt_brush.cc
@@ -4,14 +4,20 @@
 
 #include "curves_sculpt_intern.hh"
 
+#include "BKE_attribute_math.hh"
 #include "BKE_bvhutils.h"
 #include "BKE_context.h"
 #include "BKE_curves.hh"
 
+#include "DNA_meshdata_types.h"
+
 #include "ED_view3d.h"
 
 #include "UI_interface.h"
 
+#include "DNA_mesh_types.h"
+#include "DNA_meshdata_types.h"
+
 #include "BLI_enumerable_thread_specific.hh"
 #include "BLI_length_parameterize.hh"
 #include "BLI_task.hh"
@@ -318,4 +324,33 @@ CurvesSculptCommonContext::CurvesSculptCommonContext(const bContext &C)
   this->rv3d = CTX_wm_region_view3d(&C);
 }
 
+float3 compute_surface_point_normal(const MLoopTri &looptri,
+                                    const float3 &bary_coord,
+                                    const Span<float3> corner_normals)
+{
+  const int l0 = looptri.tri[0];
+  const int l1 = looptri.tri[1];
+  const int l2 = looptri.tri[2];
+
+  const float3 &l0_normal = corner_normals[l0];
+  const float3 &l1_normal = corner_normals[l1];
+  const float3 &l2_normal = corner_normals[l2];
+
+  const float3 normal = math::normalize(
+      attribute_math::mix3(bary_coord, l0_normal, l1_normal, l2_normal));
+  return normal;
+}
+
+float3 compute_bary_coord_in_triangle(const Mesh &mesh,
+                                      const MLoopTri &looptri,
+                                      const float3 &position)
+{
+  const float3 &v0 = mesh.mvert[mesh.mloop[looptri.tri[0]].v].co;
+  const float3 &v1 = mesh.mvert[mesh.mloop[looptri.tri[1]].v].co;
+  const float3 &v2 = mesh.mvert[mesh.mloop[looptri.tri[2]].v].co;
+  float3 bary_coords;
+  interp_weights_tri_v3(bary_coords, v0, v1, v2, position);
+  return bary_coords;
+}
+
 }  // namespace blender::ed::sculpt_paint
diff --git a/source/blender/editors/sculpt_paint/curves_sculpt_intern.hh b/source/blender/editors/sculpt_paint/curves_sculpt_intern.hh
index 4aaf6671cdb..5c926b1a740 100644
--- a/source/blender/editors/sculpt_paint/curves_sculpt_intern.hh
+++ b/source/blender/editors/sculpt_paint/curves_sculpt_intern.hh
@@ -97,6 +97,14 @@ IndexMask retrieve_selected_curves(const Curves &curves_id, Vector<int64_t> &r_i
 
 void move_last_point_and_resample(MutableSpan<float3> positions, const float3 &new_last_position);
 
+float3 compute_surface_point_normal(const MLoopTri &looptri,
+                                    const float3 &bary_coord,
+                                    const Span<float3> corner_normals);
+
+float3 compute_bary_coord_in_triangle(const Mesh &mesh,
+                                      const MLoopTri &looptri,
+                                      const float3 &position);
+
 class CurvesSculptCommonContext {
  public:
   const Depsgraph *depsgraph = nullptr;