Fix Pose Brush FK mode detecting wrong rotation origin

The Pose FK mode assings the rotation origin to the boundary of the last
visited face set in the floodfill operation. In some cases, the topology
of the model may make the flood fill operation to visit a face set as the
first one (assinging it to target) and visit it again as the last one
(assinging it to origin). This will make the pose brush to default the
origin and target to the brush location and not to the face sets as it
considers that there is only one possible boundary.
This adds a GSet to ensure that a particular face set is not visited
twice in the flood fill, fixing these cases.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D7984

1 files changed, 15 insertions, 9 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_pose.c b/source/blender/editors/sculpt_paint/sculpt_pose.c
index a338b5346af..dc556fa1945 100644
--- a/source/blender/editors/sculpt_paint/sculpt_pose.c
+++ b/source/blender/editors/sculpt_paint/sculpt_pose.c
@@ -830,17 +830,21 @@ static bool pose_face_sets_fk_find_masked_floodfill_cb(
   }
 
   const int to_face_set = SCULPT_vertex_face_set_get(ss, to_v);
-  if (SCULPT_vertex_has_unique_face_set(ss, to_v) &&
-      !SCULPT_vertex_has_unique_face_set(ss, from_v) &&
-      SCULPT_vertex_has_face_set(ss, from_v, to_face_set)) {
+  if (!BLI_gset_haskey(data->visited_face_sets, POINTER_FROM_INT(to_face_set))) {
+    if (SCULPT_vertex_has_unique_face_set(ss, to_v) &&
+        !SCULPT_vertex_has_unique_face_set(ss, from_v) &&
+        SCULPT_vertex_has_face_set(ss, from_v, to_face_set)) {
 
-    if (data->floodfill_it[to_v] > data->masked_face_set_it) {
-      data->masked_face_set = to_face_set;
-      data->masked_face_set_it = data->floodfill_it[to_v];
-    }
+      BLI_gset_add(data->visited_face_sets, POINTER_FROM_INT(to_face_set));
+
+      if (data->floodfill_it[to_v] >= data->masked_face_set_it) {
+        data->masked_face_set = to_face_set;
+        data->masked_face_set_it = data->floodfill_it[to_v];
+      }
 
-    if (data->target_face_set == SCULPT_FACE_SET_NONE) {
-      data->target_face_set = to_face_set;
+      if (data->target_face_set == SCULPT_FACE_SET_NONE) {
+        data->target_face_set = to_face_set;
+      }
     }
   }
 
@@ -875,8 +879,10 @@ static SculptPoseIKChain *pose_ik_chain_init_face_sets_fk(
   fdata.masked_face_set = SCULPT_FACE_SET_NONE;
   fdata.target_face_set = SCULPT_FACE_SET_NONE;
   fdata.masked_face_set_it = 0;
+  fdata.visited_face_sets = BLI_gset_int_new_ex("visited_face_sets", 3);
   SCULPT_floodfill_execute(ss, &flood, pose_face_sets_fk_find_masked_floodfill_cb, &fdata);
   SCULPT_floodfill_free(&flood);
+  BLI_gset_free(fdata.visited_face_sets, NULL);
 
   int origin_count = 0;
   float origin_acc[3] = {0.0f};