Fix mask expand creating wrong masks when the cursor is not over the mesh

This forces the mask expand modal operator to use the maximum
iteration when the cursor is not over the mesh, masking the entire
connected component.

This fixes the issue for both expanding masks and face sets.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D9105

1 files changed, 8 insertions, 2 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_mask_expand.c b/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
index e403bada316..5a921383ac3 100644
--- a/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
+++ b/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
@@ -188,8 +188,14 @@ static int sculpt_mask_expand_modal(bContext *C, wmOperator *op, const wmEvent *
     float mouse[2];
     mouse[0] = event->mval[0];
     mouse[1] = event->mval[1];
-    SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
-    mask_expand_update_it = ss->filter_cache->mask_update_it[(int)SCULPT_active_vertex_get(ss)];
+    if (SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false)) {
+      /* The cursor is over the mesh, get the update iteration from the updated active vertex. */
+      mask_expand_update_it = ss->filter_cache->mask_update_it[(int)SCULPT_active_vertex_get(ss)];
+    }
+    else {
+      /* When the cursor is outside the mesh, affect the entire connected component. */
+      mask_expand_update_it = ss->filter_cache->mask_update_last_it - 1;
+    }
   }
 
   if ((event->type == EVT_ESCKEY && event->val == KM_PRESS) ||