GP: box/lasso support for select-operations

This adds support for more adbvanced select operations
matching 3D view select operators.

Also de-duplicate box/lasso select code.

5 files changed, 132 insertions, 163 deletions

diff --git a/source/blender/blenkernel/BKE_gpencil.h b/source/blender/blenkernel/BKE_gpencil.h
index 1debfecc895..14c4e8fec32 100644
--- a/source/blender/blenkernel/BKE_gpencil.h
+++ b/source/blender/blenkernel/BKE_gpencil.h
@@ -149,6 +149,8 @@ bool BKE_gpencil_data_minmax(
 bool BKE_gpencil_stroke_minmax(
 	const struct bGPDstroke *gps, const bool use_select,
 	float r_min[3], float r_max[3]);
+bool BKE_gpencil_stroke_select_check(
+	const struct bGPDstroke *gps);
 
 struct BoundBox *BKE_gpencil_boundbox_get(struct Object *ob);
 void BKE_gpencil_centroid_3D(struct bGPdata *gpd, float r_centroid[3]);
diff --git a/source/blender/blenkernel/intern/gpencil.c b/source/blender/blenkernel/intern/gpencil.c
index e6d5514a0f1..ed551b1c109 100644
--- a/source/blender/blenkernel/intern/gpencil.c
+++ b/source/blender/blenkernel/intern/gpencil.c
@@ -1133,6 +1133,19 @@ bool BKE_gpencil_data_minmax(Object *ob, const bGPdata *gpd, float r_min[3], flo
 	return changed;
 }
 
+bool BKE_gpencil_stroke_select_check(
+        const bGPDstroke *gps)
+{
+	const bGPDspoint *pt;
+	int i;
+	for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
+		if (pt->flag & GP_SPOINT_SELECT) {
+			return true;
+		}
+	}
+	return false;
+}
+
 /* compute center of bounding box */
 void BKE_gpencil_centroid_3D(bGPdata *gpd, float r_centroid[3])
 {
diff --git a/source/blender/editors/gpencil/gpencil_intern.h b/source/blender/editors/gpencil/gpencil_intern.h
index 4656c6fc20c..a17591b87cc 100644
--- a/source/blender/editors/gpencil/gpencil_intern.h
+++ b/source/blender/editors/gpencil/gpencil_intern.h
@@ -199,14 +199,14 @@ bool gp_stroke_inside_circle(
 void gp_point_conversion_init(struct bContext *C, GP_SpaceConversion *r_gsc);
 
 void gp_point_to_xy(
-        GP_SpaceConversion *settings, struct bGPDstroke *gps, struct bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const struct bGPDstroke *gps, const struct bGPDspoint *pt,
         int *r_x, int *r_y);
 
 void gp_point_to_xy_fl(
-        GP_SpaceConversion *gsc, bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const bGPDstroke *gps, const bGPDspoint *pt,
         float *r_x, float *r_y);
 
-void gp_point_to_parent_space(bGPDspoint *pt, float diff_mat[4][4], bGPDspoint *r_pt);
+void gp_point_to_parent_space(const bGPDspoint *pt, const float diff_mat[4][4], bGPDspoint *r_pt);
 /**
  * Change points position relative to parent object
  */
diff --git a/source/blender/editors/gpencil/gpencil_select.c b/source/blender/editors/gpencil/gpencil_select.c
index cb3c0575f81..22f2c5e995a 100644
--- a/source/blender/editors/gpencil/gpencil_select.c
+++ b/source/blender/editors/gpencil/gpencil_select.c
@@ -1014,21 +1014,25 @@ void GPENCIL_OT_select_circle(wmOperatorType *ot)
 	WM_operator_properties_gesture_circle_select(ot);
 }
 
-/* ********************************************** */
-/* Box Selection */
 
-static int gpencil_box_select_exec(bContext *C, wmOperator *op)
+typedef bool (*GPencilTestFn)(
+        bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const float diff_mat[4][4], void *user_data);
+
+static int gpencil_generic_select_exec(
+        bContext *C, wmOperator *op,
+        GPencilTestFn is_inside_fn, void *user_data)
 {
 	bGPdata *gpd = ED_gpencil_data_get_active(C);
 	ToolSettings *ts = CTX_data_tool_settings(C);
 	ScrArea *sa = CTX_wm_area(C);
+	const bool strokemode = (
+	        (ts->gpencil_selectmode == GP_SELECTMODE_STROKE) &&
+	        ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0));
+	const eSelectOp sel_op = RNA_enum_get(op->ptr, "mode");
 
-	const bool select = !RNA_boolean_get(op->ptr, "deselect");
-	bool extend = RNA_boolean_get(op->ptr, "extend") && ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0);
-	const bool strokemode = (ts->gpencil_selectmode == GP_SELECTMODE_STROKE) && ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0);
 
 	GP_SpaceConversion gsc = {NULL};
-	rcti rect = {0};
 
 	bool changed = false;
 
@@ -1038,15 +1042,12 @@ static int gpencil_box_select_exec(bContext *C, wmOperator *op)
 		return OPERATOR_CANCELLED;
 	}
 
-	if (strokemode) {
-		extend = false;
-	}
-
 	/* init space conversion stuff */
 	gp_point_conversion_init(C, &gsc);
 
 	/* deselect all strokes first? */
-	if (select && !extend) {
+	if (SEL_OP_USE_PRE_DESELECT(sel_op)) {
+
 		CTX_DATA_BEGIN(C, bGPDstroke *, gps, editable_gpencil_strokes)
 		{
 			bGPDspoint *pt;
@@ -1061,9 +1062,6 @@ static int gpencil_box_select_exec(bContext *C, wmOperator *op)
 		CTX_DATA_END;
 	}
 
-	/* get settings from operator */
-	WM_operator_properties_border_to_rcti(op, &rect);
-
 	/* select/deselect points */
 	GP_EDITABLE_STROKES_BEGIN(C, gpl, gps)
 	{
@@ -1072,40 +1070,40 @@ static int gpencil_box_select_exec(bContext *C, wmOperator *op)
 		int i;
 		bool hit = false;
 		for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
-			int x0, y0;
-
 			/* convert point coords to screenspace */
-			bGPDspoint pt2;
-			gp_point_to_parent_space(pt, diff_mat, &pt2);
-			gp_point_to_xy(&gsc, gps, &pt2, &x0, &y0);
-
-			/* test if in selection rect */
-			if ((!ELEM(V2D_IS_CLIPPED, x0, y0)) && BLI_rcti_isect_pt(&rect, x0, y0)) {
-				hit = true;
-				if (select) {
-					pt->flag |= GP_SPOINT_SELECT;
-				}
-				else {
-					pt->flag &= ~GP_SPOINT_SELECT;
+			const bool is_inside = is_inside_fn(gps, pt, &gsc, diff_mat, user_data);
+
+			if (strokemode == false) {
+				const bool is_select = (pt->flag & GP_SPOINT_SELECT) != 0;
+				const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
+				if (sel_op_result != -1) {
+					SET_FLAG_FROM_TEST(pt->flag, sel_op_result, GP_SPOINT_SELECT);
+					changed = true;
 				}
-
-				changed = true;
-
-				/* if stroke mode, don't check more points */
-				if ((hit) && (strokemode)) {
+			}
+			else {
+				if (is_inside) {
+					hit = true;
 					break;
 				}
 			}
 		}
+
 		/* if stroke mode expand selection */
-		if ((hit) && (strokemode)) {
-			for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
-				if (select) {
-					pt->flag |= GP_SPOINT_SELECT;
-				}
-				else {
-					pt->flag &= ~GP_SPOINT_SELECT;
+		if (strokemode) {
+			const bool is_select = BKE_gpencil_stroke_select_check(gps);
+			const bool is_inside = hit;
+			const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
+			if (sel_op_result != -1) {
+				for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
+					if (sel_op_result) {
+						pt->flag |= GP_SPOINT_SELECT;
+					}
+					else {
+						pt->flag &= ~GP_SPOINT_SELECT;
+					}
 				}
+				changed = true;
 			}
 		}
 
@@ -1134,6 +1132,35 @@ static int gpencil_box_select_exec(bContext *C, wmOperator *op)
 	return OPERATOR_FINISHED;
 }
 
+/* ********************************************** */
+/* Box Selection */
+
+struct GP_SelectBoxUserData {
+	rcti rect;
+};
+
+static bool gpencil_test_box(
+        bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const float diff_mat[4][4], void *user_data)
+{
+	const struct GP_SelectBoxUserData *data = user_data;
+	bGPDspoint pt2;
+	int x0, y0;
+	gp_point_to_parent_space(pt, diff_mat, &pt2);
+	gp_point_to_xy(gsc, gps, &pt2, &x0, &y0);
+	return ((!ELEM(V2D_IS_CLIPPED, x0, y0)) &&
+	        BLI_rcti_isect_pt(&data->rect, x0, y0));
+}
+
+static int gpencil_box_select_exec(bContext *C, wmOperator *op)
+{
+	struct GP_SelectBoxUserData data = {0};
+	WM_operator_properties_border_to_rcti(op, &data.rect);
+	return gpencil_generic_select_exec(
+	        C, op,
+	        gpencil_test_box, &data);
+}
+
 void GPENCIL_OT_select_box(wmOperatorType *ot)
 {
 	/* identifiers */
@@ -1153,129 +1180,55 @@ void GPENCIL_OT_select_box(wmOperatorType *ot)
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO | OPTYPE_USE_EVAL_DATA;
 
 	/* rna */
-	WM_operator_properties_gesture_box_select(ot);
+	WM_operator_properties_select_operation(ot);
+	WM_operator_properties_gesture_box(ot);
 }
 
 /* ********************************************** */
 /* Lasso */
 
-static int gpencil_lasso_select_exec(bContext *C, wmOperator *op)
-{
-	bGPdata *gpd = ED_gpencil_data_get_active(C);
-	ToolSettings *ts = CTX_data_tool_settings(C);
-	GP_SpaceConversion gsc = {NULL};
-	rcti rect = {0};
+struct GP_SelectLassoUserData {
+	rcti rect;
+	const int (*mcords)[2];
+	int         mcords_len;
+};
 
-	bool extend = RNA_boolean_get(op->ptr, "extend") && ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0);
-	const bool select = !RNA_boolean_get(op->ptr, "deselect");
-	const bool strokemode = (ts->gpencil_selectmode == GP_SELECTMODE_STROKE) && ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0);
-	int mcords_tot;
-	const int (*mcords)[2] = WM_gesture_lasso_path_to_array(C, op, &mcords_tot);
+static bool gpencil_test_lasso(
+        bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const float diff_mat[4][4],
+        void *user_data)
+{
+	const struct GP_SelectLassoUserData *data = user_data;
+	bGPDspoint pt2;
+	int x0, y0;
+	gp_point_to_parent_space(pt, diff_mat, &pt2);
+	gp_point_to_xy(gsc, gps, &pt2, &x0, &y0);
+	/* test if in lasso boundbox + within the lasso noose */
+	return ((!ELEM(V2D_IS_CLIPPED, x0, y0)) &&
+	        BLI_rcti_isect_pt(&data->rect, x0, y0) &&
+	        BLI_lasso_is_point_inside(data->mcords, data->mcords_len, x0, y0, INT_MAX));
+}
 
-	bool changed = false;
+static int gpencil_lasso_select_exec(bContext *C, wmOperator *op)
+{
+	struct GP_SelectLassoUserData data = {0};
+	data.mcords = WM_gesture_lasso_path_to_array(C, op, &data.mcords_len);
 
-	/* sanity check */
-	if (mcords == NULL)
+	/* Sanity check. */
+	if (data.mcords == NULL) {
 		return OPERATOR_PASS_THROUGH;
-
-	if (strokemode) {
-		extend = false;
 	}
 
-	/* compute boundbox of lasso (for faster testing later) */
-	BLI_lasso_boundbox(&rect, mcords, mcords_tot);
+	/* Compute boundbox of lasso (for faster testing later). */
+	BLI_lasso_boundbox(&data.rect, data.mcords, data.mcords_len);
 
-	/* init space conversion stuff */
-	gp_point_conversion_init(C, &gsc);
+	int ret = gpencil_generic_select_exec(
+	        C, op,
+	        gpencil_test_lasso, &data);
 
-	/* deselect all strokes first? */
-	if (select && !extend) {
-		CTX_DATA_BEGIN(C, bGPDstroke *, gps, editable_gpencil_strokes)
-		{
-			bGPDspoint *pt;
-			int i;
+	MEM_freeN((void *)data.mcords);
 
-			for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
-				pt->flag &= ~GP_SPOINT_SELECT;
-			}
-
-			gps->flag &= ~GP_STROKE_SELECT;
-		}
-		CTX_DATA_END;
-	}
-
-	/* select/deselect points */
-	GP_EDITABLE_STROKES_BEGIN(C, gpl, gps)
-	{
-		bGPDspoint *pt;
-		int i;
-		bool hit = false;
-		for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
-			int x0, y0;
-
-			/* convert point coords to screenspace */
-			bGPDspoint pt2;
-			gp_point_to_parent_space(pt, diff_mat, &pt2);
-			gp_point_to_xy(&gsc, gps, &pt2, &x0, &y0);
-			/* test if in lasso boundbox + within the lasso noose */
-			if ((!ELEM(V2D_IS_CLIPPED, x0, y0)) && BLI_rcti_isect_pt(&rect, x0, y0) &&
-			    BLI_lasso_is_point_inside(mcords, mcords_tot, x0, y0, INT_MAX))
-			{
-				hit = true;
-				if (select) {
-					pt->flag |= GP_SPOINT_SELECT;
-				}
-				else {
-					pt->flag &= ~GP_SPOINT_SELECT;
-				}
-
-				changed = true;
-
-				/* if stroke mode, don't check more points */
-				if ((hit) && (strokemode)) {
-					break;
-				}
-			}
-		}
-
-		/* if stroke mode expand selection */
-		if ((hit) && (strokemode)) {
-			for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
-				if (select) {
-					pt->flag |= GP_SPOINT_SELECT;
-				}
-				else {
-					pt->flag &= ~GP_SPOINT_SELECT;
-				}
-			}
-		}
-
-		/* Ensure that stroke selection is in sync with its points */
-		BKE_gpencil_stroke_sync_selection(gps);
-	}
-	GP_EDITABLE_STROKES_END;
-
-	/* cleanup */
-	MEM_freeN((void *)mcords);
-
-	/* if paint mode,delete selected points */
-	if (gpd->flag & GP_DATA_STROKE_PAINTMODE) {
-		gp_delete_selected_point_wrap(C);
-		changed = true;
-		DEG_id_tag_update(&gpd->id, OB_RECALC_OB | OB_RECALC_DATA);
-	}
-
-	/* updates */
-	if (changed) {
-		DEG_id_tag_update(&gpd->id, OB_RECALC_DATA);
-
-		/* copy on write tag is needed, or else no refresh happens */
-		DEG_id_tag_update(&gpd->id, DEG_TAG_COPY_ON_WRITE);
-
-		WM_event_add_notifier(C, NC_GPENCIL | NA_SELECTED, NULL);
-	}
-
-	return OPERATOR_FINISHED;
+	return ret;
 }
 
 void GPENCIL_OT_select_lasso(wmOperatorType *ot)
@@ -1294,7 +1247,8 @@ void GPENCIL_OT_select_lasso(wmOperatorType *ot)
 	ot->flag = OPTYPE_UNDO | OPTYPE_USE_EVAL_DATA;
 
 	/* properties */
-	WM_operator_properties_gesture_lasso_select(ot);
+	WM_operator_properties_select_operation(ot);
+	WM_operator_properties_gesture_lasso(ot);
 }
 
 /* ********************************************** */
diff --git a/source/blender/editors/gpencil/gpencil_utils.c b/source/blender/editors/gpencil/gpencil_utils.c
index f58df5e2710..85ba5c55b80 100644
--- a/source/blender/editors/gpencil/gpencil_utils.c
+++ b/source/blender/editors/gpencil/gpencil_utils.c
@@ -555,7 +555,7 @@ void gp_point_conversion_init(bContext *C, GP_SpaceConversion *r_gsc)
  * \param diff_mat: Matrix with the difference between original parent matrix
  * \param[out] r_pt: Pointer to new point after apply matrix
  */
-void gp_point_to_parent_space(bGPDspoint *pt, float diff_mat[4][4], bGPDspoint *r_pt)
+void gp_point_to_parent_space(const bGPDspoint *pt, const float diff_mat[4][4], bGPDspoint *r_pt)
 {
 	float fpt[3];
 
@@ -612,12 +612,12 @@ void gp_apply_parent_point(Depsgraph *depsgraph, Object *obact, bGPdata *gpd, bG
  * \warning This assumes that the caller has already checked whether the stroke in question can be drawn.
  */
 void gp_point_to_xy(
-        GP_SpaceConversion *gsc, bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const bGPDstroke *gps, const bGPDspoint *pt,
         int *r_x, int *r_y)
 {
-	ARegion *ar = gsc->ar;
-	View2D *v2d = gsc->v2d;
-	rctf *subrect = gsc->subrect;
+	const ARegion *ar = gsc->ar;
+	const View2D *v2d = gsc->v2d;
+	const rctf *subrect = gsc->subrect;
 	int xyval[2];
 
 	/* sanity checks */
@@ -666,12 +666,12 @@ void gp_point_to_xy(
  * \warning This assumes that the caller has already checked whether the stroke in question can be drawn
  */
 void gp_point_to_xy_fl(
-        GP_SpaceConversion *gsc, bGPDstroke *gps, bGPDspoint *pt,
+        const GP_SpaceConversion *gsc, const bGPDstroke *gps, const bGPDspoint *pt,
         float *r_x, float *r_y)
 {
-	ARegion *ar = gsc->ar;
-	View2D *v2d = gsc->v2d;
-	rctf *subrect = gsc->subrect;
+	const ARegion *ar = gsc->ar;
+	const View2D *v2d = gsc->v2d;
+	const rctf *subrect = gsc->subrect;
 	float xyval[2];
 
 	/* sanity checks */