Patch [#30965] Cancel Sculpt Stroke w/ ESCAPE

If the RMB has not been released after starting a sculpt stroke, then hitting escape will cancel the stroke in progress and undo any changes to the mesh.

This is a slightly faster work-flow than using undo, is a feature available in other paint programs, and also puts in place the infrastructure to add other keys later that could tweak strokes in different ways.

5 files changed, 121 insertions, 65 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_intern.h b/source/blender/editors/sculpt_paint/paint_intern.h
index 200fd8e65c7..267dc043676 100644
--- a/source/blender/editors/sculpt_paint/paint_intern.h
+++ b/source/blender/editors/sculpt_paint/paint_intern.h
@@ -58,10 +58,11 @@ typedef void (*StrokeDone)(struct bContext *C, struct PaintStroke *stroke);
 struct PaintStroke *paint_stroke_new(struct bContext *C,
 					 StrokeGetLocation get_location, StrokeTestStart test_start,
 					 StrokeUpdateStep update_step, StrokeDone done, int event_type);
-void paint_stroke_free(struct PaintStroke *stroke);
+void paint_stroke_data_free(struct wmOperator *op);
 
 int paint_space_stroke_enabled(struct Brush *br);
 
+struct wmKeyMap *paint_stroke_modal_keymap(struct wmKeyConfig *keyconf);
 int paint_stroke_modal(struct bContext *C, struct wmOperator *op, struct wmEvent *event);
 int paint_stroke_exec(struct bContext *C, struct wmOperator *op);
 int paint_stroke_cancel(struct bContext *C, struct wmOperator *op);
diff --git a/source/blender/editors/sculpt_paint/paint_ops.c b/source/blender/editors/sculpt_paint/paint_ops.c
index 6860b551556..e218cfe8fd2 100644
--- a/source/blender/editors/sculpt_paint/paint_ops.c
+++ b/source/blender/editors/sculpt_paint/paint_ops.c
@@ -712,4 +712,7 @@ void ED_keymap_paint(wmKeyConfig *keyconf)
 	RNA_enum_set(WM_keymap_add_item(keymap, "BRUSH_OT_uv_sculpt_tool_set", PKEY, KM_PRESS, 0, 0)->ptr, "tool", UV_SCULPT_TOOL_PINCH);
 	RNA_enum_set(WM_keymap_add_item(keymap, "BRUSH_OT_uv_sculpt_tool_set", GKEY, KM_PRESS, 0, 0)->ptr, "tool", UV_SCULPT_TOOL_GRAB);
 
+	/* paint stroke */
+	keymap = paint_stroke_modal_keymap(keyconf);
+	WM_modalkeymap_assign(keymap, "SCULPT_OT_brush_stroke");
 }
diff --git a/source/blender/editors/sculpt_paint/paint_stroke.c b/source/blender/editors/sculpt_paint/paint_stroke.c
index 448fb37ef54..010278e8621 100644
--- a/source/blender/editors/sculpt_paint/paint_stroke.c
+++ b/source/blender/editors/sculpt_paint/paint_stroke.c
@@ -276,9 +276,30 @@ PaintStroke *paint_stroke_new(bContext *C,
 	return stroke;
 }
 
-void paint_stroke_free(PaintStroke *stroke)
+void paint_stroke_data_free(struct wmOperator *op)
 {
-	MEM_freeN(stroke);
+	MEM_freeN(op->customdata);
+	op->customdata= NULL;
+}
+
+static void stroke_done(struct bContext *C, struct wmOperator *op)
+{
+	struct PaintStroke *stroke = op->customdata;
+
+	if (stroke->stroke_started && stroke->done)
+		stroke->done(C, stroke);
+
+	if (stroke->timer) {
+		WM_event_remove_timer(
+			CTX_wm_manager(C),
+			CTX_wm_window(C),
+			stroke->timer);
+	}
+
+	if (stroke->smooth_stroke_cursor)
+		WM_paint_cursor_end(CTX_wm_manager(C), stroke->smooth_stroke_cursor);
+
+	paint_stroke_data_free(op);
 }
 
 /* Returns zero if the stroke dots should not be spaced, non-zero otherwise */
@@ -289,6 +310,35 @@ int paint_space_stroke_enabled(Brush *br)
 	       !ELEM4(br->sculpt_tool, SCULPT_TOOL_GRAB, SCULPT_TOOL_THUMB, SCULPT_TOOL_ROTATE, SCULPT_TOOL_SNAKE_HOOK);
 }
 
+#define PAINT_STROKE_MODAL_CANCEL 1
+
+/* called in paint_ops.c, on each regeneration of keymaps  */
+struct wmKeyMap *paint_stroke_modal_keymap(struct wmKeyConfig *keyconf)
+{
+	static struct EnumPropertyItem modal_items[] = {
+		{PAINT_STROKE_MODAL_CANCEL, "CANCEL", 0,
+		"Cancel",
+		"Cancel and undo a stroke in progress"},
+
+		{ 0 }
+	};
+
+	static const char *name= "Paint Stroke Modal";
+
+	struct wmKeyMap *keymap= WM_modalkeymap_get(keyconf, name);
+
+	/* this function is called for each spacetype, only needs to add map once */
+	if (!keymap) {
+		keymap= WM_modalkeymap_add(keyconf, name, modal_items);
+
+		/* items for modal map */
+		WM_modalkeymap_add_item(
+			keymap, ESCKEY, KM_PRESS, KM_ANY, 0, PAINT_STROKE_MODAL_CANCEL);
+	}
+
+	return keymap;
+}
+
 int paint_stroke_modal(bContext *C, wmOperator *op, wmEvent *event)
 {
 	PaintStroke *stroke = op->customdata;
@@ -319,16 +369,16 @@ int paint_stroke_modal(bContext *C, wmOperator *op, wmEvent *event)
 		//ED_region_tag_redraw(ar);
 	}
 
-	if (event->type == stroke->event_type && event->val == KM_RELEASE) {
-		/* exit stroke, free data */
-		if (stroke->smooth_stroke_cursor)
-			WM_paint_cursor_end(CTX_wm_manager(C), stroke->smooth_stroke_cursor);
-
-		if (stroke->timer)
-			WM_event_remove_timer(CTX_wm_manager(C), CTX_wm_window(C), stroke->timer);
+	/* Cancel */
+	if (event->type == EVT_MODAL_MAP && event->val == PAINT_STROKE_MODAL_CANCEL) {
+		if (op->type->cancel)
+			return op->type->cancel(C, op);
+		else
+			return paint_stroke_cancel(C, op);
+	}
 
-		stroke->done(C, stroke);
-		MEM_freeN(stroke);
+	if (event->type == stroke->event_type && event->val == KM_RELEASE) {
+		stroke_done(C, op);
 		return OPERATOR_FINISHED;
 	}
 	else if ((first) ||
@@ -383,24 +433,14 @@ int paint_stroke_exec(bContext *C, wmOperator *op)
 	}
 	RNA_END;
 
-	stroke->done(C, stroke);
-
-	MEM_freeN(stroke);
-	op->customdata = NULL;
+	stroke_done(C, op);
 
 	return OPERATOR_FINISHED;
 }
 
 int paint_stroke_cancel(bContext *C, wmOperator *op)
 {
-	PaintStroke *stroke = op->customdata;
-
-	if (stroke->done)
-		stroke->done(C, stroke);
-
-	MEM_freeN(stroke);
-	op->customdata = NULL;
-
+	stroke_done(C, op);
 	return OPERATOR_CANCELLED;
 }
 
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index 18ddc965976..9c89eb7c573 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -2421,7 +2421,7 @@ static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, P
 	ED_region_tag_redraw(vc->ar);
 }
 
-static void wpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
+static void wpaint_stroke_done(const bContext *C, struct PaintStroke *stroke)
 {
 	ToolSettings *ts = CTX_data_tool_settings(C);
 	Object *ob = CTX_data_active_object(C);
@@ -2945,7 +2945,7 @@ static void vpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, P
 	}
 }
 
-static void vpaint_stroke_done(bContext *C, struct PaintStroke *stroke)
+static void vpaint_stroke_done(const bContext *C, struct PaintStroke *stroke)
 {
 	ToolSettings *ts = CTX_data_tool_settings(C);
 	struct VPaintData *vpd = paint_stroke_mode_data(stroke);
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 51fe73bb527..ca9bb51aaef 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -264,6 +264,50 @@ typedef struct StrokeCache {
 	rcti previous_r; /* previous redraw rectangle */
 } StrokeCache;
 
+
+/*** paint mesh ***/
+
+static void paint_mesh_restore_co(Sculpt *sd, SculptSession *ss)
+{
+	StrokeCache *cache = ss->cache;
+	int i;
+
+	PBVHNode **nodes;
+	int n, totnode;
+
+	BLI_pbvh_search_gather(ss->pbvh, NULL, NULL, &nodes, &totnode);
+
+	#pragma omp parallel for schedule(guided) if (sd->flags & SCULPT_USE_OPENMP)
+	for (n = 0; n < totnode; n++) {
+		SculptUndoNode *unode;
+		
+		unode = sculpt_undo_get_node(nodes[n]);
+		if (unode) {
+			PBVHVertexIter vd;
+
+			BLI_pbvh_vertex_iter_begin(ss->pbvh, nodes[n], vd, PBVH_ITER_UNIQUE) {
+				copy_v3_v3(vd.co, unode->co[vd.i]);
+				if (vd.no) copy_v3_v3_short(vd.no, unode->no[vd.i]);
+				else normal_short_to_float_v3(vd.fno, unode->no[vd.i]);
+
+				if (vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
+			}
+			BLI_pbvh_vertex_iter_end;
+
+			BLI_pbvh_node_mark_update(nodes[n]);
+		}
+	}
+
+	if (ss->face_normals) {
+		float *fn = ss->face_normals;
+		for (i = 0; i < ss->totpoly; ++i, fn += 3)
+			copy_v3_v3(fn, cache->face_norms[i]);
+	}
+
+	if (nodes)
+		MEM_freeN(nodes);
+}
+
 /*** BVH Tree ***/
 
 /* Get a screen-space rectangle of the modified area */
@@ -3193,7 +3237,7 @@ static void sculpt_update_cache_variants(bContext *C, Sculpt *sd, Object *ob,
 	sd->special_rotation = cache->special_rotation;
 }
 
-static void sculpt_stroke_modifiers_check(bContext *C, Object *ob)
+static void sculpt_stroke_modifiers_check(const bContext *C, Object *ob)
 {
 	SculptSession *ss = ob->sculpt;
 
@@ -3329,43 +3373,7 @@ static void sculpt_restore_mesh(Sculpt *sd, SculptSession *ss)
 	     brush_use_size_pressure(ss->cache->vc->scene, brush)) ||
 	    (brush->flag & BRUSH_RESTORE_MESH))
 	{
-		StrokeCache *cache = ss->cache;
-		int i;
-
-		PBVHNode **nodes;
-		int n, totnode;
-
-		BLI_pbvh_search_gather(ss->pbvh, NULL, NULL, &nodes, &totnode);
-
-		#pragma omp parallel for schedule(guided) if (sd->flags & SCULPT_USE_OPENMP)
-		for (n = 0; n < totnode; n++) {
-			SculptUndoNode *unode;
-			
-			unode = sculpt_undo_get_node(nodes[n]);
-			if (unode) {
-				PBVHVertexIter vd;
-
-				BLI_pbvh_vertex_iter_begin(ss->pbvh, nodes[n], vd, PBVH_ITER_UNIQUE) {
-					copy_v3_v3(vd.co, unode->co[vd.i]);
-					if (vd.no) copy_v3_v3_short(vd.no, unode->no[vd.i]);
-					else normal_short_to_float_v3(vd.fno, unode->no[vd.i]);
-
-					if (vd.mvert) vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
-				}
-				BLI_pbvh_vertex_iter_end;
-
-				BLI_pbvh_node_mark_update(nodes[n]);
-			}
-		}
-
-		if (ss->face_normals) {
-			float *fn = ss->face_normals;
-			for (i = 0; i < ss->totpoly; ++i, fn += 3)
-				copy_v3_v3(fn, cache->face_norms[i]);
-		}
-
-		if (nodes)
-			MEM_freeN(nodes);
+		paint_mesh_restore_co(sd, ss);
 	}
 }
 
@@ -3551,7 +3559,7 @@ static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, wmEvent *even
 	                                          "ignore_background_click");
 
 	if (ignore_background_click && !over_mesh(C, op, event->x, event->y)) {
-		paint_stroke_free(stroke);
+		paint_stroke_data_free(op);
 		return OPERATOR_PASS_THROUGH;
 	}
 	
@@ -3583,6 +3591,10 @@ static int sculpt_brush_stroke_cancel(bContext *C, wmOperator *op)
 	SculptSession *ss = ob->sculpt;
 	Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
 
+	if (ss->cache) {
+		paint_mesh_restore_co(sd, ss);
+	}
+
 	paint_stroke_cancel(C, op);
 
 	if (ss->cache) {