Fix #35372: sculpting/painting long brush strokes with small brush size would

take up a lot of memory.

The operator was recording an array with all stroke points. However this was not
particularly useful, only sculpt mode had exec() implemented to redo the stroke,
but it was not registering the operator anyway so there was no way to access the
data after the operator was done. So no one was using this anyway.

I did now implement exec for the paint modes so you can call the operator with
stroke points from a script.

3 files changed, 75 insertions, 16 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_image.c b/source/blender/editors/sculpt_paint/paint_image.c
index 7e721277f91..a6e3b206e0f 100644
--- a/source/blender/editors/sculpt_paint/paint_image.c
+++ b/source/blender/editors/sculpt_paint/paint_image.c
@@ -478,7 +478,7 @@ void paint_brush_exit_tex(Brush *brush)
 }
 
 
-static PaintOperation *texture_paint_init(bContext *C, wmOperator *op, const wmEvent *event)
+static PaintOperation *texture_paint_init(bContext *C, wmOperator *op, float mouse[2])
 {
 	Scene *scene = CTX_data_scene(C);
 	ToolSettings *settings = scene->toolsettings;
@@ -486,11 +486,8 @@ static PaintOperation *texture_paint_init(bContext *C, wmOperator *op, const wmE
 	int mode = RNA_enum_get(op->ptr, "mode");
 	view3d_set_viewcontext(C, &pop->vc);
 
-	/* TODO Should avoid putting this here. Instead, last position should be requested
-	 * from stroke system. */
-	pop->prevmouse[0] = event->mval[0];
-	pop->prevmouse[1] = event->mval[1];
-
+	pop->prevmouse[0] = mouse[0];
+	pop->prevmouse[1] = mouse[1];
 
 	/* initialize from context */
 	if (CTX_wm_region_view3d(C)) {
@@ -609,18 +606,23 @@ static int paint_stroke_test_start(bContext *UNUSED(C), wmOperator *UNUSED(op),
 static int paint_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
 	PaintOperation *pop;
-	struct PaintStroke *stroke;
+	float mouse[2];
 	int retval;
 
-	if (!(pop = texture_paint_init(C, op, event))) {
+	/* TODO Should avoid putting this here. Instead, last position should be requested
+	 * from stroke system. */
+	mouse[0] = event->mval[0];
+	mouse[1] = event->mval[1];
+
+	if (!(pop = texture_paint_init(C, op, mouse))) {
 		return OPERATOR_CANCELLED;
 	}
 
-	stroke = op->customdata = paint_stroke_new(C, NULL, paint_stroke_test_start,
+	op->customdata = paint_stroke_new(C, NULL, paint_stroke_test_start,
 	                                  paint_stroke_update_step,
 	                                  paint_stroke_redraw,
 	                                  paint_stroke_done, event->type);
-	paint_stroke_set_mode_data(stroke, pop);
+	paint_stroke_set_mode_data(op->customdata, pop);
 	/* add modal handler */
 	WM_event_add_modal_handler(C, op);
 
@@ -631,6 +633,35 @@ static int paint_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 	return OPERATOR_RUNNING_MODAL;
 }
 
+static int paint_exec(bContext *C, wmOperator *op)
+{
+	PaintOperation *pop;
+	PropertyRNA *strokeprop;
+	PointerRNA firstpoint;
+	float mouse[2];
+
+	strokeprop = RNA_struct_find_property(op->ptr, "stroke");
+
+	if (!RNA_property_collection_lookup_int(op->ptr, strokeprop, 0, &firstpoint))
+		return OPERATOR_CANCELLED;
+
+	RNA_float_get_array(&firstpoint, "mouse", mouse);
+
+	if (!(pop = texture_paint_init(C, op, mouse))) {
+		return OPERATOR_CANCELLED;
+	}
+
+	op->customdata = paint_stroke_new(C, NULL, paint_stroke_test_start,
+	                                  paint_stroke_update_step,
+	                                  paint_stroke_redraw,
+	                                  paint_stroke_done, 0);
+	paint_stroke_set_mode_data(op->customdata, pop);
+
+	/* frees op->customdata */
+	paint_stroke_exec(C, op);
+
+	return OPERATOR_FINISHED;
+}
 
 void PAINT_OT_image_paint(wmOperatorType *ot)
 {
@@ -648,12 +679,12 @@ void PAINT_OT_image_paint(wmOperatorType *ot)
 	/* api callbacks */
 	ot->invoke = paint_invoke;
 	ot->modal = paint_stroke_modal;
-	/* ot->exec = paint_exec; <-- needs stroke property */
+	ot->exec = paint_exec;
 	ot->poll = image_paint_poll;
 	ot->cancel = paint_stroke_cancel;
 
 	/* flags */
-	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO | OPTYPE_BLOCKING;
+	ot->flag = OPTYPE_UNDO | OPTYPE_BLOCKING;
 
 	RNA_def_enum(ot->srna, "mode", stroke_mode_items, BRUSH_STROKE_NORMAL,
 	             "Paint Stroke Mode",
diff --git a/source/blender/editors/sculpt_paint/paint_stroke.c b/source/blender/editors/sculpt_paint/paint_stroke.c
index 90c0d7e3a7a..d3cb53c94fc 100644
--- a/source/blender/editors/sculpt_paint/paint_stroke.c
+++ b/source/blender/editors/sculpt_paint/paint_stroke.c
@@ -328,6 +328,10 @@ static void paint_brush_stroke_add_step(bContext *C, wmOperator *op, const wmEve
 
 	stroke->update_step(C, stroke, &itemptr);
 
+	/* don't record this for now, it takes up a lot of memory when doing long
+	 * strokes with small brush size, and operators have register disabled */
+	RNA_collection_clear(op->ptr, "stroke");
+
 	/* always redraw region if brush is shown */
 	if (ar && (paint->flags & PAINT_SHOW_BRUSH))
 		WM_paint_cursor_tag_redraw(window, ar);
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index 388b0e769e8..5feb59c6082 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -2575,6 +2575,18 @@ static int wpaint_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 	return OPERATOR_RUNNING_MODAL;
 }
 
+static int wpaint_exec(bContext *C, wmOperator *op)
+{
+	op->customdata = paint_stroke_new(C, NULL, wpaint_stroke_test_start,
+	                                  wpaint_stroke_update_step, NULL,
+	                                  wpaint_stroke_done, 0);
+
+	/* frees op->customdata */
+	paint_stroke_exec(C, op);
+
+	return OPERATOR_FINISHED;
+}
+
 static int wpaint_cancel(bContext *C, wmOperator *op)
 {
 	paint_stroke_cancel(C, op);
@@ -2593,12 +2605,12 @@ void PAINT_OT_weight_paint(wmOperatorType *ot)
 	/* api callbacks */
 	ot->invoke = wpaint_invoke;
 	ot->modal = paint_stroke_modal;
-	/* ot->exec = vpaint_exec; <-- needs stroke property */
+	ot->exec = wpaint_exec;
 	ot->poll = weight_paint_poll;
 	ot->cancel = wpaint_cancel;
 	
 	/* flags */
-	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO | OPTYPE_BLOCKING;
+	ot->flag = OPTYPE_UNDO | OPTYPE_BLOCKING;
 
 	RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
 }
@@ -3105,6 +3117,18 @@ static int vpaint_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 	return OPERATOR_RUNNING_MODAL;
 }
 
+static int vpaint_exec(bContext *C, wmOperator *op)
+{
+	op->customdata = paint_stroke_new(C, NULL, vpaint_stroke_test_start,
+	                                  vpaint_stroke_update_step, NULL,
+	                                  vpaint_stroke_done, 0);
+
+	/* frees op->customdata */
+	paint_stroke_exec(C, op);
+
+	return OPERATOR_FINISHED;
+}
+
 static int vpaint_cancel(bContext *C, wmOperator *op)
 {
 	paint_stroke_cancel(C, op);
@@ -3122,12 +3146,12 @@ void PAINT_OT_vertex_paint(wmOperatorType *ot)
 	/* api callbacks */
 	ot->invoke = vpaint_invoke;
 	ot->modal = paint_stroke_modal;
-	/* ot->exec = vpaint_exec; <-- needs stroke property */
+	ot->exec = vpaint_exec;
 	ot->poll = vertex_paint_poll;
 	ot->cancel = vpaint_cancel;
 	
 	/* flags */
-	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO | OPTYPE_BLOCKING;
+	ot->flag = OPTYPE_UNDO | OPTYPE_BLOCKING;
 
 	RNA_def_collection_runtime(ot->srna, "stroke", &RNA_OperatorStrokeElement, "Stroke", "");
 }