Fix T42984 detail flood fill not respecting mask values for smooth/sharp

curves.

Issue here is that brush curve could return negative values. This would
result in overflow of mask values. Those were not visible during real
time preview because result would be clamped.

We had two functions in the code, one of which allowed negatives but I
don't think that we really want that, users have no control over the
negative values at all anyway.

Thanks to the reporter, Leon Cheung for figuring out the issue :)

6 files changed, 8 insertions, 20 deletions

diff --git a/source/blender/blenkernel/BKE_brush.h b/source/blender/blenkernel/BKE_brush.h
index d48753590bb..42379c02e89 100644
--- a/source/blender/blenkernel/BKE_brush.h
+++ b/source/blender/blenkernel/BKE_brush.h
@@ -66,8 +66,7 @@ void BKE_brush_randomize_texture_coordinates(struct UnifiedPaintSettings *ups, b
 
 /* brush curve */
 void BKE_brush_curve_preset(struct Brush *b, int preset);
-float BKE_brush_curve_strength_clamp(struct Brush *br, float p, const float len);
-float BKE_brush_curve_strength(struct Brush *br, float p, const float len); /* used for sculpt */
+float BKE_brush_curve_strength(struct Brush *br, float p, const float len);
 
 /* sampling */
 float BKE_brush_sample_tex_3D(const Scene *scene, struct Brush *br, const float point[3],
diff --git a/source/blender/blenkernel/intern/brush.c b/source/blender/blenkernel/intern/brush.c
index cdbcffe0ce4..0a57dc5c945 100644
--- a/source/blender/blenkernel/intern/brush.c
+++ b/source/blender/blenkernel/intern/brush.c
@@ -964,7 +964,7 @@ void BKE_brush_randomize_texture_coordinates(UnifiedPaintSettings *ups, bool mas
 }
 
 /* Uses the brush curve control to find a strength value between 0 and 1 */
-float BKE_brush_curve_strength_clamp(Brush *br, float p, const float len)
+float BKE_brush_curve_strength(Brush *br, float p, const float len)
 {
 	float strength;
 
@@ -977,17 +977,6 @@ float BKE_brush_curve_strength_clamp(Brush *br, float p, const float len)
 
 	return strength;
 }
-/* same as above but can return negative values if the curve enables
- * used for sculpt only */
-float BKE_brush_curve_strength(Brush *br, float p, const float len)
-{
-	if (p >= len)
-		p = 1.0f;
-	else
-		p = p / len;
-
-	return curvemapping_evaluateF(br->curve, 0, p);
-}
 
 /* TODO: should probably be unified with BrushPainter stuff? */
 unsigned int *BKE_brush_gen_texture_cache(Brush *br, int half_side, bool use_secondary)
@@ -1045,7 +1034,7 @@ struct ImBuf *BKE_brush_gen_radial_control_imbuf(Brush *br, bool secondary)
 	for (i = 0; i < side; ++i) {
 		for (j = 0; j < side; ++j) {
 			float magn = sqrtf(pow2f(i - half) + pow2f(j - half));
-			im->rect_float[i * side + j] = BKE_brush_curve_strength_clamp(br, magn, half);
+			im->rect_float[i * side + j] = BKE_brush_curve_strength(br, magn, half);
 		}
 	}
 
diff --git a/source/blender/editors/sculpt_paint/paint_cursor.c b/source/blender/editors/sculpt_paint/paint_cursor.c
index 92cb346b61e..bd380d458cf 100644
--- a/source/blender/editors/sculpt_paint/paint_cursor.c
+++ b/source/blender/editors/sculpt_paint/paint_cursor.c
@@ -426,7 +426,7 @@ static int load_tex_cursor(Brush *br, ViewContext *vc, float zoom)
 				len = sqrtf(x * x + y * y);
 
 				if (len <= 1) {
-					float avg = BKE_brush_curve_strength_clamp(br, len, 1.0f);  /* Falloff curve */
+					float avg = BKE_brush_curve_strength(br, len, 1.0f);  /* Falloff curve */
 
 					buffer[index] = 255 - (GLubyte)(255 * avg);
 
diff --git a/source/blender/editors/sculpt_paint/paint_image_2d.c b/source/blender/editors/sculpt_paint/paint_image_2d.c
index 2298164822f..95940d89694 100644
--- a/source/blender/editors/sculpt_paint/paint_image_2d.c
+++ b/source/blender/editors/sculpt_paint/paint_image_2d.c
@@ -359,7 +359,7 @@ static unsigned short *brush_painter_curve_mask_new(BrushPainter *painter, int d
 			float xy[2] = {x + xoff, y + yoff};
 			float len = len_v2(xy);
 
-			*m = (unsigned short)(65535.0f * BKE_brush_curve_strength_clamp(brush, len, radius));
+			*m = (unsigned short)(65535.0f * BKE_brush_curve_strength(brush, len, radius));
 		}
 	}
 
diff --git a/source/blender/editors/sculpt_paint/paint_image_proj.c b/source/blender/editors/sculpt_paint/paint_image_proj.c
index 0832782db4b..93401d4ec52 100644
--- a/source/blender/editors/sculpt_paint/paint_image_proj.c
+++ b/source/blender/editors/sculpt_paint/paint_image_proj.c
@@ -4286,7 +4286,7 @@ static void *do_projectpaint_thread(void *ph_v)
 				if (dist_sq <= brush_radius_sq) {
 					dist = sqrtf(dist_sq);
 
-					falloff = BKE_brush_curve_strength_clamp(ps->brush, dist, brush_radius);
+					falloff = BKE_brush_curve_strength(ps->brush, dist, brush_radius);
 
 					if (falloff > 0.0f) {
 						float texrgb[3];
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index 644d394f194..7324cebbf49 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -935,7 +935,7 @@ static float calc_vp_strength_col_dl(VPaint *vp, ViewContext *vc, const float co
 			else {
 				factor = 1.0f;
 			}
-			return factor * BKE_brush_curve_strength_clamp(brush, dist, brush_size_pressure);
+			return factor * BKE_brush_curve_strength(brush, dist, brush_size_pressure);
 		}
 	}
 	if (rgba)
@@ -3215,7 +3215,7 @@ static void gradientVert_update(DMGradient_userData *grad_data, int index)
 	/* no need to clamp 'alpha' yet */
 
 	/* adjust weight */
-	alpha = BKE_brush_curve_strength_clamp(grad_data->brush, alpha, 1.0f);
+	alpha = BKE_brush_curve_strength(grad_data->brush, alpha, 1.0f);
 
 	if (alpha != 0.0f) {
 		MDeformVert *dv = &me->dvert[index];