Fix T57529: 2D image paint fill tool not taking into account alpha.

3 files changed, 17 insertions, 15 deletions

diff --git a/source/blender/blenlib/BLI_math_vector.h b/source/blender/blenlib/BLI_math_vector.h
index 59c9341f75c..c23c0409f81 100644
--- a/source/blender/blenlib/BLI_math_vector.h
+++ b/source/blender/blenlib/BLI_math_vector.h
@@ -272,6 +272,7 @@ MINLINE bool compare_v4v4_relative(const float a[4], const float b[4], const flo
 
 MINLINE bool compare_len_v3v3(const float a[3], const float b[3], const float limit)  ATTR_WARN_UNUSED_RESULT;
 MINLINE bool compare_len_squared_v3v3(const float a[3], const float b[3], const float limit)  ATTR_WARN_UNUSED_RESULT;
+MINLINE bool compare_len_squared_v4v4(const float a[4], const float b[4], const float limit)  ATTR_WARN_UNUSED_RESULT;
 
 MINLINE float line_point_side_v2(const float l1[2], const float l2[2], const float pt[2]) ATTR_WARN_UNUSED_RESULT;
 
diff --git a/source/blender/blenlib/intern/math_vector_inline.c b/source/blender/blenlib/intern/math_vector_inline.c
index 715e2e65c96..37eab44c6e8 100644
--- a/source/blender/blenlib/intern/math_vector_inline.c
+++ b/source/blender/blenlib/intern/math_vector_inline.c
@@ -1087,24 +1087,23 @@ MINLINE bool compare_v4v4_relative(const float v1[4], const float v2[4], const f
 
 MINLINE bool compare_len_v3v3(const float v1[3], const float v2[3], const float limit)
 {
-	float x, y, z;
-
-	x = v1[0] - v2[0];
-	y = v1[1] - v2[1];
-	z = v1[2] - v2[2];
-
-	return ((x * x + y * y + z * z) <= (limit * limit));
+	float d[3];
+	sub_v3_v3v3(d, v1, v2);
+	return (dot_v3v3(d, d) <= (limit * limit));
 }
 
 MINLINE bool compare_len_squared_v3v3(const float v1[3], const float v2[3], const float limit_sq)
 {
-	float x, y, z;
-
-	x = v1[0] - v2[0];
-	y = v1[1] - v2[1];
-	z = v1[2] - v2[2];
+	float d[3];
+	sub_v3_v3v3(d, v1, v2);
+	return (dot_v3v3(d, d) <= limit_sq);
+}
 
-	return ((x * x + y * y + z * z) <= limit_sq);
+MINLINE bool compare_len_squared_v4v4(const float v1[4], const float v2[4], const float limit_sq)
+{
+	float d[4];
+	sub_v4_v4v4(d, v1, v2);
+	return (dot_v4v4(d, d) <= limit_sq);
 }
 
 /**
diff --git a/source/blender/editors/sculpt_paint/paint_image_2d.c b/source/blender/editors/sculpt_paint/paint_image_2d.c
index ddfee5de4d7..d89d4f72d3d 100644
--- a/source/blender/editors/sculpt_paint/paint_image_2d.c
+++ b/source/blender/editors/sculpt_paint/paint_image_2d.c
@@ -1419,8 +1419,9 @@ static void paint_2d_fill_add_pixel_byte(
 		float color_f[4];
 		unsigned char *color_b = (unsigned char *)(ibuf->rect + coordinate);
 		rgba_uchar_to_float(color_f, color_b);
+		straight_to_premul_v4(color_f);
 
-		if (compare_len_squared_v3v3(color_f, color, threshold_sq)) {
+		if (compare_len_squared_v4v4(color_f, color, threshold_sq)) {
 			BLI_stack_push(stack, &coordinate);
 		}
 		BLI_BITMAP_SET(touched, coordinate, true);
@@ -1439,7 +1440,7 @@ static void paint_2d_fill_add_pixel_float(
 	coordinate = ((size_t)y_px) * ibuf->x + x_px;
 
 	if (!BLI_BITMAP_TEST(touched, coordinate)) {
-		if (compare_len_squared_v3v3(ibuf->rect_float + 4 * coordinate, color, threshold_sq)) {
+		if (compare_len_squared_v4v4(ibuf->rect_float + 4 * coordinate, color, threshold_sq)) {
 			BLI_stack_push(stack, &coordinate);
 		}
 		BLI_BITMAP_SET(touched, coordinate, true);
@@ -1546,6 +1547,7 @@ void paint_2d_bucket_fill(
 		else {
 			int pixel_color_b = *(ibuf->rect + coordinate);
 			rgba_uchar_to_float(pixel_color, (unsigned char *)&pixel_color_b);
+			straight_to_premul_v4(pixel_color);
 		}
 
 		BLI_stack_push(stack, &coordinate);