More painting fixes:

* 2D image painting with textures that contained alpha did not work correctly,
  had been broken for a while.

* 2D image panels texture (mask) panels showed wrong buttons for texture overlay.

* Texture map mode 3D now also uses masking, like Tiled and Stencil the texture
  does not move along with the brush so it works fine.

* 2D image paint View mapping did not work correct, especially noticeable with
  Rake rotation.

* Masking is now disabled for the smear tool, this can't really work because
  the original image is constantly changing and gave artifacts.

7 files changed, 86 insertions, 83 deletions

diff --git a/release/scripts/startup/bl_ui/space_image.py b/release/scripts/startup/bl_ui/space_image.py
index 458266aec46..e4c033d7223 100644
--- a/release/scripts/startup/bl_ui/space_image.py
+++ b/release/scripts/startup/bl_ui/space_image.py
@@ -728,6 +728,7 @@ class IMAGE_PT_tools_brush_texture(BrushButtonsPanel, Panel):
 
         toolsettings = context.tool_settings.image_paint
         brush = toolsettings.brush
+        tex_slot = brush.texture_slot
 
         col = layout.column()
         col.template_ID_preview(brush, "texture", new="texture.new", rows=3, cols=8)
@@ -740,13 +741,15 @@ class IMAGE_PT_tools_brush_texture(BrushButtonsPanel, Panel):
         col.label(text="Overlay:")
 
         row = col.row()
-        if brush.use_primary_overlay:
-            row.prop(brush, "use_primary_overlay", toggle=True, text="", icon='RESTRICT_VIEW_OFF')
-        else:
-            row.prop(brush, "use_primary_overlay", toggle=True, text="", icon='RESTRICT_VIEW_ON')
+        if tex_slot.map_mode != 'STENCIL':
+            if brush.use_primary_overlay:
+                row.prop(brush, "use_primary_overlay", toggle=True, text="", icon='RESTRICT_VIEW_OFF')
+            else:
+                row.prop(brush, "use_primary_overlay", toggle=True, text="", icon='RESTRICT_VIEW_ON')
+
         sub = row.row()
         sub.prop(brush, "texture_overlay_alpha", text="Alpha")
-        sub.prop(brush, "use_cursor_overlay_override", toggle=True, text="", icon='BRUSH_DATA')
+        sub.prop(brush, "use_primary_overlay_override", toggle=True, text="", icon='BRUSH_DATA')
 
 
 
@@ -778,8 +781,8 @@ class IMAGE_PT_tools_mask_texture(BrushButtonsPanel, Panel):
                 row.prop(brush, "use_secondary_overlay", toggle=True, text="", icon='RESTRICT_VIEW_ON')
         
         sub = row.row()
-        sub.prop(brush, "texture_overlay_alpha", text="Alpha")
-        sub.prop(brush, "use_cursor_overlay_override", toggle=True, text="", icon='BRUSH_DATA')
+        sub.prop(brush, "mask_overlay_alpha", text="Alpha")
+        sub.prop(brush, "use_secondary_overlay_override", toggle=True, text="", icon='BRUSH_DATA')
 
 
 class IMAGE_PT_tools_brush_tool(BrushButtonsPanel, Panel):
diff --git a/source/blender/blenkernel/BKE_brush.h b/source/blender/blenkernel/BKE_brush.h
index 0ae243483dd..60c50b989fb 100644
--- a/source/blender/blenkernel/BKE_brush.h
+++ b/source/blender/blenkernel/BKE_brush.h
@@ -80,8 +80,10 @@ float BKE_brush_sample_tex_3D(const Scene *scene, struct Brush *br, const float
                               float rgba[4], const int thread, struct ImagePool *pool);
 float BKE_brush_sample_masktex(const Scene *scene, struct Brush *br, const float point[3],
                                const int thread, struct ImagePool *pool);
-void BKE_brush_imbuf_new(const struct Scene *scene, struct Brush *brush, short flt,
-                         short texfalloff, int size, struct ImBuf **imbuf,
+
+enum BrushImBufFill { BRUSH_IMBUF_MASK, BRUSH_IMBUF_TEX, BRUSH_IMBUF_TEX_MASK };
+void BKE_brush_imbuf_new(const struct Scene *scene, struct Brush *brush, bool use_float,
+                         enum BrushImBufFill fill, int size, struct ImBuf **imbuf,
                          bool use_color_correction, bool use_brush_alpha,
                          struct ImagePool *pool, struct rctf *mapping);
 
diff --git a/source/blender/blenkernel/intern/brush.c b/source/blender/blenkernel/intern/brush.c
index 5ddc215dc2c..feb6a961596 100644
--- a/source/blender/blenkernel/intern/brush.c
+++ b/source/blender/blenkernel/intern/brush.c
@@ -765,9 +765,10 @@ static void brush_imbuf_tex_co(rctf *mapping, int x, int y, float texco[3])
 	texco[2] = 0.0f;
 }
 
-/* TODO, use define for 'texfall' arg
- * NOTE: only used for 2d brushes currently! */
-void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texfall, int bufsize,
+/* NOTE: only used for 2d brushes currently! and needs to stay in sync
+ * with brush_painter_2d_do_partial */
+void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, bool use_float,
+                         enum BrushImBufFill fill, int bufsize,
                          ImBuf **outbuf, bool use_color_correction, bool use_brush_alpha,
                          struct ImagePool *pool, rctf *mapping)
 {
@@ -780,7 +781,7 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 	float brush_rgb[3] = {1.0f, 1.0f, 1.0f};
 	int thread = 0;
 
-	imbflag = (flt) ? IB_rectfloat : IB_rect;
+	imbflag = (use_float) ? IB_rectfloat : IB_rect;
 	xoff = -bufsize / 2.0f + 0.5f;
 	yoff = -bufsize / 2.0f + 0.5f;
 	rowbytes = bufsize * 4;
@@ -790,7 +791,7 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 	else
 		ibuf = IMB_allocImBuf(bufsize, bufsize, 32, imbflag);
 
-	if (flt) {
+	if (use_float) {
 		if (brush->imagepaint_tool == PAINT_TOOL_DRAW) {
 			copy_v3_v3(brush_rgb, brush->rgb);
 			if (use_color_correction) {
@@ -805,27 +806,21 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 				xy[0] = x + xoff;
 				xy[1] = y + yoff;
 
-				if (texfall == 0) {
+				if (fill == BRUSH_IMBUF_MASK) {
 					copy_v3_v3(dstf, brush_rgb);
 					dstf[3] = alpha * BKE_brush_curve_strength_clamp(brush, len_v2(xy), radius);
 				}
-				else if (texfall == 1) {
+				else if (fill == BRUSH_IMBUF_TEX) {
 					brush_imbuf_tex_co(mapping, x, y, texco);
 					BKE_brush_sample_tex_3D(scene, brush, texco, dstf, thread, pool);
 				}
-				else if (texfall == 2) {
+				else { /* if (fill == BRUSH_IMBUF_TEX_MASK) */
 					brush_imbuf_tex_co(mapping, x, y, texco);
 					BKE_brush_sample_tex_3D(scene, brush, texco, rgba, thread, pool);
 
 					mul_v3_v3v3(dstf, rgba, brush_rgb);
 					dstf[3] = rgba[3] * alpha * BKE_brush_curve_strength_clamp(brush, len_v2(xy), radius);
 				}
-				else {
-					brush_imbuf_tex_co(mapping, x, y, texco);
-					BKE_brush_sample_tex_3D(scene, brush, texco, rgba, thread, pool);
-					copy_v3_v3(dstf, brush_rgb);
-					dstf[3] = rgba[3] * alpha * BKE_brush_curve_strength_clamp(brush, len_v2(xy), radius);
-				}
 
 				/* output premultiplied alpha image */
 				dstf[0] *= dstf[3];
@@ -838,9 +833,9 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 		float alpha_f; /* final float alpha to convert to char */
 
 		if (brush->imagepaint_tool == PAINT_TOOL_DRAW)
-			rgb_float_to_uchar(crgb, brush->rgb);
-		else
-			rgb_float_to_uchar(crgb, brush_rgb);
+			copy_v3_v3(brush_rgb, brush->rgb);
+
+		rgb_float_to_uchar(crgb, brush->rgb);
 
 		for (y = 0; y < ibuf->y; y++) {
 			dst = (unsigned char *)ibuf->rect + y * rowbytes;
@@ -849,7 +844,7 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 				xy[0] = x + xoff;
 				xy[1] = y + yoff;
 
-				if (texfall == 0) {
+				if (fill == BRUSH_IMBUF_MASK) {
 					alpha_f = alpha * BKE_brush_curve_strength(brush, len_v2(xy), radius);
 
 					dst[0] = crgb[0];
@@ -857,29 +852,19 @@ void BKE_brush_imbuf_new(const Scene *scene, Brush *brush, short flt, short texf
 					dst[2] = crgb[2];
 					dst[3] = FTOCHAR(alpha_f);
 				}
-				else if (texfall == 1) {
+				else if (fill == BRUSH_IMBUF_TEX) {
 					brush_imbuf_tex_co(mapping, x, y, texco);
 					BKE_brush_sample_tex_3D(scene, brush, texco, rgba, thread, pool);
 					rgba_float_to_uchar(dst, rgba);
 				}
-				else if (texfall == 2) {
+				else { /* if (fill == BRUSH_IMBUF_TEX_MASK) */
 					brush_imbuf_tex_co(mapping, x, y, texco);
 					BKE_brush_sample_tex_3D(scene, brush, texco, rgba, thread, pool);
-					mul_v3_v3(rgba, brush->rgb);
-					alpha_f = rgba[3] * alpha * BKE_brush_curve_strength_clamp(brush, len_v2(xy), radius);
-
-					rgb_float_to_uchar(dst, rgba);
 
-					dst[3] = FTOCHAR(alpha_f);
-				}
-				else {
-					brush_imbuf_tex_co(mapping, x, y, texco);
-					BKE_brush_sample_tex_3D(scene, brush, texco, rgba, thread, pool);
+					mul_v3_v3(rgba, brush_rgb);
 					alpha_f = rgba[3] * alpha * BKE_brush_curve_strength_clamp(brush, len_v2(xy), radius);
 
-					dst[0] = crgb[0];
-					dst[1] = crgb[1];
-					dst[2] = crgb[2];
+					rgb_float_to_uchar(dst, rgba);
 					dst[3] = FTOCHAR(alpha_f);
 				}
 			}
diff --git a/source/blender/blenlib/BLI_math_color_blend.h b/source/blender/blenlib/BLI_math_color_blend.h
index 8f3fc97d09d..d7e9bf50eae 100644
--- a/source/blender/blenlib/BLI_math_color_blend.h
+++ b/source/blender/blenlib/BLI_math_color_blend.h
@@ -49,6 +49,7 @@ MINLINE void blend_color_lighten_byte(unsigned char dst[4], const unsigned char
 MINLINE void blend_color_darken_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4]);
 MINLINE void blend_color_erase_alpha_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4]);
 MINLINE void blend_color_add_alpha_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4]);
+MINLINE void blend_color_interpolate_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4], float t);
 
 MINLINE void blend_color_mix_float(float dst[4], const float src1[4], const float src2[4]);
 MINLINE void blend_color_add_float(float dst[4], const float src1[4], const float src2[4]);
@@ -58,6 +59,7 @@ MINLINE void blend_color_lighten_float(float dst[4], const float src1[4], const
 MINLINE void blend_color_darken_float(float dst[4], const float src1[4], const float src2[4]);
 MINLINE void blend_color_erase_alpha_float(float dst[4], const float src1[4], const float src2[4]);
 MINLINE void blend_color_add_alpha_float(float dst[4], const float src1[4], const float src2[4]);
+MINLINE void blend_color_interpolate_float(float dst[4], const float src1[4], const float src2[4], float t);
 
 #if BLI_MATH_DO_INLINE
 #include "intern/math_color_blend_inline.c"
diff --git a/source/blender/blenlib/intern/math_color_blend_inline.c b/source/blender/blenlib/intern/math_color_blend_inline.c
index 121750fe919..52947fcee08 100644
--- a/source/blender/blenlib/intern/math_color_blend_inline.c
+++ b/source/blender/blenlib/intern/math_color_blend_inline.c
@@ -242,6 +242,28 @@ MINLINE void blend_color_add_alpha_byte(unsigned char dst[4], const unsigned cha
 	}
 }
 
+MINLINE void blend_color_interpolate_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4], float ft)
+{
+	/* do color interpolation, but in premultiplied space so that RGB colors
+	 * from zero alpha regions have no influence */
+	const int t = (int)(255 * ft);
+	const int mt = 255 - t;
+	int tmp = (mt * src1[3] + t * src2[3]);
+
+	if (tmp > 0) {
+		dst[0] = divide_round_i(mt * src1[0] * src1[3] + t * src2[0] * src2[3], tmp);
+		dst[1] = divide_round_i(mt * src1[1] * src1[3] + t * src2[1] * src2[3], tmp);
+		dst[2] = divide_round_i(mt * src1[2] * src1[3] + t * src2[2] * src2[3], tmp);
+		dst[3] = divide_round_i(tmp, 255);
+	}
+	else {
+		dst[0] = src1[0];
+		dst[1] = src1[1];
+		dst[2] = src1[2];
+		dst[3] = src1[3];
+	}
+}
+
 /* premultiplied alpha float blending modes */
 
 MINLINE void blend_color_mix_float(float dst[4], const float src1[4], const float src2[4])
@@ -420,4 +442,15 @@ MINLINE void blend_color_add_alpha_float(float dst[4], const float src1[4], cons
 	}
 }
 
+MINLINE void blend_color_interpolate_float(float dst[4], const float src1[4], const float src2[4], float t)
+{
+	/* interpolation, colors are premultiplied so it goes fine */
+	float mt = 1.0f - t;
+
+	dst[0] = mt * src1[0] + t * src2[0];
+	dst[1] = mt * src1[1] + t * src2[1];
+	dst[2] = mt * src1[2] + t * src2[2];
+	dst[3] = mt * src1[3] + t * src2[3];
+}
+
 #endif /* __MATH_COLOR_BLEND_INLINE_C__ */
diff --git a/source/blender/editors/sculpt_paint/paint_image_2d.c b/source/blender/editors/sculpt_paint/paint_image_2d.c
index 5cba3394a18..02bdd974aa2 100644
--- a/source/blender/editors/sculpt_paint/paint_image_2d.c
+++ b/source/blender/editors/sculpt_paint/paint_image_2d.c
@@ -337,7 +337,7 @@ static void brush_painter_2d_tiled_tex_partial_update(BrushPainter *painter, con
 		brush_painter_2d_do_partial(painter, NULL, x1, y2, x2, ibuf->y, 0, 0, pos);
 }
 
-static void brush_painter_2d_tex_mapping(ImagePaintState *s, int bufsize, const float pos[2], bool do_stencil, bool do_3D, rctf *mapping)
+static void brush_painter_2d_tex_mapping(ImagePaintState *s, int bufsize, const float pos[2], bool do_stencil, bool do_3D, bool do_view, rctf *mapping)
 {
 	float invw = 1.0f/(float)s->canvas->x;
 	float invh = 1.0f/(float)s->canvas->y;
@@ -348,7 +348,7 @@ static void brush_painter_2d_tex_mapping(ImagePaintState *s, int bufsize, const
 	ipos[0] = (int)floorf((pos[0] - bufsize / 2) + 1.0f);
 	ipos[1] = (int)floorf((pos[1] - bufsize / 2) + 1.0f);
 
-	if (do_stencil) {
+	if (do_stencil || do_view) {
 		/* map from view coordinates of brush to region coordinates */
 		UI_view2d_to_region_no_clip(s->v2d, ipos[0]*invw, ipos[1]*invh, &xmin, &ymin);
 		UI_view2d_to_region_no_clip(s->v2d, (ipos[0] + bufsize)*invw, (ipos[1] + bufsize)*invh, &xmax, &ymax);
@@ -398,7 +398,7 @@ static void brush_painter_2d_refresh_cache(ImagePaintState *s, BrushPainter *pai
 		rotation += ups->brush_rotation;
 	}
 
-	brush_painter_2d_tex_mapping(s, size, pos, do_stencil, do_3D, &painter->mapping);
+	brush_painter_2d_tex_mapping(s, size, pos, do_stencil, do_3D, do_view, &painter->mapping);
 
 	painter->pool = BKE_image_pool_new();
 
@@ -418,15 +418,19 @@ static void brush_painter_2d_refresh_cache(ImagePaintState *s, BrushPainter *pai
 		}
 
 		if (do_tiled || do_3D || do_stencil) {
-			BKE_brush_imbuf_new(scene, brush, flt, 3, size, &cache->maskibuf,
+			BKE_brush_imbuf_new(scene, brush, flt, BRUSH_IMBUF_MASK,
+			                    size, &cache->maskibuf,
 			                    use_color_correction, use_brush_alpha,
 			                    painter->pool, &painter->mapping);
+
 			brush_painter_2d_tiled_tex_partial_update(painter, pos);
 		}
-		else
-			BKE_brush_imbuf_new(scene, brush, flt, 2, size, &cache->ibuf,
+		else {
+			BKE_brush_imbuf_new(scene, brush, flt, BRUSH_IMBUF_TEX_MASK,
+			                    size, &cache->ibuf,
 			                    use_color_correction, use_brush_alpha,
 			                    painter->pool, &painter->mapping);
+		}
 
 		cache->lastsize = diameter;
 		cache->lastalpha = alpha;
@@ -784,8 +788,9 @@ static int paint_2d_canvas_set(ImagePaintState *s, Image *ima)
 	}
 
 	/* set masking */
-	s->do_masking = (s->brush->flag & BRUSH_AIRBRUSH || (s->brush->mtex.tex &&
-					 !ELEM(s->brush->mtex.brush_map_mode, MTEX_MAP_MODE_TILED, MTEX_MAP_MODE_STENCIL)))
+	s->do_masking = (s->brush->flag & BRUSH_AIRBRUSH ||
+	                 (s->brush->imagepaint_tool == PAINT_TOOL_SMEAR) ||
+	                 (s->brush->mtex.tex && !ELEM3(s->brush->mtex.brush_map_mode, MTEX_MAP_MODE_TILED, MTEX_MAP_MODE_STENCIL, MTEX_MAP_MODE_3D)))
 					 ? false : true;
 	
 	return 1;
diff --git a/source/blender/editors/sculpt_paint/paint_image_proj.c b/source/blender/editors/sculpt_paint/paint_image_proj.c
index 13b3629518f..2e0ad9e0020 100644
--- a/source/blender/editors/sculpt_paint/paint_image_proj.c
+++ b/source/blender/editors/sculpt_paint/paint_image_proj.c
@@ -3516,34 +3516,6 @@ typedef struct ProjectHandle {
 	struct ImagePool *pool;
 } ProjectHandle;
 
-static void interpolate_color_byte(unsigned char dst[4], const unsigned char src1[4], const unsigned char src2[4], const float ffac)
-{
-	/* do color interpolation, but in premultiplied space so that RGB colors
-	 * from zero alpha regions have no influence */
-	const int fac = (int)(255 * ffac);
-	const int mfac = 255 - fac;
-	int tmp = (mfac * src1[3] + fac * src2[3]);
-
-	if (tmp > 0) {
-		dst[0] = divide_round_i(mfac * src1[0] * src1[3] + fac * src2[0] * src2[3], tmp);
-		dst[1] = divide_round_i(mfac * src1[1] * src1[3] + fac * src2[1] * src2[3], tmp);
-		dst[2] = divide_round_i(mfac * src1[2] * src1[3] + fac * src2[2] * src2[3], tmp);
-		dst[3] = divide_round_i(tmp, 255);
-	}
-	else {
-		dst[0] = src1[0];
-		dst[1] = src1[1];
-		dst[2] = src1[2];
-		dst[3] = src1[3];
-	}
-}
-
-static void interpolate_color_float(float dst[4], const float src1[4], const float src2[4], const float fac)
-{
-	/* interpolation, colors are premultiplied so it goes fine */
-	interp_v4_v4v4(dst, src1, src2, fac);
-}
-
 static void do_projectpaint_clone(ProjPaintState *ps, ProjPixel *projPixel, float mask)
 {
 	const unsigned char *clone_pt = ((ProjPixelClone *)projPixel)->clonepx.ch;
@@ -3597,7 +3569,7 @@ static void do_projectpaint_smear(ProjPaintState *ps, ProjPixel *projPixel, floa
 	if (project_paint_PickColor(ps, co, NULL, rgba_ub, 1) == 0)
 		return;
 
-	interpolate_color_byte(((ProjPixelClone *)projPixel)->clonepx.ch, projPixel->pixel.ch_pt, rgba_ub, mask);
+	blend_color_interpolate_byte(((ProjPixelClone *)projPixel)->clonepx.ch, projPixel->pixel.ch_pt, rgba_ub, mask);
 	BLI_linklist_prepend_arena(smearPixels, (void *)projPixel, smearArena);
 }
 
@@ -3609,7 +3581,7 @@ static void do_projectpaint_smear_f(ProjPaintState *ps, ProjPixel *projPixel, fl
 	if (project_paint_PickColor(ps, co, rgba, NULL, 1) == 0)
 		return;
 
-	interpolate_color_float(((ProjPixelClone *)projPixel)->clonepx.f, projPixel->pixel.f_pt, rgba, mask);
+	blend_color_interpolate_float(((ProjPixelClone *)projPixel)->clonepx.f, projPixel->pixel.f_pt, rgba, mask);
 	BLI_linklist_prepend_arena(smearPixels_f, (void *)projPixel, smearArena);
 }
 
@@ -3648,7 +3620,7 @@ static void do_projectpaint_soften_f(ProjPaintState *ps, ProjPixel *projPixel, f
 
 	if (LIKELY(accum_tot != 0)) {
 		mul_v4_fl(rgba, 1.0f / (float)accum_tot);
-		interpolate_color_float(rgba, rgba, projPixel->pixel.f_pt, mask);
+		blend_color_interpolate_float(rgba, rgba, projPixel->pixel.f_pt, mask);
 		BLI_linklist_prepend_arena(softenPixels, (void *)projPixel, softenArena);
 	}
 }
@@ -3683,7 +3655,7 @@ static void do_projectpaint_soften(ProjPaintState *ps, ProjPixel *projPixel, flo
 		mul_v4_fl(rgba, 1.0f / (float)accum_tot);
 		premul_float_to_straight_uchar(rgba_ub, rgba);
 
-		interpolate_color_byte(rgba_ub, rgba_ub, projPixel->pixel.ch_pt, mask);
+		blend_color_interpolate_byte(rgba_ub, rgba_ub, projPixel->pixel.ch_pt, mask);
 		BLI_linklist_prepend_arena(softenPixels, (void *)projPixel, softenArena);
 	}
 }
@@ -4133,8 +4105,9 @@ static void project_state_init(bContext *C, Object *ob, ProjPaintState *ps, int
 		ps->blend = brush->blend;
 
 		/* disable for 3d mapping also because painting on mirrored mesh can create "stripes" */
-		ps->do_masking = (brush->flag & BRUSH_AIRBRUSH || (brush->mtex.tex &&
-		                 !ELEM(brush->mtex.brush_map_mode, MTEX_MAP_MODE_TILED, MTEX_MAP_MODE_STENCIL)))
+		ps->do_masking = (brush->flag & BRUSH_AIRBRUSH ||
+		                  (brush->imagepaint_tool == PAINT_TOOL_SMEAR) ||
+		                  (brush->mtex.tex && !ELEM3(brush->mtex.brush_map_mode, MTEX_MAP_MODE_TILED, MTEX_MAP_MODE_STENCIL, MTEX_MAP_MODE_3D)))
 		                 ? false : true;
 		ps->is_texbrush = (brush->mtex.tex && brush->imagepaint_tool == PAINT_TOOL_DRAW) ? true : false;
 		ps->is_maskbrush = (brush->mask_mtex.tex) ? true : false;