More image painting fixes:

* 2D image painting support for masking to limit the max contribution of a stroke
  to a pixel, to get it working compatible with projection painting. Not strictly
  a bugfix, but the inconsistency here was annoying.

* Fix python errors in Texture Mask panel in image editor, was missing overlay
  options.

* Clamp paint mask to 0..1 in case some texture exceeds it, this could give black
  pixels due to integer overflow.

1 files changed, 65 insertions, 14 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_image_2d.c b/source/blender/editors/sculpt_paint/paint_image_2d.c
index 1c50b048fce..d6a634ec404 100644
--- a/source/blender/editors/sculpt_paint/paint_image_2d.c
+++ b/source/blender/editors/sculpt_paint/paint_image_2d.c
@@ -133,6 +133,8 @@ typedef struct ImagePaintState {
 	char *warnpackedfile;
 	char *warnmultifile;
 
+	bool do_masking;
+
 	/* viewport texture paint only, but _not_ project paint */
 	Object *ob;
 	int faceindex;
@@ -327,7 +329,7 @@ static void brush_painter_2d_tiled_tex_partial_update(BrushPainter *painter, con
 		brush_painter_2d_do_partial(painter, NULL, x1, y2, x2, ibuf->y, 0, 0, pos);
 }
 
-static void brush_painter_2d_refresh_cache(BrushPainter *painter, const float pos[2], int use_color_correction)
+static void brush_painter_2d_refresh_cache(BrushPainter *painter, const float pos[2], bool use_color_correction, bool use_brush_alpha)
 {
 	const Scene *scene = painter->scene;
 	UnifiedPaintSettings *ups = &scene->toolsettings->unified_paint_settings;
@@ -347,7 +349,7 @@ static void brush_painter_2d_refresh_cache(BrushPainter *painter, const float po
 	}
 
 	if (diameter != cache->lastsize ||
-	    alpha != cache->lastalpha ||
+	    (use_brush_alpha && alpha != cache->lastalpha) ||
 	    brush->jitter != cache->lastjitter ||
 	    rotation != cache->last_rotation ||
 	    do_random)
@@ -365,11 +367,13 @@ static void brush_painter_2d_refresh_cache(BrushPainter *painter, const float po
 		size = (cache->size) ? cache->size : diameter;
 
 		if (do_tiled) {
-			BKE_brush_imbuf_new(scene, brush, flt, 3, size, &cache->maskibuf, use_color_correction);
+			BKE_brush_imbuf_new(scene, brush, flt, 3, size, &cache->maskibuf,
+			                    use_color_correction, use_brush_alpha);
 			brush_painter_2d_tiled_tex_partial_update(painter, pos);
 		}
 		else
-			BKE_brush_imbuf_new(scene, brush, flt, 2, size, &cache->ibuf, use_color_correction);
+			BKE_brush_imbuf_new(scene, brush, flt, 2, size, &cache->ibuf,
+			                    use_color_correction, use_brush_alpha);
 
 		cache->lastsize = diameter;
 		cache->lastalpha = alpha;
@@ -552,7 +556,8 @@ static void paint_2d_lift_smear(ImBuf *ibuf, ImBuf *ibufb, int *pos)
 	tot = paint_2d_torus_split_region(region, ibufb, ibuf);
 
 	for (a = 0; a < tot; a++)
-		IMB_rectblend(ibufb, ibuf, region[a].destx, region[a].desty,
+		IMB_rectblend(ibufb, ibufb, ibuf, NULL, 0, region[a].destx, region[a].desty,
+		              region[a].destx, region[a].desty,
 		              region[a].srcx, region[a].srcy,
 		              region[a].width, region[a].height, IMB_BLEND_COPY_RGB);
 }
@@ -565,9 +570,9 @@ static ImBuf *paint_2d_lift_clone(ImBuf *ibuf, ImBuf *ibufb, int *pos)
 	ImBuf *clonebuf = IMB_allocImBuf(w, h, ibufb->planes, ibufb->flags);
 
 	IMB_rectclip(clonebuf, ibuf, &destx, &desty, &srcx, &srcy, &w, &h);
-	IMB_rectblend(clonebuf, ibuf, destx, desty, srcx, srcy, w, h,
+	IMB_rectblend(clonebuf, clonebuf, ibuf, NULL, 0, destx, desty, destx, desty, srcx, srcy, w, h,
 	              IMB_BLEND_COPY_RGB);
-	IMB_rectblend(clonebuf, ibufb, destx, desty, destx, desty, w, h,
+	IMB_rectblend(clonebuf, clonebuf, ibufb, NULL, 0, destx, desty, destx, desty, destx, desty, w, h,
 	              IMB_BLEND_COPY_ALPHA);
 
 	return clonebuf;
@@ -582,12 +587,14 @@ static void paint_2d_convert_brushco(ImBuf *ibufb, const float pos[2], int ipos[
 static int paint_2d_op(void *state, ImBuf *ibufb, const float lastpos[2], const float pos[2])
 {
 	ImagePaintState *s = ((ImagePaintState *)state);
-	ImBuf *clonebuf = NULL, *frombuf;
+	ImBuf *clonebuf = NULL, *frombuf, *tmpbuf = NULL;
 	ImagePaintRegion region[4];
 	short torus = s->brush->flag & BRUSH_TORUS;
 	short blend = s->blend;
 	float *offset = s->brush->clone.offset;
 	float liftpos[2];
+	float brush_alpha = BKE_brush_alpha_get(s->scene, s->brush);
+	unsigned short mask_max = (unsigned short)(brush_alpha * 65535.0f);
 	int bpos[2], blastpos[2], bliftpos[2];
 	int a, tot;
 
@@ -623,19 +630,55 @@ static int paint_2d_op(void *state, ImBuf *ibufb, const float lastpos[2], const
 		tot = 1;
 	}
 
+	if (s->do_masking)
+		tmpbuf = IMB_allocImBuf(IMAPAINT_TILE_SIZE, IMAPAINT_TILE_SIZE, 32, 0);
+	
 	/* blend into canvas */
 	for (a = 0; a < tot; a++) {
 		imapaint_dirty_region(s->image, s->canvas,
 		                      region[a].destx, region[a].desty,
 		                      region[a].width, region[a].height);
-
-		IMB_rectblend(s->canvas, frombuf,
-		              region[a].destx, region[a].desty,
-		              region[a].srcx, region[a].srcy,
-		              region[a].width, region[a].height, blend);
+	
+		if (s->do_masking) {
+			/* masking, find original pixels tiles from undo buffer to composite over */
+			int tilex, tiley, tilew, tileh, tx, ty;
+
+			imapaint_region_tiles(s->canvas, region[a].destx, region[a].desty,
+			                      region[a].width, region[a].height,
+								  &tilex, &tiley, &tilew, &tileh);
+			
+			for (ty = tiley; ty <= tileh; ty++) {
+				for (tx = tilex; tx <= tilew; tx++) {
+					/* retrieve original pixels + mask from undo buffer */
+					unsigned short *mask;
+					int origx = region[a].destx - tx * IMAPAINT_TILE_SIZE;
+					int origy = region[a].desty - ty * IMAPAINT_TILE_SIZE;
+
+					if (s->canvas->rect_float)
+						tmpbuf->rect_float = image_undo_find_tile(s->image, s->canvas, tx, ty, &mask);
+					else
+						tmpbuf->rect = image_undo_find_tile(s->image, s->canvas, tx, ty, &mask);
+
+					IMB_rectblend(s->canvas, tmpbuf, frombuf, mask, mask_max,
+								  region[a].destx, region[a].desty,
+								  origx, origy,
+								  region[a].srcx, region[a].srcy,
+								  region[a].width, region[a].height, blend);
+				}
+			}
+		}
+		else {
+			/* no masking, composite brush directly onto canvas */
+			IMB_rectblend(s->canvas, s->canvas, frombuf, NULL, 0,
+						  region[a].destx, region[a].desty,
+						  region[a].destx, region[a].desty,
+						  region[a].srcx, region[a].srcy,
+						  region[a].width, region[a].height, blend);
+		}
 	}
 
 	if (clonebuf) IMB_freeImBuf(clonebuf);
+	if (tmpbuf) IMB_freeImBuf(tmpbuf);
 
 	return 1;
 }
@@ -684,6 +727,11 @@ static int paint_2d_canvas_set(ImagePaintState *s, Image *ima)
 			IMB_rect_from_float(s->clonecanvas);
 	}
 
+	/* set masking */
+	s->do_masking = (s->brush->flag & BRUSH_AIRBRUSH || (s->brush->mtex.tex &&
+					 !ELEM(s->brush->mtex.brush_map_mode, MTEX_MAP_MODE_TILED, MTEX_MAP_MODE_STENCIL)))
+					 ? false : true;
+	
 	return 1;
 }
 
@@ -691,6 +739,9 @@ static void paint_2d_canvas_free(ImagePaintState *s)
 {
 	BKE_image_release_ibuf(s->image, s->canvas, NULL);
 	BKE_image_release_ibuf(s->brush->clone.image, s->clonecanvas, NULL);
+
+	if (s->do_masking)
+		image_undo_remove_masks();
 }
 
 int paint_2d_stroke(void *ps, const int prev_mval[2], const int mval[2], int eraser)
@@ -734,7 +785,7 @@ int paint_2d_stroke(void *ps, const int prev_mval[2], const int mval[2], int era
 	 */
 	brush_painter_2d_require_imbuf(painter, ((ibuf->rect_float) ? 1 : 0), 0);
 
-	brush_painter_2d_refresh_cache(painter, newuv, is_data == false);
+	brush_painter_2d_refresh_cache(painter, newuv, is_data == false, s->do_masking);
 
 	if (paint_2d_op(s, painter->cache.ibuf, olduv, newuv)) {
 		imapaint_image_update(s->sima, s->image, ibuf, false);