Cavity mask support for texture painting.

Title says it all, options can be found in the options panel,

A slider controls the amount of cavity masking that is applied while
it's also possible to invert the mask and paint outside or inside
cavities.

Again we might greatly benefit from caching of the cavity result, but
that should only affect startup time for the stroke.

4 files changed, 115 insertions, 12 deletions

diff --git a/release/scripts/startup/bl_ui/space_view3d_toolbar.py b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
index 18a38c84582..7af8be108c1 100644
--- a/release/scripts/startup/bl_ui/space_view3d_toolbar.py
+++ b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
@@ -1721,6 +1721,12 @@ class VIEW3D_PT_tools_projectpaint(View3DPaintPanel, Panel):
         sub.active = (ipaint.use_normal_falloff)
         sub.prop(ipaint, "normal_angle", text="")
 
+        layout.prop(ipaint, "use_cavity")
+        sub = layout.column()
+        sub.active = (ipaint.use_cavity)
+        sub.prop(ipaint, "cavity_mul", slider=True)
+        sub.prop(ipaint, "invert_cavity")
+        
         layout.prop(ipaint, "seam_bleed")
         layout.prop(ipaint, "dither")
         self.unified_paint_settings(layout, context)
diff --git a/source/blender/editors/sculpt_paint/paint_image_proj.c b/source/blender/editors/sculpt_paint/paint_image_proj.c
index 098477ed2a1..439e9cdad22 100644
--- a/source/blender/editors/sculpt_paint/paint_image_proj.c
+++ b/source/blender/editors/sculpt_paint/paint_image_proj.c
@@ -214,10 +214,12 @@ typedef struct ProjPaintState {
 
 	DerivedMesh    *dm;
 	int dm_totface;
+	int dm_totedge;
 	int dm_totvert;
 	int dm_release;
 
 	MVert          *dm_mvert;
+	MEdge          *dm_medge;
 	MFace          *dm_mface;
 	MTFace         **dm_mtface;
 	MTFace         **dm_mtface_clone;    /* other UV map, use for cloning between layers */
@@ -250,11 +252,12 @@ typedef struct ProjPaintState {
 	int image_tot;              /* size of projectImages array */
 
 	float (*screenCoords)[4];   /* verts projected into floating point screen space */
-
+	float *cavities;            /* cavity amount for vertices */
 	float screenMin[2];         /* 2D bounds for mesh verts on the screen's plane (screenspace) */
 	float screenMax[2];
 	float screen_width;         /* Calculated from screenMin & screenMax */
 	float screen_height;
+	float cavity_multiplier;
 	int winx, winy;             /* from the carea or from the projection render */
 
 	/* options for projection painting */
@@ -267,6 +270,7 @@ typedef struct ProjPaintState {
 	bool  do_occlude;               /* Use raytraced occlusion? - ortherwise will paint right through to the back*/
 	bool  do_backfacecull;          /* ignore faces with normals pointing away, skips a lot of raycasts if your normals are correctly flipped */
 	bool  do_mask_normal;           /* mask out pixels based on their normals */
+	bool  do_mask_cavity;           /* mask out pixels based on cavity */
 	bool  do_new_shading_nodes;     /* cache BKE_scene_use_new_shading_nodes value */
 	float normal_angle;             /* what angle to mask at*/
 	float normal_angle_inner;
@@ -1300,6 +1304,25 @@ static float project_paint_uvpixel_mask(
 		mask = 1.0f;
 	}
 
+	if (ps->do_mask_cavity) {
+		MFace *mf = &ps->dm_mface[face_index];
+		float ca1, ca2, ca3, ca_mask;
+		ca1 = ps->cavities[mf->v1];
+		if (side == 1) {
+			ca2 = ps->cavities[mf->v3];
+			ca3 = ps->cavities[mf->v4];
+		}
+		else {
+			ca2 = ps->cavities[mf->v2];
+			ca3 = ps->cavities[mf->v3];
+		}
+
+		ca_mask = w[0] * ca1 + w[1] * ca2 + w[2] * ca3;
+		CLAMP(ca_mask, 0.0, 1.0);
+		ca_mask = 1.0f - ca_mask;
+		mask *= ca_mask;
+	}
+
 	/* calculate mask */
 	if (ps->do_mask_normal) {
 		MFace *mf = &ps->dm_mface[face_index];
@@ -3203,6 +3226,44 @@ static void proj_paint_state_screen_coords_init(ProjPaintState *ps, const int di
 	}
 }
 
+static void proj_paint_state_cavity_init(ProjPaintState *ps)
+{
+	MVert *mv;
+	MEdge *me;
+	float *cavities;
+	int a;
+
+	if (ps->do_mask_cavity) {
+		int *counter = MEM_callocN(sizeof(int) * ps->dm_totvert, "counter");
+		float (*edges)[3] = MEM_callocN(sizeof(float) * 3 * ps->dm_totvert, "edges");
+		ps->cavities = MEM_mallocN(sizeof(float) * ps->dm_totvert, "ProjectPaint Cavities");
+		cavities = ps->cavities;
+
+		for (a = 0, me = ps->dm_medge; a < ps->dm_totedge; a++, me++) {
+			float e[3];
+			sub_v3_v3v3(e, ps->dm_mvert[me->v1].co, ps->dm_mvert[me->v2].co);
+			normalize_v3(e);
+			add_v3_v3(edges[me->v2], e);
+			counter[me->v2]++;
+			sub_v3_v3(edges[me->v1], e);
+			counter[me->v1]++;
+		}
+		for (a = 0, mv = ps->dm_mvert; a < ps->dm_totvert; a++, mv++) {
+			if (counter[a] > 0) {
+				float no[3];
+				mul_v3_fl(edges[a], 1.0f / counter[a]);
+				normal_short_to_float_v3(no, mv->no);
+				cavities[a] = ps->cavity_multiplier * 10.0f * dot_v3v3(no, edges[a]);
+			}
+			else
+				cavities[a] = 0.0;
+		}
+
+		MEM_freeN(counter);
+		MEM_freeN(edges);
+	}
+}
+
 #ifndef PROJ_DEBUG_NOSEAMBLEED
 static void proj_paint_state_seam_bleed_init(ProjPaintState *ps)
 {
@@ -3329,9 +3390,14 @@ static bool proj_paint_state_dm_init(ProjPaintState *ps)
 	DM_update_materials(ps->dm, ps->ob);
 
 	ps->dm_totvert = ps->dm->getNumVerts(ps->dm);
+	ps->dm_totedge = ps->dm->getNumEdges(ps->dm);
 	ps->dm_totface = ps->dm->getNumTessFaces(ps->dm);
 
 	ps->dm_mvert = ps->dm->getVertArray(ps->dm);
+
+	if (ps->do_mask_cavity)
+		ps->dm_medge = ps->dm->getEdgeArray(ps->dm);
+
 	ps->dm_mface = ps->dm->getTessFaceArray(ps->dm);
 	ps->dm_mtface = MEM_mallocN(ps->dm_totface * sizeof(MTFace *), "proj_paint_mtfaces");
 
@@ -3733,6 +3799,8 @@ static void project_paint_begin(ProjPaintState *ps)
 	/* when using subsurf or multires, mface arrays are thrown away, we need to keep a copy */
 	proj_paint_state_non_cddm_init(ps);
 
+	proj_paint_state_cavity_init(ps);
+
 	proj_paint_state_viewport_init(ps);
 
 	/* calculate vert screen coords
@@ -3830,6 +3898,10 @@ static void project_paint_end(ProjPaintState *ps)
 		MEM_freeN(ps->blurkernel);
 	}
 
+	if (ps->do_mask_cavity) {
+		MEM_freeN(ps->cavities);
+	}
+
 	if (ps->vertFlags) MEM_freeN(ps->vertFlags);
 
 	for (a = 0; a < ps->thread_tot; a++) {
@@ -4855,12 +4927,17 @@ static void project_state_init(bContext *C, Object *ob, ProjPaintState *ps, int
 
 	/* setup projection painting data */
 	if (ps->tool != PAINT_TOOL_FILL) {
-		ps->do_backfacecull = (settings->imapaint.flag & IMAGEPAINT_PROJECT_BACKFACE) ? 0 : 1;
-		ps->do_occlude = (settings->imapaint.flag & IMAGEPAINT_PROJECT_XRAY) ? 0 : 1;
-		ps->do_mask_normal = (settings->imapaint.flag & IMAGEPAINT_PROJECT_FLAT) ? 0 : 1;
+		ps->do_backfacecull = (settings->imapaint.flag & IMAGEPAINT_PROJECT_BACKFACE) ? false : true;
+		ps->do_occlude = (settings->imapaint.flag & IMAGEPAINT_PROJECT_XRAY) ? false : true;
+		ps->do_mask_normal = (settings->imapaint.flag & IMAGEPAINT_PROJECT_FLAT) ? false : true;
+		ps->do_mask_cavity = (settings->imapaint.flag & IMAGEPAINT_PROJECT_CAVITY) ? true : false;
+		ps->cavity_multiplier = settings->imapaint.cavity_mul;
+		if (settings->imapaint.flag & IMAGEPAINT_PROJECT_CAVITY_INV) {
+			ps->cavity_multiplier *= -1;
+		}
 	}
 	else {
-		ps->do_backfacecull = ps->do_occlude = ps->do_mask_normal = 0;
+		ps->do_backfacecull = ps->do_occlude = ps->do_mask_normal = ps->do_mask_cavity = 0;
 	}
 	ps->do_new_shading_nodes = BKE_scene_use_new_shading_nodes(scene); /* only cache the value */
 
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index 9ac506696ad..d3fa9c82d39 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -850,6 +850,8 @@ typedef struct ImagePaintSettings {
 	struct Image *canvas;      /* canvas when the explicit system is used for painting */
 	float stencil_col[3];
 	float dither;              /* dither amount used when painting on byte images */
+	float cavity_mul;
+	float pad;
 } ImagePaintSettings;
 
 /* ------------------------------------------- */
@@ -1754,12 +1756,15 @@ typedef enum ImagePaintMode {
 // #define IMAGEPAINT_DRAW_TOOL_DRAWING	4 // deprecated
 
 /* projection painting only */
-#define IMAGEPAINT_PROJECT_XRAY			16
-#define IMAGEPAINT_PROJECT_BACKFACE		32
-#define IMAGEPAINT_PROJECT_FLAT			64
-#define IMAGEPAINT_PROJECT_LAYER_CLONE	128
-#define IMAGEPAINT_PROJECT_LAYER_STENCIL	256
-#define IMAGEPAINT_PROJECT_LAYER_STENCIL_INV	512
+#define IMAGEPAINT_PROJECT_XRAY			(1 << 4)
+#define IMAGEPAINT_PROJECT_BACKFACE		(1 << 5)
+#define IMAGEPAINT_PROJECT_FLAT			(1 << 6)
+#define IMAGEPAINT_PROJECT_LAYER_CLONE	(1 << 7)
+#define IMAGEPAINT_PROJECT_LAYER_STENCIL	(1 << 8)
+#define IMAGEPAINT_PROJECT_LAYER_STENCIL_INV	(1 << 9)
+#define IMAGEPAINT_PROJECT_CAVITY	(1 << 10)
+#define IMAGEPAINT_PROJECT_CAVITY_INV	(1 << 11)
+
 
 #define IMAGEPAINT_MISSING_UVS       (1 << 0)
 #define IMAGEPAINT_MISSING_MATERIAL  (1 << 1)
diff --git a/source/blender/makesrna/intern/rna_sculpt_paint.c b/source/blender/makesrna/intern/rna_sculpt_paint.c
index 0a282eeb7ac..d9e991d15a3 100644
--- a/source/blender/makesrna/intern/rna_sculpt_paint.c
+++ b/source/blender/makesrna/intern/rna_sculpt_paint.c
@@ -681,7 +681,17 @@ static void rna_def_image_paint(BlenderRNA *brna)
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", IMAGEPAINT_PROJECT_LAYER_STENCIL);
 	RNA_def_property_ui_text(prop, "Stencil Layer", "Set the mask layer from the UV map buttons");
 	RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, "rna_ImaPaint_viewport_update");
-	
+
+	prop = RNA_def_property(srna, "use_cavity", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", IMAGEPAINT_PROJECT_CAVITY);
+	RNA_def_property_ui_text(prop, "Cavity Mask", "Mask painting according to mesh geometry cavity");
+	RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
+	prop = RNA_def_property(srna, "invert_cavity", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", IMAGEPAINT_PROJECT_CAVITY_INV);
+	RNA_def_property_ui_text(prop, "Invert Cavity", "Painting inside cavity instead of outside");
+	RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
 	prop = RNA_def_property(srna, "invert_stencil", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", IMAGEPAINT_PROJECT_LAYER_STENCIL_INV);
 	RNA_def_property_ui_text(prop, "Invert", "Invert the stencil layer");
@@ -715,6 +725,11 @@ static void rna_def_image_paint(BlenderRNA *brna)
 	RNA_def_property_ui_text(prop, "Dither", "Amount of dithering when painting on byte images");
 	RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
 	
+	prop = RNA_def_property(srna, "cavity_mul", PROP_FLOAT, PROP_NONE);
+	RNA_def_property_range(prop, 0.0, 1.0);
+	RNA_def_property_ui_text(prop, "Cavity Factor", "Factor for cavity effect");
+	RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
 	prop = RNA_def_property(srna, "use_clone_layer", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", IMAGEPAINT_PROJECT_LAYER_CLONE);
 	RNA_def_property_ui_text(prop, "Clone Map",