Cavity mask support for texture painting.

Title says it all, options can be found in the options panel,

A slider controls the amount of cavity masking that is applied while
it's also possible to invert the mask and paint outside or inside
cavities.

Again we might greatly benefit from caching of the cavity result, but
that should only affect startup time for the stroke.

1 files changed, 82 insertions, 5 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_image_proj.c b/source/blender/editors/sculpt_paint/paint_image_proj.c
index 098477ed2a1..439e9cdad22 100644
--- a/source/blender/editors/sculpt_paint/paint_image_proj.c
+++ b/source/blender/editors/sculpt_paint/paint_image_proj.c
@@ -214,10 +214,12 @@ typedef struct ProjPaintState {
 
 	DerivedMesh    *dm;
 	int dm_totface;
+	int dm_totedge;
 	int dm_totvert;
 	int dm_release;
 
 	MVert          *dm_mvert;
+	MEdge          *dm_medge;
 	MFace          *dm_mface;
 	MTFace         **dm_mtface;
 	MTFace         **dm_mtface_clone;    /* other UV map, use for cloning between layers */
@@ -250,11 +252,12 @@ typedef struct ProjPaintState {
 	int image_tot;              /* size of projectImages array */
 
 	float (*screenCoords)[4];   /* verts projected into floating point screen space */
-
+	float *cavities;            /* cavity amount for vertices */
 	float screenMin[2];         /* 2D bounds for mesh verts on the screen's plane (screenspace) */
 	float screenMax[2];
 	float screen_width;         /* Calculated from screenMin & screenMax */
 	float screen_height;
+	float cavity_multiplier;
 	int winx, winy;             /* from the carea or from the projection render */
 
 	/* options for projection painting */
@@ -267,6 +270,7 @@ typedef struct ProjPaintState {
 	bool  do_occlude;               /* Use raytraced occlusion? - ortherwise will paint right through to the back*/
 	bool  do_backfacecull;          /* ignore faces with normals pointing away, skips a lot of raycasts if your normals are correctly flipped */
 	bool  do_mask_normal;           /* mask out pixels based on their normals */
+	bool  do_mask_cavity;           /* mask out pixels based on cavity */
 	bool  do_new_shading_nodes;     /* cache BKE_scene_use_new_shading_nodes value */
 	float normal_angle;             /* what angle to mask at*/
 	float normal_angle_inner;
@@ -1300,6 +1304,25 @@ static float project_paint_uvpixel_mask(
 		mask = 1.0f;
 	}
 
+	if (ps->do_mask_cavity) {
+		MFace *mf = &ps->dm_mface[face_index];
+		float ca1, ca2, ca3, ca_mask;
+		ca1 = ps->cavities[mf->v1];
+		if (side == 1) {
+			ca2 = ps->cavities[mf->v3];
+			ca3 = ps->cavities[mf->v4];
+		}
+		else {
+			ca2 = ps->cavities[mf->v2];
+			ca3 = ps->cavities[mf->v3];
+		}
+
+		ca_mask = w[0] * ca1 + w[1] * ca2 + w[2] * ca3;
+		CLAMP(ca_mask, 0.0, 1.0);
+		ca_mask = 1.0f - ca_mask;
+		mask *= ca_mask;
+	}
+
 	/* calculate mask */
 	if (ps->do_mask_normal) {
 		MFace *mf = &ps->dm_mface[face_index];
@@ -3203,6 +3226,44 @@ static void proj_paint_state_screen_coords_init(ProjPaintState *ps, const int di
 	}
 }
 
+static void proj_paint_state_cavity_init(ProjPaintState *ps)
+{
+	MVert *mv;
+	MEdge *me;
+	float *cavities;
+	int a;
+
+	if (ps->do_mask_cavity) {
+		int *counter = MEM_callocN(sizeof(int) * ps->dm_totvert, "counter");
+		float (*edges)[3] = MEM_callocN(sizeof(float) * 3 * ps->dm_totvert, "edges");
+		ps->cavities = MEM_mallocN(sizeof(float) * ps->dm_totvert, "ProjectPaint Cavities");
+		cavities = ps->cavities;
+
+		for (a = 0, me = ps->dm_medge; a < ps->dm_totedge; a++, me++) {
+			float e[3];
+			sub_v3_v3v3(e, ps->dm_mvert[me->v1].co, ps->dm_mvert[me->v2].co);
+			normalize_v3(e);
+			add_v3_v3(edges[me->v2], e);
+			counter[me->v2]++;
+			sub_v3_v3(edges[me->v1], e);
+			counter[me->v1]++;
+		}
+		for (a = 0, mv = ps->dm_mvert; a < ps->dm_totvert; a++, mv++) {
+			if (counter[a] > 0) {
+				float no[3];
+				mul_v3_fl(edges[a], 1.0f / counter[a]);
+				normal_short_to_float_v3(no, mv->no);
+				cavities[a] = ps->cavity_multiplier * 10.0f * dot_v3v3(no, edges[a]);
+			}
+			else
+				cavities[a] = 0.0;
+		}
+
+		MEM_freeN(counter);
+		MEM_freeN(edges);
+	}
+}
+
 #ifndef PROJ_DEBUG_NOSEAMBLEED
 static void proj_paint_state_seam_bleed_init(ProjPaintState *ps)
 {
@@ -3329,9 +3390,14 @@ static bool proj_paint_state_dm_init(ProjPaintState *ps)
 	DM_update_materials(ps->dm, ps->ob);
 
 	ps->dm_totvert = ps->dm->getNumVerts(ps->dm);
+	ps->dm_totedge = ps->dm->getNumEdges(ps->dm);
 	ps->dm_totface = ps->dm->getNumTessFaces(ps->dm);
 
 	ps->dm_mvert = ps->dm->getVertArray(ps->dm);
+
+	if (ps->do_mask_cavity)
+		ps->dm_medge = ps->dm->getEdgeArray(ps->dm);
+
 	ps->dm_mface = ps->dm->getTessFaceArray(ps->dm);
 	ps->dm_mtface = MEM_mallocN(ps->dm_totface * sizeof(MTFace *), "proj_paint_mtfaces");
 
@@ -3733,6 +3799,8 @@ static void project_paint_begin(ProjPaintState *ps)
 	/* when using subsurf or multires, mface arrays are thrown away, we need to keep a copy */
 	proj_paint_state_non_cddm_init(ps);
 
+	proj_paint_state_cavity_init(ps);
+
 	proj_paint_state_viewport_init(ps);
 
 	/* calculate vert screen coords
@@ -3830,6 +3898,10 @@ static void project_paint_end(ProjPaintState *ps)
 		MEM_freeN(ps->blurkernel);
 	}
 
+	if (ps->do_mask_cavity) {
+		MEM_freeN(ps->cavities);
+	}
+
 	if (ps->vertFlags) MEM_freeN(ps->vertFlags);
 
 	for (a = 0; a < ps->thread_tot; a++) {
@@ -4855,12 +4927,17 @@ static void project_state_init(bContext *C, Object *ob, ProjPaintState *ps, int
 
 	/* setup projection painting data */
 	if (ps->tool != PAINT_TOOL_FILL) {
-		ps->do_backfacecull = (settings->imapaint.flag & IMAGEPAINT_PROJECT_BACKFACE) ? 0 : 1;
-		ps->do_occlude = (settings->imapaint.flag & IMAGEPAINT_PROJECT_XRAY) ? 0 : 1;
-		ps->do_mask_normal = (settings->imapaint.flag & IMAGEPAINT_PROJECT_FLAT) ? 0 : 1;
+		ps->do_backfacecull = (settings->imapaint.flag & IMAGEPAINT_PROJECT_BACKFACE) ? false : true;
+		ps->do_occlude = (settings->imapaint.flag & IMAGEPAINT_PROJECT_XRAY) ? false : true;
+		ps->do_mask_normal = (settings->imapaint.flag & IMAGEPAINT_PROJECT_FLAT) ? false : true;
+		ps->do_mask_cavity = (settings->imapaint.flag & IMAGEPAINT_PROJECT_CAVITY) ? true : false;
+		ps->cavity_multiplier = settings->imapaint.cavity_mul;
+		if (settings->imapaint.flag & IMAGEPAINT_PROJECT_CAVITY_INV) {
+			ps->cavity_multiplier *= -1;
+		}
 	}
 	else {
-		ps->do_backfacecull = ps->do_occlude = ps->do_mask_normal = 0;
+		ps->do_backfacecull = ps->do_occlude = ps->do_mask_normal = ps->do_mask_cavity = 0;
 	}
 	ps->do_new_shading_nodes = BKE_scene_use_new_shading_nodes(scene); /* only cache the value */