Fix T44553: Dyntopo ignores front-face option

When 'Front Faces' brush option was enabled, dyntop would still adjust detail on back-faces.

3 files changed, 70 insertions, 11 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index 1df633cc966..c7ef53589aa 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -145,8 +145,10 @@ typedef enum {
 	PBVH_Subdivide = 1,
 	PBVH_Collapse = 2,
 } PBVHTopologyUpdateMode;
-bool BKE_pbvh_bmesh_update_topology(PBVH *bvh, PBVHTopologyUpdateMode mode,
-                                   const float center[3], float radius);
+bool BKE_pbvh_bmesh_update_topology(
+        PBVH *bvh, PBVHTopologyUpdateMode mode,
+        const float center[3], const float view_normal[3],
+        float radius);
 
 /* Node Access */
 
diff --git a/source/blender/blenkernel/intern/pbvh_bmesh.c b/source/blender/blenkernel/intern/pbvh_bmesh.c
index e631a4a365d..9f092bd651f 100644
--- a/source/blender/blenkernel/intern/pbvh_bmesh.c
+++ b/source/blender/blenkernel/intern/pbvh_bmesh.c
@@ -48,6 +48,9 @@
 #  include "BKE_global.h"
 #endif
 
+/* Support for only operating on front-faces */
+#define USE_EDGEQUEUE_FRONTFACE
+
 /* don't add edges into the queue multiple times */
 #define USE_EDGEQUEUE_TAG
 /**
@@ -598,6 +601,12 @@ typedef struct {
 #ifdef USE_EDGEQUEUE_EVEN_SUBDIV
 	float limit_len;
 #endif
+
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	const float *view_normal;
+	unsigned int use_view_normal : 1;
+#endif
+
 } EdgeQueue;
 
 typedef struct {
@@ -719,6 +728,14 @@ static void long_edge_queue_edge_add_recursive(
 {
 	BLI_assert(len_sq > SQUARE(limit_len));
 
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	if (eq_ctx->q->use_view_normal) {
+		if (dot_v3v3(l_edge->f->no, eq_ctx->q->view_normal) < 0.0f) {
+			return;
+		}
+	}
+#endif
+
 #ifdef USE_EDGEQUEUE_TAG
 	if (EDGE_QUEUE_TEST(l_edge->e) == false)
 #endif
@@ -787,6 +804,14 @@ static void long_edge_queue_face_add(
         EdgeQueueContext *eq_ctx,
         BMFace *f)
 {
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	if (eq_ctx->q->use_view_normal) {
+		if (dot_v3v3(f->no, eq_ctx->q->view_normal) < 0.0f) {
+			return;
+		}
+	}
+#endif
+
 	if (edge_queue_tri_in_sphere(eq_ctx->q, f)) {
 		BMLoop *l_iter;
 		BMLoop *l_first;
@@ -814,6 +839,14 @@ static void short_edge_queue_face_add(
         EdgeQueueContext *eq_ctx,
         BMFace *f)
 {
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	if (eq_ctx->q->use_view_normal) {
+		if (dot_v3v3(f->no, eq_ctx->q->view_normal) < 0.0f) {
+			return;
+		}
+	}
+#endif
+
 	if (edge_queue_tri_in_sphere(eq_ctx->q, f)) {
 		BMLoop *l_iter;
 		BMLoop *l_first;
@@ -837,7 +870,7 @@ static void short_edge_queue_face_add(
  */
 static void long_edge_queue_create(
         EdgeQueueContext *eq_ctx,
-        PBVH *bvh, const float center[3],
+        PBVH *bvh, const float center[3], const float view_normal[3],
         float radius)
 {
 	int n;
@@ -850,6 +883,13 @@ static void long_edge_queue_create(
 	eq_ctx->q->limit_len = bvh->bm_max_edge_len;
 #endif
 
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	eq_ctx->q->view_normal = view_normal;
+	eq_ctx->q->use_view_normal = (view_normal != NULL);
+#else
+	UNUSED_VARS(view_normal);
+#endif
+
 #ifdef USE_EDGEQUEUE_TAG_VERIFY
 	pbvh_bmesh_edge_tag_verify(bvh);
 #endif
@@ -886,7 +926,7 @@ static void long_edge_queue_create(
  */
 static void short_edge_queue_create(
         EdgeQueueContext *eq_ctx,
-        PBVH *bvh, const float center[3],
+        PBVH *bvh, const float center[3], const float view_normal[3],
         float radius)
 {
 	int n;
@@ -899,6 +939,13 @@ static void short_edge_queue_create(
 	eq_ctx->q->limit_len = bvh->bm_min_edge_len;
 #endif
 
+#ifdef USE_EDGEQUEUE_FRONTFACE
+	eq_ctx->q->view_normal = view_normal;
+	eq_ctx->q->use_view_normal = (view_normal != NULL);
+#else
+	UNUSED_VARS(view_normal);
+#endif
+
 	for (n = 0; n < bvh->totnode; n++) {
 		PBVHNode *node = &bvh->nodes[n];
 
@@ -1704,7 +1751,8 @@ void BKE_pbvh_build_bmesh(
 /* Collapse short edges, subdivide long edges */
 bool BKE_pbvh_bmesh_update_topology(
         PBVH *bvh, PBVHTopologyUpdateMode mode,
-        const float center[3], float radius)
+        const float center[3], const float view_normal[3],
+        float radius)
 {
 	/* 2 is enough for edge faces - manifold edge */
 	BLI_buffer_declare_static(BMLoop *, edge_loops, BLI_BUFFER_NOP, 2);
@@ -1716,12 +1764,16 @@ bool BKE_pbvh_bmesh_update_topology(
 	bool modified = false;
 	int n;
 
+	if (view_normal) {
+		BLI_assert(len_squared_v3(view_normal) != 0.0f);
+	}
+
 	if (mode & PBVH_Collapse) {
 		EdgeQueue q;
 		BLI_mempool *queue_pool = BLI_mempool_create(sizeof(BMVert *[2]), 0, 128, BLI_MEMPOOL_NOP);
 		EdgeQueueContext eq_ctx = {&q, queue_pool, bvh->bm, cd_vert_mask_offset, cd_vert_node_offset, cd_face_node_offset};
 
-		short_edge_queue_create(&eq_ctx, bvh, center, radius);
+		short_edge_queue_create(&eq_ctx, bvh, center, view_normal, radius);
 		modified |= !BLI_heap_is_empty(q.heap);
 		pbvh_bmesh_collapse_short_edges(&eq_ctx, bvh,
 		                                &deleted_faces);
@@ -1734,7 +1786,7 @@ bool BKE_pbvh_bmesh_update_topology(
 		BLI_mempool *queue_pool = BLI_mempool_create(sizeof(BMVert *[2]), 0, 128, BLI_MEMPOOL_NOP);
 		EdgeQueueContext eq_ctx = {&q, queue_pool, bvh->bm, cd_vert_mask_offset, cd_vert_node_offset, cd_face_node_offset};
 
-		long_edge_queue_create(&eq_ctx, bvh, center, radius);
+		long_edge_queue_create(&eq_ctx, bvh, center, view_normal, radius);
 		modified |= !BLI_heap_is_empty(q.heap);
 		pbvh_bmesh_subdivide_long_edges(&eq_ctx, bvh, &edge_loops);
 		BLI_heap_free(q.heap, NULL);
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index e2403a57556..0d1ee08cc75 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -3126,9 +3126,11 @@ static void sculpt_topology_update(Sculpt *sd, Object *ob, Brush *brush, Unified
 		}
 
 		if (BKE_pbvh_type(ss->pbvh) == PBVH_BMESH) {
-			BKE_pbvh_bmesh_update_topology(ss->pbvh, mode,
-			                               ss->cache->location,
-			                               ss->cache->radius);
+			BKE_pbvh_bmesh_update_topology(
+			        ss->pbvh, mode,
+			        ss->cache->location,
+			        (brush->flag & BRUSH_FRONTFACE) ? ss->cache->view_normal : NULL,
+			        ss->cache->radius);
 		}
 
 		MEM_freeN(nodes);
@@ -5167,7 +5169,10 @@ static int sculpt_detail_flood_fill_exec(bContext *C, wmOperator *UNUSED(op))
 	sculpt_undo_push_begin("Dynamic topology flood fill");
 	sculpt_undo_push_node(ob, NULL, SCULPT_UNDO_COORDS);
 
-	while (BKE_pbvh_bmesh_update_topology(ss->pbvh, PBVH_Collapse | PBVH_Subdivide, bb_min, size)) {
+	while (BKE_pbvh_bmesh_update_topology(
+	               ss->pbvh, PBVH_Collapse | PBVH_Subdivide,
+	               bb_min, NULL, size))
+	{
 		for (i = 0; i < totnodes; i++)
 			BKE_pbvh_node_mark_topology_update(nodes[i]);
 	}