Fix T63467: Edge/vertex selection isnt working properly with X-ray set to 1

Edit mode shows the mesh elements in X-ray mode even if alpha is set to
1. Now the code takes this into account so that you can still select
visible mesh elements in X-ray edit mode.

view3d_draw_legacy need to be updated with the new XRAY flag macros to
avoid crashes. Additional cleanup of the XRAY macro flags were done.

10 files changed, 24 insertions, 28 deletions

diff --git a/source/blender/draw/engines/workbench/workbench_forward.c b/source/blender/draw/engines/workbench/workbench_forward.c
index a6c41f41198..85596b22225 100644
--- a/source/blender/draw/engines/workbench/workbench_forward.c
+++ b/source/blender/draw/engines/workbench/workbench_forward.c
@@ -382,7 +382,7 @@ void workbench_forward_engine_init(WORKBENCH_Data *vedata)
 
 	/* TODO(campbell): displays but masks geometry,
 	 * only use with wire or solid-without-xray for now. */
-	if (((wpd->shading.type != OB_WIRE && XRAY_FLAG(wpd) == 0)) &&
+	if ((wpd->shading.type != OB_WIRE && !XRAY_FLAG_ENABLED(wpd)) &&
 	    (draw_ctx->rv3d && (draw_ctx->rv3d->rflag & RV3D_CLIPPING) && draw_ctx->rv3d->clipbb))
 	{
 		psl->background_pass = DRW_pass_create(
@@ -408,7 +408,7 @@ void workbench_forward_engine_init(WORKBENCH_Data *vedata)
 			noise_offset = fmodf(noise_offset + 1.0f / 8.0f, 1.0f);
 		}
 
-		if (wpd->shading.flag & XRAY_FLAG(wpd)) {
+		if (XRAY_FLAG_ENABLED(wpd)) {
 			blend_threshold = 1.0f - XRAY_ALPHA(wpd) * 0.9f;
 		}
 
diff --git a/source/blender/draw/engines/workbench/workbench_render.c b/source/blender/draw/engines/workbench/workbench_render.c
index 7ad8eea1c6e..663727285a1 100644
--- a/source/blender/draw/engines/workbench/workbench_render.c
+++ b/source/blender/draw/engines/workbench/workbench_render.c
@@ -137,7 +137,7 @@ void workbench_render(WORKBENCH_Data *data, RenderEngine *engine, RenderLayer *r
 		return;
 	}
 
-	const bool deferred = (scene->display.shading.flag & XRAY_FLAG(&scene->display)) == 0;
+	const bool deferred = !XRAY_FLAG_ENABLED(&scene->display);
 
 	if (deferred) {
 		/* Init engine. */
diff --git a/source/blender/draw/modes/edit_armature_mode.c b/source/blender/draw/modes/edit_armature_mode.c
index 26d232600d0..e2af2f10779 100644
--- a/source/blender/draw/modes/edit_armature_mode.c
+++ b/source/blender/draw/modes/edit_armature_mode.c
@@ -114,8 +114,7 @@ static void EDIT_ARMATURE_cache_populate(void *vedata, Object *ob)
 			const DRWContextState *draw_ctx = DRW_context_state_get();
 
 			int ghost = (ob->dtx & OB_DRAWXRAY) ? 1 : 0;
-			bool transp = (stl->g_data->transparent_bones || (ob->dt <= OB_WIRE)) ||
-			              (draw_ctx->v3d->shading.flag & XRAY_FLAG(draw_ctx->v3d)) != 0;
+			bool transp = (stl->g_data->transparent_bones || (ob->dt <= OB_WIRE)) || XRAY_FLAG_ENABLED(draw_ctx->v3d);
 
 			DRWArmaturePasses passes = {
 			    .bone_solid = (transp) ? psl->bone_transp[ghost] : psl->bone_solid[ghost],
diff --git a/source/blender/draw/modes/edit_mesh_mode.c b/source/blender/draw/modes/edit_mesh_mode.c
index 01bd43d9c6e..dedf7eac00a 100644
--- a/source/blender/draw/modes/edit_mesh_mode.c
+++ b/source/blender/draw/modes/edit_mesh_mode.c
@@ -387,7 +387,7 @@ static void EDIT_MESH_cache_init(void *vedata)
 	stl->g_data->do_faces = true;
 	stl->g_data->do_edges = true;
 
-	stl->g_data->do_zbufclip = ((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0;
+	stl->g_data->do_zbufclip = XRAY_FLAG_ENABLED(v3d);
 
 	stl->g_data->data_mask[0] = 0xFF; /* Face Flag */
 	stl->g_data->data_mask[1] = 0xFF; /* Edge Flag */
@@ -742,7 +742,7 @@ static void EDIT_MESH_draw_scene(void *vedata)
 		DRW_draw_pass(psl->normals);
 		DRW_draw_pass(psl->edit_face_overlay);
 
-		if (v3d->shading.type == OB_SOLID && (v3d->shading.flag & XRAY_FLAG(v3d)) == 0 &&
+		if (v3d->shading.type == OB_SOLID && !XRAY_FLAG_ENABLED(v3d) &&
 		    stl->g_data->ghost_ob == 1 && stl->g_data->edit_ob == 1)
 		{
 			/* In the case of single ghost object edit (common case for retopology):
diff --git a/source/blender/draw/modes/object_mode.c b/source/blender/draw/modes/object_mode.c
index 11575cbf227..7b6519a6863 100644
--- a/source/blender/draw/modes/object_mode.c
+++ b/source/blender/draw/modes/object_mode.c
@@ -1840,8 +1840,7 @@ static void camera_view3d_reconstruction(
 
 	BLI_assert(BLI_listbase_is_empty(&sgl->camera_path));
 	const bool is_solid_bundle = (v3d->bundle_drawtype == OB_EMPTY_SPHERE) &&
-	                             ((v3d->shading.type != OB_SOLID) ||
-	                              ((v3d->shading.flag & XRAY_FLAG(v3d)) == 0));
+	                             ((v3d->shading.type != OB_SOLID) || !XRAY_FLAG_ENABLED(v3d));
 
 	MovieTracking *tracking = &clip->tracking;
 	/* Index must start in 1, to mimic BKE_tracking_track_get_indexed. */
@@ -3155,8 +3154,7 @@ static void OBJECT_cache_populate(void *vedata, Object *ob)
 			bArmature *arm = ob->data;
 			if (arm->edbo == NULL) {
 				if (DRW_state_is_select() || !DRW_pose_mode_armature(ob, draw_ctx->obact)) {
-					bool is_wire = (v3d->shading.type == OB_WIRE) || (ob->dt <= OB_WIRE) ||
-					               (v3d->shading.flag & XRAY_FLAG(v3d)) != 0;
+					bool is_wire = (v3d->shading.type == OB_WIRE) || (ob->dt <= OB_WIRE) || XRAY_FLAG_ENABLED(v3d);
 					DRWArmaturePasses passes = {
 					    .bone_solid = (is_wire) ? NULL : sgl->bone_solid,
 					    .bone_outline = sgl->bone_outline,
diff --git a/source/blender/draw/modes/overlay_mode.c b/source/blender/draw/modes/overlay_mode.c
index 99102a22622..f2cafd51666 100644
--- a/source/blender/draw/modes/overlay_mode.c
+++ b/source/blender/draw/modes/overlay_mode.c
@@ -423,7 +423,7 @@ static void overlay_cache_finish(void *vedata)
 	View3D *v3d = ctx->v3d;
 
 	/* only in solid mode */
-	if (v3d->shading.type == OB_SOLID && (v3d->shading.flag & XRAY_FLAG(v3d)) == 0) {
+	if (v3d->shading.type == OB_SOLID && !XRAY_FLAG_ENABLED(v3d)) {
 		if (stl->g_data->ghost_stencil_test) {
 			DRW_pass_state_add(psl->face_wireframe_pass, DRW_STATE_STENCIL_EQUAL);
 		}
diff --git a/source/blender/draw/modes/pose_mode.c b/source/blender/draw/modes/pose_mode.c
index ec61eb99d52..cb90f944cbd 100644
--- a/source/blender/draw/modes/pose_mode.c
+++ b/source/blender/draw/modes/pose_mode.c
@@ -200,8 +200,7 @@ static void POSE_cache_populate(void *vedata, Object *ob)
 		}
 		if (DRW_pose_mode_armature(ob, draw_ctx->obact)) {
 			int ghost = (ob->dtx & OB_DRAWXRAY) ? 1 : 0;
-			bool transp = (ppd->transparent_bones || (ob->dt <= OB_WIRE)) ||
-			              (draw_ctx->v3d->shading.flag & XRAY_FLAG(draw_ctx->v3d)) != 0;
+			bool transp = (ppd->transparent_bones || (ob->dt <= OB_WIRE)) || XRAY_FLAG_ENABLED(draw_ctx->v3d);
 
 			DRWArmaturePasses passes = {
 			    .bone_solid = (transp) ? psl->bone_transp[ghost] : psl->bone_solid[ghost],
diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index e34014d179c..4c03be3bd5a 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -556,12 +556,11 @@ void ED_view3d_operator_properties_viewmat_get(struct wmOperator *op, int *winx,
 void ED_view3d_stop_render_preview(struct wmWindowManager *wm, struct ARegion *ar);
 void ED_view3d_shade_update(struct Main *bmain, struct View3D *v3d, struct ScrArea *sa);
 
-#define XRAY_ALPHA(v3d)   (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
-#define XRAY_FLAG(v3d)    (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
-#define XRAY_ENABLED(v3d) ((((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0) && (XRAY_ALPHA(v3d) < 1.0f))
-#define XRAY_ACTIVE(v3d)  (XRAY_ENABLED(v3d) && ((v3d)->shading.type < OB_MATERIAL))
-
-#define V3D_IS_ZBUF(v3d)     (((v3d)->shading.flag & XRAY_FLAG(v3d)) == 0)
+#define XRAY_ALPHA(v3d)        (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
+#define XRAY_FLAG(v3d)         (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
+#define XRAY_FLAG_ENABLED(v3d) (((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0)
+#define XRAY_ENABLED(v3d)      (XRAY_FLAG_ENABLED(v3d) && (XRAY_ALPHA(v3d) < 1.0f))
+#define XRAY_ACTIVE(v3d)       (XRAY_ENABLED(v3d) && ((v3d)->shading.type < OB_MATERIAL))
 
 /* view3d_draw_legacy.c */
 /* Try avoid using these more move out of legacy. */
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 155c91ec037..8b75b511319 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -215,7 +215,7 @@ bool EDBM_backbuf_border_init(
 {
 	uint *buf, *dr, buf_len;
 
-	if (vc->obedit == NULL || XRAY_ENABLED(vc->v3d)) {
+	if (vc->obedit == NULL || XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -292,7 +292,7 @@ bool EDBM_backbuf_border_mask_init(ViewContext *vc, const int mcords[][2], short
 			return false;
 		}
 	}
-	else if (XRAY_ENABLED(vc->v3d)) {
+	else if (XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -342,7 +342,7 @@ bool EDBM_backbuf_circle_init(
 			return false;
 		}
 	}
-	else if (XRAY_ENABLED(vc->v3d)) {
+	else if (XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -464,7 +464,7 @@ BMVert *EDBM_vert_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMVert *eve;
@@ -655,7 +655,7 @@ BMEdge *EDBM_edge_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMEdge *eed;
@@ -825,7 +825,7 @@ BMFace *EDBM_face_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		float dist_test = 0.0f;
 		unsigned int index;
 		BMFace *efa;
diff --git a/source/blender/editors/space_view3d/view3d_draw_legacy.c b/source/blender/editors/space_view3d/view3d_draw_legacy.c
index 4a1fd58b48a..f7fcd5cb11f 100644
--- a/source/blender/editors/space_view3d/view3d_draw_legacy.c
+++ b/source/blender/editors/space_view3d/view3d_draw_legacy.c
@@ -81,6 +81,7 @@
 #include "ED_space_api.h"
 #include "ED_screen_types.h"
 #include "ED_transform.h"
+#include "ED_view3d.h"
 
 #include "UI_interface.h"
 #include "UI_interface_icons.h"
@@ -174,12 +175,12 @@ static void validate_object_select_id(
 		/* do nothing */
 	}
 	else if ((obact_eval && (obact_eval->mode & OB_MODE_PARTICLE_EDIT)) &&
-	         V3D_IS_ZBUF(v3d))
+	         !XRAY_ENABLED(v3d))
 	{
 		/* do nothing */
 	}
 	else if ((obedit && (obedit->mode & OB_MODE_EDIT)) &&
-	         V3D_IS_ZBUF(v3d))
+	         !XRAY_FLAG_ENABLED(v3d))
 	{
 		/* do nothing */
 	}