Fix T63467: Edge/vertex selection isnt working properly with X-ray set to 1

Edit mode shows the mesh elements in X-ray mode even if alpha is set to
1. Now the code takes this into account so that you can still select
visible mesh elements in X-ray edit mode.

view3d_draw_legacy need to be updated with the new XRAY flag macros to
avoid crashes. Additional cleanup of the XRAY macro flags were done.

3 files changed, 14 insertions, 14 deletions

diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index e34014d179c..4c03be3bd5a 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -556,12 +556,11 @@ void ED_view3d_operator_properties_viewmat_get(struct wmOperator *op, int *winx,
 void ED_view3d_stop_render_preview(struct wmWindowManager *wm, struct ARegion *ar);
 void ED_view3d_shade_update(struct Main *bmain, struct View3D *v3d, struct ScrArea *sa);
 
-#define XRAY_ALPHA(v3d)   (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
-#define XRAY_FLAG(v3d)    (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
-#define XRAY_ENABLED(v3d) ((((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0) && (XRAY_ALPHA(v3d) < 1.0f))
-#define XRAY_ACTIVE(v3d)  (XRAY_ENABLED(v3d) && ((v3d)->shading.type < OB_MATERIAL))
-
-#define V3D_IS_ZBUF(v3d)     (((v3d)->shading.flag & XRAY_FLAG(v3d)) == 0)
+#define XRAY_ALPHA(v3d)        (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
+#define XRAY_FLAG(v3d)         (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
+#define XRAY_FLAG_ENABLED(v3d) (((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0)
+#define XRAY_ENABLED(v3d)      (XRAY_FLAG_ENABLED(v3d) && (XRAY_ALPHA(v3d) < 1.0f))
+#define XRAY_ACTIVE(v3d)       (XRAY_ENABLED(v3d) && ((v3d)->shading.type < OB_MATERIAL))
 
 /* view3d_draw_legacy.c */
 /* Try avoid using these more move out of legacy. */
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 155c91ec037..8b75b511319 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -215,7 +215,7 @@ bool EDBM_backbuf_border_init(
 {
 	uint *buf, *dr, buf_len;
 
-	if (vc->obedit == NULL || XRAY_ENABLED(vc->v3d)) {
+	if (vc->obedit == NULL || XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -292,7 +292,7 @@ bool EDBM_backbuf_border_mask_init(ViewContext *vc, const int mcords[][2], short
 			return false;
 		}
 	}
-	else if (XRAY_ENABLED(vc->v3d)) {
+	else if (XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -342,7 +342,7 @@ bool EDBM_backbuf_circle_init(
 			return false;
 		}
 	}
-	else if (XRAY_ENABLED(vc->v3d)) {
+	else if (XRAY_FLAG_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -464,7 +464,7 @@ BMVert *EDBM_vert_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMVert *eve;
@@ -655,7 +655,7 @@ BMEdge *EDBM_edge_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMEdge *eed;
@@ -825,7 +825,7 @@ BMFace *EDBM_face_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (!XRAY_ENABLED(vc->v3d)) {
+	if (!XRAY_FLAG_ENABLED(vc->v3d)) {
 		float dist_test = 0.0f;
 		unsigned int index;
 		BMFace *efa;
diff --git a/source/blender/editors/space_view3d/view3d_draw_legacy.c b/source/blender/editors/space_view3d/view3d_draw_legacy.c
index 4a1fd58b48a..f7fcd5cb11f 100644
--- a/source/blender/editors/space_view3d/view3d_draw_legacy.c
+++ b/source/blender/editors/space_view3d/view3d_draw_legacy.c
@@ -81,6 +81,7 @@
 #include "ED_space_api.h"
 #include "ED_screen_types.h"
 #include "ED_transform.h"
+#include "ED_view3d.h"
 
 #include "UI_interface.h"
 #include "UI_interface_icons.h"
@@ -174,12 +175,12 @@ static void validate_object_select_id(
 		/* do nothing */
 	}
 	else if ((obact_eval && (obact_eval->mode & OB_MODE_PARTICLE_EDIT)) &&
-	         V3D_IS_ZBUF(v3d))
+	         !XRAY_ENABLED(v3d))
 	{
 		/* do nothing */
 	}
 	else if ((obedit && (obedit->mode & OB_MODE_EDIT)) &&
-	         V3D_IS_ZBUF(v3d))
+	         !XRAY_FLAG_ENABLED(v3d))
 	{
 		/* do nothing */
 	}