Fix T62114: Wireframe mode selection: selects backside objects when clicking frontside object in 3d-window

Now the depth order of objects when trying to select them is only used when not in X-ray mode.
Before, this was only the case in wireframe mode regardless of X-ray settings.

I've also unified the usage of V3D_XRAY and XRAY_FLAG as they were basically copies of each other.

Reviewed By: Clément

Differential Revision: http://developer.blender.org/D4504

17 files changed, 45 insertions, 34 deletions

diff --git a/source/blender/draw/engines/gpencil/gpencil_engine.c b/source/blender/draw/engines/gpencil/gpencil_engine.c
index 56ef4b97dcf..303178c1a5e 100644
--- a/source/blender/draw/engines/gpencil/gpencil_engine.c
+++ b/source/blender/draw/engines/gpencil/gpencil_engine.c
@@ -405,7 +405,7 @@ void GPENCIL_cache_init(void *vedata)
 
 		/* xray mode */
 		if (v3d) {
-			stl->storage->is_xray = (v3d->shading.flag & V3D_XRAY_FLAG(v3d)) ? 1 : 0;
+			stl->storage->is_xray = XRAY_ACTIVE(v3d);
 		}
 		else {
 			stl->storage->is_xray = 0;
diff --git a/source/blender/draw/engines/workbench/workbench_forward.c b/source/blender/draw/engines/workbench/workbench_forward.c
index 45f1c632a05..a6c41f41198 100644
--- a/source/blender/draw/engines/workbench/workbench_forward.c
+++ b/source/blender/draw/engines/workbench/workbench_forward.c
@@ -38,6 +38,7 @@
 #include "DNA_modifier_types.h"
 #include "DNA_node_types.h"
 
+#include "ED_view3d.h"
 
 #include "GPU_shader.h"
 #include "GPU_texture.h"
diff --git a/source/blender/draw/engines/workbench/workbench_render.c b/source/blender/draw/engines/workbench/workbench_render.c
index c551a369b58..7ad8eea1c6e 100644
--- a/source/blender/draw/engines/workbench/workbench_render.c
+++ b/source/blender/draw/engines/workbench/workbench_render.c
@@ -28,6 +28,8 @@
 
 #include "DRW_render.h"
 
+#include "ED_view3d.h"
+
 #include "GPU_shader.h"
 
 #include "DEG_depsgraph.h"
diff --git a/source/blender/draw/intern/DRW_render.h b/source/blender/draw/intern/DRW_render.h
index 3d4618021e8..ce75cc0f05d 100644
--- a/source/blender/draw/intern/DRW_render.h
+++ b/source/blender/draw/intern/DRW_render.h
@@ -598,8 +598,4 @@ typedef struct DRWContextState {
 
 const DRWContextState *DRW_context_state_get(void);
 
-#define XRAY_ALPHA(v3d)   (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
-#define XRAY_FLAG(v3d)    (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
-#define XRAY_ENABLED(v3d) ((((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0) && (XRAY_ALPHA(v3d) < 1.0f))
-
 #endif /* __DRW_RENDER_H__ */
diff --git a/source/blender/draw/modes/edit_armature_mode.c b/source/blender/draw/modes/edit_armature_mode.c
index 68f091da797..26d232600d0 100644
--- a/source/blender/draw/modes/edit_armature_mode.c
+++ b/source/blender/draw/modes/edit_armature_mode.c
@@ -26,6 +26,8 @@
 #include "DNA_armature_types.h"
 #include "DNA_view3d_types.h"
 
+#include "ED_view3d.h"
+
 #include "draw_common.h"
 #include "draw_mode_engines.h"
 
diff --git a/source/blender/draw/modes/edit_mesh_mode.c b/source/blender/draw/modes/edit_mesh_mode.c
index 71c759d47a5..01bd43d9c6e 100644
--- a/source/blender/draw/modes/edit_mesh_mode.c
+++ b/source/blender/draw/modes/edit_mesh_mode.c
@@ -43,6 +43,8 @@
 #include "BLI_dynstr.h"
 #include "BLI_string_utils.h"
 
+#include "ED_view3d.h"
+
 extern char datatoc_paint_weight_vert_glsl[];
 extern char datatoc_paint_weight_frag_glsl[];
 
diff --git a/source/blender/draw/modes/object_mode.c b/source/blender/draw/modes/object_mode.c
index 563d1267885..94935da0f05 100644
--- a/source/blender/draw/modes/object_mode.c
+++ b/source/blender/draw/modes/object_mode.c
@@ -1011,8 +1011,7 @@ static void OBJECT_cache_init(void *vedata)
 	}
 
 	g_data = stl->g_data;
-	g_data->xray_enabled = XRAY_ENABLED(draw_ctx->v3d) &&
-	                       (draw_ctx->v3d->shading.type < OB_MATERIAL);
+	g_data->xray_enabled = XRAY_ACTIVE(draw_ctx->v3d);
 	g_data->xray_enabled_and_not_wire = g_data->xray_enabled && draw_ctx->v3d->shading.type > OB_WIRE;
 
 	{
diff --git a/source/blender/draw/modes/overlay_mode.c b/source/blender/draw/modes/overlay_mode.c
index 52e8f73f971..99102a22622 100644
--- a/source/blender/draw/modes/overlay_mode.c
+++ b/source/blender/draw/modes/overlay_mode.c
@@ -34,6 +34,8 @@
 #include "GPU_shader.h"
 #include "DRW_render.h"
 
+#include "ED_view3d.h"
+
 #include "draw_mode_engines.h"
 
 #ifdef __APPLE__
diff --git a/source/blender/draw/modes/pose_mode.c b/source/blender/draw/modes/pose_mode.c
index c4050bf3f1a..ec61eb99d52 100644
--- a/source/blender/draw/modes/pose_mode.c
+++ b/source/blender/draw/modes/pose_mode.c
@@ -27,6 +27,8 @@
 #include "DRW_engine.h"
 #include "DRW_render.h"
 
+#include "ED_view3d.h"
+
 /* If builtin shaders are needed */
 #include "GPU_shader.h"
 
diff --git a/source/blender/editors/armature/armature_select.c b/source/blender/editors/armature/armature_select.c
index 6e93ecc484d..4aab80bcbdd 100644
--- a/source/blender/editors/armature/armature_select.c
+++ b/source/blender/editors/armature/armature_select.c
@@ -419,7 +419,7 @@ static EditBone *get_nearest_editbonepoint(
 	if (use_cycle) {
 		static int last_mval[2] = {-100, -100};
 
-		if (vc->v3d->shading.type > OB_WIRE) {
+		if (!XRAY_ACTIVE(vc->v3d)) {
 			do_nearest = true;
 			if (len_manhattan_v2v2_int(vc->mval, last_mval) <= WM_EVENT_CURSOR_MOTION_THRESHOLD) {
 				do_nearest = false;
@@ -428,7 +428,7 @@ static EditBone *get_nearest_editbonepoint(
 		copy_v2_v2_int(last_mval, vc->mval);
 	}
 	else {
-		if (vc->v3d->shading.type > OB_WIRE) {
+		if (!XRAY_ACTIVE(vc->v3d)) {
 			do_nearest = true;
 		}
 	}
diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index 2a115f49c31..e34014d179c 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -556,8 +556,12 @@ void ED_view3d_operator_properties_viewmat_get(struct wmOperator *op, int *winx,
 void ED_view3d_stop_render_preview(struct wmWindowManager *wm, struct ARegion *ar);
 void ED_view3d_shade_update(struct Main *bmain, struct View3D *v3d, struct ScrArea *sa);
 
-#define V3D_XRAY_FLAG(v3d)   (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
-#define V3D_IS_ZBUF(v3d)     (((v3d)->shading.flag & V3D_XRAY_FLAG(v3d)) == 0)
+#define XRAY_ALPHA(v3d)   (((v3d)->shading.type == OB_WIRE) ? (v3d)->shading.xray_alpha_wire : (v3d)->shading.xray_alpha)
+#define XRAY_FLAG(v3d)    (((v3d)->shading.type == OB_WIRE) ? V3D_SHADING_XRAY_BONE : V3D_SHADING_XRAY)
+#define XRAY_ENABLED(v3d) ((((v3d)->shading.flag & XRAY_FLAG(v3d)) != 0) && (XRAY_ALPHA(v3d) < 1.0f))
+#define XRAY_ACTIVE(v3d)  (XRAY_ENABLED(v3d) && ((v3d)->shading.type < OB_MATERIAL))
+
+#define V3D_IS_ZBUF(v3d)     (((v3d)->shading.flag & XRAY_FLAG(v3d)) == 0)
 
 /* view3d_draw_legacy.c */
 /* Try avoid using these more move out of legacy. */
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 1855eeab944..c04fe58c12e 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -215,7 +215,7 @@ bool EDBM_backbuf_border_init(
 {
 	uint *buf, *dr, buf_len;
 
-	if (vc->obedit == NULL || !V3D_IS_ZBUF(vc->v3d)) {
+	if (vc->obedit == NULL || XRAY_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -292,7 +292,7 @@ bool EDBM_backbuf_border_mask_init(ViewContext *vc, const int mcords[][2], short
 			return false;
 		}
 	}
-	else if (!V3D_IS_ZBUF(vc->v3d)) {
+	else if (XRAY_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -342,7 +342,7 @@ bool EDBM_backbuf_circle_init(
 			return false;
 		}
 	}
-	else if (!V3D_IS_ZBUF(vc->v3d)) {
+	else if (XRAY_ENABLED(vc->v3d)) {
 		return false;
 	}
 
@@ -464,7 +464,7 @@ BMVert *EDBM_vert_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (V3D_IS_ZBUF(vc->v3d)) {
+	if (!XRAY_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMVert *eve;
@@ -655,7 +655,7 @@ BMEdge *EDBM_edge_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (V3D_IS_ZBUF(vc->v3d)) {
+	if (!XRAY_ENABLED(vc->v3d)) {
 		uint dist_px = (uint)ED_view3d_backbuf_sample_size_clamp(vc->ar, *r_dist);
 		unsigned int index;
 		BMEdge *eed;
@@ -825,7 +825,7 @@ BMFace *EDBM_face_find_nearest_ex(
 {
 	BMesh *bm = vc->em->bm;
 
-	if (V3D_IS_ZBUF(vc->v3d)) {
+	if (!XRAY_ENABLED(vc->v3d)) {
 		float dist_test = 0.0f;
 		unsigned int index;
 		BMFace *efa;
diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index 112d453c44a..efc969a333e 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -446,7 +446,7 @@ static void PE_set_view3d_data(bContext *C, PEData *data)
 
 	ED_view3d_viewcontext_init(C, &data->vc);
 
-	if (V3D_IS_ZBUF(data->vc.v3d)) {
+	if (!XRAY_ENABLED(data->vc.v3d)) {
 		if (data->vc.v3d->flag & V3D_INVALID_BACKBUF) {
 			/* needed or else the draw matrix can be incorrect */
 			view3d_operator_needs_opengl(C);
@@ -503,7 +503,7 @@ static bool key_test_depth(const PEData *data, const float co[3], const int scre
 	float depth;
 
 	/* nothing to do */
-	if (!V3D_IS_ZBUF(v3d))
+	if (XRAY_ENABLED(v3d))
 		return true;
 
 	/* used to calculate here but all callers have  the screen_co already, so pass as arg */
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index ea44f19c4fe..3ff6a7146c6 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -943,7 +943,7 @@ static void do_lasso_select_meshobject__doSelectVert(void *userData, MVert *mv,
 }
 static bool do_lasso_select_paintvert(ViewContext *vc, const int mcords[][2], short moves, const eSelectOp sel_op)
 {
-	const bool use_zbuf = V3D_IS_ZBUF(vc->v3d);
+	const bool use_zbuf = !XRAY_ENABLED(vc->v3d);
 	Object *ob = vc->obact;
 	Mesh *me = ob->data;
 	rcti rect;
@@ -1472,7 +1472,7 @@ static int mixed_bones_object_selectbuffer_extended(
 
 	/* define if we use solid nearest select or not */
 	if (use_cycle) {
-		if (v3d->shading.type > OB_WIRE) {
+		if (!XRAY_ACTIVE(v3d)) {
 			do_nearest = true;
 			if (len_manhattan_v2v2_int(mval, last_mval) <= WM_EVENT_CURSOR_MOTION_THRESHOLD) {
 				do_nearest = false;
@@ -1481,7 +1481,7 @@ static int mixed_bones_object_selectbuffer_extended(
 		copy_v2_v2_int(last_mval, mval);
 	}
 	else {
-		if (v3d->shading.type > OB_WIRE) {
+		if (!XRAY_ACTIVE(v3d)) {
 			do_nearest = true;
 		}
 	}
@@ -1630,7 +1630,7 @@ Base *ED_view3d_give_base_under_cursor(bContext *C, const int mval[2])
 
 	ED_view3d_viewcontext_init(C, &vc);
 
-	const bool do_nearest = (vc.v3d->shading.type > OB_WIRE);
+	const bool do_nearest = !XRAY_ACTIVE(vc.v3d);
 	const int hits = mixed_bones_object_selectbuffer(
 	        &vc, buffer, mval, VIEW3D_SELECT_FILTER_NOP, do_nearest);
 
@@ -1996,7 +1996,7 @@ static bool ed_wpaint_vertex_select_pick(
         bool extend, bool deselect, bool toggle, Object *obact)
 {
 	View3D *v3d = CTX_wm_view3d(C);
-	const bool use_zbuf = V3D_IS_ZBUF(v3d);
+	const bool use_zbuf = !XRAY_ENABLED(v3d);
 
 	Mesh *me = obact->data; /* already checked for NULL */
 	uint index = 0;
@@ -2246,7 +2246,7 @@ static void do_paintvert_box_select__doSelectVert(void *userData, MVert *mv, con
 static bool do_paintvert_box_select(
         ViewContext *vc, const rcti *rect, const eSelectOp sel_op)
 {
-	const bool use_zbuf = V3D_IS_ZBUF(vc->v3d);
+	const bool use_zbuf = !XRAY_ENABLED(vc->v3d);
 	Mesh *me;
 	MVert *mvert;
 	unsigned int *rt;
@@ -3096,7 +3096,7 @@ static void paint_vertsel_circle_select_doSelectVert(void *userData, MVert *mv,
 static bool paint_vertsel_circle_select(ViewContext *vc, const eSelectOp sel_op, const int mval[2], float rad)
 {
 	BLI_assert(ELEM(sel_op, SEL_OP_SET, SEL_OP_ADD, SEL_OP_SUB));
-	const bool use_zbuf = V3D_IS_ZBUF(vc->v3d);
+	const bool use_zbuf = !XRAY_ENABLED(vc->v3d);
 	Object *ob = vc->obact;
 	Mesh *me = ob->data;
 	bool bbsel;
diff --git a/source/blender/editors/space_view3d/view3d_view.c b/source/blender/editors/space_view3d/view3d_view.c
index 21716f28b85..4e5dddf4742 100644
--- a/source/blender/editors/space_view3d/view3d_view.c
+++ b/source/blender/editors/space_view3d/view3d_view.c
@@ -1049,7 +1049,7 @@ int view3d_opengl_select(
 	 * the object & bone view locking takes 'rect' into account, see: T51629. */
 	ED_view3d_draw_setup_view(vc->win, depsgraph, scene, ar, v3d, vc->rv3d->viewmat, NULL, &rect);
 
-	if (v3d->shading.type > OB_WIRE) {
+	if (!XRAY_ACTIVE(v3d)) {
 		GPU_depth_test(true);
 	}
 
@@ -1058,7 +1058,7 @@ int view3d_opengl_select(
 	}
 
 	/* If in xray mode, we select the wires in priority. */
-	if ((v3d->shading.flag & V3D_XRAY_FLAG(v3d)) && use_nearest) {
+	if (XRAY_ACTIVE(v3d) && use_nearest) {
 		/* We need to call "GPU_select_*" API's inside DRW_draw_select_loop
 		 * because the OpenGL context created & destroyed inside this function. */
 		struct DrawSelectLoopUserData drw_select_loop_user_data = {
@@ -1092,7 +1092,8 @@ int view3d_opengl_select(
 			.rect = &rect,
 			.gpu_select_mode = gpu_select_mode,
 		};
-		draw_surface = (v3d->shading.type > OB_WIRE) || ((v3d->shading.flag & V3D_XRAY_FLAG(v3d)) == 0);
+		/* If are not in wireframe mode, we need to use the mesh surfaces to check for hits */
+		draw_surface = (v3d->shading.type > OB_WIRE) || !XRAY_ENABLED(v3d);
 		DRW_draw_select_loop(
 		        depsgraph, ar, v3d,
 		        use_obedit_skip, draw_surface, use_nearest, &rect,
@@ -1104,7 +1105,7 @@ int view3d_opengl_select(
 	G.f &= ~G_FLAG_PICKSEL;
 	ED_view3d_draw_setup_view(vc->win, depsgraph, scene, ar, v3d, vc->rv3d->viewmat, NULL, NULL);
 
-	if (v3d->shading.type > OB_WIRE) {
+	if (!XRAY_ACTIVE(v3d)) {
 		GPU_depth_test(false);
 	}
 
diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index e785268b80a..d937273e3e3 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -7094,7 +7094,7 @@ static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *t
 	if (t->spacetype == SPACE_VIEW3D) {
 		v3d = t->sa ? t->sa->spacedata.first : NULL;
 		rv3d = t->ar ? t->ar->regiondata : NULL;
-		use_occlude_geometry = (v3d && TRANS_DATA_CONTAINER_FIRST_OK(t)->obedit->dt > OB_WIRE && v3d->shading.type > OB_WIRE);
+		use_occlude_geometry = (v3d && TRANS_DATA_CONTAINER_FIRST_OK(t)->obedit->dt > OB_WIRE && !XRAY_ENABLED(v3d));
 	}
 
 	calcEdgeSlide_mval_range(t, tc, sld, sv_table, loop_nr, mval, use_occlude_geometry, true);
@@ -7289,7 +7289,7 @@ static bool createEdgeSlideVerts_single_side(TransInfo *t, TransDataContainer *t
 	if (t->spacetype == SPACE_VIEW3D) {
 		v3d = t->sa ? t->sa->spacedata.first : NULL;
 		rv3d = t->ar ? t->ar->regiondata : NULL;
-		use_occlude_geometry = (v3d && TRANS_DATA_CONTAINER_FIRST_OK(t)->obedit->dt > OB_WIRE && v3d->shading.type > OB_WIRE);
+		use_occlude_geometry = (v3d && TRANS_DATA_CONTAINER_FIRST_OK(t)->obedit->dt > OB_WIRE && !XRAY_ENABLED(v3d));
 	}
 
 	calcEdgeSlide_mval_range(t, tc, sld, sv_table, loop_nr, mval, use_occlude_geometry, false);
diff --git a/source/blender/editors/transform/transform_snap_object.c b/source/blender/editors/transform/transform_snap_object.c
index 7ca62b89bf9..f2692f57a35 100644
--- a/source/blender/editors/transform/transform_snap_object.c
+++ b/source/blender/editors/transform/transform_snap_object.c
@@ -678,7 +678,7 @@ static bool raycastObj(
 		{
 			if (use_occlusion_test) {
 				if (use_obedit && sctx->use_v3d &&
-				    !V3D_IS_ZBUF(sctx->v3d_data.v3d))
+				    XRAY_ENABLED(sctx->v3d_data.v3d))
 				{
 					/* Use of occlude geometry in editing mode disabled. */
 					return false;
@@ -2541,7 +2541,7 @@ static short transform_snap_context_project_view3d_mixed_impl(
 	const RegionView3D *rv3d = ar->regiondata;
 
 	bool use_occlusion_test =
-	        params->use_occlusion_test && V3D_IS_ZBUF(sctx->v3d_data.v3d);
+	        params->use_occlusion_test && !XRAY_ENABLED(sctx->v3d_data.v3d);
 
 	if (snap_to_flag & SCE_SNAP_MODE_FACE || use_occlusion_test) {
 		float ray_start[3], ray_normal[3];