Fix T47164: [Scene.raycast] - True result when it should be False.

We cannot use FLT_MAX as initi distance for raycast...

Renamed TRANSFORM_DIST_MAX_RAY to BVH_RAYCAST_DIST_MAX, moved it into BLI_kdopbvh,
and use in RNA raycast callbacks (and all other places using that API).

15 files changed, 32 insertions, 25 deletions

diff --git a/source/blender/blenkernel/intern/constraint.c b/source/blender/blenkernel/intern/constraint.c
index ec3dd9272e1..dabe606fa55 100644
--- a/source/blender/blenkernel/intern/constraint.c
+++ b/source/blender/blenkernel/intern/constraint.c
@@ -3478,7 +3478,7 @@ static void shrinkwrap_get_tarmat(bConstraint *con, bConstraintOb *cob, bConstra
 
 					/* TODO should use FLT_MAX.. but normal projection doenst yet supports it */
 					hit.index = -1;
-					hit.dist = (scon->projLimit == 0.0f) ? 100000.0f : scon->projLimit;
+					hit.dist = (scon->projLimit == 0.0f) ? BVH_RAYCAST_DIST_MAX : scon->projLimit;
 
 					switch (scon->projAxis) {
 						case OB_POSX: case OB_POSY: case OB_POSZ:
@@ -4123,7 +4123,7 @@ static void followtrack_evaluate(bConstraint *con, bConstraintOb *cob, ListBase
 
 					bvhtree_from_mesh_looptri(&treeData, target, 0.0f, 4, 6);
 
-					hit.dist = FLT_MAX;
+					hit.dist = BVH_RAYCAST_DIST_MAX;
 					hit.index = -1;
 
 					result = BLI_bvhtree_ray_cast(treeData.tree, ray_start, ray_nor, 0.0f, &hit, treeData.raycast_callback, &treeData);
diff --git a/source/blender/blenkernel/intern/dynamicpaint.c b/source/blender/blenkernel/intern/dynamicpaint.c
index 67a6d067aba..3ed148023a3 100644
--- a/source/blender/blenkernel/intern/dynamicpaint.c
+++ b/source/blender/blenkernel/intern/dynamicpaint.c
@@ -3274,7 +3274,7 @@ static int dynamicPaint_paintMesh(DynamicPaintSurface *surface,
 							add_v3_fl(ray_start, 0.001f);
 
 							hit.index = -1;
-							hit.dist = 9999;
+							hit.dist = BVH_RAYCAST_DIST_MAX;
 							nearest.index = -1;
 							nearest.dist_sq = brush_radius * brush_radius; /* find_nearest uses squared distance */
 
@@ -3304,7 +3304,7 @@ static int dynamicPaint_paintMesh(DynamicPaintSurface *surface,
 										 * point is at least surrounded by two brush faces */
 										negate_v3(ray_dir);
 										hit.index = -1;
-										hit.dist = 9999;
+										hit.dist = BVH_RAYCAST_DIST_MAX;
 
 										BLI_bvhtree_ray_cast(treeData.tree, ray_start, ray_dir, 0.0f, &hit, mesh_tris_spherecast_dp, &treeData);
 
diff --git a/source/blender/blenkernel/intern/shrinkwrap.c b/source/blender/blenkernel/intern/shrinkwrap.c
index 40c58cea3d9..bbed4ccfae7 100644
--- a/source/blender/blenkernel/intern/shrinkwrap.c
+++ b/source/blender/blenkernel/intern/shrinkwrap.c
@@ -326,7 +326,7 @@ static void shrinkwrap_calc_normal_projection(ShrinkwrapCalcData *calc, bool for
 
 
 			hit.index = -1;
-			hit.dist = 10000.0f; /* TODO: we should use FLT_MAX here, but sweepsphere code isn't prepared for that */
+			hit.dist = BVH_RAYCAST_DIST_MAX; /* TODO: we should use FLT_MAX here, but sweepsphere code isn't prepared for that */
 
 			/* Project over positive direction of axis */
 			if (calc->smd->shrinkOpts & MOD_SHRINKWRAP_PROJECT_ALLOW_POS_DIR) {
diff --git a/source/blender/blenlib/BLI_kdopbvh.h b/source/blender/blenlib/BLI_kdopbvh.h
index 0c359c7f090..d91992fff99 100644
--- a/source/blender/blenlib/BLI_kdopbvh.h
+++ b/source/blender/blenlib/BLI_kdopbvh.h
@@ -77,6 +77,7 @@ enum {
 	BVH_RAYCAST_WATERTIGHT		= (1 << 0),
 };
 #define BVH_RAYCAST_DEFAULT (BVH_RAYCAST_WATERTIGHT)
+#define BVH_RAYCAST_DIST_MAX (FLT_MAX / 2.0f)
 
 /* callback must update nearest in case it finds a nearest result */
 typedef void (*BVHTree_NearestPointCallback)(void *userdata, int index, const float co[3], BVHTreeNearest *nearest);
diff --git a/source/blender/blenlib/intern/BLI_kdopbvh.c b/source/blender/blenlib/intern/BLI_kdopbvh.c
index 3752751085f..f1ed49f74e0 100644
--- a/source/blender/blenlib/intern/BLI_kdopbvh.c
+++ b/source/blender/blenlib/intern/BLI_kdopbvh.c
@@ -1611,7 +1611,7 @@ static void dfs_raycast_all(BVHRayCastData *data, BVHNode *node)
 	if (node->totnode == 0) {
 		if (data->callback) {
 			data->hit.index = -1;
-			data->hit.dist = FLT_MAX;
+			data->hit.dist = BVH_RAYCAST_DIST_MAX;
 			data->callback(data->userdata, node->index, &data->ray, &data->hit);
 		}
 		else {
@@ -1720,7 +1720,7 @@ int BLI_bvhtree_ray_cast_ex(
 	}
 	else {
 		data.hit.index = -1;
-		data.hit.dist = FLT_MAX;
+		data.hit.dist = BVH_RAYCAST_DIST_MAX;
 	}
 
 	if (root) {
@@ -1747,7 +1747,7 @@ float BLI_bvhtree_bb_raycast(const float bv[6], const float light_start[3], cons
 	BVHRayCastData data;
 	float dist;
 
-	data.hit.dist = FLT_MAX;
+	data.hit.dist = BVH_RAYCAST_DIST_MAX;
 	
 	/* get light direction */
 	sub_v3_v3v3(data.ray.direction, light_end, light_start);
@@ -1792,7 +1792,7 @@ int BLI_bvhtree_ray_cast_all_ex(
 	bvhtree_ray_cast_data_precalc(&data, flag);
 
 	data.hit.index = -1;
-	data.hit.dist = FLT_MAX;
+	data.hit.dist = BVH_RAYCAST_DIST_MAX;
 
 	if (root) {
 		dfs_raycast_all(&data, root);
diff --git a/source/blender/bmesh/intern/bmesh_polygon_edgenet.c b/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
index bd7bdc74c1e..1ee96d4e759 100644
--- a/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
+++ b/source/blender/bmesh/intern/bmesh_polygon_edgenet.c
@@ -808,7 +808,7 @@ static BMEdge *test_edges_isect_2d_ray(
 	BLI_ASSERT_UNIT_V2(dir);
 
 	hit.index = -1;
-	hit.dist = FLT_MAX;
+	hit.dist = BVH_RAYCAST_DIST_MAX;
 
 	struct Edges_VertRay_BVHTreeTest user_data = {0};
 	user_data.edge_arr = args->edge_arr;
diff --git a/source/blender/bmesh/tools/bmesh_intersect.c b/source/blender/bmesh/tools/bmesh_intersect.c
index abcc4b132ea..e6437aa6c59 100644
--- a/source/blender/bmesh/tools/bmesh_intersect.c
+++ b/source/blender/bmesh/tools/bmesh_intersect.c
@@ -899,7 +899,7 @@ static int isect_bvhtree_point_v3(
 	 * keeps calling the intersect callback.
 	 */
 	hit.index = -1;
-	hit.dist = FLT_MAX;
+	hit.dist = BVH_RAYCAST_DIST_MAX;
 
 	BLI_bvhtree_ray_cast(tree,
 	                     co, dir,
diff --git a/source/blender/editors/armature/meshlaplacian.c b/source/blender/editors/armature/meshlaplacian.c
index 4bda8285029..9500fd59b8b 100644
--- a/source/blender/editors/armature/meshlaplacian.c
+++ b/source/blender/editors/armature/meshlaplacian.c
@@ -1008,7 +1008,7 @@ static MDefBoundIsect *meshdeform_ray_tree_intersect(MeshDeformBind *mdb, const
 	isect_mdef.vec_length = normalize_v3_v3(vec_normal, isect_mdef.vec);
 
 	hit.index = -1;
-	hit.dist = FLT_MAX;
+	hit.dist = BVH_RAYCAST_DIST_MAX;
 	if (BLI_bvhtree_ray_cast(mdb->bvhtree, isect_mdef.start, vec_normal,
 	                         0.0, &hit, harmonic_ray_callback, &data) != -1)
 	{
diff --git a/source/blender/editors/include/ED_transform.h b/source/blender/editors/include/ED_transform.h
index 73f7cd2ba7c..db8085a6696 100644
--- a/source/blender/editors/include/ED_transform.h
+++ b/source/blender/editors/include/ED_transform.h
@@ -109,6 +109,7 @@ struct TransInfo;
 struct Base;
 struct Scene;
 struct Object;
+struct wmOperator;
 
 /* UNUSED */
 // int BIF_snappingSupported(struct Object *obedit);
@@ -175,7 +176,6 @@ typedef enum SnapSelect {
 } SnapSelect;
 
 #define SNAP_MIN_DISTANCE 30
-#define TRANSFORM_DIST_MAX_RAY (FLT_MAX / 2.0f)
 
 bool peelObjectsTransForm(
         struct TransInfo *t, const float mval[2], SnapSelect snap_select,
diff --git a/source/blender/editors/space_view3d/view3d_edit.c b/source/blender/editors/space_view3d/view3d_edit.c
index 6b98d2b9784..0f407658228 100644
--- a/source/blender/editors/space_view3d/view3d_edit.c
+++ b/source/blender/editors/space_view3d/view3d_edit.c
@@ -41,6 +41,7 @@
 #include "MEM_guardedalloc.h"
 
 #include "BLI_blenlib.h"
+#include "BLI_kdopbvh.h"
 #include "BLI_math.h"
 #include "BLI_utildefines.h"
 
@@ -5183,7 +5184,7 @@ bool ED_view3d_snap_from_ray(
         float r_co[3])
 {
 	float r_no_dummy[3];
-	float ray_dist = TRANSFORM_DIST_MAX_RAY;
+	float ray_dist = BVH_RAYCAST_DIST_MAX;
 	bool ret;
 
 	struct Object *obedit = scene->obedit;
@@ -5223,7 +5224,7 @@ bool ED_view3d_snap_from_region(
         float r_co[3], float r_no[3])
 {
 	float r_no_dummy[3];
-	float ray_dist = TRANSFORM_DIST_MAX_RAY;
+	float ray_dist = BVH_RAYCAST_DIST_MAX;
 	bool is_hit = false;
 	float *r_no_ptr = r_no ? r_no : r_no_dummy;
 
@@ -5236,7 +5237,7 @@ bool ED_view3d_snap_from_region(
 	for (int i = 0; i < 3; i++) {
 		if (elem_test[i] && (is_hit == false || use_depth)) {
 			if (use_depth == false) {
-				ray_dist = TRANSFORM_DIST_MAX_RAY;
+				ray_dist = BVH_RAYCAST_DIST_MAX;
 			}
 			if (snapObjectsEx(
 			        scene, v3d, ar, NULL, obedit,
diff --git a/source/blender/editors/space_view3d/view3d_walk.c b/source/blender/editors/space_view3d/view3d_walk.c
index 8b3ea2a1fee..022b305e5ac 100644
--- a/source/blender/editors/space_view3d/view3d_walk.c
+++ b/source/blender/editors/space_view3d/view3d_walk.c
@@ -33,6 +33,7 @@
 
 #include "BLI_math.h"
 #include "BLI_blenlib.h"
+#include "BLI_kdopbvh.h"
 #include "BLI_utildefines.h"
 
 #include "BKE_context.h"
@@ -410,7 +411,7 @@ static bool walk_floor_distance_get(bContext *C, RegionView3D *rv3d, WalkInfo *w
 	float dvec_tmp[3];
 	bool ret;
 
-	*r_distance = TRANSFORM_DIST_MAX_RAY;
+	*r_distance = BVH_RAYCAST_DIST_MAX;
 
 	copy_v3_v3(ray_start, rv3d->viewinv[3]);
 
@@ -441,7 +442,7 @@ static bool walk_ray_cast(bContext *C, RegionView3D *rv3d, WalkInfo *walk, float
 	float mat[3][3]; /* 3x3 copy of the view matrix so we can move along the view axis */
 	bool ret;
 
-	*ray_distance = TRANSFORM_DIST_MAX_RAY;
+	*ray_distance = BVH_RAYCAST_DIST_MAX;
 
 	copy_v3_v3(ray_start, rv3d->viewinv[3]);
 	copy_m3_m4(mat, rv3d->viewinv);
diff --git a/source/blender/editors/transform/transform_snap.c b/source/blender/editors/transform/transform_snap.c
index 359e191c90d..869f03e416a 100644
--- a/source/blender/editors/transform/transform_snap.c
+++ b/source/blender/editors/transform/transform_snap.c
@@ -1544,7 +1544,7 @@ static bool snapDerivedMesh(
 	if (totvert > 0) {
 		float imat[4][4];
 		float timat[3][3]; /* transpose inverse matrix for normals */
-		float ray_start_local[3], ray_normal_local[3], local_scale, len_diff = TRANSFORM_DIST_MAX_RAY;
+		float ray_start_local[3], ray_normal_local[3], local_scale, len_diff = BVH_RAYCAST_DIST_MAX;
 
 		invert_m4_m4(imat, obmat);
 		transpose_m3_m4(timat, imat);
@@ -1637,7 +1637,7 @@ static bool snapDerivedMesh(
 
 				hit.index = -1;
 				hit.dist = *ray_depth;
-				if (hit.dist != TRANSFORM_DIST_MAX_RAY) {
+				if (hit.dist != BVH_RAYCAST_DIST_MAX) {
 					hit.dist *= local_scale;
 					hit.dist -= len_diff;
 				}
@@ -2081,7 +2081,7 @@ bool snapObjectsTransform(
         TransInfo *t, const float mval[2], SnapSelect snap_select,
         float r_loc[3], float r_no[3], float *r_dist_px)
 {
-	float ray_dist = TRANSFORM_DIST_MAX_RAY;
+	float ray_dist = BVH_RAYCAST_DIST_MAX;
 	Object *obedit = NULL;
 	Base *base_act = NULL;
 
@@ -2109,7 +2109,7 @@ bool snapObjectsContext(
 	Scene *scene = CTX_data_scene(C);
 	ARegion *ar = CTX_wm_region(C);
 	Object *obedit = CTX_data_edit_object(C);
-	float ray_dist = TRANSFORM_DIST_MAX_RAY;
+	float ray_dist = BVH_RAYCAST_DIST_MAX;
 
 	return snapObjects(
 	        scene, v3d, ar, scene->basact, obedit,
diff --git a/source/blender/makesrna/intern/rna_object_api.c b/source/blender/makesrna/intern/rna_object_api.c
index 5645f018cad..7cc69831dce 100644
--- a/source/blender/makesrna/intern/rna_object_api.c
+++ b/source/blender/makesrna/intern/rna_object_api.c
@@ -35,6 +35,7 @@
 #include <time.h>
 
 #include "BLI_utildefines.h"
+#include "BLI_kdopbvh.h"
 
 #include "RNA_define.h"
 
@@ -613,7 +614,8 @@ void RNA_api_object(StructRNA *srna)
 	RNA_def_property_flag(parm, PROP_REQUIRED);
 	parm = RNA_def_float_vector(func, "direction", 3, NULL, -FLT_MAX, FLT_MAX, "", "", -1e4, 1e4);
 	RNA_def_property_flag(parm, PROP_REQUIRED);
-	RNA_def_float(func, "distance", FLT_MAX, 0.0, FLT_MAX, "", "Maximum distance", 0.0, FLT_MAX);
+	RNA_def_float(func, "distance", BVH_RAYCAST_DIST_MAX, 0.0, BVH_RAYCAST_DIST_MAX,
+	              "", "Maximum distance", 0.0, BVH_RAYCAST_DIST_MAX);
 
 	/* return location and normal */
 	parm = RNA_def_boolean(func, "result", 0, "", "");
diff --git a/source/blender/makesrna/intern/rna_scene_api.c b/source/blender/makesrna/intern/rna_scene_api.c
index b31681f1b6e..6d7506f3911 100644
--- a/source/blender/makesrna/intern/rna_scene_api.c
+++ b/source/blender/makesrna/intern/rna_scene_api.c
@@ -33,6 +33,7 @@
 #include <stdio.h>
 
 #include "BLI_utildefines.h"
+#include "BLI_kdopbvh.h"
 #include "BLI_path_util.h"
 
 #include "RNA_define.h"
@@ -232,7 +233,8 @@ void RNA_api_scene(StructRNA *srna)
 	RNA_def_property_flag(parm, PROP_REQUIRED);
 	parm = RNA_def_float_vector(func, "direction", 3, NULL, -FLT_MAX, FLT_MAX, "", "", -1e4, 1e4);
 	RNA_def_property_flag(parm, PROP_REQUIRED);
-	RNA_def_float(func, "distance", FLT_MAX, 0.0, FLT_MAX, "", "Maximum distance", 0.0, FLT_MAX);
+	RNA_def_float(func, "distance", BVH_RAYCAST_DIST_MAX, 0.0, BVH_RAYCAST_DIST_MAX,
+	              "", "Maximum distance", 0.0, BVH_RAYCAST_DIST_MAX);
 
 	/* return location and normal */
 	parm = RNA_def_boolean(func, "result", 0, "", "");
diff --git a/source/blender/render/intern/source/bake_api.c b/source/blender/render/intern/source/bake_api.c
index 9a26be81ef3..16585be0ef9 100644
--- a/source/blender/render/intern/source/bake_api.c
+++ b/source/blender/render/intern/source/bake_api.c
@@ -299,7 +299,7 @@ static bool cast_ray_highpoly(
 
 		hits[i].index = -1;
 		/* TODO: we should use FLT_MAX here, but sweepsphere code isn't prepared for that */
-		hits[i].dist = 10000.0f;
+		hits[i].dist = BVH_RAYCAST_DIST_MAX;
 
 		/* transform the ray from the world space to the highpoly space */
 		mul_v3_m4v3(co_high, highpoly[i].imat, co);