fix issue with shrinkwrap face projection distance comparisons when using both positive and negative projection.

- don't attempt to convert the 'dist' value between local/target space, since all the projections are done in target space
  and dist isnt used afterwards. Also, this could fail with non uniform scale - overwriting ray casts with larger dist values.
- added an assert to check larger dist values never overwrite smaller ones.
- remove use of sasqrt() since the value is checked beforehand anyway.

1 files changed, 23 insertions, 3 deletions

diff --git a/source/blender/blenkernel/intern/shrinkwrap.c b/source/blender/blenkernel/intern/shrinkwrap.c
index 12472afe339..96faec389df 100644
--- a/source/blender/blenkernel/intern/shrinkwrap.c
+++ b/source/blender/blenkernel/intern/shrinkwrap.c
@@ -159,7 +159,9 @@ static void shrinkwrap_calc_nearest_vertex(ShrinkwrapCalcData *calc)
 		float *co = calc->vertexCos[i];
 		float tmp_co[3];
 		float weight = defvert_array_find_weight_safe(calc->dvert, i, calc->vgroup);
-		if (weight == 0.0f) continue;
+		if (weight == 0.0f) {
+			continue;
+		}
 
 
 		/* Convert the vertex to tree coordinates */
@@ -188,8 +190,10 @@ static void shrinkwrap_calc_nearest_vertex(ShrinkwrapCalcData *calc)
 		if (nearest.index != -1) {
 			/* Adjusting the vertex weight,
 			 * so that after interpolating it keeps a certain distance from the nearest position */
-			float dist = sasqrt(nearest.dist);
-			if (dist > FLT_EPSILON) weight *= (dist - calc->keepDist) / dist;
+			if (nearest.dist > FLT_EPSILON) {
+				const float dist = sqrtf(nearest.dist);
+				weight *= (dist - calc->keepDist) / dist;
+			}
 
 			/* Convert the coordinates back to mesh coordinates */
 			copy_v3_v3(tmp_co, nearest.co);
@@ -202,6 +206,7 @@ static void shrinkwrap_calc_nearest_vertex(ShrinkwrapCalcData *calc)
 	free_bvhtree_from_mesh(&treeData);
 }
 
+
 /*
  * This function raycast a single vertex and updates the hit if the "hit" is considered valid.
  * Returns TRUE if "hit" was updated.
@@ -215,6 +220,11 @@ int normal_projection_project_vertex(char options, const float vert[3], const fl
                                      BVHTree *tree, BVHTreeRayHit *hit,
                                      BVHTree_RayCastCallback callback, void *userdata)
 {
+	/* don't use this because this dist value could be incompatible
+	 * this value used by the callback for comparing prev/new dist values.
+	 * also, at the moment there is no need to have a corrected 'dist' value */
+// #define USE_DIST_CORRECT
+
 	float tmp_co[3], tmp_no[3];
 	const float *co, *no;
 	BVHTreeRayHit hit_tmp;
@@ -232,7 +242,9 @@ int normal_projection_project_vertex(char options, const float vert[3], const fl
 		space_transform_apply_normal(transf, tmp_no);
 		no = tmp_no;
 
+#ifdef USE_DIST_CORRECT
 		hit_tmp.dist *= mat4_to_scale(((SpaceTransform *)transf)->local2target);
+#endif
 	}
 	else {
 		co = vert;
@@ -262,9 +274,13 @@ int normal_projection_project_vertex(char options, const float vert[3], const fl
 		if (transf) {
 			/* Inverting space transform (TODO make coeherent with the initial dist readjust) */
 			space_transform_invert(transf, hit_tmp.co);
+#ifdef USE_DIST_CORRECT
 			hit_tmp.dist = len_v3v3(vert, hit_tmp.co);
+#endif
 		}
 
+		BLI_assert(hit_tmp.dist <= hit->dist);
+
 		memcpy(hit, &hit_tmp, sizeof(hit_tmp));
 		return TRUE;
 	}
@@ -281,6 +297,10 @@ static void shrinkwrap_calc_normal_projection(ShrinkwrapCalcData *calc)
 	float proj_axis[3]      = {0.0f, 0.0f, 0.0f};
 
 	/* Raycast and tree stuff */
+
+	/** \note 'hit.dist' is kept in the targets space, this is only used
+	 * for finding the best hit, to get the real dist,
+	 * measure the len_v3v3() from the input coord to hit.co */
 	BVHTreeRayHit hit;
 	BVHTreeFromMesh treeData = NULL_BVHTreeFromMesh;