Fix missing check if isect_plane_plane_v3 fails to find an intersection.

5 files changed, 39 insertions, 25 deletions

diff --git a/source/blender/blenkernel/BKE_camera.h b/source/blender/blenkernel/BKE_camera.h
index a07558cc8af..49ba9c9d875 100644
--- a/source/blender/blenkernel/BKE_camera.h
+++ b/source/blender/blenkernel/BKE_camera.h
@@ -117,8 +117,8 @@ void BKE_camera_view_frame_ex(struct Scene *scene, struct Camera *camera, float
 
 void BKE_camera_view_frame(struct Scene *scene, struct Camera *camera, float r_vec[4][3]);
 
-int BKE_camera_view_frame_fit_to_scene(struct Scene *scene, struct View3D *v3d, struct Object *camera_ob,
-                                       float r_co[3]);
+bool BKE_camera_view_frame_fit_to_scene(struct Scene *scene, struct View3D *v3d, struct Object *camera_ob,
+                                        float r_co[3]);
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/blenkernel/intern/camera.c b/source/blender/blenkernel/intern/camera.c
index aee8ede4062..9dbada4d58f 100644
--- a/source/blender/blenkernel/intern/camera.c
+++ b/source/blender/blenkernel/intern/camera.c
@@ -480,7 +480,7 @@ static void camera_to_frame_view_cb(const float co[3], void *user_data)
 
 /* don't move the camera, just yield the fit location */
 /* only valid for perspective cameras */
-int BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object *camera_ob, float r_co[3])
+bool BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object *camera_ob, float r_co[3])
 {
 	float shift[2];
 	float plane_tx[4][3];
@@ -527,7 +527,7 @@ int BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object
 	                                camera_to_frame_view_cb, &data_cb);
 
 	if (data_cb.tot <= 1) {
-		return FALSE;
+		return false;
 	}
 	else {
 		float plane_isect_1[3], plane_isect_1_no[3], plane_isect_1_other[3];
@@ -540,12 +540,15 @@ int BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object
 			mul_v3_v3fl(plane_tx[i], data_cb.normal_tx[i], sqrtf_signed(data_cb.dist_vals_sq[i]));
 		}
 
-		isect_plane_plane_v3(plane_isect_1, plane_isect_1_no,
-		                     plane_tx[0], data_cb.normal_tx[0],
-		                     plane_tx[2], data_cb.normal_tx[2]);
-		isect_plane_plane_v3(plane_isect_2, plane_isect_2_no,
-		                     plane_tx[1], data_cb.normal_tx[1],
-		                     plane_tx[3], data_cb.normal_tx[3]);
+		if ((!isect_plane_plane_v3(plane_isect_1, plane_isect_1_no,
+		                           plane_tx[0], data_cb.normal_tx[0],
+		                           plane_tx[2], data_cb.normal_tx[2])) ||
+		    (!isect_plane_plane_v3(plane_isect_2, plane_isect_2_no,
+		                           plane_tx[1], data_cb.normal_tx[1],
+		                           plane_tx[3], data_cb.normal_tx[3])))
+		{
+			return false;
+		}
 
 		add_v3_v3v3(plane_isect_1_other, plane_isect_1, plane_isect_1_no);
 		add_v3_v3v3(plane_isect_2_other, plane_isect_2, plane_isect_2_no);
@@ -554,7 +557,7 @@ int BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object
 		                       plane_isect_2, plane_isect_2_other,
 		                       plane_isect_pt_1, plane_isect_pt_2) == 0)
 		{
-			return FALSE;
+			return false;
 		}
 		else {
 			float cam_plane_no[3] = {0.0f, 0.0f, -1.0f};
@@ -582,7 +585,7 @@ int BKE_camera_view_frame_fit_to_scene(Scene *scene, struct View3D *v3d, Object
 			}
 
 
-			return TRUE;
+			return true;
 		}
 	}
 }
diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index e99f89036b3..e800369a682 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -140,11 +140,11 @@ bool isect_ray_plane_v3(const float p1[3], const float d[3],
 
 bool isect_point_planes_v3(float (*planes)[4], int totplane, const float p[3]);
 bool isect_line_plane_v3(float out[3], const float l1[3], const float l2[3],
-                         const float plane_co[3], const float plane_no[3]);
+                         const float plane_co[3], const float plane_no[3]) ATTR_WARN_UNUSED_RESULT;
 
-void isect_plane_plane_v3(float r_isect_co[3], float r_isect_no[3],
+bool isect_plane_plane_v3(float r_isect_co[3], float r_isect_no[3],
                           const float plane_a_co[3], const float plane_a_no[3],
-                          const float plane_b_co[3], const float plane_b_no[3]);
+                          const float plane_b_co[3], const float plane_b_no[3]) ATTR_WARN_UNUSED_RESULT;
 
 /* line/ray triangle */
 bool isect_line_tri_v3(const float p1[3], const float p2[3],
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index 833a08cec60..bf5ae8fab4f 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -1185,7 +1185,7 @@ bool isect_line_plane_v3(float out[3],
  *
  * \note return normal isn't unit length
  */
-void isect_plane_plane_v3(float r_isect_co[3], float r_isect_no[3],
+bool isect_plane_plane_v3(float r_isect_co[3], float r_isect_no[3],
                           const float plane_a_co[3], const float plane_a_no[3],
                           const float plane_b_co[3], const float plane_b_no[3])
 {
@@ -1193,7 +1193,7 @@ void isect_plane_plane_v3(float r_isect_co[3], float r_isect_no[3],
 	cross_v3_v3v3(r_isect_no, plane_a_no, plane_b_no); /* direction is simply the cross product */
 	cross_v3_v3v3(plane_a_co_other, plane_a_no, r_isect_no);
 	add_v3_v3(plane_a_co_other, plane_a_co);
-	isect_line_plane_v3(r_isect_co, plane_a_co, plane_a_co_other, plane_b_co, plane_b_no);
+	return isect_line_plane_v3(r_isect_co, plane_a_co, plane_a_co_other, plane_b_co, plane_b_no);
 }
 
 
diff --git a/source/blender/python/mathutils/mathutils_geometry.c b/source/blender/python/mathutils/mathutils_geometry.c
index 53e428cf73c..29e7779b89a 100644
--- a/source/blender/python/mathutils/mathutils_geometry.c
+++ b/source/blender/python/mathutils/mathutils_geometry.c
@@ -626,11 +626,11 @@ PyDoc_STRVAR(M_Geometry_intersect_plane_plane_doc,
 "   :arg plane_b_no: Normal of the second plane\n"
 "   :type plane_b_no: :class:`mathutils.Vector`\n"
 "   :return: The line of the intersection represented as a point and a vector\n"
-"   :rtype: tuple pair of :class:`mathutils.Vector`\n"
+"   :rtype: tuple pair of :class:`mathutils.Vector` or None if the intersection can't be calculated\n"
 );
 static PyObject *M_Geometry_intersect_plane_plane(PyObject *UNUSED(self), PyObject *args)
 {
-	PyObject *ret;
+	PyObject *ret, *ret_co, *ret_no;
 	VectorObject *plane_a_co, *plane_a_no, *plane_b_co, *plane_b_no;
 
 	float isect_co[3];
@@ -660,15 +660,26 @@ static PyObject *M_Geometry_intersect_plane_plane(PyObject *UNUSED(self), PyObje
 		return NULL;
 	}
 
-	isect_plane_plane_v3(isect_co, isect_no,
-	                     plane_a_co->vec, plane_a_no->vec,
-	                     plane_b_co->vec, plane_b_no->vec);
+	if (isect_plane_plane_v3(isect_co, isect_no,
+	                         plane_a_co->vec, plane_a_no->vec,
+	                         plane_b_co->vec, plane_b_no->vec))
+	{
+		normalize_v3(isect_no);
+
+		ret_co = Vector_CreatePyObject(isect_co, 3, Py_NEW, NULL);
+		ret_no = Vector_CreatePyObject(isect_no, 3, Py_NEW, NULL);
+	}
+	else {
+		ret_co = Py_None;
+		ret_no = Py_None;
 
-	normalize_v3(isect_no);
+		Py_INCREF(ret_co);
+		Py_INCREF(ret_no);
+	}
 
 	ret = PyTuple_New(2);
-	PyTuple_SET_ITEM(ret, 0, Vector_CreatePyObject(isect_co, 3, Py_NEW, NULL));
-	PyTuple_SET_ITEM(ret, 1, Vector_CreatePyObject(isect_no, 3, Py_NEW, NULL));
+	PyTuple_SET_ITEM(ret, 0, ret_co);
+	PyTuple_SET_ITEM(ret, 1, ret_no);
 	return ret;
 }