fix [#33758] Blender crashes when user goes to choose "build navigation mesh"

Give useful reports when the 'MESH_OT_navmesh_make' fails too.

1 files changed, 31 insertions, 15 deletions

diff --git a/source/blender/editors/mesh/mesh_navmesh.c b/source/blender/editors/mesh/mesh_navmesh.c
index 1a81cab8de7..a7f77d0d483 100644
--- a/source/blender/editors/mesh/mesh_navmesh.c
+++ b/source/blender/editors/mesh/mesh_navmesh.c
@@ -157,8 +157,9 @@ static void createVertsTrisData(bContext *C, LinkNode *obs, int *nverts_r, float
 	*tris_r = tris;
 }
 
-static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
-                        struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh)
+static bool buildNavMesh(const RecastData *recastParams, int nverts, float *verts, int ntris, int *tris,
+                         struct recast_polyMesh **pmesh, struct recast_polyMeshDetail **dmesh,
+                         ReportList *reports)
 {
 	float bmin[3], bmax[3];
 	struct recast_heightfield *solid;
@@ -185,14 +186,20 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 	/* Set the area where the navigation will be build. */
 	recast_calcGridSize(bmin, bmax, recastParams->cellsize, &width, &height);
 
+	/* zero dimensions cause zero alloc later on [#33758] */
+	if (width <= 0 || height <= 0) {
+		BKE_report(reports, RPT_ERROR, "Object has no volume");
+		return false;
+	}
+
 	/* ** Step 2: Rasterize input polygon soup ** */
 	/* Allocate voxel heightfield where we rasterize our input data to */
 	solid = recast_newHeightfield();
 
 	if (!recast_createHeightfield(solid, width, height, bmin, bmax, recastParams->cellsize, recastParams->cellheight)) {
 		recast_destroyHeightfield(solid);
-
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to create hight field");
+		return false;
 	}
 
 	/* Allocate array that can hold triangle flags */
@@ -215,7 +222,8 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 		recast_destroyHeightfield(solid);
 		recast_destroyCompactHeightfield(chf);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to create compact hight field");
+		return false;
 	}
 
 	recast_destroyHeightfield(solid);
@@ -224,21 +232,24 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 	if (!recast_erodeWalkableArea(walkableRadius, chf)) {
 		recast_destroyCompactHeightfield(chf);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to erode walkable area");
+		return false;
 	}
 
 	/* Prepare for region partitioning, by calculating distance field along the walkable surface */
 	if (!recast_buildDistanceField(chf)) {
 		recast_destroyCompactHeightfield(chf);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to build distance field");
+		return false;
 	}
 
 	/* Partition the walkable surface into simple regions without holes */
 	if (!recast_buildRegions(chf, 0, minRegionArea, mergeRegionArea)) {
 		recast_destroyCompactHeightfield(chf);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to build regions");
+		return false;
 	}
 
 	/* ** Step 5: Trace and simplify region contours ** */
@@ -249,7 +260,8 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 		recast_destroyCompactHeightfield(chf);
 		recast_destroyContourSet(cset);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to build contours");
+		return false;
 	}
 
 	/* ** Step 6: Build polygons mesh from contours ** */
@@ -259,7 +271,8 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 		recast_destroyContourSet(cset);
 		recast_destroyPolyMesh(*pmesh);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to build poly mesh");
+		return false;
 	}
 
 
@@ -272,13 +285,14 @@ static int buildNavMesh(const RecastData *recastParams, int nverts, float *verts
 		recast_destroyPolyMesh(*pmesh);
 		recast_destroyPolyMeshDetail(*dmesh);
 
-		return 0;
+		BKE_report(reports, RPT_ERROR, "Faled to build poly mesh detail");
+		return false;
 	}
 
 	recast_destroyCompactHeightfield(chf);
 	recast_destroyContourSet(cset);
 
-	return 1;
+	return true;
 }
 
 static Object *createRepresentation(bContext *C, struct recast_polyMesh *pmesh, struct recast_polyMeshDetail *dmesh, Base *base)
@@ -437,6 +451,7 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
 	if (obs) {
 		struct recast_polyMesh *pmesh = NULL;
 		struct recast_polyMeshDetail *dmesh = NULL;
+		bool ok;
 
 		int nverts = 0, ntris = 0;
 		int *tris = 0;
@@ -444,13 +459,14 @@ static int navmesh_create_exec(bContext *C, wmOperator *op)
 
 		createVertsTrisData(C, obs, &nverts, &verts, &ntris, &tris);
 		BLI_linklist_free(obs, NULL);
-		buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh);
-		createRepresentation(C, pmesh, dmesh, navmeshBase);
+		if ((ok = buildNavMesh(&scene->gm.recastData, nverts, verts, ntris, tris, &pmesh, &dmesh, op->reports))) {
+			createRepresentation(C, pmesh, dmesh, navmeshBase);
+		}
 
 		MEM_freeN(verts);
 		MEM_freeN(tris);
 
-		return OPERATOR_FINISHED;
+		return ok ? OPERATOR_FINISHED : OPERATOR_CANCELLED;
 	}
 	else {
 		BKE_report(op->reports, RPT_ERROR, "No mesh objects found");