BGE: Navmesh fixes and improvements

The navigation mesh functionality was broken for quite a while. This patch
contains fixes: recalculating tesselations before getting the number of
tesselation faces (it otherwise returned 0) before calculating the navmesh,
and calling `DM_ensure_tessface()` on the navmesh's `DerivedMesh` object
(which fixes visualisation in Blender). This allows one to create a new
navmesh, which also works in the BGE.

Furthermore, the patch adds several return values, and shows more error
messages when things go wrong. In several places in the navmesh creation
code, return codes weren't checked and errors silently ignored.

Reviewers: nicks, brita_, campbellbarton, lordloki, moguri, panzergame

Reviewed By: panzergame

Differential Revision: https://developer.blender.org/D1435

3 files changed, 34 insertions, 7 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index b06cd72a5a3..0b1b4310ca4 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -2116,6 +2116,8 @@ static void mesh_calc_modifiers(
 			finaldm->release(finaldm);
 			finaldm = tdm;
 		}
+
+		DM_ensure_tessface(finaldm);
 	}
 #endif /* WITH_GAMEENGINE */
 
diff --git a/source/blender/blenkernel/intern/navmesh_conversion.c b/source/blender/blenkernel/intern/navmesh_conversion.c
index 6c3f4d545d3..35bcca52f63 100644
--- a/source/blender/blenkernel/intern/navmesh_conversion.c
+++ b/source/blender/blenkernel/intern/navmesh_conversion.c
@@ -123,6 +123,11 @@ int buildRawVertIndicesData(DerivedMesh *dm, int *nverts_r, float **verts_r,
 		printf("Converting navmesh: Error! Too many vertices. Max number of vertices %d\n", 0xffff);
 		return 0;
 	}
+	if (nverts == 0) {
+		printf("Converting navmesh: Error! There are no vertices!\n");
+		return 0;
+	}
+
 	verts = MEM_mallocN(sizeof(float[3]) * nverts, "buildRawVertIndicesData verts");
 	dm->getVertCos(dm, (float(*)[3])verts);
 
@@ -132,7 +137,13 @@ int buildRawVertIndicesData(DerivedMesh *dm, int *nverts_r, float **verts_r,
 	}
 
 	/* calculate number of tris */
+	dm->recalcTessellation(dm);
 	nfaces = dm->getNumTessFaces(dm);
+	if (nfaces == 0) {
+		printf("Converting navmesh: Error! There are %i vertices, but no faces!\n", nverts);
+		return 0;
+	}
+
 	faces = dm->getTessFaceArray(dm);
 	ntris = nfaces;
 	for (fi = 0; fi < nfaces; fi++) {
@@ -387,7 +398,12 @@ int buildNavMeshData(const int nverts, const float *verts,
 
 
 	/* build adjacency info for detailed mesh triangles */
-	recast_buildMeshAdjacency(dtris, ndtris, nverts, 3);
+	if (!recast_buildMeshAdjacency(dtris, ndtris, nverts, 3)) {
+		printf("Converting navmesh: Error! Unable to build mesh adjacency information\n");
+		MEM_freeN(trisMapping);
+		MEM_freeN(dtrisToPolysMap);
+		return 0;
+	}
 
 	/* create detailed mesh description for each navigation polygon */
 	npolys = dtrisToPolysMap[ndtris - 1];
@@ -415,7 +431,10 @@ int buildNavMeshData(const int nverts, const float *verts,
 	polys = MEM_callocN(sizeof(unsigned short) * npolys * vertsPerPoly * 2, "buildNavMeshData polys");
 	memset(polys, 0xff, sizeof(unsigned short) * vertsPerPoly * 2 * npolys);
 
-	buildPolygonsByDetailedMeshes(vertsPerPoly, npolys, polys, dmeshes, verts, dtris, dtrisToPolysMap);
+	if (!buildPolygonsByDetailedMeshes(vertsPerPoly, npolys, polys, dmeshes, verts, dtris, dtrisToPolysMap)) {
+		printf("Converting navmesh: Error! Unable to build polygons from detailed mesh\n");
+		goto fail;
+	}
 
 	*ndtris_r = ndtris;
 	*npolys_r = npolys;
@@ -449,7 +468,7 @@ int buildNavMeshDataByDerivedMesh(DerivedMesh *dm, int *vertsPerPoly,
 
 	res = buildRawVertIndicesData(dm, nverts, verts, &ntris, &tris, trisToFacesMap, &recastData);
 	if (!res) {
-		printf("Converting navmesh: Error! Can't get vertices and indices from mesh\n");
+		printf("Converting navmesh: Error! Can't get raw vertices and indices from mesh\n");
 		goto exit;
 	}
 
@@ -457,7 +476,7 @@ int buildNavMeshDataByDerivedMesh(DerivedMesh *dm, int *vertsPerPoly,
 	                       ndtris, dtris, npolys, dmeshes, polys, vertsPerPoly,
 	                       dtrisToPolysMap, dtrisToTrisMap);
 	if (!res) {
-		printf("Converting navmesh: Error! Can't get vertices and indices from mesh\n");
+		printf("Converting navmesh: Error! Can't build navmesh data from mesh\n");
 		goto exit;
 	}
 
diff --git a/source/gameengine/Ketsji/KX_NavMeshObject.cpp b/source/gameengine/Ketsji/KX_NavMeshObject.cpp
index 891eb00cae6..b8907ca5c6b 100644
--- a/source/gameengine/Ketsji/KX_NavMeshObject.cpp
+++ b/source/gameengine/Ketsji/KX_NavMeshObject.cpp
@@ -98,12 +98,14 @@ void KX_NavMeshObject::ProcessReplica()
 {
 	KX_GameObject::ProcessReplica();
 	m_navMesh = NULL;  /* without this, building frees the navmesh we copied from */
-	BuildNavMesh();
+	if (!BuildNavMesh()) {
+		std::cout << "Error in " << __func__ << ": unable to build navigation mesh" << std::endl;
+		return;
+	}
 	KX_Scene* scene = KX_GetActiveScene();
 	KX_ObstacleSimulation* obssimulation = scene->GetObstacleSimulation();
 	if (obssimulation)
 		obssimulation->AddObstaclesForNavMesh(this);
-
 }
 
 bool KX_NavMeshObject::BuildVertIndArrays(float *&vertices, int& nverts,
@@ -321,7 +323,11 @@ bool KX_NavMeshObject::BuildNavMesh()
 		}
 	}
 
-	buildMeshAdjacency(polys, npolys, nverts, vertsPerPoly);
+	if (!buildMeshAdjacency(polys, npolys, nverts, vertsPerPoly)) {
+		std::cout << __func__ << ": unable to build mesh adjacency information." << std::endl;
+		delete[] vertices;
+		return false;
+	}
 	
 	float cs = 0.2f;