fix crash for recent navmesh edits when setting a non-mesh object to a navmesh.

also minor cleanup.

4 files changed, 42 insertions, 33 deletions

diff --git a/release/scripts/startup/bl_ui/properties_game.py b/release/scripts/startup/bl_ui/properties_game.py
index 6925836c5a2..55ab3313579 100644
--- a/release/scripts/startup/bl_ui/properties_game.py
+++ b/release/scripts/startup/bl_ui/properties_game.py
@@ -47,8 +47,9 @@ class PHYSICS_PT_game_physics(PhysicsButtonsPanel, Panel):
         layout.prop(game, "physics_type")
         layout.separator()
 
-        #if game.physics_type == 'DYNAMIC':
-        if game.physics_type in {'DYNAMIC', 'RIGID_BODY'}:
+        physics_type = game.physics_type
+
+        if physics_type in {'DYNAMIC', 'RIGID_BODY'}:
             split = layout.split()
 
             col = split.column()
@@ -108,7 +109,7 @@ class PHYSICS_PT_game_physics(PhysicsButtonsPanel, Panel):
             col.prop(game, "lock_rotation_y", text="Y")
             col.prop(game, "lock_rotation_z", text="Z")
 
-        elif game.physics_type == 'SOFT_BODY':
+        elif physics_type == 'SOFT_BODY':
             col = layout.column()
             col.prop(game, "use_actor")
             col.prop(game, "use_ghost")
@@ -143,7 +144,7 @@ class PHYSICS_PT_game_physics(PhysicsButtonsPanel, Panel):
             sub.active = (soft.use_cluster_rigid_to_softbody or soft.use_cluster_soft_to_softbody)
             sub.prop(soft, "cluster_iterations", text="Iterations")
 
-        elif game.physics_type == 'STATIC':
+        elif physics_type == 'STATIC':
             col = layout.column()
             col.prop(game, "use_actor")
             col.prop(game, "use_ghost")
@@ -164,9 +165,10 @@ class PHYSICS_PT_game_physics(PhysicsButtonsPanel, Panel):
             subsub.active = game.use_anisotropic_friction
             subsub.prop(game, "friction_coefficients", text="", slider=True)
 
-        elif game.physics_type in {'SENSOR', 'INVISIBLE', 'NO_COLLISION', 'OCCLUDE'}:
+        elif physics_type in {'SENSOR', 'INVISIBLE', 'NO_COLLISION', 'OCCLUDE'}:
             layout.prop(ob, "hide_render", text="Invisible")
-        elif game.physics_type == 'NAVMESH':
+
+        elif physics_type == 'NAVMESH':
             layout.operator("mesh.assign_navpolygon")
             layout.operator("mesh.assign_new_navpolygon")
 
diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt
index 856810b048a..bfadff06400 100644
--- a/source/blender/blenkernel/CMakeLists.txt
+++ b/source/blender/blenkernel/CMakeLists.txt
@@ -353,7 +353,7 @@ endif()
 
 if(WITH_GAMEENGINE)
 	list(APPEND INC_SYS
-	../../../extern/recastnavigation
+		../../../extern/recastnavigation
 	)
 	list(APPEND SRC
 		intern/navmesh_conversion.c
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 9c90ec23d3c..6b9a65ddfa2 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -2113,7 +2113,7 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 
 #ifdef WITH_GAMEENGINE
 	/* NavMesh - this is a hack but saves having a NavMesh modifier */
-	if (ob->body_type == OB_BODY_TYPE_NAVMESH && finaldm->type == DM_TYPE_CDDM) {
+	if ((ob->gameflag & OB_NAVMESH) && (finaldm->type == DM_TYPE_CDDM)) {
 		DerivedMesh *tdm;
 		tdm= navmesh_dm_createNavMeshForVisualization(finaldm);
 		if (finaldm != tdm) {
@@ -2966,7 +2966,7 @@ BM_INLINE int navmesh_bit(int a, int b)
 	return (a & (1 << b)) >> b;
 }
 
-BM_INLINE void navmesh_intToCol(int i, float* col)
+static void navmesh_intToCol(int i, float* col)
 {
 	int	r = navmesh_bit(i, 0) + navmesh_bit(i, 3) * 2 + 1;
 	int	g = navmesh_bit(i, 1) + navmesh_bit(i, 4) * 2 + 1;
@@ -3034,8 +3034,8 @@ static void navmesh_DM_drawFacesTex(DerivedMesh *dm, int (*setDrawOptions)(MTFac
 }
 
 static void navmesh_DM_drawFacesSolid(DerivedMesh *dm,
-								float (*partial_redraw_planes)[4],
-								int UNUSED(fast), int (*setMaterial)(int, void *attribs))
+                                      float (*partial_redraw_planes)[4],
+                                      int UNUSED(fast), int (*setMaterial)(int, void *attribs))
 {
 	(void) partial_redraw_planes;
 	(void) setMaterial;
@@ -3056,54 +3056,50 @@ static DerivedMesh *navmesh_dm_createNavMeshForVisualization(DerivedMesh *dm)
 	int res;
 
 	result = CDDM_copy(dm);
-	if (!CustomData_has_layer(&result->faceData, CD_RECAST))
-	{
+	if (!CustomData_has_layer(&result->faceData, CD_RECAST)) {
 		int *sourceRecastData = (int*)CustomData_get_layer(&dm->faceData, CD_RECAST);
 		CustomData_add_layer_named(&result->faceData, CD_RECAST, CD_DUPLICATE,
 			sourceRecastData, maxFaces, "recastData");
 	}
 	recastData = (int*)CustomData_get_layer(&result->faceData, CD_RECAST);
+
+	/* note: This is not good design! - really should not be doing this */
 	result->drawFacesTex =  navmesh_DM_drawFacesTex;
 	result->drawFacesSolid = navmesh_DM_drawFacesSolid;
 
 
-	//process mesh
+	/* process mesh */
 	res  = buildNavMeshDataByDerivedMesh(dm, &vertsPerPoly, &nverts, &verts, &ndtris, &dtris,
-										&npolys, &dmeshes, &polys, &dtrisToPolysMap, &dtrisToTrisMap,
-										&trisToFacesMap);
-	if (res)
-	{
+	                                     &npolys, &dmeshes, &polys, &dtrisToPolysMap, &dtrisToTrisMap,
+	                                     &trisToFacesMap);
+	if (res) {
 		size_t polyIdx;
 
-		//invalidate concave polygon
-		for (polyIdx=0; polyIdx<(size_t)npolys; polyIdx++)
-		{
+		/* invalidate concave polygon */
+		for (polyIdx=0; polyIdx<(size_t)npolys; polyIdx++) {
 			unsigned short* poly = &polys[polyIdx*2*vertsPerPoly];
-			if (!polyIsConvex(poly, vertsPerPoly, verts))
-			{
-				//set negative polygon idx to all faces
+			if (!polyIsConvex(poly, vertsPerPoly, verts)) {
+				/* set negative polygon idx to all faces */
 				unsigned short *dmesh = &dmeshes[4*polyIdx];
 				unsigned short tbase = dmesh[2];
 				unsigned short tnum = dmesh[3];
 				unsigned short ti;
 
-				for (ti=0; ti<tnum; ti++)
-				{
+				for (ti=0; ti<tnum; ti++) {
 					unsigned short triidx = dtrisToTrisMap[tbase+ti];
 					unsigned short faceidx = trisToFacesMap[triidx];
-					if (recastData[faceidx]>0)
+					if (recastData[faceidx] > 0) {
 						recastData[faceidx] = -recastData[faceidx];
+					}
 				}
 			}
 		}
-
 	}
-	else
-	{
+	else {
 		printf("Error during creation polygon infos\n");
 	}
 
-	//clean up
+	/* clean up */
 	if (verts!=NULL)
 		MEM_freeN(verts);
 	if (dtris!=NULL)
diff --git a/source/blender/makesrna/intern/rna_object.c b/source/blender/makesrna/intern/rna_object.c
index 25d5baf74b0..96ffa6b2ed4 100644
--- a/source/blender/makesrna/intern/rna_object.c
+++ b/source/blender/makesrna/intern/rna_object.c
@@ -890,6 +890,7 @@ static int rna_GameObjectSettings_physics_type_get(PointerRNA *ptr)
 static void rna_GameObjectSettings_physics_type_set(PointerRNA *ptr, int value)
 {
 	Object *ob= (Object*)ptr->id.data;
+	const int was_navmesh= (ob->gameflag & OB_NAVMESH);
 	ob->body_type= value;
 
 	switch (ob->body_type) {
@@ -906,8 +907,10 @@ static void rna_GameObjectSettings_physics_type_set(PointerRNA *ptr, int value)
 		ob->gameflag |= OB_NAVMESH;
 		ob->gameflag &= ~(OB_SENSOR|OB_COLLISION|OB_DYNAMIC|OB_OCCLUDER);
 
-		/* could be moved into mesh UI but for now ensure mesh data layer */
-		BKE_mesh_ensure_navmesh(ob->data);
+		if (ob->type == OB_MESH) {
+			/* could be moved into mesh UI but for now ensure mesh data layer */
+			BKE_mesh_ensure_navmesh(ob->data);
+		}
 
 		break;
 	case OB_BODY_TYPE_NO_COLLISION:
@@ -940,6 +943,14 @@ static void rna_GameObjectSettings_physics_type_set(PointerRNA *ptr, int value)
 			ob->bsoft = bsbNew();
 		break;
 	}
+
+	if (was_navmesh != (ob->gameflag & OB_NAVMESH)) {
+		if (ob->type == OB_MESH) {
+			DAG_id_tag_update(ptr->id.data, OB_RECALC_DATA);
+			WM_main_add_notifier(NC_OBJECT|ND_DRAW, ptr->id.data);
+		}
+	}
+
 }
 
 static PointerRNA rna_Object_active_particle_system_get(PointerRNA *ptr)