Fix potential bad behavior, and cleanup/refactor a bit BKE_mesh_ensure_normals_for_display().

This is merely making behaviors of this function a bit more explicit,
and avoid re-adding another CD_NORMAL layer to polys in the (unlikely)
case it would already have one.

It also handles CD_MASK_NORMAL in cd_dirty_poly, but this is more like
future-proof thing, this is not used anywhere currently afaik.

1 files changed, 14 insertions, 13 deletions

diff --git a/source/blender/blenkernel/intern/mesh_evaluate.c b/source/blender/blenkernel/intern/mesh_evaluate.c
index a5338b3a6be..1e4c9a6950a 100644
--- a/source/blender/blenkernel/intern/mesh_evaluate.c
+++ b/source/blender/blenkernel/intern/mesh_evaluate.c
@@ -355,27 +355,28 @@ void BKE_mesh_ensure_normals(Mesh *mesh)
  */
 void BKE_mesh_ensure_normals_for_display(Mesh *mesh)
 {
-	/* Note: mesh *may* have a poly CD_NORMAL layer (generated by a modifier needing poly normals e.g.).
-	 *       We do not use it here, though. And it should be tagged as temp!
-	 */
-	/* BLI_assert((CustomData_has_layer(&mesh->pdata, CD_NORMAL) == false)); */
-
-	if (mesh->runtime.cd_dirty_vert & CD_MASK_NORMAL || !CustomData_has_layer(&mesh->pdata, CD_NORMAL)) {
-		float (*poly_nors)[3] = NULL;
-		poly_nors = MEM_malloc_arrayN((size_t)mesh->totpoly, sizeof(*poly_nors), __func__);
+	float (*poly_nors)[3] = CustomData_get_layer(&mesh->pdata, CD_NORMAL);
+	const bool do_vert_normals = (mesh->runtime.cd_dirty_vert & CD_MASK_NORMAL) != 0;
+	const bool do_poly_normals = (mesh->runtime.cd_dirty_poly & CD_MASK_NORMAL || poly_nors == NULL);
 
-		/* if normals are dirty we want to calculate vertex normals too */
-		bool only_face_normals = !(mesh->runtime.cd_dirty_vert & CD_MASK_NORMAL);
+	if (do_vert_normals || do_poly_normals) {
+		const bool do_add_poly_nors_cddata = (poly_nors == NULL);
+		if (do_add_poly_nors_cddata) {
+			poly_nors = MEM_malloc_arrayN((size_t)mesh->totpoly, sizeof(*poly_nors), __func__);
+		}
 
-		/* calculate face normals */
+		/* calculate poly/vert normals */
 		BKE_mesh_calc_normals_poly(
 		        mesh->mvert, NULL, mesh->totvert, mesh->mloop, mesh->mpoly,
 		        mesh->totloop, mesh->totpoly, poly_nors,
-		        only_face_normals);
+		        !do_vert_normals);
 
-		CustomData_add_layer(&mesh->pdata, CD_NORMAL, CD_ASSIGN, poly_nors, mesh->totpoly);
+		if (do_add_poly_nors_cddata) {
+			CustomData_add_layer(&mesh->pdata, CD_NORMAL, CD_ASSIGN, poly_nors, mesh->totpoly);
+		}
 
 		mesh->runtime.cd_dirty_vert &= ~CD_MASK_NORMAL;
+		mesh->runtime.cd_dirty_poly &= ~CD_MASK_NORMAL;
 	}
 }