'Fix' T61942: Triangulate modifier breaks custom normals.

Not a bug, but supporting preservation of custom normals in that
specific modifier makes sense, in game pipeline contexts.

Could also ease work of IO add-ons that want to export
triangulated geometry...

4 files changed, 61 insertions, 7 deletions

diff --git a/release/scripts/startup/bl_ui/properties_data_modifier.py b/release/scripts/startup/bl_ui/properties_data_modifier.py
index 36deb55978f..d08cb0ac41f 100644
--- a/release/scripts/startup/bl_ui/properties_data_modifier.py
+++ b/release/scripts/startup/bl_ui/properties_data_modifier.py
@@ -1366,6 +1366,7 @@ class DATA_PT_modifiers(ModifierButtonsPanel, Panel):
         col = row.column()
         col.label(text="Quad Method:")
         col.prop(md, "quad_method", text="")
+        col.prop(md, "keep_custom_normals")
         col = row.column()
         col.label(text="Ngon Method:")
         col.prop(md, "ngon_method", text="")
diff --git a/source/blender/makesdna/DNA_modifier_types.h b/source/blender/makesdna/DNA_modifier_types.h
index 4139b5eb91c..5aec9b95797 100644
--- a/source/blender/makesdna/DNA_modifier_types.h
+++ b/source/blender/makesdna/DNA_modifier_types.h
@@ -1552,11 +1552,13 @@ typedef struct TriangulateModifierData {
 	char _pad[4];
 } TriangulateModifierData;
 
-#ifdef DNA_DEPRECATED
+/* TriangulateModifierData.flag */
 enum {
-	MOD_TRIANGULATE_BEAUTY = (1 << 0), /* deprecated */
-};
+#ifdef DNA_DEPRECATED
+	MOD_TRIANGULATE_BEAUTY = (1 << 0),  /* deprecated */
 #endif
+	MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS = 1 << 1,
+};
 
 /* Triangulate methods - NGons */
 enum {
diff --git a/source/blender/makesrna/intern/rna_modifier.c b/source/blender/makesrna/intern/rna_modifier.c
index ae22fbd5e54..31c78533151 100644
--- a/source/blender/makesrna/intern/rna_modifier.c
+++ b/source/blender/makesrna/intern/rna_modifier.c
@@ -4353,6 +4353,13 @@ static void rna_def_modifier_triangulate(BlenderRNA *brna)
 	RNA_def_property_enum_items(prop, rna_enum_modifier_triangulate_ngon_method_items);
 	RNA_def_property_ui_text(prop, "Polygon Method", "Method for splitting the polygons into triangles");
 	RNA_def_property_update(prop, 0, "rna_Modifier_update");
+
+	prop = RNA_def_property(srna, "keep_custom_normals", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS);
+	RNA_def_property_ui_text(prop, "Keep Normals",
+	                         "Try to preserve custom normals (WARNING: depending on chosen triangulation method, "
+	                         "shading may not be fully preserved, 'Fixed' method usually gives the best result here)");
+	RNA_def_property_update(prop, 0, "rna_Modifier_update");
 }
 
 static void rna_def_modifier_meshcache(BlenderRNA *brna)
diff --git a/source/blender/modifiers/intern/MOD_triangulate.c b/source/blender/modifiers/intern/MOD_triangulate.c
index 2baaff9a676..4f6276492ea 100644
--- a/source/blender/modifiers/intern/MOD_triangulate.c
+++ b/source/blender/modifiers/intern/MOD_triangulate.c
@@ -18,6 +18,8 @@
  * \ingroup modifiers
  */
 
+#include "MEM_guardedalloc.h"
+
 #include "BLI_utildefines.h"
 
 #include "DNA_mesh_types.h"
@@ -32,26 +34,68 @@
 
 #include "MOD_modifiertypes.h"
 
-static Mesh *triangulate_mesh(Mesh *mesh, const int quad_method, const int ngon_method)
+static Mesh *triangulate_mesh(Mesh *mesh, const int quad_method, const int ngon_method, const int flag)
 {
 	Mesh *result;
 	BMesh *bm;
 	int total_edges, i;
 	MEdge *me;
+	CustomDataMask cddata_masks = CD_MASK_ORIGINDEX;
+
+	bool keep_clnors = (flag & MOD_TRIANGULATE_KEEP_CUSTOMLOOP_NORMALS) != 0;
+
+	if (keep_clnors) {
+		BKE_mesh_calc_normals_split(mesh);
+		/* We need that one to 'survive' to/from BMesh conversions. */
+		CustomData_clear_layer_flag(&mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
+		cddata_masks |= CD_MASK_NORMAL;  /* TODO: once D4421 is in, only request CD_NORMAL on loop data... */
+	}
 
 	bm = BKE_mesh_to_bmesh_ex(
 	        mesh,
 	        &((struct BMeshCreateParams){0}),
 	        &((struct BMeshFromMeshParams){
 	            .calc_face_normal = true,
-	            .cd_mask_extra = CD_MASK_ORIGINDEX,
+	            .cd_mask_extra = cddata_masks,
 	        }));
 
 	BM_mesh_triangulate(bm, quad_method, ngon_method, false, NULL, NULL, NULL);
 
-	result = BKE_mesh_from_bmesh_for_eval_nomain(bm, 0);
+	result = BKE_mesh_from_bmesh_for_eval_nomain(bm, cddata_masks);
 	BM_mesh_free(bm);
 
+
+	if (keep_clnors) {
+		bool free_pnors = false;
+		float (*pnors)[3] = CustomData_get_layer(&result->pdata, CD_NORMAL);
+		if (pnors == NULL) {
+			pnors = MEM_mallocN(sizeof(*pnors) * result->totpoly, __func__);
+			BKE_mesh_calc_normals_poly(
+			            result->mvert, NULL, result->totvert,
+			            result->mloop, result->mpoly, result->totloop, result->totpoly, pnors, false);
+			free_pnors = true;
+		}
+
+		float (*lnors)[3] = CustomData_get_layer(&result->ldata, CD_NORMAL);
+		short (*clnors)[2] = CustomData_get_layer(&result->ldata, CD_CUSTOMLOOPNORMAL);
+		if (clnors == NULL) {
+			clnors = CustomData_add_layer(&result->ldata, CD_CUSTOMLOOPNORMAL, CD_CALLOC, NULL, result->totloop);
+		}
+
+		BKE_mesh_normals_loop_custom_set(
+		            result->mvert, result->totvert,
+		            result->medge, result->totedge,
+		            result->mloop, lnors, result->totloop,
+		            result->mpoly, pnors, result->totpoly, clnors);
+
+		/* Do some cleanup, we do not want those temp data to stay around. */
+		CustomData_set_layer_flag(&mesh->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
+		CustomData_set_layer_flag(&result->ldata, CD_NORMAL, CD_FLAG_TEMPORARY);
+		if (free_pnors) {
+			MEM_freeN(pnors);
+		}
+	}
+
 	total_edges = result->totedge;
 	me = result->medge;
 
@@ -82,7 +126,7 @@ static Mesh *applyModifier(
 {
 	TriangulateModifierData *tmd = (TriangulateModifierData *)md;
 	Mesh *result;
-	if (!(result = triangulate_mesh(mesh, tmd->quad_method, tmd->ngon_method))) {
+	if (!(result = triangulate_mesh(mesh, tmd->quad_method, tmd->ngon_method, tmd->flag))) {
 		return mesh;
 	}