Fix T51746: normal map tangents not working correctly when there are no UV maps.

Patch by Alexander, with some refactoring by Brecht.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D2709

5 files changed, 59 insertions, 43 deletions

diff --git a/source/blender/blenkernel/BKE_DerivedMesh.h b/source/blender/blenkernel/BKE_DerivedMesh.h
index 789bc8df7e5..63ff1149a68 100644
--- a/source/blender/blenkernel/BKE_DerivedMesh.h
+++ b/source/blender/blenkernel/BKE_DerivedMesh.h
@@ -201,7 +201,7 @@ struct DerivedMesh {
 	/* use for converting to BMesh which doesn't store bevel weight and edge crease by default */
 	char cd_flag;
 
-	char tangent_mask; /* which tangent layers are calculated */
+	short tangent_mask; /* which tangent layers are calculated */
 
 	/** Calculate vert and face normals */
 	void (*calcNormals)(DerivedMesh *dm);
@@ -791,11 +791,13 @@ void DM_calc_tangents_names_from_gpu(
 void DM_add_named_tangent_layer_for_uv(
         CustomData *uv_data, CustomData *tan_data, int numLoopData,
         const char *layer_name);
+
+#define DM_TANGENT_MASK_ORCO (1 << 9)
 void DM_calc_loop_tangents_step_0(
         const CustomData *loopData, bool calc_active_tangent,
         const char (*tangent_names)[MAX_NAME], int tangent_names_count,
         bool *rcalc_act, bool *rcalc_ren, int *ract_uv_n, int *rren_uv_n,
-        char *ract_uv_name, char *rren_uv_name, char *rtangent_mask);
+        char *ract_uv_name, char *rren_uv_name, short *rtangent_mask);
 void DM_calc_loop_tangents(
         DerivedMesh *dm, bool calc_active_tangent, const char (*tangent_names)[MAX_NAME],
         int tangent_names_count);
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index eba5814d897..7eea8224ba1 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -3322,7 +3322,7 @@ void DM_calc_loop_tangents_step_0(
         const CustomData *loopData, bool calc_active_tangent,
         const char (*tangent_names)[MAX_NAME], int tangent_names_count,
         bool *rcalc_act, bool *rcalc_ren, int *ract_uv_n, int *rren_uv_n,
-        char *ract_uv_name, char *rren_uv_name, char *rtangent_mask)
+        char *ract_uv_name, char *rren_uv_name, short *rtangent_mask)
 {
 	/* Active uv in viewport */
 	int layer_index = CustomData_get_layer_index(loopData, CD_MLOOPUV);
@@ -3377,21 +3377,22 @@ void DM_calc_loop_tangents_step_0(
 		if (add)
 			*rtangent_mask |= 1 << n;
 	}
+
+	if (uv_layer_num == 0)
+		*rtangent_mask |= DM_TANGENT_MASK_ORCO;
 }
 
 void DM_calc_loop_tangents(
         DerivedMesh *dm, bool calc_active_tangent,
         const char (*tangent_names)[MAX_NAME], int tangent_names_count)
 {
-	if (CustomData_number_of_layers(&dm->loopData, CD_MLOOPUV) == 0)
-		return;
 	int act_uv_n = -1;
 	int ren_uv_n = -1;
 	bool calc_act = false;
 	bool calc_ren = false;
 	char act_uv_name[MAX_NAME];
 	char ren_uv_name[MAX_NAME];
-	char tangent_mask = 0;
+	short tangent_mask = 0;
 	DM_calc_loop_tangents_step_0(
 	        &dm->loopData, calc_active_tangent, tangent_names, tangent_names_count,
 	        &calc_act, &calc_ren, &act_uv_n, &ren_uv_n, act_uv_name, ren_uv_name, &tangent_mask);
@@ -3404,6 +3405,8 @@ void DM_calc_loop_tangents(
 		for (int i = 0; i < tangent_names_count; i++)
 			if (tangent_names[i][0])
 				DM_add_named_tangent_layer_for_uv(&dm->loopData, &dm->loopData, dm->numLoopData, tangent_names[i]);
+		if ((tangent_mask & DM_TANGENT_MASK_ORCO) && CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, "") == -1)
+			CustomData_add_layer_named(&dm->loopData, CD_TANGENT, CD_CALLOC, NULL, dm->numLoopData, "");
 		if (calc_act && act_uv_name[0])
 			DM_add_named_tangent_layer_for_uv(&dm->loopData, &dm->loopData, dm->numLoopData, act_uv_name);
 		if (calc_ren && ren_uv_name[0])
@@ -3465,19 +3468,24 @@ void DM_calc_loop_tangents(
 
 				mesh2tangent->orco = NULL;
 				mesh2tangent->mloopuv = CustomData_get_layer_named(&dm->loopData, CD_MLOOPUV, dm->loopData.layers[index].name);
+
+				/* Fill the resulting tangent_mask */
 				if (!mesh2tangent->mloopuv) {
 					mesh2tangent->orco = dm->getVertDataArray(dm, CD_ORCO);
 					if (!mesh2tangent->orco)
 						continue;
+
+					dm->tangent_mask |= DM_TANGENT_MASK_ORCO;
+				}
+				else {
+					int uv_ind = CustomData_get_named_layer_index(&dm->loopData, CD_MLOOPUV, dm->loopData.layers[index].name);
+					int uv_start = CustomData_get_layer_index(&dm->loopData, CD_MLOOPUV);
+					BLI_assert(uv_ind != -1 && uv_start != -1);
+					BLI_assert(uv_ind - uv_start < MAX_MTFACE);
+					dm->tangent_mask |= 1 << (uv_ind - uv_start);
 				}
-				mesh2tangent->tangent = dm->loopData.layers[index].data;
 
-				/* Fill the resulting tangent_mask */
-				int uv_ind = CustomData_get_named_layer_index(&dm->loopData, CD_MLOOPUV, dm->loopData.layers[index].name);
-				int uv_start = CustomData_get_layer_index(&dm->loopData, CD_MLOOPUV);
-				BLI_assert(uv_ind != -1 && uv_start != -1);
-				BLI_assert(uv_ind - uv_start < MAX_MTFACE);
-				dm->tangent_mask |= 1 << (uv_ind - uv_start);
+				mesh2tangent->tangent = dm->loopData.layers[index].data;
 				BLI_task_pool_push(task_pool, DM_calc_loop_tangents_thread, mesh2tangent, false, TASK_PRIORITY_LOW);
 			}
 
@@ -3493,21 +3501,19 @@ void DM_calc_loop_tangents(
 
 #endif
 
-		int uv_index, tan_index;
-
 		/* Update active layer index */
-		uv_index = CustomData_get_layer_index_n(&dm->loopData, CD_MLOOPUV, act_uv_n);
-		if (uv_index != -1) {
-			tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, dm->loopData.layers[uv_index].name);
+		int act_uv_index = CustomData_get_layer_index_n(&dm->loopData, CD_MLOOPUV, act_uv_n);
+		if (act_uv_index != -1) {
+			int tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, dm->loopData.layers[act_uv_index].name);
 			CustomData_set_layer_active_index(&dm->loopData, CD_TANGENT, tan_index);
-		}
+		} /* else tangent has been built from orco */
 
 		/* Update render layer index */
-		uv_index = CustomData_get_layer_index_n(&dm->loopData, CD_MLOOPUV, ren_uv_n);
-		if (uv_index != -1) {
-			tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, dm->loopData.layers[uv_index].name);
+		int ren_uv_index = CustomData_get_layer_index_n(&dm->loopData, CD_MLOOPUV, ren_uv_n);
+		if (ren_uv_index != -1) {
+			int tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, dm->loopData.layers[ren_uv_index].name);
 			CustomData_set_layer_render_index(&dm->loopData, CD_TANGENT, tan_index);
-		}
+		} /* else tangent has been built from orco */
 	}
 }
 
diff --git a/source/blender/blenkernel/intern/editderivedmesh.c b/source/blender/blenkernel/intern/editderivedmesh.c
index 4a45bf1534c..a91983c001c 100644
--- a/source/blender/blenkernel/intern/editderivedmesh.c
+++ b/source/blender/blenkernel/intern/editderivedmesh.c
@@ -489,8 +489,6 @@ static void emDM_calc_loop_tangents(
 	EditDerivedBMesh *bmdm = (EditDerivedBMesh *)dm;
 	BMEditMesh *em = bmdm->em;
 	BMesh *bm = bmdm->em->bm;
-	if (CustomData_number_of_layers(&bm->ldata, CD_MLOOPUV) == 0)
-		return;
 
 	int act_uv_n = -1;
 	int ren_uv_n = -1;
@@ -498,7 +496,7 @@ static void emDM_calc_loop_tangents(
 	bool calc_ren = false;
 	char act_uv_name[MAX_NAME];
 	char ren_uv_name[MAX_NAME];
-	char tangent_mask = 0;
+	short tangent_mask = 0;
 
 	DM_calc_loop_tangents_step_0(
 	        &bm->ldata, calc_active_tangent, tangent_names, tangent_names_count,
@@ -508,6 +506,8 @@ static void emDM_calc_loop_tangents(
 		for (int i = 0; i < tangent_names_count; i++)
 			if (tangent_names[i][0])
 				DM_add_named_tangent_layer_for_uv(&bm->ldata, &dm->loopData, dm->numLoopData, tangent_names[i]);
+		if ((tangent_mask & DM_TANGENT_MASK_ORCO) && CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, "") == -1)
+			CustomData_add_layer_named(&dm->loopData, CD_TANGENT, CD_CALLOC, NULL, dm->numLoopData, "");
 		if (calc_act && act_uv_name[0])
 			DM_add_named_tangent_layer_for_uv(&bm->ldata, &dm->loopData, dm->numLoopData, act_uv_name);
 		if (calc_ren && ren_uv_name[0])
@@ -574,7 +574,17 @@ static void emDM_calc_loop_tangents(
 						continue;
 					/* needed for orco lookups */
 					htype_index |= BM_VERT;
+					dm->tangent_mask |= DM_TANGENT_MASK_ORCO;
+				}
+				else {
+					/* Fill the resulting tangent_mask */
+					int uv_ind = CustomData_get_named_layer_index(&bm->ldata, CD_MLOOPUV, dm->loopData.layers[index].name);
+					int uv_start = CustomData_get_layer_index(&bm->ldata, CD_MLOOPUV);
+					BLI_assert(uv_ind != -1 && uv_start != -1);
+					BLI_assert(uv_ind - uv_start < MAX_MTFACE);
+					dm->tangent_mask |= 1 << (uv_ind - uv_start);
 				}
+
 				if (mesh2tangent->precomputedFaceNormals) {
 					/* needed for face normal lookups */
 					htype_index |= BM_FACE;
@@ -584,12 +594,6 @@ static void emDM_calc_loop_tangents(
 				mesh2tangent->looptris = (const BMLoop *(*)[3])em->looptris;
 				mesh2tangent->tangent = dm->loopData.layers[index].data;
 
-				/* Fill the resulting tangent_mask */
-				int uv_ind = CustomData_get_named_layer_index(&bm->ldata, CD_MLOOPUV, dm->loopData.layers[index].name);
-				int uv_start = CustomData_get_layer_index(&bm->ldata, CD_MLOOPUV);
-				BLI_assert(uv_ind != -1 && uv_start != -1);
-				BLI_assert(uv_ind - uv_start < MAX_MTFACE);
-				dm->tangent_mask |= 1 << (uv_ind - uv_start);
 				BLI_task_pool_push(task_pool, emDM_calc_loop_tangents_thread, mesh2tangent, false, TASK_PRIORITY_LOW);
 			}
 
@@ -604,15 +608,20 @@ static void emDM_calc_loop_tangents(
 #undef USE_LOOPTRI_DETECT_QUADS
 #endif
 	}
+
 	/* Update active layer index */
-	int uv_index = CustomData_get_layer_index_n(&bm->ldata, CD_MLOOPUV, act_uv_n);
-	int tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, bm->ldata.layers[uv_index].name);
-	CustomData_set_layer_active_index(&dm->loopData, CD_TANGENT, tan_index);
+	int act_uv_index = CustomData_get_layer_index_n(&bm->ldata, CD_MLOOPUV, act_uv_n);
+	if (act_uv_index >= 0) {
+		int tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, bm->ldata.layers[act_uv_index].name);
+		CustomData_set_layer_active_index(&dm->loopData, CD_TANGENT, tan_index);
+	} /* else tangent has been built from orco */
 
 	/* Update render layer index */
-	uv_index = CustomData_get_layer_index_n(&bm->ldata, CD_MLOOPUV, ren_uv_n);
-	tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, bm->ldata.layers[uv_index].name);
-	CustomData_set_layer_render_index(&dm->loopData, CD_TANGENT, tan_index);
+	int ren_uv_index = CustomData_get_layer_index_n(&bm->ldata, CD_MLOOPUV, ren_uv_n);
+	if (ren_uv_index >= 0) {
+		int tan_index = CustomData_get_named_layer_index(&dm->loopData, CD_TANGENT, bm->ldata.layers[ren_uv_index].name);
+		CustomData_set_layer_render_index(&dm->loopData, CD_TANGENT, tan_index);
+	} /* else tangent has been built from orco */
 }
 
 /** \} */
diff --git a/source/blender/render/intern/include/render_types.h b/source/blender/render/intern/include/render_types.h
index f0323340899..f9942bef61f 100644
--- a/source/blender/render/intern/include/render_types.h
+++ b/source/blender/render/intern/include/render_types.h
@@ -335,7 +335,7 @@ typedef struct ObjectRen {
 	char (*mcol)[MAX_CUSTOMDATA_LAYER_NAME];
 	int  actmtface, actmcol, bakemtface;
 
-	char tangent_mask; /* which tangent layer should be calculated */
+	short tangent_mask; /* which tangent layer should be calculated */
 
 	float obmat[4][4];	/* only used in convertblender.c, for instancing */
 
diff --git a/source/blender/render/intern/source/convertblender.c b/source/blender/render/intern/source/convertblender.c
index 41ff8e046e7..4876d2d193e 100644
--- a/source/blender/render/intern/source/convertblender.c
+++ b/source/blender/render/intern/source/convertblender.c
@@ -3439,10 +3439,9 @@ static void init_render_mesh(Render *re, ObjectRen *obr, int timeoffset)
 										if (need_nmap_tangent_concrete || need_tangent) {
 											int uv_start = CustomData_get_layer_index(&dm->faceData, CD_MTFACE);
 											int uv_index = CustomData_get_named_layer_index(&dm->faceData, CD_MTFACE, layer->name);
-											BLI_assert(uv_start >= 0 && uv_index >= 0);
-											if ((uv_start < 0 || uv_index < 0))
-												continue;
-											int n = uv_index - uv_start;
+
+											/* if there are no UVs, orco tangents are in first slot */
+											int n = (uv_start >= 0 && uv_index >= 0) ? uv_index - uv_start : 0;
 
 											const float *tangent = (const float *) layer->data;
 											float *ftang = RE_vlakren_get_nmap_tangent(obr, vlr, n, true);