Fix T94952: normals maps don't render correctly with GPU subdivision

A simple case of missing the tangent VBO. The tangents are computed from
the coarse mesh, and interpolated on the GPU for the final mesh. Code for
initializing the tangents, and the vertex format for the VBO was
factored out of the coarse extraction routine, to be shared with the
subdivision routine.

9 files changed, 163 insertions, 37 deletions

diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.cc b/source/blender/draw/intern/draw_cache_extract_mesh.cc
index 4056f31bd5e..8b4962f24b0 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.cc
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.cc
@@ -835,6 +835,7 @@ static void mesh_buffer_cache_create_requested_subdiv(MeshBatchCache *cache,
   EXTRACT_ADD_REQUESTED(vbo, edituv_data);
   /* Make sure UVs are computed before edituv stuffs. */
   EXTRACT_ADD_REQUESTED(vbo, uv);
+  EXTRACT_ADD_REQUESTED(vbo, tan);
   EXTRACT_ADD_REQUESTED(vbo, edituv_stretch_area);
   EXTRACT_ADD_REQUESTED(vbo, edituv_stretch_angle);
   EXTRACT_ADD_REQUESTED(ibo, lines_adjacency);
@@ -848,6 +849,7 @@ static void mesh_buffer_cache_create_requested_subdiv(MeshBatchCache *cache,
     return;
   }
 
+  mesh_render_data_update_looptris(mr, MR_ITER_LOOPTRI, MR_DATA_LOOPTRI);
   mesh_render_data_update_loose_geom(mr, mbc, MR_ITER_LEDGE | MR_ITER_LVERT, MR_DATA_LOOSE_GEOM);
 
   void *data_stack = MEM_mallocN(extractors.data_size_total(), __func__);
diff --git a/source/blender/draw/intern/draw_cache_impl_subdivision.cc b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
index 8c42eea1689..f88c1a0e945 100644
--- a/source/blender/draw/intern/draw_cache_impl_subdivision.cc
+++ b/source/blender/draw/intern/draw_cache_impl_subdivision.cc
@@ -1317,7 +1317,8 @@ void draw_subdiv_interp_custom_data(const DRWSubdivCache *cache,
                                     GPUVertBuf *src_data,
                                     GPUVertBuf *dst_data,
                                     int dimensions,
-                                    int dst_offset)
+                                    int dst_offset,
+                                    bool compress_to_u16)
 {
   GPUShader *shader = nullptr;
 
@@ -1337,10 +1338,17 @@ void draw_subdiv_interp_custom_data(const DRWSubdivCache *cache,
                                "#define DIMENSIONS 3\n");
   }
   else if (dimensions == 4) {
-    shader = get_subdiv_shader(SHADER_COMP_CUSTOM_DATA_INTERP_4D,
-                               "#define SUBDIV_POLYGON_OFFSET\n"
-                               "#define DIMENSIONS 4\n"
-                               "#define GPU_FETCH_U16_TO_FLOAT\n");
+    if (compress_to_u16) {
+      shader = get_subdiv_shader(SHADER_COMP_CUSTOM_DATA_INTERP_4D,
+                                 "#define SUBDIV_POLYGON_OFFSET\n"
+                                 "#define DIMENSIONS 4\n"
+                                 "#define GPU_FETCH_U16_TO_FLOAT\n");
+    }
+    else {
+      shader = get_subdiv_shader(SHADER_COMP_CUSTOM_DATA_INTERP_4D,
+                                 "#define SUBDIV_POLYGON_OFFSET\n"
+                                 "#define DIMENSIONS 4\n");
+    }
   }
   else {
     /* Crash if dimensions are not supported. */
diff --git a/source/blender/draw/intern/draw_subdivision.h b/source/blender/draw/intern/draw_subdivision.h
index 2f339cf18f8..5942797ef8f 100644
--- a/source/blender/draw/intern/draw_subdivision.h
+++ b/source/blender/draw/intern/draw_subdivision.h
@@ -197,7 +197,8 @@ void draw_subdiv_interp_custom_data(const DRWSubdivCache *cache,
                                     struct GPUVertBuf *src_data,
                                     struct GPUVertBuf *dst_data,
                                     int dimensions,
-                                    int dst_offset);
+                                    int dst_offset,
+                                    bool compress_to_u16);
 
 void draw_subdiv_extract_uvs(const DRWSubdivCache *cache,
                              struct GPUVertBuf *uvs,
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_attributes.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_attributes.cc
index b846da3f016..8568b87ba69 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_attributes.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_attributes.cc
@@ -409,7 +409,7 @@ static void extract_attr_init_subdiv(const DRWSubdivCache *subdiv_cache,
   /* Ensure data is uploaded properly. */
   GPU_vertbuf_tag_dirty(src_data);
   draw_subdiv_interp_custom_data(
-      subdiv_cache, src_data, dst_buffer, static_cast<int>(dimensions), 0);
+      subdiv_cache, src_data, dst_buffer, static_cast<int>(dimensions), 0, false);
 
   GPU_vertbuf_discard(src_data);
 }
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
index 0645683d58b..a844a116a87 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
@@ -257,7 +257,8 @@ static void extract_pos_nor_init_subdiv(const DRWSubdivCache *subdiv_cache,
     GPU_vertbuf_init_build_on_device(
         dst_custom_normals, get_custom_normals_format(), subdiv_cache->num_subdiv_loops);
 
-    draw_subdiv_interp_custom_data(subdiv_cache, src_custom_normals, dst_custom_normals, 3, 0);
+    draw_subdiv_interp_custom_data(
+        subdiv_cache, src_custom_normals, dst_custom_normals, 3, 0, false);
 
     draw_subdiv_finalize_custom_normals(subdiv_cache, dst_custom_normals, vbo);
 
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_sculpt_data.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_sculpt_data.cc
index 753fbe7e0e2..4eef69411ca 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_sculpt_data.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_sculpt_data.cc
@@ -167,7 +167,7 @@ static void extract_sculpt_data_init_subdiv(const DRWSubdivCache *subdiv_cache,
     GPU_vertbuf_init_build_on_device(
         subdiv_mask_vbo, &mask_format, subdiv_cache->num_subdiv_loops);
 
-    draw_subdiv_interp_custom_data(subdiv_cache, mask_vbo, subdiv_mask_vbo, 1, 0);
+    draw_subdiv_interp_custom_data(subdiv_cache, mask_vbo, subdiv_mask_vbo, 1, 0, false);
   }
 
   /* Then, gather face sets. */
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_tan.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_tan.cc
index 03d1b327689..cce559e07f0 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_tan.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_tan.cc
@@ -32,22 +32,26 @@
 
 #include "extract_mesh.h"
 
+#include "draw_subdivision.h"
+
 namespace blender::draw {
 
 /* ---------------------------------------------------------------------- */
 /** \name Extract Tangent layers
  * \{ */
 
-static void extract_tan_ex_init(const MeshRenderData *mr,
-                                struct MeshBatchCache *cache,
-                                GPUVertBuf *vbo,
-                                const bool do_hq)
+static void extract_tan_init_common(const MeshRenderData *mr,
+                                    struct MeshBatchCache *cache,
+                                    GPUVertFormat *format,
+                                    GPUVertCompType comp_type,
+                                    GPUVertFetchMode fetch_mode,
+                                    CustomData *r_loop_data,
+                                    int *r_v_len,
+                                    int *r_tan_len,
+                                    char r_tangent_names[MAX_MTFACE][MAX_CUSTOMDATA_LAYER_NAME],
+                                    bool *r_use_orco_tan)
 {
-  GPUVertCompType comp_type = do_hq ? GPU_COMP_I16 : GPU_COMP_I10;
-  GPUVertFetchMode fetch_mode = GPU_FETCH_INT_TO_FLOAT_UNIT;
-
-  GPUVertFormat format = {0};
-  GPU_vertformat_deinterleave(&format);
+  GPU_vertformat_deinterleave(format);
 
   CustomData *cd_ldata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->ldata : &mr->me->ldata;
   CustomData *cd_vdata = (mr->extract_type == MR_EXTRACT_BMESH) ? &mr->bm->vdata : &mr->me->vdata;
@@ -57,7 +61,6 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
   bool use_orco_tan = cache->cd_used.tan_orco != 0;
 
   int tan_len = 0;
-  char tangent_names[MAX_MTFACE][MAX_CUSTOMDATA_LAYER_NAME];
 
   /* FIXME(T91838): This is to avoid a crash when orco tangent was requested but there are valid
    * uv layers. It would be better to fix the root cause. */
@@ -73,17 +76,17 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
       GPU_vertformat_safe_attr_name(layer_name, attr_safe_name, GPU_MAX_SAFE_ATTR_NAME);
       /* Tangent layer name. */
       BLI_snprintf(attr_name, sizeof(attr_name), "t%s", attr_safe_name);
-      GPU_vertformat_attr_add(&format, attr_name, comp_type, 4, fetch_mode);
+      GPU_vertformat_attr_add(format, attr_name, comp_type, 4, fetch_mode);
       /* Active render layer name. */
       if (i == CustomData_get_render_layer(cd_ldata, CD_MLOOPUV)) {
-        GPU_vertformat_alias_add(&format, "t");
+        GPU_vertformat_alias_add(format, "t");
       }
       /* Active display layer name. */
       if (i == CustomData_get_active_layer(cd_ldata, CD_MLOOPUV)) {
-        GPU_vertformat_alias_add(&format, "at");
+        GPU_vertformat_alias_add(format, "at");
       }
 
-      BLI_strncpy(tangent_names[tan_len++], layer_name, MAX_CUSTOMDATA_LAYER_NAME);
+      BLI_strncpy(r_tangent_names[tan_len++], layer_name, MAX_CUSTOMDATA_LAYER_NAME);
     }
   }
   if (use_orco_tan && orco == nullptr) {
@@ -110,20 +113,19 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
   }
 
   /* Start Fresh */
-  CustomData loop_data;
-  CustomData_reset(&loop_data);
+  CustomData_reset(r_loop_data);
   if (tan_len != 0 || use_orco_tan) {
     short tangent_mask = 0;
     bool calc_active_tangent = false;
     if (mr->extract_type == MR_EXTRACT_BMESH) {
       BKE_editmesh_loop_tangent_calc(mr->edit_bmesh,
                                      calc_active_tangent,
-                                     tangent_names,
+                                     r_tangent_names,
                                      tan_len,
                                      mr->poly_normals,
                                      mr->loop_normals,
                                      orco,
-                                     &loop_data,
+                                     r_loop_data,
                                      mr->loop_len,
                                      &tangent_mask);
     }
@@ -136,13 +138,13 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
                                     mr->tri_len,
                                     cd_ldata,
                                     calc_active_tangent,
-                                    tangent_names,
+                                    r_tangent_names,
                                     tan_len,
                                     mr->vert_normals,
                                     mr->poly_normals,
                                     mr->loop_normals,
                                     orco,
-                                    &loop_data,
+                                    r_loop_data,
                                     mr->loop_len,
                                     &tangent_mask);
     }
@@ -150,12 +152,12 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
 
   if (use_orco_tan) {
     char attr_name[32], attr_safe_name[GPU_MAX_SAFE_ATTR_NAME];
-    const char *layer_name = CustomData_get_layer_name(&loop_data, CD_TANGENT, 0);
+    const char *layer_name = CustomData_get_layer_name(r_loop_data, CD_TANGENT, 0);
     GPU_vertformat_safe_attr_name(layer_name, attr_safe_name, GPU_MAX_SAFE_ATTR_NAME);
     BLI_snprintf(attr_name, sizeof(*attr_name), "t%s", attr_safe_name);
-    GPU_vertformat_attr_add(&format, attr_name, comp_type, 4, fetch_mode);
-    GPU_vertformat_alias_add(&format, "t");
-    GPU_vertformat_alias_add(&format, "at");
+    GPU_vertformat_attr_add(format, attr_name, comp_type, 4, fetch_mode);
+    GPU_vertformat_alias_add(format, "t");
+    GPU_vertformat_alias_add(format, "at");
   }
 
   if (orco_allocated) {
@@ -163,12 +165,42 @@ static void extract_tan_ex_init(const MeshRenderData *mr,
   }
 
   int v_len = mr->loop_len;
-  if (format.attr_len == 0) {
-    GPU_vertformat_attr_add(&format, "dummy", GPU_COMP_F32, 1, GPU_FETCH_FLOAT);
+  if (format->attr_len == 0) {
+    GPU_vertformat_attr_add(format, "dummy", GPU_COMP_F32, 1, GPU_FETCH_FLOAT);
     /* VBO will not be used, only allocate minimum of memory. */
     v_len = 1;
   }
 
+  *r_use_orco_tan = use_orco_tan;
+  *r_v_len = v_len;
+  *r_tan_len = tan_len;
+}
+
+static void extract_tan_ex_init(const MeshRenderData *mr,
+                                struct MeshBatchCache *cache,
+                                GPUVertBuf *vbo,
+                                const bool do_hq)
+{
+  GPUVertCompType comp_type = do_hq ? GPU_COMP_I16 : GPU_COMP_I10;
+  GPUVertFetchMode fetch_mode = GPU_FETCH_INT_TO_FLOAT_UNIT;
+
+  GPUVertFormat format = {0};
+  CustomData loop_data;
+  int v_len = 0;
+  int tan_len = 0;
+  bool use_orco_tan;
+  char tangent_names[MAX_MTFACE][MAX_CUSTOMDATA_LAYER_NAME];
+  extract_tan_init_common(mr,
+                          cache,
+                          &format,
+                          comp_type,
+                          fetch_mode,
+                          &loop_data,
+                          &v_len,
+                          &tan_len,
+                          tangent_names,
+                          &use_orco_tan);
+
   GPU_vertbuf_init_with_format(vbo, &format);
   GPU_vertbuf_data_alloc(vbo, v_len);
 
@@ -227,10 +259,92 @@ static void extract_tan_init(const MeshRenderData *mr,
   extract_tan_ex_init(mr, cache, vbo, false);
 }
 
+static GPUVertFormat *get_coarse_tan_format()
+{
+  static GPUVertFormat format = {0};
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "tan", GPU_COMP_F32, 4, GPU_FETCH_FLOAT);
+  }
+  return &format;
+}
+
+static void extract_tan_init_subdiv(const DRWSubdivCache *subdiv_cache,
+                                    const MeshRenderData *mr,
+                                    struct MeshBatchCache *cache,
+                                    void *buffer,
+                                    void *UNUSED(data))
+{
+  GPUVertCompType comp_type = GPU_COMP_F32;
+  GPUVertFetchMode fetch_mode = GPU_FETCH_FLOAT;
+  GPUVertFormat format = {0};
+  CustomData loop_data;
+  int coarse_len = 0;
+  int tan_len = 0;
+  bool use_orco_tan;
+  char tangent_names[MAX_MTFACE][MAX_CUSTOMDATA_LAYER_NAME];
+  extract_tan_init_common(mr,
+                          cache,
+                          &format,
+                          comp_type,
+                          fetch_mode,
+                          &loop_data,
+                          &coarse_len,
+                          &tan_len,
+                          tangent_names,
+                          &use_orco_tan);
+
+  GPUVertBuf *dst_buffer = static_cast<GPUVertBuf *>(buffer);
+  GPU_vertbuf_init_build_on_device(dst_buffer, &format, subdiv_cache->num_subdiv_loops);
+
+  GPUVertBuf *coarse_vbo = GPU_vertbuf_calloc();
+  /* Dynamic as we upload and interpolate layers one at a time. */
+  GPU_vertbuf_init_with_format_ex(coarse_vbo, get_coarse_tan_format(), GPU_USAGE_DYNAMIC);
+  GPU_vertbuf_data_alloc(coarse_vbo, coarse_len);
+
+  /* Index of the tangent layer in the compact buffer. Used layers are stored in a single buffer.
+   */
+  int pack_layer_index = 0;
+  for (int i = 0; i < tan_len; i++) {
+    float(*tan_data)[4] = (float(*)[4])GPU_vertbuf_get_data(coarse_vbo);
+    const char *name = tangent_names[i];
+    float(*layer_data)[4] = (float(*)[4])CustomData_get_layer_named(&loop_data, CD_TANGENT, name);
+    for (int ml_index = 0; ml_index < mr->loop_len; ml_index++) {
+      copy_v3_v3(*tan_data, layer_data[ml_index]);
+      (*tan_data)[3] = (layer_data[ml_index][3] > 0.0f) ? 1.0f : -1.0f;
+      tan_data++;
+    }
+
+    /* Ensure data is uploaded properly. */
+    GPU_vertbuf_tag_dirty(coarse_vbo);
+    /* Include stride in offset. */
+    const int dst_offset = (int)subdiv_cache->num_subdiv_loops * 4 * pack_layer_index++;
+    draw_subdiv_interp_custom_data(subdiv_cache, coarse_vbo, dst_buffer, 4, dst_offset, false);
+  }
+  if (use_orco_tan) {
+    float(*tan_data)[4] = (float(*)[4])GPU_vertbuf_get_data(coarse_vbo);
+    float(*layer_data)[4] = (float(*)[4])CustomData_get_layer_n(&loop_data, CD_TANGENT, 0);
+    for (int ml_index = 0; ml_index < mr->loop_len; ml_index++) {
+      copy_v3_v3(*tan_data, layer_data[ml_index]);
+      (*tan_data)[3] = (layer_data[ml_index][3] > 0.0f) ? 1.0f : -1.0f;
+      tan_data++;
+    }
+
+    /* Ensure data is uploaded properly. */
+    GPU_vertbuf_tag_dirty(coarse_vbo);
+    /* Include stride in offset. */
+    const int dst_offset = (int)subdiv_cache->num_subdiv_loops * 4 * pack_layer_index++;
+    draw_subdiv_interp_custom_data(subdiv_cache, coarse_vbo, dst_buffer, 4, dst_offset, true);
+  }
+
+  CustomData_free(&loop_data, mr->loop_len);
+  GPU_vertbuf_discard(coarse_vbo);
+}
+
 constexpr MeshExtract create_extractor_tan()
 {
   MeshExtract extractor = {nullptr};
   extractor.init = extract_tan_init;
+  extractor.init_subdiv = extract_tan_init_subdiv;
   extractor.data_type = MR_DATA_POLY_NOR | MR_DATA_TAN_LOOP_NOR | MR_DATA_LOOPTRI;
   extractor.data_size = 0;
   extractor.use_threading = false;
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_vcol.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_vcol.cc
index a0307b9b2cd..324ac4bae37 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_vcol.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_vcol.cc
@@ -180,7 +180,7 @@ static void extract_vcol_init_subdiv(const DRWSubdivCache *subdiv_cache,
 
       /* Ensure data is uploaded properly. */
       GPU_vertbuf_tag_dirty(src_data);
-      draw_subdiv_interp_custom_data(subdiv_cache, src_data, dst_buffer, 4, dst_offset);
+      draw_subdiv_interp_custom_data(subdiv_cache, src_data, dst_buffer, 4, dst_offset, true);
     }
   }
 
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_weights.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_weights.cc
index bb608014c53..b5727367412 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_weights.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_weights.cc
@@ -201,7 +201,7 @@ static void extract_weights_init_subdiv(const DRWSubdivCache *subdiv_cache,
     }
   }
 
-  draw_subdiv_interp_custom_data(subdiv_cache, coarse_weights, vbo, 1, 0);
+  draw_subdiv_interp_custom_data(subdiv_cache, coarse_weights, vbo, 1, 0, false);
 
   GPU_vertbuf_discard(coarse_weights);
 }