Fix T84459: Face normals not displaying (AMD GPU)

This is part two of the fix for T84459.
Issue appears to be caused by AMD graphics driver later than 20.11.1 and
affects older GPUs (Polaris/FIJI cards). Drawing normals in edit mode
uses the same OpenGL data type for storing normals that is known to be
faulty.

This change fixes the face dot normals by using GPU_COMP_I16.

1 files changed, 82 insertions, 6 deletions

diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.c b/source/blender/draw/intern/draw_cache_extract_mesh.c
index 6eeb3c7abdc..8185519ef33 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.c
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.c
@@ -3976,7 +3976,78 @@ static const MeshExtract extract_fdots_nor = {
 /** \} */
 
 /* ---------------------------------------------------------------------- */
-/** \name Extract Facedots Normal and edit flag
+/** \name Extract Facedots High Quality Normal and edit flag
+ * \{ */
+static void *extract_fdots_nor_hq_init(const MeshRenderData *mr, void *buf)
+{
+  static GPUVertFormat format = {0};
+  if (format.attr_len == 0) {
+    GPU_vertformat_attr_add(&format, "norAndFlag", GPU_COMP_I16, 4, GPU_FETCH_INT_TO_FLOAT_UNIT);
+  }
+  GPUVertBuf *vbo = buf;
+  GPU_vertbuf_init_with_format(vbo, &format);
+  GPU_vertbuf_data_alloc(vbo, mr->poly_len);
+
+  return NULL;
+}
+
+static void extract_fdots_nor_hq_finish(const MeshRenderData *mr, void *buf, void *UNUSED(data))
+{
+  static float invalid_normal[3] = {0.0f, 0.0f, 0.0f};
+  GPUVertBuf *vbo = buf;
+  short *nor = (short *)vbo->data;
+  BMFace *efa;
+
+  /* Quicker than doing it for each loop. */
+  if (mr->extract_type == MR_EXTRACT_BMESH) {
+    for (int f = 0; f < mr->poly_len; f++) {
+      efa = BM_face_at_index(mr->bm, f);
+      const bool is_face_hidden = BM_elem_flag_test(efa, BM_ELEM_HIDDEN);
+      if (is_face_hidden || (mr->extract_type == MR_EXTRACT_MAPPED && mr->p_origindex &&
+                             mr->p_origindex[f] == ORIGINDEX_NONE)) {
+        normal_float_to_short_v3(&nor[f * 4], invalid_normal);
+        nor[f * 4 + 3] = NOR_AND_FLAG_HIDDEN;
+      }
+      else {
+        normal_float_to_short_v3(&nor[f * 4], efa->no);
+        /* Select / Active Flag. */
+        nor[f * 4 + 3] = (BM_elem_flag_test(efa, BM_ELEM_SELECT) ?
+                              ((efa == mr->efa_act) ? NOR_AND_FLAG_ACTIVE : NOR_AND_FLAG_SELECT) :
+                              NOR_AND_FLAG_DEFAULT);
+      }
+    }
+  }
+  else {
+    for (int f = 0; f < mr->poly_len; f++) {
+      efa = bm_original_face_get(mr, f);
+      const bool is_face_hidden = efa && BM_elem_flag_test(efa, BM_ELEM_HIDDEN);
+      if (is_face_hidden || (mr->extract_type == MR_EXTRACT_MAPPED && mr->p_origindex &&
+                             mr->p_origindex[f] == ORIGINDEX_NONE)) {
+        normal_float_to_short_v3(&nor[f * 4], invalid_normal);
+        nor[f * 4 + 3] = NOR_AND_FLAG_HIDDEN;
+      }
+      else {
+        normal_float_to_short_v3(&nor[f * 4], efa->no);
+        /* Select / Active Flag. */
+        nor[f * 4 + 3] = (BM_elem_flag_test(efa, BM_ELEM_SELECT) ?
+                              ((efa == mr->efa_act) ? NOR_AND_FLAG_ACTIVE : NOR_AND_FLAG_SELECT) :
+                              NOR_AND_FLAG_DEFAULT);
+      }
+    }
+  }
+}
+
+static const MeshExtract extract_fdots_nor_hq = {
+    .init = extract_fdots_nor_hq_init,
+    .finish = extract_fdots_nor_hq_finish,
+    .data_flag = MR_DATA_POLY_NOR,
+    .use_threading = false,
+};
+
+/** \} */
+
+/* ---------------------------------------------------------------------- */
+/** \name Extract Facedots UV
  * \{ */
 
 typedef struct MeshExtract_FdotUV_Data {
@@ -4545,11 +4616,16 @@ static void extract_task_create(TaskPool *task_pool,
   BLI_assert(scene != NULL);
   const bool do_hq_normals = (scene->r.perf_flag & SCE_PERF_HQ_NORMALS) != 0 ||
                              GPU_use_hq_normals_workaround();
-  if (do_hq_normals && (extract == &extract_lnor)) {
-    extract = &extract_lnor_hq;
-  }
-  if (do_hq_normals && (extract == &extract_tan)) {
-    extract = &extract_tan_hq;
+  if (do_hq_normals) {
+    if (extract == &extract_lnor) {
+      extract = &extract_lnor_hq;
+    }
+    else if (extract == &extract_tan) {
+      extract = &extract_tan_hq;
+    }
+    else if (extract == &extract_fdots_nor) {
+      extract = &extract_fdots_nor_hq;
+    }
   }
 
   /* Divide extraction of the VBO/IBO into sensible chunks of works. */