Fix T77342: Fail to draw loose edges

The underlying issue is that the loose edge extraction cannot be
threaded. This patch will extract the loose edges during initialization
of the render mesh data.

When working on this patch the mesh_render_data_update was split into
multiple functions to identify what part was failing. These functions
would also help us with debugging.

Reviewed By: Clément Foucault

Differential Revision: https://developer.blender.org/D7962

1 files changed, 197 insertions, 160 deletions

diff --git a/source/blender/draw/intern/draw_cache_extract_mesh.c b/source/blender/draw/intern/draw_cache_extract_mesh.c
index 06462d5b9c5..6013bdb2883 100644
--- a/source/blender/draw/intern/draw_cache_extract_mesh.c
+++ b/source/blender/draw/intern/draw_cache_extract_mesh.c
@@ -134,6 +134,183 @@ typedef struct MeshRenderData {
   float (*poly_normals)[3];
   int *lverts, *ledges;
 } MeshRenderData;
+
+static void mesh_render_data_update_loose_geom(MeshRenderData *mr,
+                                               const eMRIterType iter_type,
+                                               const eMRDataType UNUSED(data_flag))
+{
+  if (mr->extract_type != MR_EXTRACT_BMESH) {
+    /* Mesh */
+    if (iter_type & (MR_ITER_LEDGE | MR_ITER_LVERT)) {
+      mr->vert_loose_len = 0;
+      mr->edge_loose_len = 0;
+
+      BLI_bitmap *lvert_map = BLI_BITMAP_NEW(mr->vert_len, "lvert map");
+
+      mr->ledges = MEM_mallocN(mr->edge_len * sizeof(int), __func__);
+      const MEdge *medge = mr->medge;
+      for (int e = 0; e < mr->edge_len; e++, medge++) {
+        if (medge->flag & ME_LOOSEEDGE) {
+          mr->ledges[mr->edge_loose_len++] = e;
+        }
+        /* Tag verts as not loose. */
+        BLI_BITMAP_ENABLE(lvert_map, medge->v1);
+        BLI_BITMAP_ENABLE(lvert_map, medge->v2);
+      }
+      if (mr->edge_loose_len < mr->edge_len) {
+        mr->ledges = MEM_reallocN(mr->ledges, mr->edge_loose_len * sizeof(*mr->ledges));
+      }
+
+      mr->lverts = MEM_mallocN(mr->vert_len * sizeof(*mr->lverts), __func__);
+      for (int v = 0; v < mr->vert_len; v++) {
+        if (!BLI_BITMAP_TEST(lvert_map, v)) {
+          mr->lverts[mr->vert_loose_len++] = v;
+        }
+      }
+      if (mr->vert_loose_len < mr->vert_len) {
+        mr->lverts = MEM_reallocN(mr->lverts, mr->vert_loose_len * sizeof(*mr->lverts));
+      }
+
+      MEM_freeN(lvert_map);
+
+      mr->loop_loose_len = mr->vert_loose_len + mr->edge_loose_len * 2;
+    }
+  }
+  else {
+    /* BMesh */
+    BMesh *bm = mr->bm;
+    if (iter_type & (MR_ITER_LEDGE | MR_ITER_LVERT)) {
+      int elem_id;
+      BMIter iter;
+      BMVert *eve;
+      BMEdge *ede;
+      mr->vert_loose_len = 0;
+      mr->edge_loose_len = 0;
+
+      mr->lverts = MEM_mallocN(mr->vert_len * sizeof(*mr->lverts), __func__);
+      BM_ITER_MESH_INDEX (eve, &iter, bm, BM_VERTS_OF_MESH, elem_id) {
+        if (eve->e == NULL) {
+          mr->lverts[mr->vert_loose_len++] = elem_id;
+        }
+      }
+      if (mr->vert_loose_len < mr->vert_len) {
+        mr->lverts = MEM_reallocN(mr->lverts, mr->vert_loose_len * sizeof(*mr->lverts));
+      }
+
+      mr->ledges = MEM_mallocN(mr->edge_len * sizeof(*mr->ledges), __func__);
+      BM_ITER_MESH_INDEX (ede, &iter, bm, BM_EDGES_OF_MESH, elem_id) {
+        if (ede->l == NULL) {
+          mr->ledges[mr->edge_loose_len++] = elem_id;
+        }
+      }
+      if (mr->edge_loose_len < mr->edge_len) {
+        mr->ledges = MEM_reallocN(mr->ledges, mr->edge_loose_len * sizeof(*mr->ledges));
+      }
+
+      mr->loop_loose_len = mr->vert_loose_len + mr->edge_loose_len * 2;
+    }
+  }
+}
+
+/* Part of the creation of the MeshRenderData that happens in a thread. */
+static void mesh_render_data_update_looptris(MeshRenderData *mr,
+                                             const eMRIterType iter_type,
+                                             const eMRDataType data_flag)
+{
+  Mesh *me = mr->me;
+  if (mr->extract_type != MR_EXTRACT_BMESH) {
+    /* Mesh */
+    if ((iter_type & MR_ITER_LOOPTRI) || (data_flag & MR_DATA_LOOPTRI)) {
+      mr->mlooptri = MEM_mallocN(sizeof(*mr->mlooptri) * mr->tri_len, "MR_DATATYPE_LOOPTRI");
+      BKE_mesh_recalc_looptri(
+          me->mloop, me->mpoly, me->mvert, me->totloop, me->totpoly, mr->mlooptri);
+    }
+  }
+  else {
+    /* BMesh */
+    if ((iter_type & MR_ITER_LOOPTRI) || (data_flag & MR_DATA_LOOPTRI)) {
+      /* Edit mode ensures this is valid, no need to calculate. */
+      BLI_assert((mr->bm->totloop == 0) || (mr->edit_bmesh->looptris != NULL));
+    }
+  }
+}
+
+static void mesh_render_data_update_normals(MeshRenderData *mr,
+                                            const eMRIterType UNUSED(iter_type),
+                                            const eMRDataType data_flag)
+{
+  Mesh *me = mr->me;
+  const bool is_auto_smooth = (me->flag & ME_AUTOSMOOTH) != 0;
+  const float split_angle = is_auto_smooth ? me->smoothresh : (float)M_PI;
+
+  if (mr->extract_type != MR_EXTRACT_BMESH) {
+    /* Mesh */
+    if (data_flag & (MR_DATA_POLY_NOR | MR_DATA_LOOP_NOR | MR_DATA_TAN_LOOP_NOR)) {
+      mr->poly_normals = MEM_mallocN(sizeof(*mr->poly_normals) * mr->poly_len, __func__);
+      BKE_mesh_calc_normals_poly((MVert *)mr->mvert,
+                                 NULL,
+                                 mr->vert_len,
+                                 mr->mloop,
+                                 mr->mpoly,
+                                 mr->loop_len,
+                                 mr->poly_len,
+                                 mr->poly_normals,
+                                 true);
+    }
+    if (((data_flag & MR_DATA_LOOP_NOR) && is_auto_smooth) || (data_flag & MR_DATA_TAN_LOOP_NOR)) {
+      mr->loop_normals = MEM_mallocN(sizeof(*mr->loop_normals) * mr->loop_len, __func__);
+      short(*clnors)[2] = CustomData_get_layer(&mr->me->ldata, CD_CUSTOMLOOPNORMAL);
+      BKE_mesh_normals_loop_split(mr->me->mvert,
+                                  mr->vert_len,
+                                  mr->me->medge,
+                                  mr->edge_len,
+                                  mr->me->mloop,
+                                  mr->loop_normals,
+                                  mr->loop_len,
+                                  mr->me->mpoly,
+                                  mr->poly_normals,
+                                  mr->poly_len,
+                                  is_auto_smooth,
+                                  split_angle,
+                                  NULL,
+                                  clnors,
+                                  NULL);
+    }
+  }
+  else {
+    /* BMesh */
+    if (data_flag & MR_DATA_POLY_NOR) {
+      /* Use bmface->no instead. */
+    }
+    if (((data_flag & MR_DATA_LOOP_NOR) && is_auto_smooth) || (data_flag & MR_DATA_TAN_LOOP_NOR)) {
+
+      const float(*vert_coords)[3] = NULL;
+      const float(*vert_normals)[3] = NULL;
+      const float(*poly_normals)[3] = NULL;
+
+      if (mr->edit_data && mr->edit_data->vertexCos) {
+        vert_coords = mr->bm_vert_coords;
+        vert_normals = mr->bm_vert_normals;
+        poly_normals = mr->bm_poly_normals;
+      }
+
+      mr->loop_normals = MEM_mallocN(sizeof(*mr->loop_normals) * mr->loop_len, __func__);
+      int clnors_offset = CustomData_get_offset(&mr->bm->ldata, CD_CUSTOMLOOPNORMAL);
+      BM_loops_calc_normal_vcos(mr->bm,
+                                vert_coords,
+                                vert_normals,
+                                poly_normals,
+                                is_auto_smooth,
+                                split_angle,
+                                mr->loop_normals,
+                                NULL,
+                                NULL,
+                                clnors_offset,
+                                false);
+    }
+  }
+}
+
 static MeshRenderData *mesh_render_data_create(Mesh *me,
                                                const bool is_editmode,
                                                const bool is_paint_mode,
@@ -141,7 +318,9 @@ static MeshRenderData *mesh_render_data_create(Mesh *me,
                                                const bool do_final,
                                                const bool do_uvedit,
                                                const DRW_MeshCDMask *UNUSED(cd_used),
-                                               const ToolSettings *ts)
+                                               const ToolSettings *ts,
+                                               const eMRIterType iter_type,
+                                               const eMRDataType data_flag)
 {
   MeshRenderData *mr = MEM_callocN(sizeof(*mr), __func__);
   mr->toolsettings = ts;
@@ -249,165 +428,11 @@ static MeshRenderData *mesh_render_data_create(Mesh *me,
     mr->poly_len = bm->totface;
     mr->tri_len = poly_to_tri_count(mr->poly_len, mr->loop_len);
   }
+  mesh_render_data_update_loose_geom(mr, iter_type, data_flag);
 
   return mr;
 }
 
-/* Part of the creation of the MeshRenderData that happens in a thread. */
-static void mesh_render_data_update(MeshRenderData *mr,
-                                    const eMRIterType iter_type,
-                                    const eMRDataType data_flag)
-{
-  Mesh *me = mr->me;
-  const bool is_auto_smooth = (me->flag & ME_AUTOSMOOTH) != 0;
-  const float split_angle = is_auto_smooth ? me->smoothresh : (float)M_PI;
-
-  if (mr->extract_type != MR_EXTRACT_BMESH) {
-    /* Mesh */
-    if (data_flag & (MR_DATA_POLY_NOR | MR_DATA_LOOP_NOR | MR_DATA_TAN_LOOP_NOR)) {
-      mr->poly_normals = MEM_mallocN(sizeof(*mr->poly_normals) * mr->poly_len, __func__);
-      BKE_mesh_calc_normals_poly((MVert *)mr->mvert,
-                                 NULL,
-                                 mr->vert_len,
-                                 mr->mloop,
-                                 mr->mpoly,
-                                 mr->loop_len,
-                                 mr->poly_len,
-                                 mr->poly_normals,
-                                 true);
-    }
-    if (((data_flag & MR_DATA_LOOP_NOR) && is_auto_smooth) || (data_flag & MR_DATA_TAN_LOOP_NOR)) {
-      mr->loop_normals = MEM_mallocN(sizeof(*mr->loop_normals) * mr->loop_len, __func__);
-      short(*clnors)[2] = CustomData_get_layer(&mr->me->ldata, CD_CUSTOMLOOPNORMAL);
-      BKE_mesh_normals_loop_split(mr->me->mvert,
-                                  mr->vert_len,
-                                  mr->me->medge,
-                                  mr->edge_len,
-                                  mr->me->mloop,
-                                  mr->loop_normals,
-                                  mr->loop_len,
-                                  mr->me->mpoly,
-                                  mr->poly_normals,
-                                  mr->poly_len,
-                                  is_auto_smooth,
-                                  split_angle,
-                                  NULL,
-                                  clnors,
-                                  NULL);
-    }
-    if ((iter_type & MR_ITER_LOOPTRI) || (data_flag & MR_DATA_LOOPTRI)) {
-      mr->mlooptri = MEM_mallocN(sizeof(*mr->mlooptri) * mr->tri_len, "MR_DATATYPE_LOOPTRI");
-      BKE_mesh_recalc_looptri(mr->me->mloop,
-                              mr->me->mpoly,
-                              mr->me->mvert,
-                              mr->me->totloop,
-                              mr->me->totpoly,
-                              mr->mlooptri);
-    }
-    if (iter_type & (MR_ITER_LEDGE | MR_ITER_LVERT)) {
-      mr->vert_loose_len = 0;
-      mr->edge_loose_len = 0;
-
-      BLI_bitmap *lvert_map = BLI_BITMAP_NEW(mr->vert_len, "lvert map");
-
-      mr->ledges = MEM_mallocN(mr->edge_len * sizeof(int), __func__);
-      const MEdge *medge = mr->medge;
-      for (int e = 0; e < mr->edge_len; e++, medge++) {
-        if (medge->flag & ME_LOOSEEDGE) {
-          mr->ledges[mr->edge_loose_len++] = e;
-        }
-        /* Tag verts as not loose. */
-        BLI_BITMAP_ENABLE(lvert_map, medge->v1);
-        BLI_BITMAP_ENABLE(lvert_map, medge->v2);
-      }
-      if (mr->edge_loose_len < mr->edge_len) {
-        mr->ledges = MEM_reallocN(mr->ledges, mr->edge_loose_len * sizeof(*mr->ledges));
-      }
-
-      mr->lverts = MEM_mallocN(mr->vert_len * sizeof(*mr->lverts), __func__);
-      for (int v = 0; v < mr->vert_len; v++) {
-        if (!BLI_BITMAP_TEST(lvert_map, v)) {
-          mr->lverts[mr->vert_loose_len++] = v;
-        }
-      }
-      if (mr->vert_loose_len < mr->vert_len) {
-        mr->lverts = MEM_reallocN(mr->lverts, mr->vert_loose_len * sizeof(*mr->lverts));
-      }
-
-      MEM_freeN(lvert_map);
-
-      mr->loop_loose_len = mr->vert_loose_len + mr->edge_loose_len * 2;
-    }
-  }
-  else {
-    /* BMesh */
-    BMesh *bm = mr->bm;
-    if (data_flag & MR_DATA_POLY_NOR) {
-      /* Use bmface->no instead. */
-    }
-    if (((data_flag & MR_DATA_LOOP_NOR) && is_auto_smooth) || (data_flag & MR_DATA_TAN_LOOP_NOR)) {
-
-      const float(*vert_coords)[3] = NULL;
-      const float(*vert_normals)[3] = NULL;
-      const float(*poly_normals)[3] = NULL;
-
-      if (mr->edit_data && mr->edit_data->vertexCos) {
-        vert_coords = mr->bm_vert_coords;
-        vert_normals = mr->bm_vert_normals;
-        poly_normals = mr->bm_poly_normals;
-      }
-
-      mr->loop_normals = MEM_mallocN(sizeof(*mr->loop_normals) * mr->loop_len, __func__);
-      int clnors_offset = CustomData_get_offset(&mr->bm->ldata, CD_CUSTOMLOOPNORMAL);
-      BM_loops_calc_normal_vcos(mr->bm,
-                                vert_coords,
-                                vert_normals,
-                                poly_normals,
-                                is_auto_smooth,
-                                split_angle,
-                                mr->loop_normals,
-                                NULL,
-                                NULL,
-                                clnors_offset,
-                                false);
-    }
-    if ((iter_type & MR_ITER_LOOPTRI) || (data_flag & MR_DATA_LOOPTRI)) {
-      /* Edit mode ensures this is valid, no need to calculate. */
-      BLI_assert((bm->totloop == 0) || (mr->edit_bmesh->looptris != NULL));
-    }
-    if (iter_type & (MR_ITER_LEDGE | MR_ITER_LVERT)) {
-      int elem_id;
-      BMIter iter;
-      BMVert *eve;
-      BMEdge *ede;
-      mr->vert_loose_len = 0;
-      mr->edge_loose_len = 0;
-
-      mr->lverts = MEM_mallocN(mr->vert_len * sizeof(*mr->lverts), __func__);
-      BM_ITER_MESH_INDEX (eve, &iter, bm, BM_VERTS_OF_MESH, elem_id) {
-        if (eve->e == NULL) {
-          mr->lverts[mr->vert_loose_len++] = elem_id;
-        }
-      }
-      if (mr->vert_loose_len < mr->vert_len) {
-        mr->lverts = MEM_reallocN(mr->lverts, mr->vert_loose_len * sizeof(*mr->lverts));
-      }
-
-      mr->ledges = MEM_mallocN(mr->edge_len * sizeof(*mr->ledges), __func__);
-      BM_ITER_MESH_INDEX (ede, &iter, bm, BM_EDGES_OF_MESH, elem_id) {
-        if (ede->l == NULL) {
-          mr->ledges[mr->edge_loose_len++] = elem_id;
-        }
-      }
-      if (mr->edge_loose_len < mr->edge_len) {
-        mr->ledges = MEM_reallocN(mr->ledges, mr->edge_loose_len * sizeof(*mr->ledges));
-      }
-
-      mr->loop_loose_len = mr->vert_loose_len + mr->edge_loose_len * 2;
-    }
-  }
-}
-
 static void mesh_render_data_free(MeshRenderData *mr)
 {
   MEM_SAFE_FREE(mr->mlooptri);
@@ -4677,8 +4702,12 @@ static void mesh_render_data_update_task_data_free(MeshRenderDataUpdateTaskData
 static void mesh_extract_render_data_node_exec(void *__restrict task_data)
 {
   MeshRenderDataUpdateTaskData *update_task_data = task_data;
-  mesh_render_data_update(
-      update_task_data->mr, update_task_data->iter_type, update_task_data->data_flag);
+  MeshRenderData *mr = update_task_data->mr;
+  const eMRIterType iter_type = update_task_data->iter_type;
+  const eMRDataType data_flag = update_task_data->data_flag;
+
+  mesh_render_data_update_normals(mr, iter_type, data_flag);
+  mesh_render_data_update_looptris(mr, iter_type, data_flag);
 }
 
 static struct TaskNode *mesh_extract_render_data_node_create(struct TaskGraph *task_graph,
@@ -4986,8 +5015,16 @@ void mesh_buffer_cache_create_requested(struct TaskGraph *task_graph,
   double rdata_start = PIL_check_seconds_timer();
 #endif
 
-  MeshRenderData *mr = mesh_render_data_create(
-      me, is_editmode, is_paint_mode, obmat, do_final, do_uvedit, cd_layer_used, ts);
+  MeshRenderData *mr = mesh_render_data_create(me,
+                                               is_editmode,
+                                               is_paint_mode,
+                                               obmat,
+                                               do_final,
+                                               do_uvedit,
+                                               cd_layer_used,
+                                               ts,
+                                               iter_flag,
+                                               data_flag);
   mr->cache = cache; /* HACK */
   mr->use_hide = use_hide;
   mr->use_subsurf_fdots = use_subsurf_fdots;