Fix T95177: GPU subdiv crashes mirror modifier in edit-mode

The issue has two causes: on one hand origin indices were not handled
properly, on the other hand the extraction type (Mesh, BMesh, or mapped)
was not detected correctly.

For the second case reuse the MeshRenderData creation from the coarse
code path so that we make the same decisions. Loose geometry extraction
had to be updated to properly handle the BMesh cases.

For the origin indices, in some cases (for edges and faces), the arrays
used by the subdivision code already have the origin indices baked into
them, so mapping them a second time through the origin index layer is
wrong, and could cause out of bounds accesses.

For vertices especially, we would use two arrays: one for mapping
subdivision vertices to coarse vertices, and another one to map coarse
vertices to subdivision loops used for the selection index buffer. The
second one is now removed (which saves a bit of memory) as it is did not
have the proper data setup for use with the origin indices and we can
easily compute it using the first array anyway.

1 files changed, 60 insertions, 26 deletions

diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
index 5d2ea923658..c92cf554fda 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
@@ -265,9 +265,6 @@ static void extract_pos_nor_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache
   }
 
   GPUVertBuf *vbo = static_cast<GPUVertBuf *>(buffer);
-  const Mesh *coarse_mesh = subdiv_cache->mesh;
-  const MEdge *coarse_edges = coarse_mesh->medge;
-  const MVert *coarse_verts = coarse_mesh->mvert;
   uint offset = subdiv_cache->num_subdiv_loops;
 
   /* TODO(kevindietrich) : replace this when compressed normals are supported. */
@@ -277,38 +274,75 @@ static void extract_pos_nor_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache
     float flag;
   };
 
-  SubdivPosNorLoop edge_data[2];
-  for (int i = 0; i < loose_geom->edge_len; i++) {
-    const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
-    const MVert *loose_vert1 = &coarse_verts[loose_edge->v1];
-    const MVert *loose_vert2 = &coarse_verts[loose_edge->v2];
+  if (mr->extract_type == MR_EXTRACT_MESH) {
+    const Mesh *coarse_mesh = subdiv_cache->mesh;
+    const MEdge *coarse_edges = coarse_mesh->medge;
+    const MVert *coarse_verts = coarse_mesh->mvert;
 
-    copy_v3_v3(edge_data[0].pos, loose_vert1->co);
-    copy_v3_v3(edge_data[0].nor, mr->vert_normals[loose_edge->v1]);
-    edge_data[0].flag = 0.0f;
+    SubdivPosNorLoop edge_data[2];
+    memset(&edge_data, 0, sizeof(SubdivPosNorLoop) * 2);
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
+      const MVert *loose_vert1 = &coarse_verts[loose_edge->v1];
+      const MVert *loose_vert2 = &coarse_verts[loose_edge->v2];
 
-    copy_v3_v3(edge_data[1].pos, loose_vert2->co);
-    copy_v3_v3(edge_data[1].nor, mr->vert_normals[loose_edge->v2]);
-    edge_data[1].flag = 0.0f;
+      copy_v3_v3(edge_data[0].pos, loose_vert1->co);
+      copy_v3_v3(edge_data[1].pos, loose_vert2->co);
 
-    GPU_vertbuf_update_sub(
-        vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
 
-    offset += 2;
+      offset += 2;
+    }
+
+    SubdivPosNorLoop vert_data;
+    memset(&vert_data, 0, sizeof(SubdivPosNorLoop));
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const MVert *loose_vertex = &coarse_verts[loose_geom->verts[i]];
+
+      copy_v3_v3(vert_data.pos, loose_vertex->co);
+
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+
+      offset += 1;
+    }
   }
+  else {
+    BMesh *bm = subdiv_cache->bm;
 
-  SubdivPosNorLoop vert_data;
-  vert_data.flag = 0.0f;
-  for (int i = 0; i < loose_geom->vert_len; i++) {
-    const MVert *loose_vertex = &coarse_verts[loose_geom->verts[i]];
+    SubdivPosNorLoop edge_data[2];
+    memset(&edge_data, 0, sizeof(SubdivPosNorLoop) * 2);
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const BMEdge *loose_edge = BM_edge_at_index(bm, loose_geom->edges[i]);
+      const BMVert *loose_vert1 = loose_edge->v1;
+      const BMVert *loose_vert2 = loose_edge->v2;
 
-    copy_v3_v3(vert_data.pos, loose_vertex->co);
-    copy_v3_v3(vert_data.nor, mr->vert_normals[loose_geom->verts[i]]);
+      copy_v3_v3(edge_data[0].pos, loose_vert1->co);
+      copy_v3_v3(edge_data[0].nor, loose_vert1->no);
 
-    GPU_vertbuf_update_sub(
-        vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+      copy_v3_v3(edge_data[1].pos, loose_vert2->co);
+      copy_v3_v3(edge_data[1].nor, loose_vert2->no);
 
-    offset += 1;
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
+
+      offset += 2;
+    }
+
+    SubdivPosNorLoop vert_data;
+    memset(&vert_data, 0, sizeof(SubdivPosNorLoop));
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const BMVert *loose_vertex = BM_vert_at_index(bm, loose_geom->verts[i]);
+
+      copy_v3_v3(vert_data.pos, loose_vertex->co);
+      copy_v3_v3(vert_data.nor, loose_vertex->no);
+
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+
+      offset += 1;
+    }
   }
 }