Fix T95177: GPU subdiv crashes mirror modifier in edit-mode

The issue has two causes: on one hand origin indices were not handled
properly, on the other hand the extraction type (Mesh, BMesh, or mapped)
was not detected correctly.

For the second case reuse the MeshRenderData creation from the coarse
code path so that we make the same decisions. Loose geometry extraction
had to be updated to properly handle the BMesh cases.

For the origin indices, in some cases (for edges and faces), the arrays
used by the subdivision code already have the origin indices baked into
them, so mapping them a second time through the origin index layer is
wrong, and could cause out of bounds accesses.

For vertices especially, we would use two arrays: one for mapping
subdivision vertices to coarse vertices, and another one to map coarse
vertices to subdivision loops used for the selection index buffer. The
second one is now removed (which saves a bit of memory) as it is did not
have the proper data setup for use with the origin indices and we can
easily compute it using the first array anyway.

5 files changed, 205 insertions, 71 deletions

diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
index c3f89ab96ee..2bb4cc0fb47 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_ibo_points.cc
@@ -163,16 +163,55 @@ static void extract_points_init_subdiv(const DRWSubdivCache *subdiv_cache,
                                        void *data)
 {
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(data);
-  /* Copy the points as the data upload will free them. */
-  elb->data = (uint *)MEM_dupallocN(subdiv_cache->point_indices);
-  elb->index_len = mr->vert_len;
-  elb->index_min = 0;
-  elb->index_max = subdiv_cache->num_subdiv_loops + mr->loop_loose_len;
-  elb->prim_type = GPU_PRIM_POINTS;
+  GPU_indexbuf_init(
+      elb, GPU_PRIM_POINTS, mr->vert_len, subdiv_cache->num_subdiv_loops + mr->loop_loose_len);
+}
+
+static void extract_points_iter_subdiv_common(GPUIndexBufBuilder *elb,
+                                              const MeshRenderData *mr,
+                                              const DRWSubdivCache *subdiv_cache,
+                                              uint subdiv_quad_index)
+{
+  int *subdiv_loop_vert_index = (int *)GPU_vertbuf_get_data(subdiv_cache->verts_orig_index);
+  uint start_loop_idx = subdiv_quad_index * 4;
+  uint end_loop_idx = (subdiv_quad_index + 1) * 4;
+  for (uint i = start_loop_idx; i < end_loop_idx; i++) {
+    int coarse_vertex_index = subdiv_loop_vert_index[i];
+
+    if (coarse_vertex_index == -1) {
+      continue;
+    }
+
+    if (mr->v_origindex && mr->v_origindex[coarse_vertex_index] == -1) {
+      continue;
+    }
+
+    GPU_indexbuf_set_point_vert(elb, coarse_vertex_index, i);
+  }
+}
+
+static void extract_points_iter_subdiv_bm(const DRWSubdivCache *subdiv_cache,
+                                          const MeshRenderData *mr,
+                                          void *_data,
+                                          uint subdiv_quad_index,
+                                          const BMFace *UNUSED(coarse_quad))
+{
+  GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
+}
+
+static void extract_points_iter_subdiv_mesh(const DRWSubdivCache *subdiv_cache,
+                                            const MeshRenderData *mr,
+                                            void *_data,
+                                            uint subdiv_quad_index,
+                                            const MPoly *UNUSED(coarse_quad))
+{
+  GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(_data);
+  extract_points_iter_subdiv_common(elb, mr, subdiv_cache, subdiv_quad_index);
 }
 
 static void extract_points_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
-                                             const MeshRenderData *UNUSED(mr),
+                                             const MeshRenderData *mr,
                                              const MeshExtractLooseGeom *loose_geom,
                                              void *UNUSED(buffer),
                                              void *data)
@@ -183,28 +222,38 @@ static void extract_points_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
   }
 
   GPUIndexBufBuilder *elb = static_cast<GPUIndexBufBuilder *>(data);
-
-  const Mesh *coarse_mesh = subdiv_cache->mesh;
-  const MEdge *coarse_edges = coarse_mesh->medge;
-
   uint offset = subdiv_cache->num_subdiv_loops;
 
-  for (int i = 0; i < loose_geom->edge_len; i++) {
-    const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
-    if (elb->data[loose_edge->v1] == -1u) {
-      GPU_indexbuf_set_point_vert(elb, loose_edge->v1, offset);
+  if (mr->extract_type == MR_EXTRACT_MESH) {
+    const Mesh *coarse_mesh = subdiv_cache->mesh;
+    const MEdge *coarse_edges = coarse_mesh->medge;
+
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
+      vert_set_mesh(elb, mr, loose_edge->v1, offset);
+      vert_set_mesh(elb, mr, loose_edge->v2, offset + 1);
+      offset += 2;
     }
-    if (elb->data[loose_edge->v2] == -1u) {
-      GPU_indexbuf_set_point_vert(elb, loose_edge->v2, offset + 1);
+
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      vert_set_mesh(elb, mr, loose_geom->verts[i], offset);
+      offset += 1;
     }
-    offset += 2;
   }
+  else {
+    BMesh *bm = mr->bm;
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const BMEdge *loose_edge = BM_edge_at_index(bm, loose_geom->edges[i]);
+      vert_set_bm(elb, loose_edge->v1, offset);
+      vert_set_bm(elb, loose_edge->v2, offset + 1);
+      offset += 2;
+    }
 
-  for (int i = 0; i < loose_geom->vert_len; i++) {
-    if (elb->data[loose_geom->verts[i]] == -1u) {
-      GPU_indexbuf_set_point_vert(elb, loose_geom->verts[i], offset);
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const BMVert *loose_vert = BM_vert_at_index(bm, loose_geom->verts[i]);
+      vert_set_bm(elb, loose_vert, offset);
+      offset += 1;
     }
-    offset += 1;
   }
 }
 
@@ -232,6 +281,8 @@ constexpr MeshExtract create_extractor_points()
   extractor.task_reduce = extract_points_task_reduce;
   extractor.finish = extract_points_finish;
   extractor.init_subdiv = extract_points_init_subdiv;
+  extractor.iter_subdiv_bm = extract_points_iter_subdiv_bm;
+  extractor.iter_subdiv_mesh = extract_points_iter_subdiv_mesh;
   extractor.iter_loose_geom_subdiv = extract_points_loose_geom_subdiv;
   extractor.finish_subdiv = extract_points_finish_subdiv;
   extractor.use_threading = true;
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edit_data.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edit_data.cc
index 0002b95c867..046e4b9e4bb 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edit_data.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edit_data.cc
@@ -288,21 +288,24 @@ static void extract_edit_data_iter_subdiv_bm(const DRWSubdivCache *subdiv_cache,
     memset(edit_loop_data, 0, sizeof(EditLoopData));
 
     if (vert_origindex != -1) {
-      const BMVert *eve = bm_original_vert_get(mr, vert_origindex);
+      const BMVert *eve = mr->v_origindex ? bm_original_vert_get(mr, vert_origindex) :
+                                            BM_vert_at_index(mr->bm, vert_origindex);
       if (eve) {
         mesh_render_data_vert_flag(mr, eve, edit_loop_data);
       }
     }
 
     if (edge_origindex != -1) {
-      const BMEdge *eed = bm_original_edge_get(mr, edge_origindex);
-      if (eed) {
-        mesh_render_data_edge_flag(mr, eed, edit_loop_data);
-      }
+      /* NOTE: #subdiv_loop_edge_index already has the origindex layer baked in. */
+      const BMEdge *eed = BM_edge_at_index(mr->bm, edge_origindex);
+      mesh_render_data_edge_flag(mr, eed, edit_loop_data);
     }
 
-    /* The -1 parameter is for edit_uvs, which we don't do here. */
-    mesh_render_data_face_flag(mr, coarse_quad, -1, edit_loop_data);
+    /* coarse_quad can be null when called by the mesh iteration below. */
+    if (coarse_quad) {
+      /* The -1 parameter is for edit_uvs, which we don't do here. */
+      mesh_render_data_face_flag(mr, coarse_quad, -1, edit_loop_data);
+    }
   }
 }
 
@@ -333,7 +336,9 @@ static void extract_edit_data_loose_geom_subdiv(const DRWSubdivCache *subdiv_cac
     const int offset = subdiv_cache->num_subdiv_loops + ledge_index * 2;
     EditLoopData *data = &vbo_data[offset];
     memset(data, 0, sizeof(EditLoopData));
-    BMEdge *eed = bm_original_edge_get(mr, loose_geom->edges[ledge_index]);
+    const int edge_index = loose_geom->edges[ledge_index];
+    BMEdge *eed = mr->e_origindex ? bm_original_edge_get(mr, edge_index) :
+                                    BM_edge_at_index(mr->bm, edge_index);
     mesh_render_data_edge_flag(mr, eed, &data[0]);
     data[1] = data[0];
     mesh_render_data_vert_flag(mr, eed->v1, &data[0]);
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edituv_data.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edituv_data.cc
index b25e40690c9..3b0233ac267 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edituv_data.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_edituv_data.cc
@@ -159,7 +159,7 @@ static void extract_edituv_data_iter_subdiv_bm(const DRWSubdivCache *subdiv_cach
     memset(edit_loop_data, 0, sizeof(EditLoopData));
 
     if (vert_origindex != -1 && edge_origindex != -1) {
-      BMEdge *eed = bm_original_edge_get(mr, edge_origindex);
+      BMEdge *eed = BM_edge_at_index(mr->bm, edge_origindex);
       /* Loop on an edge endpoint. */
       BMLoop *l = BM_face_edge_share_loop(const_cast<BMFace *>(coarse_quad), eed);
       mesh_render_data_loop_flag(mr, l, data->cd_ofs, edit_loop_data);
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
index 5d2ea923658..c92cf554fda 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_pos_nor.cc
@@ -265,9 +265,6 @@ static void extract_pos_nor_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache
   }
 
   GPUVertBuf *vbo = static_cast<GPUVertBuf *>(buffer);
-  const Mesh *coarse_mesh = subdiv_cache->mesh;
-  const MEdge *coarse_edges = coarse_mesh->medge;
-  const MVert *coarse_verts = coarse_mesh->mvert;
   uint offset = subdiv_cache->num_subdiv_loops;
 
   /* TODO(kevindietrich) : replace this when compressed normals are supported. */
@@ -277,38 +274,75 @@ static void extract_pos_nor_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache
     float flag;
   };
 
-  SubdivPosNorLoop edge_data[2];
-  for (int i = 0; i < loose_geom->edge_len; i++) {
-    const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
-    const MVert *loose_vert1 = &coarse_verts[loose_edge->v1];
-    const MVert *loose_vert2 = &coarse_verts[loose_edge->v2];
+  if (mr->extract_type == MR_EXTRACT_MESH) {
+    const Mesh *coarse_mesh = subdiv_cache->mesh;
+    const MEdge *coarse_edges = coarse_mesh->medge;
+    const MVert *coarse_verts = coarse_mesh->mvert;
 
-    copy_v3_v3(edge_data[0].pos, loose_vert1->co);
-    copy_v3_v3(edge_data[0].nor, mr->vert_normals[loose_edge->v1]);
-    edge_data[0].flag = 0.0f;
+    SubdivPosNorLoop edge_data[2];
+    memset(&edge_data, 0, sizeof(SubdivPosNorLoop) * 2);
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
+      const MVert *loose_vert1 = &coarse_verts[loose_edge->v1];
+      const MVert *loose_vert2 = &coarse_verts[loose_edge->v2];
 
-    copy_v3_v3(edge_data[1].pos, loose_vert2->co);
-    copy_v3_v3(edge_data[1].nor, mr->vert_normals[loose_edge->v2]);
-    edge_data[1].flag = 0.0f;
+      copy_v3_v3(edge_data[0].pos, loose_vert1->co);
+      copy_v3_v3(edge_data[1].pos, loose_vert2->co);
 
-    GPU_vertbuf_update_sub(
-        vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
 
-    offset += 2;
+      offset += 2;
+    }
+
+    SubdivPosNorLoop vert_data;
+    memset(&vert_data, 0, sizeof(SubdivPosNorLoop));
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const MVert *loose_vertex = &coarse_verts[loose_geom->verts[i]];
+
+      copy_v3_v3(vert_data.pos, loose_vertex->co);
+
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+
+      offset += 1;
+    }
   }
+  else {
+    BMesh *bm = subdiv_cache->bm;
 
-  SubdivPosNorLoop vert_data;
-  vert_data.flag = 0.0f;
-  for (int i = 0; i < loose_geom->vert_len; i++) {
-    const MVert *loose_vertex = &coarse_verts[loose_geom->verts[i]];
+    SubdivPosNorLoop edge_data[2];
+    memset(&edge_data, 0, sizeof(SubdivPosNorLoop) * 2);
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const BMEdge *loose_edge = BM_edge_at_index(bm, loose_geom->edges[i]);
+      const BMVert *loose_vert1 = loose_edge->v1;
+      const BMVert *loose_vert2 = loose_edge->v2;
 
-    copy_v3_v3(vert_data.pos, loose_vertex->co);
-    copy_v3_v3(vert_data.nor, mr->vert_normals[loose_geom->verts[i]]);
+      copy_v3_v3(edge_data[0].pos, loose_vert1->co);
+      copy_v3_v3(edge_data[0].nor, loose_vert1->no);
 
-    GPU_vertbuf_update_sub(
-        vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+      copy_v3_v3(edge_data[1].pos, loose_vert2->co);
+      copy_v3_v3(edge_data[1].nor, loose_vert2->no);
 
-    offset += 1;
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop) * 2, &edge_data);
+
+      offset += 2;
+    }
+
+    SubdivPosNorLoop vert_data;
+    memset(&vert_data, 0, sizeof(SubdivPosNorLoop));
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const BMVert *loose_vertex = BM_vert_at_index(bm, loose_geom->verts[i]);
+
+      copy_v3_v3(vert_data.pos, loose_vertex->co);
+      copy_v3_v3(vert_data.nor, loose_vertex->no);
+
+      GPU_vertbuf_update_sub(
+          vbo, offset * sizeof(SubdivPosNorLoop), sizeof(SubdivPosNorLoop), &vert_data);
+
+      offset += 1;
+    }
   }
 }
 
diff --git a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_select_idx.cc b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_select_idx.cc
index 33c27b45627..8d3df75b782 100644
--- a/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_select_idx.cc
+++ b/source/blender/draw/intern/mesh_extractors/extract_mesh_vbo_select_idx.cc
@@ -206,13 +206,28 @@ static void extract_vert_idx_init_subdiv(const DRWSubdivCache *subdiv_cache,
   GPUVertBuf *vbo = static_cast<GPUVertBuf *>(buf);
   /* Each element points to an element in the ibo.points. */
   draw_subdiv_init_origindex_buffer(vbo,
-                                    subdiv_cache->subdiv_loop_subdiv_vert_index,
+                                    (int *)GPU_vertbuf_get_data(subdiv_cache->verts_orig_index),
                                     subdiv_cache->num_subdiv_loops,
                                     mr->loop_loose_len);
+
+  if (!mr->v_origindex) {
+    return;
+  }
+
+  /* Remap the vertex indices to those pointed by the origin indices layer. At this point, the
+   * VBO data is a copy of #verts_orig_index which contains the coarse vertices indices, so
+   * the memory can both be accessed for lookup and immediately overwritten. */
+  int *vbo_data = static_cast<int *>(GPU_vertbuf_get_data(vbo));
+  for (int i = 0; i < subdiv_cache->num_subdiv_loops; i++) {
+    if (vbo_data[i] == -1) {
+      continue;
+    }
+    vbo_data[i] = mr->v_origindex[vbo_data[i]];
+  }
 }
 
 static void extract_vert_idx_loose_geom_subdiv(const DRWSubdivCache *subdiv_cache,
-                                               const MeshRenderData *UNUSED(mr),
+                                               const MeshRenderData *mr,
                                                const MeshExtractLooseGeom *loose_geom,
                                                void *buffer,
                                                void *UNUSED(data))
@@ -224,20 +239,37 @@ static void extract_vert_idx_loose_geom_subdiv(const DRWSubdivCache *subdiv_cach
 
   GPUVertBuf *vbo = static_cast<GPUVertBuf *>(buffer);
   uint *vert_idx_data = (uint *)GPU_vertbuf_get_data(vbo);
-  const Mesh *coarse_mesh = subdiv_cache->mesh;
-  const MEdge *coarse_edges = coarse_mesh->medge;
   uint offset = subdiv_cache->num_subdiv_loops;
 
-  for (int i = 0; i < loose_geom->edge_len; i++) {
-    const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
-    vert_idx_data[offset] = loose_edge->v1;
-    vert_idx_data[offset + 1] = loose_edge->v2;
-    offset += 2;
+  if (mr->extract_type == MR_EXTRACT_MESH) {
+    const Mesh *coarse_mesh = subdiv_cache->mesh;
+    const MEdge *coarse_edges = coarse_mesh->medge;
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const MEdge *loose_edge = &coarse_edges[loose_geom->edges[i]];
+      vert_idx_data[offset] = loose_edge->v1;
+      vert_idx_data[offset + 1] = loose_edge->v2;
+      offset += 2;
+    }
+
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      vert_idx_data[offset] = loose_geom->verts[i];
+      offset += 1;
+    }
   }
-
-  for (int i = 0; i < loose_geom->vert_len; i++) {
-    vert_idx_data[offset] = loose_geom->verts[i];
-    offset += 1;
+  else {
+    BMesh *bm = mr->bm;
+    for (int i = 0; i < loose_geom->edge_len; i++) {
+      const BMEdge *loose_edge = BM_edge_at_index(bm, loose_geom->edges[i]);
+      vert_idx_data[offset] = BM_elem_index_get(loose_edge->v1);
+      vert_idx_data[offset + 1] = BM_elem_index_get(loose_edge->v2);
+      offset += 2;
+    }
+
+    for (int i = 0; i < loose_geom->vert_len; i++) {
+      const BMVert *loose_vert = BM_vert_at_index(bm, loose_geom->verts[i]);
+      vert_idx_data[offset] = BM_elem_index_get(loose_vert);
+      offset += 1;
+    }
   }
 }
 
@@ -278,7 +310,7 @@ static void extract_edge_idx_loose_geom_subdiv(const DRWSubdivCache *subdiv_cach
 }
 
 static void extract_poly_idx_init_subdiv(const DRWSubdivCache *subdiv_cache,
-                                         const MeshRenderData *UNUSED(mr),
+                                         const MeshRenderData *mr,
                                          MeshBatchCache *UNUSED(cache),
                                          void *buf,
                                          void *UNUSED(data))
@@ -286,6 +318,18 @@ static void extract_poly_idx_init_subdiv(const DRWSubdivCache *subdiv_cache,
   GPUVertBuf *vbo = static_cast<GPUVertBuf *>(buf);
   draw_subdiv_init_origindex_buffer(
       vbo, subdiv_cache->subdiv_loop_poly_index, subdiv_cache->num_subdiv_loops, 0);
+
+  if (!mr->p_origindex) {
+    return;
+  }
+
+  /* Remap the polygon indices to those pointed by the origin indices layer. At this point, the
+   * VBO data is a copy of #subdiv_loop_poly_index which contains the coarse polygon indices, so
+   * the memory can both be accessed for lookup and immediately overwritten. */
+  int *vbo_data = static_cast<int *>(GPU_vertbuf_get_data(vbo));
+  for (int i = 0; i < subdiv_cache->num_subdiv_loops; i++) {
+    vbo_data[i] = mr->p_origindex[vbo_data[i]];
+  }
 }
 
 constexpr MeshExtract create_extractor_poly_idx()