Overlay Engine: Refactor & Cleanup

This is the unification of all overlays into one overlay engine as described in T65347.

I went over all the code making it more future proof with less hacks and removing old / not relevent parts.

Goals / Acheivements:
- Remove internal shader usage (only drw shaders)
- Remove viewportSize and viewportSizeInv and put them in gloabl ubo
- Fixed some drawing issues: Missing probe option and Missing Alt+B clipping of some shader
- Remove old (legacy) shaders dependancy (not using view UBO).
- Less shader variation (less compilation time at first load and less patching needed for vulkan)
- removed some geom shaders when I could
- Remove static e_data (except shaders storage where it is OK)
- Clear the way to fix some anoying limitations (dithered transparency, background image compositing etc...)
- Wireframe drawing now uses the same batching capabilities as workbench & eevee (indirect drawing).
- Reduced complexity, removed ~3000 Lines of code in draw (also removed a lot of unused shader in GPU).
- Post AA to avoid complexity and cost of MSAA.

Remaining issues:
- ~~Armature edits, overlay toggles, (... others?) are not refreshing viewport after AA is complete~~
- FXAA is not the best for wires, maybe investigate SMAA
- Maybe do something more temporally stable for AA.
- ~~Paint overlays are not working with AA.~~
- ~~infront objects are difficult to select.~~
- ~~the infront wires sometimes goes through they solid counterpart (missing clear maybe?) (toggle overlays on-off when using infront+wireframe overlay in solid shading)~~

Note: I made some decision to change slightly the appearance of some objects to simplify their drawing. Namely the empty arrows end (which is now hollow/wire) and distance points of the cameras/spots being done by lines.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D6296

4 files changed, 64 insertions, 119 deletions

diff --git a/source/blender/gpu/intern/gpu_batch.c b/source/blender/gpu/intern/gpu_batch.c
index 23692cb20ec..717983f94d8 100644
--- a/source/blender/gpu/intern/gpu_batch.c
+++ b/source/blender/gpu/intern/gpu_batch.c
@@ -103,7 +103,9 @@ void GPU_batch_init_ex(
   for (int v = 1; v < GPU_BATCH_VBO_MAX_LEN; v++) {
     batch->verts[v] = NULL;
   }
-  batch->inst = NULL;
+  for (int v = 0; v < GPU_BATCH_INST_VBO_MAX_LEN; v++) {
+    batch->inst[v] = NULL;
+  }
   batch->elem = elem;
   batch->gl_prim_type = convert_prim_type_to_gl(prim_type);
   batch->phase = GPU_BATCH_READY_TO_DRAW;
@@ -129,7 +131,8 @@ void GPU_batch_clear(GPUBatch *batch)
     GPU_indexbuf_discard(batch->elem);
   }
   if (batch->owns_flag & GPU_BATCH_OWNS_INSTANCES) {
-    GPU_vertbuf_discard(batch->inst);
+    GPU_vertbuf_discard(batch->inst[0]);
+    GPU_VERTBUF_DISCARD_SAFE(batch->inst[1]);
   }
   if ((batch->owns_flag & ~GPU_BATCH_OWNS_INDEX) != 0) {
     for (int v = 0; v < GPU_BATCH_VBO_MAX_LEN; v++) {
@@ -171,10 +174,11 @@ void GPU_batch_instbuf_set(GPUBatch *batch, GPUVertBuf *inst, bool own_vbo)
   /* redo the bindings */
   GPU_batch_vao_cache_clear(batch);
 
-  if (batch->inst != NULL && (batch->owns_flag & GPU_BATCH_OWNS_INSTANCES)) {
-    GPU_vertbuf_discard(batch->inst);
+  if (batch->inst[0] != NULL && (batch->owns_flag & GPU_BATCH_OWNS_INSTANCES)) {
+    GPU_vertbuf_discard(batch->inst[0]);
+    GPU_VERTBUF_DISCARD_SAFE(batch->inst[1]);
   }
-  batch->inst = inst;
+  batch->inst[0] = inst;
 
   if (own_vbo) {
     batch->owns_flag |= GPU_BATCH_OWNS_INSTANCES;
@@ -203,6 +207,37 @@ void GPU_batch_elembuf_set(GPUBatch *batch, GPUIndexBuf *elem, bool own_ibo)
   }
 }
 
+/* A bit of a quick hack. Should be streamlined as the vbos handling */
+int GPU_batch_instbuf_add_ex(GPUBatch *batch, GPUVertBuf *insts, bool own_vbo)
+{
+  /* redo the bindings */
+  GPU_batch_vao_cache_clear(batch);
+
+  for (uint v = 0; v < GPU_BATCH_INST_VBO_MAX_LEN; v++) {
+    if (batch->inst[v] == NULL) {
+#if TRUST_NO_ONE
+      /* for now all VertexBuffers must have same vertex_len */
+      if (batch->inst[0] != NULL) {
+        /* Allow for different size of vertex buf (will choose the smallest number of verts). */
+        // assert(insts->vertex_len == batch->inst[0]->vertex_len);
+        assert(own_vbo == ((batch->owns_flag & GPU_BATCH_OWNS_INSTANCES) != 0));
+      }
+#endif
+      batch->inst[v] = insts;
+      if (own_vbo) {
+        batch->owns_flag |= GPU_BATCH_OWNS_INSTANCES;
+      }
+      return v;
+    }
+  }
+
+  /* we only make it this far if there is no room for another GPUVertBuf */
+#if TRUST_NO_ONE
+  assert(false);
+#endif
+  return -1;
+}
+
 /* Returns the index of verts in the batch. */
 int GPU_batch_vertbuf_add_ex(GPUBatch *batch, GPUVertBuf *verts, bool own_vbo)
 {
@@ -458,8 +493,10 @@ static void batch_update_program_bindings(GPUBatch *batch, uint i_first)
       create_bindings(batch->verts[v], batch->interface, 0, false);
     }
   }
-  if (batch->inst) {
-    create_bindings(batch->inst, batch->interface, i_first, true);
+  for (int v = GPU_BATCH_INST_VBO_MAX_LEN - 1; v > -1; v--) {
+    if (batch->inst[v]) {
+      create_bindings(batch->inst[v], batch->interface, i_first, true);
+    }
   }
   if (batch->elem) {
     GPU_indexbuf_use(batch->elem);
@@ -639,7 +676,11 @@ void GPU_batch_draw_advanced(GPUBatch *batch, int v_first, int v_count, int i_fi
     v_count = (batch->elem) ? batch->elem->index_len : batch->verts[0]->vertex_len;
   }
   if (i_count == 0) {
-    i_count = (batch->inst) ? batch->inst->vertex_len : 1;
+    i_count = (batch->inst[0]) ? batch->inst[0]->vertex_len : 1;
+    /* Meh. This is to be able to use different numbers of verts in instance vbos. */
+    if (batch->inst[1] && i_count > batch->inst[1]->vertex_len) {
+      i_count = batch->inst[1]->vertex_len;
+    }
   }
 
   if (v_count == 0 || i_count == 0) {
diff --git a/source/blender/gpu/intern/gpu_shader.c b/source/blender/gpu/intern/gpu_shader.c
index 5df73d1a0c6..ae2c7864e36 100644
--- a/source/blender/gpu/intern/gpu_shader.c
+++ b/source/blender/gpu/intern/gpu_shader.c
@@ -102,20 +102,7 @@ extern char datatoc_gpu_shader_3D_smooth_color_frag_glsl[];
 extern char datatoc_gpu_shader_3D_passthrough_vert_glsl[];
 extern char datatoc_gpu_shader_3D_clipped_uniform_color_vert_glsl[];
 
-extern char datatoc_gpu_shader_instance_vert_glsl[];
 extern char datatoc_gpu_shader_instance_variying_size_variying_color_vert_glsl[];
-extern char datatoc_gpu_shader_instance_variying_size_variying_id_vert_glsl[];
-extern char datatoc_gpu_shader_instance_objectspace_variying_color_vert_glsl[];
-extern char datatoc_gpu_shader_instance_screenspace_variying_color_vert_glsl[];
-extern char datatoc_gpu_shader_instance_screen_aligned_vert_glsl[];
-extern char datatoc_gpu_shader_instance_camera_vert_glsl[];
-extern char datatoc_gpu_shader_instance_distance_line_vert_glsl[];
-extern char datatoc_gpu_shader_instance_edges_variying_color_geom_glsl[];
-extern char datatoc_gpu_shader_instance_edges_variying_color_vert_glsl[];
-extern char datatoc_gpu_shader_instance_mball_handles_vert_glsl[];
-
-extern char datatoc_gpu_shader_3D_groundpoint_vert_glsl[];
-extern char datatoc_gpu_shader_3D_groundline_geom_glsl[];
 
 extern char datatoc_gpu_shader_point_uniform_color_frag_glsl[];
 extern char datatoc_gpu_shader_point_uniform_color_aa_frag_glsl[];
@@ -141,14 +128,10 @@ extern char datatoc_gpu_shader_2D_edituvs_stretch_vert_glsl[];
 
 extern char datatoc_gpu_shader_2D_line_dashed_uniform_color_vert_glsl[];
 extern char datatoc_gpu_shader_2D_line_dashed_frag_glsl[];
-extern char datatoc_gpu_shader_2D_line_dashed_geom_glsl[];
 extern char datatoc_gpu_shader_3D_line_dashed_uniform_color_vert_glsl[];
 
 extern char datatoc_gpu_shader_text_vert_glsl[];
-extern char datatoc_gpu_shader_text_geom_glsl[];
 extern char datatoc_gpu_shader_text_frag_glsl[];
-extern char datatoc_gpu_shader_text_simple_vert_glsl[];
-extern char datatoc_gpu_shader_text_simple_geom_glsl[];
 extern char datatoc_gpu_shader_keyframe_diamond_vert_glsl[];
 extern char datatoc_gpu_shader_keyframe_diamond_frag_glsl[];
 
@@ -885,13 +868,6 @@ static const GPUShaderStages builtin_shader_stages[GPU_SHADER_BUILTIN_LEN] = {
     [GPU_SHADER_TEXT] =
         {
             .vert = datatoc_gpu_shader_text_vert_glsl,
-            .geom = datatoc_gpu_shader_text_geom_glsl,
-            .frag = datatoc_gpu_shader_text_frag_glsl,
-        },
-    [GPU_SHADER_TEXT_SIMPLE] =
-        {
-            .vert = datatoc_gpu_shader_text_simple_vert_glsl,
-            .geom = datatoc_gpu_shader_text_simple_geom_glsl,
             .frag = datatoc_gpu_shader_text_frag_glsl,
         },
     [GPU_SHADER_KEYFRAME_DIAMOND] =
@@ -1105,70 +1081,17 @@ static const GPUShaderStages builtin_shader_stages[GPU_SHADER_BUILTIN_LEN] = {
             .frag = datatoc_gpu_shader_uniform_color_frag_glsl,
         },
 
-    [GPU_SHADER_3D_GROUNDPOINT] =
-        {
-            .vert = datatoc_gpu_shader_3D_groundpoint_vert_glsl,
-            .frag = datatoc_gpu_shader_point_uniform_color_frag_glsl,
-        },
-    [GPU_SHADER_3D_GROUNDLINE] =
-        {
-            .vert = datatoc_gpu_shader_3D_passthrough_vert_glsl,
-            .geom = datatoc_gpu_shader_3D_groundline_geom_glsl,
-            .frag = datatoc_gpu_shader_uniform_color_frag_glsl,
-        },
-
     [GPU_SHADER_2D_LINE_DASHED_UNIFORM_COLOR] =
         {
             .vert = datatoc_gpu_shader_2D_line_dashed_uniform_color_vert_glsl,
-            .geom = datatoc_gpu_shader_2D_line_dashed_geom_glsl,
             .frag = datatoc_gpu_shader_2D_line_dashed_frag_glsl,
         },
     [GPU_SHADER_3D_LINE_DASHED_UNIFORM_COLOR] =
         {
             .vert = datatoc_gpu_shader_3D_line_dashed_uniform_color_vert_glsl,
-            .geom = datatoc_gpu_shader_2D_line_dashed_geom_glsl,
             .frag = datatoc_gpu_shader_2D_line_dashed_frag_glsl,
         },
 
-    [GPU_SHADER_3D_OBJECTSPACE_SIMPLE_LIGHTING_VARIYING_COLOR] =
-        {
-            .vert = datatoc_gpu_shader_instance_objectspace_variying_color_vert_glsl,
-            .frag = datatoc_gpu_shader_simple_lighting_frag_glsl,
-            .defs = "#define USE_INSTANCE_COLOR\n",
-        },
-    [GPU_SHADER_3D_OBJECTSPACE_VARIYING_COLOR] =
-        {
-            .vert = datatoc_gpu_shader_instance_objectspace_variying_color_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-    [GPU_SHADER_3D_SCREENSPACE_VARIYING_COLOR] =
-        {
-            .vert = datatoc_gpu_shader_instance_screenspace_variying_color_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-    [GPU_SHADER_3D_INSTANCE_SCREEN_ALIGNED_AXIS] =
-        {
-            .vert = datatoc_gpu_shader_instance_screen_aligned_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-            .defs = "#define AXIS_NAME\n",
-        },
-    [GPU_SHADER_3D_INSTANCE_SCREEN_ALIGNED] =
-        {
-            .vert = datatoc_gpu_shader_instance_screen_aligned_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-
-    [GPU_SHADER_CAMERA] =
-        {
-            .vert = datatoc_gpu_shader_instance_camera_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-    [GPU_SHADER_DISTANCE_LINES] =
-        {
-            .vert = datatoc_gpu_shader_instance_distance_line_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-
     [GPU_SHADER_2D_POINT_FIXED_SIZE_UNIFORM_COLOR] =
         {
             .vert = datatoc_gpu_shader_2D_vert_glsl,
@@ -1225,34 +1148,12 @@ static const GPUShaderStages builtin_shader_stages[GPU_SHADER_BUILTIN_LEN] = {
             .frag = datatoc_gpu_shader_point_uniform_color_outline_aa_frag_glsl,
         },
 
-    [GPU_SHADER_INSTANCE_UNIFORM_COLOR] =
-        {
-            .vert = datatoc_gpu_shader_instance_vert_glsl,
-            .frag = datatoc_gpu_shader_uniform_color_frag_glsl,
-        },
-    [GPU_SHADER_INSTANCE_VARIYING_ID_VARIYING_SIZE] =
-        {
-            .vert = datatoc_gpu_shader_instance_variying_size_variying_id_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_id_frag_glsl,
-            .defs = "#define UNIFORM_SCALE\n",
-        },
     [GPU_SHADER_INSTANCE_VARIYING_COLOR_VARIYING_SIZE] =
         {
             .vert = datatoc_gpu_shader_instance_variying_size_variying_color_vert_glsl,
             .frag = datatoc_gpu_shader_flat_color_frag_glsl,
             .defs = "#define UNIFORM_SCALE\n",
         },
-    [GPU_SHADER_INSTANCE_VARIYING_COLOR_VARIYING_SCALE] =
-        {
-            .vert = datatoc_gpu_shader_instance_variying_size_variying_color_vert_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
-    [GPU_SHADER_INSTANCE_EDGES_VARIYING_COLOR] =
-        {
-            .vert = datatoc_gpu_shader_instance_edges_variying_color_vert_glsl,
-            .geom = datatoc_gpu_shader_instance_edges_variying_color_geom_glsl,
-            .frag = datatoc_gpu_shader_flat_color_frag_glsl,
-        },
 
     [GPU_SHADER_2D_AREA_EDGES] =
         {
@@ -1366,22 +1267,11 @@ GPUShader *GPU_shader_get_builtin_shader_with_config(eGPUBuiltinShader shader,
                       GPU_SHADER_3D_UNIFORM_COLOR,
                       GPU_SHADER_3D_SMOOTH_COLOR,
                       GPU_SHADER_3D_DEPTH_ONLY,
-                      GPU_SHADER_CAMERA,
                       GPU_SHADER_INSTANCE_VARIYING_COLOR_VARIYING_SIZE,
-                      GPU_SHADER_INSTANCE_VARIYING_COLOR_VARIYING_SCALE,
                       GPU_SHADER_3D_POINT_UNIFORM_SIZE_UNIFORM_COLOR_OUTLINE_AA,
                       GPU_SHADER_3D_POINT_UNIFORM_SIZE_UNIFORM_COLOR_AA,
-                      GPU_SHADER_3D_SCREENSPACE_VARIYING_COLOR,
-                      GPU_SHADER_3D_INSTANCE_SCREEN_ALIGNED,
-
-                      GPU_SHADER_3D_GROUNDLINE,
-                      GPU_SHADER_3D_GROUNDPOINT,
-                      GPU_SHADER_DISTANCE_LINES,
-                      GPU_SHADER_INSTANCE_EDGES_VARIYING_COLOR) ||
-                 ELEM(shader,
                       GPU_SHADER_3D_FLAT_COLOR,
-                      GPU_SHADER_3D_LINE_DASHED_UNIFORM_COLOR,
-                      GPU_SHADER_INSTANCE_VARIYING_ID_VARIYING_SIZE));
+                      GPU_SHADER_3D_LINE_DASHED_UNIFORM_COLOR));
       const char *world_clip_lib = datatoc_gpu_shader_cfg_world_clip_lib_glsl;
       const char *world_clip_def = "#define USE_WORLD_CLIP_PLANES\n";
       /* In rare cases geometry shaders calculate clipping themselves. */
diff --git a/source/blender/gpu/intern/gpu_vertex_buffer.c b/source/blender/gpu/intern/gpu_vertex_buffer.c
index eecbcb9ec94..1df7e68e08b 100644
--- a/source/blender/gpu/intern/gpu_vertex_buffer.c
+++ b/source/blender/gpu/intern/gpu_vertex_buffer.c
@@ -196,6 +196,19 @@ void GPU_vertbuf_attr_fill(GPUVertBuf *verts, uint a_idx, const void *data)
   GPU_vertbuf_attr_fill_stride(verts, a_idx, stride, data);
 }
 
+/** Fills a whole vertex (all attribs). Data must match packed layout.  */
+void GPU_vertbuf_vert_set(GPUVertBuf *verts, uint v_idx, const void *data)
+{
+  const GPUVertFormat *format = &verts->format;
+
+#if TRUST_NO_ONE
+  assert(v_idx < verts->vertex_alloc);
+  assert(verts->data != NULL);
+#endif
+  verts->dirty = true;
+  memcpy((GLubyte *)verts->data + v_idx * format->stride, data, format->stride);
+}
+
 void GPU_vertbuf_attr_fill_stride(GPUVertBuf *verts, uint a_idx, uint stride, const void *data)
 {
   const GPUVertFormat *format = &verts->format;
diff --git a/source/blender/gpu/intern/gpu_viewport.c b/source/blender/gpu/intern/gpu_viewport.c
index 519cd918b53..972ff87ebcf 100644
--- a/source/blender/gpu/intern/gpu_viewport.c
+++ b/source/blender/gpu/intern/gpu_viewport.c
@@ -145,6 +145,7 @@ GPUViewport *GPU_viewport_create_from_offscreen(struct GPUOffScreen *ofs)
     GPU_framebuffer_ensure_config(
         &viewport->fbl->depth_only_fb,
         {GPU_ATTACHMENT_TEXTURE(viewport->txl->depth), GPU_ATTACHMENT_NONE});
+    /* TODO infront buffer */
   }
 
   return viewport;