Overlay Engine: Refactor & Cleanup

This is the unification of all overlays into one overlay engine as described in T65347.

I went over all the code making it more future proof with less hacks and removing old / not relevent parts.

Goals / Acheivements:
- Remove internal shader usage (only drw shaders)
- Remove viewportSize and viewportSizeInv and put them in gloabl ubo
- Fixed some drawing issues: Missing probe option and Missing Alt+B clipping of some shader
- Remove old (legacy) shaders dependancy (not using view UBO).
- Less shader variation (less compilation time at first load and less patching needed for vulkan)
- removed some geom shaders when I could
- Remove static e_data (except shaders storage where it is OK)
- Clear the way to fix some anoying limitations (dithered transparency, background image compositing etc...)
- Wireframe drawing now uses the same batching capabilities as workbench & eevee (indirect drawing).
- Reduced complexity, removed ~3000 Lines of code in draw (also removed a lot of unused shader in GPU).
- Post AA to avoid complexity and cost of MSAA.

Remaining issues:
- ~~Armature edits, overlay toggles, (... others?) are not refreshing viewport after AA is complete~~
- FXAA is not the best for wires, maybe investigate SMAA
- Maybe do something more temporally stable for AA.
- ~~Paint overlays are not working with AA.~~
- ~~infront objects are difficult to select.~~
- ~~the infront wires sometimes goes through they solid counterpart (missing clear maybe?) (toggle overlays on-off when using infront+wireframe overlay in solid shading)~~

Note: I made some decision to change slightly the appearance of some objects to simplify their drawing. Namely the empty arrows end (which is now hollow/wire) and distance points of the cameras/spots being done by lines.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D6296

1 files changed, 26 insertions, 24 deletions

diff --git a/source/blender/draw/engines/basic/basic_engine.c b/source/blender/draw/engines/basic/basic_engine.c
index 0dd1a4fd686..fcbe227ca1b 100644
--- a/source/blender/draw/engines/basic/basic_engine.c
+++ b/source/blender/draw/engines/basic/basic_engine.c
@@ -46,8 +46,8 @@ typedef struct BASIC_StorageList {
 } BASIC_StorageList;
 
 typedef struct BASIC_PassList {
-  struct DRWPass *depth_pass;
-  struct DRWPass *depth_pass_cull;
+  struct DRWPass *depth_pass[2];
+  struct DRWPass *depth_pass_cull[2];
 } BASIC_PassList;
 
 typedef struct BASIC_Data {
@@ -70,8 +70,8 @@ static struct {
 } e_data = {{{NULL}}}; /* Engine data */
 
 typedef struct BASIC_PrivateData {
-  DRWShadingGroup *depth_shgrp;
-  DRWShadingGroup *depth_shgrp_cull;
+  DRWShadingGroup *depth_shgrp[2];
+  DRWShadingGroup *depth_shgrp_cull[2];
 } BASIC_PrivateData; /* Transient data */
 
 /* Functions */
@@ -97,24 +97,21 @@ static void basic_cache_init(void *vedata)
 
   if (!stl->g_data) {
     /* Alloc transient pointers */
-    stl->g_data = MEM_mallocN(sizeof(*stl->g_data), __func__);
+    stl->g_data = MEM_callocN(sizeof(*stl->g_data), __func__);
   }
 
-  {
-    psl->depth_pass = DRW_pass_create("Depth Pass",
-                                      DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL);
-    stl->g_data->depth_shgrp = DRW_shgroup_create(sh_data->depth, psl->depth_pass);
-    if (draw_ctx->sh_cfg == GPU_SHADER_CFG_CLIPPED) {
-      DRW_shgroup_state_enable(stl->g_data->depth_shgrp, DRW_STATE_CLIP_PLANES);
-    }
+  /* Twice for normal and infront objects. */
+  for (int i = 0; i < 2; i++) {
+    DRWState clip_state = (draw_ctx->sh_cfg == GPU_SHADER_CFG_CLIPPED) ? DRW_STATE_CLIP_PLANES : 0;
+    DRWState infront_state = (DRW_state_is_select() && (i == 1)) ? DRW_STATE_IN_FRONT_SELECT : 0;
+    DRWState state = DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL;
 
-    psl->depth_pass_cull = DRW_pass_create("Depth Pass Cull",
-                                           DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL |
-                                               DRW_STATE_CULL_BACK);
-    stl->g_data->depth_shgrp_cull = DRW_shgroup_create(sh_data->depth, psl->depth_pass_cull);
-    if (draw_ctx->sh_cfg == GPU_SHADER_CFG_CLIPPED) {
-      DRW_shgroup_state_enable(stl->g_data->depth_shgrp_cull, DRW_STATE_CLIP_PLANES);
-    }
+    DRW_PASS_CREATE(psl->depth_pass[i], state | clip_state | infront_state);
+    stl->g_data->depth_shgrp[i] = DRW_shgroup_create(sh_data->depth, psl->depth_pass[i]);
+
+    state |= DRW_STATE_CULL_BACK;
+    DRW_PASS_CREATE(psl->depth_pass_cull[i], state | clip_state | infront_state);
+    stl->g_data->depth_shgrp_cull[i] = DRW_shgroup_create(sh_data->depth, psl->depth_pass_cull[i]);
   }
 }
 
@@ -128,6 +125,8 @@ static void basic_cache_populate(void *vedata, Object *ob)
     return;
   }
 
+  bool do_in_front = (ob->dtx & OB_DRAWXRAY) != 0;
+
   const DRWContextState *draw_ctx = DRW_context_state_get();
   if (ob != draw_ctx->object_edit) {
     for (ParticleSystem *psys = ob->particlesystem.first; psys != NULL; psys = psys->next) {
@@ -138,7 +137,7 @@ static void basic_cache_populate(void *vedata, Object *ob)
       const int draw_as = (part->draw_as == PART_DRAW_REND) ? part->ren_as : part->draw_as;
       if (draw_as == PART_DRAW_PATH) {
         struct GPUBatch *hairs = DRW_cache_particles_get_hair(ob, psys, NULL);
-        DRW_shgroup_call(stl->g_data->depth_shgrp, hairs, NULL);
+        DRW_shgroup_call(stl->g_data->depth_shgrp[do_in_front], hairs, NULL);
       }
     }
   }
@@ -155,7 +154,7 @@ static void basic_cache_populate(void *vedata, Object *ob)
       /* Avoid losing flat objects when in ortho views (see T56549) */
       struct GPUBatch *geom = DRW_cache_object_all_edges_get(ob);
       if (geom) {
-        DRW_shgroup_call(stl->g_data->depth_shgrp, geom, ob);
+        DRW_shgroup_call(stl->g_data->depth_shgrp[do_in_front], geom, ob);
       }
       return;
     }
@@ -165,7 +164,8 @@ static void basic_cache_populate(void *vedata, Object *ob)
                                !DRW_state_is_image_render();
   const bool do_cull = (draw_ctx->v3d &&
                         (draw_ctx->v3d->shading.flag & V3D_SHADING_BACKFACE_CULLING));
-  DRWShadingGroup *shgrp = (do_cull) ? stl->g_data->depth_shgrp_cull : stl->g_data->depth_shgrp;
+  DRWShadingGroup *shgrp = (do_cull) ? stl->g_data->depth_shgrp_cull[do_in_front] :
+                                       stl->g_data->depth_shgrp[do_in_front];
 
   if (use_sculpt_pbvh) {
     DRW_shgroup_call_sculpt(shgrp, ob, false, false, false);
@@ -189,8 +189,10 @@ static void basic_draw_scene(void *vedata)
 {
   BASIC_PassList *psl = ((BASIC_Data *)vedata)->psl;
 
-  DRW_draw_pass(psl->depth_pass);
-  DRW_draw_pass(psl->depth_pass_cull);
+  DRW_draw_pass(psl->depth_pass[0]);
+  DRW_draw_pass(psl->depth_pass_cull[0]);
+  DRW_draw_pass(psl->depth_pass[1]);
+  DRW_draw_pass(psl->depth_pass_cull[1]);
 }
 
 static void basic_engine_free(void)