Overlay Engine: Refactor & Cleanup

This is the unification of all overlays into one overlay engine as described in T65347.

I went over all the code making it more future proof with less hacks and removing old / not relevent parts.

Goals / Acheivements:
- Remove internal shader usage (only drw shaders)
- Remove viewportSize and viewportSizeInv and put them in gloabl ubo
- Fixed some drawing issues: Missing probe option and Missing Alt+B clipping of some shader
- Remove old (legacy) shaders dependancy (not using view UBO).
- Less shader variation (less compilation time at first load and less patching needed for vulkan)
- removed some geom shaders when I could
- Remove static e_data (except shaders storage where it is OK)
- Clear the way to fix some anoying limitations (dithered transparency, background image compositing etc...)
- Wireframe drawing now uses the same batching capabilities as workbench & eevee (indirect drawing).
- Reduced complexity, removed ~3000 Lines of code in draw (also removed a lot of unused shader in GPU).
- Post AA to avoid complexity and cost of MSAA.

Remaining issues:
- ~~Armature edits, overlay toggles, (... others?) are not refreshing viewport after AA is complete~~
- FXAA is not the best for wires, maybe investigate SMAA
- Maybe do something more temporally stable for AA.
- ~~Paint overlays are not working with AA.~~
- ~~infront objects are difficult to select.~~
- ~~the infront wires sometimes goes through they solid counterpart (missing clear maybe?) (toggle overlays on-off when using infront+wireframe overlay in solid shading)~~

Note: I made some decision to change slightly the appearance of some objects to simplify their drawing. Namely the empty arrows end (which is now hollow/wire) and distance points of the cameras/spots being done by lines.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D6296

1 files changed, 144 insertions, 0 deletions

diff --git a/source/blender/draw/engines/overlay/shaders/wireframe_vert.glsl b/source/blender/draw/engines/overlay/shaders/wireframe_vert.glsl
new file mode 100644
index 00000000000..78ce8fd8a8f
--- /dev/null
+++ b/source/blender/draw/engines/overlay/shaders/wireframe_vert.glsl
@@ -0,0 +1,144 @@
+
+uniform float wireStepParam;
+
+in vec3 pos;
+in vec3 nor;
+in float wd; /* wiredata */
+
+float get_edge_sharpness(float wd)
+{
+  return ((wd == 0.0) ? -1.5 : wd) + wireStepParam;
+}
+
+/* Geometry shader version */
+#if defined(SELECT_EDGES) || defined(USE_GEOM)
+out vec3 finalColor_g;
+out float edgeSharpness_g;
+
+#else /* USE_GEOM */
+
+flat out vec2 edgeStart;
+noperspective out vec2 edgePos;
+
+out vec3 finalColor;
+flat out float edgeSharpness;
+#  define finalColor_g finalColor
+#  define edgeSharpness_g edgeSharpness
+
+#endif /* SELECT_EDGES */
+
+uniform bool useColoring;
+uniform bool isTransform;
+uniform bool isObjectColor;
+uniform bool isRandomColor;
+
+void wire_color_get(out vec3 rim_col, out vec3 wire_col)
+{
+  int flag = int(abs(ObjectInfo.w));
+  bool is_selected = (flag & DRW_BASE_SELECTED) != 0;
+  bool is_from_dupli = (flag & DRW_BASE_FROM_DUPLI) != 0;
+  bool is_from_set = (flag & DRW_BASE_FROM_SET) != 0;
+  bool is_active = (flag & DRW_BASE_ACTIVE) != 0;
+
+  if (is_from_set) {
+    rim_col = colorDupli.rgb;
+    wire_col = colorDupli.rgb;
+  }
+  else if (is_from_dupli) {
+    if (is_selected) {
+      if (isTransform) {
+        rim_col = colorTransform.rgb;
+      }
+      else {
+        rim_col = colorDupliSelect.rgb;
+      }
+    }
+    else {
+      rim_col = colorDupli.rgb;
+    }
+    wire_col = colorDupli.rgb;
+  }
+  else if (is_selected && useColoring) {
+    if (isTransform) {
+      rim_col = colorTransform.rgb;
+    }
+    else if (is_active) {
+      rim_col = colorActive.rgb;
+    }
+    else {
+      rim_col = colorSelect.rgb;
+    }
+    wire_col = colorWire.rgb;
+  }
+  else {
+    rim_col = colorWire.rgb;
+    wire_col = colorBackground.rgb;
+  }
+}
+
+vec3 hsv_to_rgb(vec3 hsv)
+{
+  vec3 nrgb = abs(hsv.x * 6.0 - vec3(3.0, 2.0, 4.0)) * vec3(1, -1, -1) + vec3(-1, 2, 2);
+  nrgb = clamp(nrgb, 0.0, 1.0);
+  return ((nrgb - 1.0) * hsv.y + 1.0) * hsv.z;
+}
+
+void wire_object_color_get(out vec3 rim_col, out vec3 wire_col)
+{
+  int flag = int(abs(ObjectInfo.w));
+  bool is_selected = (flag & DRW_BASE_SELECTED) != 0;
+
+  if (isObjectColor) {
+    rim_col = wire_col = ObjectColor.rgb * 0.5;
+  }
+  else {
+    float hue = ObjectInfo.z;
+    vec3 hsv = vec3(hue, 0.75, 0.8);
+    rim_col = wire_col = hsv_to_rgb(hsv);
+  }
+
+  if (is_selected && useColoring) {
+    /* "Normalize" color. */
+    wire_col += 1e-4; /* Avoid division by 0. */
+    float brightness = max(wire_col.x, max(wire_col.y, wire_col.z));
+    wire_col *= 0.5 / brightness;
+    rim_col += 0.75;
+  }
+  else {
+    rim_col *= 0.5;
+    wire_col += 0.5;
+  }
+}
+
+void main()
+{
+  vec3 wpos = point_object_to_world(pos);
+  gl_Position = point_world_to_ndc(wpos);
+
+#if !(defined(SELECT_EDGES) || defined(USE_GEOM))
+  /* Convert to screen position [0..sizeVp]. */
+  edgePos = edgeStart = ((gl_Position.xy / gl_Position.w) * 0.5 + 0.5) * sizeViewport.xy;
+#endif
+
+  edgeSharpness_g = get_edge_sharpness(wd);
+
+  vec3 rim_col, wire_col;
+  if (isObjectColor || isRandomColor) {
+    wire_object_color_get(rim_col, wire_col);
+  }
+  else {
+    wire_color_get(rim_col, wire_col);
+  }
+
+  vec3 wnor = normalize(normal_object_to_world(nor));
+  float facing = dot(wnor, ViewMatrixInverse[2].xyz);
+  facing = clamp(abs(facing), 0.0, 1.0);
+
+  vec3 final_front_col = mix(rim_col, wire_col, 0.4);
+  vec3 final_rim_col = mix(rim_col, wire_col, 0.1);
+  finalColor_g = mix(final_rim_col, final_front_col, facing);
+
+#ifdef USE_WORLD_CLIP_PLANES
+  world_clip_planes_calc_clip_distance(wpos);
+#endif
+}