Fix T65114 Overlay: Armature: Bones with negative scale have wrong display

This moves the backface culling to the fragment shader to avoid all the
limitations of the current system. This has a cost but it is unlikely that
bone drawing will be a bottleneck.

5 files changed, 16 insertions, 1 deletions

diff --git a/source/blender/draw/engines/overlay/overlay_armature.c b/source/blender/draw/engines/overlay/overlay_armature.c
index 0b77fcad265..9fde3ec0902 100644
--- a/source/blender/draw/engines/overlay/overlay_armature.c
+++ b/source/blender/draw/engines/overlay/overlay_armature.c
@@ -186,7 +186,7 @@ void OVERLAY_armature_cache_init(OVERLAY_Data *vedata)
     cb->custom_shapes_ghash = BLI_ghash_ptr_new(__func__);
 
     DRWState infront_state = (DRW_state_is_select() && (i == 1)) ? DRW_STATE_IN_FRONT_SELECT : 0;
-    state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_CULL_BACK |
+    state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL |
             (pd->armature.transparent ? DRW_STATE_BLEND_ALPHA : DRW_STATE_WRITE_DEPTH);
     DRW_PASS_CREATE(*p_armature_ps, state | pd->clipping_state | infront_state);
 
diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl
index ffc941149c7..db5975ea226 100644
--- a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl
+++ b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_geom.glsl
@@ -7,6 +7,7 @@ in vec3 vPos[];
 in vec2 ssPos[];
 in vec2 ssNor[];
 in vec4 vColSize[];
+in int inverted[];
 
 flat out vec4 finalColor;
 flat out vec2 edgeStart;
@@ -39,6 +40,7 @@ void main(void)
     }
   }
 
+  n0 = (inverted[0] == 1) ? -n0 : n0;
   /* Don't outline if concave edge. */
   if (dot(n0, v13) > 0.0001) {
     return;
diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl
index cd9368a997a..b720be0c7d1 100644
--- a/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/armature_shape_outline_vert.glsl
@@ -12,6 +12,7 @@ out vec3 vPos;
 out vec2 ssPos;
 out vec2 ssNor;
 out vec4 vColSize;
+out int inverted;
 
 /* project to screen space */
 vec2 proj(vec4 pos)
@@ -30,6 +31,8 @@ void main()
   vPos = viewpos.xyz;
   pPos = ProjectionMatrix * viewpos;
 
+  inverted = int(dot(cross(model_mat[0].xyz, model_mat[1].xyz), model_mat[2].xyz) < 0.0);
+
   /* This is slow and run per vertex, but it's still faster than
    * doing it per instance on CPU and sending it on via instance attribute. */
   mat3 normal_mat = transpose(inverse(mat3(model_mat)));
diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl
index 54cd807edaa..17e8d0da5d9 100644
--- a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl
+++ b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_frag.glsl
@@ -2,12 +2,19 @@
 uniform float alpha = 0.6;
 
 in vec4 finalColor;
+flat in int inverted;
 
 layout(location = 0) out vec4 fragColor;
 layout(location = 1) out vec4 lineOutput;
 
 void main()
 {
+  /* Manual backface cullling.. Not ideal for performance
+   * but needed for view clarity in xray mode and support
+   * for inverted bone matrices. */
+  if ((inverted == 1) == gl_FrontFacing) {
+    discard;
+  }
   fragColor = vec4(finalColor.rgb, alpha);
   lineOutput = vec4(0.0);
 }
diff --git a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl
index 8284bd43adc..a6af7214de1 100644
--- a/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/armature_shape_solid_vert.glsl
@@ -7,6 +7,7 @@ in vec3 nor;
 in mat4 inst_obmat;
 
 out vec4 finalColor;
+flat out int inverted;
 
 void main()
 {
@@ -18,6 +19,8 @@ void main()
   mat3 normal_mat = transpose(inverse(mat3(model_mat)));
   vec3 normal = normalize(normal_world_to_view(normal_mat * nor));
 
+  inverted = int(dot(cross(model_mat[0].xyz, model_mat[1].xyz), model_mat[2].xyz) < 0.0);
+
   /* Do lighting at an angle to avoid flat shading on front facing bone. */
   const vec3 light = vec3(0.1, 0.1, 0.8);
   float n = dot(normal, light);