Fix T74536: Grease pencil immediately crashes on macOS

It seems like OSX drivers are using standard attributes for passing
gl_VertexID and gl_InstanceID to the vertex shader, and count them in the
limit of MAX_VERTEX_ATTRIBS.

This patch make sure to never use more than 13 attributes by packing some
attributes together.

3 files changed, 81 insertions, 49 deletions

diff --git a/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl b/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
index 8774b633467..5e84b3592e4 100644
--- a/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
+++ b/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
@@ -218,35 +218,53 @@ uniform float strokeIndexOffset = 0.0;
  * - ma3 reference the next adjacency point.
  * Note that we are rendering quad instances and not using any index buffer (except for fills).
  */
-in vec4 ma;
-in vec4 ma1;
-in vec4 ma2;
-in vec4 ma3;
-#  define strength1 ma1.y
-#  define strength2 ma2.y
-#  define stroke_id1 ma1.z
-#  define point_id1 ma1.w
+/* x is material index, y is stroke_id, z is point_id, w is aspect & rotation & hardness packed. */
+in ivec4 ma;
+in ivec4 ma1;
+in ivec4 ma2;
+in ivec4 ma3;
 /* Position contains thickness in 4th component. */
 in vec4 pos;  /* Prev adj vert */
 in vec4 pos1; /* Current edge */
 in vec4 pos2; /* Current edge */
 in vec4 pos3; /* Next adj vert */
-#  define thickness1 pos1.w
-#  define thickness2 pos2.w
-/* xy is UV for fills, z is U of stroke, w is cosine of UV angle with sign of sine.  */
+/* xy is UV for fills, z is U of stroke, w is strength.  */
 in vec4 uv1;
 in vec4 uv2;
-
 in vec4 col1;
 in vec4 col2;
-
 in vec4 fcol1;
+/* WARNING: Max attrib count is actually 14 because OSX OpenGL implementation
+ * considers gl_VertexID and gl_InstanceID as vertex attribute. (see T74536) */
+#  define stroke_id1 ma1.y
+#  define point_id1 ma1.z
+#  define thickness1 pos1.w
+#  define thickness2 pos2.w
+#  define strength1 uv1.w
+#  define strength2 uv2.w
+/* Packed! need to be decoded. */
+#  define hardness1 ma1.w
+#  define hardness2 ma2.w
+#  define uvrot1 ma1.w
+#  define aspect1 ma1.w
+
+vec2 decode_aspect(int packed_data)
+{
+  float asp = float(uint(packed_data) & 0x1FFu) * (1.0 / 255.0);
+  return (asp > 1.0) ? vec2(1.0, (asp - 1.0)) : vec2(asp, 1.0);
+}
 
-/* hard.x is aspect. */
-in vec2 hard1;
-in vec2 hard2;
-#  define aspect1 hard1.x
-#  define aspect2 hard2.x
+float decode_uvrot(int packed_data)
+{
+  uint udata = uint(packed_data);
+  float uvrot = 1e-8 + float((udata & 0x1FE00u) >> 9u) * (1.0 / 255.0);
+  return ((udata & 0x20000u) != 0u) ? -uvrot : uvrot;
+}
+
+float decode_hardness(int packed_data)
+{
+  return float((uint(packed_data) & 0x3FC0000u) >> 18u) * (1.0 / 255.0);
+}
 
 void discard_vert()
 {
@@ -356,13 +374,13 @@ void stroke_vertex()
 #  endif
 
   /* Special Case. Stroke with single vert are rendered as dots. Do not discard them. */
-  if (!is_dot && ma.x == -1.0 && ma2.x == -1.0) {
+  if (!is_dot && ma.x == -1 && ma2.x == -1) {
     is_dot = true;
     is_squares = false;
   }
 
   /* Enpoints, we discard the vertices. */
-  if (ma1.x == -1.0 || (!is_dot && ma2.x == -1.0)) {
+  if (ma1.x == -1 || (!is_dot && ma2.x == -1)) {
     discard_vert();
     return;
   }
@@ -398,7 +416,7 @@ void stroke_vertex()
   thickness = stroke_thickness_modulate(thickness);
 
   finalUvs = vec2(x, y) * 0.5 + 0.5;
-  strokeHardeness = (use_curr) ? hard1.y : hard2.y;
+  strokeHardeness = decode_hardness(use_curr ? hardness1 : hardness2);
 
   if (is_dot) {
 #  ifdef GP_MATERIAL_BUFFER_LEN
@@ -408,7 +426,7 @@ void stroke_vertex()
     vec2 x_axis;
 #  ifdef GP_MATERIAL_BUFFER_LEN
     if (alignement == GP_STROKE_ALIGNMENT_STROKE) {
-      x_axis = (ma2.x == -1.0) ? line_adj : line;
+      x_axis = (ma2.x == -1) ? line_adj : line;
     }
     else if (alignement == GP_STROKE_ALIGNMENT_FIXED) {
       /* Default for no-material drawing. */
@@ -423,25 +441,20 @@ void stroke_vertex()
     }
 
     /* Rotation: Encoded as Cos + Sin sign. */
-    float rot_sin = sqrt(1.0 - uv1.w * uv1.w) * sign(uv1.w);
-    float rot_cos = abs(uv1.w);
+    float uv_rot = decode_uvrot(uvrot1);
+    float rot_sin = sqrt(1.0 - uv_rot * uv_rot) * sign(uv_rot);
+    float rot_cos = abs(uv_rot);
     x_axis = mat2(rot_cos, -rot_sin, rot_sin, rot_cos) * x_axis;
 
     vec2 y_axis = rotate_90deg(x_axis);
 
-    strokeAspect.x = aspect1;
+    strokeAspect = decode_aspect(aspect1);
 
-    if (strokeAspect.x > 1.0) {
-      strokeAspect.y = strokeAspect.x;
-      strokeAspect.x = 1.0;
-    }
-    else {
-      strokeAspect.x = 1.0 / strokeAspect.x;
-      strokeAspect.y = 1.0;
-    }
+    x *= strokeAspect.x;
+    y *= strokeAspect.y;
 
-    x /= strokeAspect.x;
-    y /= strokeAspect.y;
+    /* Invert for vertex shader. */
+    strokeAspect = 1.0 / strokeAspect;
 
     gl_Position.xy += (x * x_axis + y * y_axis) * sizeViewportInv.xy * thickness;
 
diff --git a/source/blender/draw/engines/overlay/shaders/edit_gpencil_vert.glsl b/source/blender/draw/engines/overlay/shaders/edit_gpencil_vert.glsl
index cb03ad44615..3a52e0c73b7 100644
--- a/source/blender/draw/engines/overlay/shaders/edit_gpencil_vert.glsl
+++ b/source/blender/draw/engines/overlay/shaders/edit_gpencil_vert.glsl
@@ -9,7 +9,7 @@ uniform vec4 gpEditColor;
 uniform sampler1D weightTex;
 
 in vec3 pos;
-in float ma;
+in int ma;
 in uint vflag;
 in float weight;
 
@@ -93,7 +93,7 @@ void main()
 #endif
 
   /* Discard unwanted padding vertices. */
-  if (ma == -1.0 || (is_multiframe && !doMultiframe)) {
+  if (ma == -1 || (is_multiframe && !doMultiframe)) {
     discard_vert();
   }
 
diff --git a/source/blender/draw/intern/draw_cache_impl_gpencil.c b/source/blender/draw/intern/draw_cache_impl_gpencil.c
index c6cedd3f1d2..1464cb9364d 100644
--- a/source/blender/draw/intern/draw_cache_impl_gpencil.c
+++ b/source/blender/draw/intern/draw_cache_impl_gpencil.c
@@ -166,23 +166,20 @@ void DRW_gpencil_batch_cache_free(bGPdata *gpd)
 
 /* MUST match the format below. */
 typedef struct gpStrokeVert {
-  /** Mat is float because we need to pack other float attribs with it. */
-  float mat, strength, stroke_id, point_id;
+  int32_t mat, stroke_id, point_id, packed_asp_hard_rot;
   /** Position and thickness packed in the same attribute. */
   float pos[3], thickness;
-  float uv_fill[2], u_stroke, v_rot;
-  /** Aspect ratio and hardnes. */
-  float aspect_ratio, hardness;
+  /** UV and strength packed in the same attribute. */
+  float uv_fill[2], u_stroke, strength;
 } gpStrokeVert;
 
 static GPUVertFormat *gpencil_stroke_format(void)
 {
   static GPUVertFormat format = {0};
   if (format.attr_len == 0) {
-    GPU_vertformat_attr_add(&format, "ma", GPU_COMP_F32, 4, GPU_FETCH_FLOAT);
+    GPU_vertformat_attr_add(&format, "ma", GPU_COMP_I32, 4, GPU_FETCH_INT);
     GPU_vertformat_attr_add(&format, "pos", GPU_COMP_F32, 4, GPU_FETCH_FLOAT);
     GPU_vertformat_attr_add(&format, "uv", GPU_COMP_F32, 4, GPU_FETCH_FLOAT);
-    GPU_vertformat_attr_add(&format, "hard", GPU_COMP_F32, 2, GPU_FETCH_FLOAT);
     /* IMPORTANT: This means having only 4 attributes to fit into GPU module limit of 16 attrib. */
     GPU_vertformat_multiload_enable(&format, 4);
   }
@@ -249,6 +246,29 @@ static int gpencil_stroke_is_cyclic(const bGPDstroke *gps)
   return ((gps->flag & GP_STROKE_CYCLIC) != 0) && (gps->totpoints > 2);
 }
 
+BLI_INLINE int32_t pack_rotation_aspect_hardness(float rot, float asp, float hard)
+{
+  int32_t packed = 0;
+  /* Aspect uses 9 bits */
+  float asp_normalized = (asp > 1.0f) ? (1.0f / asp) : asp;
+  packed |= (int32_t)unit_float_to_uchar_clamp(asp_normalized);
+  /* Store if inversed in the 9th bit. */
+  if (asp > 1.0f) {
+    packed |= 1 << 8;
+  }
+  /* Rotation uses 9 bits */
+  /* Rotation are in [-90°..90°] range, so we can encode the sign of the angle + the cosine
+   * because the cosine will always be positive. */
+  packed |= (int32_t)unit_float_to_uchar_clamp(cosf(rot)) << 9;
+  /* Store sine sign in 9th bit. */
+  if (rot < 0.0f) {
+    packed |= 1 << 17;
+  }
+  /* Hardness uses 8 bits */
+  packed |= (int32_t)unit_float_to_uchar_clamp(hard) << 18;
+  return packed;
+}
+
 static void gpencil_buffer_add_point(gpStrokeVert *verts,
                                      gpColorVert *cols,
                                      const bGPDstroke *gps,
@@ -275,15 +295,14 @@ static void gpencil_buffer_add_point(gpStrokeVert *verts,
   vert->u_stroke = pt->uv_fac;
   vert->stroke_id = gps->runtime.stroke_start;
   vert->point_id = v;
-  /* Rotation are in [-90°..90°] range, so we can encode the sign of the angle + the cosine
-   * because the cosine will always be positive. */
-  vert->v_rot = cosf(pt->uv_rot) * signf(pt->uv_rot);
   vert->thickness = max_ff(0.0f, gps->thickness * pt->pressure) * (round_cap1 ? 1.0 : -1.0);
   /* Tag endpoint material to -1 so they get discarded by vertex shader. */
   vert->mat = (is_endpoint) ? -1 : (gps->mat_nr % GP_MATERIAL_BUFFER_LEN);
 
-  vert->aspect_ratio = gps->aspect_ratio[0] / max_ff(gps->aspect_ratio[1], 1e-8);
-  vert->hardness = gps->hardeness;
+  float aspect_ratio = gps->aspect_ratio[0] / max_ff(gps->aspect_ratio[1], 1e-8);
+
+  vert->packed_asp_hard_rot = pack_rotation_aspect_hardness(
+      pt->uv_rot, aspect_ratio, gps->hardeness);
 }
 
 static void gpencil_buffer_add_stroke(gpStrokeVert *verts,