MSL: Adjust gl_SampleMaskIn for sample-shading and/or fixed sample mask.

Vulkan specifies that the Sample Mask Test occurs before fragment shading.
This means gl_SampleMaskIn should be influenced by both sample-shading and
VkPipelineMultisampleStateCreateInfo::pSampleMask.

CTS tests dEQP-VK.pipeline.multisample_shader_builtin.* bear this out.

For sample-shading, gl_SampleMaskIn should only have a single bit set,
Since Metal does not filter for this, apply a bitmask based on gl_SampleID.

For a fixed sample mask, since Metal is unaware of
VkPipelineMultisampleStateCreateInfo::pSampleMask, we need to ensure that
we apply it to both gl_SampleMaskIn and gl_SampleMask. This has the side
effect of a redundant application of pSampleMask if the shader already
includes gl_SampleMaskIn when setting gl_SampleMask, but I don't see an
easy way around this.

Also, simplify the logic for including the fixed sample mask in gl_ShaderMask,
and print the fixed sample mask as a hex value for readability of bits.

4 files changed, 24 insertions, 4 deletions

diff --git a/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.force-sample.frag b/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.force-sample.frag
new file mode 100644
index 00000000..626fe4c7
--- /dev/null
+++ b/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.force-sample.frag
@@ -0,0 +1,20 @@
+#include <metal_stdlib>
+#include <simd/simd.h>
+
+using namespace metal;
+
+struct main0_out
+{
+    float4 FragColor [[color(0)]];
+    uint gl_SampleMask [[sample_mask]];
+};
+
+fragment main0_out main0(uint gl_SampleMaskIn [[sample_mask]], uint gl_SampleID [[sample_id]])
+{
+    main0_out out = {};
+    out.FragColor = float4(1.0);
+    out.gl_SampleMask = (gl_SampleMaskIn & 0x22 & (1 << gl_SampleID));
+    out.gl_SampleMask &= 0x22;
+    return out;
+}
+
diff --git a/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.frag b/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.frag
index 21ca7178..f478901b 100644
--- a/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.frag
+++ b/reference/opt/shaders-msl/frag/sample-mask-in-and-out.fixed-sample-mask.frag
@@ -13,8 +13,8 @@ fragment main0_out main0(uint gl_SampleMaskIn [[sample_mask]])
 {
     main0_out out = {};
     out.FragColor = float4(1.0);
-    out.gl_SampleMask = gl_SampleMaskIn;
-    out.gl_SampleMask &= 34;
+    out.gl_SampleMask = (gl_SampleMaskIn & 0x22);
+    out.gl_SampleMask &= 0x22;
     return out;
 }
 
diff --git a/reference/opt/shaders-msl/frag/sample-mask-not-used.fixed-sample-mask.frag b/reference/opt/shaders-msl/frag/sample-mask-not-used.fixed-sample-mask.frag
index 040c6414..d04f2033 100644
--- a/reference/opt/shaders-msl/frag/sample-mask-not-used.fixed-sample-mask.frag
+++ b/reference/opt/shaders-msl/frag/sample-mask-not-used.fixed-sample-mask.frag
@@ -13,7 +13,7 @@ fragment main0_out main0()
 {
     main0_out out = {};
     out.FragColor = float4(1.0);
-    out.gl_SampleMask = 34;
+    out.gl_SampleMask = 0x22;
     return out;
 }
 
diff --git a/reference/opt/shaders-msl/frag/sample-mask.fixed-sample-mask.frag b/reference/opt/shaders-msl/frag/sample-mask.fixed-sample-mask.frag
index 20444779..76306b5a 100644
--- a/reference/opt/shaders-msl/frag/sample-mask.fixed-sample-mask.frag
+++ b/reference/opt/shaders-msl/frag/sample-mask.fixed-sample-mask.frag
@@ -14,7 +14,7 @@ fragment main0_out main0()
     main0_out out = {};
     out.FragColor = float4(1.0);
     out.gl_SampleMask = 0;
-    out.gl_SampleMask &= 34;
+    out.gl_SampleMask &= 0x22;
     return out;
 }