Fixup names of anonymous inner structs.

Just like we try to fixup struct names for block types, inner structs
can be "anonymous" structs. HLSL codegen from DXC tends to emit this,
and emitting dummy struct names tends to break GL linkage on some
drivers.

10 files changed, 537 insertions, 0 deletions

diff --git a/reference/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/reference/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..128a8c52
--- /dev/null
+++ b/reference/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,89 @@
+struct anon_aa
+{
+    int foo;
+};
+
+struct anon_ab
+{
+    int foo;
+};
+
+struct anon_a
+{
+    anon_aa _aa;
+    anon_ab ab;
+};
+
+struct anon_ba
+{
+    int foo;
+};
+
+struct anon_bb
+{
+    int foo;
+};
+
+struct anon_b
+{
+    anon_ba _ba;
+    anon_bb bb;
+};
+
+struct VertexData
+{
+    anon_a _a;
+    anon_b b;
+};
+
+struct anon_ca
+{
+    int foo;
+};
+
+struct anon_c
+{
+    anon_ca _ca;
+};
+
+struct anon_da
+{
+    int foo;
+};
+
+struct anon_d
+{
+    anon_da da;
+};
+
+struct anon_e
+{
+    int a;
+};
+
+cbuffer UBO : register(b0)
+{
+    anon_c _16_c : packoffset(c0);
+    anon_d _16_d : packoffset(c1);
+};
+
+RWByteAddressBuffer _19 : register(u1);
+
+static VertexData _3;
+
+struct SPIRV_Cross_Input
+{
+    anon_a VertexData__a : TEXCOORD0;
+    anon_b VertexData_b : TEXCOORD2;
+};
+
+void frag_main()
+{
+}
+
+void main(SPIRV_Cross_Input stage_input)
+{
+    _3._a = stage_input.VertexData__a;
+    _3.b = stage_input.VertexData_b;
+    frag_main();
+}
diff --git a/reference/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/reference/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..e4397f82
--- /dev/null
+++ b/reference/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,85 @@
+#include <metal_stdlib>
+#include <simd/simd.h>
+
+using namespace metal;
+
+struct anon_aa
+{
+    int foo;
+};
+
+struct anon_ab
+{
+    int foo;
+};
+
+struct anon_a
+{
+    anon_aa _aa;
+    anon_ab ab;
+};
+
+struct anon_ba
+{
+    int foo;
+};
+
+struct anon_bb
+{
+    int foo;
+};
+
+struct anon_b
+{
+    anon_ba _ba;
+    anon_bb bb;
+};
+
+struct VertexData
+{
+    anon_a _a;
+    anon_b b;
+};
+
+struct anon_ca
+{
+    int foo;
+};
+
+struct anon_c
+{
+    anon_ca _ca;
+};
+
+struct anon_da
+{
+    int foo;
+};
+
+struct anon_d
+{
+    anon_da da;
+};
+
+struct UBO
+{
+    anon_c _c;
+    anon_d d;
+};
+
+struct anon_e
+{
+    int a;
+};
+
+struct SSBO
+{
+    anon_e _m0;
+    anon_e _e;
+    anon_e f;
+};
+
+fragment void main0()
+{
+}
+
diff --git a/reference/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/reference/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..b2d8919a
--- /dev/null
+++ b/reference/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,63 @@
+#version 450
+
+struct anon_aa
+{
+    int foo;
+};
+
+struct anon_ab
+{
+    int foo;
+};
+
+struct anon_a
+{
+    anon_aa _aa;
+    anon_ab ab;
+};
+
+struct anon_ba
+{
+    int foo;
+};
+
+struct anon_bb
+{
+    int foo;
+};
+
+struct anon_b
+{
+    anon_ba _ba;
+    anon_bb bb;
+};
+
+struct anon_ca
+{
+    int foo;
+};
+
+struct anon_c
+{
+    anon_ca _ca;
+};
+
+struct anon_da
+{
+    int foo;
+};
+
+struct anon_d
+{
+    anon_da da;
+};
+
+struct anon_e
+{
+    int a;
+};
+
+void main()
+{
+}
+
diff --git a/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..6782b124
--- /dev/null
+++ b/shaders-hlsl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,83 @@
+; SPIR-V
+; Version: 1.0
+; Generator: Khronos Glslang Reference Front End; 10
+; Bound: 27
+; Schema: 0
+               OpCapability Shader
+          %1 = OpExtInstImport "GLSL.std.450"
+               OpMemoryModel Logical GLSL450
+               OpEntryPoint Fragment %main "main" %_
+               OpExecutionMode %main OriginUpperLeft
+               OpSource GLSL 450
+               OpName %main "main"
+               OpMemberName %AA 0 "foo"
+               OpMemberName %AB 0 "foo"
+               OpMemberName %A 0 "_aa"
+               OpMemberName %A 1 "ab"
+               OpMemberName %BA 0 "foo"
+               OpMemberName %BB 0 "foo"
+               OpMemberName %B 0 "_ba"
+               OpMemberName %B 1 "bb"
+               OpName %VertexData "VertexData"
+               OpMemberName %VertexData 0 "_a"
+               OpMemberName %VertexData 1 "b"
+               OpName %_ ""
+               OpMemberName %CA 0 "foo"
+               OpMemberName %C 0 "_ca"
+               OpMemberName %DA 0 "foo"
+               OpMemberName %D 0 "da"
+               OpName %UBO "UBO"
+               OpMemberName %UBO 0 "_c"
+               OpMemberName %UBO 1 "d"
+               OpName %__0 ""
+               OpMemberName %E 0 "a"
+               OpName %SSBO "SSBO"
+               ;OpMemberName %SSBO 0 "e" Test that we don't try to assign bogus aliases.
+               OpMemberName %SSBO 1 "_e"
+               OpMemberName %SSBO 2 "f"
+               OpName %__1 ""
+               OpDecorate %VertexData Block
+               OpDecorate %_ Location 0
+               OpMemberDecorate %CA 0 Offset 0
+               OpMemberDecorate %C 0 Offset 0
+               OpMemberDecorate %DA 0 Offset 0
+               OpMemberDecorate %D 0 Offset 0
+               OpMemberDecorate %UBO 0 Offset 0
+               OpMemberDecorate %UBO 1 Offset 16
+               OpDecorate %UBO Block
+               OpDecorate %__0 DescriptorSet 0
+               OpDecorate %__0 Binding 0
+               OpMemberDecorate %E 0 Offset 0
+               OpMemberDecorate %SSBO 0 Offset 0
+               OpMemberDecorate %SSBO 1 Offset 4
+               OpMemberDecorate %SSBO 2 Offset 8
+               OpDecorate %SSBO BufferBlock
+               OpDecorate %__1 DescriptorSet 0
+               OpDecorate %__1 Binding 1
+       %void = OpTypeVoid
+          %3 = OpTypeFunction %void
+        %int = OpTypeInt 32 1
+         %AA = OpTypeStruct %int
+         %AB = OpTypeStruct %int
+          %A = OpTypeStruct %AA %AB
+         %BA = OpTypeStruct %int
+         %BB = OpTypeStruct %int
+          %B = OpTypeStruct %BA %BB
+ %VertexData = OpTypeStruct %A %B
+%_ptr_Input_VertexData = OpTypePointer Input %VertexData
+          %_ = OpVariable %_ptr_Input_VertexData Input
+         %CA = OpTypeStruct %int
+          %C = OpTypeStruct %CA
+         %DA = OpTypeStruct %int
+          %D = OpTypeStruct %DA
+        %UBO = OpTypeStruct %C %D
+%_ptr_Uniform_UBO = OpTypePointer Uniform %UBO
+        %__0 = OpVariable %_ptr_Uniform_UBO Uniform
+          %E = OpTypeStruct %int
+       %SSBO = OpTypeStruct %E %E %E
+%_ptr_Uniform_SSBO = OpTypePointer Uniform %SSBO
+        %__1 = OpVariable %_ptr_Uniform_SSBO Uniform
+       %main = OpFunction %void None %3
+          %5 = OpLabel
+               OpReturn
+               OpFunctionEnd
diff --git a/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..6782b124
--- /dev/null
+++ b/shaders-msl-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,83 @@
+; SPIR-V
+; Version: 1.0
+; Generator: Khronos Glslang Reference Front End; 10
+; Bound: 27
+; Schema: 0
+               OpCapability Shader
+          %1 = OpExtInstImport "GLSL.std.450"
+               OpMemoryModel Logical GLSL450
+               OpEntryPoint Fragment %main "main" %_
+               OpExecutionMode %main OriginUpperLeft
+               OpSource GLSL 450
+               OpName %main "main"
+               OpMemberName %AA 0 "foo"
+               OpMemberName %AB 0 "foo"
+               OpMemberName %A 0 "_aa"
+               OpMemberName %A 1 "ab"
+               OpMemberName %BA 0 "foo"
+               OpMemberName %BB 0 "foo"
+               OpMemberName %B 0 "_ba"
+               OpMemberName %B 1 "bb"
+               OpName %VertexData "VertexData"
+               OpMemberName %VertexData 0 "_a"
+               OpMemberName %VertexData 1 "b"
+               OpName %_ ""
+               OpMemberName %CA 0 "foo"
+               OpMemberName %C 0 "_ca"
+               OpMemberName %DA 0 "foo"
+               OpMemberName %D 0 "da"
+               OpName %UBO "UBO"
+               OpMemberName %UBO 0 "_c"
+               OpMemberName %UBO 1 "d"
+               OpName %__0 ""
+               OpMemberName %E 0 "a"
+               OpName %SSBO "SSBO"
+               ;OpMemberName %SSBO 0 "e" Test that we don't try to assign bogus aliases.
+               OpMemberName %SSBO 1 "_e"
+               OpMemberName %SSBO 2 "f"
+               OpName %__1 ""
+               OpDecorate %VertexData Block
+               OpDecorate %_ Location 0
+               OpMemberDecorate %CA 0 Offset 0
+               OpMemberDecorate %C 0 Offset 0
+               OpMemberDecorate %DA 0 Offset 0
+               OpMemberDecorate %D 0 Offset 0
+               OpMemberDecorate %UBO 0 Offset 0
+               OpMemberDecorate %UBO 1 Offset 16
+               OpDecorate %UBO Block
+               OpDecorate %__0 DescriptorSet 0
+               OpDecorate %__0 Binding 0
+               OpMemberDecorate %E 0 Offset 0
+               OpMemberDecorate %SSBO 0 Offset 0
+               OpMemberDecorate %SSBO 1 Offset 4
+               OpMemberDecorate %SSBO 2 Offset 8
+               OpDecorate %SSBO BufferBlock
+               OpDecorate %__1 DescriptorSet 0
+               OpDecorate %__1 Binding 1
+       %void = OpTypeVoid
+          %3 = OpTypeFunction %void
+        %int = OpTypeInt 32 1
+         %AA = OpTypeStruct %int
+         %AB = OpTypeStruct %int
+          %A = OpTypeStruct %AA %AB
+         %BA = OpTypeStruct %int
+         %BB = OpTypeStruct %int
+          %B = OpTypeStruct %BA %BB
+ %VertexData = OpTypeStruct %A %B
+%_ptr_Input_VertexData = OpTypePointer Input %VertexData
+          %_ = OpVariable %_ptr_Input_VertexData Input
+         %CA = OpTypeStruct %int
+          %C = OpTypeStruct %CA
+         %DA = OpTypeStruct %int
+          %D = OpTypeStruct %DA
+        %UBO = OpTypeStruct %C %D
+%_ptr_Uniform_UBO = OpTypePointer Uniform %UBO
+        %__0 = OpVariable %_ptr_Uniform_UBO Uniform
+          %E = OpTypeStruct %int
+       %SSBO = OpTypeStruct %E %E %E
+%_ptr_Uniform_SSBO = OpTypePointer Uniform %SSBO
+        %__1 = OpVariable %_ptr_Uniform_SSBO Uniform
+       %main = OpFunction %void None %3
+          %5 = OpLabel
+               OpReturn
+               OpFunctionEnd
diff --git a/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag b/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
new file mode 100644
index 00000000..6782b124
--- /dev/null
+++ b/shaders-no-opt/asm/frag/anonymous-inner-struct-names.asm.frag
@@ -0,0 +1,83 @@
+; SPIR-V
+; Version: 1.0
+; Generator: Khronos Glslang Reference Front End; 10
+; Bound: 27
+; Schema: 0
+               OpCapability Shader
+          %1 = OpExtInstImport "GLSL.std.450"
+               OpMemoryModel Logical GLSL450
+               OpEntryPoint Fragment %main "main" %_
+               OpExecutionMode %main OriginUpperLeft
+               OpSource GLSL 450
+               OpName %main "main"
+               OpMemberName %AA 0 "foo"
+               OpMemberName %AB 0 "foo"
+               OpMemberName %A 0 "_aa"
+               OpMemberName %A 1 "ab"
+               OpMemberName %BA 0 "foo"
+               OpMemberName %BB 0 "foo"
+               OpMemberName %B 0 "_ba"
+               OpMemberName %B 1 "bb"
+               OpName %VertexData "VertexData"
+               OpMemberName %VertexData 0 "_a"
+               OpMemberName %VertexData 1 "b"
+               OpName %_ ""
+               OpMemberName %CA 0 "foo"
+               OpMemberName %C 0 "_ca"
+               OpMemberName %DA 0 "foo"
+               OpMemberName %D 0 "da"
+               OpName %UBO "UBO"
+               OpMemberName %UBO 0 "_c"
+               OpMemberName %UBO 1 "d"
+               OpName %__0 ""
+               OpMemberName %E 0 "a"
+               OpName %SSBO "SSBO"
+               ;OpMemberName %SSBO 0 "e" Test that we don't try to assign bogus aliases.
+               OpMemberName %SSBO 1 "_e"
+               OpMemberName %SSBO 2 "f"
+               OpName %__1 ""
+               OpDecorate %VertexData Block
+               OpDecorate %_ Location 0
+               OpMemberDecorate %CA 0 Offset 0
+               OpMemberDecorate %C 0 Offset 0
+               OpMemberDecorate %DA 0 Offset 0
+               OpMemberDecorate %D 0 Offset 0
+               OpMemberDecorate %UBO 0 Offset 0
+               OpMemberDecorate %UBO 1 Offset 16
+               OpDecorate %UBO Block
+               OpDecorate %__0 DescriptorSet 0
+               OpDecorate %__0 Binding 0
+               OpMemberDecorate %E 0 Offset 0
+               OpMemberDecorate %SSBO 0 Offset 0
+               OpMemberDecorate %SSBO 1 Offset 4
+               OpMemberDecorate %SSBO 2 Offset 8
+               OpDecorate %SSBO BufferBlock
+               OpDecorate %__1 DescriptorSet 0
+               OpDecorate %__1 Binding 1
+       %void = OpTypeVoid
+          %3 = OpTypeFunction %void
+        %int = OpTypeInt 32 1
+         %AA = OpTypeStruct %int
+         %AB = OpTypeStruct %int
+          %A = OpTypeStruct %AA %AB
+         %BA = OpTypeStruct %int
+         %BB = OpTypeStruct %int
+          %B = OpTypeStruct %BA %BB
+ %VertexData = OpTypeStruct %A %B
+%_ptr_Input_VertexData = OpTypePointer Input %VertexData
+          %_ = OpVariable %_ptr_Input_VertexData Input
+         %CA = OpTypeStruct %int
+          %C = OpTypeStruct %CA
+         %DA = OpTypeStruct %int
+          %D = OpTypeStruct %DA
+        %UBO = OpTypeStruct %C %D
+%_ptr_Uniform_UBO = OpTypePointer Uniform %UBO
+        %__0 = OpVariable %_ptr_Uniform_UBO Uniform
+          %E = OpTypeStruct %int
+       %SSBO = OpTypeStruct %E %E %E
+%_ptr_Uniform_SSBO = OpTypePointer Uniform %SSBO
+        %__1 = OpVariable %_ptr_Uniform_SSBO Uniform
+       %main = OpFunction %void None %3
+          %5 = OpLabel
+               OpReturn
+               OpFunctionEnd
diff --git a/spirv_glsl.cpp b/spirv_glsl.cpp
index 52cef4ec..f25c3b42 100644
--- a/spirv_glsl.cpp
+++ b/spirv_glsl.cpp
@@ -656,6 +656,7 @@ string CompilerGLSL::compile()
 		backend.support_case_fallthrough = false;
 
 	// Scan the SPIR-V to find trivial uses of extensions.
+	fixup_anonymous_struct_names();
 	fixup_type_alias();
 	reorder_type_alias();
 	build_function_control_flow_graphs_and_analyze();
@@ -15791,6 +15792,52 @@ void CompilerGLSL::reset_name_caches()
 	function_overloads.clear();
 }
 
+void CompilerGLSL::fixup_anonymous_struct_names(std::unordered_set<uint32_t> &visited, const SPIRType &type)
+{
+	if (visited.count(type.self))
+		return;
+	visited.insert(type.self);
+
+	for (uint32_t i = 0; i < uint32_t(type.member_types.size()); i++)
+	{
+		auto &mbr_type = get<SPIRType>(type.member_types[i]);
+
+		if (mbr_type.basetype == SPIRType::Struct)
+		{
+			// If there are multiple aliases, the output might be somewhat unpredictable,
+			// but the only real alternative in that case is to do nothing, which isn't any better.
+			// This check should be fine in practice.
+			if (get_name(mbr_type.self).empty() && !get_member_name(type.self, i).empty())
+			{
+				auto anon_name = join("anon_", get_member_name(type.self, i));
+				ParsedIR::sanitize_underscores(anon_name);
+				set_name(mbr_type.self, anon_name);
+			}
+
+			fixup_anonymous_struct_names(visited, mbr_type);
+		}
+	}
+}
+
+void CompilerGLSL::fixup_anonymous_struct_names()
+{
+	// HLSL codegen can often end up emitting anonymous structs inside blocks, which
+	// breaks GL linking since all names must match ...
+	// Try to emit sensible code, so attempt to find such structs and emit anon_$member.
+
+	// Breaks exponential explosion with weird type trees.
+	std::unordered_set<uint32_t> visited;
+
+	ir.for_each_typed_id<SPIRType>([&](uint32_t, SPIRType &type) {
+		if (type.basetype == SPIRType::Struct &&
+		    (has_decoration(type.self, DecorationBlock) ||
+		     has_decoration(type.self, DecorationBufferBlock)))
+		{
+			fixup_anonymous_struct_names(visited, type);
+		}
+	});
+}
+
 void CompilerGLSL::fixup_type_alias()
 {
 	// Due to how some backends work, the "master" type of type_alias must be a block-like type if it exists.
diff --git a/spirv_glsl.hpp b/spirv_glsl.hpp
index b892e0c3..7297239f 100644
--- a/spirv_glsl.hpp
+++ b/spirv_glsl.hpp
@@ -946,6 +946,8 @@ protected:
 
 	void fixup_type_alias();
 	void reorder_type_alias();
+	void fixup_anonymous_struct_names();
+	void fixup_anonymous_struct_names(std::unordered_set<uint32_t> &visited, const SPIRType &type);
 
 	static const char *vector_swizzle(int vecsize, int index);
 
diff --git a/spirv_hlsl.cpp b/spirv_hlsl.cpp
index d0d0e15a..e44ed846 100644
--- a/spirv_hlsl.cpp
+++ b/spirv_hlsl.cpp
@@ -5820,6 +5820,7 @@ string CompilerHLSL::compile()
 	// SM 4.1 does not support precise for some reason.
 	backend.support_precise_qualifier = hlsl_options.shader_model >= 50 || hlsl_options.shader_model == 40;
 
+	fixup_anonymous_struct_names();
 	fixup_type_alias();
 	reorder_type_alias();
 	build_function_control_flow_graphs_and_analyze();
diff --git a/spirv_msl.cpp b/spirv_msl.cpp
index d46dc484..75551b46 100644
--- a/spirv_msl.cpp
+++ b/spirv_msl.cpp
@@ -1371,6 +1371,7 @@ string CompilerMSL::compile()
 	for (auto &id : next_metal_resource_ids)
 		id = 0;
 
+	fixup_anonymous_struct_names();
 	fixup_type_alias();
 	replace_illegal_names();
 	sync_entry_point_aliases_and_names();