[mono][aot] Prefer concrete instances instead of gshared ones for met… (#78182)

* [mono][aot] Prefer concrete instances instead of gshared ones for methods containing type equality checks.

For example, calls to Vector<T>.IsSupported cannot be optimized away in a method where T
is T_BYTE because its written as:

```
            get => (typeof(T) == typeof(byte)) ||
                   (typeof(T) == typeof(double)) ||
```

and T_BYTE could be instantiated with an enum whose basetype is byte.

Fixes some of the issues in https://github.com/dotnet/runtime/issues/78163.

* Avoid an assert when compiling Vector<object> instances.

2 files changed, 17 insertions, 6 deletions

diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c
index a2348e0f379..eaa0b333baa 100644
--- a/src/mono/mono/mini/intrinsics.c
+++ b/src/mono/mono/mini/intrinsics.c
@@ -762,7 +762,7 @@ get_class_from_ldtoken_ins (MonoInst *ins)
  * their relation (EQ/NE/NONE).
  */
 static CompRelation
-get_rttype_ins_relation (MonoInst *ins1, MonoInst *ins2)
+get_rttype_ins_relation (MonoInst *ins1, MonoInst *ins2, gboolean *vtype_constrained_gparam)
 {
 	MonoClass *k1 = get_class_from_ldtoken_ins (ins1);
 	MonoClass *k2 = get_class_from_ldtoken_ins (ins2);
@@ -783,6 +783,7 @@ get_rttype_ins_relation (MonoInst *ins1, MonoInst *ins2)
 				if (MONO_TYPE_IS_PRIMITIVE (t2) || MONO_TYPE_ISSTRUCT (t2))
 					rel = CMP_NE;
 			} else if (MONO_TYPE_IS_PRIMITIVE (constraint1)) {
+				*vtype_constrained_gparam = TRUE;
 				if (MONO_TYPE_IS_PRIMITIVE (t2) && constraint1->type != t2->type)
 					rel = CMP_NE;
 				else if (MONO_TYPE_IS_REFERENCE (t2))
@@ -1879,7 +1880,8 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
 		}
 	} else if (cmethod->klass == mono_defaults.systemtype_class && !strcmp (cmethod->name, "op_Equality") &&
 			args [0]->klass == mono_defaults.runtimetype_class && args [1]->klass == mono_defaults.runtimetype_class) {
-		CompRelation rel = get_rttype_ins_relation (args [0], args [1]);
+		gboolean vtype_constrained_gparam = FALSE;
+		CompRelation rel = get_rttype_ins_relation (args [0], args [1], &vtype_constrained_gparam);
 		if (rel == CMP_EQ) {
 			if (cfg->verbose_level > 2)
 				printf ("-> true\n");
@@ -1894,11 +1896,16 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
 			ins->dreg = alloc_preg (cfg);
 			ins->type = STACK_I4;
 			MONO_ADD_INS (cfg->cbb, ins);
+
+			/* Type checks in gshared methods like Vector<T>.IsSupported can not be optimized away if the type is T_BYTE etc. */
+			if (cfg->gshared && !cfg->gsharedvt && vtype_constrained_gparam)
+				cfg->prefer_instances = TRUE;
 		}
 		return ins;
 	} else if (cmethod->klass == mono_defaults.systemtype_class && !strcmp (cmethod->name, "op_Inequality") &&
 			args [0]->klass == mono_defaults.runtimetype_class && args [1]->klass == mono_defaults.runtimetype_class) {
-		CompRelation rel = get_rttype_ins_relation (args [0], args [1]);
+		gboolean vtype_constrained_gparam = FALSE;
+		CompRelation rel = get_rttype_ins_relation (args [0], args [1], &vtype_constrained_gparam);
 		if (rel == CMP_NE) {
 			if (cfg->verbose_level > 2)
 				printf ("-> true\n");
@@ -1913,6 +1920,9 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
 			ins->dreg = alloc_preg (cfg);
 			ins->type = STACK_I4;
 			MONO_ADD_INS (cfg->cbb, ins);
+
+			if (cfg->gshared && !cfg->gsharedvt && vtype_constrained_gparam)
+				cfg->prefer_instances = TRUE;
 		}
 		return ins;
 	} else if (cmethod->klass == mono_defaults.systemtype_class && !strcmp (cmethod->name, "get_IsValueType") &&
diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c
index 5b52af08953..fbe696fe41e 100644
--- a/src/mono/mono/mini/simd-intrinsics.c
+++ b/src/mono/mono/mini/simd-intrinsics.c
@@ -1697,6 +1697,10 @@ emit_vector64_vector128_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
 
 	MonoClass *klass = cmethod->klass;
 	MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0];
+
+	if (!MONO_TYPE_IS_VECTOR_PRIMITIVE (etype))
+		return NULL;
+
 	int size = mono_class_value_size (klass, NULL);
 	int esize = mono_class_value_size (mono_class_from_mono_type_internal (etype), NULL);
 	g_assert (size > 0);
@@ -1704,9 +1708,6 @@ emit_vector64_vector128_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign
 	int len = size / esize;
 	MonoTypeEnum arg0_type = fsig->param_count > 0 ? get_underlying_type (fsig->params [0]) : MONO_TYPE_VOID;
 
-	if (!MONO_TYPE_IS_VECTOR_PRIMITIVE (etype))
-		return NULL;
-
 	if (cfg->verbose_level > 1) {
 		char *name = mono_method_full_name (cmethod, TRUE);
 		printf ("  SIMD intrinsic %s\n", name);