[2018-06] [arm64] Remove the limitation on the number of nullable arguments for dyncalls (#11266)

* [arm64] Remove the limitation on the number of nullable arguments for dyncalls. (#11186)

* [arm64] Remove the limitation on the number of nullable arguments for dyncalls.

Fixes https://github.com/xamarin/xamarin-macios/issues/4984.

* [amd64] Remove the limitation on the number of nullable arguments for dyncalls.

* [jit] Add tests for dyncalls with lots of nullable arguments.

* fix merge conflict

5 files changed, 89 insertions, 13 deletions

diff --git a/mono/mini/aot-tests.cs b/mono/mini/aot-tests.cs
index c69b1c0f8b4..e062e6dc342 100644
--- a/mono/mini/aot-tests.cs
+++ b/mono/mini/aot-tests.cs
@@ -282,6 +282,14 @@ class Tests
 		}
 
 		[MethodImplAttribute (MethodImplOptions.NoInlining)]
+		public static int NullableMany(long? i1, long? i2, long? i3, long? i4, long? i5, long? i6, long? i7, long? i8,
+										long? i11, long? i12, long? i13, long? i14, long? i15, long? i16, long? i17, long? i18,
+										long? i21, long? i22, long? i23, long? i24, long? i25, long? i26, long? i27, long? i28,
+										long? i31, long? i32, long? i33, long? i34, long? i35, long? i36, long? i37, long? i38) {
+			return (int)((i1 + i8 + i11 + i18 + i21 + i28 + i31 + i38).Value);
+		}
+
+		[MethodImplAttribute (MethodImplOptions.NoInlining)]
 		public static Nullable<T> GetNull<T>() where T : struct {
 			return null;
 		}
@@ -321,6 +329,11 @@ class Tests
 		res2 = (int?)typeof (NullableMethods).GetMethod ("GetNull").MakeGenericMethod (new Type [] { typeof (int) }).Invoke (null, new object [] { });
 		if (res2.HasValue)
 			return 5;
+
+		NullableMethods.NullableMany (1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8);
+		res2 = (int?)typeof (NullableMethods).GetMethod ("NullableMany").Invoke (null, new object [] { 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L });
+		if (res2 != 36)
+			return 6;
 		return 0;
 	}
 
diff --git a/mono/mini/mini-amd64.c b/mono/mini/mini-amd64.c
index a4b3c5ece9c..5facae19d81 100644
--- a/mono/mini/mini-amd64.c
+++ b/mono/mini/mini-amd64.c
@@ -2553,7 +2553,7 @@ mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
 typedef struct {
 	MonoMethodSignature *sig;
 	CallInfo *cinfo;
-	int nstack_args;
+	int nstack_args, nullable_area;
 } ArchDynCallInfo;
 
 static gboolean
@@ -2603,7 +2603,7 @@ mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
 {
 	ArchDynCallInfo *info;
 	CallInfo *cinfo;
-	int i;
+	int i, aindex;
 
 	cinfo = get_call_info (NULL, sig);
 
@@ -2628,6 +2628,34 @@ mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
 			break;
 		}
 	}
+
+	for (aindex = 0; aindex < sig->param_count; aindex++) {
+		MonoType *t = sig->params [aindex];
+		ArgInfo *ainfo = &cinfo->args [aindex + sig->hasthis];
+
+		if (t->byref)
+			continue;
+
+		switch (t->type) {
+		case MONO_TYPE_GENERICINST:
+			if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
+				MonoClass *klass = mono_class_from_mono_type (t);
+				int size;
+
+				if (!(ainfo->storage == ArgValuetypeInReg || ainfo->storage == ArgOnStack)) {
+					/* Nullables need a temporary buffer, its stored at the end of DynCallArgs.regs after the stack args */
+					size = mono_class_value_size (klass, NULL);
+					info->nullable_area += size;
+				}
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	info->nullable_area = ALIGN_TO (info->nullable_area, 16);
+
 	/* Align to 16 bytes */
 	if (info->nstack_args & 1)
 		info->nstack_args ++;
@@ -2655,7 +2683,7 @@ mono_arch_dyn_call_get_buf_size (MonoDynCallInfo *info)
 	ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
 
 	/* Extend the 'regs' field dynamically */
-	return sizeof (DynCallArgs) + (ainfo->nstack_args * sizeof (mgreg_t));
+	return sizeof (DynCallArgs) + (ainfo->nstack_args * sizeof (mgreg_t)) + ainfo->nullable_area;
 }
 
 #define PTR_TO_GREG(ptr) (mgreg_t)(ptr)
@@ -2684,6 +2712,7 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 	MonoMethodSignature *sig = dinfo->sig;
 	int buffer_offset = 0;
 	static int param_reg_to_index [16];
+	guint8 *nullable_buffer;
 	static gboolean param_reg_to_index_inited;
 
 	if (!param_reg_to_index_inited) {
@@ -2702,6 +2731,12 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 	freg = 0;
 	pindex = 0;
 
+	/* Stored after the stack arguments */
+	nullable_buffer = (guint8*)&(p->regs [PARAM_REGS + (dinfo->nstack_args * sizeof (mgreg_t))]);
+
+	if (dinfo->nullable_area)
+		printf ("%d\n", dinfo->nullable_area);
+
 	if (sig->hasthis || dinfo->cinfo->vret_arg_index == 1) {
 		p->regs [greg ++] = PTR_TO_GREG(*(args [arg_index ++]));
 		if (!sig->hasthis)
@@ -2786,9 +2821,13 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 					int size;
 
 					size = mono_class_value_size (klass, NULL);
-					nullable_buf = p->buffer + buffer_offset;
-					buffer_offset += size;
-					g_assert (buffer_offset <= 256);
+					if (ainfo->storage == ArgValuetypeInReg || ainfo->storage == ArgOnStack) {
+						nullable_buf = g_alloca (size);
+					} else {
+						nullable_buf = nullable_buffer + buffer_offset;
+						buffer_offset += size;
+						g_assert (buffer_offset <= dinfo->nullable_area);
+					}
 
 					/* The argument pointed to by arg is either a boxed vtype or null */
 					mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
diff --git a/mono/mini/mini-amd64.h b/mono/mini/mini-amd64.h
index c8c077dbe13..71fb85c23a4 100644
--- a/mono/mini/mini-amd64.h
+++ b/mono/mini/mini-amd64.h
@@ -272,7 +272,6 @@ typedef struct {
 	double fregs [8];
 	mgreg_t has_fp;
 	mgreg_t nstack_args;
-	guint8 buffer [256];
 	/* This should come last as the structure is dynamically extended */
 	mgreg_t regs [PARAM_REGS];
 } DynCallArgs;
diff --git a/mono/mini/mini-arm64.c b/mono/mini/mini-arm64.c
index b4707111a36..dd2fd2e92d2 100644
--- a/mono/mini/mini-arm64.c
+++ b/mono/mini/mini-arm64.c
@@ -1583,7 +1583,7 @@ typedef struct {
 	CallInfo *cinfo;
 	MonoType *rtype;
 	MonoType **param_types;
-	int n_fpargs, n_fpret;
+	int n_fpargs, n_fpret, nullable_area;
 } ArchDynCallInfo;
 
 static gboolean
@@ -1636,7 +1636,7 @@ mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
 {
 	ArchDynCallInfo *info;
 	CallInfo *cinfo;
-	int i;
+	int i, aindex;
 
 	cinfo = get_call_info (NULL, sig);
 
@@ -1665,6 +1665,28 @@ mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
 	default:
 		break;
 	}
+
+	for (aindex = 0; aindex < sig->param_count; aindex++) {
+		MonoType *t = info->param_types [aindex];
+
+		if (t->byref)
+			continue;
+
+		switch (t->type) {
+		case MONO_TYPE_GENERICINST:
+			if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
+				MonoClass *klass = mono_class_from_mono_type (t);
+				int size;
+
+				/* Nullables need a temporary buffer, its stored at the end of DynCallArgs.regs after the stack args */
+				size = mono_class_value_size (klass, NULL);
+				info->nullable_area += size;
+			}
+			break;
+		default:
+			break;
+		}
+	}
 	
 	return (MonoDynCallInfo*)info;
 }
@@ -1685,7 +1707,7 @@ mono_arch_dyn_call_get_buf_size (MonoDynCallInfo *info)
 	ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
 
 	g_assert (ainfo->cinfo->stack_usage % MONO_ARCH_FRAME_ALIGNMENT == 0);
-	return sizeof (DynCallArgs) + ainfo->cinfo->stack_usage;
+	return sizeof (DynCallArgs) + ainfo->cinfo->stack_usage + ainfo->nullable_area;
 }
 
 static double
@@ -1713,6 +1735,7 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 	MonoMethodSignature *sig = dinfo->sig;
 	CallInfo *cinfo = dinfo->cinfo;
 	int buffer_offset = 0;
+	guint8 *nullable_buffer;
 
 	p->res = 0;
 	p->ret = ret;
@@ -1724,6 +1747,9 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 	greg = 0;
 	pindex = 0;
 
+	/* Stored after the stack arguments */
+	nullable_buffer = (guint8*)&(p->regs [PARAM_REGS + 1 + (cinfo->stack_usage / sizeof (mgreg_t))]);
+
 	if (sig->hasthis)
 		p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
 
@@ -1829,9 +1855,9 @@ mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, g
 					 * if the nullable param is passed by ref.
 					 */
 					size = mono_class_value_size (klass, NULL);
-					nullable_buf = p->buffer + buffer_offset;
+					nullable_buf = nullable_buffer + buffer_offset;
 					buffer_offset += size;
-					g_assert (buffer_offset <= 256);
+					g_assert (buffer_offset <= dinfo->nullable_area);
 
 					/* The argument pointed to by arg is either a boxed vtype or null */
 					mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
diff --git a/mono/mini/mini-arm64.h b/mono/mini/mini-arm64.h
index 96fb0b955a6..c1b0143d160 100644
--- a/mono/mini/mini-arm64.h
+++ b/mono/mini/mini-arm64.h
@@ -93,7 +93,6 @@ typedef struct {
 	guint8 *ret;
 	double fpregs [FP_PARAM_REGS];
 	int n_fpargs, n_fpret, n_stackargs;
-	guint8 buffer [256];
 	/* This should come last as the structure is dynamically extended */
 	/* The +1 is for r8 */
 	mgreg_t regs [PARAM_REGS + 1];