[interp] Fix virtual calls in mixed mode (#18526)

* [interp] Enable virtual calls to compiled code

If we have an aot-ed version of a method we should always go through it, because the IL for that method could be stripped.

* [interp] Disable jit call for wrappers

Some generic wrappers don't seem to work in mixed mode, probably due to the fact that the transition already reuses a gshared wrapper.

3 files changed, 64 insertions, 34 deletions

diff --git a/mono/mini/interp/interp-internals.h b/mono/mini/interp/interp-internals.h
index 86fc3a1da62..e062e2c4fda 100644
--- a/mono/mini/interp/interp-internals.h
+++ b/mono/mini/interp/interp-internals.h
@@ -123,6 +123,13 @@ typedef struct _InterpFrame InterpFrame;
 typedef void (*MonoFuncV) (void);
 typedef void (*MonoPIFunc) (void *callme, void *margs);
 
+
+typedef enum {
+	IMETHOD_CODE_INTERP,
+	IMETHOD_CODE_COMPILED,
+	IMETHOD_CODE_UNKNOWN
+} InterpMethodCodeType;
+
 /* 
  * Structure representing a method transformed for the interpreter 
  * This is domain specific
@@ -163,6 +170,7 @@ typedef struct _InterpMethod
 	MonoJitInfo *jinfo;
 	MonoDomain *domain;
 	MonoProfilerCallInstrumentationFlags prof_flags;
+	InterpMethodCodeType code_type;
 #ifdef ENABLE_EXPERIMENT_TIERED
 	MiniTieredCounter tiered_counter;
 #endif
@@ -261,6 +269,9 @@ mono_interp_get_imethod (MonoDomain *domain, MonoMethod *method, MonoError *erro
 void
 mono_interp_print_code (InterpMethod *imethod);
 
+gboolean
+mono_interp_jit_call_supported (MonoMethod *method, MonoMethodSignature *sig);
+
 static inline int
 mint_type(MonoType *type_)
 {
diff --git a/mono/mini/interp/interp.c b/mono/mini/interp/interp.c
index d1617eb9c56..a2ec894aa40 100644
--- a/mono/mini/interp/interp.c
+++ b/mono/mini/interp/interp.c
@@ -552,6 +552,7 @@ mono_interp_get_imethod (MonoDomain *domain, MonoMethod *method, MonoError *erro
 	imethod->param_count = sig->param_count;
 	imethod->hasthis = sig->hasthis;
 	imethod->vararg = sig->call_convention == MONO_CALL_VARARG;
+	imethod->code_type = IMETHOD_CODE_UNKNOWN;
 	if (imethod->method->string_ctor)
 		imethod->rtype = m_class_get_byval_arg (mono_defaults.string_class);
 	else
@@ -2282,7 +2283,7 @@ jit_call_cb (gpointer arg)
 	}
 }
 
-static MONO_NEVER_INLINE stackval *
+static MONO_NEVER_INLINE void
 do_jit_call (stackval *sp, unsigned char *vt_sp, ThreadContext *context, InterpFrame *frame, InterpMethod *rmethod, MonoError *error)
 {
 	MonoMethodSignature *sig;
@@ -2310,7 +2311,7 @@ do_jit_call (stackval *sp, unsigned char *vt_sp, ThreadContext *context, InterpF
 		mono_error_assert_ok (error);
 
 		gpointer addr = mono_jit_compile_method_jit_only (method, error);
-		return_val_if_nok (error, NULL);
+		return_if_nok (error);
 		g_assert (addr);
 
 		rmethod->jit_addr = addr;
@@ -2322,10 +2323,6 @@ do_jit_call (stackval *sp, unsigned char *vt_sp, ThreadContext *context, InterpF
 		sig = rmethod->jit_sig;
 	}
 
-	sp -= sig->param_count;
-	if (sig->hasthis)
-		--sp;
-
 	ftndesc.addr = rmethod->jit_addr;
 	ftndesc.arg = NULL;
 
@@ -2407,7 +2404,7 @@ do_jit_call (stackval *sp, unsigned char *vt_sp, ThreadContext *context, InterpF
 			MonoObject *obj = mono_llvm_load_exception ();
 			g_assert (obj);
 			mono_error_set_exception_instance (error, (MonoException*)obj);
-			return sp;
+			return;
 		}
 	} else {
 		jit_call_cb (&cb_data);
@@ -2475,8 +2472,6 @@ do_jit_call (stackval *sp, unsigned char *vt_sp, ThreadContext *context, InterpF
 		g_assert_not_reached ();
 		break;
 	}
-
-	return sp;
 }
 
 static MONO_NEVER_INLINE void
@@ -3866,8 +3861,6 @@ main_loop:
 			gboolean is_void = *ip == MINT_VCALLVIRT_FAST;
 			int slot;
 
-			// FIXME Have it handle also remoting calls and use a single opcode for virtual calls
-
 			frame->ip = ip;
 
 			imethod = (InterpMethod*)frame->imethod->data_items [ip [1]];
@@ -3888,30 +3881,51 @@ main_loop:
 				gpointer unboxed = mono_object_unbox_internal (this_arg);
 				sp [0].data.p = unboxed;
 			}
+retry_callvirt_fast:
+			if (imethod->code_type == IMETHOD_CODE_INTERP) {
+				SAVE_INTERP_STATE (frame);
+
+				if (G_UNLIKELY (!imethod->transformed)) {
+					MonoException *ex;
+					gboolean tracing;
+
+					child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval);
+					method_entry (context, child_frame, &tracing, &ex);
+					if (G_UNLIKELY (ex)) {
+						frame = child_frame;
+						frame->ip = NULL;
+						THROW_EX (ex, NULL);
+						EXCEPTION_CHECKPOINT;
+					}
+				} else {
+					child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval);
+					alloc_stack_data (context, child_frame, imethod->alloca_size);
+				}
 
-			SAVE_INTERP_STATE (frame);
+				frame = child_frame;
+				INIT_INTERP_STATE (frame, NULL);
+				clause_args = NULL;
+			} else if (imethod->code_type == IMETHOD_CODE_COMPILED) {
+				error_init_reuse (error);
+				do_jit_call (sp, vt_sp, context, frame, imethod, error);
+				if (!is_ok (error)) {
+					MonoException *ex = mono_error_convert_to_exception (error);
+					THROW_EX (ex, ip);
+				}
 
-			if (G_UNLIKELY (!imethod->transformed)) {
-				MonoException *ex;
-				gboolean tracing;
+				CHECK_RESUME_STATE (context);
 
-				child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval);
-				method_entry (context, child_frame, &tracing, &ex);
-				if (G_UNLIKELY (ex)) {
-					frame = child_frame;
-					frame->ip = NULL;
-					THROW_EX (ex, NULL);
-					EXCEPTION_CHECKPOINT;
-				}
+				if (imethod->rtype->type != MONO_TYPE_VOID)
+					sp++;
 			} else {
-				child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval);
-				alloc_stack_data (context, child_frame, imethod->alloca_size);
+				MonoMethodSignature *sig = mono_method_signature_internal (imethod->method);
+				if (mono_interp_jit_call_supported (imethod->method, sig))
+					imethod->code_type = IMETHOD_CODE_COMPILED;
+				else
+					imethod->code_type = IMETHOD_CODE_INTERP;
+				goto retry_callvirt_fast;
 			}
 
-			frame = child_frame;
-			INIT_INTERP_STATE (frame, NULL);
-			clause_args = NULL;
-
 			MINT_IN_BREAK;
 		}
 		MINT_IN_CASE(MINT_CALL_VARARG) {
@@ -3949,6 +3963,7 @@ main_loop:
 		MINT_IN_CASE(MINT_CALL)
 		MINT_IN_CASE(MINT_CALLVIRT)
 		MINT_IN_CASE(MINT_VCALLVIRT) {
+			// FIXME CALLVIRT opcodes are not used on netcore. We should kill them.
 			stackval *retval;
 			gboolean is_void = *ip == MINT_VCALL || *ip == MINT_VCALLVIRT;
 			gboolean is_virtual = *ip == MINT_CALLVIRT || *ip == MINT_VCALLVIRT;
@@ -4016,7 +4031,8 @@ main_loop:
 			InterpMethod *rmethod = (InterpMethod*)frame->imethod->data_items [ip [1]];
 			error_init_reuse (error);
 			frame->ip = ip;
-			sp = do_jit_call (sp, vt_sp, context, frame, rmethod, error);
+			sp -= rmethod->param_count + rmethod->hasthis;
+			do_jit_call (sp, vt_sp, context, frame, rmethod, error);
 			if (!is_ok (error)) {
 				MonoException *ex = mono_error_convert_to_exception (error);
 				THROW_EX (ex, ip);
@@ -4037,7 +4053,8 @@ main_loop:
 			error_init_reuse (error);
 			frame->ip = ip;
 
-			sp = do_jit_call (sp, vt_sp, context, frame, rmethod, error);
+			sp -= rmethod->param_count + rmethod->hasthis;
+			do_jit_call (sp, vt_sp, context, frame, rmethod, error);
 			if (!is_ok (error)) {
 				MonoException *ex = mono_error_convert_to_exception (error);
 				THROW_EX (ex, ip);
diff --git a/mono/mini/interp/transform.c b/mono/mini/interp/transform.c
index b59dec7bbbc..ad7154ef66b 100644
--- a/mono/mini/interp/transform.c
+++ b/mono/mini/interp/transform.c
@@ -716,8 +716,8 @@ get_data_item_index (TransformData *td, void *ptr)
 	return index;
 }
 
-static gboolean
-jit_call_supported (MonoMethod *method, MonoMethodSignature *sig)
+gboolean
+mono_interp_jit_call_supported (MonoMethod *method, MonoMethodSignature *sig)
 {
 	GSList *l;
 
@@ -733,6 +733,8 @@ jit_call_supported (MonoMethod *method, MonoMethodSignature *sig)
 		return FALSE;
 	if (method->string_ctor)
 		return FALSE;
+	if (method->wrapper_type != MONO_WRAPPER_NONE)
+		return FALSE;
 
 	if (mono_aot_only && m_class_get_image (method->klass)->aot_module && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) {
 		ERROR_DECL (error);
@@ -2376,7 +2378,7 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target
 			td->last_ins->data [1] = get_data_item_index (td, mono_method_signature_internal (target_method));
 		}
 #endif
-	} else if (!calli && !is_virtual && jit_call_supported (target_method, csignature)) {
+	} else if (!calli && !is_virtual && mono_interp_jit_call_supported (target_method, csignature)) {
 		interp_add_ins (td, MINT_JIT_CALL);
 		td->last_ins->data [0] = get_data_item_index (td, (void *)mono_interp_get_imethod (domain, target_method, error));
 		mono_error_assert_ok (error);