diff options
author | Zoltan Varga <vargaz@gmail.com> | 2019-11-26 06:31:32 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-11-26 06:31:32 +0300 |
commit | c4136ebd94567bb29557c94e7480e24fe72ffdac (patch) | |
tree | 1bedb126535a88396b48e92a7fca24281517202e | |
parent | 5b4cd5ce50e293548fd097622e41010807fcf2b8 (diff) |
[interp] Non-recursive interpreter (#16461)
* Aling alloca_size to 8.
* Prevent some false pinning in the monitor-resurrection.cs test when using the non-recursive interpreter.
* [runtime] Add a mono_compiler_barrier () function.
* Add a MONO_GET_SP opcode to pass to enter/exit_gc_safe_region, ldloca might not return a native stack address with the interpreter.
* Add an interpreter/gc callback to mark the interpreter stack.
* Copy only an integer in ENDFILTER.
* Add a native_stack_addr field to InterpFrame so the EH code has something to compare against native stack addresses.
* Add a FrameStack structure to allow allocation of InterpFrame structures and stack data outside the native stack.
* Allocate InterpFrames/stack data from a frame stack instead of the native stack.
* Make some calls non-recursive.
* Remove an assert.
* Use non-recursive calls in CALLVIRT_FAST as well.
* Fix stack size allocation for vararg calls.
* Add some micro optimizations to the calling code.
* Add more micro optimizations.
* [dtest] disable Crash test on interpreter
* Add some implicit conversions to STSFLD to fix a corefx test failure.
* Disable the new test.
-rw-r--r-- | mcs/class/Mono.Debugger.Soft/Test/dtest.cs | 1 | ||||
-rw-r--r-- | mono/cil/cil-opcodes.xml | 1 | ||||
-rw-r--r-- | mono/cil/opcode.def | 1 | ||||
-rw-r--r-- | mono/metadata/gc-internals.h | 4 | ||||
-rw-r--r-- | mono/metadata/marshal-ilgen.c | 25 | ||||
-rw-r--r-- | mono/metadata/sgen-mono.c | 12 | ||||
-rw-r--r-- | mono/mini/ee.h | 2 | ||||
-rw-r--r-- | mono/mini/iltests.il | 14 | ||||
-rw-r--r-- | mono/mini/interp-stubs.c | 12 | ||||
-rw-r--r-- | mono/mini/interp/interp-internals.h | 33 | ||||
-rw-r--r-- | mono/mini/interp/interp.c | 686 | ||||
-rw-r--r-- | mono/mini/interp/mintops.def | 1 | ||||
-rw-r--r-- | mono/mini/interp/transform.c | 39 | ||||
-rw-r--r-- | mono/mini/method-to-ir.c | 7 | ||||
-rw-r--r-- | mono/mini/mini-exceptions.c | 5 | ||||
-rw-r--r-- | mono/mini/mini-gc.c | 4 | ||||
-rw-r--r-- | mono/tests/monitor-resurrection.cs | 2 | ||||
-rw-r--r-- | mono/utils/mono-membar.h | 14 |
18 files changed, 677 insertions, 186 deletions
diff --git a/mcs/class/Mono.Debugger.Soft/Test/dtest.cs b/mcs/class/Mono.Debugger.Soft/Test/dtest.cs index 52da50a45db..599d3952e63 100644 --- a/mcs/class/Mono.Debugger.Soft/Test/dtest.cs +++ b/mcs/class/Mono.Debugger.Soft/Test/dtest.cs @@ -2476,6 +2476,7 @@ public class DebuggerTests [Test] [Category("NotOnWindows")] [Category ("AndroidSdksNotWorking")] + [Category ("NotWorkingRuntimeInterpreter")] /* See https://github.com/mono/mono/commit/0a90cf4303f8bea334bf826155b86a7269c61373 */ public void Crash () { string [] existingCrashFileEntries = Directory.GetFiles (".", "mono_crash*.json"); diff --git a/mono/cil/cil-opcodes.xml b/mono/cil/cil-opcodes.xml index d1c6145c3b8..365b258eaf3 100644 --- a/mono/cil/cil-opcodes.xml +++ b/mono/cil/cil-opcodes.xml @@ -324,4 +324,5 @@ <opcode name="mono_ldptr_profiler_allocation_count" input="Pop0" output="PushI" args="InlineNone" o1="0xF0" o2="0x1D" flow="next" /> <opcode name="mono_ld_delegate_method_ptr" input="Pop1" output="PushI" args="InlineNone" o1="0xF0" o2="0x1E" flow="next" /> <opcode name="mono_rethrow" input="PopRef" output="Push0" args="InlineNone" o1="0xF0" o2="0x1F" flow="throw" type="Objmodel" /> +<opcode name="mono_get_sp" input="Pop0" output="PushI" args="InlineNone" o1="0xF0" o2="0x20" flow="next" /> </opdesc> diff --git a/mono/cil/opcode.def b/mono/cil/opcode.def index c23ccd66303..43722f6d328 100644 --- a/mono/cil/opcode.def +++ b/mono/cil/opcode.def @@ -324,6 +324,7 @@ OPDEF(CEE_MONO_GET_RGCTX_ARG, "mono_get_rgctx_arg", Pop0, PushI, InlineNone, 0, OPDEF(CEE_MONO_LDPTR_PROFILER_ALLOCATION_COUNT, "mono_ldptr_profiler_allocation_count", Pop0, PushI, InlineNone, 0, 2, 0xF0, 0x1D, NEXT) OPDEF(CEE_MONO_LD_DELEGATE_METHOD_PTR, "mono_ld_delegate_method_ptr", Pop1, PushI, InlineNone, 0, 2, 0xF0, 0x1E, NEXT) OPDEF(CEE_MONO_RETHROW, "mono_rethrow", PopRef, Push0, InlineNone, 0, 2, 0xF0, 0x1F, ERROR) +OPDEF(CEE_MONO_GET_SP, "mono_get_sp", Pop0, PushI, InlineNone, 0, 2, 0xF0, 0x20, NEXT) #ifndef OPALIAS #define _MONO_CIL_OPALIAS_DEFINED_ #define OPALIAS(a,s,r) diff --git a/mono/metadata/gc-internals.h b/mono/metadata/gc-internals.h index 437ac385bde..fcd57e7cbc8 100644 --- a/mono/metadata/gc-internals.h +++ b/mono/metadata/gc-internals.h @@ -268,6 +268,10 @@ typedef struct { * tracking the provenances of objects. */ gpointer (*get_provenance_func) (void); + /* + * Same as thread_mark_func, mark the intepreter frames. + */ + void (*interp_mark_func) (gpointer thread_info, GcScanFunc func, gpointer gc_data, gboolean precise); } MonoGCCallbacks; /* Set the callback functions callable by the GC */ diff --git a/mono/metadata/marshal-ilgen.c b/mono/metadata/marshal-ilgen.c index 3f8daf3fc66..9cc0188bf31 100644 --- a/mono/metadata/marshal-ilgen.c +++ b/mono/metadata/marshal-ilgen.c @@ -1760,7 +1760,6 @@ mono_mb_emit_auto_layout_exception (MonoMethodBuilder *mb, MonoClass *klass) typedef struct EmitGCSafeTransitionBuilder { MonoMethodBuilder *mb; gboolean func_param; - int coop_gc_stack_dummy; int coop_gc_var; #ifndef DISABLE_COM int coop_cominterop_fnptr; @@ -1772,7 +1771,6 @@ gc_safe_transition_builder_init (GCSafeTransitionBuilder *builder, MonoMethodBui { builder->mb = mb; builder->func_param = func_param; - builder->coop_gc_stack_dummy = -1; builder->coop_gc_var = -1; #ifndef DISABLE_COM builder->coop_cominterop_fnptr = -1; @@ -1791,9 +1789,7 @@ static void gc_safe_transition_builder_add_locals (GCSafeTransitionBuilder *builder) { MonoType *int_type = mono_get_int_type(); - /* local 4, dummy local used to get a stack address for suspend funcs */ - builder->coop_gc_stack_dummy = mono_mb_add_local (builder->mb, int_type); - /* local 5, the local to be used when calling the suspend funcs */ + /* local 4, the local to be used when calling the suspend funcs */ builder->coop_gc_var = mono_mb_add_local (builder->mb, int_type); #ifndef DISABLE_COM if (!builder->func_param && MONO_CLASS_IS_IMPORT (builder->mb->method->klass)) { @@ -1826,7 +1822,8 @@ gc_safe_transition_builder_emit_enter (GCSafeTransitionBuilder *builder, MonoMet } #endif - mono_mb_emit_ldloc_addr (builder->mb, builder->coop_gc_stack_dummy); + mono_mb_emit_byte (builder->mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (builder->mb, CEE_MONO_GET_SP); mono_mb_emit_icall (builder->mb, mono_threads_enter_gc_safe_region_unbalanced); mono_mb_emit_stloc (builder->mb, builder->coop_gc_var); } @@ -1840,7 +1837,8 @@ static void gc_safe_transition_builder_emit_exit (GCSafeTransitionBuilder *builder) { mono_mb_emit_ldloc (builder->mb, builder->coop_gc_var); - mono_mb_emit_ldloc_addr (builder->mb, builder->coop_gc_stack_dummy); + mono_mb_emit_byte (builder->mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (builder->mb, CEE_MONO_GET_SP); mono_mb_emit_icall (builder->mb, mono_threads_exit_gc_safe_region_unbalanced); } @@ -1848,7 +1846,6 @@ static void gc_safe_transition_builder_cleanup (GCSafeTransitionBuilder *builder) { builder->mb = NULL; - builder->coop_gc_stack_dummy = -1; builder->coop_gc_var = -1; #ifndef DISABLE_COM builder->coop_cominterop_fnptr = -1; @@ -4227,7 +4224,7 @@ emit_thunk_invoke_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethod *method, Mono MonoImage *image = get_method_image (method); MonoMethodSignature *sig = mono_method_signature_internal (method); int param_count = sig->param_count + sig->hasthis + 1; - int pos_leave, coop_gc_var = 0, coop_gc_stack_dummy = 0; + int pos_leave, coop_gc_var = 0; MonoExceptionClause *clause; MonoType *object_type = mono_get_object_type (); #if defined (TARGET_WASM) @@ -4244,9 +4241,7 @@ emit_thunk_invoke_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethod *method, Mono mono_mb_add_local (mb, sig->ret); if (do_blocking_transition) { - /* local 4, dummy local used to get a stack address for suspend funcs */ - coop_gc_stack_dummy = mono_mb_add_local (mb, mono_get_int_type ()); - /* local 5, the local to be used when calling the suspend funcs */ + /* local 4, the local to be used when calling the suspend funcs */ coop_gc_var = mono_mb_add_local (mb, mono_get_int_type ()); } @@ -4256,7 +4251,8 @@ emit_thunk_invoke_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethod *method, Mono mono_mb_emit_byte (mb, CEE_STIND_REF); if (do_blocking_transition) { - mono_mb_emit_ldloc_addr (mb, coop_gc_stack_dummy); + mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (mb, CEE_MONO_GET_SP); mono_mb_emit_icall (mb, mono_threads_enter_gc_unsafe_region_unbalanced); mono_mb_emit_stloc (mb, coop_gc_var); } @@ -4332,7 +4328,8 @@ emit_thunk_invoke_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethod *method, Mono if (do_blocking_transition) { mono_mb_emit_ldloc (mb, coop_gc_var); - mono_mb_emit_ldloc_addr (mb, coop_gc_stack_dummy); + mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (mb, CEE_MONO_GET_SP); mono_mb_emit_icall (mb, mono_threads_exit_gc_unsafe_region_unbalanced); } diff --git a/mono/metadata/sgen-mono.c b/mono/metadata/sgen-mono.c index d350a86231b..ae37cce1485 100644 --- a/mono/metadata/sgen-mono.c +++ b/mono/metadata/sgen-mono.c @@ -2293,6 +2293,10 @@ sgen_client_scan_thread_data (void *start_nursery, void *end_nursery, gboolean p SGEN_TV_GETTIME (scan_thread_data_start); + if (gc_callbacks.interp_mark_func) + /* The interpreter code uses only compiler write barriers so have to synchronize with it */ + mono_memory_barrier_process_wide (); + FOREACH_THREAD_EXCLUDE (info, MONO_THREAD_INFO_FLAGS_NO_GC) { int skip_reason = 0; void *aligned_stack_start; @@ -2354,6 +2358,14 @@ sgen_client_scan_thread_data (void *start_nursery, void *end_nursery, gboolean p } } } + if (gc_callbacks.interp_mark_func) { + PinHandleStackInteriorPtrData ud; + memset (&ud, 0, sizeof (ud)); + ud.start_nursery = (void**)start_nursery; + ud.end_nursery = (void**)end_nursery; + SGEN_LOG (3, "Scanning thread %p interp stack", info); + gc_callbacks.interp_mark_func (&info->client_info.info, pin_handle_stack_interior_ptrs, &ud, precise); + } if (info->client_info.info.handle_stack) { /* Make two passes over the handle stack. On the imprecise pass, pin all diff --git a/mono/mini/ee.h b/mono/mini/ee.h index 57ff732e5bd..d2a6f20590f 100644 --- a/mono/mini/ee.h +++ b/mono/mini/ee.h @@ -51,6 +51,7 @@ typedef gpointer MonoInterpFrameHandle; MONO_EE_CALLBACK (gpointer, frame_get_local, (MonoInterpFrameHandle frame, int pos)) \ MONO_EE_CALLBACK (gpointer, frame_get_this, (MonoInterpFrameHandle frame)) \ MONO_EE_CALLBACK (gpointer, frame_get_res, (MonoInterpFrameHandle frame)) \ + MONO_EE_CALLBACK (gpointer, frame_get_native_stack_addr, (MonoInterpFrameHandle frame)) \ MONO_EE_CALLBACK (void, frame_arg_to_data, (MonoInterpFrameHandle frame, MonoMethodSignature *sig, int index, gpointer data)) \ MONO_EE_CALLBACK (void, data_to_frame_arg, (MonoInterpFrameHandle frame, MonoMethodSignature *sig, int index, gconstpointer data)) \ MONO_EE_CALLBACK (gpointer, frame_arg_to_storage, (MonoInterpFrameHandle frame, MonoMethodSignature *sig, int index)) \ @@ -60,6 +61,7 @@ typedef gpointer MonoInterpFrameHandle; MONO_EE_CALLBACK (void, stop_single_stepping, (void)) \ MONO_EE_CALLBACK (void, free_context, (gpointer)) \ MONO_EE_CALLBACK (void, cleanup, (void)) \ + MONO_EE_CALLBACK (void, mark_stack, (gpointer thread_info, GcScanFunc func, gpointer gc_data, gboolean precise)) \ typedef struct _MonoEECallbacks { diff --git a/mono/mini/iltests.il b/mono/mini/iltests.il index ad921256278..d28016e68fd 100644 --- a/mono/mini/iltests.il +++ b/mono/mini/iltests.il @@ -3335,4 +3335,18 @@ L_3: IL_0012: ldloc.2 IL_0013: ret } // end of method Tests::test_10_rconv_to_u8_ovf_un + + .field private static int64 TestLong + +/* Disabled until its fixed on i386 + .method public static int32 test_1_stsfld_conv () cil managed + { + ldc.i4.2 + stsfld int64 Tests::TestLong + ldsfld int64 Tests::TestLong + ldc.i8 2 + ceq + ret + } +*/ } diff --git a/mono/mini/interp-stubs.c b/mono/mini/interp-stubs.c index 3bbc4011515..922b671c2f3 100644 --- a/mono/mini/interp-stubs.c +++ b/mono/mini/interp-stubs.c @@ -71,6 +71,13 @@ stub_frame_get_res (MonoInterpFrameHandle frame) return NULL; } +static gpointer +stub_frame_get_native_stack_addr (MonoInterpFrameHandle frame) +{ + g_assert_not_reached (); + return NULL; +} + static void stub_start_single_stepping (void) { @@ -213,6 +220,11 @@ stub_free_context (gpointer context) g_assert_not_reached (); } +static void +stub_mark_stack (gpointer thread_data, GcScanFunc func, gpointer gc_data, gboolean precise) +{ +} + #undef MONO_EE_CALLBACK #define MONO_EE_CALLBACK(ret, name, sig) stub_ ## name, diff --git a/mono/mini/interp/interp-internals.h b/mono/mini/interp/interp-internals.h index f2bc64e2636..181d3067d9f 100644 --- a/mono/mini/interp/interp-internals.h +++ b/mono/mini/interp/interp-internals.h @@ -176,14 +176,43 @@ typedef struct _InterpMethod #endif } InterpMethod; +typedef struct _StackFragment StackFragment; +struct _StackFragment { + guint8 *pos, *end; + struct _StackFragment *next; + double data [1]; +}; + +typedef struct { + StackFragment *first, *last, *current; + /* For GC sync */ + int inited; +} FrameStack; + +/* State of the interpreter main loop */ +typedef struct { + stackval *sp; + unsigned char *vt_sp; + const unsigned short *ip; + GSList *finally_ips; + gpointer clause_args; +} InterpState; + struct _InterpFrame { InterpFrame *parent; /* parent */ InterpMethod *imethod; /* parent */ stackval *retval; /* parent */ stackval *stack_args; /* parent */ stackval *stack; + /* An address on the native stack associated with the frame, used during EH */ + gpointer native_stack_addr; + /* Stack fragments this frame was allocated from */ + StackFragment *iframe_frag, *data_frag; /* exception info */ const unsigned short *ip; + /* State saved before calls */ + /* This is valid if state.ip != NULL */ + InterpState state; }; #define frame_locals(frame) (((guchar*)((frame)->stack)) + (frame)->imethod->stack_size + (frame)->imethod->vt_stack_size) @@ -199,6 +228,10 @@ typedef struct { MonoJitExceptionInfo *handler_ei; /* Exception that is being thrown. Set with rest of resume state */ guint32 exc_gchandle; + /* Stack of InterpFrames */ + FrameStack iframe_stack; + /* Stack of frame data */ + FrameStack data_stack; } ThreadContext; typedef struct { diff --git a/mono/mini/interp/interp.c b/mono/mini/interp/interp.c index 37712682465..c9564584da5 100644 --- a/mono/mini/interp/interp.c +++ b/mono/mini/interp/interp.c @@ -25,7 +25,8 @@ #include <mono/utils/gc_wrapper.h> #include <mono/utils/mono-math.h> #include <mono/utils/mono-counters.h> -#include "mono/utils/mono-tls-inline.h" +#include <mono/utils/mono-tls-inline.h> +#include <mono/utils/mono-membar.h> #ifdef HAVE_ALLOCA_H # include <alloca.h> @@ -99,14 +100,179 @@ typedef struct { InterpFrame *base_frame; } FrameClauseArgs; +/* + * This code synchronizes with interp_mark_stack () using compiler memory barriers. + */ + +static StackFragment* +stack_frag_new (int size) +{ + StackFragment *frag = (StackFragment*)g_malloc (size); + + frag->pos = (guint8*)&frag->data; + frag->end = (guint8*)frag + size; + frag->next = NULL; + return frag; +} + static void -init_frame (InterpFrame *frame, InterpFrame *parent_frame, InterpMethod *rmethod, stackval *method_args, stackval *method_retval) +frame_stack_init (FrameStack *stack, int size) +{ + StackFragment *frag; + + frag = stack_frag_new (size); + stack->first = stack->last = stack->current = frag; + mono_compiler_barrier (); + stack->inited = 1; +} + +static StackFragment* +add_frag (FrameStack *stack, int size) +{ + StackFragment *new_frag; + + // FIXME: + int frag_size = 4096; + if (size > frag_size) + frag_size = size + sizeof (StackFragment); + new_frag = stack_frag_new (frag_size); + mono_compiler_barrier (); + stack->last->next = new_frag; + stack->last = new_frag; + stack->current = new_frag; + return new_frag; +} + +static MONO_ALWAYS_INLINE gpointer +frame_stack_alloc_ovf (FrameStack *stack, int size, StackFragment **out_frag) +{ + StackFragment *current = stack->current; + gpointer res; + + if (current->next && current->next->pos + size <= current->next->end) { + current = stack->current = current->next; + current->pos = (guint8*)¤t->data; + } else { + current = add_frag (stack, size); + } + g_assert (current->pos + size <= current->end); + res = (gpointer)current->pos; + current->pos += size; + + mono_compiler_barrier (); + + if (out_frag) + *out_frag = current; + return res; +} + +static MONO_ALWAYS_INLINE gpointer +frame_stack_alloc (FrameStack *stack, int size, StackFragment **out_frag) { - frame->parent = parent_frame; - frame->stack_args = method_args; - frame->retval = method_retval; - frame->imethod = rmethod; + StackFragment *current = stack->current; + gpointer res; + + if (G_LIKELY (current->pos + size <= current->end)) { + res = current->pos; + current->pos += size; + mono_compiler_barrier (); + + if (out_frag) + *out_frag = current; + return res; + } else { + return frame_stack_alloc_ovf (stack, size, out_frag); + } +} + +static MONO_ALWAYS_INLINE void +frame_stack_pop (FrameStack *stack, StackFragment *frag, gpointer pos) +{ + g_assert ((guint8*)pos >= (guint8*)&frag->data && (guint8*)pos <= (guint8*)frag->end); + stack->current = frag; + mono_compiler_barrier (); + stack->current->pos = (guint8*)pos; + mono_compiler_barrier (); + //memset (stack->current->pos, 0, stack->current->end - stack->current->pos); +} + +static void +frame_stack_free (FrameStack *stack) +{ + stack->inited = 0; + mono_compiler_barrier (); + StackFragment *frag = stack->first; + while (frag) { + StackFragment *next = frag->next; + g_free (frag); + frag = next; + } +} + +/* + * alloc_frame: + * + * Allocate a new frame from the frame stack. + */ +static MONO_ALWAYS_INLINE InterpFrame* +alloc_frame (ThreadContext *ctx, gpointer native_stack_addr, InterpFrame *parent, InterpMethod *imethod, stackval *args, stackval *retval) +{ + StackFragment *frag; + InterpFrame *frame; + + // FIXME: Add stack overflow checks + frame = (InterpFrame*)frame_stack_alloc (&ctx->iframe_stack, sizeof (InterpFrame), &frag); + + frame->iframe_frag = frag; + frame->parent = parent; + frame->native_stack_addr = native_stack_addr; + frame->imethod = imethod; + frame->stack_args = args; + frame->retval = retval; + frame->stack = NULL; frame->ip = NULL; + frame->state.ip = NULL; + + return frame; +} + +/* + * alloc_data_stack: + * + * Allocate stack space for a frame. + */ +static MONO_ALWAYS_INLINE void +alloc_stack_data (ThreadContext *ctx, InterpFrame *frame, int size) +{ + StackFragment *frag; + gpointer res; + + res = frame_stack_alloc (&ctx->data_stack, size, &frag); + + frame->stack = (stackval*)res; + frame->data_frag = frag; +} + +static gpointer +alloc_extra_stack_data (ThreadContext *ctx, int size) +{ + StackFragment *frag; + + return frame_stack_alloc (&ctx->data_stack, size, &frag); +} + +/* + * pop_frame: + * + * Pop FRAME and its child frames from the frame stack. + * FRAME stays valid until the next alloc_frame () call. + */ +static void +pop_frame (ThreadContext *context, InterpFrame *frame) +{ + if (frame->stack) + frame_stack_pop (&context->data_stack, frame->data_frag, frame->stack); + frame_stack_pop (&context->iframe_stack, frame->iframe_frag, frame); } #define interp_exec_method(frame, context, error) interp_exec_method_full ((frame), (context), NULL, error) @@ -186,7 +352,6 @@ debug_enter (InterpFrame *frame, int *tracing) } } - #define DEBUG_LEAVE() \ if (tracing) { \ char *mn, *args; \ @@ -276,6 +441,12 @@ get_context (void) ThreadContext *context = (ThreadContext *) mono_native_tls_get_value (thread_context_id); if (context == NULL) { context = g_new0 (ThreadContext, 1); + /* + * Use two stacks, one for InterpFrame structures, one for data. + * This is useful because InterpFrame structures don't need to be GC tracked. + */ + frame_stack_init (&context->iframe_stack, 8192); + frame_stack_init (&context->data_stack, 8192); set_context (context); } return context; @@ -286,6 +457,8 @@ interp_free_context (gpointer ctx) { ThreadContext *context = (ThreadContext*)ctx; + frame_stack_free (&context->iframe_stack); + frame_stack_free (&context->data_stack); g_free (context); } @@ -915,7 +1088,7 @@ interp_throw (ThreadContext *context, MonoException *ex, InterpFrame *frame, con MonoContext ctx; memset (&ctx, 0, sizeof (MonoContext)); - MONO_CONTEXT_SET_SP (&ctx, frame); + MONO_CONTEXT_SET_SP (&ctx, frame->native_stack_addr); /* * Call the JIT EH code. The EH code will call back to us using: @@ -1690,7 +1863,7 @@ dump_args (InterpFrame *inv) static MonoObject* interp_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObject **exc, MonoError *error) { - InterpFrame frame; + InterpFrame *frame; ThreadContext *context = get_context (); MonoMethodSignature *sig = mono_method_signature_internal (method); MonoClass *klass = mono_class_from_mono_type_internal (sig->ret); @@ -1722,9 +1895,9 @@ interp_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObject InterpMethod *imethod = mono_interp_get_imethod (domain, invoke_wrapper, error); mono_error_assert_ok (error); - init_frame (&frame, NULL, imethod, args, &result); + frame = alloc_frame (context, &result, NULL, imethod, args, &result); - interp_exec_method (&frame, context, error); + interp_exec_method (frame, context, error); if (context->has_resume_state) { // This can happen on wasm !? @@ -1750,11 +1923,12 @@ typedef struct { static void interp_entry (InterpEntryData *data) { - InterpFrame frame; + InterpFrame *frame; InterpMethod *rmethod; ThreadContext *context; stackval result; stackval *args; + stackval *retval; MonoMethod *method; MonoMethodSignature *sig; MonoType *type; @@ -1813,23 +1987,24 @@ interp_entry (InterpEntryData *data) } memset (&result, 0, sizeof (result)); - init_frame (&frame, NULL, data->rmethod, args, &result); + retval = &result; + frame = alloc_frame (context, &result, NULL, data->rmethod, args, retval); type = rmethod->rtype; switch (type->type) { case MONO_TYPE_GENERICINST: if (!MONO_TYPE_IS_REFERENCE (type)) - frame.retval->data.vt = data->res; + retval->data.vt = data->res; break; case MONO_TYPE_VALUETYPE: - frame.retval->data.vt = data->res; + retval->data.vt = data->res; break; default: break; } ERROR_DECL (error); - interp_exec_method (&frame, context, error); + interp_exec_method (frame, context, error); g_assert (!context->has_resume_state); @@ -1849,11 +2024,11 @@ interp_entry (InterpEntryData *data) break; case MONO_TYPE_OBJECT: /* No need for a write barrier */ - *(MonoObject**)data->res = (MonoObject*)frame.retval->data.p; + *(MonoObject**)data->res = (MonoObject*)retval->data.p; break; case MONO_TYPE_GENERICINST: if (MONO_TYPE_IS_REFERENCE (type)) { - *(MonoObject**)data->res = (MonoObject*)frame.retval->data.p; + *(MonoObject**)data->res = (MonoObject*)retval->data.p; } else { /* Already set before the call */ } @@ -1862,7 +2037,7 @@ interp_entry (InterpEntryData *data) /* Already set before the call */ break; default: - stackval_to_data (type, frame.retval, data->res, FALSE); + stackval_to_data (type, retval, data->res, FALSE); break; } } @@ -2524,7 +2699,7 @@ interp_entry_general (gpointer this_arg, gpointer res, gpointer *args, gpointer static void interp_entry_from_trampoline (gpointer ccontext_untyped, gpointer rmethod_untyped) { - InterpFrame frame; + InterpFrame *frame; ThreadContext *context; stackval result; stackval *args; @@ -2551,22 +2726,22 @@ interp_entry_from_trampoline (gpointer ccontext_untyped, gpointer rmethod_untype args = (stackval*)alloca (sizeof (stackval) * (sig->param_count + (sig->hasthis ? 1 : 0))); - init_frame (&frame, NULL, rmethod, args, &result); + frame = alloc_frame (context, &result, NULL, rmethod, args, &result); /* Allocate storage for value types */ for (i = 0; i < sig->param_count; i++) { MonoType *type = sig->params [i]; - alloc_storage_for_stackval (&frame.stack_args [i + sig->hasthis], type, sig->pinvoke); + alloc_storage_for_stackval (&frame->stack_args [i + sig->hasthis], type, sig->pinvoke); } if (sig->ret->type != MONO_TYPE_VOID) - alloc_storage_for_stackval (frame.retval, sig->ret, sig->pinvoke); + alloc_storage_for_stackval (frame->retval, sig->ret, sig->pinvoke); /* Copy the args saved in the trampoline to the frame stack */ - mono_arch_get_native_call_context_args (ccontext, &frame, sig); + mono_arch_get_native_call_context_args (ccontext, frame, sig); ERROR_DECL (error); - interp_exec_method (&frame, context, error); + interp_exec_method (frame, context, error); g_assert (!context->has_resume_state); @@ -2574,7 +2749,8 @@ interp_entry_from_trampoline (gpointer ccontext_untyped, gpointer rmethod_untype mono_threads_detach_coop (orig_domain, &attach_cookie); /* Write back the return value */ - mono_arch_set_native_call_context_ret (ccontext, &frame, sig); + /* 'frame' is still valid */ + mono_arch_set_native_call_context_ret (ccontext, frame, sig); } #else @@ -3217,33 +3393,6 @@ mono_interp_store_remote_field_vt (InterpFrame* frame, const guint16* ip, stackv return ALIGN_TO (i32, MINT_VT_ALIGNMENT); } -static MONO_ALWAYS_INLINE stackval* -mono_interp_call (InterpFrame *frame, ThreadContext *context, InterpFrame *child_frame, const guint16 *ip, stackval *sp, guchar *vt_sp, gboolean is_virtual, gboolean is_void) -{ - frame->ip = ip; - - child_frame->imethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; - if (!is_void) { - sp->data.p = vt_sp; - child_frame->retval = sp; - } - - /* decrement by the actual number of args */ - sp -= ip [2]; - - if (is_virtual) { - MonoObject *this_arg = (MonoObject*)sp->data.p; - - child_frame->imethod = get_virtual_method (child_frame->imethod, this_arg->vtable); - if (m_class_is_valuetype (this_arg->vtable->klass) && m_class_is_valuetype (child_frame->imethod->method->klass)) { - /* unbox */ - gpointer unboxed = mono_object_unbox_internal (this_arg); - sp [0].data.p = unboxed; - } - } - return sp; -} - // varargs in wasm consumes extra linear stack per call-site. // These g_warning/g_error wrappers fix that. It is not the // small wasm stack, but conserving it is still desirable. @@ -3265,27 +3414,82 @@ g_error_xsx (const char *format, int x1, const char *s, int x2) g_error (format, x1, s, x2); } +static MONO_ALWAYS_INLINE void +method_entry (ThreadContext *context, InterpFrame *frame, gboolean *out_tracing, MonoException **out_ex) +{ +#if DEBUG_INTERP + debug_enter (frame, out_tracing); +#endif + + *out_ex = NULL; + if (!G_UNLIKELY (frame->imethod->transformed)) { +#if DEBUG_INTERP + char *mn = mono_method_full_name (frame->imethod->method, TRUE); + g_print ("(%p) Transforming %s\n", mono_thread_internal_current (), mn); + g_free (mn); +#endif + + frame->ip = NULL; + MonoException *ex = do_transform_method (frame, context); + if (ex) { + *out_ex = ex; + return; + } + } + + alloc_stack_data (context, frame, frame->imethod->alloca_size); +} + +/* Save the state of the interpeter main loop into FRAME */ +#define SAVE_INTERP_STATE(frame) do { \ + frame->state.ip = ip; \ + frame->state.sp = sp; \ + frame->state.vt_sp = vt_sp; \ + frame->state.finally_ips = finally_ips; \ + frame->state.clause_args = clause_args; \ + } while (0) + +/* Load and clear state from FRAME */ +#define LOAD_INTERP_STATE(frame) do { \ + ip = frame->state.ip; \ + sp = frame->state.sp; \ + vt_sp = frame->state.vt_sp; \ + finally_ips = frame->state.finally_ips; \ + clause_args = (FrameClauseArgs*)frame->state.clause_args; \ + locals = (unsigned char *)frame->stack + frame->imethod->stack_size + frame->imethod->vt_stack_size; \ + frame->state.ip = NULL; \ + } while (0) + +/* Initialize interpreter state for executing FRAME */ +#define INIT_INTERP_STATE(frame, _clause_args) do { \ + ip = _clause_args ? ((FrameClauseArgs*)_clause_args)->start_with_ip : (frame)->imethod->code; \ + sp = (frame)->stack; \ + vt_sp = (unsigned char *) sp + (frame)->imethod->stack_size; \ + locals = (unsigned char *) vt_sp + (frame)->imethod->vt_stack_size; \ + finally_ips = NULL; \ + } while (0) + /* - * If EXIT_AT_FINALLY is not -1, exit after exiting the finally clause with that index. - * If BASE_FRAME is not NULL, copy arguments/locals from BASE_FRAME. + * If CLAUSE_ARGS is non-null, start executing from it. * The ERROR argument is used to avoid declaring an error object for every interp frame, its not used * to return error information. - * - * Currently this method uses 0x88 of stack space on 64bit gcc. Make sure to keep it under control. + * FRAME is only valid until the next call to alloc_frame (). */ static void interp_exec_method_full (InterpFrame *frame, ThreadContext *context, FrameClauseArgs *clause_args, MonoError *error) { - InterpFrame child_frame; - GSList *finally_ips = NULL; + /* Interpreter main loop state (InterpState) */ const guint16 *ip = NULL; stackval *sp; + unsigned char *vt_sp; + unsigned char *locals = NULL; + GSList *finally_ips = NULL; + + InterpFrame *child_frame; #if DEBUG_INTERP gint tracing = global_tracing; unsigned char *vtalloc; #endif - unsigned char *vt_sp; - unsigned char *locals = NULL; #if USE_COMPUTED_GOTO static void * const in_labels[] = { #define OPDEF(a,b,c,d,e,f) &&LAB_ ## a, @@ -3312,22 +3516,21 @@ interp_exec_method_full (InterpFrame *frame, ThreadContext *context, FrameClause } if (!clause_args) { - frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); - ip = frame->imethod->code; + //frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); + alloc_stack_data (context, frame, frame->imethod->alloca_size); } else { - ip = clause_args->start_with_ip; if (clause_args->base_frame) { - frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); + //frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); + alloc_stack_data (context, frame, frame->imethod->alloca_size); memcpy (frame->stack, clause_args->base_frame->stack, frame->imethod->alloca_size); } } - sp = frame->stack; - vt_sp = (unsigned char *) sp + frame->imethod->stack_size; + + INIT_INTERP_STATE (frame, clause_args); + #if DEBUG_INTERP vtalloc = vt_sp; #endif - locals = (unsigned char *) vt_sp + frame->imethod->vt_stack_size; - child_frame.parent = frame; if (clause_args && clause_args->filter_exception) { sp->data.p = clause_args->filter_exception; @@ -3510,7 +3713,8 @@ main_loop: * than the callee stack frame (at the interp level) */ if (realloc_frame) { - frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); + //frame->stack = (stackval*)g_alloca (frame->imethod->alloca_size); + alloc_stack_data (context, frame, frame->imethod->alloca_size); memset (frame->stack, 0, frame->imethod->alloca_size); sp = frame->stack; } @@ -3524,23 +3728,24 @@ main_loop: } MINT_IN_CASE(MINT_CALLI) { MonoMethodSignature *csignature; + stackval *retval; frame->ip = ip; csignature = (MonoMethodSignature*)frame->imethod->data_items [ip [1]]; ip += 2; --sp; - child_frame.imethod = (InterpMethod*)sp->data.p; + retval = sp; + child_frame = alloc_frame (context, &retval, frame, (InterpMethod*)sp->data.p, NULL, retval); sp->data.p = vt_sp; - child_frame.retval = sp; /* decrement by the actual number of args */ sp -= csignature->param_count; if (csignature->hasthis) --sp; - if (child_frame.imethod->method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) { - child_frame.imethod = mono_interp_get_imethod (frame->imethod->domain, mono_marshal_get_native_wrapper (child_frame.imethod->method, FALSE, FALSE), error); + if (child_frame->imethod->method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) { + child_frame->imethod = mono_interp_get_imethod (frame->imethod->domain, mono_marshal_get_native_wrapper (child_frame->imethod->method, FALSE, FALSE), error); mono_interp_error_cleanup (error); /* FIXME: don't swallow the error */ } @@ -3553,9 +3758,14 @@ main_loop: } } - if (csignature->ret->type != MONO_TYPE_VOID) - goto common_call; - goto common_vcall; + child_frame->stack_args = sp; + interp_exec_method (child_frame, context, error); + CHECK_RESUME_STATE (context); + if (csignature->ret->type != MONO_TYPE_VOID) { + *sp = *retval; + sp++; + } + MINT_IN_BREAK; } MINT_IN_CASE(MINT_CALLI_NAT_FAST) { gpointer target_ip = sp [-1].data.p; @@ -3573,75 +3783,105 @@ main_loop: MINT_IN_BREAK; } MINT_IN_CASE(MINT_CALLI_NAT) { + stackval *retval; + MonoMethodSignature* csignature; frame->ip = ip; - MonoMethodSignature* csignature = (MonoMethodSignature*)frame->imethod->data_items [ip [1]]; + csignature = (MonoMethodSignature*)frame->imethod->data_items [ip [1]]; ip += 3; --sp; guchar* const code = (guchar*)sp->data.p; - child_frame.imethod = NULL; + retval = sp; + child_frame = alloc_frame (context, &retval, frame, NULL, NULL, retval); sp->data.p = vt_sp; - child_frame.retval = sp; /* decrement by the actual number of args */ sp -= csignature->param_count; if (csignature->hasthis) --sp; - child_frame.stack_args = sp; + child_frame->stack_args = sp; if (frame->imethod->method->dynamic && csignature->pinvoke) { - mono_interp_calli_nat_dynamic_pinvoke (&child_frame, code, context, csignature, error); + mono_interp_calli_nat_dynamic_pinvoke (child_frame, code, context, csignature, error); } else { const gboolean save_last_error = ip [-3 + 2]; - ves_pinvoke_method (&child_frame, csignature, (MonoFuncV) code, context, save_last_error); + ves_pinvoke_method (child_frame, csignature, (MonoFuncV) code, context, save_last_error); } + CHECK_RESUME_STATE (context); - /* need to handle typedbyref ... */ - if (csignature->ret->type != MONO_TYPE_VOID) - goto call_return; - goto vcall_return; + if (csignature->ret->type != MONO_TYPE_VOID) { + *sp = *retval; + sp++; + } + MINT_IN_BREAK; } MINT_IN_CASE(MINT_CALLVIRT_FAST) MINT_IN_CASE(MINT_VCALLVIRT_FAST) { + stackval *retval; MonoObject *this_arg; - InterpMethod *target_imethod; + InterpMethod *imethod; + gboolean is_void = *ip == MINT_VCALLVIRT_FAST; int slot; // FIXME Have it handle also remoting calls and use a single opcode for virtual calls frame->ip = ip; - target_imethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; + imethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; slot = (gint16)ip [2]; ip += 3; sp->data.p = vt_sp; - child_frame.retval = sp; + + retval = is_void ? NULL : sp; /* decrement by the actual number of args */ - sp -= target_imethod->param_count + target_imethod->hasthis; + sp -= imethod->param_count + imethod->hasthis; this_arg = (MonoObject*)sp->data.p; - child_frame.imethod = get_virtual_method_fast (target_imethod, this_arg->vtable, slot); - if (m_class_is_valuetype (this_arg->vtable->klass) && m_class_is_valuetype (child_frame.imethod->method->klass)) { + imethod = get_virtual_method_fast (imethod, this_arg->vtable, slot); + if (m_class_is_valuetype (this_arg->vtable->klass) && m_class_is_valuetype (imethod->method->klass)) { /* unbox */ gpointer unboxed = mono_object_unbox_internal (this_arg); sp [0].data.p = unboxed; } - const gboolean is_void = ip [-3] == MINT_VCALLVIRT_FAST; - if (!is_void) - goto common_call; - goto common_vcall; + + SAVE_INTERP_STATE (frame); + + if (G_UNLIKELY (!imethod->transformed)) { + MonoException *ex; + gboolean tracing; + + child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval); + method_entry (context, child_frame, &tracing, &ex); + if (G_UNLIKELY (ex)) { + frame = child_frame; + frame->ip = NULL; + THROW_EX (ex, NULL); + EXCEPTION_CHECKPOINT; + } + } else { + child_frame = alloc_frame (context, &retval, frame, imethod, sp, retval); + alloc_stack_data (context, child_frame, imethod->alloca_size); + } + + frame = child_frame; + INIT_INTERP_STATE (frame, NULL); + clause_args = NULL; + + MINT_IN_BREAK; } MINT_IN_CASE(MINT_CALL_VARARG) { + stackval *retval; int num_varargs = 0; MonoMethodSignature *csig; frame->ip = ip; - child_frame.imethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; + retval = sp; + child_frame = alloc_frame (context, &retval, frame, (InterpMethod*)frame->imethod->data_items [ip [1]], NULL, retval); /* The real signature for vararg calls */ csig = (MonoMethodSignature*) frame->imethod->data_items [ip [2]]; /* Push all vararg arguments from normal sp to vt_sp together with the signature */ @@ -3650,55 +3890,87 @@ main_loop: ip += 3; sp->data.p = vt_sp; - child_frame.retval = sp; + child_frame->retval = sp; /* decrement by the actual number of args */ - sp -= child_frame.imethod->param_count + child_frame.imethod->hasthis + num_varargs; + sp -= child_frame->imethod->param_count + child_frame->imethod->hasthis + num_varargs; - if (csig->ret->type != MONO_TYPE_VOID) - goto common_call; - goto common_vcall; - } - MINT_IN_CASE(MINT_CALL) - sp = mono_interp_call (frame, context, &child_frame, ip, sp, vt_sp, FALSE, FALSE); -#ifdef ENABLE_EXPERIMENT_TIERED - ip += 5; -#else - ip += 3; -#endif -common_call: - child_frame.stack_args = sp; - interp_exec_method (&child_frame, context, error); -call_return: - /* need to handle typedbyref ... */ - *sp = *child_frame.retval; - sp++; -vcall_return: + child_frame->stack_args = sp; + interp_exec_method (child_frame, context, error); CHECK_RESUME_STATE (context); + if (csig->ret->type != MONO_TYPE_VOID) { + *sp = *retval; + sp++; + } MINT_IN_BREAK; - + } MINT_IN_CASE(MINT_VCALL) - sp = mono_interp_call (frame, context, &child_frame, ip, sp, vt_sp, FALSE, TRUE); + MINT_IN_CASE(MINT_CALL) + MINT_IN_CASE(MINT_CALLVIRT) + MINT_IN_CASE(MINT_VCALLVIRT) { + stackval *retval; + gboolean is_void = *ip == MINT_VCALL || *ip == MINT_VCALLVIRT; + gboolean is_virtual = *ip == MINT_CALLVIRT || *ip == MINT_VCALLVIRT; + gpointer native_stack_addr = frame->native_stack_addr ? (gpointer)((guint8*)frame->native_stack_addr - 1) : (gpointer)&retval; + InterpMethod *imethod; + + imethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; + sp->data.p = vt_sp; + retval = is_void ? NULL : sp; + + /* decrement by the actual number of args */ + sp -= ip [2]; + + if (is_virtual) { + MonoObject *this_arg = (MonoObject*)sp->data.p; + + imethod = get_virtual_method (imethod, this_arg->vtable); + if (m_class_is_valuetype (this_arg->vtable->klass) && m_class_is_valuetype (imethod->method->klass)) { + /* unbox */ + gpointer unboxed = mono_object_unbox_internal (this_arg); + sp [0].data.p = unboxed; + } + } + + frame->ip = ip; #ifdef ENABLE_EXPERIMENT_TIERED ip += 5; #else ip += 3; #endif -common_vcall: - child_frame.stack_args = sp; - interp_exec_method (&child_frame, context, error); - goto vcall_return; - MINT_IN_CASE(MINT_CALLVIRT) - sp = mono_interp_call (frame, context, &child_frame, ip, sp, vt_sp, TRUE, FALSE); - ip += 3; - goto common_call; + /* + * Make a non-recursive call by loading the new interpreter state based on child frame, + * and going back to the main loop. + */ + SAVE_INTERP_STATE (frame); + + if (G_UNLIKELY (!imethod->transformed)) { + MonoException *ex; + gboolean tracing; + + child_frame = alloc_frame (context, native_stack_addr, frame, imethod, sp, retval); + method_entry (context, child_frame, &tracing, &ex); + if (G_UNLIKELY (ex)) { + frame = child_frame; + frame->ip = NULL; + THROW_EX (ex, NULL); + EXCEPTION_CHECKPOINT; + } + } else { + child_frame = alloc_frame (context, native_stack_addr, frame, imethod, sp, retval); + alloc_stack_data (context, child_frame, imethod->alloca_size); +#if DEBUG_INTERP + debug_enter (child_frame, out_tracing); +#endif + } - MINT_IN_CASE(MINT_VCALLVIRT) - sp = mono_interp_call (frame, context, &child_frame, ip, sp, vt_sp, TRUE, TRUE); - ip += 3; - goto common_vcall; + frame = child_frame; + INIT_INTERP_STATE (frame, NULL); + clause_args = NULL; + MINT_IN_BREAK; + } MINT_IN_CASE(MINT_JIT_CALL) { InterpMethod *rmethod = (InterpMethod*)frame->imethod->data_items [ip [1]]; error_init_reuse (error); @@ -3771,7 +4043,7 @@ common_vcall: --sp; *frame->retval = *sp; if (sp > frame->stack) - g_warning_d ("ret: more values on stack: %d", sp -frame->stack); + g_warning_d ("ret: more values on stack: %d", sp - frame->stack); goto exit_frame; MINT_IN_CASE(MINT_RET_VOID) if (sp > frame->stack) @@ -4751,7 +5023,6 @@ common_vcall: MINT_IN_BREAK; } MINT_IN_CASE(MINT_NEWOBJ_FAST) { - MonoVTable *vtable = (MonoVTable*) frame->imethod->data_items [ip [3]]; INIT_VTABLE (vtable); MonoObject *o; // See the comment about GC safety. @@ -4781,10 +5052,8 @@ common_vcall: InterpMethod *ctor_method = (InterpMethod*) frame->imethod->data_items [imethod_index]; frame->ip = ip; - child_frame.imethod = ctor_method; - child_frame.stack_args = sp; - - interp_exec_method (&child_frame, context, error); + child_frame = alloc_frame (context, &vtable, frame, ctor_method, sp, NULL); + interp_exec_method (child_frame, context, error); CHECK_RESUME_STATE (context); sp [0].data.o = o; sp++; @@ -4795,34 +5064,34 @@ common_vcall: } MINT_IN_CASE(MINT_NEWOBJ_VT_FAST) MINT_IN_CASE(MINT_NEWOBJ_VTST_FAST) { - + int dummy; // This is split up to: // - conserve stack // - keep exception handling and resume mostly in the main function frame->ip = ip; - child_frame.imethod = (InterpMethod*) frame->imethod->data_items [ip [1]]; + child_frame = alloc_frame (context, &dummy, frame, (InterpMethod*) frame->imethod->data_items [ip [1]], NULL, NULL); guint16 const param_count = ip [2]; if (param_count) { sp -= param_count; memmove (sp + 1, sp, param_count * sizeof (stackval)); } - child_frame.stack_args = sp; + child_frame->stack_args = sp; gboolean const vtst = *ip == MINT_NEWOBJ_VTST_FAST; if (vtst) { memset (vt_sp, 0, ip [3]); sp->data.p = vt_sp; ip += 4; - interp_exec_method (&child_frame, context, error); + interp_exec_method (child_frame, context, error); CHECK_RESUME_STATE (context); sp->data.p = vt_sp; } else { ip += 3; - mono_interp_newobj_vt (&child_frame, context, error); + mono_interp_newobj_vt (child_frame, context, error); CHECK_RESUME_STATE (context); } ++sp; @@ -4830,7 +5099,7 @@ common_vcall: } MINT_IN_CASE(MINT_NEWOBJ) { - + int dummy; // This is split up to: // - conserve stack // - keep exception handling and resume mostly in the main function @@ -4840,10 +5109,8 @@ common_vcall: guint32 const token = ip [1]; ip += 2; // FIXME: Do this after throw? - child_frame.ip = NULL; - - child_frame.imethod = (InterpMethod*)frame->imethod->data_items [token]; - MonoMethodSignature* const csig = mono_method_signature_internal (child_frame.imethod->method); + child_frame = alloc_frame (context, &dummy, frame, (InterpMethod*)frame->imethod->data_items [token], NULL, NULL); + MonoMethodSignature* const csig = mono_method_signature_internal (child_frame->imethod->method); g_assert (csig->hasthis); if (csig->param_count) { @@ -4851,9 +5118,9 @@ common_vcall: memmove (sp + 1, sp, csig->param_count * sizeof (stackval)); } - child_frame.stack_args = sp; + child_frame->stack_args = sp; - MonoException* const exc = mono_interp_newobj (&child_frame, context, error, vt_sp); + MonoException* const exc = mono_interp_newobj (child_frame, context, error, vt_sp); if (exc) THROW_EX (exc, ip); CHECK_RESUME_STATE (context); @@ -6046,7 +6313,7 @@ common_vcall: MINT_IN_CASE(MINT_LEAVE_S) MINT_IN_CASE(MINT_LEAVE_CHECK) MINT_IN_CASE(MINT_LEAVE_S_CHECK) { - + int dummy; // Leave is split into pieces in order to consume less stack, // but not have to change how exception handling macros access labels and locals. @@ -6059,9 +6326,8 @@ common_vcall: gboolean const check = opcode == MINT_LEAVE_CHECK || opcode == MINT_LEAVE_S_CHECK; if (check && frame->imethod->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE) { - child_frame.parent = frame; - child_frame.imethod = NULL; - MonoException *abort_exc = mono_interp_leave (&child_frame); + child_frame = alloc_frame (context, &dummy, frame, NULL, NULL, NULL); + MonoException *abort_exc = mono_interp_leave (child_frame); if (abort_exc) THROW_EX (abort_exc, frame->ip); } @@ -6078,9 +6344,8 @@ common_vcall: #endif // FIXME Null check for frame->imethod follows deref. if (frame->imethod == NULL || (method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) - || (method->iflags & (METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL | METHOD_IMPL_ATTRIBUTE_RUNTIME))) { + || (method->iflags & (METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL | METHOD_IMPL_ATTRIBUTE_RUNTIME))) goto exit_frame; - } guint32 const ip_offset = frame->ip - frame->imethod->code; finally_ips = g_slist_prepend (finally_ips, (void *)endfinally_ip); @@ -6165,6 +6430,11 @@ common_vcall: ++sp; ++ip; MINT_IN_BREAK; + MINT_IN_CASE(MINT_MONO_GET_SP) + sp->data.p = &frame; + ++sp; + ++ip; + MINT_IN_BREAK; MINT_IN_CASE(MINT_SDB_INTR_LOC) if (G_UNLIKELY (ss_enabled)) { typedef void (*T) (void); @@ -6594,7 +6864,8 @@ common_vcall: goto abort_label; int len = sp [-1].data.i; - sp [-1].data.p = alloca (len); + //sp [-1].data.p = alloca (len); + sp [-1].data.p = alloc_extra_stack_data (context, ALIGN_TO (len, 8)); if (frame->imethod->init_locals) memset (sp [-1].data.p, 0, len); @@ -6603,7 +6874,7 @@ common_vcall: } MINT_IN_CASE(MINT_ENDFILTER) /* top of stack is result of filter */ - frame->retval = &sp [-1]; + frame->retval->data.i = sp [-1].data.i; goto exit_frame; MINT_IN_CASE(MINT_INITOBJ) --sp; @@ -6762,6 +7033,7 @@ invalid_cast_label: THROW_EX (mono_get_exception_invalid_cast (), ip); resume: g_assert (context->has_resume_state); + g_assert (frame->imethod); if (frame == context->handler_frame && (!clause_args || context->handler_ip < clause_args->end_at_ip)) { /* Set the current execution state to the resume state in context */ @@ -6783,9 +7055,35 @@ resume: exit_frame: error_init_reuse (error); + g_assert (frame->imethod); + if (clause_args && clause_args->base_frame) memcpy (clause_args->base_frame->stack, frame->stack, frame->imethod->alloca_size); + if (!clause_args && frame->parent && frame->parent->state.ip) { + /* Return to the main loop after a non-recursive interpreter call */ + //printf ("R: %s -> %s %p\n", mono_method_get_full_name (frame->imethod->method), mono_method_get_full_name (frame->parent->imethod->method), frame->parent->state.ip); + stackval *retval = frame->retval; + + InterpFrame *child_frame = frame; + + frame = frame->parent; + LOAD_INTERP_STATE (frame); + + pop_frame (context, child_frame); + + CHECK_RESUME_STATE (context); + + if (retval) { + *sp = *retval; + sp ++; + } + goto main_loop; + } + + if (!clause_args) + pop_frame (context, frame); + DEBUG_LEAVE (); } @@ -6869,14 +7167,20 @@ interp_run_finally (StackFrameInfo *frame, int clause_index, gpointer handler_ip ThreadContext *context = get_context (); const unsigned short *old_ip = iframe->ip; FrameClauseArgs clause_args; + const guint16 *saved_ip; memset (&clause_args, 0, sizeof (FrameClauseArgs)); clause_args.start_with_ip = (const guint16*)handler_ip; clause_args.end_at_ip = (const guint16*)handler_ip_end; clause_args.exit_clause = clause_index; + saved_ip = iframe->state.ip; + iframe->state.ip = NULL; + ERROR_DECL (error); interp_exec_method_full (iframe, context, &clause_args, error); + iframe->state.ip = saved_ip; + iframe->state.clause_args = NULL; if (context->has_resume_state) { return TRUE; } else { @@ -6896,7 +7200,7 @@ interp_run_filter (StackFrameInfo *frame, MonoException *ex, int clause_index, g { InterpFrame *iframe = (InterpFrame*)frame->interp_frame; ThreadContext *context = get_context (); - InterpFrame child_frame; + InterpFrame *child_frame; stackval retval; FrameClauseArgs clause_args; @@ -6904,11 +7208,7 @@ interp_run_filter (StackFrameInfo *frame, MonoException *ex, int clause_index, g * Have to run the clause in a new frame which is a copy of IFRAME, since * during debugging, there are two copies of the frame on the stack. */ - memset (&child_frame, 0, sizeof (InterpFrame)); - child_frame.imethod = iframe->imethod; - child_frame.retval = &retval; - child_frame.parent = iframe; - child_frame.stack_args = iframe->stack_args; + child_frame = alloc_frame (context, &retval, iframe, iframe->imethod, iframe->stack_args, &retval); memset (&clause_args, 0, sizeof (FrameClauseArgs)); clause_args.start_with_ip = (const guint16*)handler_ip; @@ -6917,9 +7217,9 @@ interp_run_filter (StackFrameInfo *frame, MonoException *ex, int clause_index, g clause_args.base_frame = iframe; ERROR_DECL (error); - interp_exec_method_full (&child_frame, context, &clause_args, error); + interp_exec_method_full (child_frame, context, &clause_args, error); /* ENDFILTER stores the result into child_frame->retval */ - return child_frame.retval->data.i ? TRUE : FALSE; + return retval.data.i ? TRUE : FALSE; } typedef struct { @@ -6973,7 +7273,7 @@ interp_frame_iter_next (MonoInterpStackIter *iter, StackFrameInfo *frame) frame->managed = TRUE; } frame->ji = iframe->imethod->jinfo; - frame->frame_addr = iframe; + frame->frame_addr = iframe->native_stack_addr; stack_iter->current = iframe->parent; @@ -7080,6 +7380,14 @@ interp_frame_get_res (MonoInterpFrameHandle frame) return stackval_to_data_addr (sig->ret, iframe->retval); } +static gpointer +interp_frame_get_native_stack_addr (MonoInterpFrameHandle frame) +{ + InterpFrame *iframe = (InterpFrame*)frame; + + return iframe->native_stack_addr; +} + static void interp_start_single_stepping (void) { @@ -7092,6 +7400,48 @@ interp_stop_single_stepping (void) ss_enabled = FALSE; } +/* + * interp_mark_stack: + * + * Mark the interpreter stack frames for a thread. + * + */ +static void +interp_mark_stack (gpointer thread_data, GcScanFunc func, gpointer gc_data, gboolean precise) +{ + MonoThreadInfo *info = (MonoThreadInfo*)thread_data; + + if (!mono_use_interpreter) + return; + if (precise) + return; + + /* + * We explicitly mark the frames instead of registering the stack fragments as GC roots, so + * we have to process less data and avoid false pinning from data which is above 'pos'. + * + * The stack frame handling code uses compiler write barriers only, but the calling code + * in sgen-mono.c already did a mono_memory_barrier_process_wide () so we can + * process these data structures normally. + */ + MonoJitTlsData *jit_tls = (MonoJitTlsData *)info->tls [TLS_KEY_JIT_TLS]; + if (!jit_tls) + return; + + ThreadContext *context = (ThreadContext*)jit_tls->interp_context; + if (!context || !context->data_stack.inited) + return; + + StackFragment *frag; + for (frag = context->data_stack.first; frag; frag = frag->next) { + // FIXME: Scan the whole area with 1 call + for (gpointer *p = (gpointer*)&frag->data; p < (gpointer*)frag->pos; ++p) + func (p, gc_data); + if (frag == context->data_stack.current) + break; + } +} + #if COUNT_OPS static int diff --git a/mono/mini/interp/mintops.def b/mono/mini/interp/mintops.def index 04def43ed3b..20335a53acd 100644 --- a/mono/mini/interp/mintops.def +++ b/mono/mini/interp/mintops.def @@ -744,6 +744,7 @@ OPDEF(MINT_MONO_RETOBJ, "mono_retobj", 1, Pop1, Push0, MintOpNoArgs) OPDEF(MINT_MONO_ATOMIC_STORE_I4, "mono_atomic.store.i4", 1, Pop2, Push0, MintOpNoArgs) OPDEF(MINT_MONO_MEMORY_BARRIER, "mono_memory_barrier", 1, Pop0, Push0, MintOpNoArgs) OPDEF(MINT_MONO_LDDOMAIN, "mono_lddomain", 1, Pop0, Push1, MintOpNoArgs) +OPDEF(MINT_MONO_GET_SP, "mono_get_sp", 1, Pop0, Push1, MintOpNoArgs) OPDEF(MINT_SDB_INTR_LOC, "sdb_intr_loc", 1, Pop0, Push0, MintOpNoArgs) OPDEF(MINT_SDB_SEQ_POINT, "sdb_seq_point", 1, Pop0, Push0, MintOpNoArgs) diff --git a/mono/mini/interp/transform.c b/mono/mini/interp/transform.c index ba277f46301..05cccab2f8a 100644 --- a/mono/mini/interp/transform.c +++ b/mono/mini/interp/transform.c @@ -2387,7 +2387,8 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target for (i = csignature->sentinelpos; i < csignature->param_count; ++i) { int align, arg_size; arg_size = mono_type_stack_size (csignature->params [i], &align); - vararg_stack += ALIGN_TO (arg_size, align); + vararg_stack = ALIGN_TO (vararg_stack, align); + vararg_stack += arg_size; } /* allocate space for the pointer to varargs space start */ vararg_stack += sizeof (gpointer); @@ -3173,6 +3174,33 @@ interp_emit_load_const (TransformData *td, gpointer field_addr, int mt) return TRUE; } +/* + * emit_convert: + * + * Emit some implicit conversions which are not part of the .net spec, but are allowed by MS.NET. + */ +static void +emit_convert (TransformData *td, int stack_type, MonoType *ftype) +{ + ftype = mini_get_underlying_type (ftype); + + // FIXME: Add more + switch (ftype->type) { + case MONO_TYPE_I8: { + switch (stack_type) { + case STACK_TYPE_I4: + interp_add_ins (td, MINT_CONV_I8_I4); + break; + default: + break; + } + break; + } + default: + break; + } +} + static void interp_emit_sfld_access (TransformData *td, MonoClassField *field, MonoClass *field_class, int mt, gboolean is_load, MonoError *error) { @@ -4942,6 +4970,8 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, MonoType *ftype = mono_field_get_type_internal (field); mt = mint_type (ftype); + emit_convert (td, td->sp [-1].type, ftype); + /* the vtable of the field might not be initialized at this point */ MonoClass *fld_klass = mono_class_from_mono_type_internal (ftype); mono_class_vtable_checked (domain, fld_klass, error); @@ -5908,6 +5938,11 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, save_last_error = TRUE; ++td->ip; break; + case CEE_MONO_GET_SP: + interp_add_ins (td, MINT_MONO_GET_SP); + PUSH_SIMPLE_TYPE (td, STACK_TYPE_I); + ++td->ip; + break; default: g_error ("transform.c: Unimplemented opcode: 0xF0 %02x at 0x%x\n", *td->ip, td->ip-header->code); } @@ -7432,7 +7467,7 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG rtm->stack_size = ALIGN_TO (rtm->stack_size, MINT_VT_ALIGNMENT); rtm->vt_stack_size = td->max_vt_sp; rtm->total_locals_size = td->total_locals_size; - rtm->alloca_size = rtm->total_locals_size + rtm->vt_stack_size + rtm->stack_size; + rtm->alloca_size = ALIGN_TO (rtm->total_locals_size + rtm->vt_stack_size + rtm->stack_size, 8); rtm->data_items = (gpointer*)mono_domain_alloc0 (domain, td->n_data_items * sizeof (td->data_items [0])); memcpy (rtm->data_items, td->data_items, td->n_data_items * sizeof (td->data_items [0])); diff --git a/mono/mini/method-to-ir.c b/mono/mini/method-to-ir.c index 8eab40a1a27..e77fad2bb6c 100644 --- a/mono/mini/method-to-ir.c +++ b/mono/mini/method-to-ir.c @@ -10705,6 +10705,13 @@ mono_ldptr: *sp++ = ins; break; + case MONO_CEE_MONO_GET_SP: { + /* Used by COOP only, so this is good enough */ + MonoInst *var = mono_compile_create_var (cfg, mono_get_int_type (), OP_LOCAL); + EMIT_NEW_VARLOADA (cfg, ins, var, NULL); + *sp++ = ins; + break; + } case MONO_CEE_ARGLIST: { /* somewhat similar to LDTOKEN */ diff --git a/mono/mini/mini-exceptions.c b/mono/mini/mini-exceptions.c index e13151ca29f..e3b9e30b709 100644 --- a/mono/mini/mini-exceptions.c +++ b/mono/mini/mini-exceptions.c @@ -761,7 +761,10 @@ unwinder_unwind_frame (Unwinder *unwinder, unwinder->in_interp = mini_get_interp_callbacks ()->frame_iter_next (&unwinder->interp_iter, frame); if (frame->type == FRAME_TYPE_INTERP) { parent = mini_get_interp_callbacks ()->frame_get_parent (frame->interp_frame); - unwinder->last_frame_addr = parent; + if (parent) + unwinder->last_frame_addr = mini_get_interp_callbacks ()->frame_get_native_stack_addr (parent); + else + unwinder->last_frame_addr = NULL; } if (!unwinder->in_interp) return unwinder_unwind_frame (unwinder, domain, jit_tls, prev_ji, ctx, new_ctx, trace, lmf, save_locations, frame); diff --git a/mono/mini/mini-gc.c b/mono/mini/mini-gc.c index 4c9ee81d9ee..d5ede5a0342 100644 --- a/mono/mini/mini-gc.c +++ b/mono/mini/mini-gc.c @@ -2542,6 +2542,8 @@ mini_gc_init (void) /* Comment this out to disable precise stack marking */ cb.thread_mark_func = thread_mark_func; cb.get_provenance_func = get_provenance_func; + if (mono_use_interpreter) + cb.interp_mark_func = mini_get_interp_callbacks ()->mark_stack; mono_gc_set_gc_callbacks (&cb); logfile = mono_gc_get_logfile (); @@ -2604,6 +2606,8 @@ mini_gc_init (void) MonoGCCallbacks cb; memset (&cb, 0, sizeof (cb)); cb.get_provenance_func = get_provenance_func; + if (mono_use_interpreter) + cb.interp_mark_func = mini_get_interp_callbacks ()->mark_stack; mono_gc_set_gc_callbacks (&cb); } diff --git a/mono/tests/monitor-resurrection.cs b/mono/tests/monitor-resurrection.cs index 205d10ca16e..1d52ec47727 100644 --- a/mono/tests/monitor-resurrection.cs +++ b/mono/tests/monitor-resurrection.cs @@ -40,7 +40,7 @@ public class Foo { public static int Main() { /* Allocate an object down the stack so it doesn't get pinned */ - CreateFoo (100); + typeof (Foo).GetMethod ("CreateFoo").Invoke (null, new object [] { 100 }); Console.WriteLine ("."); diff --git a/mono/utils/mono-membar.h b/mono/utils/mono-membar.h index 5aa985ac695..711a419cca0 100644 --- a/mono/utils/mono-membar.h +++ b/mono/utils/mono-membar.h @@ -15,6 +15,11 @@ #include <glib.h> +/* + * Memory barrier which only affects the compiler. + * mono_memory_barrier_process_wide () should be uses to synchronize with code which uses this. + */ +//#define mono_compiler_barrier() asm volatile("": : :"memory") #ifdef TARGET_WASM @@ -30,6 +35,8 @@ static inline void mono_memory_write_barrier (void) { } +#define mono_compiler_barrier() asm volatile("": : :"memory") + #elif _MSC_VER #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -61,7 +68,11 @@ static inline void mono_memory_write_barrier (void) _WriteBarrier (); MemoryBarrier (); } + +#define mono_compiler_barrier() _ReadWriteBarrier () + #elif defined(USE_GCC_ATOMIC_OPS) + static inline void mono_memory_barrier (void) { __sync_synchronize (); @@ -76,6 +87,9 @@ static inline void mono_memory_write_barrier (void) { mono_memory_barrier (); } + +#define mono_compiler_barrier() asm volatile("": : :"memory") + #else #error "Don't know how to do memory barriers!" #endif |