diff options
Diffstat (limited to '3rdparty/lua/src/lj_dispatch.c')
| -rw-r--r-- | 3rdparty/lua/src/lj_dispatch.c | 988 |
1 files changed, 494 insertions, 494 deletions
diff --git a/3rdparty/lua/src/lj_dispatch.c b/3rdparty/lua/src/lj_dispatch.c index d57f1a6..3e714d4 100644 --- a/3rdparty/lua/src/lj_dispatch.c +++ b/3rdparty/lua/src/lj_dispatch.c @@ -1,494 +1,494 @@ -/* -** Instruction dispatch handling. -** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h -*/ - -#define lj_dispatch_c -#define LUA_CORE - -#include "lj_obj.h" -#include "lj_err.h" -#include "lj_func.h" -#include "lj_str.h" -#include "lj_tab.h" -#include "lj_meta.h" -#include "lj_debug.h" -#include "lj_state.h" -#include "lj_frame.h" -#include "lj_bc.h" -#include "lj_ff.h" -#if LJ_HASJIT -#include "lj_jit.h" -#endif -#if LJ_HASFFI -#include "lj_ccallback.h" -#endif -#include "lj_trace.h" -#include "lj_dispatch.h" -#include "lj_vm.h" -#include "luajit.h" - -/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */ -LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC); - -/* -- Dispatch table management ------------------------------------------- */ - -#if LJ_TARGET_MIPS -#include <math.h> -LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L, - lua_State *co); - -#define GOTFUNC(name) (ASMFunction)name, -static const ASMFunction dispatch_got[] = { - GOTDEF(GOTFUNC) -}; -#undef GOTFUNC -#endif - -/* Initialize instruction dispatch table and hot counters. */ -void lj_dispatch_init(GG_State *GG) -{ - uint32_t i; - ASMFunction *disp = GG->dispatch; - for (i = 0; i < GG_LEN_SDISP; i++) - disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]); - for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) - disp[i] = makeasmfunc(lj_bc_ofs[i]); - /* The JIT engine is off by default. luaopen_jit() turns it on. */ - disp[BC_FORL] = disp[BC_IFORL]; - disp[BC_ITERL] = disp[BC_IITERL]; - disp[BC_LOOP] = disp[BC_ILOOP]; - disp[BC_FUNCF] = disp[BC_IFUNCF]; - disp[BC_FUNCV] = disp[BC_IFUNCV]; - GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0); - for (i = 0; i < GG_NUM_ASMFF; i++) - GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0); -#if LJ_TARGET_MIPS - memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4); -#endif -} - -#if LJ_HASJIT -/* Initialize hotcount table. */ -void lj_dispatch_init_hotcount(global_State *g) -{ - int32_t hotloop = G2J(g)->param[JIT_P_hotloop]; - HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1); - HotCount *hotcount = G2GG(g)->hotcount; - uint32_t i; - for (i = 0; i < HOTCOUNT_SIZE; i++) - hotcount[i] = start; -} -#endif - -/* Internal dispatch mode bits. */ -#define DISPMODE_JIT 0x01 /* JIT compiler on. */ -#define DISPMODE_REC 0x02 /* Recording active. */ -#define DISPMODE_INS 0x04 /* Override instruction dispatch. */ -#define DISPMODE_CALL 0x08 /* Override call dispatch. */ -#define DISPMODE_RET 0x10 /* Override return dispatch. */ - -/* Update dispatch table depending on various flags. */ -void lj_dispatch_update(global_State *g) -{ - uint8_t oldmode = g->dispatchmode; - uint8_t mode = 0; -#if LJ_HASJIT - mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0; - mode |= G2J(g)->state != LJ_TRACE_IDLE ? - (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0; -#endif - mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0; - mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0; - mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0; - if (oldmode != mode) { /* Mode changed? */ - ASMFunction *disp = G2GG(g)->dispatch; - ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv; - g->dispatchmode = mode; - - /* Hotcount if JIT is on, but not while recording. */ - if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) { - f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]); - f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]); - f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]); - f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]); - f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]); - } else { /* Otherwise use the non-hotcounting instructions. */ - f_forl = disp[GG_LEN_DDISP+BC_IFORL]; - f_iterl = disp[GG_LEN_DDISP+BC_IITERL]; - f_loop = disp[GG_LEN_DDISP+BC_ILOOP]; - f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]); - f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]); - } - /* Init static counting instruction dispatch first (may be copied below). */ - disp[GG_LEN_DDISP+BC_FORL] = f_forl; - disp[GG_LEN_DDISP+BC_ITERL] = f_iterl; - disp[GG_LEN_DDISP+BC_LOOP] = f_loop; - - /* Set dynamic instruction dispatch. */ - if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) { - /* Need to update the whole table. */ - if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { /* No ins dispatch? */ - /* Copy static dispatch table to dynamic dispatch table. */ - memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction)); - /* Overwrite with dynamic return dispatch. */ - if ((mode & DISPMODE_RET)) { - disp[BC_RETM] = lj_vm_rethook; - disp[BC_RET] = lj_vm_rethook; - disp[BC_RET0] = lj_vm_rethook; - disp[BC_RET1] = lj_vm_rethook; - } - } else { - /* The recording dispatch also checks for hooks. */ - ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook; - uint32_t i; - for (i = 0; i < GG_LEN_SDISP; i++) - disp[i] = f; - } - } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { - /* Otherwise set dynamic counting ins. */ - disp[BC_FORL] = f_forl; - disp[BC_ITERL] = f_iterl; - disp[BC_LOOP] = f_loop; - /* Set dynamic return dispatch. */ - if ((mode & DISPMODE_RET)) { - disp[BC_RETM] = lj_vm_rethook; - disp[BC_RET] = lj_vm_rethook; - disp[BC_RET0] = lj_vm_rethook; - disp[BC_RET1] = lj_vm_rethook; - } else { - disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM]; - disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET]; - disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0]; - disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1]; - } - } - - /* Set dynamic call dispatch. */ - if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */ - uint32_t i; - if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */ - for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) - disp[i] = makeasmfunc(lj_bc_ofs[i]); - } else { - for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++) - disp[i] = lj_vm_callhook; - } - } - if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */ - disp[BC_FUNCF] = f_funcf; - disp[BC_FUNCV] = f_funcv; - } - -#if LJ_HASJIT - /* Reset hotcounts for JIT off to on transition. */ - if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT)) - lj_dispatch_init_hotcount(g); -#endif - } -} - -/* -- JIT mode setting ---------------------------------------------------- */ - -#if LJ_HASJIT -/* Set JIT mode for a single prototype. */ -static void setptmode(global_State *g, GCproto *pt, int mode) -{ - if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */ - pt->flags &= ~PROTO_NOJIT; - lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */ - } else { /* Flush and/or disable JIT compilation. */ - if (!(mode & LUAJIT_MODE_FLUSH)) - pt->flags |= PROTO_NOJIT; - lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */ - } -} - -/* Recursively set the JIT mode for all children of a prototype. */ -static void setptmode_all(global_State *g, GCproto *pt, int mode) -{ - ptrdiff_t i; - if (!(pt->flags & PROTO_CHILD)) return; - for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) { - GCobj *o = proto_kgc(pt, i); - if (o->gch.gct == ~LJ_TPROTO) { - setptmode(g, gco2pt(o), mode); - setptmode_all(g, gco2pt(o), mode); - } - } -} -#endif - -/* Public API function: control the JIT engine. */ -int luaJIT_setmode(lua_State *L, int idx, int mode) -{ - global_State *g = G(L); - int mm = mode & LUAJIT_MODE_MASK; - lj_trace_abort(g); /* Abort recording on any state change. */ - /* Avoid pulling the rug from under our own feet. */ - if ((g->hookmask & HOOK_GC)) - lj_err_caller(L, LJ_ERR_NOGCMM); - switch (mm) { -#if LJ_HASJIT - case LUAJIT_MODE_ENGINE: - if ((mode & LUAJIT_MODE_FLUSH)) { - lj_trace_flushall(L); - } else { - if (!(mode & LUAJIT_MODE_ON)) - G2J(g)->flags &= ~(uint32_t)JIT_F_ON; -#if LJ_TARGET_X86ORX64 - else if ((G2J(g)->flags & JIT_F_SSE2)) - G2J(g)->flags |= (uint32_t)JIT_F_ON; - else - return 0; /* Don't turn on JIT compiler without SSE2 support. */ -#else - else - G2J(g)->flags |= (uint32_t)JIT_F_ON; -#endif - lj_dispatch_update(g); - } - break; - case LUAJIT_MODE_FUNC: - case LUAJIT_MODE_ALLFUNC: - case LUAJIT_MODE_ALLSUBFUNC: { - cTValue *tv = idx == 0 ? frame_prev(L->base-1) : - idx > 0 ? L->base + (idx-1) : L->top + idx; - GCproto *pt; - if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn)) - pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */ - else if (tvisproto(tv)) - pt = protoV(tv); - else - return 0; /* Failed. */ - if (mm != LUAJIT_MODE_ALLSUBFUNC) - setptmode(g, pt, mode); - if (mm != LUAJIT_MODE_FUNC) - setptmode_all(g, pt, mode); - break; - } - case LUAJIT_MODE_TRACE: - if (!(mode & LUAJIT_MODE_FLUSH)) - return 0; /* Failed. */ - lj_trace_flush(G2J(g), idx); - break; -#else - case LUAJIT_MODE_ENGINE: - case LUAJIT_MODE_FUNC: - case LUAJIT_MODE_ALLFUNC: - case LUAJIT_MODE_ALLSUBFUNC: - UNUSED(idx); - if ((mode & LUAJIT_MODE_ON)) - return 0; /* Failed. */ - break; -#endif - case LUAJIT_MODE_WRAPCFUNC: - if ((mode & LUAJIT_MODE_ON)) { - if (idx != 0) { - cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx; - if (tvislightud(tv)) - g->wrapf = (lua_CFunction)lightudV(tv); - else - return 0; /* Failed. */ - } else { - return 0; /* Failed. */ - } - g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0); - } else { - g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0); - } - break; - default: - return 0; /* Failed. */ - } - return 1; /* OK. */ -} - -/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */ -LUA_API void LUAJIT_VERSION_SYM(void) -{ -} - -/* -- Hooks --------------------------------------------------------------- */ - -/* This function can be called asynchronously (e.g. during a signal). */ -LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count) -{ - global_State *g = G(L); - mask &= HOOK_EVENTMASK; - if (func == NULL || mask == 0) { mask = 0; func = NULL; } /* Consistency. */ - g->hookf = func; - g->hookcount = g->hookcstart = (int32_t)count; - g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask); - lj_trace_abort(g); /* Abort recording on any hook change. */ - lj_dispatch_update(g); - return 1; -} - -LUA_API lua_Hook lua_gethook(lua_State *L) -{ - return G(L)->hookf; -} - -LUA_API int lua_gethookmask(lua_State *L) -{ - return G(L)->hookmask & HOOK_EVENTMASK; -} - -LUA_API int lua_gethookcount(lua_State *L) -{ - return (int)G(L)->hookcstart; -} - -/* Call a hook. */ -static void callhook(lua_State *L, int event, BCLine line) -{ - global_State *g = G(L); - lua_Hook hookf = g->hookf; - if (hookf && !hook_active(g)) { - lua_Debug ar; - lj_trace_abort(g); /* Abort recording on any hook call. */ - ar.event = event; - ar.currentline = line; - /* Top frame, nextframe = NULL. */ - ar.i_ci = (int)((L->base-1) - tvref(L->stack)); - lj_state_checkstack(L, 1+LUA_MINSTACK); - hook_enter(g); - hookf(L, &ar); - lua_assert(hook_active(g)); - hook_leave(g); - } -} - -/* -- Dispatch callbacks -------------------------------------------------- */ - -/* Calculate number of used stack slots in the current frame. */ -static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres) -{ - BCIns ins = pc[-1]; - if (bc_op(ins) == BC_UCLO) - ins = pc[bc_j(ins)]; - switch (bc_op(ins)) { - case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1; - case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1; - case BC_TSETM: return bc_a(ins) + nres-1; - default: return pt->framesize; - } -} - -/* Instruction dispatch. Used by instr/line/return hooks or when recording. */ -void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc) -{ - ERRNO_SAVE - GCfunc *fn = curr_func(L); - GCproto *pt = funcproto(fn); - void *cf = cframe_raw(L->cframe); - const BCIns *oldpc = cframe_pc(cf); - global_State *g = G(L); - BCReg slots; - setcframe_pc(cf, pc); - slots = cur_topslot(pt, pc, cframe_multres_n(cf)); - L->top = L->base + slots; /* Fix top. */ -#if LJ_HASJIT - { - jit_State *J = G2J(g); - if (J->state != LJ_TRACE_IDLE) { -#ifdef LUA_USE_ASSERT - ptrdiff_t delta = L->top - L->base; -#endif - J->L = L; - lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */ - lua_assert(L->top - L->base == delta); - } - } -#endif - if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) { - g->hookcount = g->hookcstart; - callhook(L, LUA_HOOKCOUNT, -1); - L->top = L->base + slots; /* Fix top again. */ - } - if ((g->hookmask & LUA_MASKLINE)) { - BCPos npc = proto_bcpos(pt, pc) - 1; - BCPos opc = proto_bcpos(pt, oldpc) - 1; - BCLine line = lj_debug_line(pt, npc); - if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) { - callhook(L, LUA_HOOKLINE, line); - L->top = L->base + slots; /* Fix top again. */ - } - } - if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1]))) - callhook(L, LUA_HOOKRET, -1); - ERRNO_RESTORE -} - -/* Initialize call. Ensure stack space and return # of missing parameters. */ -static int call_init(lua_State *L, GCfunc *fn) -{ - if (isluafunc(fn)) { - GCproto *pt = funcproto(fn); - int numparams = pt->numparams; - int gotparams = (int)(L->top - L->base); - int need = pt->framesize; - if ((pt->flags & PROTO_VARARG)) need += 1+gotparams; - lj_state_checkstack(L, (MSize)need); - numparams -= gotparams; - return numparams >= 0 ? numparams : 0; - } else { - lj_state_checkstack(L, LUA_MINSTACK); - return 0; - } -} - -/* Call dispatch. Used by call hooks, hot calls or when recording. */ -ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc) -{ - ERRNO_SAVE - GCfunc *fn = curr_func(L); - BCOp op; - global_State *g = G(L); -#if LJ_HASJIT - jit_State *J = G2J(g); -#endif - int missing = call_init(L, fn); -#if LJ_HASJIT - J->L = L; - if ((uintptr_t)pc & 1) { /* Marker for hot call. */ -#ifdef LUA_USE_ASSERT - ptrdiff_t delta = L->top - L->base; -#endif - pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1); - lj_trace_hot(J, pc); - lua_assert(L->top - L->base == delta); - goto out; - } else if (J->state != LJ_TRACE_IDLE && - !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) { -#ifdef LUA_USE_ASSERT - ptrdiff_t delta = L->top - L->base; -#endif - /* Record the FUNC* bytecodes, too. */ - lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */ - lua_assert(L->top - L->base == delta); - } -#endif - if ((g->hookmask & LUA_MASKCALL)) { - int i; - for (i = 0; i < missing; i++) /* Add missing parameters. */ - setnilV(L->top++); - callhook(L, LUA_HOOKCALL, -1); - /* Preserve modifications of missing parameters by lua_setlocal(). */ - while (missing-- > 0 && tvisnil(L->top - 1)) - L->top--; - } -#if LJ_HASJIT -out: -#endif - op = bc_op(pc[-1]); /* Get FUNC* op. */ -#if LJ_HASJIT - /* Use the non-hotcounting variants if JIT is off or while recording. */ - if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) && - (op == BC_FUNCF || op == BC_FUNCV)) - op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF); -#endif - ERRNO_RESTORE - return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */ -} - +/*
+** Instruction dispatch handling.
+** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
+*/
+
+#define lj_dispatch_c
+#define LUA_CORE
+
+#include "lj_obj.h"
+#include "lj_err.h"
+#include "lj_func.h"
+#include "lj_str.h"
+#include "lj_tab.h"
+#include "lj_meta.h"
+#include "lj_debug.h"
+#include "lj_state.h"
+#include "lj_frame.h"
+#include "lj_bc.h"
+#include "lj_ff.h"
+#if LJ_HASJIT
+#include "lj_jit.h"
+#endif
+#if LJ_HASFFI
+#include "lj_ccallback.h"
+#endif
+#include "lj_trace.h"
+#include "lj_dispatch.h"
+#include "lj_vm.h"
+#include "luajit.h"
+
+/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
+LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
+
+/* -- Dispatch table management ------------------------------------------- */
+
+#if LJ_TARGET_MIPS
+#include <math.h>
+LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
+ lua_State *co);
+
+#define GOTFUNC(name) (ASMFunction)name,
+static const ASMFunction dispatch_got[] = {
+ GOTDEF(GOTFUNC)
+};
+#undef GOTFUNC
+#endif
+
+/* Initialize instruction dispatch table and hot counters. */
+void lj_dispatch_init(GG_State *GG)
+{
+ uint32_t i;
+ ASMFunction *disp = GG->dispatch;
+ for (i = 0; i < GG_LEN_SDISP; i++)
+ disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
+ for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
+ disp[i] = makeasmfunc(lj_bc_ofs[i]);
+ /* The JIT engine is off by default. luaopen_jit() turns it on. */
+ disp[BC_FORL] = disp[BC_IFORL];
+ disp[BC_ITERL] = disp[BC_IITERL];
+ disp[BC_LOOP] = disp[BC_ILOOP];
+ disp[BC_FUNCF] = disp[BC_IFUNCF];
+ disp[BC_FUNCV] = disp[BC_IFUNCV];
+ GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
+ for (i = 0; i < GG_NUM_ASMFF; i++)
+ GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
+#if LJ_TARGET_MIPS
+ memcpy(GG->got, dispatch_got, LJ_GOT__MAX*4);
+#endif
+}
+
+#if LJ_HASJIT
+/* Initialize hotcount table. */
+void lj_dispatch_init_hotcount(global_State *g)
+{
+ int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
+ HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
+ HotCount *hotcount = G2GG(g)->hotcount;
+ uint32_t i;
+ for (i = 0; i < HOTCOUNT_SIZE; i++)
+ hotcount[i] = start;
+}
+#endif
+
+/* Internal dispatch mode bits. */
+#define DISPMODE_JIT 0x01 /* JIT compiler on. */
+#define DISPMODE_REC 0x02 /* Recording active. */
+#define DISPMODE_INS 0x04 /* Override instruction dispatch. */
+#define DISPMODE_CALL 0x08 /* Override call dispatch. */
+#define DISPMODE_RET 0x10 /* Override return dispatch. */
+
+/* Update dispatch table depending on various flags. */
+void lj_dispatch_update(global_State *g)
+{
+ uint8_t oldmode = g->dispatchmode;
+ uint8_t mode = 0;
+#if LJ_HASJIT
+ mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
+ mode |= G2J(g)->state != LJ_TRACE_IDLE ?
+ (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
+#endif
+ mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
+ mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
+ mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
+ if (oldmode != mode) { /* Mode changed? */
+ ASMFunction *disp = G2GG(g)->dispatch;
+ ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
+ g->dispatchmode = mode;
+
+ /* Hotcount if JIT is on, but not while recording. */
+ if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
+ f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
+ f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
+ f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
+ f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
+ f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
+ } else { /* Otherwise use the non-hotcounting instructions. */
+ f_forl = disp[GG_LEN_DDISP+BC_IFORL];
+ f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
+ f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
+ f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
+ f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
+ }
+ /* Init static counting instruction dispatch first (may be copied below). */
+ disp[GG_LEN_DDISP+BC_FORL] = f_forl;
+ disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
+ disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
+
+ /* Set dynamic instruction dispatch. */
+ if ((oldmode ^ mode) & (DISPMODE_REC|DISPMODE_INS)) {
+ /* Need to update the whole table. */
+ if (!(mode & (DISPMODE_REC|DISPMODE_INS))) { /* No ins dispatch? */
+ /* Copy static dispatch table to dynamic dispatch table. */
+ memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
+ /* Overwrite with dynamic return dispatch. */
+ if ((mode & DISPMODE_RET)) {
+ disp[BC_RETM] = lj_vm_rethook;
+ disp[BC_RET] = lj_vm_rethook;
+ disp[BC_RET0] = lj_vm_rethook;
+ disp[BC_RET1] = lj_vm_rethook;
+ }
+ } else {
+ /* The recording dispatch also checks for hooks. */
+ ASMFunction f = (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
+ uint32_t i;
+ for (i = 0; i < GG_LEN_SDISP; i++)
+ disp[i] = f;
+ }
+ } else if (!(mode & (DISPMODE_REC|DISPMODE_INS))) {
+ /* Otherwise set dynamic counting ins. */
+ disp[BC_FORL] = f_forl;
+ disp[BC_ITERL] = f_iterl;
+ disp[BC_LOOP] = f_loop;
+ /* Set dynamic return dispatch. */
+ if ((mode & DISPMODE_RET)) {
+ disp[BC_RETM] = lj_vm_rethook;
+ disp[BC_RET] = lj_vm_rethook;
+ disp[BC_RET0] = lj_vm_rethook;
+ disp[BC_RET1] = lj_vm_rethook;
+ } else {
+ disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
+ disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
+ disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
+ disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
+ }
+ }
+
+ /* Set dynamic call dispatch. */
+ if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */
+ uint32_t i;
+ if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */
+ for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
+ disp[i] = makeasmfunc(lj_bc_ofs[i]);
+ } else {
+ for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
+ disp[i] = lj_vm_callhook;
+ }
+ }
+ if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */
+ disp[BC_FUNCF] = f_funcf;
+ disp[BC_FUNCV] = f_funcv;
+ }
+
+#if LJ_HASJIT
+ /* Reset hotcounts for JIT off to on transition. */
+ if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
+ lj_dispatch_init_hotcount(g);
+#endif
+ }
+}
+
+/* -- JIT mode setting ---------------------------------------------------- */
+
+#if LJ_HASJIT
+/* Set JIT mode for a single prototype. */
+static void setptmode(global_State *g, GCproto *pt, int mode)
+{
+ if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */
+ pt->flags &= ~PROTO_NOJIT;
+ lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */
+ } else { /* Flush and/or disable JIT compilation. */
+ if (!(mode & LUAJIT_MODE_FLUSH))
+ pt->flags |= PROTO_NOJIT;
+ lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */
+ }
+}
+
+/* Recursively set the JIT mode for all children of a prototype. */
+static void setptmode_all(global_State *g, GCproto *pt, int mode)
+{
+ ptrdiff_t i;
+ if (!(pt->flags & PROTO_CHILD)) return;
+ for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
+ GCobj *o = proto_kgc(pt, i);
+ if (o->gch.gct == ~LJ_TPROTO) {
+ setptmode(g, gco2pt(o), mode);
+ setptmode_all(g, gco2pt(o), mode);
+ }
+ }
+}
+#endif
+
+/* Public API function: control the JIT engine. */
+int luaJIT_setmode(lua_State *L, int idx, int mode)
+{
+ global_State *g = G(L);
+ int mm = mode & LUAJIT_MODE_MASK;
+ lj_trace_abort(g); /* Abort recording on any state change. */
+ /* Avoid pulling the rug from under our own feet. */
+ if ((g->hookmask & HOOK_GC))
+ lj_err_caller(L, LJ_ERR_NOGCMM);
+ switch (mm) {
+#if LJ_HASJIT
+ case LUAJIT_MODE_ENGINE:
+ if ((mode & LUAJIT_MODE_FLUSH)) {
+ lj_trace_flushall(L);
+ } else {
+ if (!(mode & LUAJIT_MODE_ON))
+ G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
+#if LJ_TARGET_X86ORX64
+ else if ((G2J(g)->flags & JIT_F_SSE2))
+ G2J(g)->flags |= (uint32_t)JIT_F_ON;
+ else
+ return 0; /* Don't turn on JIT compiler without SSE2 support. */
+#else
+ else
+ G2J(g)->flags |= (uint32_t)JIT_F_ON;
+#endif
+ lj_dispatch_update(g);
+ }
+ break;
+ case LUAJIT_MODE_FUNC:
+ case LUAJIT_MODE_ALLFUNC:
+ case LUAJIT_MODE_ALLSUBFUNC: {
+ cTValue *tv = idx == 0 ? frame_prev(L->base-1) :
+ idx > 0 ? L->base + (idx-1) : L->top + idx;
+ GCproto *pt;
+ if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
+ pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */
+ else if (tvisproto(tv))
+ pt = protoV(tv);
+ else
+ return 0; /* Failed. */
+ if (mm != LUAJIT_MODE_ALLSUBFUNC)
+ setptmode(g, pt, mode);
+ if (mm != LUAJIT_MODE_FUNC)
+ setptmode_all(g, pt, mode);
+ break;
+ }
+ case LUAJIT_MODE_TRACE:
+ if (!(mode & LUAJIT_MODE_FLUSH))
+ return 0; /* Failed. */
+ lj_trace_flush(G2J(g), idx);
+ break;
+#else
+ case LUAJIT_MODE_ENGINE:
+ case LUAJIT_MODE_FUNC:
+ case LUAJIT_MODE_ALLFUNC:
+ case LUAJIT_MODE_ALLSUBFUNC:
+ UNUSED(idx);
+ if ((mode & LUAJIT_MODE_ON))
+ return 0; /* Failed. */
+ break;
+#endif
+ case LUAJIT_MODE_WRAPCFUNC:
+ if ((mode & LUAJIT_MODE_ON)) {
+ if (idx != 0) {
+ cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
+ if (tvislightud(tv))
+ g->wrapf = (lua_CFunction)lightudV(tv);
+ else
+ return 0; /* Failed. */
+ } else {
+ return 0; /* Failed. */
+ }
+ g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
+ } else {
+ g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
+ }
+ break;
+ default:
+ return 0; /* Failed. */
+ }
+ return 1; /* OK. */
+}
+
+/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
+LUA_API void LUAJIT_VERSION_SYM(void)
+{
+}
+
+/* -- Hooks --------------------------------------------------------------- */
+
+/* This function can be called asynchronously (e.g. during a signal). */
+LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
+{
+ global_State *g = G(L);
+ mask &= HOOK_EVENTMASK;
+ if (func == NULL || mask == 0) { mask = 0; func = NULL; } /* Consistency. */
+ g->hookf = func;
+ g->hookcount = g->hookcstart = (int32_t)count;
+ g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
+ lj_trace_abort(g); /* Abort recording on any hook change. */
+ lj_dispatch_update(g);
+ return 1;
+}
+
+LUA_API lua_Hook lua_gethook(lua_State *L)
+{
+ return G(L)->hookf;
+}
+
+LUA_API int lua_gethookmask(lua_State *L)
+{
+ return G(L)->hookmask & HOOK_EVENTMASK;
+}
+
+LUA_API int lua_gethookcount(lua_State *L)
+{
+ return (int)G(L)->hookcstart;
+}
+
+/* Call a hook. */
+static void callhook(lua_State *L, int event, BCLine line)
+{
+ global_State *g = G(L);
+ lua_Hook hookf = g->hookf;
+ if (hookf && !hook_active(g)) {
+ lua_Debug ar;
+ lj_trace_abort(g); /* Abort recording on any hook call. */
+ ar.event = event;
+ ar.currentline = line;
+ /* Top frame, nextframe = NULL. */
+ ar.i_ci = (int)((L->base-1) - tvref(L->stack));
+ lj_state_checkstack(L, 1+LUA_MINSTACK);
+ hook_enter(g);
+ hookf(L, &ar);
+ lua_assert(hook_active(g));
+ hook_leave(g);
+ }
+}
+
+/* -- Dispatch callbacks -------------------------------------------------- */
+
+/* Calculate number of used stack slots in the current frame. */
+static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
+{
+ BCIns ins = pc[-1];
+ if (bc_op(ins) == BC_UCLO)
+ ins = pc[bc_j(ins)];
+ switch (bc_op(ins)) {
+ case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1;
+ case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
+ case BC_TSETM: return bc_a(ins) + nres-1;
+ default: return pt->framesize;
+ }
+}
+
+/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
+void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
+{
+ ERRNO_SAVE
+ GCfunc *fn = curr_func(L);
+ GCproto *pt = funcproto(fn);
+ void *cf = cframe_raw(L->cframe);
+ const BCIns *oldpc = cframe_pc(cf);
+ global_State *g = G(L);
+ BCReg slots;
+ setcframe_pc(cf, pc);
+ slots = cur_topslot(pt, pc, cframe_multres_n(cf));
+ L->top = L->base + slots; /* Fix top. */
+#if LJ_HASJIT
+ {
+ jit_State *J = G2J(g);
+ if (J->state != LJ_TRACE_IDLE) {
+#ifdef LUA_USE_ASSERT
+ ptrdiff_t delta = L->top - L->base;
+#endif
+ J->L = L;
+ lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
+ lua_assert(L->top - L->base == delta);
+ }
+ }
+#endif
+ if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
+ g->hookcount = g->hookcstart;
+ callhook(L, LUA_HOOKCOUNT, -1);
+ L->top = L->base + slots; /* Fix top again. */
+ }
+ if ((g->hookmask & LUA_MASKLINE)) {
+ BCPos npc = proto_bcpos(pt, pc) - 1;
+ BCPos opc = proto_bcpos(pt, oldpc) - 1;
+ BCLine line = lj_debug_line(pt, npc);
+ if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
+ callhook(L, LUA_HOOKLINE, line);
+ L->top = L->base + slots; /* Fix top again. */
+ }
+ }
+ if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
+ callhook(L, LUA_HOOKRET, -1);
+ ERRNO_RESTORE
+}
+
+/* Initialize call. Ensure stack space and return # of missing parameters. */
+static int call_init(lua_State *L, GCfunc *fn)
+{
+ if (isluafunc(fn)) {
+ GCproto *pt = funcproto(fn);
+ int numparams = pt->numparams;
+ int gotparams = (int)(L->top - L->base);
+ int need = pt->framesize;
+ if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
+ lj_state_checkstack(L, (MSize)need);
+ numparams -= gotparams;
+ return numparams >= 0 ? numparams : 0;
+ } else {
+ lj_state_checkstack(L, LUA_MINSTACK);
+ return 0;
+ }
+}
+
+/* Call dispatch. Used by call hooks, hot calls or when recording. */
+ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
+{
+ ERRNO_SAVE
+ GCfunc *fn = curr_func(L);
+ BCOp op;
+ global_State *g = G(L);
+#if LJ_HASJIT
+ jit_State *J = G2J(g);
+#endif
+ int missing = call_init(L, fn);
+#if LJ_HASJIT
+ J->L = L;
+ if ((uintptr_t)pc & 1) { /* Marker for hot call. */
+#ifdef LUA_USE_ASSERT
+ ptrdiff_t delta = L->top - L->base;
+#endif
+ pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
+ lj_trace_hot(J, pc);
+ lua_assert(L->top - L->base == delta);
+ goto out;
+ } else if (J->state != LJ_TRACE_IDLE &&
+ !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
+#ifdef LUA_USE_ASSERT
+ ptrdiff_t delta = L->top - L->base;
+#endif
+ /* Record the FUNC* bytecodes, too. */
+ lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
+ lua_assert(L->top - L->base == delta);
+ }
+#endif
+ if ((g->hookmask & LUA_MASKCALL)) {
+ int i;
+ for (i = 0; i < missing; i++) /* Add missing parameters. */
+ setnilV(L->top++);
+ callhook(L, LUA_HOOKCALL, -1);
+ /* Preserve modifications of missing parameters by lua_setlocal(). */
+ while (missing-- > 0 && tvisnil(L->top - 1))
+ L->top--;
+ }
+#if LJ_HASJIT
+out:
+#endif
+ op = bc_op(pc[-1]); /* Get FUNC* op. */
+#if LJ_HASJIT
+ /* Use the non-hotcounting variants if JIT is off or while recording. */
+ if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
+ (op == BC_FUNCF || op == BC_FUNCV))
+ op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
+#endif
+ ERRNO_RESTORE
+ return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
+}
+
|