/* ************************************************************************ */ /* */ /* Neko Virtual Machine */ /* Copyright (c)2005 Motion-Twin */ /* */ /* This library is free software; you can redistribute it and/or */ /* modify it under the terms of the GNU Lesser General Public */ /* License as published by the Free Software Foundation; either */ /* version 2.1 of the License, or (at your option) any later version. */ /* */ /* This library is distributed in the hope that it will be useful, */ /* but WITHOUT ANY WARRANTY; without even the implied warranty of */ /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU */ /* Lesser General Public License or the LICENSE file for more details. */ /* */ /* ************************************************************************ */ #include "vm.h" #include "neko_mod.h" #include "objtable.h" #include #include #include #ifdef NEKO_POSIX # include # include # define USE_MMAP #endif #define tmp_alloc alloc_private #define tmp_free(ptr) #if defined(NEKO_X86) && !defined(NEKO_MAC) #define JIT_ENABLE #endif #ifdef NEKO_MAC #define STACK_ALIGN #endif #if defined(NEKO_WINDOWS) && defined(_DEBUG_XX) #define STACK_ALIGN #define STACK_ALIGN_DEBUG #endif //#define JIT_DEBUG #ifdef JIT_ENABLE #define PARAMETER_TABLE #include "opcodes.h" extern field id_add, id_radd, id_sub, id_rsub, id_mult, id_rmult, id_div, id_rdiv, id_mod, id_rmod; extern field id_get, id_set; extern int neko_stack_expand( int_val *sp, int_val *csp, neko_vm *vm ); extern value neko_append_int( neko_vm *vm, value str, int x, bool way ); extern value neko_append_strings( value s1, value s2 ); extern value neko_alloc_module_function( void *m, int_val pos, int nargs ); extern void neko_process_trap( neko_vm *vm ); extern void neko_setup_trap( neko_vm *vm ); extern value NEKO_TYPEOF[]; typedef union { void *p; unsigned char *b; unsigned int *w; char *c; int *i; } jit_buffer; typedef struct _jlist { int pos; int target; struct _jlist *next; } jlist; typedef struct { jit_buffer buf; void *baseptr; neko_module *module; int curpc; int size; int *pos; int debug_wait; jlist *jumps; jlist *traps; } jit_ctx; enum Special { VThis, VEnv, VModule, VVm, VSpMax, VTrap, }; #define MODE_CALLBACK 1 #define MODE_PC_CUR 0 #define MODE_PC_ARG 2 enum PushInfosMode { CALLBACK, PC_CUR, PC_ARG }; enum CallMode { NORMAL, THIS_CALL, TAIL_CALL, }; enum Operation { OP_ADD, OP_SUB, OP_MUL, OP_DIV, OP_MOD, OP_LAST, OP_GET, OP_SET }; enum IOperation { IOP_SHL, IOP_SHR, IOP_USHR, IOP_AND, IOP_OR, IOP_XOR, }; #define Eax 0 #define Ebx 3 #define Ecx 1 #define Edx 2 #define Esp 4 #define Ebp 5 #define Esi 6 #define Edi 7 #define ACC Eax #define SP Edi #define CSP Esi #define VM Ebx #define TMP Ecx #define TMP2 Edx #define B(bv) *buf.b++ = bv #define W(wv) *buf.w++ = wv #define Mult1 0 #define Mult2 1 #define Mult4 2 #define Mult8 3 #define JAlways 0 #define JLt 0x82 #define JGte 0x83 #define JEq 0x84 #define JNeq 0x85 #define JLte 0x86 #define JGt 0x87 #define JSignLt 0x8C #define JSignGte 0x8D #define JSignLte 0x8E #define JSignGt 0x8F #define ERROR { tmp_free(ctx->pos); tmp_free(ctx->baseptr); failure("JIT error"); } #define CONST(v) ((int)(int_val)(v)) #define PATCH_JUMP(local) if( local != NULL ) { \ int delta = (int)((int_val)buf.p - ((int_val)local + 1)); \ if( sizeof(*local) == sizeof(int) ) \ *local = delta - 3; \ else { \ if( delta > 127 || delta < -127 ) \ ERROR; \ *local = (char)delta; \ } \ } \ #define FIELD(n) ((n) * 4) #define VMFIELD(f) ((int)(int_val)&((neko_vm*)0)->f) #define FUNFIELD(f) ((int)(int_val)&((vfunction*)0)->f) #define POS() ((int)((int_val)ctx->buf.p - (int_val)ctx->baseptr)) #define GET_PC() CONST(ctx->module->code + ctx->curpc) #define INIT_BUFFER register jit_buffer buf = ctx->buf #define END_BUFFER ctx->buf = buf #define MOD_RM(mod,reg,rm) B((mod << 6) | (reg << 3) | rm) #define SIB MOD_RM #define IS_SBYTE(c) ( (c) >= -128 && (c) < 128 ) #define OP_RM(op,mod,reg,rm) { B(op); MOD_RM(mod,reg,rm); } #define OP_ADDR(op,addr,reg,rm) { B(op); \ MOD_RM(((addr) == 0 && reg != Ebp)?0:(IS_SBYTE(addr)?1:2),rm,reg); \ if( reg == Esp ) B(0x24); \ if( (addr) == 0 && reg != Ebp ) {} \ else if IS_SBYTE(addr) B(addr); \ else W(addr); } // OPCODES : // _r : register // _c : constant // _b : 8-bit constant // _a : [constant] // _i : [reg] // _p : [reg + constant:idx] // _x : [reg + reg:idx * mult] #define XRet() B(0xC3) #define XMov_rr(dst,src) OP_RM(0x8B,3,dst,src) #define XMov_rc(dst,cst) B(0xB8+(dst)); W(cst) #define XMov_rp(dst,reg,idx) OP_ADDR(0x8B,idx,reg,dst) #define XMov_ra(dst,addr) OP_RM(0x8B,0,dst,5); W(addr) #define XMov_rx(dst,r,idx,mult) OP_RM(0x8B,0,dst,4); SIB(Mult##mult,idx,r) #define XMov_pr(dst,idx,src) OP_ADDR(0x89,idx,dst,src) #define XMov_pc(dst,idx,c) OP_ADDR(0xC7,idx,dst,0); W(c) #define XMov_ar(addr,reg) B(0x3E); if( reg == Eax ) { B(0xA3); } else { OP_RM(0x89,0,reg,5); }; W(addr) #define XMov_xr(r,idx,mult,src) OP_RM(0x89,0,src,4); SIB(Mult##mult,idx,r) #define XCall_r(r) OP_RM(0xFF,3,2,r) #define XCall_m_debug(v) { \ int *l; \ XMov_rr(TMP,Ebp); \ XSub_rr(TMP,Esp); \ XAnd_rc(TMP,15); \ XCmp_rc(TMP,0); \ XJump(JEq,l); \ XShr_rc(TMP,2); \ XPush_r(TMP); \ XPush_c(CONST(__LINE__)); \ XMov_rc(TMP,CONST(debug_method_call)); \ XCall_r(TMP); \ PATCH_JUMP(l); \ XMov_rc(TMP,CONST(v)); \ XCall_r(TMP); \ } #define XCall_m_real(v) XMov_rc(TMP,CONST(v)); XCall_r(TMP); #ifdef STACK_ALIGN_DEBUG # define XCall_m XCall_m_debug #else # define XCall_m XCall_m_real #endif #define XCall_d(delta) B(0xE8); W(delta) #define XPush_r(r) B(0x50+(r)) #define XPush_c(cst) B(0x68); W(cst) #define XPush_p(reg,idx) OP_ADDR(0xFF,idx,reg,6) #define XAdd_rc(reg,cst) if IS_SBYTE(cst) { OP_RM(0x83,3,0,reg); B(cst); } else { OP_RM(0x81,3,0,reg); W(cst); } #define XAdd_rr(dst,src) OP_RM(0x03,3,dst,src) #define XSub_rc(reg,cst) if IS_SBYTE(cst) { OP_RM(0x83,3,5,reg); B(cst); } else { OP_RM(0x81,3,5,reg); W(cst); } #define XSub_rr(dst,src) OP_RM(0x2B,3,dst,src) #define XCmp_rr(r1,r2) OP_RM(0x3B,3,r1,r2) #define XCmp_rc(reg,cst) if( reg == Eax ) { B(0x3D); } else { OP_RM(0x81,3,7,reg); }; W(cst) #define XCmp_rb(reg,byte) OP_RM(0x83,3,7,reg); B(byte) #define XJump(how,local) if( (how) == JAlways ) { B(0xE9); } else { B(0x0F); B(how); }; local = buf.i; W(0) #define XJump_near(local) B(0xEB); local = buf.c; B(0) #define XJump_r(reg) OP_RM(0xFF,3,4,reg) #define XPop_r(reg) B(0x58 + (reg)) #define XTest_rc(r,cst) if( r == Eax ) { B(0xA9); W(cst); } else { B(0xF7); MOD_RM(3,0,r); W(cst); } #define XTest_rr(r,src) B(0x85); MOD_RM(3,r,src) #define XAnd_rc(r,cst) if( r == Eax ) { B(0x25); W(cst); } else { B(0x81); MOD_RM(3,4,r); W(cst); } #define XAnd_rr(r,src) B(0x23); MOD_RM(3,r,src) #define XOr_rc(r,cst) if( r == Eax ) { B(0x0D); W(cst); } else { B(0x81); MOD_RM(3,1,r); W(cst); } #define XOr_rr(r,src) B(0x0B); MOD_RM(3,r,src) #define XXor_rc(r,cst) if( r == Eax ) { B(0x35); W(cst); } else { B(0x81); MOD_RM(3,6,r); W(cst); } #define XXor_rr(r,src) B(0x33); MOD_RM(3,r,src) #define shift_r(r,spec) B(0xD3); MOD_RM(3,spec,r); #define shift_c(r,n,spec) if( (n) == 1 ) { B(0xD1); MOD_RM(3,spec,r); } else { B(0xC1); MOD_RM(3,spec,r); B(n); } #define XShl_rr(r,src) if( src != Ecx ) ERROR; shift_r(r,4) #define XShl_rc(r,n) shift_c(r,n,4) #define XShr_rr(r,src) if( src != Ecx ) ERROR; shift_r(r,7) #define XShr_rc(r,n) shift_c(r,n,7) #define XUShr_rr(r,src) if( src != Ecx ) ERROR; shift_r(r,5) #define XUShr_rc(r,n) shift_c(r,n,5) #define XIMul_rr(dst,src) B(0x0F); B(0xAF); MOD_RM(3,dst,src) #define XIDiv_r(r) B(0xF7); MOD_RM(3,7,r) #define XCdq() B(0x99); // FPU #define XFAddp() B(0xDE); B(0xC1) #define XFSubp() B(0xDE); B(0xE9) #define XFMulp() B(0xDE); B(0xC9) #define XFDivp() B(0xDE); B(0xF9) #define XFStp_i(r) B(0xDD); MOD_RM(0,3,r); if( r == Esp ) B(0x24) #define XFLd_i(r) B(0xDD); MOD_RM(0,0,r); if( r == Esp ) B(0x24) #define XFILd_i(r) B(0xDB); MOD_RM(0,0,r); if( r == Esp ) B(0x24) #define is_int(r,flag,local) { XTest_rc(r,1); XJump((flag)?JNeq:JEq,local); } #ifdef STACK_ALIGN # define stack_pad(n) stack_push(Esp,n) # define stack_pop_pad(n,n2) stack_pop(Esp,((n) + (n2))) #else # define stack_pad(n) # define stack_pop_pad(n,n2) stack_pop(Esp,(n)) #endif #define stack_push(r,n) \ if( (n) != 0 ) { \ if( (r) == CSP ) { \ XAdd_rc(r,(n) * 4); \ } else { \ XSub_rc(r,(n) * 4); \ } \ } #define stack_pop(r,n) \ if( (n) != 0 ) { \ if( (r) == CSP ) { \ XSub_rc(r,(n) * 4); \ } else { \ XAdd_rc(r,(n) * 4); \ } \ } #define begin_call() { XMov_pr(VM,VMFIELD(sp),SP); XMov_pr(VM,VMFIELD(csp),CSP); } #define end_call() { XMov_rp(SP,VM,VMFIELD(sp)); XMov_rp(CSP,VM,VMFIELD(csp)); } #define label(code) { XMov_rc(TMP2,CONST(code)); XCall_r(TMP2); } #define todo(str) { int *loop; XMov_rc(TMP2,CONST(str)); XJump(JAlways,loop); *loop = -5; } #define pop(n) if( (n) != 0 ) { \ int i = (n); \ while( i-- > 0 ) { \ XMov_pc(SP,FIELD(i),0); \ } \ stack_pop(SP,n); \ } #define pop_loop(n) { \ char *start; \ int *loop; \ XMov_rc(TMP,n); \ start = buf.c; \ XMov_pc(SP,FIELD(0),0); \ stack_pop(SP,1); \ XSub_rc(TMP,1); \ XCmp_rc(TMP,0); \ XJump(JNeq,loop); \ *loop = (int)(start - buf.c); \ } #ifdef STACK_ALIGN # define PAD_OPT(x) (x) #else # define PAD_OPT(x) 0 #endif #define runtime_error(msg_id,in_label) { \ if( in_label ) { stack_pad(2); } else { stack_pad(1); } \ XPush_c(CONST(strings[msg_id])); \ if( in_label ) { \ XMov_rp(TMP2,Esp,FIELD(2+PAD_OPT(2))); \ XPush_r(TMP2); \ } else { \ XPush_c(GET_PC()); \ } \ label(code->runtime_error); \ } #define get_var_r(reg,v) { \ switch( v ) { \ case VThis: \ XMov_rp(reg,VM,VMFIELD(vthis)); \ break; \ case VEnv: \ XMov_rp(reg,VM,VMFIELD(env)); \ break; \ case VModule: \ XMov_rp(reg,VM,VMFIELD(jit_val)); \ break; \ case VVm: \ XMov_rr(reg,VM); \ break; \ case VSpMax: \ XMov_rp(reg,VM,VMFIELD(spmax)); \ break; \ case VTrap: \ XMov_rp(reg,VM,VMFIELD(trap)); \ break; \ default: \ ERROR; \ break; \ } \ } #define get_var_p(reg,idx,v) { \ switch( v ) { \ case VThis: \ XMov_rp(TMP,VM,VMFIELD(vthis)); \ XMov_pr(reg,idx,TMP); \ break; \ case VEnv: \ XMov_rp(TMP,VM,VMFIELD(env)); \ XMov_pr(reg,idx,TMP); \ break; \ case VModule: \ XMov_rp(TMP,VM,VMFIELD(jit_val)); \ XMov_pr(reg,idx,TMP); \ break; \ case VVm: \ XMov_pr(reg,idx,VM); \ break; \ case VSpMax: \ XMov_rp(TMP,VM,VMFIELD(spmax)); \ XMov_pr(reg,idx,TMP); \ break; \ case VTrap: \ XMov_rp(TMP,VM,VMFIELD(trap)); \ XMov_pr(reg,idx,TMP); \ break; \ default: \ ERROR; \ break; \ } \ } #define set_var_r(v,reg) { \ switch( v ) { \ case VThis: \ XMov_pr(VM,VMFIELD(vthis),reg); \ break; \ case VEnv: \ XMov_pr(VM,VMFIELD(env),reg); \ break; \ case VTrap: \ XMov_pr(VM,VMFIELD(trap),reg); \ break; \ case VModule: \ XMov_pr(VM,VMFIELD(jit_val),reg); \ break; \ default: \ ERROR; \ break; \ } \ } #define set_var_p(v,reg,idx) { \ switch( v ) { \ case VThis: \ XMov_rp(TMP,reg,idx); \ XMov_pr(VM,VMFIELD(vthis),TMP); \ break; \ case VEnv: \ XMov_rp(TMP,reg,idx); \ XMov_pr(VM,VMFIELD(env),TMP); \ break; \ case VTrap: \ XMov_rp(TMP,reg,idx); \ XMov_pr(VM,VMFIELD(trap),TMP); \ break; \ case VModule: \ XMov_rp(TMP,reg,idx); \ XMov_pr(VM,VMFIELD(jit_val),TMP); \ break; \ default: \ ERROR; \ break; \ } \ } #define jump(how,targ) { \ jlist *j = (jlist*)alloc(sizeof(jlist)); \ void *jcode; \ j->target = (int)((int_val*)(int_val)(targ) - ctx->module->code); \ j->next = ctx->jumps; \ ctx->jumps = j; \ XJump(how,jcode); \ j->pos = (int)((int_val)jcode - (int_val)ctx->baseptr); \ } #define setup_before_call(mode,is_callb) { \ push_infos(is_callb?CALLBACK:PC_ARG); \ if( !is_callb ) { XPush_r(ACC); } \ if( mode == THIS_CALL ) { \ set_var_p(VThis,SP,FIELD(0)); \ pop(1); \ } \ set_var_p(VEnv,ACC,FUNFIELD(env)); \ } #define restore_after_call(nargs,pad) { \ int *jok; \ XCmp_rc(ACC,0); \ XJump(JNeq,jok); \ XMov_rp(ACC,Esp,FIELD(nargs+PAD_OPT(pad))); \ XMov_rp(ACC,ACC,FUNFIELD(module)); \ stack_pad(-1); \ XPush_r(ACC); \ XCall_m(val_throw); \ PATCH_JUMP(jok); \ stack_pop_pad(1+nargs,pad); \ pop_infos(); \ } #define NARGS (CALL_MAX_ARGS + 1) #define MAX_ENV 8 typedef struct { char *boot; char *stack_expand; char *runtime_error; char *call_normal_jit[NARGS]; char *call_this_jit[NARGS]; char *call_tail_jit[NARGS]; char *call_normal_prim[NARGS]; char *call_this_prim[NARGS]; char *call_tail_prim[NARGS]; char *call_normal_fun[NARGS]; char *call_this_fun[NARGS]; char *call_tail_fun[NARGS]; char *make_env[MAX_ENV]; char *make_env_n; char *add; char *oop[OP_LAST]; char *oop_r[OP_LAST]; char *oo_get; char *oo_set; char *handle_trap; char *invalid_access; } jit_code; char *jit_boot_seq = NULL; char *jit_handle_trap = NULL; static jit_code *code; static value *strings; static const char *cstrings[] = { "Stack overflow", // 0 "Reading Outside Env", // 1 "Writing Outside Env", // 2 "Invalid call", // 3 "Invalid array access", // 4 "Invalid field access", // 5 "Invalid environment", // 6 "+", // 7 "-", // 8 "*", // 9 "/", // 10 "%", // 11 "<<", // 12 ">>", // 13 ">>>", // 14 "&", // 15 "|", // 16 "^", // 17 "$apply", // 18 "Invalid End Trap", // 19 "$hash", // 20 "Unsupported operation", // 21 }; #define DEFINE_PROC(p,arg) ctx->buf = buf; jit_##p(ctx,arg); buf = ctx->buf #define push_infos(arg) DEFINE_PROC(push_infos,arg) #define test(arg) DEFINE_PROC(test,arg) #define call(mode,nargs) ctx->buf = buf; jit_call(ctx,mode,nargs); buf = ctx->buf #define number_op(arg) DEFINE_PROC(number_op,arg) #define array_access(p) DEFINE_PROC(array_access,p) #define int_op(arg) DEFINE_PROC(int_op,arg) #ifdef STACK_ALIGN_DEBUG #include static void debug_method_call( int line, int stack ) { printf("Stack align error line %d (%d)\n" , line ,stack); exit(-1); } #endif static jit_ctx *jit_init_context( void *ptr, int size ) { jit_ctx *c = (jit_ctx*)alloc(sizeof(jit_ctx)); c->size = size; c->baseptr = ptr; c->buf.p = ptr; c->pos = NULL; c->curpc = 0; c->debug_wait = 0; c->jumps = NULL; c->traps = NULL; return c; } static void jit_finalize_context( jit_ctx *ctx ) { jlist *l; int nbytes = POS(); if( nbytes == 0 || nbytes > ctx->size ) *(int*)0xAABBCC = 0; l = ctx->jumps; while( l != NULL ) { *(int*)((char*)ctx->baseptr + l->pos) = ctx->pos[l->target] - (l->pos + 4); l = l->next; } l = ctx->traps; while( l != NULL ) { *(int*)((char*)ctx->baseptr + l->pos) = ctx->pos[l->target] + (int)(int_val)ctx->baseptr; l = l->next; } } static void jit_push_infos( jit_ctx *ctx, enum PushInfosMode callb ) { INIT_BUFFER; stack_push(CSP,4); if( callb == CALLBACK ) { XMov_pc(CSP,FIELD(-3),CONST(callback_return)); get_var_p(CSP,FIELD(-2),VEnv); get_var_p(CSP,FIELD(-1),VThis); XMov_pc(CSP,FIELD(0),0); } else { if( callb == PC_CUR ) { XMov_pc(CSP,FIELD(-3),GET_PC()); } else { // PC_ARG : on the stack XMov_rp(TMP2,Esp,FIELD(1)); XMov_pr(CSP,FIELD(-3),TMP2); } get_var_p(CSP,FIELD(-2),VEnv); get_var_p(CSP,FIELD(-1),VThis); get_var_p(CSP,FIELD(0),VModule) } END_BUFFER; } #define pop_infos() { \ set_var_p(VModule,CSP,FIELD(0)); \ set_var_p(VThis,CSP,FIELD(-1)); \ set_var_p(VEnv,CSP,FIELD(-2)); \ XMov_pc(CSP,FIELD(0),0); \ XMov_pc(CSP,FIELD(-1),0); \ XMov_pc(CSP,FIELD(-2),0); \ XMov_pc(CSP,FIELD(-3),0); \ stack_pop(CSP,4); \ } static void jit_boot( jit_ctx *ctx, void *_ ) { INIT_BUFFER; XPush_r(Ebp); XPush_r(Ebx); XPush_r(Esi); # ifdef STACK_ALIGN_DEBUG XMov_rr(Ebp,Esp); // ALIGNED STACK # endif XPush_r(Edi); XMov_rp(VM,Esp,FIELD(5)); get_var_r(TMP,VModule); XPush_r(TMP); set_var_p(VModule,Esp,FIELD(9)); XMov_rp(TMP,Esp,FIELD(7)); XMov_rp(ACC,Esp,FIELD(8)); end_call(); XCall_r(TMP); begin_call(); XPop_r(TMP); set_var_r(VModule,TMP); XPop_r(Edi); XPop_r(Esi); XPop_r(Ebx); XPop_r(Ebp); XRet(); END_BUFFER; } static void jit_trap( jit_ctx *ctx, int n ) { INIT_BUFFER; XMov_rp(VM,Esp,FIELD(1)); get_var_r(Ebp,VThis); // restore vm stack_pad(3); XPush_r(VM); XCall_m(neko_process_trap); stack_pop_pad(1,3); // restore registers end_call(); XMov_rr(ACC,Ebp); XMov_rp(Ebp,VM,VMFIELD(start)+FIELD(1)); XMov_rp(Esp,VM,VMFIELD(start)+FIELD(2)); XMov_rp(TMP2,VM,VMFIELD(start)+FIELD(3)); // restore vm jmp_buf XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(3),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(2),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(1),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start),TMP); XPush_r(TMP2); XRet(); END_BUFFER; } static void jit_stack_expand( jit_ctx *ctx, int _ ) { int *jresize, *jdone; int max = MAX_STACK_PER_FUNCTION; INIT_BUFFER; stack_push(CSP,max); XCmp_rr(SP,CSP); XJump(JLt,jresize); stack_pop(CSP,max); XRet(); PATCH_JUMP(jresize); stack_pop(CSP,max); XPush_r(ACC); XPush_r(VM); XPush_r(CSP); XPush_r(SP); XCall_m(neko_stack_expand); XCmp_rb(ACC,0); XJump(JNeq,jdone); stack_pad(-1); XPush_c(CONST(strings[0])); // Stack overflow XCall_m(val_throw); PATCH_JUMP(jdone); XMov_rp(ACC,Esp,FIELD(3)); end_call(); stack_pop(Esp,4); XRet(); END_BUFFER; } static void jit_runtime_error( jit_ctx *ctx, void *unused ) { INIT_BUFFER; push_infos(PC_ARG); // pc begin_call(); XMov_rp(TMP,Esp,FIELD(2)); // msg on stack XPush_r(TMP); XCall_m(val_throw); END_BUFFER; } static void jit_invalid_access( jit_ctx *ctx, int _ ) { INIT_BUFFER; int *jnext; // if( val_field_name(f) == val_null ) RuntimeError("Invalid field access") stack_pad(1); XMov_rp(TMP,Esp,FIELD(2)); // field XPush_r(TMP); XCall_m(val_field_name); stack_pop_pad(1,1); XCmp_rc(ACC,CONST(val_null)); XJump(JNeq,jnext); runtime_error(5,true); // else { // b = alloc_buffer("Invalid field access : "); PATCH_JUMP(jnext); XPush_r(ACC); XPush_c(CONST("Invalid field access : ")); XCall_m(alloc_buffer); stack_pop(Esp,1); // val_buffer(b,v); XPush_r(ACC); XCall_m(val_buffer); // buffer_to_string(b); XCall_m(buffer_to_string); stack_pop(Esp,2); push_infos(PC_ARG); // pc begin_call(); XPush_r(ACC); XCall_m(val_throw); END_BUFFER; } static void jit_test( jit_ctx *ctx, int how ) { INIT_BUFFER; int *jnot1, *jnot2; char *jend; // call val_compare(sp[0],acc) XPush_r(ACC); XMov_rp(TMP,SP,FIELD(0)); XPush_r(TMP); begin_call(); XCall_m(val_compare); end_call(); stack_pop(Esp,2); pop(1); // test ok and != invalid_comparison XCmp_rc(ACC,0); XJump(how,jnot1); XCmp_rc(ACC,invalid_comparison); XJump(JEq,jnot2); XMov_rc(ACC,CONST(val_true)); XJump_near(jend); PATCH_JUMP(jnot1); PATCH_JUMP(jnot2); XMov_rc(ACC,CONST(val_false)); PATCH_JUMP(jend); END_BUFFER; } static void jit_call( jit_ctx *ctx, int mode, int nargs ) { INIT_BUFFER; int *jerr, *jother, *jerr2; char *jend1, *jend2, *jend3; // if( is_int ) : error is_int(ACC,1,jerr); // if( type == jit ) XMov_rp(TMP,ACC,FUNFIELD(t)); XCmp_rb(TMP,VAL_JITFUN); XJump(JNeq,jother); XPush_c(GET_PC()); switch( mode ) { case NORMAL: label(code->call_normal_jit[nargs]); break; case THIS_CALL: label(code->call_this_jit[nargs]); break; case TAIL_CALL: label(code->call_tail_jit[nargs]); break; } if( mode == TAIL_CALL ) jend1 = NULL; else { XJump_near(jend1); } // else if( type == prim ) PATCH_JUMP(jother); XCmp_rb(TMP,VAL_PRIMITIVE); XJump(JNeq,jother); XPush_c(GET_PC()); switch( mode ) { case NORMAL: label(code->call_normal_prim[nargs]); break; case THIS_CALL: label(code->call_this_prim[nargs]); break; case TAIL_CALL: label(code->call_tail_prim[nargs]); break; } XJump_near(jend2); // else if( type == function ) PATCH_JUMP(jother); XCmp_rb(TMP,VAL_FUNCTION); XJump(JNeq,jerr2); XPush_c(GET_PC()); switch( mode ) { case NORMAL: label(code->call_normal_fun[nargs]); break; case THIS_CALL: label(code->call_this_fun[nargs]); break; case TAIL_CALL: label(code->call_tail_fun[nargs]); break; } XJump_near(jend3); // else error PATCH_JUMP(jerr); PATCH_JUMP(jerr2); runtime_error(3,false); // Invalid call // end PATCH_JUMP(jend1); PATCH_JUMP(jend2); PATCH_JUMP(jend3); stack_pop(Esp,1); // pushed pc END_BUFFER; } static void jit_call_jit( jit_ctx *ctx, int nargs, int mode ) { INIT_BUFFER; int *jerr; // check arg count XMov_rp(TMP,ACC,FUNFIELD(nargs)); XCmp_rb(TMP,nargs); XJump(JNeq,jerr); if( mode == TAIL_CALL ) { // pop PC and EIP from the stack stack_pop(Esp,2); set_var_p(VModule,ACC,FUNFIELD(module)); set_var_p(VEnv,ACC,FUNFIELD(env)); XMov_rp(TMP,ACC,FUNFIELD(addr)); XJump_r(TMP); } else { push_infos(PC_ARG); set_var_p(VModule,ACC,FUNFIELD(module)); set_var_p(VEnv,ACC,FUNFIELD(env)); if( mode == THIS_CALL ) { set_var_p(VThis,SP,FIELD(0)); pop(1); } XMov_rp(TMP,ACC,FUNFIELD(addr)); stack_pad(1); XCall_r(TMP); stack_pad(-1); pop_infos(); XRet(); } PATCH_JUMP(jerr); runtime_error(3,true); // Invalid call END_BUFFER; } static void jit_call_prim( jit_ctx *ctx, int nargs, int mode ) { INIT_BUFFER; int *jvararg, *jerr; int i; # ifdef STACK_ALIGN int pad_size = 4 - ((2+nargs)%4); # endif // check arg count XMov_rp(TMP,ACC,FUNFIELD(nargs)); XCmp_rb(TMP,nargs); XJump(JNeq,jvararg); // push args from VMSP to PROCSP setup_before_call(mode,false); stack_pad(pad_size); for(i=0;i sp XMov_rr(TMP,ACC); XMov_rp(ACC,SP,FIELD(0)); // is_int(acc) && is_int(sp) is_int(ACC,false,jnot_int1); is_int(TMP,false,jnot_int2); XShr_rc(ACC,1); XShr_rc(TMP,1); if( op != OP_DIV ) { switch( op ) { case OP_ADD: XAdd_rr(ACC,TMP); break; case OP_SUB: XSub_rr(ACC,TMP); break; case OP_MUL: XIMul_rr(ACC,TMP); break; case OP_MOD: XCmp_rc(TMP,0); XJump(JEq,jmod0); XCdq(); XIDiv_r(TMP); XMov_rr(ACC,Edx); break; default: ERROR; break; } XShl_rc(ACC,1); XOr_rc(ACC,1); } XJump(JAlways,jend); // is_int(acc) && is_number(sp) PATCH_JUMP(jnot_int2); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jerr1); XShr_rc(ACC,1); XPush_r(ACC); XFILd_i(Esp); XAdd_rc(TMP,4); XFLd_i(TMP); stack_pop(Esp,1); XJump(JAlways,jfloat1); // is_number(acc) ? PATCH_JUMP(jnot_int1); XMov_rp(TMP2,ACC,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jerr2); // is_number(acc) && is_number(sp) is_int(TMP,true,jint); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jerr3); XAdd_rc(ACC,4); XFLd_i(ACC); XAdd_rc(TMP,4); XFLd_i(TMP); XJump_near(jfloat2); // is_number(acc) && is_int(sp) PATCH_JUMP(jint); XAdd_rc(ACC,4); XFLd_i(ACC); XShr_rc(TMP,1); XPush_r(TMP); XFILd_i(Esp); stack_pop(Esp,1); XJump_near(jfloat3); // is_object(acc) ? PATCH_JUMP(jerr2); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jnext); XPush_c(GET_PC()); label(code->oop[op]); stack_pop(Esp,1); # ifdef STACK_ALIGN_DEBUG XJump(JAlways,jend2); # else XJump_near(jend2); # endif // is_object(sp) ? PATCH_JUMP(jnext); is_int(TMP,true,jerr4); XMov_rp(TMP2,TMP,FIELD(0)); PATCH_JUMP(jerr1); PATCH_JUMP(jerr3); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr5); XPush_c(GET_PC()); label(code->oop_r[op]); stack_pop(Esp,1); # ifdef STACK_ALIGN_DEBUG XJump(JAlways,jend3); # else XJump_near(jend3); # endif // error if( op == OP_MOD ) { PATCH_JUMP(jmod0); } PATCH_JUMP(jerr5); PATCH_JUMP(jerr4); runtime_error(7 + op,false); // division is always float if( op == OP_DIV ) { PATCH_JUMP(jend); XPush_r(ACC); XFILd_i(Esp); XPush_r(TMP); XFILd_i(Esp); stack_pop(Esp,2); } // perform operation PATCH_JUMP(jfloat1); PATCH_JUMP(jfloat2); PATCH_JUMP(jfloat3); switch( op ) { case OP_ADD: XFAddp(); break; case OP_SUB: XFSubp(); break; case OP_DIV: XFDivp(); break; case OP_MUL: XFMulp(); break; case OP_MOD: stack_push(Esp,2); XFStp_i(Esp); stack_push(Esp,2); XFStp_i(Esp); XCall_m(fmod); stack_pop(Esp,2); break; default: ERROR; break; } if( op != OP_MOD ) { stack_push(Esp,2); } XFStp_i(Esp); XCall_m(alloc_float); stack_pop(Esp,2); if( op != OP_DIV ) PATCH_JUMP(jend); PATCH_JUMP(jend2); PATCH_JUMP(jend3); pop(1); END_BUFFER; } static void jit_int_op( jit_ctx *ctx, enum IOperation op ) { INIT_BUFFER; int *jerr1, *jerr2; char *jend; is_int(ACC,false,jerr1); XMov_rr(TMP,ACC); XShr_rc(TMP,1); XMov_rp(ACC,SP,FIELD(0)); is_int(ACC,false,jerr2); XShr_rc(ACC,1); switch( op ) { case IOP_SHL: XShl_rr(ACC,TMP); break; case IOP_SHR: XShr_rr(ACC,TMP); break; case IOP_USHR: XUShr_rr(ACC,TMP); break; case IOP_AND: XAnd_rr(ACC,TMP); break; case IOP_OR: XOr_rr(ACC,TMP); break; case IOP_XOR: XXor_rr(ACC,TMP); break; default: ERROR; } XShl_rc(ACC,1); XOr_rc(ACC,1); XJump_near(jend); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(12 + op,false); PATCH_JUMP(jend); pop(1); END_BUFFER; } #define END_COUNTER 5 #define END() if( nends >= END_COUNTER ) ERROR; XJump(JAlways,jends[nends]); nends++ static void jit_add( jit_ctx *ctx, int _ ) { INIT_BUFFER; int nends = 0; int *jends[END_COUNTER]; int *jnot_int, *jnext, *jsp_object1, *jnot_float, *jbadd1, *jnot_string, *jappint1, *jsp_object2; int *joop1, *jnot_object, *joop2, *jerr1, *jbadd3, *jerr2; int *jalloc1, *jalloc2, *jalloc3, *jbadd2; // is_int(acc) && is_int(sp) is_int(ACC,false,jnot_int); // ! we are sure that sp is not an integer ! // is_int(acc) && is_float(sp) -> push(val_int(acc)+val_float(sp)) XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jnext); XAdd_rc(TMP,4); XFLd_i(TMP); stack_push(Esp,1); XShr_rc(ACC,1); XPush_r(ACC); XFILd_i(Esp); XFAddp(); XFStp_i(Esp); XJump(JAlways,jalloc1); // is_int(acc) && is_string(sp) -> prepare args for append_int(vm,sp,val_int(acc),true) PATCH_JUMP(jnext); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_STRING); XJump(JNeq,jsp_object1); stack_pad(3); XPush_c(1); XShr_rc(ACC,1); XPush_r(ACC); XPush_r(TMP);; XPush_r(VM); XJump(JAlways,jappint1); // is_float(acc) PATCH_JUMP(jnot_int); XMov_rp(TMP2,ACC,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jnot_float); // is_float(acc) && is_int(sp) -> push(val_float(acc)+val_int(sp)) is_int(TMP,false,jnext); stack_push(Esp,1); XShr_rc(TMP,1); XPush_r(TMP); XFILd_i(Esp); XAdd_rc(ACC,4); XFLd_i(ACC); XFAddp(); XFStp_i(Esp); XJump(JAlways,jalloc2); // is_float(acc) && is_float(sp) -> push(val_float(acc)+val_float(sp)) PATCH_JUMP(jnext); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jnext); stack_push(Esp,2); XAdd_rc(TMP,4); XFLd_i(TMP); XAdd_rc(ACC,4); XFLd_i(ACC); XFAddp(); XFStp_i(Esp); XJump(JAlways,jalloc3); // is_float(acc) && is_string(sp) -> BUFADD PATCH_JUMP(jnext); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_STRING); XJump(JEq,jbadd1); XJump(JAlways,jsp_object2); // is_string(acc) PATCH_JUMP(jnot_float); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_STRING); XJump(JNeq,jnot_string); // is_string(acc) && is_int(sp) -> prepare args for append_int(vm,acc,val_int(sp),false) is_int(TMP,false,jnext); stack_pad(3); XPush_c(0); XShr_rc(TMP,1); XPush_r(TMP); XPush_r(ACC); XPush_r(VM); // call append_int PATCH_JUMP(jappint1); XCall_m(neko_append_int); stack_pop_pad(4,3); END(); // is_string(acc) && is_string(sp) -> append_strings(sp,acc) PATCH_JUMP(jnext); XMov_rp(TMP2,TMP,FIELD(0)); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_STRING); XJump(JNeq,jnext); stack_pad(1); XPush_r(ACC); XPush_r(TMP); XCall_m(neko_append_strings); stack_pop_pad(2,1); END(); // is_string(acc) && is_object(sp) -> object op (else buffer) PATCH_JUMP(jnext); XCmp_rb(TMP2,VAL_OBJECT); XJump(JEq,joop1); XJump(JAlways,jbadd2); // is_object(acc) PATCH_JUMP(jnot_string); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jnot_object); // is_object(acc) && is_object(sp) -> object op (else object rop) is_int(TMP,true,jnext); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_OBJECT); XJump(JEq,joop2); PATCH_JUMP(jnext); label(code->oop_r[OP_ADD]); END(); // is_other(acc) && !is_int(sp) && is_string(sp) -> BUF PATCH_JUMP(jnot_object); is_int(TMP,true,jerr1); XMov_rp(TMP2,TMP,FIELD(0)); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_STRING); XJump(JEq,jbadd3); // is_other(acc) && is_object(sp) -> OOP else Error PATCH_JUMP(jsp_object1); PATCH_JUMP(jsp_object2); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr2); // object op PATCH_JUMP(joop1); PATCH_JUMP(joop2); label(code->oop[OP_ADD]); END(); // errors PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(7,true); // "+" // append using buffer PATCH_JUMP(jbadd1); PATCH_JUMP(jbadd2); PATCH_JUMP(jbadd3); XPush_r(ACC); XPush_r(TMP); // alloc_buffer(NULL) XPush_c(0); XCall_m(alloc_buffer); XMov_pr(Esp,FIELD(0),ACC); begin_call(); XCall_m(val_buffer); XMov_rp(TMP,Esp,FIELD(2)); XMov_pr(Esp,FIELD(1),TMP); XCall_m(val_buffer); end_call(); XCall_m(buffer_to_string); stack_pop(Esp,3); END(); // call alloc_float PATCH_JUMP(jalloc1); PATCH_JUMP(jalloc2); PATCH_JUMP(jalloc3); XCall_m(alloc_float); stack_pop(Esp,2); // end while( nends-- > 0 ) { PATCH_JUMP(jends[nends]); } XRet(); END_BUFFER; } static void jit_array_access( jit_ctx *ctx, int n ) { INIT_BUFFER; int *jerr1, *jerr2; char *jend1, *jend2 = NULL, *jend3; int *jnot_array, *jbounds = NULL; is_int(ACC,true,jerr1); XMov_rp(TMP,ACC,0); XMov_rr(TMP2,TMP); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_ARRAY); XJump(JNeq,jnot_array); if( n > 0 ) { XUShr_rc(TMP,3); XCmp_rc(TMP,n); XJump(JLte,jbounds); } XMov_rp(ACC,ACC,FIELD(n + 1)); XJump_near(jend1); if( n > 0 ) { PATCH_JUMP(jbounds); XMov_rc(ACC,CONST(val_null)); XJump_near(jend2); } PATCH_JUMP(jnot_array); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr2); XPush_c(GET_PC()); XMov_rr(TMP,ACC); XMov_rc(ACC,CONST(alloc_int(n))); label(code->oo_get); stack_pop(Esp,1); XJump_near(jend3); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(4,false); PATCH_JUMP(jend1); PATCH_JUMP(jend2); PATCH_JUMP(jend3); END_BUFFER; } static void jit_make_env( jit_ctx *ctx, int esize ) { INIT_BUFFER; int *jerr1, *jerr2, *jok; int i; if( esize == -1 ) { XMov_rr(TMP2,TMP); // store esize } // check type t_function or t_jit is_int(ACC,true,jerr1); XMov_rp(TMP,ACC,FUNFIELD(t)); XCmp_rb(TMP,VAL_JITFUN); XJump(JEq,jok); XCmp_rb(TMP,VAL_FUNCTION); XJump(JNeq,jerr2); PATCH_JUMP(jok); // prepare args for alloc_module_function XPush_r(TMP); // acc->type stack_push(Esp,1); // empty cell stack_pad(2); XMov_rp(TMP,ACC,FUNFIELD(nargs)); XPush_r(TMP); XMov_rp(TMP,ACC,FUNFIELD(addr)); XPush_r(TMP); XMov_rp(TMP,ACC,FUNFIELD(module)); XPush_r(TMP); // call alloc_array(n) stack_pad(3); if( esize == -1 ) { XPush_r(TMP2); } else { XPush_c(esize); } XCall_m(alloc_array); if( esize == -1 ) { char *start; int *loop; XPop_r(TMP2); stack_pad(-3); // fill array start = buf.c; XMov_rp(TMP,SP,FIELD(0)); XMov_xr(ACC,TMP2,4,TMP); XMov_pc(SP,FIELD(0),0); stack_pop(SP,1); XSub_rc(TMP2,1); XCmp_rc(TMP2,0); XJump(JNeq,loop); *loop = (int)(start - buf.c); } else { stack_pop_pad(1,3); // fill array for(i=0;ioo_get); stack_pop(Esp,1); XJump_near(jend3); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); PATCH_JUMP(jerr3); runtime_error(4,false); // Invalid array access PATCH_JUMP(jend1); PATCH_JUMP(jend2); PATCH_JUMP(jend3); break; } case AccIndex: array_access(p); break; case AccIndex0: array_access(0); break; case AccIndex1: array_access(1); break; case AccField: { int *jerr1, *jerr2, *jend1, *loop, *no_resolver; char *jend2, *jend3, *start; is_int(ACC,true,jerr1); XMov_rp(TMP,ACC,FIELD(0)); XCmp_rb(TMP,VAL_OBJECT); XJump(JNeq,jerr2); XPush_r(ACC); XPush_r(VM); stack_pad(1); XMov_rr(VM,ACC); XPush_c(p); start = buf.c; XMov_rp(TMP,VM,FIELD(1)); XPush_r(TMP); XCall_m(otable_find); XCmp_rc(ACC,0); XJump(JNeq,jend1); stack_pop(Esp,1); XMov_rp(VM,VM,FIELD(2)); // acc = acc->proto XCmp_rc(VM,0); XJump(JNeq,loop); *loop = (int)(start - buf.c); stack_pop_pad(1,1); XPop_r(VM); XPop_r(ACC); XMov_rp(TMP,VM,VMFIELD(resolver)); XCmp_rc(TMP,0); XJump(JEq,no_resolver); XPush_c(CONST(alloc_int(p))); XPush_r(ACC); XPush_r(TMP); begin_call(); XCall_m(val_call2); end_call(); stack_pop(Esp,3); XJump_near(jend3); PATCH_JUMP(no_resolver); XMov_rc(ACC,CONST(val_null)); XJump_near(jend2); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); XPush_c(p); XPush_c(GET_PC()); label(code->invalid_access); PATCH_JUMP(jend1); stack_pop_pad(2,2); XPop_r(VM); stack_pop(Esp,1); XMov_rp(ACC,ACC,FIELD(0)); PATCH_JUMP(jend2); PATCH_JUMP(jend3); break; } case SetStack: XMov_pr(SP,FIELD(p),ACC); break; case SetGlobal: XMov_ar(CONST(p),ACC); break; case SetEnv: get_var_r(TMP,VEnv); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rc(TMP2,(p << 3) | VAL_ARRAY); XJump(JGt,jok); runtime_error(2,false); // Writing Outside Env PATCH_JUMP(jok); XMov_pr(TMP,FIELD(p+1),ACC); // val_array_ptr(env)[p] = acc break; case SetThis: set_var_r(VThis,ACC); break; case SetField: { int *jerr1, *jerr2; char *jend; XMov_rp(TMP,SP,FIELD(0)); is_int(TMP,true,jerr1); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr2); // call otable_replace(table,field,acc) stack_pad(2); XPush_r(ACC); XPush_c(p); XMov_rp(TMP,TMP,FIELD(1)); XPush_r(TMP); XCall_m(otable_replace); stack_pop(Esp,3); XMov_rp(ACC,Esp,FIELD(-1)); stack_pad(-2); XJump_near(jend); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(5,false); PATCH_JUMP(jend); pop(1); break; } case SetArray: { int *jerr1, *jerr2, *jerr3, *jnot_array, *jend1; char *jend2, *jend3; XMov_rp(TMP,SP,FIELD(0)); // sp[0] : array/object is_int(TMP,true,jerr1); XMov_rp(TMP2,TMP,FIELD(0)); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_ARRAY); XJump(JNeq,jnot_array); XMov_rp(TMP2,SP,FIELD(1)); // sp[1] : index is_int(TMP2,false,jerr2); XMov_rp(TMP,TMP,FIELD(0)); // tmp = tmp->type XShr_rc(TMP2,1); XUShr_rc(TMP,3); XCmp_rr(TMP2,TMP); XJump(JGte,jend1); XMov_rp(TMP,SP,FIELD(0)); XAdd_rc(TMP2,1); XMov_xr(TMP,TMP2,4,ACC); XJump_near(jend2); PATCH_JUMP(jnot_array); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr3); XMov_rp(TMP2,SP,FIELD(1)); // index XPush_r(TMP2); XPush_c(GET_PC()); label(code->oo_set); stack_pop(Esp,2); XJump_near(jend3); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); PATCH_JUMP(jerr3); runtime_error(4,false); PATCH_JUMP(jend1); PATCH_JUMP(jend2); PATCH_JUMP(jend3); pop(2); break; } case SetIndex: { int *jerr1, *jerr2, *jnot_array, *jend1; char *jend2, *jend3; XMov_rp(TMP,SP,FIELD(0)); // sp[0] : array / object pop(1); is_int(TMP,true,jerr1); XMov_rp(TMP2,TMP,FIELD(0)); XAnd_rc(TMP2,7); XCmp_rb(TMP2,VAL_ARRAY); XJump(JNeq,jnot_array); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rc(TMP2,(p << 3) | VAL_ARRAY); // fake header XJump(JLte,jend1); XMov_pr(TMP,FIELD(p + 1),ACC); XJump_near(jend2); PATCH_JUMP(jnot_array); XCmp_rb(TMP2,VAL_OBJECT); XJump(JNeq,jerr2); XPush_c(CONST(alloc_int(p))); XPush_c(GET_PC()); label(code->oo_set); stack_pop(Esp,2); XJump_near(jend3); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(4,false); PATCH_JUMP(jend1); PATCH_JUMP(jend2); PATCH_JUMP(jend3); break; } case Push: stack_push(SP,1); XMov_pr(SP,FIELD(0),ACC); break; case Pop: if( p > 10 ) { pop_loop(p); } else { pop(p); } break; case Jump: jump(JAlways,p); break; case JumpIf: XCmp_rc(ACC,CONST(val_true)); jump(JEq,p); break; case JumpIfNot: XCmp_rc(ACC,CONST(val_true)); jump(JNeq,p); break; case Neq: { int *jnot1, *jnot2; char *jend; // call val_compare(sp[0],acc) XPush_r(ACC); XMov_rp(TMP,SP,FIELD(0)); XPush_r(TMP); begin_call(); XCall_m(val_compare); end_call(); stack_pop(Esp,2); pop(1); // test if ok and != invalid_comparison XCmp_rc(ACC,0); XJump(JNeq,jnot1); XCmp_rc(ACC,invalid_comparison); XJump(JEq,jnot2); XMov_rc(ACC,CONST(val_false)); XJump_near(jend); PATCH_JUMP(jnot1); PATCH_JUMP(jnot2); XMov_rc(ACC,CONST(val_true)); PATCH_JUMP(jend); break; } case Eq: test(JNeq); break; case Gt: test(JSignLte); break; case Gte: test(JSignLt); break; case Lt: test(JSignGte); break; case Lte: test(JSignGt); break; case Bool: case Not: { int *jfalse1, *jfalse2, *jfalse3; char *jend; XCmp_rc(ACC,CONST(val_false)); XJump(JEq,jfalse1); XCmp_rc(ACC,CONST(val_null)); XJump(JEq,jfalse2); XCmp_rc(ACC,CONST(alloc_int(0))); XJump(JEq,jfalse3); XMov_rc(ACC,CONST((op == Bool)?val_true:val_false)); XJump_near(jend); PATCH_JUMP(jfalse1); PATCH_JUMP(jfalse2); PATCH_JUMP(jfalse3); XMov_rc(ACC,CONST((op == Bool)?val_false:val_true)); PATCH_JUMP(jend); break; } case IsNull: { int *jnext; char *jend; XCmp_rc(ACC,CONST(val_null)); XJump(JNeq,jnext); XMov_rc(ACC,CONST(val_true)); XJump_near(jend); PATCH_JUMP(jnext); XMov_rc(ACC,CONST(val_false)); PATCH_JUMP(jend); break; } case IsNotNull: { int *jnext; char *jend; XCmp_rc(ACC,CONST(val_null)); XJump(JNeq,jnext); XMov_rc(ACC,CONST(val_false)); XJump_near(jend); PATCH_JUMP(jnext); XMov_rc(ACC,CONST(val_true)); PATCH_JUMP(jend); break; } case Call: call(NORMAL,p); break; case ObjCall: call(THIS_CALL,p); break; case TailCall: { int stack = (p >> 3); int nargs = (p & 7); int i = nargs; while( i > 0 ) { i--; XMov_rp(TMP,SP,FIELD(i)); XMov_pr(SP,FIELD(stack - nargs + i),TMP); } if( stack - nargs > 10 ) { pop_loop(stack - nargs); } else { pop(stack - nargs); } call(TAIL_CALL,nargs); // in case we return from a Primitive XRet(); } break; case Ret: if( p > 10 ) { pop_loop(p); } else { pop(p); } XRet(); break; case Add: { int *jnot_int, *jnot_int2, *jint, *jnot_float1, *jnot_float2; char *jend, *jend2; // tmp = sp[0] XMov_rp(TMP,SP,FIELD(0)); // is_int(acc) && is_int(sp) is_int(ACC,false,jnot_int); is_int(TMP,false,jnot_int2); XAdd_rr(ACC,TMP); XSub_rc(ACC,1); XJump_near(jend); // is_float(acc) && is_float(sp) PATCH_JUMP(jnot_int); XMov_rp(TMP2,ACC,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jnot_float1); is_int(TMP,true,jint); XMov_rp(TMP2,TMP,FIELD(0)); XCmp_rb(TMP2,VAL_FLOAT); XJump(JNeq,jnot_float2); XAdd_rc(ACC,4); XFLd_i(ACC); XAdd_rc(TMP,4); XFLd_i(TMP); XFAddp(); stack_push(Esp,2); XFStp_i(Esp); XCall_m(alloc_float); stack_pop(Esp,2); XJump_near(jend2); // else... PATCH_JUMP(jint); PATCH_JUMP(jnot_float1); PATCH_JUMP(jnot_float2); PATCH_JUMP(jnot_int2); XPush_c(GET_PC()); label(code->add); stack_pop(Esp,1); PATCH_JUMP(jend); PATCH_JUMP(jend2); pop(1); break; } case Sub: number_op(OP_SUB); break; case Div: number_op(OP_DIV); break; case Mult: number_op(OP_MUL); break; case Mod: number_op(OP_MOD); break; case Shl: int_op(IOP_SHL); break; case Shr: int_op(IOP_SHR); break; case UShr: int_op(IOP_USHR); break; case And: int_op(IOP_AND); break; case Or: int_op(IOP_OR); break; case Xor: int_op(IOP_XOR); break; case New: XPush_r(ACC); XCall_m(alloc_object); stack_pop(Esp,1); break; case MakeArray: stack_pad(3); XPush_r(ACC); XPush_c(p + 1); XCall_m(alloc_array); XMov_rp(TMP,Esp,FIELD(1)); // tmp = saved acc XMov_pr(ACC,FIELD(1),TMP); // val_array_ptr(acc)[0] = tmp stack_pop_pad(2,3); if( p < 6 ) { i = 0; while( p > 0 ) { p--; i++; XMov_rp(TMP,SP,FIELD(p)); XMov_pr(ACC,FIELD(i + 1),TMP); XMov_pc(SP,FIELD(p),0); } stack_pop(SP,i); } else { char *start; int *loop; XMov_rc(TMP2,(p+1)); start = buf.c; XMov_rp(TMP,SP,FIELD(0)); XMov_pc(SP,FIELD(0),0); XMov_xr(ACC,TMP2,4,TMP); stack_pop(SP,1); XSub_rc(TMP2,1); XCmp_rc(TMP2,1); XJump(JNeq,loop); *loop = (int)(start - buf.c); } break; case MakeEnv: XPush_c(GET_PC()); if( p >= MAX_ENV ) { XMov_rc(TMP,p); label(code->make_env_n); } else { label(code->make_env[p]); } stack_pop(Esp,1); break; case Last: XRet(); break; case Apply: { int *jerr1, *jerr2, *jnext, *jcall1, *jcall2, *jdone; char *jend, *start; is_int(ACC,true,jerr1); XMov_rp(TMP,ACC,FIELD(0)); XAnd_rc(TMP,7); XCmp_rb(TMP,VAL_FUNCTION); XJump(JNeq,jerr2); XMov_rp(TMP,ACC,FUNFIELD(nargs)); // what do we do depending of the number of args ? XCmp_rb(TMP,p); XJump(JSignGt,jnext); XJump(JEq,jcall1); XCmp_rb(TMP,VAR_ARGS); XJump(JEq,jcall2); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(18,false); // $apply // build the apply PATCH_JUMP(jnext); XPush_r(ACC); XPush_r(TMP); XAdd_rc(TMP,1); XPush_r(TMP); XCall_m(alloc_array); stack_pop(Esp,1); XPop_r(TMP); XPop_r(TMP2); // fill the env array XMov_pr(ACC,FIELD(1),TMP2); // env[0] = f i = p; while( i-- > 0 ) { // env[*pc-i] = sp[i] XMov_rp(TMP2,SP,FIELD(i)); XMov_pr(ACC,FIELD(1+p-i),TMP2); } pop(p); // fargs = fargs - *pc XSub_rc(TMP,p); // prepare stack for alloc_apply XPush_r(ACC); XPush_r(TMP); // fill the env array with null's XAdd_rc(ACC,(2+p) * 4 ); start = buf.c; XCmp_rb(TMP,0); XJump(JEq,jdone); XMov_pc(ACC,FIELD(0),CONST(val_null)); XAdd_rc(ACC,4); XSub_rc(TMP,1); // jump back to "start" { char *back; XJump_near(back); *back = (char)(start - (back + 1)); } // call alloc_apply PATCH_JUMP(jdone); XCall_m(neko_alloc_apply); stack_pop(Esp,2); XJump_near(jend); // direct call PATCH_JUMP(jcall1); PATCH_JUMP(jcall2); call(NORMAL,p); PATCH_JUMP(jend); break; } case Trap: { // save some vm->start on the stack XMov_rp(TMP,VM,VMFIELD(start)); XPush_r(TMP); XMov_rp(TMP,VM,VMFIELD(start)+FIELD(1)); XPush_r(TMP); XMov_rp(TMP,VM,VMFIELD(start)+FIELD(2)); XPush_r(TMP); XMov_rp(TMP,VM,VMFIELD(start)+FIELD(3)); XPush_r(TMP); // save magic, ebp , esp and ret_pc XMov_pc(VM,VMFIELD(start),CONST(jit_handle_trap)); XMov_pr(VM,VMFIELD(start)+FIELD(1),Ebp); XMov_pr(VM,VMFIELD(start)+FIELD(2),Esp); XMov_pc(VM,VMFIELD(start)+FIELD(3),-1); { jlist *t = (jlist*)alloc(sizeof(jlist)); ctx->buf = buf; t->pos = POS() - 4; t->target = (int)((int_val*)(int_val)p - ctx->module->code); t->next = ctx->traps; ctx->traps = t; } // neko_setup_trap(vm) XPush_r(ACC); stack_pad(3); XPush_r(VM); begin_call(); XCall_m(neko_setup_trap); end_call(); stack_pop(Esp,1); stack_pad(-3); XPop_r(ACC); break; } case EndTrap: { // check spmax - trap = sp get_var_r(TMP,VSpMax); get_var_r(TMP2,VTrap); XShl_rc(TMP2,2); XSub_rr(TMP,TMP2); XCmp_rr(TMP,SP); XJump(JEq,jok); runtime_error(19,false); // Invalid End Trap PATCH_JUMP(jok); // restore VM jmp_buf XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(3),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(2),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start)+FIELD(1),TMP); XPop_r(TMP); XMov_pr(VM,VMFIELD(start),TMP); // trap = val_int(sp[5]) XMov_rp(TMP,SP,FIELD(5)); XShr_rc(TMP,1); set_var_r(VTrap,TMP); pop(6); break; } case TypeOf: { int *jnot_int; char *jend; is_int(ACC,false,jnot_int); XMov_rc(ACC,CONST(alloc_int(1))); // t_int != VAL_INT XJump_near(jend); PATCH_JUMP(jnot_int); XMov_rp(TMP,ACC,FIELD(0)); XAnd_rc(TMP,7); XMov_rc(TMP2,CONST(NEKO_TYPEOF)); XMov_rx(ACC,TMP2,TMP,4); PATCH_JUMP(jend); break; } case Compare: { int *jint; char *jend; // val_compare(sp[0],acc) XPush_r(ACC); XMov_rp(TMP,SP,FIELD(0)); XPush_r(TMP); begin_call(); XCall_m(val_compare); end_call(); stack_pop(Esp,2); XCmp_rc(ACC,invalid_comparison); XJump(JNeq,jint); XMov_rc(ACC,CONST(val_null)); XJump_near(jend); PATCH_JUMP(jint); XShl_rc(ACC,1); XOr_rc(ACC,1); PATCH_JUMP(jend); pop(1); break; } case PhysCompare: { int *jeq, *jlow; char *jend1, *jend2; XMov_rp(TMP,SP,FIELD(0)); pop(1); XCmp_rr(ACC,TMP); XJump(JEq,jeq); XJump(JSignLt,jlow); XMov_rc(ACC,CONST(alloc_int(-1))); XJump_near(jend1); PATCH_JUMP(jlow); XMov_rc(ACC,CONST(alloc_int(1))); XJump_near(jend2); PATCH_JUMP(jeq); XMov_rc(ACC,CONST(alloc_int(0))); PATCH_JUMP(jend1); PATCH_JUMP(jend2); break; } case Hash: { int *jerr1, *jerr2; char *jend; is_int(ACC,true,jerr1); XMov_rp(TMP,ACC,FIELD(0)); XAnd_rc(TMP,7); XCmp_rb(TMP,VAL_STRING); XJump(JNeq,jerr2); begin_call(); XAdd_rc(ACC,4); // val_string(acc) XPush_r(ACC); XCall_m(val_id); stack_pop(Esp,1); XShl_rc(ACC,1); XOr_rc(ACC,1); XJump_near(jend); PATCH_JUMP(jerr1); PATCH_JUMP(jerr2); runtime_error(20,false); // $hash PATCH_JUMP(jend); break; } case JumpTable: { int njumps = p / 2; int *jok1, *jok2; char *jnext1, *jnext2; is_int(ACC,true,jok1); XMov_rc(TMP2,njumps); XJump_near(jnext1); PATCH_JUMP(jok1); XCmp_rc(ACC,p); XJump(JLt,jok2); XMov_rc(TMP2,njumps); XJump_near(jnext2); PATCH_JUMP(jok2); XMov_rr(TMP2,ACC); XShr_rc(TMP2,1); PATCH_JUMP(jnext1); PATCH_JUMP(jnext2); get_var_r(TMP,VModule); // tmp = jit + tmp2 * 5 XMov_rp(TMP,TMP,FIELD(0)); // m->jit XAdd_rr(TMP,TMP2); XShl_rc(TMP2,2); XAdd_rr(TMP,TMP2); ctx->debug_wait = njumps; ctx->buf = buf; { int add_size = 6; int small_add_size = 3; int jump_rsize = 2; int delta = POS() + add_size + jump_rsize; if( IS_SBYTE(delta) ) delta += small_add_size - add_size; XAdd_rc(TMP,delta); XJump_r(TMP); } break; } default: ERROR; } END_BUFFER; } #if defined(STACK_ALIGN_DEBUG) || defined(JIT_DEBUG) # define MAX_OP_SIZE 1000 # define MAX_BUF_SIZE 1000 #else # define MAX_OP_SIZE 298 // Apply(4) + label(stack_expand) # define MAX_BUF_SIZE 500 #endif #define FILL_BUFFER(f,param,ptr) \ { \ jit_ctx *ctx; \ char *buf = alloc_private(MAX_BUF_SIZE); \ int size; \ ctx = jit_init_context(buf,MAX_BUF_SIZE); \ f(ctx,param); \ size = POS(); \ buf = alloc_jit_mem(size); \ code->ptr = buf; \ memcpy(buf,ctx->baseptr,size); \ ctx->buf.p = buf + size; \ ctx->baseptr = buf; \ jit_finalize_context(ctx); \ } #ifdef USE_MMAP static void free_jit_mem( void *_p ) { int *p = (int*)_p - 1; munmap(p,*p); } static void free_jit_abstract( value v ) { free_jit_mem(val_data(v)); } static char *alloc_jit_mem( int size ) { int *p; // add space for size size += sizeof(int); // round to next page size += (4096 - size%4096); p = (int*)mmap(NULL,size,PROT_READ|PROT_WRITE|PROT_EXEC,(MAP_PRIVATE|MAP_ANON),-1,0); if( p == (int*)-1 ) val_throw(alloc_string("Failed to allocate JIT memory")); *p = size; return (char*)(p + 1); } #else # define alloc_jit_mem alloc_private #endif void neko_init_jit() { int nstrings = sizeof(cstrings) / sizeof(const char *); int i; strings = alloc_root(nstrings); for(i=0;iboot; jit_handle_trap = code->handle_trap; } void neko_free_jit() { # ifdef USE_MMAP int i; for(i=0;inglobals && !val_is_function(m->globals[k]) ) k++; if( k == m->nglobals ) { *faddr = -1; return 0; } *faddr = (int_val*)((vfunction*)m->globals[k])->addr - m->code; return k; } void neko_module_jit( neko_module *m ) { unsigned int i = 0; int_val faddr; unsigned int fcursor = next_function(m,0,&faddr); jit_ctx *ctx = jit_init_context(NULL,0); ctx->pos = (int*)tmp_alloc(sizeof(int)*(m->codesize + 1)); ctx->module = m; while( i <= m->codesize ) { enum OPCODE op = m->code[i]; int curpos = POS(); ctx->pos[i] = curpos; ctx->curpc = i + 2; // resize buffer if( curpos + MAX_OP_SIZE > ctx->size ) { int nsize = ctx->size ? (ctx->size * 4) / 3 : ((m->codesize + 1) * 20); char *buf2; if( nsize - curpos < MAX_OP_SIZE ) nsize = curpos + MAX_OP_SIZE; buf2 = tmp_alloc(nsize); memcpy(buf2,ctx->baseptr,curpos); tmp_free(ctx->baseptr); ctx->baseptr = buf2; ctx->buf.p = buf2 + curpos; ctx->size = nsize; } // begin of function : check stack overflow if( faddr == i ) { INIT_BUFFER; label(code->stack_expand); END_BUFFER; fcursor = next_function(m,fcursor+1,&faddr); } // --------- debug --------- # ifdef JIT_DEBUG if( ctx->debug_wait ) ctx->debug_wait--; else { INIT_BUFFER; XPush_r(ACC); // val_print(acc) XPush_r(ACC); XCall_m_real(val_print); stack_pop(Esp,1); // val_print(pc_pos) XPush_c(CONST(alloc_int(i))); XCall_m_real(val_print); stack_pop(Esp,1); // val_print(alloc_int(spmax - sp)) get_var_r(TMP,VSpMax); XSub_rr(TMP,SP); XShr_rc(TMP,1); XOr_rc(TMP,1); XPush_r(TMP); XCall_m_real(val_print); stack_pop(Esp,1); // val_print(alloc_int(csp - spmin)) XMov_rp(TMP2,VM,VMFIELD(spmin)); XMov_rr(TMP,CSP); XSub_rr(TMP,TMP2); XShr_rc(TMP,1); XOr_rc(TMP,1); XPush_r(TMP); XCall_m_real(val_print); stack_pop(Esp,1); XPop_r(ACC); END_BUFFER; } # endif i++; jit_opcode(ctx,op,(int)m->code[i]); # ifdef _DEBUG { int bytes = POS() - curpos; if( bytes > MAX_OP_SIZE ) ERROR; } # endif i += parameter_table[op]; } // FINALIZE { int csize = POS(); char *rbuf = alloc_jit_mem(csize); memcpy(rbuf,ctx->baseptr,csize); tmp_free(ctx->baseptr); ctx->baseptr = rbuf; ctx->buf.p = rbuf + csize; ctx->size = csize; # ifdef USE_MMAP m->jit_gc = alloc_abstract(NULL,rbuf); val_gc(m->jit_gc,free_jit_abstract); # endif # ifdef JIT_DEBUG printf("Jit size = %d ( x%.1f )\n",csize,csize * 1.0 / ((m->codesize + 1) * 4)); # endif jit_finalize_context(ctx); } // UPDATE GLOBALS { for(i=0;inglobals;i++) { vfunction *f = (vfunction*)m->globals[i]; if( !val_is_int(f) && val_tag(f) == VAL_FUNCTION && f->module == m ) { int pc = (int)((int_val*)f->addr - m->code); f->t = VAL_JITFUN; f->addr = (char*)ctx->baseptr + ctx->pos[pc]; } } } m->jit = ctx->baseptr; tmp_free(ctx->pos); } #else // JIT_ENABLE char *jit_boot_seq = NULL; char *jit_handle_trap = (char*)&jit_boot_seq; void neko_init_jit() { } void neko_free_jit() { } int neko_can_jit() { return 0; } void neko_module_jit( neko_module *m ) { } #endif /* ************************************************************************ */