From 681da49618d6153361f0bf2162abff818d0827f0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 15 May 2020 13:20:21 +0300 Subject: [PATCH 1/3] JIT refactoring to allow run-time changes of JIT options (triggers, optimization_level, debug flags, etc) --- ext/opcache/ZendAccelerator.c | 26 +- ext/opcache/ZendAccelerator.h | 9 - ext/opcache/jit/zend_jit.c | 472 +++++++++++++------------- ext/opcache/jit/zend_jit.h | 96 ++++-- ext/opcache/jit/zend_jit_disasm_x86.c | 27 +- ext/opcache/jit/zend_jit_gdb.c | 2 +- ext/opcache/jit/zend_jit_internal.h | 51 +-- ext/opcache/jit/zend_jit_trace.c | 71 ++-- ext/opcache/jit/zend_jit_vm_helpers.c | 16 +- ext/opcache/jit/zend_jit_x86.dasc | 165 +++++---- ext/opcache/zend_accelerator_module.c | 46 ++- ext/opcache/zend_file_cache.c | 6 +- ext/opcache/zend_persist.c | 9 +- 13 files changed, 528 insertions(+), 468 deletions(-) diff --git a/ext/opcache/ZendAccelerator.c b/ext/opcache/ZendAccelerator.c index 0642ca2c6d4a4..2b858bc228767 100644 --- a/ext/opcache/ZendAccelerator.c +++ b/ext/opcache/ZendAccelerator.c @@ -2854,6 +2854,10 @@ static int accel_startup(zend_extension *extension) accel_globals_ctor(&accel_globals); #endif +#ifdef HAVE_JIT + zend_jit_init(); +#endif + #ifdef ZEND_WIN32 # if !defined(__has_feature) || !__has_feature(address_sanitizer) _setmaxstdio(2048); /* The default configuration is limited to 512 stdio files */ @@ -2937,8 +2941,7 @@ static int accel_post_startup(void) size_t jit_size = 0; zend_bool reattached = 0; - if (ZCG(accel_directives).jit && - ZCG(accel_directives).jit_buffer_size) { + if (JIT_G(enabled) && JIT_G(buffer_size)) { size_t page_size; # ifdef _WIN32 @@ -2952,12 +2955,9 @@ static int accel_post_startup(void) zend_accel_error(ACCEL_LOG_FATAL, "Failure to initialize shared memory structures - can't get page size."); abort(); } - jit_size = ZCG(accel_directives).jit_buffer_size; + jit_size = JIT_G(buffer_size); jit_size = ZEND_MM_ALIGNED_SIZE_EX(jit_size, page_size); shm_size += jit_size; - } else { - ZCG(accel_directives).jit = 0; - ZCG(accel_directives).jit_buffer_size = 0; } switch (zend_shared_alloc_startup(shm_size, jit_size)) { @@ -3010,13 +3010,13 @@ static int accel_post_startup(void) zend_shared_alloc_lock(); #ifdef HAVE_JIT - if (ZCG(accel_directives).jit && - ZCG(accel_directives).jit_buffer_size && - ZSMMG(reserved) && - zend_jit_startup(ZCG(accel_directives).jit, ZSMMG(reserved), jit_size, reattached) == SUCCESS) { - ZCG(jit_enabled) = 1; - } else { - ZCG(jit_enabled) = 0; + if (JIT_G(enabled)) { + if (JIT_G(buffer_size) == 0 + || !ZSMMG(reserved) + || zend_jit_startup(ZSMMG(reserved), jit_size, reattached) != SUCCESS) { + JIT_G(enabled) = 0; + JIT_G(on) = 0; + } } #endif zend_shared_alloc_save_state(); diff --git a/ext/opcache/ZendAccelerator.h b/ext/opcache/ZendAccelerator.h index bb884d9cc9214..7dd9267e8fe46 100644 --- a/ext/opcache/ZendAccelerator.h +++ b/ext/opcache/ZendAccelerator.h @@ -191,12 +191,6 @@ typedef struct _zend_accel_directives { #ifdef ZEND_WIN32 char *cache_id; #endif -#ifdef HAVE_JIT - zend_long jit; - zend_long jit_buffer_size; - zend_long jit_debug; - zend_long jit_bisect_limit; -#endif } zend_accel_directives; typedef struct _zend_accel_globals { @@ -227,9 +221,6 @@ typedef struct _zend_accel_globals { void *arena_mem; zend_persistent_script *current_persistent_script; zend_bool is_immutable_class; -#ifdef HAVE_JIT - zend_bool jit_enabled; -#endif /* cache to save hash lookup on the same INCLUDE opcode */ const zend_op *cache_opline; zend_persistent_script *cache_persistent_script; diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index bcc1fa863c1d4..db0f9d3fb1520 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -22,6 +22,7 @@ #include "Zend/zend_vm.h" #include "Zend/zend_exceptions.h" #include "Zend/zend_constants.h" +#include "Zend/zend_ini.h" #include "zend_smart_str.h" #include "jit/zend_jit.h" @@ -82,11 +83,6 @@ typedef struct _zend_jit_stub { #define JIT_STUB(name) \ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} -static zend_uchar zend_jit_level = 0; -static zend_uchar zend_jit_trigger = 0; -static zend_uchar zend_jit_reg_alloc = 0; -static zend_uchar zend_jit_cpu_flags = 0; - zend_ulong zend_jit_profile_counter = 0; int zend_jit_profile_counter_rid = -1; @@ -105,9 +101,11 @@ static zend_long jit_bisect_pos = 0; static const void *zend_jit_runtime_jit_handler = NULL; static const void *zend_jit_profile_jit_handler = NULL; -static const void *zend_jit_func_counter_handler = NULL; -static const void *zend_jit_ret_counter_handler = NULL; -static const void *zend_jit_loop_counter_handler = NULL; +static const void *zend_jit_func_hot_counter_handler = NULL; +static const void *zend_jit_loop_hot_counter_handler = NULL; +static const void *zend_jit_func_trace_counter_handler = NULL; +static const void *zend_jit_ret_trace_counter_handler = NULL; +static const void *zend_jit_loop_trace_counter_handler = NULL; static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa); static void ZEND_FASTCALL zend_runtime_jit(void); @@ -199,9 +197,11 @@ ZEND_EXT_API void zend_jit_status(zval *ret) { zval stats; array_init(&stats); - add_assoc_long(&stats, "level", zend_jit_level); - add_assoc_long(&stats, "trigger", zend_jit_trigger); - add_assoc_long(&stats, "reg-alloc", zend_jit_reg_alloc); + add_assoc_bool(&stats, "enabled", JIT_G(enabled)); + add_assoc_bool(&stats, "on", JIT_G(on)); + add_assoc_long(&stats, "kind", JIT_G(trigger)); + add_assoc_long(&stats, "opt_level", JIT_G(opt_level)); + add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags)); if (dasm_buf) { add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); @@ -349,14 +349,14 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) if (!name) { - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { str = zend_jit_func_name(op_array); if (str) { name = ZSTR_VAL(str); } } #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); zend_jit_disasm( name, @@ -367,9 +367,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, size); } } else { - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) { zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); - if (trace_num || (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) != 0) { + if (trace_num || (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) != 0) { zend_jit_disasm( name, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, @@ -384,7 +384,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #endif #ifdef HAVE_GDB - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) { if (name) { zend_jit_gdb_register( name, @@ -396,7 +396,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #endif #ifdef HAVE_OPROFILE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) { zend_jit_oprofile_register( name, entry, @@ -405,13 +405,13 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #endif #ifdef HAVE_PERFTOOLS - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { if (name) { zend_jit_perf_map_register( name, entry, size); - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { zend_jit_perf_jitdump_register( name, entry, @@ -422,7 +422,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, #endif #ifdef HAVE_VTUNE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_VTUNE) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) { if (name) { zend_jit_vtune_register( name, @@ -656,7 +656,7 @@ static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script } #endif - if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) + if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) @@ -683,7 +683,7 @@ static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa) { - if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) + if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) && ssa->cfg.blocks && op_array->last_try_catch == 0 && !(op_array->fn_flags & ZEND_ACC_GENERATOR) @@ -1221,7 +1221,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss } - if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { + if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Register hinting (a cheap way for register coalescing) */ for (i = 0; i < ssa->vars_count; i++) { if (intervals[i]) { @@ -1787,7 +1787,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } if (list) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); ival = list; while (ival) { @@ -1815,7 +1815,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array ival = next; } - if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { + if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) { /* Naive SSA resolution */ for (i = 0; i < ssa->vars_count; i++) { if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) { @@ -1929,7 +1929,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array } } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); for (i = 0; i < ssa->vars_count; i++) { ival = intervals[i]; @@ -1970,10 +1970,10 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; zend_class_entry *ce; - if (ZCG(accel_directives).jit_bisect_limit) { + if (JIT_G(bisect_limit)) { jit_bisect_pos++; - if (jit_bisect_pos >= ZCG(accel_directives).jit_bisect_limit) { - if (jit_bisect_pos == ZCG(accel_directives).jit_bisect_limit) { + if (jit_bisect_pos >= JIT_G(bisect_limit)) { + if (jit_bisect_pos == JIT_G(bisect_limit)) { fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n", op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", op_array->scope ? "::" : "", @@ -1984,7 +1984,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } } - if (zend_jit_reg_alloc) { + if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { checkpoint = zend_arena_checkpoint(CG(arena)); ra = zend_jit_allocate_registers(op_array, ssa); } @@ -2017,7 +2017,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) { if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { /* pass */ - } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && + } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE && ssa->cfg.blocks[b].len == 1 && (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) && op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) { @@ -2079,13 +2079,13 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } else { if (recv_emitted) { zend_jit_jmp(&dasm_state, b); - } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && + } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE && ssa->cfg.blocks[b].len == 1 && (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) { /* don't generate code for BB with single opcode */ dasm_free(&dasm_state); - if (zend_jit_reg_alloc) { + if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { zend_arena_release(&CG(arena), checkpoint); } return SUCCESS; @@ -2094,13 +2094,13 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_prologue(&dasm_state); recv_emitted = 1; } - } else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && + } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE && ssa->cfg.blocks[b].len == 1 && (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) { /* don't generate code for BB with single opcode */ dasm_free(&dasm_state); - if (zend_jit_reg_alloc) { + if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { zend_arena_release(&CG(arena), checkpoint); } return SUCCESS; @@ -2113,7 +2113,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op is_terminated = 0; zend_jit_label(&dasm_state, b); - if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { + if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) && ssa->cfg.blocks[b].start != 0 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP @@ -2145,7 +2145,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (!ssa->cfg.blocks[b].len) { continue; } - if ((zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) && ra) { + if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) { zend_ssa_phi *phi = ssa->blocks[b].phis; while (phi) { @@ -2185,7 +2185,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op call_level++; } - if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -2892,7 +2892,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op case ZEND_SWITCH_STRING: break; case ZEND_JMP: - if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { + if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { const zend_op *target = OP_JMP_ADDR(opline, opline->op1); if (!zend_jit_set_ip(&dasm_state, target)) { @@ -2972,7 +2972,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { zend_class_entry *ce = NULL; - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) { if (ssa->ops && ssa->var_info) { zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; if (res_ssa->ce && !res_ssa->is_instanceof) { @@ -2995,7 +2995,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op const zend_op *next_opline = opline + 1; zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]); - if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { + if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { zend_jit_call(&dasm_state, next_opline, b + 1); is_terminated = 1; } else { @@ -3027,7 +3027,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } dasm_free(&dasm_state); - if (zend_jit_reg_alloc) { + if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { zend_arena_release(&CG(arena), checkpoint); } return SUCCESS; @@ -3036,7 +3036,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op if (dasm_state) { dasm_free(&dasm_state); } - if (zend_jit_reg_alloc) { + if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) { zend_arena_release(&CG(arena), checkpoint); } return FAILURE; @@ -3072,7 +3072,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (zend_jit_collect_calls(op_array, script) != SUCCESS) { ZEND_SET_FUNC_INFO(op_array, NULL); goto jit_failure; @@ -3088,7 +3088,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa); } @@ -3096,7 +3096,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons goto jit_failure; } - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { ZEND_SET_FUNC_INFO(op_array, NULL); } @@ -3212,8 +3212,8 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) zend_cfg cfg; uint32_t i; - ZEND_ASSERT(zend_jit_func_counter_handler != NULL); - ZEND_ASSERT(zend_jit_loop_counter_handler != NULL); + ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL); + ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL); if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { return FAILURE; @@ -3240,13 +3240,13 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array) } } - opline->handler = (const void*)zend_jit_func_counter_handler; + opline->handler = (const void*)zend_jit_func_hot_counter_handler; for (i = 0; i < cfg.blocks_count; i++) { if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) && (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) { op_array->opcodes[cfg.blocks[i].start].handler = - (const void*)zend_jit_loop_counter_handler; + (const void*)zend_jit_loop_hot_counter_handler; } } @@ -3279,7 +3279,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) return FAILURE; } - if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { + if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) { zend_op *opline = op_array->opcodes; /* Set run-time JIT handler */ @@ -3293,7 +3293,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) opline->handler = (const void*)zend_jit_runtime_jit_handler; return SUCCESS; - } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) { zend_op *opline = op_array->opcodes; ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); @@ -3308,13 +3308,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script) } return SUCCESS; - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { return zend_jit_setup_hot_counters(op_array); - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { return zend_jit_setup_hot_trace_counters(op_array); - } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) { return zend_real_jit_func(op_array, script, NULL); - } else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) { if (zend_needs_manual_jit(op_array)) { return zend_real_jit_func(op_array, script, NULL); } else { @@ -3345,20 +3345,20 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) zend_analyze_call_graph(&CG(arena), script, &call_graph); - if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC || - zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST || - zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS || - zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC || + JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST || + JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS || + JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { for (i = 0; i < call_graph.op_arrays_count; i++) { ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { goto jit_failure; } } - } else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD || - zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD || + JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) { - if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { + if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) { int do_jit = 0; for (i = 0; i < call_graph.op_arrays_count; i++) { if (zend_needs_manual_jit(call_graph.op_arrays[i])) { @@ -3391,7 +3391,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } for (i = 0; i < call_graph.op_arrays_count; i++) { - if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT && !zend_needs_manual_jit(call_graph.op_arrays[i])) { continue; } @@ -3404,9 +3404,9 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { for (i = 0; i < call_graph.op_arrays_count; i++) { - if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT && !zend_needs_manual_jit(call_graph.op_arrays[i])) { continue; } @@ -3418,7 +3418,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) } for (i = 0; i < call_graph.op_arrays_count; i++) { - if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && + if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT && !zend_needs_manual_jit(call_graph.op_arrays[i])) { continue; } @@ -3451,13 +3451,13 @@ ZEND_EXT_API int zend_jit_script(zend_script *script) ZEND_EXT_API void zend_jit_unprotect(void) { #ifdef HAVE_MPROTECT - if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } } #elif _WIN32 - if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { DWORD old; if (!VirtualProtect(dasm_buf, dasm_size, PAGE_READWRITE, &old)) { @@ -3470,13 +3470,13 @@ ZEND_EXT_API void zend_jit_unprotect(void) ZEND_EXT_API void zend_jit_protect(void) { #ifdef HAVE_MPROTECT - if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } } #elif _WIN32 - if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { + if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { DWORD old; if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) { @@ -3505,83 +3505,21 @@ static int zend_jit_make_stubs(void) } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { - return 0; - } - zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit", 0); - if (!zend_jit_runtime_jit_handler) { - return 0; - } - } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) { - return 0; - } - zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit", 0); - if (!zend_jit_profile_jit_handler) { - return 0; - } - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) { - return 0; - } - zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0); - if (!zend_jit_func_counter_handler) { - return 0; - } - - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) { - return 0; - } - zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0); - if (!zend_jit_loop_counter_handler) { - return 0; - } - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_func_trace_counter_stub(&dasm_state)) { - return 0; - } - zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0); - if (!zend_jit_func_counter_handler) { - return 0; - } - - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_ret_trace_counter_stub(&dasm_state)) { - return 0; - } - zend_jit_ret_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_ret_counter", 0); - if (!zend_jit_ret_counter_handler) { - return 0; - } - - dasm_setup(&dasm_state, dasm_actions); - if (!zend_jit_hybrid_loop_trace_counter_stub(&dasm_state)) { - return 0; - } - zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0); - if (!zend_jit_loop_counter_handler) { - return 0; - } - } + zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit]; + zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit]; + zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter]; + zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter]; + zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter]; + zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter]; + zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter]; } else { - if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { - zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; - } else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { - zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; - zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper; - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { - zend_jit_func_counter_handler = (const void*)zend_jit_func_trace_helper; - zend_jit_ret_counter_handler = (const void*)zend_jit_ret_trace_helper; - zend_jit_loop_counter_handler = (const void*)zend_jit_loop_trace_helper; - } + zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; + zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; + zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper; + zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper; + zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper; + zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper; + zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper; } dasm_free(&dasm_state); @@ -3594,20 +3532,120 @@ static void zend_jit_globals_ctor(zend_jit_globals *jit_globals) zend_jit_trace_init_caches(); } -ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached) +static int zend_jit_parse_config_num(zend_long jit) { - int ret; + if (jit == 0) { + JIT_G(on) = 0; + return SUCCESS; + } + + if (jit < 0) return FAILURE; + + if (jit % 10 == 0 || jit % 10 > 5) return FAILURE; + JIT_G(opt_level) = jit % 10; + + jit /= 10; + if (jit % 10 > 5) return FAILURE; + JIT_G(trigger) = jit % 10; + + jit /= 10; + if (jit % 10 > 2) return FAILURE; + JIT_G(opt_flags) = jit % 10; + + jit /= 10; + if (jit % 10 > 1) return FAILURE; + JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0); + + if (jit / 10 != 0) return FAILURE; + + JIT_G(on) = 1; + + return SUCCESS; +} + +ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage) +{ + zend_ulong num; + + if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) { + if (stage == ZEND_INI_STAGE_RUNTIME) { + zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)"); + } + return FAILURE; + } + + if (ZSTR_LEN(jit) == 0 + || zend_string_equals_literal_ci(jit, "disable") + || zend_string_equals_literal_ci(jit, "disabled")) { + JIT_G(enabled) = 0; + JIT_G(on) = 0; + return SUCCESS; + } else if (zend_string_equals_literal_ci(jit, "0") + || zend_string_equals_literal_ci(jit, "off") + || zend_string_equals_literal_ci(jit, "no") + || zend_string_equals_literal_ci(jit, "false")) { + JIT_G(enabled) = 1; + JIT_G(on) = 0; + return SUCCESS; + } else if (zend_string_equals_literal_ci(jit, "1") + || zend_string_equals_literal_ci(jit, "on") + || zend_string_equals_literal_ci(jit, "yes") + || zend_string_equals_literal_ci(jit, "true")) { + JIT_G(enabled) = 1; + JIT_G(on) = 1; + JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT; + JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD; + JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX; + return SUCCESS; + } else if (ZEND_HANDLE_NUMERIC(jit, num)) { + if (zend_jit_parse_config_num((zend_long)num) != SUCCESS) { + goto failure; + } + JIT_G(enabled) = 1; + return SUCCESS; + } + +failure: + zend_error(E_WARNING, "Invalid opcache.jit setting"); + JIT_G(enabled) = 0; + JIT_G(on) = 0; + return FAILURE; +} + +ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage) +{ + if (stage != ZEND_INI_STAGE_STARTUP) { + if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) { + if (stage == ZEND_INI_STAGE_RUNTIME) { + zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup"); + } + return FAILURE; + } +#ifdef HAVE_DISASM + if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { + if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) { + // TODO: error reporting and cleanup ??? + return FAILURE; + } + // TODO: symbols for JIT-ed code compiled before are missing ??? + } +#endif + } + return SUCCESS; +} +ZEND_EXT_API void zend_jit_init(void) +{ #ifdef ZTS zend_jit_globals_id = ts_allocate_id(&zend_jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL); #else zend_jit_globals_ctor(&jit_globals); #endif +} - zend_jit_level = ZEND_JIT_LEVEL(jit); - zend_jit_trigger = ZEND_JIT_TRIGGER(jit); - zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); - zend_jit_cpu_flags = ZEND_JIT_CPU_FLAGS(jit); +ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, zend_bool reattached) +{ + int ret; zend_jit_vm_kind = zend_vm_kind(); if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && @@ -3623,16 +3661,14 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo return FAILURE; } - if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(); - } + zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(); #ifdef HAVE_GDB zend_jit_gdb_init(); #endif #ifdef HAVE_OPROFILE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) { if (!zend_jit_oprofile_startup()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -3644,7 +3680,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo dasm_size = size; #ifdef HAVE_MPROTECT - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) { fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); } @@ -3654,7 +3690,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo } } #elif _WIN32 - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { DWORD old; if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) { @@ -3681,7 +3717,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo } #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { if (!zend_jit_disasm_init()) { // TODO: error reporting and cleanup ??? return FAILURE; @@ -3690,7 +3726,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo #endif #ifdef HAVE_PERFTOOLS - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { zend_jit_perf_jitdump_open(); } #endif @@ -3714,10 +3750,8 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo #endif } - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { - if (zend_jit_trace_startup() != SUCCESS) { - return FAILURE; - } + if (zend_jit_trace_startup() != SUCCESS) { + return FAILURE; } return SUCCESS; @@ -3726,113 +3760,73 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo ZEND_EXT_API void zend_jit_shutdown(void) { #ifdef HAVE_OPROFILE - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) { zend_jit_oprofile_shutdown(); } #endif #ifdef HAVE_GDB - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) { zend_jit_gdb_unregister(); } #endif #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { - zend_jit_disasm_shutdown(); - } + zend_jit_disasm_shutdown(); #endif #ifdef HAVE_PERFTOOLS - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { zend_jit_perf_jitdump_close(); } #endif } -ZEND_EXT_API void zend_jit_activate(void) +static void zend_jit_reset_counters(void) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { - int i; - - for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { - zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT; - } - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { - int i; - - for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { - zend_jit_hot_counters[i] = ZEND_JIT_TRACE_COUNTER_INIT; - } + int i; - zend_jit_trace_reset_caches(); + for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { + zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT; } } -ZEND_EXT_API void zend_jit_deactivate(void) +ZEND_EXT_API void zend_jit_activate(void) { - if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { - if (!zend_jit_profile_counter) { - return; - } else { - zend_class_entry *ce; - - zend_shared_alloc_lock(); - SHM_UNPROTECT(); - zend_jit_unprotect(); - - zend_jit_check_funcs(EG(function_table), 0); - ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) { - if (ce->type == ZEND_INTERNAL_CLASS) { - break; - } - zend_jit_check_funcs(&ce->function_table, 1); - } ZEND_HASH_FOREACH_END(); - - zend_jit_protect(); - SHM_PROTECT(); - zend_shared_alloc_unlock(); - - zend_jit_profile_counter = 0; + zend_jit_profile_counter = 0; + if (JIT_G(on)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) { + zend_jit_reset_counters(); + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + zend_jit_reset_counters(); + zend_jit_trace_reset_caches(); } } } -#else /* HAVE_JIT */ - -ZEND_EXT_API int zend_jit_op_array(const zend_op_array *op_array, zend_script *script) -{ - return FAILURE; -} - -ZEND_EXT_API int zend_jit_script(zend_script *script) -{ - return FAILURE; -} - -ZEND_EXT_API void zend_jit_unprotect(void) -{ -} - -ZEND_EXT_API void zend_jit_protect(void) +ZEND_EXT_API void zend_jit_deactivate(void) { -} + if (zend_jit_profile_counter) { + zend_class_entry *ce; -ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size) -{ - return FAILURE; -} + zend_shared_alloc_lock(); + SHM_UNPROTECT(); + zend_jit_unprotect(); -ZEND_EXT_API void zend_jit_shutdown(void) -{ -} + zend_jit_check_funcs(EG(function_table), 0); + ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) { + if (ce->type == ZEND_INTERNAL_CLASS) { + break; + } + zend_jit_check_funcs(&ce->function_table, 1); + } ZEND_HASH_FOREACH_END(); -ZEND_EXT_API void zend_jit_activate(void) -{ -} + zend_jit_protect(); + SHM_PROTECT(); + zend_shared_alloc_unlock(); -ZEND_EXT_API void zend_jit_deactivate(void) -{ + zend_jit_profile_counter = 0; + } } #endif /* HAVE_JIT */ diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 3f8196b309836..82bf417d5f0f3 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -26,8 +26,6 @@ #define ZEND_JIT_LEVEL_OPT_FUNCS 4 /* optimized JIT based on Type-Inference and call-tree */ #define ZEND_JIT_LEVEL_OPT_SCRIPT 5 /* optimized JIT based on Type-Inference and inner-procedure analysis */ -#define ZEND_JIT_LEVEL(n) ((n) % 10) - #define ZEND_JIT_ON_SCRIPT_LOAD 0 #define ZEND_JIT_ON_FIRST_EXEC 1 #define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first request functions */ @@ -35,20 +33,17 @@ #define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */ #define ZEND_JIT_ON_HOT_TRACE 5 /* trace functions after N calls or loop iterations */ -#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) - -#define ZEND_JIT_REG_ALLOC_NONE 0 /* no register allocation */ -#define ZEND_JIT_REG_ALLOC_ENABLED 1 /* local linear scan register allocation */ -#define ZEND_JIT_REG_ALLOC_GLOBAL 2 /* global linear scan register allocation */ - -#define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) - -#define ZEND_JIT_CPU_AVX 1 /* use AVX instructions, if available */ - -#define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10) +#define ZEND_JIT_REG_ALLOC_LOCAL (1<<0) /* local linear scan register allocation */ +#define ZEND_JIT_REG_ALLOC_GLOBAL (1<<1) /* global linear scan register allocation */ +#define ZEND_JIT_CPU_AVX (1<<2) /* use AVX instructions, if available */ +//#define ZEND_JIT_LEVEL(n) ((n) % 10) +//#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) +//#define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) +//#define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10) -#define ZEND_JIT_DEFAULT "1205" +#define ZEND_JIT_DEFAULT_OPTIONS "1205" +#define ZEND_JIT_DEFAULT_BUFFER_SIZE "0" /* Makes profile based JIT (opcache.jit=2*) to generate code only for most @@ -57,12 +52,13 @@ */ #define ZEND_JIT_PROF_THRESHOLD 0.005 -/* Hot Counters based JIT parameters. +/* Hot/Trace Counters based JIT parameters. * TODO: this setting should be configurable */ -#define ZEND_JIT_HOT_FUNC_COST 1 -#define ZEND_JIT_HOT_LOOP_COST 2 -#define ZEND_JIT_HOT_COUNTER_INIT 127 +#define ZEND_JIT_COUNTER_FUNC_COST 1 +#define ZEND_JIT_COUNTER_RET_COST 15 +#define ZEND_JIT_COUNTER_LOOP_COST 2 +#define ZEND_JIT_COUNTER_INIT 127 #define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_SSA (1<<1) @@ -86,11 +82,73 @@ #define ZEND_JIT_DEBUG_TRACE_TSSA (1<<19) #define ZEND_JIT_DEBUG_TRACE_EXIT_INFO (1<<20) +#define ZEND_JIT_DEBUG_PERSISTENT 0x1f0 /* profile and debbuger flags can't be changed at run-time */ + +#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */ +#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */ +#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */ +#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */ +#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */ + +#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */ +#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */ +#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */ +#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */ +#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */ + +#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */ +#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */ + +#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attemts to record/compile a root trace */ +#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attemts to record/compile a side trace */ + +#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */ + +typedef struct _zend_jit_trace_rec zend_jit_trace_rec; +typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame; +typedef struct _sym_node zend_sym_node; + +typedef struct _zend_jit_globals { + zend_bool enabled; + zend_bool on; + uint8_t trigger; + uint8_t opt_level; + uint32_t opt_flags; + + const char *options; + zend_long buffer_size; + zend_long debug; + zend_long bisect_limit; + + zend_sym_node *symbols; /* symbols for disassembler */ + + zend_jit_trace_rec *current_trace; + zend_jit_trace_stack_frame *current_frame; + + const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; + uint32_t bad_root_slot; + + uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS]; +} zend_jit_globals; + +#ifdef ZTS +# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v) +extern int zend_jit_globals_id; +#else +# define JIT_G(v) (jit_globals.v) +extern zend_jit_globals jit_globals; +#endif + ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_EXT_API int zend_jit_script(zend_script *script); ZEND_EXT_API void zend_jit_unprotect(void); ZEND_EXT_API void zend_jit_protect(void); -ZEND_EXT_API int zend_jit_startup(zend_long jit, void *jit_buffer, size_t size, zend_bool reattached); +ZEND_EXT_API void zend_jit_init(void); +ZEND_EXT_API int zend_jit_config(zend_string *jit_options, int stage); +ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage); +ZEND_EXT_API int zend_jit_startup(void *jit_buffer, size_t size, zend_bool reattached); ZEND_EXT_API void zend_jit_shutdown(void); ZEND_EXT_API void zend_jit_activate(void); ZEND_EXT_API void zend_jit_deactivate(void); diff --git a/ext/opcache/jit/zend_jit_disasm_x86.c b/ext/opcache/jit/zend_jit_disasm_x86.c index 44b23e1a19df8..81addae041603 100644 --- a/ext/opcache/jit/zend_jit_disasm_x86.c +++ b/ext/opcache/jit/zend_jit_disasm_x86.c @@ -50,16 +50,14 @@ static void zend_jit_disasm_add_symbol(const char *name, static struct ud ud; -typedef struct _sym_node { +struct _sym_node { uint64_t addr; uint64_t end; struct _sym_node *parent; struct _sym_node *child[2]; unsigned char info; char name[1]; -} zend_sym_node; - -static zend_sym_node *symbols = NULL; +}; static void zend_syms_rotateleft(zend_sym_node *p) { zend_sym_node *r = p->child[1]; @@ -69,7 +67,7 @@ static void zend_syms_rotateleft(zend_sym_node *p) { } r->parent = p->parent; if (p->parent == NULL) { - symbols = r; + JIT_G(symbols) = r; } else if (p->parent->child[0] == p) { p->parent->child[0] = r; } else { @@ -87,7 +85,7 @@ static void zend_syms_rotateright(zend_sym_node *p) { } l->parent = p->parent; if (p->parent == NULL) { - symbols = l; + JIT_G(symbols) = l; } else if (p->parent->child[1] == p) { p->parent->child[1] = l; } else { @@ -113,8 +111,8 @@ static void zend_jit_disasm_add_symbol(const char *name, memcpy((char*)&sym->name, name, len + 1); sym->parent = sym->child[0] = sym->child[1] = NULL; sym->info = 1; - if (symbols) { - zend_sym_node *node = symbols; + if (JIT_G(symbols)) { + zend_sym_node *node = JIT_G(symbols); /* insert it into rbtree */ do { @@ -147,7 +145,7 @@ static void zend_jit_disasm_add_symbol(const char *name, } while (1); /* fix rbtree after instering */ - while (sym && sym != symbols && sym->parent->info == 1) { + while (sym && sym != JIT_G(symbols) && sym->parent->info == 1) { if (sym->parent == sym->parent->parent->child[0]) { node = sym->parent->parent->child[1]; if (node && node->info == 1) { @@ -183,9 +181,9 @@ static void zend_jit_disasm_add_symbol(const char *name, } } } else { - symbols = sym; + JIT_G(symbols) = sym; } - symbols->info = 0; + JIT_G(symbols)->info = 0; } static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) { @@ -201,7 +199,7 @@ static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) { static const char* zend_jit_disasm_find_symbol(uint64_t addr, int64_t *offset) { - zend_sym_node *node = symbols; + zend_sym_node *node = JIT_G(symbols); while (node) { if (addr < node->addr) { node = node->child[0]; @@ -523,5 +521,8 @@ static int zend_jit_disasm_init(void) static void zend_jit_disasm_shutdown(void) { - zend_jit_disasm_destroy_symbols(symbols); + if (JIT_G(symbols)) { + zend_jit_disasm_destroy_symbols(JIT_G(symbols)); + JIT_G(symbols) = NULL; + } } diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 2db7246372b48..5f9847327caed 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -485,7 +485,7 @@ static void zend_jit_gdb_init(void) /* This might enable registration of all JIT-ed code, but unfortunately, * in case of many functions, this takes enormous time. */ if (zend_gdb_present()) { - ZCG(accel_directives).jit_debug |= ZEND_JIT_DEBUG_GDB; + JIT_G(debug) |= ZEND_JIT_DEBUG_GDB; } #endif } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index b139c4071e703..178cfb3f85ccb 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -132,31 +132,6 @@ int ZEND_FASTCALL zend_jit_check_constant(const zval *key); #define zend_jit_opline_hash(opline) \ zend_jit_hash(opline) -#define ZEND_JIT_TRACE_FUNC_COST (1*250) -#define ZEND_JIT_TRACE_RET_COST (15*250) -#define ZEND_JIT_TRACE_LOOP_COST (2*250) -#define ZEND_JIT_TRACE_COUNTER_INIT (127*250) - -#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */ -#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */ -#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */ -#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */ -#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */ - -#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */ -#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */ -#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */ -#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */ -#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */ - -#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */ -#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */ - -#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attemts to record/compile a root trace */ -#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attemts to record/compile a side trace */ - -#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */ - #define ZEND_JIT_TRACE_STOP(_) \ _(LOOP, "loop") \ _(RECURSIVE_CALL, "recursive call") \ @@ -275,7 +250,7 @@ typedef enum _zend_jit_trace_op { #define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \ (_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT) -typedef struct _zend_jit_trace_rec { +struct _zend_jit_trace_rec { union { struct { ZEND_ENDIAN_LOHI( uint8_t op, /* zend_jit_trace_op */ @@ -304,7 +279,7 @@ typedef struct _zend_jit_trace_rec { const zend_op *opline; const zend_class_entry *ce; }; -} zend_jit_trace_rec; +}; #define ZEND_JIT_TRACE_START_REC_SIZE 2 @@ -370,8 +345,6 @@ typedef struct _zend_jit_trace_info { //uint32_t loop_offset; } zend_jit_trace_info; -typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame; - struct _zend_jit_trace_stack_frame { zend_jit_trace_stack_frame *call; zend_jit_trace_stack_frame *prev; @@ -446,26 +419,6 @@ struct _zend_jit_trace_stack_frame { (frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \ } while (0) -typedef struct _zend_jit_globals { - zend_jit_trace_rec *current_trace; - zend_jit_trace_stack_frame *current_frame; - - const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; - uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; - uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS]; - uint32_t bad_root_slot; - - uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS]; -} zend_jit_globals; - -#ifdef ZTS -# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v) -extern int zend_jit_globals_id; -#else -# define JIT_G(v) (jit_globals.v) -extern zend_jit_globals jit_globals; -#endif - ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS); diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index cb766b907f7dc..6a18fed00943a 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -83,7 +83,7 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n) dasm_free(&dasm_state); #ifdef HAVE_DISASM - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { uint32_t i; for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { @@ -443,13 +443,13 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc jit_extension->func_info.return_value_used = -1; ssa = &jit_extension->func_info.ssa; - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) { do { if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) { break; } - if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) { break; } @@ -463,7 +463,7 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc break; } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa); } return ssa; @@ -1722,7 +1722,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } - if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_TSSA)) { + if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) { if (parent_trace) { fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s() %s:%d\n", ZEND_JIT_TRACE_NUM, @@ -2281,7 +2281,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace } if (list) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM); ival = list; while (ival) { @@ -2381,7 +2381,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace return NULL; } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM); for (i = 0; i < ssa->vars_count; i++) { ival = intervals[i]; @@ -2443,7 +2443,7 @@ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offse ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; } ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; - next_opline->handler = (const void*)zend_jit_ret_counter_handler; + next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler; } } @@ -2487,7 +2487,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } /* Register allocation */ - if (zend_jit_reg_alloc && zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) + && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num); } @@ -2723,7 +2724,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par frame->call_level++; } - if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { switch (opline->opcode) { case ZEND_PRE_INC: case ZEND_PRE_DEC: @@ -4276,7 +4277,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace zend_shared_alloc_unlock(); - if ((ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 && ret == ZEND_JIT_TRACE_STOP_COMPILED && t->exit_count > 0) { zend_jit_dump_exit_info(t); @@ -4602,7 +4603,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const return 0; } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n", trace_num, zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), @@ -4622,7 +4623,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { /* op_array may be already deallocated */ - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { fprintf(stderr, "---- TRACE %d abort (%s)\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4630,12 +4631,12 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const goto blacklist; } - if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { + if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { zend_jit_dump_trace(trace_buffer, NULL); } if (ZEND_JIT_TRACE_STOP_OK(stop)) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) { if (stop == ZEND_JIT_TRACE_STOP_LINK) { uint32_t idx = trace_buffer[1].last; uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler); @@ -4650,7 +4651,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const } stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset); if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) { fprintf(stderr, "---- TRACE %d %s\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4660,7 +4661,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const } } else { abort: - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { fprintf(stderr, "---- TRACE %d abort (%s)\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4668,7 +4669,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const blacklist: if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { fprintf(stderr, "---- TRACE %d blacklisted\n", trace_num); } @@ -4681,7 +4682,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const } } - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { fprintf(stderr, "\n"); } @@ -4847,7 +4848,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace zend_shared_alloc_unlock(); - if ((ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 + if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 && ret == ZEND_JIT_TRACE_STOP_COMPILED && t->exit_count > 0) { zend_jit_dump_exit_info(t); @@ -4869,7 +4870,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 return 0; } - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s() %s:%d\n", trace_num, parent_num, exit_num, EX(func)->op_array.function_name ? @@ -4892,7 +4893,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { /* op_array may be already deallocated */ - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { fprintf(stderr, "---- TRACE %d abort (%s)\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4900,12 +4901,12 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 goto blacklist; } - if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { + if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { zend_jit_dump_trace(trace_buffer, NULL); } if (ZEND_JIT_TRACE_STOP_OK(stop)) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) { if (stop == ZEND_JIT_TRACE_STOP_LINK) { uint32_t idx = trace_buffer[1].last; uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);; @@ -4929,7 +4930,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset); } if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) { fprintf(stderr, "---- TRACE %d %s\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4939,7 +4940,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 } } else { abort: - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { fprintf(stderr, "---- TRACE %d abort (%s)\n", trace_num, zend_jit_trace_stop_description[stop]); @@ -4948,14 +4949,14 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) || zend_jit_trace_exit_is_bad(parent_num, exit_num)) { zend_jit_blacklist_trace_exit(parent_num, exit_num); - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { fprintf(stderr, "---- EXIT %d/%d blacklisted\n", parent_num, exit_num); } } } - if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { + if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { fprintf(stderr, "\n"); } @@ -5010,7 +5011,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes && EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last); - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) { fprintf(stderr, " TRACE %d exit %d %s() %s:%d\n", trace_num, exit_num, @@ -5023,7 +5024,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) { if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) { zend_jit_blacklist_trace_exit(trace_num, exit_num); - if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { + if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { fprintf(stderr, "---- EXIT %d/%d blacklisted\n", trace_num, exit_num); } @@ -5063,9 +5064,9 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) zend_cfg cfg; uint32_t i; - ZEND_ASSERT(zend_jit_func_counter_handler != NULL); - ZEND_ASSERT(zend_jit_ret_counter_handler != NULL); - ZEND_ASSERT(zend_jit_loop_counter_handler != NULL); + ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL); + ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL); + ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL); ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { @@ -5100,7 +5101,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { /* function entry */ - opline->handler = (const void*)zend_jit_func_counter_handler; + opline->handler = (const void*)zend_jit_func_trace_counter_handler; ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; @@ -5114,7 +5115,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) /* loop header */ opline = op_array->opcodes + cfg.blocks[i].start; if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { - opline->handler = (const void*)zend_jit_loop_counter_handler; + opline->handler = (const void*)zend_jit_loop_trace_counter_handler; if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index 4a15dca129e4f..d88da124d0a98 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -198,10 +198,10 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_H const zend_op *opline = EX(opline); #endif - *(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST; + *(jit_extension->counter) -= ZEND_JIT_COUNTER_FUNC_COST; if (UNEXPECTED(*(jit_extension->counter) <= 0)) { - *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; + *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { @@ -218,10 +218,10 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H const zend_op *opline = EX(opline); #endif - *(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST; + *(jit_extension->counter) -= ZEND_JIT_COUNTER_LOOP_COST; if (UNEXPECTED(*(jit_extension->counter) <= 0)) { - *(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; + *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT; zend_jit_hot_func(execute_data, opline); ZEND_OPCODE_RETURN(); } else { @@ -291,7 +291,7 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost; if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) { - *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_TRACE_COUNTER_INIT; + *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT; if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) { #ifndef HAVE_GCC_GLOBAL_REGS return -1; @@ -312,17 +312,17 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS) { - ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_FUNC_COST); + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_FUNC_COST); } ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS) { - ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_RET_COST); + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_RET_COST); } ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS) { - ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_LOOP_COST); + ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_LOOP_COST); } #define TRACE_RECORD(_op, _info, _ptr) \ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 4af3481c51f44..6297b49a2c6fc 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -130,8 +130,6 @@ const char* zend_reg_name[] = { # define GCC_GLOBAL_REGS 0 #endif -static uint32_t zend_jit_x86_flags = 0; - #if ZTS static size_t tsrm_ls_cache_tcb_offset = 0; static size_t tsrm_tls_index; @@ -527,7 +525,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | avx_ins op1, op2 || } else { | sse_ins op1, op2 @@ -569,7 +567,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SSE_GET_LONG, reg, lval || if (lval == 0) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) || } else { | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) @@ -584,7 +582,7 @@ static void* dasm_labels[zend_lb_MAX]; |.else | mov r0, lval |.endif -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 || } else { | cvtsi2sd, xmm(reg-ZREG_XMM0), r0 @@ -596,13 +594,13 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_MODE(addr) == IS_CONST_ZVAL) { | SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else { | cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } || } else if (Z_MODE(addr) == IS_REG) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) || } else { | cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) @@ -870,7 +868,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) @@ -924,7 +922,7 @@ static void* dasm_labels[zend_lb_MAX]; || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? || Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +|| if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) @@ -2128,6 +2126,10 @@ static int zend_jit_double_one_stub(dasm_State **Dst) static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + |->hybrid_runtime_jit: | EXT_CALL zend_runtime_jit, r0 | JMP_IP @@ -2136,6 +2138,10 @@ static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) { + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + |->hybrid_profile_jit: | // ++zend_jit_profile_counter; | .if X64 @@ -2191,13 +2197,17 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) * } * */ -static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) +static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) { - |->hybrid_func_counter: + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_func_hot_counter: | mov r0, EX->func | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] - | sub word [r2], ZEND_JIT_HOT_FUNC_COST + | sub word [r2], ZEND_JIT_COUNTER_FUNC_COST | jle >1 | GET_IP r2 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] @@ -2216,7 +2226,7 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) | jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | .endif |1: - | mov word [r2], ZEND_JIT_HOT_COUNTER_INIT + | mov word [r2], ZEND_JIT_COUNTER_INIT | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 @@ -2224,13 +2234,17 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) return 1; } -static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) +static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) { - |->hybrid_loop_counter: + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_loop_hot_counter: | mov r0, EX->func | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] - | sub word [r2], ZEND_JIT_HOT_LOOP_COST + | sub word [r2], ZEND_JIT_COUNTER_LOOP_COST | jle >1 | GET_IP r2 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] @@ -2249,7 +2263,7 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) | jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | .endif |1: - | mov word [r2], ZEND_JIT_HOT_COUNTER_INIT + | mov word [r2], ZEND_JIT_COUNTER_INIT | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 @@ -2267,7 +2281,7 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) | jle >1 | jmp aword [IP + r1] |1: - | mov word [r2], ZEND_JIT_TRACE_COUNTER_INIT + | mov word [r2], ZEND_JIT_COUNTER_INIT | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_trace_hot_root, r0 @@ -2283,27 +2297,39 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) { - |->hybrid_func_counter: + |->hybrid_func_trace_counter: - return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_FUNC_COST); + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_FUNC_COST); } static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) { - |->hybrid_ret_counter: + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_ret_trace_counter: - return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_RET_COST); + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_RET_COST); } static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) { - |->hybrid_loop_counter: + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_loop_trace_counter: - return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_LOOP_COST); + return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_LOOP_COST); } static int zend_jit_trace_halt_stub(dasm_State **Dst) { + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + |->trace_halt: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | add r4, HYBRID_SPAD @@ -2535,6 +2561,13 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(trace_halt), JIT_STUB(trace_exit), JIT_STUB(trace_escape), + JIT_STUB(hybrid_runtime_jit), + JIT_STUB(hybrid_profile_jit), + JIT_STUB(hybrid_func_hot_counter), + JIT_STUB(hybrid_loop_hot_counter), + JIT_STUB(hybrid_func_trace_counter), + JIT_STUB(hybrid_ret_trace_counter), + JIT_STUB(hybrid_loop_trace_counter), JIT_STUB(double_one), #ifdef CONTEXT_THREADED_JIT JIT_STUB(context_threaded_call), @@ -2553,9 +2586,9 @@ static int zend_jit_setup(void) zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); return FAILURE; } - if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { - if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { - zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; + if (JIT_G(opt_flags)) { + if (!zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { + JIT_G(opt_flags) &= ~ZEND_JIT_CPU_AVX; } } @@ -3634,13 +3667,13 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_ } | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] } } else { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] @@ -3769,7 +3802,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 } else { | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 @@ -3799,7 +3832,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; | SSE_GET_ZVAL_LVAL result_reg, op1_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | AVX_MATH opcode, result_reg, result_reg, op2_addr } else { | SSE_MATH opcode, result_reg, op2_addr @@ -3831,7 +3864,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, result_reg = ZREG_XMM0; } | SSE_GET_ZVAL_LVAL result_reg, op2_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | AVX_MATH opcode, result_reg, result_reg, op1_addr } else { | SSE_MATH opcode, result_reg, op1_addr @@ -3849,7 +3882,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, result_reg = ZREG_XMM0; tmp_reg = ZREG_XMM1; } - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { zend_reg op1_reg; if (Z_MODE(op1_addr) == IS_REG) { @@ -3897,7 +3930,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, result_reg = ZREG_XMM0; } - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { zend_reg op1_reg; zend_jit_addr val_addr; @@ -4610,7 +4643,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); const void *exit_addr = NULL; - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -4645,7 +4678,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { | jbe &exit_addr } else { | jbe >2 // NOT_FOUND @@ -4674,7 +4707,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.endif if (type == BP_JIT_IS) { | jbe >9 // NOT_FOUND - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { | jbe &exit_addr } else { | jbe >2 // NOT_FOUND @@ -4713,13 +4746,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jmp &exit_addr } else { | jmp >2 // NOT_FOUND } } - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jmp &exit_addr } else { | jmp >2 // NOT_FOUND @@ -4728,7 +4761,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | EXT_CALL zend_hash_index_find, r0 | test r0, r0 - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jz &exit_addr } else { | jz >2 // NOT_FOUND @@ -4737,7 +4770,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |2: switch (type) { case BP_VAR_R: - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | // retval = &EG(uninitialized_zval); | UNDEFINED_OFFSET opline @@ -4757,14 +4790,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o break; case BP_VAR_RW: |2: - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { | SAVE_VALID_OPLINE opline, r0 | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { | jmp >8 } |4: @@ -4848,7 +4881,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL _zend_hash_find_known_hash, r0 } | test r0, r0 - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jz &exit_addr } else { | jz >2 // NOT_FOUND @@ -4863,7 +4896,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |2: switch (type) { case BP_VAR_R: - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); | UNDEFINED_INDEX opline | jmp >9 @@ -7351,7 +7384,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) { | vxorps xmm0, xmm0, xmm0 } else { | xorps xmm0, xmm0 @@ -7615,7 +7648,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | cmp r2, FCARG1a } - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -7997,7 +8030,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | // CACHE_PTR(opline->result.num, fbc); | mov r1, EX->run_time_cache | mov aword [r1 + opline->result.num], r0 - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -8588,7 +8621,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |2: } - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !JIT_G(current_frame)->call || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call)) { @@ -8682,7 +8715,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { @@ -8690,7 +8723,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend /* Don't generate code that always throws exception */ return 0; } - } else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8828,7 +8861,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } if (opline->opcode == ZEND_SEND_VAR_EX) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { @@ -8853,7 +8886,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend |.code } } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { @@ -8895,7 +8928,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >7 - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -8915,7 +8948,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend |.code } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { @@ -8962,7 +8995,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | cmp cl, IS_REFERENCE | je >7 } - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { @@ -9032,7 +9065,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline, cons { uint32_t arg_num = opline->op2.num; - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { @@ -9545,7 +9578,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze } | EXT_CALL zend_jit_leave_func_helper, r0 - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame))) { // TODO: try to avoid this check ??? @@ -9693,7 +9726,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { return_value_used = 1; } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { @@ -10334,7 +10367,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | cmp dword EX->This.u2.num_args, arg_num | jae >5 @@ -10559,7 +10592,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, cons op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -10647,7 +10680,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, cons | jmp >9 } - if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { |7: if (opline->opcode == ZEND_FETCH_OBJ_R) { | SAVE_VALID_OPLINE opline, r1 @@ -10840,7 +10873,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze { zend_jit_addr res_addr = RES_ADDR(); - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { @@ -11210,7 +11243,7 @@ static zend_bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) return 0; } - if (zend_jit_reg_alloc < ZEND_JIT_REG_ALLOC_GLOBAL) { + if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { /* Disable global register allocation, * register allocation for SSA variables connected through Phi functions */ @@ -11545,7 +11578,7 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend break; } - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (ssa_op == ssa->ops && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { @@ -11556,7 +11589,7 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend #if ZTS /* %r0 is used to check EG(vm_interrupt) */ - if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (ssa_op == ssa->ops && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { diff --git a/ext/opcache/zend_accelerator_module.c b/ext/opcache/zend_accelerator_module.c index bb5ac37e93cd4..ec721f77b897c 100644 --- a/ext/opcache/zend_accelerator_module.c +++ b/ext/opcache/zend_accelerator_module.c @@ -194,6 +194,28 @@ static ZEND_INI_MH(OnUpdateFileCache) return SUCCESS; } +#ifdef HAVE_JIT +static ZEND_INI_MH(OnUpdateJit) +{ + if (zend_jit_config(new_value, stage) == SUCCESS) { + return OnUpdateString(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage); + } + return FAILURE; +} + +static ZEND_INI_MH(OnUpdateJitDebug) +{ + zend_long *p = (zend_long *) ZEND_INI_GET_ADDR(); + zend_long val = zend_atol(ZSTR_VAL(new_value), ZSTR_LEN(new_value)); + + if (zend_jit_debug_config(*p, val, stage) == SUCCESS) { + *p = val; + return SUCCESS; + } + return FAILURE; +} +#endif + ZEND_INI_BEGIN() STD_PHP_INI_BOOLEAN("opcache.enable" , "1", PHP_INI_ALL, OnEnable, enabled , zend_accel_globals, accel_globals) STD_PHP_INI_BOOLEAN("opcache.use_cwd" , "1", PHP_INI_SYSTEM, OnUpdateBool, accel_directives.use_cwd , zend_accel_globals, accel_globals) @@ -251,10 +273,10 @@ ZEND_INI_BEGIN() STD_PHP_INI_ENTRY("opcache.cache_id" , "" , PHP_INI_SYSTEM, OnUpdateString, accel_directives.cache_id, zend_accel_globals, accel_globals) #endif #ifdef HAVE_JIT - STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_DEFAULT, PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit, zend_accel_globals, accel_globals) - STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_buffer_size, zend_accel_globals, accel_globals) - STD_PHP_INI_ENTRY("opcache.jit_debug" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_debug, zend_accel_globals, accel_globals) - STD_PHP_INI_ENTRY("opcache.jit_bisect_limit" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_bisect_limit, zend_accel_globals, accel_globals) + STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_DEFAULT_OPTIONS, PHP_INI_ALL, OnUpdateJit, options, zend_jit_globals, jit_globals) + STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , ZEND_JIT_DEFAULT_BUFFER_SIZE, PHP_INI_SYSTEM, OnUpdateLong, buffer_size, zend_jit_globals, jit_globals) + STD_PHP_INI_ENTRY("opcache.jit_debug" , "0", PHP_INI_ALL, OnUpdateJitDebug, debug, zend_jit_globals, jit_globals) + STD_PHP_INI_ENTRY("opcache.jit_bisect_limit" , "0", PHP_INI_ALL, OnUpdateLong, bisect_limit, zend_jit_globals, jit_globals) #endif ZEND_INI_END() @@ -387,8 +409,12 @@ void zend_accel_info(ZEND_MODULE_INFO_FUNC_ARGS) php_info_print_table_row(2, "File Cache", "Disabled"); } #if HAVE_JIT - if (ZCG(jit_enabled)) { - php_info_print_table_row(2, "JIT", "Enabled"); + if (JIT_G(enabled)) { + if (JIT_G(on)) { + php_info_print_table_row(2, "JIT", "On"); + } else { + php_info_print_table_row(2, "JIT", "Off"); + } } else { php_info_print_table_row(2, "JIT", "Disabled"); } @@ -722,10 +748,10 @@ ZEND_FUNCTION(opcache_get_configuration) add_assoc_string(&directives, "opcache.cache_id", STRING_NOT_NULL(ZCG(accel_directives).cache_id)); #endif #ifdef HAVE_JIT - add_assoc_long(&directives, "opcache.jit", ZCG(accel_directives).jit); - add_assoc_long(&directives, "opcache.jit_buffer_size", ZCG(accel_directives).jit_buffer_size); - add_assoc_long(&directives, "opcache.jit_debug", ZCG(accel_directives).jit_debug); - add_assoc_long(&directives, "opcache.jit_bisect_limit", ZCG(accel_directives).jit_bisect_limit); + add_assoc_string(&directives, "opcache.jit", JIT_G(options)); + add_assoc_long(&directives, "opcache.jit_buffer_size", JIT_G(buffer_size)); + add_assoc_long(&directives, "opcache.jit_debug", JIT_G(debug)); + add_assoc_long(&directives, "opcache.jit_bisect_limit", JIT_G(bisect_limit)); #endif add_assoc_zval(return_value, "directives", &directives); diff --git a/ext/opcache/zend_file_cache.c b/ext/opcache/zend_file_cache.c index eb74187e5d419..4d87c03fe05f0 100644 --- a/ext/opcache/zend_file_cache.c +++ b/ext/opcache/zend_file_cache.c @@ -33,6 +33,10 @@ #include "zend_accelerator_util_funcs.h" #include "zend_accelerator_hash.h" +#if HAVE_JIT +#include "jit/zend_jit.h" +#endif + #include #include #include @@ -892,7 +896,7 @@ int zend_file_cache_script_store(zend_persistent_script *script, int in_shm) #ifdef HAVE_JIT /* FIXME: dump jited codes out to file cache? */ - if (ZCG(jit_enabled)) { + if (JIT_G(on)) { return FAILURE; } #endif diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index 6a158e73f9e33..23f1a5fb61e5f 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -566,8 +566,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); #ifdef HAVE_JIT - if (ZCG(jit_enabled) && - ZEND_JIT_LEVEL(ZCG(accel_directives).jit) <= ZEND_JIT_LEVEL_OPT_FUNCS && + if (JIT_G(on) && JIT_G(opt_level) <= ZEND_JIT_LEVEL_OPT_FUNCS && !ZCG(current_persistent_script)->corrupted) { zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); } @@ -1122,7 +1121,7 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); #ifdef HAVE_JIT - if (ZCG(jit_enabled) && for_shm) { + if (JIT_G(on) && for_shm) { zend_jit_unprotect(); } #endif @@ -1143,8 +1142,8 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script } #ifdef HAVE_JIT - if (ZCG(jit_enabled) && for_shm) { - if (ZEND_JIT_LEVEL(ZCG(accel_directives).jit) >= ZEND_JIT_LEVEL_OPT_SCRIPT) { + if (JIT_G(on) && for_shm) { + if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_SCRIPT) { zend_jit_script(&script->script); } zend_jit_protect(); From 38b844a38a913c07b763ce3e9eeac64a04c099e6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 15 May 2020 13:25:18 +0300 Subject: [PATCH 2/3] Disable JIT for test --- ext/opcache/tests/bug78185.phpt | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/tests/bug78185.phpt b/ext/opcache/tests/bug78185.phpt index 926bb964047e1..120f2b61090b8 100644 --- a/ext/opcache/tests/bug78185.phpt +++ b/ext/opcache/tests/bug78185.phpt @@ -5,6 +5,7 @@ opcache.enable_cli=1 opcache.optimization_level=-1 opcache.file_cache={PWD} opcache.file_cache_only=1 +opcache.jit=0 --SKIPIF-- --FILE-- From bc6d87c36e608eb87702264aec5bc16375070756 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 15 May 2020 14:07:44 +0300 Subject: [PATCH 3/3] Fix ZTS build --- ext/opcache/jit/zend_jit.c | 4 ++-- ext/opcache/jit/zend_jit.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index db0f9d3fb1520..586f52929557e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -38,7 +38,7 @@ #include "jit/zend_jit_internal.h" #ifdef ZTS -int zend_jit_globals_id; +int jit_globals_id; #else zend_jit_globals jit_globals; #endif @@ -3637,7 +3637,7 @@ ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int ZEND_EXT_API void zend_jit_init(void) { #ifdef ZTS - zend_jit_globals_id = ts_allocate_id(&zend_jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL); + jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL); #else zend_jit_globals_ctor(&jit_globals); #endif diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 82bf417d5f0f3..699d390e76fc1 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -134,8 +134,8 @@ typedef struct _zend_jit_globals { } zend_jit_globals; #ifdef ZTS -# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v) -extern int zend_jit_globals_id; +# define JIT_G(v) ZEND_TSRMG(jit_globals_id, zend_jit_globals *, v) +extern int jit_globals_id; #else # define JIT_G(v) (jit_globals.v) extern zend_jit_globals jit_globals;