Link to the compiled function to improve performance

When JIT is recording, backtrack the trace if encountering
a compiled inline function and link to this function later.
This reduces the runtime compilation overhead and duplicated
JITTed code. Smaller code size has better cache efficiency,
which brings 1.7% performance gain in our benchmark on x86.

Signed-off-by: Wang, Xue   <[email protected]>
Signed-off-by: Yang, Lin A <[email protected]>
Signed-off-by: Su, Tao     <[email protected]>

Author: wxue1

ext/opcache/jit/zend_jit.h

@@ -116,6 +116,7 @@ typedef struct _zend_jit_globals {
 	zend_long   max_recursive_returns; /* max number of recursive inlined return unrolls */
 	zend_long   max_polymorphic_calls; /* max number of inlined polymorphic calls */
 	zend_long   max_trace_length; 	   /* max length of a single trace */
+	zend_long   jit_trace_inline_func_limit; /* max length of inlined function in a trace */
 
 	zend_sym_node *symbols;            /* symbols for disassembler */
 

ext/opcache/jit/zend_jit_vm_helpers.c

@@ -562,10 +562,14 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
 	uint8_t  trace_flags, op1_type, op2_type, op3_type;
 	zend_class_entry *ce1, *ce2;
 	const zend_op *link_to_enter_opline = NULL;
+	/* Remember the first opline of inline function*/
+	const zend_op *link_to_inline_func_opline = NULL;
 	int backtrack_link_to_enter = -1;
 	int backtrack_recursion = -1;
 	int backtrack_ret_recursion = -1;
 	int backtrack_ret_recursion_level = 0;
+	/* Remember the index of inline function opline in the trace buffer */
+	int backtrack_link_to_inline_func = -1;
 	int loop_unroll_limit = 0;
 	int last_loop = -1;
 	int last_loop_level = -1;
@@ -922,6 +926,13 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
 				} else if (count >= JIT_G(max_recursive_calls)) {
 					stop = ZEND_JIT_TRACE_STOP_DEEP_RECURSION;
 					break;
+				/* If the inline function is JITTed, backtrack this trace. */
+				} else if ( idx > JIT_G(jit_trace_inline_func_limit) && \
+					backtrack_link_to_inline_func < 0 && \
+					(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
+					backtrack_link_to_inline_func = idx;
+					link_to_inline_func_opline = opline;
+					break;
 				}
 
 				unrolled_calls[ret_level + level] = &EX(func)->op_array;
@@ -1156,6 +1167,11 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
 			ret_level = backtrack_ret_recursion_level;
 			stop = ZEND_JIT_TRACE_STOP_RECURSIVE_RET;
 			end_opline = orig_opline;
+		} else if (backtrack_link_to_inline_func > 0) {
+			/* Reset the index and stop flag to link to the compiled inline function. */
+			idx = backtrack_link_to_inline_func;
+			stop = ZEND_JIT_TRACE_STOP_LINK;
+			end_opline = link_to_inline_func_opline;
 		} else if (backtrack_link_to_enter > 0) {
 			if (stop == ZEND_JIT_TRACE_STOP_DEEP_RECURSION
 			 && zend_jit_trace_bad_stop_event(orig_opline, JIT_G(blacklist_root_trace) / 2) ==

ext/opcache/zend_accelerator_module.c

@@ -336,6 +336,7 @@ ZEND_INI_BEGIN()
 	STD_PHP_INI_ENTRY("opcache.jit_max_recursive_returns"     , "2",                          PHP_INI_ALL,    OnUpdateUnrollR,  max_recursive_returns, zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_max_polymorphic_calls"     , "2",                          PHP_INI_ALL,    OnUpdateLong,     max_polymorphic_calls, zend_jit_globals, jit_globals)
     STD_PHP_INI_ENTRY("opcache.jit_max_trace_length"          , "1024",                       PHP_INI_ALL,    OnUpdateMaxTraceLength, max_trace_length, zend_jit_globals, jit_globals)
+	STD_PHP_INI_ENTRY("opcache.jit_trace_inline_func_limit"     , "16",                       PHP_INI_ALL,    OnUpdateLong,     jit_trace_inline_func_limit, zend_jit_globals, jit_globals)
 #endif
 ZEND_INI_END()
 
@@ -851,6 +852,7 @@ ZEND_FUNCTION(opcache_get_configuration)
 	add_assoc_long(&directives,   "opcache.jit_max_side_traces", JIT_G(max_side_traces));
 	add_assoc_long(&directives,   "opcache.jit_prof_threshold", JIT_G(prof_threshold));
 	add_assoc_long(&directives,   "opcache.jit_max_trace_length", JIT_G(max_trace_length));
+	add_assoc_long(&directives,   "opcache.jit_trace_inline_func_limit", JIT_G(jit_trace_inline_func_limit));
 #endif
 
 	add_assoc_zval(return_value, "directives", &directives);