[wasm] More jiterpreter cleanup (#78519)

    Move more jiterpreter configuration and constants into options
    Fix should_generate_trace_here not scanning across multiple basic blocks
    Disable specialized JIT call in threaded wasm mode (though I think it might work, it's better to turn it off to be sure for now)
    Introduces genmintops.py, a script that automatically generates mintops.ts from mintops.def
    Adjust typescript config to make it able to find the generated mintops.ts (and fix ESLint on Linux)
    Unroll memsets below a certain size into raw wasm opcodes, because v8 generates expensive function calls for memset and memcpy. Unrolling memcpy is a TODO for later
    Rename "always generate" to "disable heuristic" to more accurately describe what it does
    Fix jiterpreter_dump_stats hiding errors if startup failed before cwraps were ready
    Misc. code cleanup

Author: kg

src/mono/mono/mini/interp/interp.c

@@ -2147,7 +2147,7 @@ typedef struct {
 	gpointer *many_args;
 } InterpEntryData;
 
-static gboolean
+static MONO_ALWAYS_INLINE gboolean
 is_method_multicastdelegate_invoke (MonoMethod *method)
 {
 	return m_class_get_parent (method->klass) == mono_defaults.multicastdelegate_class && !strcmp (method->name, "Invoke");
@@ -2672,16 +2672,17 @@ do_jit_call (ThreadContext *context, stackval *ret_sp, stackval *sp, InterpFrame
 			goto epilogue;
 		} else {
 			int count = cinfo->hit_count;
-			if (count == JITERPRETER_JIT_CALL_TRAMPOLINE_HIT_COUNT) {
+			if (count == mono_opt_jiterpreter_jit_call_trampoline_hit_count) {
 				void *fn = cinfo->no_wrapper ? cinfo->addr : cinfo->wrapper;
 				mono_interp_jit_wasm_jit_call_trampoline (
 					rmethod, cinfo, fn, rmethod->hasthis, rmethod->param_count,
 					rmethod->arg_offsets, mono_aot_mode == MONO_AOT_MODE_LLVMONLY_INTERP
 				);
 			} else {
-				if (count <= JITERPRETER_JIT_CALL_QUEUE_FLUSH_THRESHOLD)
+				int excess = count - mono_opt_jiterpreter_jit_call_queue_flush_threshold;
+				if (excess <= 0)
 					cinfo->hit_count++;
-				if (count == JITERPRETER_JIT_CALL_QUEUE_FLUSH_THRESHOLD)
+				if (excess == 0)
 					mono_interp_flush_jitcall_queue ();
 			}
 		}

src/mono/mono/mini/interp/jiterpreter.c

@@ -840,32 +840,35 @@ jiterp_should_abort_trace (InterpInst *ins, gboolean *inside_branch_block)
 }
 
 static gboolean
-should_generate_trace_here (InterpBasicBlock *bb, InterpInst *last_ins) {
+should_generate_trace_here (InterpBasicBlock *bb) {
 	int current_trace_length = 0;
 	// A preceding trace may have been in a branch block, but we only care whether the current
 	//  trace will have a branch block opened, because that determines whether calls and branches
 	//  will unconditionally abort the trace or not.
 	gboolean inside_branch_block = FALSE;
 
-	// We scan forward through the entire method body starting from the current block, not just
-	//  the current block (since the actual trace compiler doesn't know about block boundaries).
-	for (InterpInst *ins = bb->first_ins; (ins != NULL) && (ins != last_ins); ins = ins->next) {
-		int category = jiterp_should_abort_trace(ins, &inside_branch_block);
-		switch (category) {
-			case TRACE_ABORT: {
-				jiterpreter_abort_counts[ins->opcode]++;
-				return current_trace_length >= mono_opt_jiterpreter_minimum_trace_length;
+	while (bb) {
+		// We scan forward through the entire method body starting from the current block, not just
+		//  the current block (since the actual trace compiler doesn't know about block boundaries).
+		for (InterpInst *ins = bb->first_ins; ins != NULL; ins = ins->next) {
+			int category = jiterp_should_abort_trace(ins, &inside_branch_block);
+			switch (category) {
+				case TRACE_ABORT:
+					jiterpreter_abort_counts[ins->opcode]++;
+					return current_trace_length >= mono_opt_jiterpreter_minimum_trace_length;
+				case TRACE_IGNORE:
+					break;
+				default:
+					current_trace_length++;
+					break;
 			}
-			case TRACE_IGNORE:
-				break;
-			default:
-				current_trace_length++;
-				break;
+
+			// Once we know the trace is long enough we can stop scanning.
+			if (current_trace_length >= mono_opt_jiterpreter_minimum_trace_length)
+				return TRUE;
 		}
 
-		// Once we know the trace is long enough we can stop scanning.
-		if (current_trace_length >= mono_opt_jiterpreter_minimum_trace_length)
-			return TRUE;
+		bb = bb->next_bb;
 	}
 
 	return FALSE;
@@ -908,12 +911,12 @@ jiterp_insert_entry_points (void *_td)
 		//  multiple times and waste some work. At present this is unavoidable because
 		//  control flow means we can end up with two traces covering different subsets
 		//  of the same method in order to handle loops and resuming
-		gboolean should_generate = enabled && should_generate_trace_here(bb, td->last_ins);
+		gboolean should_generate = enabled && should_generate_trace_here(bb);
 
 		if (mono_opt_jiterpreter_call_resume_enabled && bb->contains_call_instruction)
 			enter_at_next = TRUE;
 
-		if (mono_opt_jiterpreter_always_generate)
+		if (mono_opt_jiterpreter_disable_heuristic)
 			should_generate = TRUE;
 
 		if (enabled && should_generate) {

src/mono/mono/mini/interp/jiterpreter.h

@@ -3,9 +3,15 @@
 
 #ifdef HOST_BROWSER
 
+#ifdef DISABLE_THREADS
+#define JITERPRETER_ENABLE_JIT_CALL_TRAMPOLINES 1
 // enables specialized mono_llvm_cpp_catch_exception replacement (see jiterpreter-jit-call.ts)
 // works even if the jiterpreter is otherwise disabled.
 #define JITERPRETER_ENABLE_SPECIALIZED_JIT_CALL 1
+#else
+#define JITERPRETER_ENABLE_JIT_CALL_TRAMPOLINES 0
+#define JITERPRETER_ENABLE_SPECIALIZED_JIT_CALL 0
+#endif // DISABLE_THREADS
 
 // mono_interp_tier_prepare_jiterpreter will return these special values if it doesn't
 //  have a function pointer for a specific entry point.
@@ -14,13 +20,6 @@
 // NOT_JITTED indicates that the trace was not jitted and it should be turned into a NOP
 #define JITERPRETER_NOT_JITTED 1
 
-#define JITERPRETER_ENABLE_JIT_CALL_TRAMPOLINES 1
-// After a do_jit_call call site is hit this many times, we will queue it to be jitted
-#define JITERPRETER_JIT_CALL_TRAMPOLINE_HIT_COUNT 2999
-// If a do_jit_call site is hit this many times without being jitted (due to waiting in
-//  the queue), we will flush the queue immediately
-#define JITERPRETER_JIT_CALL_QUEUE_FLUSH_THRESHOLD 10000
-
 typedef const ptrdiff_t (*JiterpreterThunk) (void *frame, void *pLocals);
 typedef void (*WasmJitCallThunk) (void *extra_arg, void *ret_sp, void *sp, gboolean *thrown);
 typedef void (*WasmDoJitCall) (gpointer cb, gpointer arg, gboolean *out_thrown);

src/mono/mono/mini/interp/mintops.def

@@ -8,6 +8,11 @@
  * optype describes the contents of the instruction, following the dreg/sreg offsets.
  */
 
+/*
+ * This file is parsed by genmintops.py to generate typescript during the wasm build process,
+ *  so if you make any changes to its syntax you will need to update that script.
+ */
+
 OPDEF(MINT_NOP, "nop", 1, 0, 0, MintOpNoArgs)
 OPDEF(MINT_NIY, "niy", 1, 0, 0, MintOpNoArgs)
 OPDEF(MINT_DEF, "def", 2, 1, 0, MintOpNoArgs)

src/mono/mono/mini/interp/transform.c

@@ -10071,7 +10071,8 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG
 		interp_optimize_code (td);
 		interp_alloc_offsets (td);
 #if HOST_BROWSER
-		jiterp_insert_entry_points (td);
+		if (mono_interp_tiering_enabled ())
+			jiterp_insert_entry_points (td);
 #endif
 	}
 

src/mono/mono/utils/options-def.h

@@ -65,21 +65,21 @@ DEFINE_BOOL(aot_lazy_assembly_load, "aot-lazy-assembly-load", FALSE, "Load assem
 
 // the jiterpreter is not yet thread safe due to the need to synchronize function pointers
 //  and wasm modules between threads. before these can be enabled we need to implement all that
-#if FEATURE_WASM_THREADS
+#ifdef DISABLE_THREADS
 // traces_enabled controls whether the jiterpreter will JIT individual interpreter opcode traces
-DEFINE_BOOL_READONLY(jiterpreter_traces_enabled, "jiterpreter-traces-enabled", FALSE, "JIT interpreter opcode traces into WASM")
+DEFINE_BOOL(jiterpreter_traces_enabled, "jiterpreter-traces-enabled", FALSE, "JIT interpreter opcode traces into WASM")
 // interp_entry_enabled controls whether specialized interp_entry wrappers will be jitted
-DEFINE_BOOL_READONLY(jiterpreter_interp_entry_enabled, "jiterpreter-interp-entry-enabled", FALSE, "JIT specialized WASM interp_entry wrappers")
+DEFINE_BOOL(jiterpreter_interp_entry_enabled, "jiterpreter-interp-entry-enabled", FALSE, "JIT specialized WASM interp_entry wrappers")
 // jit_call_enabled controls whether do_jit_call will use specialized trampolines for hot call sites
-DEFINE_BOOL_READONLY(jiterpreter_jit_call_enabled, "jiterpreter-jit-call-enabled", FALSE, "JIT specialized WASM do_jit_call trampolines")
+DEFINE_BOOL(jiterpreter_jit_call_enabled, "jiterpreter-jit-call-enabled", FALSE, "JIT specialized WASM do_jit_call trampolines")
 #else
 // traces_enabled controls whether the jiterpreter will JIT individual interpreter opcode traces
-DEFINE_BOOL(jiterpreter_traces_enabled, "jiterpreter-traces-enabled", FALSE, "JIT interpreter opcode traces into WASM")
+DEFINE_BOOL_READONLY(jiterpreter_traces_enabled, "jiterpreter-traces-enabled", FALSE, "JIT interpreter opcode traces into WASM")
 // interp_entry_enabled controls whether specialized interp_entry wrappers will be jitted
-DEFINE_BOOL(jiterpreter_interp_entry_enabled, "jiterpreter-interp-entry-enabled", FALSE, "JIT specialized WASM interp_entry wrappers")
+DEFINE_BOOL_READONLY(jiterpreter_interp_entry_enabled, "jiterpreter-interp-entry-enabled", FALSE, "JIT specialized WASM interp_entry wrappers")
 // jit_call_enabled controls whether do_jit_call will use specialized trampolines for hot call sites
-DEFINE_BOOL(jiterpreter_jit_call_enabled, "jiterpreter-jit-call-enabled", FALSE, "JIT specialized WASM do_jit_call trampolines")
-#endif // FEATURE_WASM_THREADS
+DEFINE_BOOL_READONLY(jiterpreter_jit_call_enabled, "jiterpreter-jit-call-enabled", FALSE, "JIT specialized WASM do_jit_call trampolines")
+#endif // DISABLE_THREADS
 
 // enables using WASM try/catch_all instructions where appropriate (currently only do_jit_call),
 //  will be automatically turned off if the instructions are not available.
@@ -93,16 +93,24 @@ DEFINE_BOOL(jiterpreter_call_resume_enabled, "jiterpreter-call-resume-enabled",
 // For locations where the jiterpreter heuristic says we will be unable to generate
 //  a trace, insert an entry point opcode anyway. This enables collecting accurate
 //  stats for options like estimateHeat, but raises overhead.
-DEFINE_BOOL(jiterpreter_always_generate, "jiterpreter-always-generate", FALSE, "Always insert trace entry points for more accurate statistics")
+DEFINE_BOOL(jiterpreter_disable_heuristic, "jiterpreter-disable-heuristic", FALSE, "Always insert trace entry points for more accurate statistics")
 // Automatically prints stats at app exit or when jiterpreter_dump_stats is called
 DEFINE_BOOL(jiterpreter_stats_enabled, "jiterpreter-stats-enabled", FALSE, "Automatically print jiterpreter statistics")
 // Continue counting hits for traces that fail to compile and use it to estimate
 //  the relative importance of the opcode that caused them to abort
 DEFINE_BOOL(jiterpreter_estimate_heat, "jiterpreter-estimate-heat", FALSE, "Maintain accurate hit count for all trace entry points")
 // Count the number of times a trace bails out (branch taken, etc) and for what reason
 DEFINE_BOOL(jiterpreter_count_bailouts, "jiterpreter-count-bailouts", FALSE, "Maintain accurate count of all trace bailouts based on cause")
+// Dump the wasm blob for all compiled traces
+DEFINE_BOOL(jiterpreter_dump_traces, "jiterpreter-dump-traces", FALSE, "Dump the wasm blob for all compiled traces to the console")
 // any trace that doesn't have at least this many meaningful (non-nop) opcodes in it will be rejected
 DEFINE_INT(jiterpreter_minimum_trace_length, "jiterpreter-minimum-trace-length", 8, "Reject traces shorter than this number of meaningful opcodes")
+// once a trace entry point is inserted, we only actually JIT code for it once it's been hit this many times
+DEFINE_INT(jiterpreter_minimum_trace_hit_count, "jiterpreter-minimum-trace-hit-count", 10000, "JIT trace entry points once they are hit this many times")
+// After a do_jit_call call site is hit this many times, we will queue it to be jitted
+DEFINE_INT(jiterpreter_jit_call_trampoline_hit_count, "jiterpreter-jit-call-hit-count", 3000, "Queue specialized do_jit_call trampoline for JIT after this many hits")
+// After a do_jit_call call site is hit this many times without being jitted, we will flush the JIT queue
+DEFINE_INT(jiterpreter_jit_call_queue_flush_threshold, "jiterpreter-jit-call-queue-flush-threshold", 10000, "Flush the do_jit_call JIT queue after an unJITted call site has this many hits")
 #endif // HOST_BROWSER
 
 /* Cleanup */

src/mono/mono/utils/options.c

@@ -90,7 +90,7 @@ mono_options_print_usage (void)
 static GHashTable *_option_hash = NULL;
 
 static GHashTable *
-get_option_hash ()
+get_option_hash (void)
 {
 	GHashTable *result;