Merge pull request #36166 from JuliaLang/jn/ml-matches-leaf-cache

Create a leaf-types cache (so we'll first hit the call-site cache LRU, then the TypeMap search, then this hash lookup, before going to the defs list), and use this new cache for the ml-matches search also (when applicable/useful).

Author: vtjnash

base/errorshow.jl

@@ -176,7 +176,7 @@ function showerror(io::IO, ex::InexactError)
     Experimental.show_error_hints(io, ex)
 end
 
-typesof(args...) = Tuple{Any[ Core.Typeof(a) for a in args ]...}
+typesof(@nospecialize args...) = Tuple{Any[ Core.Typeof(args[i]) for i in 1:length(args) ]...}
 
 function print_with_compare(io::IO, @nospecialize(a::DataType), @nospecialize(b::DataType), color::Symbol)
     if a.name === b.name

src/ast.c

@@ -962,7 +962,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule
     jl_value_t *result;
     JL_TRY {
         margs[0] = jl_toplevel_eval(*ctx, margs[0]);
-        jl_method_instance_t *mfunc = jl_method_lookup(margs, nargs, 1, world);
+        jl_method_instance_t *mfunc = jl_method_lookup(margs, nargs, world);
         JL_GC_PROMISE_ROOTED(mfunc);
         if (mfunc == NULL) {
             jl_method_error(margs[0], &margs[1], nargs, world);

src/builtins.c

@@ -990,8 +990,7 @@ JL_CALLABLE(jl_f_applicable)
 {
     JL_NARGSV(applicable, 1);
     size_t world = jl_get_ptls_states()->world_age;
-    return jl_method_lookup(args, nargs, 1, world) != NULL ?
-        jl_true : jl_false;
+    return jl_method_lookup(args, nargs, world) != NULL ? jl_true : jl_false;
 }
 
 JL_CALLABLE(jl_f_invoke)

src/datatype.c

@@ -49,6 +49,7 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo
     mt->name = jl_demangle_typename(name);
     mt->module = module;
     mt->defs = jl_nothing;
+    mt->leafcache = (jl_array_t*)jl_an_empty_vec_any;
     mt->cache = jl_nothing;
     mt->max_args = 0;
     mt->kwsorter = NULL;

src/dump.c

@@ -2237,9 +2237,6 @@ STATIC_INLINE jl_value_t *verify_type(jl_value_t *v) JL_NOTSAFEPOINT
 }
 #endif
 
-jl_datatype_t *jl_lookup_cache_type_(jl_datatype_t *type);
-void jl_cache_type_(jl_datatype_t *type);
-
 static jl_datatype_t *jl_recache_type(jl_datatype_t *dt) JL_GC_DISABLED
 {
     jl_datatype_t *t; // the type after unique'ing

src/gf.c

@@ -37,7 +37,11 @@ static int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_t *val)
     jl_array_t *a = *pa;
     size_t orig, index, iter, empty_slot;
     size_t newsz, sz = hash_size(a);
-    assert(sz >= 1);
+    if (sz == 0) {
+        a = jl_alloc_vec_any(HT_N_INLINE);
+        sz = hash_size(a);
+        *pa = a;
+    }
     size_t maxprobe = max_probe(sz);
     void **tab = (void **)a->data;
 
@@ -108,7 +112,8 @@ static int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_t *val)
 jl_value_t **jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT
 {
     size_t sz = hash_size(a);
-    assert(sz >= 1);
+    if (sz == 0)
+        return NULL;
     size_t maxprobe = max_probe(sz);
     void **tab = (void **)a->data;
     uint_t hv = keyhash(key);

src/iddict.c

@@ -721,7 +721,6 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     else {
         jl_init_types();
         jl_init_codegen();
-        jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally
     }
 
     jl_init_tasks();

src/init.c

@@ -822,17 +822,9 @@ static void cache_insert_type_linear(jl_datatype_t *type, ssize_t insert_at)
 #ifndef NDEBUG
 static int is_cacheable(jl_datatype_t *type)
 {
-    // only cache types whose behavior will not depend on the identities
-    // of contained TypeVars
-    assert(jl_is_datatype(type));
-    jl_svec_t *t = type->parameters;
-    if (jl_svec_len(t) == 0 && jl_emptytuple_type != NULL) // Tuple{} is the only type eligible for this that doesn't have parameters
-        return 0;
-    // cache abstract types with no free type vars
-    if (jl_is_abstracttype(type))
-        return !jl_has_free_typevars((jl_value_t*)type);
-    // ... or concrete types
-    return jl_is_concrete_type((jl_value_t*)type);
+    // ensure cache only contains types whose behavior will not depend on the
+    // identities of contained TypeVars
+    return !jl_has_free_typevars((jl_value_t*)type);
 }
 #endif
 
@@ -1943,18 +1935,19 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_methtable_type->name->mt = jl_nonfunction_mt;
     jl_methtable_type->super = jl_any_type;
     jl_methtable_type->parameters = jl_emptysvec;
-    jl_methtable_type->name->names = jl_perm_symsvec(11, "name", "defs",
-                                                     "cache", "max_args",
+    jl_methtable_type->name->names = jl_perm_symsvec(12, "name", "defs",
+                                                     "leafcache", "cache", "max_args",
                                                      "kwsorter", "module",
                                                      "backedges", "", "", "offs", "");
-    jl_methtable_type->types = jl_svec(11, jl_symbol_type, jl_any_type, jl_any_type, jl_any_type/*jl_long*/,
+    jl_methtable_type->types = jl_svec(12, jl_symbol_type, jl_any_type, jl_any_type,
+                                       jl_any_type, jl_any_type/*jl_long*/,
                                        jl_any_type, jl_any_type/*module*/,
                                        jl_any_type/*any vector*/, jl_any_type/*long*/, jl_any_type/*int32*/,
                                        jl_any_type/*uint8*/, jl_any_type/*uint8*/);
     jl_methtable_type->instance = NULL;
     jl_methtable_type->abstract = 0;
     jl_methtable_type->mutabl = 1;
-    jl_methtable_type->ninitialized = 4;
+    jl_methtable_type->ninitialized = 5;
     jl_precompute_memoized_dt(jl_methtable_type, 1);
 
     jl_symbol_type->name = jl_new_typename_in(jl_symbol("Symbol"), core);
@@ -2152,6 +2145,9 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_array_symbol_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_symbol_type, jl_box_long(1));
     jl_array_uint8_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint8_type, jl_box_long(1));
     jl_array_int32_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_int32_type, jl_box_long(1));
+    jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally
+    jl_nonfunction_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any;
+    jl_type_type_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any;
 
     jl_expr_type =
         jl_new_datatype(jl_symbol("Expr"), core,
@@ -2466,18 +2462,18 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_svecset(jl_typename_type->types, 1, jl_module_type);
     jl_svecset(jl_typename_type->types, 6, jl_long_type);
     jl_svecset(jl_typename_type->types, 3, jl_type_type);
-    jl_svecset(jl_methtable_type->types, 3, jl_long_type);
-    jl_svecset(jl_methtable_type->types, 5, jl_module_type);
-    jl_svecset(jl_methtable_type->types, 6, jl_array_any_type);
+    jl_svecset(jl_methtable_type->types, 4, jl_long_type);
+    jl_svecset(jl_methtable_type->types, 6, jl_module_type);
+    jl_svecset(jl_methtable_type->types, 7, jl_array_any_type);
 #ifdef __LP64__
-    jl_svecset(jl_methtable_type->types, 7, jl_int64_type); // unsigned long
-    jl_svecset(jl_methtable_type->types, 8, jl_int64_type); // uint32_t plus alignment
+    jl_svecset(jl_methtable_type->types, 8, jl_int64_type); // unsigned long
+    jl_svecset(jl_methtable_type->types, 9, jl_int64_type); // uint32_t plus alignment
 #else
-    jl_svecset(jl_methtable_type->types, 7, jl_int32_type); // DWORD
-    jl_svecset(jl_methtable_type->types, 8, jl_int32_type); // uint32_t
+    jl_svecset(jl_methtable_type->types, 8, jl_int32_type); // DWORD
+    jl_svecset(jl_methtable_type->types, 9, jl_int32_type); // uint32_t
 #endif
-    jl_svecset(jl_methtable_type->types, 9, jl_uint8_type);
     jl_svecset(jl_methtable_type->types, 10, jl_uint8_type);
+    jl_svecset(jl_methtable_type->types, 11, jl_uint8_type);
     jl_svecset(jl_method_type->types, 13, jl_method_instance_type);
     jl_svecset(jl_method_instance_type->types, 5, jl_code_instance_type);
     jl_svecset(jl_code_instance_type->types, 8, jl_voidpointer_type);

src/jltypes.c

@@ -537,6 +537,7 @@ typedef struct _jl_methtable_t {
     JL_DATA_TYPE
     jl_sym_t *name; // sometimes a hack used by serialization to handle kwsorter
     jl_typemap_t *defs;
+    jl_array_t *leafcache;
     jl_typemap_t *cache;
     intptr_t max_args;  // max # of non-vararg arguments in a signature
     jl_value_t *kwsorter;  // keyword argument sorter function

src/julia.h

@@ -462,6 +462,8 @@ jl_datatype_t *jl_new_uninitialized_datatype(void);
 void jl_precompute_memoized_dt(jl_datatype_t *dt, int cacheable);
 jl_datatype_t *jl_wrap_Type(jl_value_t *t);  // x -> Type{x}
 jl_value_t *jl_wrap_vararg(jl_value_t *t, jl_value_t *n);
+jl_datatype_t *jl_lookup_cache_type_(jl_datatype_t *type);
+void jl_cache_type_(jl_datatype_t *type);
 void jl_assign_bits(void *dest, jl_value_t *bits) JL_NOTSAFEPOINT;
 void set_nth_field(jl_datatype_t *st, void *v, size_t i, jl_value_t *rhs) JL_NOTSAFEPOINT;
 jl_expr_t *jl_exprn(jl_sym_t *head, size_t n);
@@ -482,7 +484,7 @@ int jl_is_toplevel_only_expr(jl_value_t *e) JL_NOTSAFEPOINT;
 jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule);
 void jl_linenumber_to_lineinfo(jl_code_info_t *ci, jl_value_t *name);
 
-jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, int cache, size_t world);
+jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t world);
 jl_value_t *jl_gf_invoke(jl_value_t *types, jl_value_t *f, jl_value_t **args, size_t nargs);
 jl_method_instance_t *jl_lookup_generic(jl_value_t **args, uint32_t nargs, uint32_t callsite, size_t world) JL_ALWAYS_LEAFTYPE;
 JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous,
@@ -1047,13 +1049,12 @@ struct jl_typemap_info {
     jl_datatype_t **jl_contains; // the type that is being put in this
 };
 
-jl_typemap_entry_t *jl_typemap_insert(jl_typemap_t **cache,
-                                      jl_value_t *parent JL_PROPAGATES_ROOT,
-                                      jl_tupletype_t *type,
-                                      jl_tupletype_t *simpletype, jl_svec_t *guardsigs,
-                                      jl_value_t *newvalue, int8_t offs,
-                                      const struct jl_typemap_info *tparams,
-                                      size_t min_world, size_t max_world);
+void jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent,
+        jl_typemap_entry_t *newrec, int8_t offs,
+        const struct jl_typemap_info *tparams);
+jl_typemap_entry_t *jl_typemap_alloc(
+        jl_tupletype_t *type, jl_tupletype_t *simpletype, jl_svec_t *guardsigs,
+        jl_value_t *newvalue, size_t min_world, size_t max_world);
 
 struct jl_typemap_assoc {
     // inputs

src/julia_internal.h

@@ -314,12 +314,6 @@ static void jl_compile_all_defs(void)
     JL_GC_POP();
 }
 
-static int precompile_enq_all_cache__(jl_typemap_entry_t *l, void *closure)
-{
-    jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)l->func.linfo);
-    return 1;
-}
-
 static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closure)
 {
     assert(jl_is_method_instance(mi));
@@ -370,7 +364,6 @@ static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *c
 static void precompile_enq_all_specializations_(jl_methtable_t *mt, void *env)
 {
     jl_typemap_visitor(mt->defs, precompile_enq_all_specializations__, env);
-    jl_typemap_visitor(mt->cache, precompile_enq_all_cache__, env);
 }
 
 void jl_compile_now(jl_method_instance_t *mi);

src/precompile.c

@@ -872,18 +872,34 @@ static void jl_typemap_level_insert_(
     jl_typemap_list_insert_(map, &cache->linear, (jl_value_t*)cache, newrec, tparams);
 }
 
-jl_typemap_entry_t *jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent,
-                                      jl_tupletype_t *type,
-                                      jl_tupletype_t *simpletype, jl_svec_t *guardsigs,
-                                      jl_value_t *newvalue, int8_t offs,
-                                      const struct jl_typemap_info *tparams,
-                                      size_t min_world, size_t max_world)
+jl_typemap_entry_t *jl_typemap_alloc(
+        jl_tupletype_t *type, jl_tupletype_t *simpletype, jl_svec_t *guardsigs,
+        jl_value_t *newvalue, size_t min_world, size_t max_world)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
     assert(min_world > 0 && max_world > 0);
     if (!simpletype)
         simpletype = (jl_tupletype_t*)jl_nothing;
     jl_value_t *ttype = jl_unwrap_unionall((jl_value_t*)type);
+    assert(jl_is_tuple_type(ttype));
+    // compute the complexity of this type signature
+    int isva = jl_is_va_tuple((jl_datatype_t*)ttype);
+    int issimplesig = !jl_is_unionall(type); // a TypeVar environment needs a complex matching test
+    int isleafsig = issimplesig && !isva; // entirely leaf types don't need to be sorted
+    size_t i, l;
+    for (i = 0, l = jl_nparams(ttype); i < l && issimplesig; i++) {
+        jl_value_t *decl = jl_tparam(ttype, i);
+        if (jl_is_kind(decl))
+            isleafsig = 0; // Type{} may have a higher priority than a kind
+        else if (jl_is_type_type(decl))
+            isleafsig = 0; // Type{} may need special processing to compute the match
+        else if (jl_is_vararg_type(decl))
+            isleafsig = 0; // makes iteration easier when the endpoints are the same
+        else if (decl == (jl_value_t*)jl_any_type)
+            isleafsig = 0; // Any needs to go in the general cache
+        else if (!jl_is_concrete_type(decl)) // anything else needs to go through the general subtyping test
+            isleafsig = issimplesig = 0;
+    }
 
     jl_typemap_entry_t *newrec =
         (jl_typemap_entry_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_entry_t),
@@ -895,32 +911,19 @@ jl_typemap_entry_t *jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent,
     newrec->next = (jl_typemap_entry_t*)jl_nothing;
     newrec->min_world = min_world;
     newrec->max_world = max_world;
-    // compute the complexity of this type signature
-    newrec->va = jl_is_va_tuple((jl_datatype_t*)ttype);
-    newrec->issimplesig = !jl_is_unionall(type); // a TypeVar environment needs a complex matching test
-    newrec->isleafsig = newrec->issimplesig && !newrec->va; // entirely leaf types don't need to be sorted
-    JL_GC_PUSH1(&newrec);
-    assert(jl_is_tuple_type(ttype));
-    size_t i, l;
-    for (i = 0, l = jl_nparams(ttype); i < l && newrec->issimplesig; i++) {
-        jl_value_t *decl = jl_tparam(ttype, i);
-        if (jl_is_kind(decl))
-            newrec->isleafsig = 0; // Type{} may have a higher priority than a kind
-        else if (jl_is_type_type(decl))
-            newrec->isleafsig = 0; // Type{} may need special processing to compute the match
-        else if (jl_is_vararg_type(decl))
-            newrec->isleafsig = 0; // makes iteration easier when the endpoints are the same
-        else if (decl == (jl_value_t*)jl_any_type)
-            newrec->isleafsig = 0; // Any needs to go in the general cache
-        else if (!jl_is_concrete_type(decl)) // anything else needs to go through the general subtyping test
-            newrec->isleafsig = newrec->issimplesig = 0;
-    }
-    // TODO: assert that guardsigs == jl_emptysvec && simplesig == jl_nothing if isleafsig and optimize with that knowledge?
-    jl_typemap_insert_generic(*cache, cache, parent, newrec, offs, tparams);
-    JL_GC_POP();
+    newrec->va = isva;
+    newrec->issimplesig = issimplesig;
+    newrec->isleafsig = isleafsig;
     return newrec;
 }
 
+void jl_typemap_insert(jl_typemap_t **cache, jl_value_t *parent,
+        jl_typemap_entry_t *newrec, int8_t offs,
+        const struct jl_typemap_info *tparams)
+{
+    jl_typemap_insert_generic(*cache, cache, parent, newrec, offs, tparams);
+}
+
 static void jl_typemap_list_insert_sorted(
         jl_typemap_t *map, jl_typemap_entry_t **pml, jl_value_t *parent,
         jl_typemap_entry_t *newrec, const struct jl_typemap_info *tparams)

src/typemap.c

@@ -397,15 +397,15 @@ end
 # Returns the return type. example: get_type(:(Base.strip("", ' ')), Main) returns (String, true)
 function try_get_type(sym::Expr, fn::Module)
     val, found = get_value(sym, fn)
-    found && return Base.typesof(val).parameters[1], found
+    found && return Core.Typeof(val), found
     if sym.head === :call
         # getfield call is special cased as the evaluation of getfield provides good type information,
         # is inexpensive and it is also performed in the complete_symbol function.
         a1 = sym.args[1]
         if isa(a1,GlobalRef) && isconst(a1.mod,a1.name) && isdefined(a1.mod,a1.name) &&
             eval(a1) === Core.getfield
             val, found = get_value_getfield(sym, Main)
-            return found ? Base.typesof(val).parameters[1] : Any, found
+            return found ? Core.Typeof(val) : Any, found
         end
         return get_type_call(sym)
     elseif sym.head === :thunk
@@ -432,7 +432,7 @@ end
 
 function get_type(sym, fn::Module)
     val, found = get_value(sym, fn)
-    return found ? Base.typesof(val).parameters[1] : Any, found
+    return found ? Core.Typeof(val) : Any, found
 end
 
 # Method completion on function call expression that look like :(max(1))

stdlib/REPL/src/REPLCompletions.jl

@@ -390,6 +390,7 @@ end
         t = Timer(0) do t
             tc[] += 1
         end
+        Libc.systemsleep(0.005)
         @test isopen(t)
         Base.process_events()
         @test !isopen(t)
@@ -402,6 +403,7 @@ end
         t = Timer(0) do t
             tc[] += 1
         end
+        Libc.systemsleep(0.005)
         @test isopen(t)
         close(t)
         @test !isopen(t)

test/channels.jl

@@ -507,7 +507,6 @@ f18888() = nothing
 let
     world = Core.Compiler.get_world_counter()
     m = first(methods(f18888, Tuple{}))
-    @test isempty(m.specializations)
     ft = typeof(f18888)
 
     code_typed(f18888, Tuple{}; optimize=false)