hook up stdlib to the standard test running system (#24701)

* hook up stdlib to the standard test running system

* fixups from review

Author: KristofferC

test/Makefile

@@ -1,11 +1,13 @@
 SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 JULIAHOME := $(abspath $(SRCDIR)/..)
 BUILDDIR := .
+STDLIBDIR := $(abspath $(JULIAHOME)/stdlib)
 include $(JULIAHOME)/Make.inc
 # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE
 
 TESTGROUPS = linalg sparse unicode strings dates
-TESTS = all $(TESTGROUPS) \
+TESTS = all stdlib $(TESTGROUPS) \
+        $(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \
 		$(filter-out TestHelpers runtests testdefs, \
 			$(patsubst $(SRCDIR)/%.jl,%,$(wildcard $(SRCDIR)/*.jl))) \
 		$(foreach group,$(TESTGROUPS), \

test/choosetests.jl

@@ -1,5 +1,8 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+const STDLIB_DIR = joinpath(JULIA_HOME, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
+const STDLIBS = readdir(STDLIB_DIR)
+
 @doc """
 
 `tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be
@@ -165,6 +168,20 @@ function choosetests(choices = [])
         filter!(x -> !(x in net_required_for), tests)
     end
 
+    if "stdlib" in skip_tests
+        filter!(x -> (x != "stdlib" && !(x in STDLIBS)) , tests)
+    elseif "stdlib" in tests
+        filter!(x -> (x != "stdlib" && !(x in STDLIBS)) , tests)
+        prepend!(tests, STDLIBS)
+    end
+
+    if startswith(string(Sys.ARCH), "arm")
+        # Remove profile from default tests on ARM since it currently segfaults
+        # Allow explicitly adding it for testing
+        warn("Skipping Profile tests")
+        filter!(x -> (x != "Profile"), tests)
+    end
+
     filter!(x -> !(x in skip_tests), tests)
 
     tests, net_on, exit_on_error, seed

test/runtests.jl

@@ -13,15 +13,30 @@ else
     typemax(Csize_t)
 end
 
+function test_path(test)
+    if test in STDLIBS
+        test_file = joinpath(STDLIB_DIR, test, "test", "runtests")
+        if !isfile(test_file * ".jl")
+            error("Standard library $test did not provide a `test/runtests.jl` file")
+        end
+        return test_file
+    else
+        return test
+    end
+end
+
 const node1_tests = String[]
 function move_to_node1(t)
     if t in tests
         splice!(tests, findfirst(equalto(t), tests))
         push!(node1_tests, t)
     end
 end
+
 # Base.compile only works from node 1, so compile test is handled specially
 move_to_node1("compile")
+move_to_node1("SharedArrays")
+
 # In a constrained memory environment, run the "distributed" test after all other tests
 # since it starts a lot of workers and can easily exceed the maximum memory
 max_worker_rss != typemax(Csize_t) && move_to_node1("distributed")
@@ -55,7 +70,7 @@ cd(dirname(@__FILE__)) do
                     local resp
                     wrkr = p
                     try
-                        resp = remotecall_fetch(runtests, wrkr, test; seed=seed)
+                        resp = remotecall_fetch(runtests, wrkr, test, test_path(test); seed=seed)
                     catch e
                         resp = [e]
                     end
@@ -105,9 +120,11 @@ cd(dirname(@__FILE__)) do
         # and either way, append the results
         # to the overall aggregator
         n > 1 && print("\tFrom worker 1:\t")
+        isolate = true
+        t == "SharedArrays" && (isolate = false)
         local resp
         try
-            resp = eval(Expr(:call, () -> runtests(t, seed=seed))) # runtests is defined by the include above
+            resp = eval(Expr(:call, () -> runtests(t, test_path(t), isolate, seed=seed))) # runtests is defined by the include above
         catch e
             resp = [e]
         end

test/stdlib.jl

@@ -2,7 +2,7 @@
 
 using Test
 
-function runtests(name, isolate=true; seed=nothing)
+function runtests(name, path, isolate=true; seed=nothing)
     old_print_setting = Test.TESTSET_PRINT_ENABLE[]
     Test.TESTSET_PRINT_ENABLE[] = false
     try
@@ -19,7 +19,7 @@ function runtests(name, isolate=true; seed=nothing)
             @timed @testset $"$name" begin
                 # srand(nothing) will fail
                 $seed != nothing && srand($seed)
-                include($"$name.jl")
+                include($"$path.jl")
             end
         end
         res_and_time_data = eval(m, ex)