Skip to content

WIP: Fixing Julia tests #508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

WIP: Fixing Julia tests #508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a583832
Fixing Julia tests
goerch Dec 10, 2021
7606d47
Trying an alternative fix for evaluate_limited!
goerch Dec 10, 2021
064c240
Fixing run_test_by_eval and increasing test isolation
goerch Dec 11, 2021
4d52046
Increasing test isolation
goerch Dec 12, 2021
9bc95a4
Assertion hit by subtype tests commented
goerch Dec 13, 2021
f183577
Optional unlimited evaluation of Julia tests
goerch Dec 17, 2021
50f35b6
Reasonable performance with time budgeting call sites...
goerch Dec 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions results.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
Julia Version 1.8.0-DEV.1169
Commit f2f31336a1 (2021-12-22 11:27 UTC)
Platform Info:
OS: Windows (x86_64-w64-mingw32)
CPU: Intel(R) Core(TM) i7-10710U CPU @ 1.10GHz
WORD_SIZE: 64
LIBM: libopenlibm
LLVM: libLLVM-12.0.1 (ORCJIT, skylake)
Test run at: 2021-12-23T13:30:26.035

Maximum number of statements per lowered expression: 10000000

| Test file | Passes | Fails | Errors | Broken | Aborted blocks |
| --------- | ------:| -----:| ------:| ------:| --------------:|
| ambiguous | 107 | 0 | 0 | 2 | 0 |
| subarray | 318316 | 0 | 0 | 0 | 0 |
| strings/basic | 87674 | 0 | 0 | 0 | 0 |
| strings/search | 876 | 0 | 0 | 0 | 0 |
| strings/util | 1147 | 0 | 0 | 0 | 0 |
| strings/io | 12764 | 0 | 0 | 0 | 0 |
| strings/types | 2302691 | 0 | 0 | 0 | 0 |
| unicode/utf8 | 19 | 0 | 0 | 0 | 0 |
| core | X | X | X | X | X |
| worlds | ☠️ | ☠️ | ☠️ | ☠️ | ☠️ |
| atomics | 3444 | 0 | 0 | 0 | 0 |
| keywordargs | 151 | 0 | 0 | 0 | 0 |
| numbers | 1578757 | 1 | 0 | 2 | 0 |
| subtype | 337674 | 0 | 0 | 19 | 0 |
| char | 1628 | 0 | 0 | 0 | 0 |
| triplequote | 29 | 0 | 0 | 0 | 0 |
| intrinsics | 301 | 0 | 0 | 0 | 0 |
| dict | 144420 | 0 | 0 | 0 | 0 |
| hashing | 12519 | 0 | 0 | 0 | 0 |
| iobuffer | 209 | 0 | 0 | 0 | 0 |
| staged | 59 | 5 | 0 | 0 | 0 |
| offsetarray | 484 | 3 | 0 | 3 | 0 |
| arrayops | 2031 | 0 | 0 | 2 | 0 |
| tuple | 625 | 0 | 0 | 0 | 0 |
| reduce | 8580 | 2 | 0 | 0 | 0 |
| reducedim | 865 | 0 | 0 | 0 | 0 |
| abstractarray | 55892 | 0 | 0 | 24795 | 0 |
| intfuncs | 227902 | 0 | 0 | 0 | 0 |
| simdloop | 240 | 0 | 0 | 0 | 0 |
| vecelement | 678 | 0 | 0 | 0 | 0 |
| rational | 98639 | 0 | 0 | 1 | 0 |
| bitarray | 914001 | 1 | 0 | 0 | 0 |
| copy | 533 | 0 | 0 | 0 | 0 |
| math | 148967 | 1 | 11 | 0 | 0 |
| fastmath | 946 | 0 | 0 | 0 | 0 |
| functional | 98 | 0 | 0 | 0 | 0 |
| iterators | 10164 | 0 | 0 | 0 | 0 |
| operators | 13039 | 0 | 1 | 0 | 0 |
| ordering | 37 | 0 | 0 | 0 | 0 |
| path | 1051 | 0 | 0 | 12 | 0 |
| ccall | X | X | X | X | X |
| parse | 16098 | 0 | 0 | 0 | 0 |
| loading | 168472 | 2 | 1 | 0 | 0 |
| gmp | 2357 | 0 | 0 | 0 | 0 |
| sorting | 16096 | 0 | 0 | 10 | 0 |
| spawn | 230 | 1 | 1 | 4 | 0 |
| backtrace | 13 | 9 | 16 | 1 | 0 |
| exceptions | 43 | 21 | 6 | 0 | 0 |
| file | X | X | X | X | X |
| read | 3870 | 0 | 0 | 0 | 0 |
| version | 2452 | 0 | 0 | 0 | 0 |
| namedtuple | 215 | 0 | 0 | 0 | 0 |
| mpfr | 1135 | 0 | 0 | 1 | 0 |
| broadcast | 509 | 2 | 0 | 0 | 0 |
| complex | 8432 | 0 | 0 | 5 | 0 |
| floatapprox | 49 | 0 | 0 | 0 | 0 |
| reflection | X | X | X | X | X |
| regex | 130 | 0 | 0 | 0 | 0 |
| float16 | 237 | 0 | 0 | 0 | 0 |
| combinatorics | 170 | 0 | 0 | 0 | 0 |
| sysinfo | 4 | 0 | 0 | 0 | 0 |
| env | 94 | 0 | 0 | 0 | 0 |
| rounding | 112720 | 0 | 0 | 0 | 0 |
| ranges | 12110652 | 2 | 0 | 327682 | 0 |
| mod2pi | 80 | 0 | 0 | 0 | 0 |
| euler | 12 | 0 | 0 | 0 | 0 |
| show | 128879 | 0 | 0 | 8 | 0 |
| client | 2 | 3 | 0 | 0 | 0 |
| errorshow | X | X | X | X | X |
| sets | 3594 | 0 | 0 | 1 | 0 |
| goto | 19 | 0 | 0 | 0 | 0 |
| llvmcall2 | 7 | 0 | 0 | 0 | 0 |
| ryu | 31215 | 0 | 0 | 0 | 0 |
| some | 72 | 0 | 0 | 0 | 0 |
| meta | 69 | 0 | 0 | 0 | 0 |
| stacktraces | X | X | X | X | X |
| docs | 237 | 1 | 0 | 0 | 0 |
| misc | X | X | X | X | X |
| threads | X | X | X | X | X |
| stress | 0 | 0 | 0 | 0 | 0 |
| binaryplatforms | 341 | 0 | 0 | 0 | 0 |
| atexit | 40 | 0 | 0 | 0 | 0 |
| enums | 99 | 0 | 0 | 0 | 0 |
| cmdlineargs | 255 | 0 | 0 | 3 | 0 |
| int | 524698 | 0 | 0 | 0 | 0 |
| interpreter | 3 | 0 | 0 | 0 | 0 |
| checked | 1239 | 0 | 0 | 0 | 0 |
| bitset | 195 | 0 | 0 | 0 | 0 |
| floatfuncs | 215 | 0 | 0 | 0 | 0 |
| precompile | X | X | X | X | X |
| boundscheck | X | X | X | X | X |
| error | 31 | 0 | 0 | 0 | 0 |
| cartesian | 343 | 0 | 0 | 3 | 0 |
| osutils | 57 | 0 | 0 | 0 | 0 |
| channels | 252 | 6 | 0 | 0 | 0 |
| iostream | 50 | 0 | 0 | 0 | 0 |
| secretbuffer | 27 | 0 | 0 | 0 | 0 |
| specificity | X | X | X | X | X |
| reinterpretarray | 232 | 0 | 0 | 0 | 0 |
| syntax | X | X | X | X | X |
| corelogging | 230 | 1 | 0 | 0 | 0 |
| missing | 565 | 0 | 0 | 1 | 0 |
| asyncmap | 304 | 0 | 0 | 0 | 0 |
| smallarrayshrink | 36 | 0 | 0 | 0 | 0 |
| opaque_closure | X | X | X | X | X |
| filesystem | 4 | 0 | 0 | 0 | 0 |
| download | X | X | X | X | X |
12 changes: 8 additions & 4 deletions src/construct.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const compiled_modules = Set{Module}()

const junk_framedata = FrameData[] # to allow re-use of allocated memory (this is otherwise a bottleneck)
const junk_frames = Frame[]
disable_recycle() = true
debug_recycle() = false
@noinline function _check_frame_not_in_junk(frame)
@assert frame.framedata ∉ junk_framedata
Expand All @@ -43,8 +44,8 @@ end

@inline function recycle(frame)
debug_recycle() && _check_frame_not_in_junk(frame)
push!(junk_framedata, frame.framedata)
push!(junk_frames, frame)
disable_recycle() || push!(junk_framedata, frame.framedata)
disable_recycle() || push!(junk_frames, frame)
end

function return_from(frame::Frame)
Expand Down Expand Up @@ -264,14 +265,16 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e
ng, ns = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any}), length(src.slotflags)
if length(junk_framedata) > 0
olddata = pop!(junk_framedata)
locals, ssavalues, sparams = olddata.locals, olddata.ssavalues, olddata.sparams
locals, ssavalues, times, sparams = olddata.locals, olddata.ssavalues, olddata.times, olddata.sparams
exception_frames, last_reference = olddata.exception_frames, olddata.last_reference
last_exception = olddata.last_exception
callargs = olddata.callargs
resize!(locals, ns)
fill!(locals, nothing)
resize!(ssavalues, 0)
resize!(ssavalues, ng)
resize!(times, 0)
resize!(times, ng)
# for check_isdefined to work properly, we need sparams to start out unassigned
resize!(sparams, 0)
empty!(exception_frames)
Expand All @@ -280,6 +283,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e
else
locals = Vector{Union{Nothing,Some{Any}}}(nothing, ns)
ssavalues = Vector{Any}(undef, ng)
times = Vector{UInt64}(undef, ng)
sparams = Vector{Any}(undef, 0)
exception_frames = Int[]
last_reference = Vector{Int}(undef, ns)
Expand Down Expand Up @@ -310,7 +314,7 @@ function prepare_framedata(framecode, argvals::Vector{Any}, lenv::SimpleVector=e
isa(T, TypeVar) && continue # only fill concrete types
sparams[i] = T
end
FrameData(locals, ssavalues, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs)
FrameData(locals, ssavalues, times, sparams, exception_frames, last_exception, caller_will_catch_err, last_reference, callargs)
end

"""
Expand Down
3 changes: 2 additions & 1 deletion src/interpret.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,8 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev
# show_stackloc(frame)
# @show node
# end
@assert is_leaf(frame)
# The following assertion is hit by Julia subtype tests
# @assert is_leaf(frame)
@static VERSION >= v"1.8.0-DEV.370" && coverage_visit_line!(frame)
local rhs
# For debugging:
Expand Down
1 change: 1 addition & 0 deletions src/types.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ Important fields:
struct FrameData
locals::Vector{Union{Nothing,Some{Any}}}
ssavalues::Vector{Any}
times::Vector{UInt64}
sparams::Vector{Any}
exception_frames::Vector{Int}
last_exception::Base.RefValue{Any}
Expand Down
35 changes: 12 additions & 23 deletions test/juliatests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ if !isdefined(Main, :read_and_parse)
include("utils.jl")
end

const juliadir = dirname(dirname(Sys.BINDIR))
const testdir = joinpath(juliadir, "test")
const juliadir = dirname(Sys.BINDIR)
const testdir = joinpath(juliadir, "share/julia/test")
if isdir(testdir)
include(joinpath(testdir, "choosetests.jl"))
else
Expand All @@ -28,7 +28,7 @@ function test_path(test)
end
end

nstmts = 10^4 # very quick, aborts a lot
nstmts = 10^7 # very quick, aborts a lot
outputfile = "results.md"
i = 1
while i <= length(ARGS)
Expand All @@ -45,11 +45,12 @@ while i <= length(ARGS)
end
end

tests, _, exit_on_error, seed = choosetests(ARGS)
tests, _, exit_on_error, seed = choosetests(["--skip", "llvmcall", "compiler", "stdlib", ARGS...])

function spin_up_worker()
p = addprocs(1)[1]
remotecall_wait(include, p, "utils.jl")
remotecall_wait(include, p, "test/utils.jl")
remotecall_wait(JuliaInterpreter.clear_caches, p)
remotecall_wait(configure_test, p)
return p
end
Expand All @@ -58,29 +59,16 @@ function spin_up_workers(n)
procs = addprocs(n)
@sync begin
@async for p in procs
remotecall_wait(include, p, "utils.jl")
remotecall_wait(include, p, "test/utils.jl")
remotecall_wait(JuliaInterpreter.clear_caches, p)
remotecall_wait(configure_test, p)
end
end
return procs
end

# Really, we're just going to skip all the tests that run on node1
const node1_tests = String[]
function move_to_node1(t)
if t in tests
splice!(tests, findfirst(isequal(t), tests))
push!(node1_tests, t)
end
nothing
end
move_to_node1("precompile")
move_to_node1("SharedArrays")
move_to_node1("stress")
move_to_node1("Distributed")

@testset "Julia tests" begin
nworkers = min(Sys.CPU_THREADS, length(tests))
nworkers = Threads.nthreads()
println("Using $nworkers workers")
results = Dict{String,Any}()
tests0 = copy(tests)
Expand All @@ -99,7 +87,8 @@ move_to_node1("Distributed")
try
resp = disable_sigint() do
p = spin_up_worker()
result = remotecall_fetch(run_test_by_eval, p, test, fullpath, nstmts)
# result = remotecall_fetch(run_test_by_limited_eval, p, test, fullpath, nstmts)
result = remotecall_fetch(run_test_by_eval, p, test, fullpath)
rmprocs(p; waitfor=5)
result
end
Expand Down Expand Up @@ -137,7 +126,7 @@ move_to_node1("Distributed")
catch
end
end
foreach(wait, all_tasks)
foreach(wait, [task for task in all_tasks if isa(task, Task)])
end

open(outputfile, "w") do io
Expand Down
Loading