Skip to content

Commit ee525ff

Browse files
committed
Load framework test files in deterministic order
`Dir.glob` doesn't guarantee the order of its results: https://ruby-doc.org/core-2.6.5/Dir.html#method-c-glob > Case sensitivity depends on your system (File::FNM_CASEFOLD is > ignored), as does the order in which the results are returned. Minitest stores a list of all test cases in the order that they were defined; it shuffles them before they're run, but doesn't sort them: https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L1048 https://github.com/seattlerb/minitest/blob/v5.13.0/lib/minitest.rb#L156 This means that the order in which framework tests run is platform dependent, and running a test command that failed in CI locally won't necessarily reproduce the error, even when the same seed is provided. `Rake::FileList` resolves glob patterns to a sorted list of files: https://github.com/ruby/rake/blob/v13.0.1/lib/rake/file_list.rb#L408 By using `Rake::FileList` instead of `Dir.glob`, framework tests will always run in the same order when given the same seed, and reproducing order dependent CI failures will be easier.
1 parent 3578f69 commit ee525ff

File tree

5 files changed

+8
-8
lines changed

5 files changed

+8
-8
lines changed

actioncable/Rakefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ task :package
1212

1313
Rake::TestTask.new do |t|
1414
t.libs << "test"
15-
t.test_files = Dir.glob("#{__dir__}/test/**/*_test.rb")
15+
t.test_files = FileList["#{__dir__}/test/**/*_test.rb"]
1616
t.warning = true
1717
t.verbose = true
1818
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

actionpack/Rakefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
require "rake/testtask"
44

5-
test_files = Dir.glob("test/**/*_test.rb")
5+
test_files = FileList["test/**/*_test.rb"]
66

77
desc "Default Task"
88
task default: :test

actionview/Rakefile

+3-3
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ namespace :test do
2323

2424
Rake::TestTask.new(:template) do |t|
2525
t.libs << "test"
26-
t.test_files = Dir.glob("test/template/**/*_test.rb")
26+
t.test_files = FileList["test/template/**/*_test.rb"]
2727
t.warning = true
2828
t.verbose = true
2929
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@@ -81,7 +81,7 @@ namespace :test do
8181
# Active Record Integration Tests
8282
Rake::TestTask.new(:active_record) do |t|
8383
t.libs << "test"
84-
t.test_files = Dir.glob("test/activerecord/*_test.rb")
84+
t.test_files = FileList["test/activerecord/*_test.rb"]
8585
t.warning = true
8686
t.verbose = true
8787
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)
@@ -90,7 +90,7 @@ namespace :test do
9090
# Action Pack Integration Tests
9191
Rake::TestTask.new(:action_pack) do |t|
9292
t.libs << "test"
93-
t.test_files = Dir.glob("test/actionpack/**/*_test.rb")
93+
t.test_files = FileList["test/actionpack/**/*_test.rb"]
9494
t.warning = true
9595
t.verbose = true
9696
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

activemodel/Rakefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ task :package
88

99
Rake::TestTask.new do |t|
1010
t.libs << "test"
11-
t.test_files = Dir.glob("#{__dir__}/test/cases/**/*_test.rb")
11+
t.test_files = FileList["#{__dir__}/test/cases/**/*_test.rb"]
1212
t.warning = true
1313
t.verbose = true
1414
t.ruby_opts = ["--dev"] if defined?(JRUBY_VERSION)

activerecord/Rakefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,9 @@ end
5252
Rake::TestTask.new(adapter => "#{adapter}:env") { |t|
5353
adapter_short = adapter == "db2" ? adapter : adapter[/^[a-z0-9]+/]
5454
t.libs << "test"
55-
t.test_files = (Dir.glob("test/cases/**/*_test.rb").reject {
55+
t.test_files = (FileList["test/cases/**/*_test.rb"].reject {
5656
|x| x.include?("/adapters/")
57-
} + Dir.glob("test/cases/adapters/#{adapter_short}/**/*_test.rb"))
57+
} + FileList["test/cases/adapters/#{adapter_short}/**/*_test.rb"])
5858

5959
t.warning = true
6060
t.verbose = true

0 commit comments

Comments
 (0)