Improve spill performance: Disable re-validation of spilled files (#15454)

* add skip_validation and comment

* add benchmark

* fmt

* update sqllogictests and fmt Cargo.toml

* use SpillManager in benchmark

* fix clippy

* upgrade dependencies

* Update comment

Co-authored-by: Daniël Heres <[email protected]>

---------

Co-authored-by: zeb <[email protected]>
Co-authored-by: Daniël Heres <[email protected]>
Co-authored-by: Andrew Lamb <[email protected]>

Author: 4 people

Cargo.lock

@@ -87,19 +87,19 @@ ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
 apache-avro = { version = "0.17", default-features = false }
-arrow = { version = "54.2.1", features = [
+arrow = { version = "54.3.0", features = [
     "prettyprint",
     "chrono-tz",
 ] }
-arrow-buffer = { version = "54.1.0", default-features = false }
-arrow-flight = { version = "54.2.1", features = [
+arrow-buffer = { version = "54.3.0", default-features = false }
+arrow-flight = { version = "54.3.0", features = [
     "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "54.2.0", default-features = false, features = [
+arrow-ipc = { version = "54.3.0", default-features = false, features = [
     "lz4",
 ] }
-arrow-ord = { version = "54.1.0", default-features = false }
-arrow-schema = { version = "54.1.0", default-features = false }
+arrow-ord = { version = "54.3.0", default-features = false }
+arrow-schema = { version = "54.3.0", default-features = false }
 async-trait = "0.1.88"
 bigdecimal = "0.4.7"
 bytes = "1.10"
@@ -149,7 +149,7 @@ itertools = "0.14"
 log = "^0.4"
 object_store = { version = "0.11.0", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "54.2.1", default-features = false, features = [
+parquet = { version = "54.3.0", default-features = false, features = [
     "arrow",
     "async",
     "object_store",

Cargo.toml

@@ -72,6 +72,7 @@ insta = { workspace = true }
 rand = { workspace = true }
 rstest = { workspace = true }
 rstest_reuse = "0.7.0"
+tempfile = "3.19.1"
 tokio = { workspace = true, features = [
     "rt-multi-thread",
     "fs",
@@ -81,3 +82,7 @@ tokio = { workspace = true, features = [
 [[bench]]
 harness = false
 name = "partial_ordering"
+
+[[bench]]
+harness = false
+name = "spill_io"