Improve spill performance: Disable re-validation of spilled files (apache#15454)

* add skip_validation and comment * add benchmark * fmt * update sqllogictests and fmt Cargo.toml * use SpillManager in benchmark * fix clippy * upgrade dependencies * Update comment Co-authored-by: Daniël Heres <danielheres@gmail.com> --------- Co-authored-by: zeb <zeb@zeb-2.local> Co-authored-by: Daniël Heres <danielheres@gmail.com> Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Author: 4 people

Cargo.lock

@@ -87,19 +87,19 @@ ahash = { version = "0.8", default-features = false, features = [
  "runtime-rng",
 ] }
 apache-avro = { version = "0.17", default-features = false }
-arrow = { version = "54.2.1", features = [
+arrow = { version = "54.3.0", features = [
  "prettyprint",
  "chrono-tz",
 ] }
-arrow-buffer = { version = "54.1.0", default-features = false }
-arrow-flight = { version = "54.2.1", features = [
+arrow-buffer = { version = "54.3.0", default-features = false }
+arrow-flight = { version = "54.3.0", features = [
  "flight-sql-experimental",
 ] }
-arrow-ipc = { version = "54.2.0", default-features = false, features = [
+arrow-ipc = { version = "54.3.0", default-features = false, features = [
  "lz4",
 ] }
-arrow-ord = { version = "54.1.0", default-features = false }
-arrow-schema = { version = "54.1.0", default-features = false }
+arrow-ord = { version = "54.3.0", default-features = false }
+arrow-schema = { version = "54.3.0", default-features = false }
 async-trait = "0.1.88"
 bigdecimal = "0.4.7"
 bytes = "1.10"
@@ -149,7 +149,7 @@ itertools = "0.14"
 log = "^0.4"
 object_store = { version = "0.11.0", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "54.2.1", default-features = false, features = [
+parquet = { version = "54.3.0", default-features = false, features = [
  "arrow",
  "async",
  "object_store",

Cargo.toml

@@ -72,6 +72,7 @@ insta = { workspace = true }
 rand = { workspace = true }
 rstest = { workspace = true }
 rstest_reuse = "0.7.0"
+tempfile = "3.19.1"
 tokio = { workspace = true, features = [
  "rt-multi-thread",
  "fs",
@@ -81,3 +82,7 @@ tokio = { workspace = true, features = [
 [[bench]]
 harness = false
 name = "partial_ordering"
+
+[[bench]]
+harness = false
+name = "spill_io"