Merge branch 'main' into improve_combine_hashes

Author: alamb

.github/workflows/audit.yml

@@ -42,7 +42,7 @@ jobs:
  steps:
  - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
  - name: Install cargo-audit
- uses: taiki-e/install-action@d850aa816998e5cf15f67a78c7b933f2a5033f8a # v2.63.3
+ uses: taiki-e/install-action@69e777b377e4ec209ddad9426ae3e0c1008b0ef3 # v2.64.0
  with:
  tool: cargo-audit
  - name: Run audit check

.github/workflows/rust.yml

@@ -421,7 +421,7 @@ jobs:
  sudo apt-get update -qq
  sudo apt-get install -y -qq clang
  - name: Setup wasm-pack
- uses: taiki-e/install-action@d850aa816998e5cf15f67a78c7b933f2a5033f8a # v2.63.3
+ uses: taiki-e/install-action@69e777b377e4ec209ddad9426ae3e0c1008b0ef3 # v2.64.0
  with:
  tool: wasm-pack
  - name: Run tests with headless mode
@@ -724,7 +724,7 @@ jobs:
  - name: Setup Rust toolchain
  uses: ./.github/actions/setup-builder
  - name: Install cargo-msrv
- uses: taiki-e/install-action@d850aa816998e5cf15f67a78c7b933f2a5033f8a # v2.63.3
+ uses: taiki-e/install-action@69e777b377e4ec209ddad9426ae3e0c1008b0ef3 # v2.64.0
  with:
  tool: cargo-msrv
 

Cargo.lock

@@ -71,7 +71,7 @@ resolver = "2"
 
 [workspace.package]
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
-edition = "2021"
+edition = "2024"
 homepage = "https://datafusion.apache.org"
 license = "Apache-2.0"
 readme = "README.md"

Cargo.toml

@@ -33,16 +33,16 @@ use datafusion::datasource::listing::{
 };
 use datafusion::datasource::{MemTable, TableProvider};
 use datafusion::error::Result;
+use datafusion::execution::SessionStateBuilder;
 use datafusion::execution::memory_pool::FairSpillPool;
 use datafusion::execution::runtime_env::RuntimeEnvBuilder;
-use datafusion::execution::SessionStateBuilder;
 use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::{collect, displayable};
 use datafusion::prelude::*;
 use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult};
 use datafusion_common::instant::Instant;
 use datafusion_common::utils::get_available_parallelism;
-use datafusion_common::{exec_err, DEFAULT_PARQUET_EXTENSION};
+use datafusion_common::{DEFAULT_PARQUET_EXTENSION, exec_err};
 use datafusion_common::{human_readable_size, units};
 
 #[derive(Debug, StructOpt)]

benchmarks/src/bin/external_aggr.rs

@@ -199,21 +199,18 @@ fn run_query(args: &[String], results: &mut Vec<QueryResult>) -> Result<()> {
 
  // Look for lines that contain execution time / memory stats
  while let Some(line) = iter.next() {
- if let Some((query, duration_ms)) = parse_query_time(line) {
- if let Some(next_line) = iter.peek() {
- if let Some((peak_rss, peak_commit, page_faults)) =
- parse_vm_line(next_line)
- {
- results.push(QueryResult {
- query,
- duration_ms,
- peak_rss,
- peak_commit,
- page_faults,
- });
- break;
- }
- }
+ if let Some((query, duration_ms)) = parse_query_time(line)
+ && let Some(next_line) = iter.peek()
+ && let Some((peak_rss, peak_commit, page_faults)) = parse_vm_line(next_line)
+ {
+ results.push(QueryResult {
+ query,
+ duration_ms,
+ peak_rss,
+ peak_commit,
+ page_faults,
+ });
+ break;
  }
  }
 

benchmarks/src/bin/mem_profile.rs

@@ -25,22 +25,22 @@ use arrow::array::Array;
 use arrow::datatypes::DataType;
 use arrow::record_batch::RecordBatch;
 use datafusion::common::{Result, ScalarValue};
-use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::file_format::FileFormat;
+use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::listing::{ListingOptions, ListingTableUrl};
-use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::execution::TaskContext;
-use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
 use datafusion::prelude::*;
 use datafusion_common::instant::Instant;
 use futures::TryStreamExt;
 use object_store::ObjectStore;
-use parquet::arrow::async_writer::ParquetObjectWriter;
 use parquet::arrow::AsyncArrowWriter;
+use parquet::arrow::async_writer::ParquetObjectWriter;
+use rand::Rng;
 use rand::distr::Alphanumeric;
 use rand::rngs::ThreadRng;
-use rand::Rng;
 use structopt::StructOpt;
 use tokio::runtime::Runtime;
 use tokio_util::sync::CancellationToken;

benchmarks/src/cancellation.rs

@@ -19,7 +19,7 @@ use std::fs;
 use std::io::ErrorKind;
 use std::path::{Path, PathBuf};
 
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt, QueryResult};
+use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};
 use datafusion::logical_expr::{ExplainFormat, ExplainOption};
 use datafusion::{
  error::{DataFusionError, Result},
@@ -156,7 +156,9 @@ impl RunOpt {
  .any(|opt| opt.contains("prefer_existing_sort=true"));
 
  if !has_prefer_sort {
- println!("ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true");
+ println!(
+ "ℹ️ Consider using -c datafusion.optimizer.prefer_existing_sort=true"
+ );
  println!("ℹ️ to optimize queries while maintaining parallelism");
  }
  }

benchmarks/src/clickbench.rs

@@ -20,11 +20,11 @@
 //! - [H2O AI Benchmark](https://duckdb.org/2023/04/14/h2oai.html)
 //! - [Extended window function benchmark](https://duckdb.org/2024/06/26/benchmarks-over-time.html#window-functions-benchmark)
 
-use crate::util::{print_memory_stats, BenchmarkRun, CommonOpt};
+use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};
 use datafusion::logical_expr::{ExplainFormat, ExplainOption};
 use datafusion::{error::Result, prelude::SessionContext};
 use datafusion_common::{
- exec_datafusion_err, instant::Instant, internal_err, DataFusionError, TableReference,
+ DataFusionError, TableReference, exec_datafusion_err, instant::Instant, internal_err,
 };
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;

benchmarks/src/h2o.rs

@@ -19,7 +19,7 @@ use crate::util::{BenchmarkRun, CommonOpt, QueryResult};
 use datafusion::physical_plan::execute_stream;
 use datafusion::{error::Result, prelude::SessionContext};
 use datafusion_common::instant::Instant;
-use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError};
+use datafusion_common::{DataFusionError, exec_datafusion_err, exec_err};
 use structopt::StructOpt;
 
 use futures::StreamExt;