Skip to content

perf(catalog): cut datafusion-catalog compile time ~6x #24325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Dandandan merged 2 commits into from
Aug 13, 2026
Merged

perf(catalog): cut datafusion-catalog compile time ~6x #24325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5766348
perf(catalog): cut `datafusion-catalog` compile time ~4.6x
Dandandan Aug 13, 2026
c7d4714
perf(catalog): convert the last `async fn` in the `ListingTable`-adja…
Dandandan Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 65 additions & 20 deletions datafusion/catalog/src/cte_worktable.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
//! CteWorkTable implementation used for recursive queries

use std::borrow::Cow;
use std::future::ready;
use std::sync::Arc;

use arrow::datatypes::SchemaRef;
Expand All @@ -26,6 +27,7 @@ use datafusion_common::error::Result;
use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableType};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::work_table::WorkTableExec;
use futures::future::BoxFuture;

use crate::{ScanArgs, ScanResult, Session, TableProvider};

Expand Down Expand Up @@ -78,30 +80,35 @@ impl TableProvider for CteWorkTable {
TableType::Temporary
}

async fn scan(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
filters: &[Expr],
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
projection: Option<&'life2 Vec<usize>>,
filters: &'life3 [Expr],
limit: Option<usize>,
) -> Result<Arc<dyn ExecutionPlan>> {
let options = ScanArgs::default()
.with_projection(projection.map(|p| p.as_slice()))
.with_filters(Some(filters))
.with_limit(limit);
Ok(self.scan_with_args(state, options).await?.into_inner())
) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Self: 'async_trait,
{
self.scan_boxed(state, projection, filters, limit)
}

async fn scan_with_args<'a>(
&self,
_state: &dyn Session,
fn scan_with_args<'a, 'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
args: ScanArgs<'a>,
) -> Result<ScanResult> {
Ok(ScanResult::new(Arc::new(WorkTableExec::new(
self.name.clone(),
Arc::clone(&self.table_schema),
args.projection().map(|p| p.to_vec()),
)?)))
) -> BoxFuture<'async_trait, Result<ScanResult>>
where
'a: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
Box::pin(ready(self.scan_with_args_inner(state, &args)))
}

fn supports_filters_pushdown(
Expand All @@ -115,3 +122,41 @@ impl TableProvider for CteWorkTable {
])
}
}

impl CteWorkTable {
fn scan_with_args_inner<'a>(
&self,
_state: &dyn Session,
args: &ScanArgs<'a>,
) -> Result<ScanResult> {
Ok(ScanResult::new(Arc::new(WorkTableExec::new(
self.name.clone(),
Arc::clone(&self.table_schema),
args.projection().map(|p| p.to_vec()),
)?)))
}

fn scan_boxed<'a>(
&'a self,
state: &'a dyn Session,
projection: Option<&'a Vec<usize>>,
filters: &'a [Expr],
limit: Option<usize>,
) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.scan_inner(state, projection, filters, limit))
}

async fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
filters: &[Expr],
limit: Option<usize>,
) -> Result<Arc<dyn ExecutionPlan>> {
let options = ScanArgs::default()
.with_projection(projection.map(|p| p.as_slice()))
.with_filters(Some(filters))
.with_limit(limit);
Ok(self.scan_with_args(state, options).await?.into_inner())
}
}
138 changes: 118 additions & 20 deletions datafusion/catalog/src/memory/table.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

use std::collections::HashMap;
use std::fmt::Debug;
use std::future::ready;
use std::sync::Arc;

use crate::TableProvider;
Expand Down Expand Up @@ -51,6 +52,7 @@ use datafusion_physical_plan::{
use datafusion_session::Session;

use async_trait::async_trait;
use futures::future::BoxFuture;
use log::debug;
use parking_lot::Mutex;
use tokio::sync::RwLock;
Expand Down Expand Up @@ -184,7 +186,95 @@ impl TableProvider for MemTable {
TableType::Base
}

async fn scan(
fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>(

@alamb alamb Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is amazing -- I think it would help future readers if we left some comments pointing back at an explanation of why this particular implementation helps compile time

@alamb alamb Aug 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will make a follow on PR to add comments to the relevant places

&'life0 self,
state: &'life1 dyn Session,
projection: Option<&'life2 Vec<usize>>,
filters: &'life3 [Expr],
limit: Option<usize>,
) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
'life3: 'async_trait,
Self: 'async_trait,
{
self.scan_boxed(state, projection, filters, limit)
}

/// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].
///
/// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].
///
/// # Arguments
///
/// * `state` - The [`SessionState`] containing the context for executing the plan.
/// * `input` - The [`ExecutionPlan`] to execute and insert.
///
/// # Returns
///
/// * A plan that returns the number of rows written.
///
/// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
fn insert_into<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
input: Arc<dyn ExecutionPlan>,
insert_op: InsertOp,
) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
self.insert_into_boxed(state, input, insert_op)
}

fn get_column_default(&self, column: &str) -> Option<&Expr> {
self.column_defaults.get(column)
}

fn delete_from<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
filters: Vec<Expr>,
) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
self.delete_from_boxed(state, filters)
}

fn update<'life0, 'life1, 'async_trait>(
&'life0 self,
state: &'life1 dyn Session,
assignments: Vec<(String, Expr)>,
filters: Vec<Expr>,
) -> BoxFuture<'async_trait, Result<Arc<dyn ExecutionPlan>>>
where
'life0: 'async_trait,
'life1: 'async_trait,
Self: 'async_trait,
{
self.update_boxed(state, assignments, filters)
}
}

impl MemTable {
fn scan_boxed<'a>(
&'a self,
state: &'a dyn Session,
projection: Option<&'a Vec<usize>>,
filters: &'a [Expr],
limit: Option<usize>,
) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.scan_inner(state, projection, filters, limit))
}

async fn scan_inner(
&self,
state: &dyn Session,
projection: Option<&Vec<usize>>,
Expand Down Expand Up @@ -225,21 +315,16 @@ impl TableProvider for MemTable {
Ok(DataSourceExec::from_data_source(source))
}

/// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`].
///
/// The [`ExecutionPlan`] must have the same schema as this [`MemTable`].
///
/// # Arguments
///
/// * `state` - The [`SessionState`] containing the context for executing the plan.
/// * `input` - The [`ExecutionPlan`] to execute and insert.
///
/// # Returns
///
/// * A plan that returns the number of rows written.
///
/// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html
async fn insert_into(
fn insert_into_boxed<'a>(
&'a self,
state: &'a dyn Session,
input: Arc<dyn ExecutionPlan>,
insert_op: InsertOp,
) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
Box::pin(ready(self.insert_into_inner(state, input, insert_op)))
}

fn insert_into_inner(
&self,
_state: &dyn Session,
input: Arc<dyn ExecutionPlan>,
Expand All @@ -260,11 +345,15 @@ impl TableProvider for MemTable {
Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None)))
}

fn get_column_default(&self, column: &str) -> Option<&Expr> {
self.column_defaults.get(column)
fn delete_from_boxed<'a>(
&'a self,
state: &'a dyn Session,
filters: Vec<Expr>,
) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.delete_from_inner(state, filters))
}

async fn delete_from(
async fn delete_from_inner(
&self,
state: &dyn Session,
filters: Vec<Expr>,
Expand Down Expand Up @@ -328,7 +417,16 @@ impl TableProvider for MemTable {
Ok(Arc::new(DmlResultExec::new(total_deleted)))
}

async fn update(
fn update_boxed<'a>(
&'a self,
state: &'a dyn Session,
assignments: Vec<(String, Expr)>,
filters: Vec<Expr>,
) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {
Box::pin(self.update_inner(state, assignments, filters))
}

async fn update_inner(
&self,
state: &dyn Session,
assignments: Vec<(String, Expr)>,
Expand Down
Loading
Loading