ci: add check for doc comment formatting

ci/scripts/rust_fmt.sh

@@ -18,4 +18,13 @@
 # under the License.
 
 set -ex
-cargo fmt --all -- --check
+
+# Install nightly toolchain (skips if already installed)
+rustup toolchain install nightly --component rustfmt
+
+# Use nightly rustfmt to check formatting including doc comments
+# This requires nightly because format_code_in_doc_comments is an unstable feature
+if ! cargo +nightly fmt --all -- --check --config format_code_in_doc_comments=true; then
+    echo "To fix, run: cargo +nightly fmt --all -- --config format_code_in_doc_comments=true"
+    exit 1
+fi

datafusion-examples/examples/data_io/remote_catalog.rs

@@ -16,7 +16,6 @@
 // under the License.
 
 //! See `main.rs` for how to run it.
-//!
 /// This example shows how to implement the DataFusion [`CatalogProvider`] API
 /// for catalogs that are remote (require network access) and/or offer only
 /// asynchronous APIs such as [Polaris], [Unity], and [Hive].

datafusion-examples/examples/udf/simple_udaf.rs

@@ -16,7 +16,6 @@
 // under the License.
 
 //! See `main.rs` for how to run it.
-//!
 /// In this example we will declare a single-type, single return type UDAF that computes the geometric mean.
 /// The geometric mean is described here: https://en.wikipedia.org/wiki/Geometric_mean
 use datafusion::arrow::{

datafusion/common/src/scalar/mod.rs

@@ -2735,8 +2735,8 @@ impl ScalarValue {
     /// ```
     /// use arrow::array::{Int32Array, ListArray};
     /// use arrow::datatypes::{DataType, Int32Type};
-    /// use datafusion_common::cast::as_list_array;
     /// use datafusion_common::ScalarValue;
+    /// use datafusion_common::cast::as_list_array;
     ///
     /// let scalars = vec![
     ///     ScalarValue::Int32(Some(1)),
@@ -2796,8 +2796,8 @@ impl ScalarValue {
     /// ```
     /// use arrow::array::{Int32Array, ListArray};
     /// use arrow::datatypes::{DataType, Int32Type};
-    /// use datafusion_common::cast::as_list_array;
     /// use datafusion_common::ScalarValue;
+    /// use datafusion_common::cast::as_list_array;
     ///
     /// let scalars = vec![
     ///     ScalarValue::Int32(Some(1)),
@@ -2840,8 +2840,8 @@ impl ScalarValue {
     /// ```
     /// use arrow::array::{Int32Array, LargeListArray};
     /// use arrow::datatypes::{DataType, Int32Type};
-    /// use datafusion_common::cast::as_large_list_array;
     /// use datafusion_common::ScalarValue;
+    /// use datafusion_common::cast::as_large_list_array;
     ///
     /// let scalars = vec![
     ///     ScalarValue::Int32(Some(1)),
@@ -3318,8 +3318,8 @@ impl ScalarValue {
     /// ```
     /// use arrow::array::ListArray;
     /// use arrow::datatypes::{DataType, Int32Type};
-    /// use datafusion_common::utils::SingleRowListArrayBuilder;
     /// use datafusion_common::ScalarValue;
+    /// use datafusion_common::utils::SingleRowListArrayBuilder;
     /// use std::sync::Arc;
     ///
     /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![

datafusion/core/src/dataframe/mod.rs

@@ -1380,6 +1380,7 @@ impl DataFrame {
     ///     .read_csv("tests/data/example.csv", CsvReadOptions::new())
     ///     .await?;
     /// let count = df.count().await?; // 1
+    ///
     /// # assert_eq!(count, 1);
     /// # Ok(())
     /// # }

datafusion/core/src/execution/context/mod.rs

@@ -443,7 +443,7 @@ impl SessionContext {
     /// # use datafusion::execution::SessionStateBuilder;
     /// # use datafusion_optimizer::push_down_filter::PushDownFilter;
     /// let my_rule = PushDownFilter {}; // pretend it is a new rule
-    ///                                  // Create a new builder with a custom optimizer rule
+    /// // Create a new builder with a custom optimizer rule
     /// let context: SessionContext = SessionStateBuilder::new()
     ///     .with_optimizer_rule(Arc::new(my_rule))
     ///     .build()
@@ -635,9 +635,10 @@ impl SessionContext {
     ///     .sql_with_options("CREATE TABLE foo (x INTEGER)", options)
     ///     .await
     ///     .unwrap_err();
-    /// assert!(err
-    ///     .to_string()
-    ///     .starts_with("Error during planning: DDL not supported: CreateMemoryTable"));
+    /// assert!(
+    ///     err.to_string()
+    ///         .starts_with("Error during planning: DDL not supported: CreateMemoryTable")
+    /// );
     /// # Ok(())
     /// # }
     /// ```

datafusion/core/src/lib.rs

@@ -491,7 +491,7 @@
 //! consumes it immediately as well.
 //!
 //! ```text
-//!
+//! 
 //! Step 3: FilterExec calls next()       Step 2: ProjectionExec calls
 //!         on input Stream                  next() on input Stream
 //!         ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─      ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐

datafusion/core/src/physical_planner.rs

@@ -1906,7 +1906,6 @@ fn get_physical_expr_pair(
 /// splitting AND conjunctions into individual expressions.
 /// Column qualifiers are stripped so expressions can be evaluated against
 /// the TableProvider's schema.
-///
 fn extract_dml_filters(input: &Arc<LogicalPlan>) -> Result<Vec<Expr>> {
     let mut filters = Vec::new();
 
@@ -1944,7 +1943,6 @@ fn strip_column_qualifiers(expr: Expr) -> Result<Expr> {
 /// For UPDATE statements, the SQL planner encodes assignments as a projection
 /// over the source table. This function extracts column name and expression pairs
 /// from the projection. Column qualifiers are stripped from the expressions.
-///
 fn extract_update_assignments(input: &Arc<LogicalPlan>) -> Result<Vec<(String, Expr)>> {
     // The UPDATE input plan structure is:
     // Projection(updated columns as expressions with aliases)

datafusion/datasource-parquet/src/access_plan.rs

@@ -42,11 +42,11 @@ use parquet::file::metadata::RowGroupMetaData;
 /// // Use parquet reader RowSelector to specify scanning rows 100-200 and 350-400
 /// // in a row group that has 1000 rows
 /// let row_selection = RowSelection::from(vec![
-///    RowSelector::skip(100),
-///    RowSelector::select(100),
-///    RowSelector::skip(150),
-///    RowSelector::select(50),
-///    RowSelector::skip(600),  // skip last 600 rows
+///     RowSelector::skip(100),
+///     RowSelector::select(100),
+///     RowSelector::skip(150),
+///     RowSelector::select(50),
+///     RowSelector::skip(600), // skip last 600 rows
 /// ]);
 /// access_plan.scan_selection(1, row_selection);
 /// access_plan.skip(2); // skip row group 2

datafusion/doc/src/lib.rs

@@ -196,7 +196,7 @@ impl Default for DocSection {
 /// Example:
 ///
 /// ```rust
-///
+/// 
 /// # fn main() {
 ///     use datafusion_doc::{DocSection, Documentation};
 ///     let doc_section = DocSection {

datafusion/expr-common/src/interval_arithmetic.rs

@@ -1317,7 +1317,7 @@ fn min_of_bounds(first: &ScalarValue, second: &ScalarValue) -> ScalarValue {
 /// Example usage:
 /// ```
 /// use datafusion_common::DataFusionError;
-/// use datafusion_expr_common::interval_arithmetic::{satisfy_greater, Interval};
+/// use datafusion_expr_common::interval_arithmetic::{Interval, satisfy_greater};
 ///
 /// let left = Interval::make(Some(-1000.0_f32), Some(1000.0_f32))?;
 /// let right = Interval::make(Some(500.0_f32), Some(2000.0_f32))?;

datafusion/expr-common/src/signature.rs

@@ -967,7 +967,7 @@ fn get_data_types(native_type: &NativeType) -> Vec<DataType> {
 /// # Examples
 ///
 /// ```
-/// use datafusion_common::types::{logical_binary, logical_string, NativeType};
+/// use datafusion_common::types::{NativeType, logical_binary, logical_string};
 /// use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
 ///
 /// // Exact coercion that only accepts timestamp types

datafusion/expr/src/expr.rs

@@ -1851,7 +1851,7 @@ impl Expr {
     /// Example
     /// ```
     /// # use datafusion_common::Column;
-    /// use datafusion_expr::{col, Expr};
+    /// use datafusion_expr::{Expr, col};
     /// let expr = col("foo");
     /// assert_eq!(expr.try_as_col(), Some(&Column::from("foo")));
     ///

datafusion/expr/src/logical_plan/ddl.rs

@@ -255,8 +255,9 @@ impl CreateExternalTable {
     ///     TableReference::bare("my_table"),
     ///     "/path/to/data",
     ///     "parquet",
-    ///     Arc::new(DFSchema::empty())
-    /// ).build();
+    ///     Arc::new(DFSchema::empty()),
+    /// )
+    /// .build();
     /// ```
     pub fn builder(
         name: impl Into<TableReference>,

datafusion/expr/src/logical_plan/plan.rs

@@ -1587,7 +1587,7 @@ impl LogicalPlan {
     ///
     /// ```
     /// use arrow::datatypes::{DataType, Field, Schema};
-    /// use datafusion_expr::{col, lit, logical_plan::table_scan, LogicalPlanBuilder};
+    /// use datafusion_expr::{LogicalPlanBuilder, col, lit, logical_plan::table_scan};
     /// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
     /// let plan = table_scan(Some("t1"), &schema, None)
     ///     .unwrap()
@@ -1629,7 +1629,7 @@ impl LogicalPlan {
     ///
     /// ```
     /// use arrow::datatypes::{DataType, Field, Schema};
-    /// use datafusion_expr::{col, lit, logical_plan::table_scan, LogicalPlanBuilder};
+    /// use datafusion_expr::{LogicalPlanBuilder, col, lit, logical_plan::table_scan};
     /// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
     /// let plan = table_scan(Some("t1"), &schema, None)
     ///     .unwrap()
@@ -1694,7 +1694,7 @@ impl LogicalPlan {
     ///
     /// ```
     /// use arrow::datatypes::{DataType, Field, Schema};
-    /// use datafusion_expr::{col, lit, logical_plan::table_scan, LogicalPlanBuilder};
+    /// use datafusion_expr::{LogicalPlanBuilder, col, lit, logical_plan::table_scan};
     /// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
     /// let plan = table_scan(Some("t1"), &schema, None)
     ///     .unwrap()
@@ -1752,7 +1752,7 @@ impl LogicalPlan {
     /// ```
     /// ```
     /// use arrow::datatypes::{DataType, Field, Schema};
-    /// use datafusion_expr::{col, lit, logical_plan::table_scan, LogicalPlanBuilder};
+    /// use datafusion_expr::{LogicalPlanBuilder, col, lit, logical_plan::table_scan};
     /// let schema = Schema::new(vec![Field::new("id", DataType::Int32, false)]);
     /// let plan = table_scan(Some("t1"), &schema, None)
     ///     .unwrap()

datafusion/expr/src/predicate_bounds.rs

@@ -45,7 +45,6 @@ use datafusion_expr_common::operator::Operator;
 ///
 /// The function returns a [NullableInterval] that describes the possible boolean values the
 /// predicate can evaluate to.
-///
 pub(super) fn evaluate_bounds(
     predicate: &Expr,
     certainly_null_expr: Option<&Expr>,

datafusion/expr/src/udf.rs

@@ -142,7 +142,7 @@ impl ScalarUDF {
     ///
     /// # Example
     /// ```no_run
-    /// use datafusion_expr::{col, lit, ScalarUDF};
+    /// use datafusion_expr::{ScalarUDF, col, lit};
     /// # fn my_udf() -> ScalarUDF { unimplemented!() }
     /// let my_func: ScalarUDF = my_udf();
     /// // Create an expr for `my_func(a, 12.3)`

datafusion/functions-aggregate-common/src/tdigest.rs

@@ -523,7 +523,7 @@ impl TDigest {
     /// [`ScalarValue::Float64`]:
     ///
     /// ```text
-    ///
+    /// 
     ///    ┌────────┬────────┬────────┬───────┬────────┬────────┐
     ///    │max_size│  sum   │ count  │  max  │  min   │centroid│
     ///    └────────┴────────┴────────┴───────┴────────┴────────┘

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -148,7 +148,7 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use datafusion_expr::execution_props::ExecutionProps;
     /// use datafusion_expr::simplify::SimplifyContext;
     /// use datafusion_expr::simplify::SimplifyInfo;
-    /// use datafusion_expr::{col, lit, Expr};
+    /// use datafusion_expr::{Expr, col, lit};
     /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     /// use std::sync::Arc;
     ///
@@ -290,7 +290,7 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use datafusion_expr::execution_props::ExecutionProps;
     /// use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
     /// use datafusion_expr::simplify::SimplifyContext;
-    /// use datafusion_expr::{col, lit, Expr};
+    /// use datafusion_expr::{Expr, col, lit};
     /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     ///
     /// let schema = Schema::new(vec![
@@ -352,7 +352,7 @@ impl<S: SimplifyInfo> ExprSimplifier<S> {
     /// use datafusion_expr::execution_props::ExecutionProps;
     /// use datafusion_expr::interval_arithmetic::{Interval, NullableInterval};
     /// use datafusion_expr::simplify::SimplifyContext;
-    /// use datafusion_expr::{col, lit, Expr};
+    /// use datafusion_expr::{Expr, col, lit};
     /// use datafusion_optimizer::simplify_expressions::ExprSimplifier;
     ///
     /// let schema = Schema::new(vec![

datafusion/physical-expr/src/expressions/case/literal_lookup_table/mod.rs

@@ -63,7 +63,6 @@ use std::fmt::Debug;
 ///     ELSE <optional-fallback_literal>
 /// END
 /// ```
-///
 #[derive(Debug)]
 pub(in super::super) struct LiteralLookupTable {
     /// The lookup table to use for evaluating the CASE expression
@@ -252,7 +251,6 @@ pub(super) trait WhenLiteralIndexMap: Debug + Send + Sync {
     ///
     /// the returned vector will be:
     /// - `[0, 2, else_index, 1, 0]`
-    ///
     fn map_to_when_indices(
         &self,
         array: &ArrayRef,

datafusion/physical-expr/src/intervals/cp_solver.rs

@@ -562,11 +562,11 @@ impl ExprIntervalGraph {
     /// use arrow::datatypes::Field;
     /// use arrow::datatypes::Schema;
     /// use datafusion_common::ScalarValue;
-    /// use datafusion_expr::interval_arithmetic::Interval;
     /// use datafusion_expr::Operator;
+    /// use datafusion_expr::interval_arithmetic::Interval;
+    /// use datafusion_physical_expr::PhysicalExpr;
     /// use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};
     /// use datafusion_physical_expr::intervals::cp_solver::ExprIntervalGraph;
-    /// use datafusion_physical_expr::PhysicalExpr;
     /// use std::sync::Arc;
     ///
     /// let expr = Arc::new(BinaryExpr::new(

datafusion/physical-expr/src/projection.rs

@@ -260,12 +260,12 @@ impl ProjectionExprs {
     /// # Example
     ///
     /// ```rust
-    /// use std::sync::Arc;
     /// use arrow::datatypes::{DataType, Field, Schema};
     /// use datafusion_common::Result;
+    /// use datafusion_physical_expr::PhysicalExpr;
     /// use datafusion_physical_expr::expressions::Column;
     /// use datafusion_physical_expr::projection::ProjectionExprs;
-    /// use datafusion_physical_expr::PhysicalExpr;
+    /// use std::sync::Arc;
     ///
     /// // Create a schema and projection
     /// let schema = Arc::new(Schema::new(vec![
@@ -516,10 +516,10 @@ impl ProjectionExprs {
     ///
     /// ```rust
     /// use arrow::datatypes::{DataType, Field, Schema};
-    /// use datafusion_common::stats::{ColumnStatistics, Precision, Statistics};
-    /// use datafusion_physical_expr::projection::ProjectionExprs;
     /// use datafusion_common::Result;
     /// use datafusion_common::ScalarValue;
+    /// use datafusion_common::stats::{ColumnStatistics, Precision, Statistics};
+    /// use datafusion_physical_expr::projection::ProjectionExprs;
     /// use std::sync::Arc;
     ///
     /// fn main() -> Result<()> {

datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs

@@ -180,7 +180,6 @@ pub struct GroupValuesColumn<const STREAMING: bool> {
     /// We don't really store the actual `group values` in `hashtable`,
     /// instead we store the `group indices` pointing to values in `GroupValues`.
     /// And we use [`GroupIndexView`] to represent such `group indices` in table.
-    ///
     map: HashTable<(u64, GroupIndexView)>,
 
     /// The size of `map` in bytes

datafusion/physical-plan/src/aggregates/row_hash.rs

@@ -234,7 +234,7 @@ enum OutOfMemoryMode {
 /// # Architecture
 ///
 /// ```text
-///
+/// 
 ///     Assigns a consecutive group           internally stores aggregate values
 ///     index for each unique set                     for all groups
 ///         of group values

datafusion/physical-plan/src/joins/hash_join/exec.rs

@@ -211,7 +211,7 @@ impl JoinLeftData {
 ///
 ///
 /// ```text
-///
+/// 
 ///  Original build-side data   Inserting build-side values into hashmap    Concatenated build-side batch
 ///                                                                         ┌───────────────────────────┐
 ///                             hashmap.insert(row-hash, row-idx + offset)  │                      idx  │

datafusion/physical-plan/src/joins/hash_join/stream.rs

@@ -110,7 +110,7 @@ impl BuildSide {
 /// Expected state transitions performed by HashJoinStream are:
 ///
 /// ```text
-///
+/// 
 ///       WaitBuildSide
 ///             │
 ///             ▼