Skip to content

Commit 21169b9

Browse files
authored
Structify ConfigOptions (#4517) (#4771)
* Structify ConfigOptions (#4517) * Simplify * Fix environment variables * Tweaks * Update datafusion-cli * Format * Misc cleanup * Further fixes * Format * Update config.md * Flatten configuration * Document macro
1 parent 6c81c10 commit 21169b9

File tree

22 files changed

+715
-906
lines changed

22 files changed

+715
-906
lines changed

benchmarks/src/bin/h2o.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@
1818
//! DataFusion h2o benchmarks
1919
2020
use datafusion::arrow::datatypes::{DataType, Field, Schema};
21+
use datafusion::config::ConfigOptions;
2122
use datafusion::datasource::file_format::csv::CsvFormat;
2223
use datafusion::datasource::listing::{
2324
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
2425
};
2526
use datafusion::datasource::MemTable;
26-
use datafusion::prelude::{CsvReadOptions, SessionConfig};
27+
use datafusion::prelude::CsvReadOptions;
2728
use datafusion::{arrow::util::pretty, error::Result, prelude::SessionContext};
2829
use std::path::PathBuf;
2930
use std::sync::Arc;
@@ -63,9 +64,10 @@ async fn main() -> Result<()> {
6364

6465
async fn group_by(opt: &GroupBy) -> Result<()> {
6566
let path = opt.path.to_str().unwrap();
66-
let config = SessionConfig::from_env().with_batch_size(65535);
67+
let mut config = ConfigOptions::from_env()?;
68+
config.execution.batch_size = 65535;
6769

68-
let ctx = SessionContext::with_config(config);
70+
let ctx = SessionContext::with_config(config.into());
6971

7072
let schema = Schema::new(vec![
7173
Field::new("id1", DataType::Utf8, false),

datafusion-cli/src/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ pub async fn main() -> Result<()> {
9696
env::set_current_dir(p).unwrap();
9797
};
9898

99-
let mut session_config = SessionConfig::from_env().with_information_schema(true);
99+
let mut session_config = SessionConfig::from_env()?.with_information_schema(true);
100100

101101
if let Some(batch_size) = args.batch_size {
102102
session_config = session_config.with_batch_size(batch_size);

datafusion/core/src/bin/print_config_docs.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use datafusion::config::BuiltInConfigs;
18+
use datafusion::config::ConfigOptions;
1919

2020
fn main() {
21-
let docs = BuiltInConfigs::generate_config_markdown();
21+
let docs = ConfigOptions::generate_config_markdown();
2222
println!("{docs}");
2323
}

datafusion/core/src/catalog/information_schema.rs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ use arrow::{
2727
record_batch::RecordBatch,
2828
};
2929

30-
use crate::config::ConfigOptions;
30+
use crate::config::{ConfigEntry, ConfigOptions};
3131
use crate::datasource::streaming::{PartitionStream, StreamingTable};
3232
use crate::datasource::TableProvider;
3333
use crate::execution::context::TaskContext;
@@ -162,8 +162,8 @@ impl InformationSchemaConfig {
162162
config_options: &ConfigOptions,
163163
builder: &mut InformationSchemaDfSettingsBuilder,
164164
) {
165-
for (name, setting) in config_options.options() {
166-
builder.add_setting(name, setting.to_string());
165+
for entry in config_options.entries() {
166+
builder.add_setting(entry);
167167
}
168168
}
169169
}
@@ -611,7 +611,7 @@ impl InformationSchemaDfSettings {
611611
fn new(config: InformationSchemaConfig) -> Self {
612612
let schema = Arc::new(Schema::new(vec![
613613
Field::new("name", DataType::Utf8, false),
614-
Field::new("setting", DataType::Utf8, false),
614+
Field::new("setting", DataType::Utf8, true),
615615
]));
616616

617617
Self { schema, config }
@@ -656,9 +656,9 @@ struct InformationSchemaDfSettingsBuilder {
656656
}
657657

658658
impl InformationSchemaDfSettingsBuilder {
659-
fn add_setting(&mut self, name: impl AsRef<str>, setting: impl AsRef<str>) {
660-
self.names.append_value(name.as_ref());
661-
self.settings.append_value(setting.as_ref());
659+
fn add_setting(&mut self, entry: ConfigEntry) {
660+
self.names.append_value(entry.key);
661+
self.settings.append_option(entry.value);
662662
}
663663

664664
fn finish(&mut self) -> RecordBatch {

0 commit comments

Comments
 (0)