Structify ConfigOptions (#4517) (#4771)

* Structify ConfigOptions (#4517)

* Simplify

* Fix environment variables

* Tweaks

* Update datafusion-cli

* Format

* Misc cleanup

* Further fixes

* Format

* Update config.md

* Flatten configuration

* Document macro

Author: tustvold

benchmarks/src/bin/h2o.rs

@@ -18,12 +18,13 @@
 //! DataFusion h2o benchmarks
 
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
+use datafusion::config::ConfigOptions;
 use datafusion::datasource::file_format::csv::CsvFormat;
 use datafusion::datasource::listing::{
     ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
 };
 use datafusion::datasource::MemTable;
-use datafusion::prelude::{CsvReadOptions, SessionConfig};
+use datafusion::prelude::CsvReadOptions;
 use datafusion::{arrow::util::pretty, error::Result, prelude::SessionContext};
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -63,9 +64,10 @@ async fn main() -> Result<()> {
 
 async fn group_by(opt: &GroupBy) -> Result<()> {
     let path = opt.path.to_str().unwrap();
-    let config = SessionConfig::from_env().with_batch_size(65535);
+    let mut config = ConfigOptions::from_env()?;
+    config.execution.batch_size = 65535;
 
-    let ctx = SessionContext::with_config(config);
+    let ctx = SessionContext::with_config(config.into());
 
     let schema = Schema::new(vec![
         Field::new("id1", DataType::Utf8, false),

datafusion-cli/src/main.rs

@@ -96,7 +96,7 @@ pub async fn main() -> Result<()> {
         env::set_current_dir(p).unwrap();
     };
 
-    let mut session_config = SessionConfig::from_env().with_information_schema(true);
+    let mut session_config = SessionConfig::from_env()?.with_information_schema(true);
 
     if let Some(batch_size) = args.batch_size {
         session_config = session_config.with_batch_size(batch_size);

datafusion/core/src/bin/print_config_docs.rs

@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion::config::BuiltInConfigs;
+use datafusion::config::ConfigOptions;
 
 fn main() {
-    let docs = BuiltInConfigs::generate_config_markdown();
+    let docs = ConfigOptions::generate_config_markdown();
     println!("{docs}");
 }

datafusion/core/src/catalog/information_schema.rs

@@ -27,7 +27,7 @@ use arrow::{
     record_batch::RecordBatch,
 };
 
-use crate::config::ConfigOptions;
+use crate::config::{ConfigEntry, ConfigOptions};
 use crate::datasource::streaming::{PartitionStream, StreamingTable};
 use crate::datasource::TableProvider;
 use crate::execution::context::TaskContext;
@@ -162,8 +162,8 @@ impl InformationSchemaConfig {
         config_options: &ConfigOptions,
         builder: &mut InformationSchemaDfSettingsBuilder,
     ) {
-        for (name, setting) in config_options.options() {
-            builder.add_setting(name, setting.to_string());
+        for entry in config_options.entries() {
+            builder.add_setting(entry);
         }
     }
 }
@@ -611,7 +611,7 @@ impl InformationSchemaDfSettings {
     fn new(config: InformationSchemaConfig) -> Self {
         let schema = Arc::new(Schema::new(vec![
             Field::new("name", DataType::Utf8, false),
-            Field::new("setting", DataType::Utf8, false),
+            Field::new("setting", DataType::Utf8, true),
         ]));
 
         Self { schema, config }
@@ -656,9 +656,9 @@ struct InformationSchemaDfSettingsBuilder {
 }
 
 impl InformationSchemaDfSettingsBuilder {
-    fn add_setting(&mut self, name: impl AsRef<str>, setting: impl AsRef<str>) {
-        self.names.append_value(name.as_ref());
-        self.settings.append_value(setting.as_ref());
+    fn add_setting(&mut self, entry: ConfigEntry) {
+        self.names.append_value(entry.key);
+        self.settings.append_option(entry.value);
     }
 
     fn finish(&mut self) -> RecordBatch {