make format prefix optional for format options in COPY (#9723)

* make format prefix optional for format options in COPY

* fix clippy lint error

* Add negative test case for unknown option

* Improve test comments

---------

Co-authored-by: Andrew Lamb <[email protected]>

Author: tinfoil-knight

datafusion/sql/src/statement.rs

@@ -850,7 +850,16 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                     return plan_err!("Unsupported Value in COPY statement {}", value);
                 }
             };
-            options.insert(key.to_lowercase(), value_string.to_lowercase());
+            if !(&key.contains('.')) {
+                // If config does not belong to any namespace, assume it is
+                // a format option and apply the format prefix for backwards
+                // compatibility.
+
+                let renamed_key = format!("format.{}", key);
+                options.insert(renamed_key.to_lowercase(), value_string.to_lowercase());
+            } else {
+                options.insert(key.to_lowercase(), value_string.to_lowercase());
+            }
         }
 
         let file_type = if let Some(file_type) = statement.stored_as {

datafusion/sqllogictest/test_files/copy.slt

@@ -474,6 +474,45 @@ select * from validate_arrow;
 1 Foo
 2 Bar
 
+# Format Options Support without the 'format.' prefix
+
+# Copy with format options for Parquet without the 'format.' prefix
+query IT
+COPY source_table TO 'test_files/scratch/copy/format_table.parquet'
+OPTIONS (
+    compression snappy,
+    'compression::col1' 'zstd(5)'
+);
+----
+2
+
+# Copy with format options for JSON without the 'format.' prefix
+query IT
+COPY source_table  to 'test_files/scratch/copy/format_table'
+STORED AS JSON OPTIONS (compression gzip);
+----
+2
+
+# Copy with format options for CSV without the 'format.' prefix
+query IT
+COPY source_table to 'test_files/scratch/copy/format_table.csv'
+OPTIONS (
+    has_header false,
+    compression xz,
+    datetime_format '%FT%H:%M:%S.%9f',
+    delimiter ';',
+    null_value 'NULLVAL'
+);
+----
+2
+
+# Copy with unknown format options without the 'format.' prefix to ensure error is sensible
+query error DataFusion error: Invalid or Unsupported Configuration: Config value "unknown_option" not found on CsvOptions
+COPY source_table to 'test_files/scratch/copy/format_table2.csv'
+OPTIONS (
+    unknown_option  false,
+);
+
 
 # Error cases: