Remove FORMAT <..> backwards compatibility options from COPY (#9985)

* Revert "Add test for reading back file created with FORMAT options (#9753)"

This reverts commit b50f3aa.

* Revert "support format in options of COPY command (#9744)"

This reverts commit 40fb1b8.

* update docs and example to remove old syntax

Author: tinfoil-knight

datafusion/sql/src/parser.rs

@@ -87,11 +87,11 @@ impl fmt::Display for ExplainStatement {
 ///
 /// ```sql
 /// COPY lineitem  TO 'lineitem'
-///  (format parquet,
+/// STORED AS PARQUET (
 ///   partitions 16,
 ///   row_group_limit_rows 100000,
 ///   row_group_limit_bytes 200000
-///  )
+/// )
 ///
 /// COPY (SELECT l_orderkey from lineitem) to 'lineitem.parquet';
 /// ```

datafusion/sql/src/statement.rs

@@ -850,7 +850,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                     return plan_err!("Unsupported Value in COPY statement {}", value);
                 }
             };
-            if !(key.contains('.') || key == "format") {
+            if !(&key.contains('.')) {
                 // If config does not belong to any namespace, assume it is
                 // a format option and apply the format prefix for backwards
                 // compatibility.
@@ -866,16 +866,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
             FileType::from_str(&file_type).map_err(|_| {
                 DataFusionError::Configuration(format!("Unknown FileType {}", file_type))
             })?
-        } else if let Some(format) = options.remove("format") {
-            // try to infer file format from the "format" key in options
-            FileType::from_str(&format)
-                .map_err(|e| DataFusionError::Configuration(format!("{}", e)))?
         } else {
             let e = || {
                 DataFusionError::Configuration(
-                    "Format not explicitly set and unable to get file extension! Use STORED AS to define file format."
-                        .to_string(),
-                )
+                "Format not explicitly set and unable to get file extension! Use STORED AS to define file format."
+                    .to_string(),
+            )
             };
             // try to infer file format from file extension
             let extension: &str = &Path::new(&statement.target)

datafusion/sql/tests/sql_integration.rs

@@ -444,18 +444,6 @@ CopyTo: format=csv output_url=output.csv options: ()
     quick_test(sql, plan);
 }
 
-#[test]
-fn plan_copy_stored_as_priority() {
-    let sql = "COPY (select * from (values (1))) to 'output/' STORED AS CSV OPTIONS (format json)";
-    let plan = r#"
-CopyTo: format=csv output_url=output/ options: (format json)
-  Projection: column1
-    Values: (Int64(1))
-    "#
-    .trim();
-    quick_test(sql, plan);
-}
-
 #[test]
 fn plan_insert() {
     let sql =

datafusion/sqllogictest/test_files/copy.slt

@@ -514,65 +514,6 @@ OPTIONS (
 );
 
 
-# Format Options Support with format in OPTIONS
-#
-# i.e. COPY { table_name | query } TO 'file_name' OPTIONS (format <format-name>, ...)
-
-# Ensure that the format is set in the OPTIONS, not extension
-query I
-COPY (select * from (values (1))) to 'test_files/scratch/copy/foo.dat'
-OPTIONS (format parquet);
-----
-1
-
-statement ok
-CREATE EXTERNAL TABLE foo_dat STORED AS PARQUET LOCATION 'test_files/scratch/copy/foo.dat';
-
-query I
-select * from foo_dat;
-----
-1
-
-statement ok
-DROP TABLE foo_dat;
-
-
-query I
-COPY (select * from (values (1))) to 'test_files/scratch/copy'
-OPTIONS (format parquet);
-----
-1
-
-query I
-COPY (select * from (values (1))) to 'test_files/scratch/copy/'
-OPTIONS (format parquet, compression 'zstd(10)');
-----
-1
-
-query I
-COPY (select * from (values (1))) to 'test_files/scratch/copy/'
-OPTIONS (format json, compression gzip);
-----
-1
-
-query I
-COPY (select * from (values (1))) to 'test_files/scratch/copy/'
-OPTIONS (
-    format csv,
-    has_header false,
-    compression xz,
-    datetime_format '%FT%H:%M:%S.%9f',
-    delimiter ';',
-    null_value 'NULLVAL'
-);
-----
-1
-
-query error DataFusion error: Invalid or Unsupported Configuration: This feature is not implemented: Unknown FileType: NOTVALIDFORMAT
-COPY (select * from (values (1))) to 'test_files/scratch/copy/'
-OPTIONS (format notvalidformat, compression 'zstd(5)');
-
-
 # Error cases:
 
 # Copy from table with options

docs/source/user-guide/sql/dml.md

@@ -44,8 +44,7 @@ separate hive-style directories.
 The output format is determined by the first match of the following rules:
 
 1. Value of `STORED AS`
-2. Value of the `OPTION (FORMAT ..)`
-3. Filename extension (e.g. `foo.parquet` implies `PARQUET` format)
+2. Filename extension (e.g. `foo.parquet` implies `PARQUET` format)
 
 For a detailed list of valid OPTIONS, see [Write Options](write_options).