Implementation to export CSV direct to ZIP or GZIP and format Date,Time,Timestamp and Number.

README.md

@@ -18,9 +18,45 @@ CSV2J COPY (
   SELECT 1 AS ID, 10.99 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION
   UNION ALL
   SELECT 2 AS ID, 7.50 AS AMOUNT, TIMESTAMP '2023-01-31 21:59:58.987' AS DAT_CREATION
-) TO '/tmp/file.csv' WITH CSV HEADER
+) TO '/tmp/file.csv' WITH CSV [ZIP|GZIP] HEADER
 ```
 
+### Formatting Date,Time,Datetime and Numbers
+#### Selecting language 
+```
+LANGUAGE '<Language Tag>' -- en-US pt-BR fr-FR
+```
+Default is System Language<br>
+More language tag see [Supported Locales]("https://www.oracle.com/java/technologies/javase/jdk8-jre8-suported-locales.html#util-text")
+
+#### Using custom templates
+```
+DATETIMEFORMAT 'yyyy-MM-dd HH:mm:ss.SSS'
+DATEFORMAT 'yyyy-MM-dd'
+TIMEFORMAT 'HH:mm:ss.SSS'
+NUMBERFORMAT '#00'
+DECIMALFORMAT '#00.00'
+```
+More patterns details see 
+[SimpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html) 
+and [DecimalFormat](https://docs.oracle.com/javase/7/docs/api/java/text/DecimalFormat.html)
+
+### Full Example
+```sql
+CSV2J COPY (
+  SELECT 1234567890 AS ID, 1234567890.995 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58.987' AS DAT_CREATION, DATE '2023-03-11' as DAT_BORN, TIME '21:47:01.001' as TIME_BORN
+  UNION ALL
+  SELECT 1 AS ID, 1.12345 AS AMOUNT, TIMESTAMP '2022-01-31 23:59:58' AS DAT_CREATION, DATE '1978-04-16' as DAT_BORN, TIME '23:35:00' as TIME_BORN
+) TO '/tmp/file.csv' WITH CSV GZIP HEADER
+LANGUAGE 'pt-BR'
+DATETIMEFORMAT 'yyyy-MM-dd HH:mm:ss.SSS'
+DATEFORMAT 'yyyy-MM-dd'
+TIMEFORMAT 'HH:mm:ss.SSS'
+NUMBERFORMAT '#00'
+DECIMALFORMAT '#00.00'
+```
+
+
 ## Using
 
 ### SQL client software application

build.gradle

@@ -41,6 +41,8 @@ dependencies {
   testImplementation group: 'org.postgresql', name: 'postgresql', version: '9.4.1212'
   testImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
 
+  testImplementation group: 'org.apache.commons', name: 'commons-compress', version: '1.22'
+
 }
 
 tasks.named('test') {

gradlew.bat

@@ -33,7 +33,7 @@ set APP_HOME=%DIRNAME%
 for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
 
 @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" "-Djavax.net.ssl.trustStorePassword=changeit"
 
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome

src/main/antlr/PostgreSQLLexer.g4

@@ -2192,6 +2192,41 @@ LOOP
 OPEN
    : 'OPEN'
    ;
+
+ZIP
+   : 'ZIP'
+   ;
+
+GZIP
+   : 'GZIP'
+   ;
+
+BZIP2
+   : 'BZIP2'
+   ;
+
+DATEFORMAT
+   : 'DATEFORMAT'
+   ;
+
+TIMEFORMAT
+   : 'TIMEFORMAT'
+   ;
+
+DATETIMEFORMAT
+   : 'DATETIMEFORMAT'
+   ;
+
+NUMBERFORMAT
+   : 'NUMBERFORMAT'
+   ;
+
+DECIMALFORMAT
+   : 'DECIMALFORMAT'
+   ;
+
+
+
    //
 
    // IDENTIFIERS (4.1.1)

src/main/antlr/PostgreSQLParser.g4

@@ -660,9 +660,18 @@ copy2csv_opt_list
 copy2csv_opt_item
    : DELIMITER sconst
    | CSV
+   | GZIP
+   | ZIP
+   | BZIP2
    | HEADER_P
    | CREATE_TABLE_P
    | ENCODING sconst
+   | LANGUAGE sconst
+   | DATEFORMAT sconst
+   | TIMEFORMAT sconst
+   | DATETIMEFORMAT sconst
+   | NUMBERFORMAT sconst
+   | DECIMALFORMAT sconst
    ;
 
 createstmt

src/main/java/com/mageddo/csv2jdbc/CopyCsvStatement.java

@@ -83,6 +83,49 @@ public CopyCsvStatement validateIsCsv() {
     return this;
   }
 
+  public boolean isZIP() {
+    return this.options.containsKey(Option.ZIP);
+  }
+
+  public boolean isGZIP() {
+    return this.options.containsKey(Option.GZIP);
+  }
+
+  public boolean isBZIP2() {
+    return this.options.containsKey(Option.BZIP2);
+  }
+
+  public String getCompression() {
+    if( isGZIP() ) return Option.GZIP;
+    if( isZIP() ) return Option.ZIP;
+    if( isBZIP2() ) return Option.BZIP2;
+    return "";
+  }
+
+  protected void setFile(Path file) {
+    this.file = file;
+  }
+
+  public String getLanguage() {
+    return this.options.getOrDefault(Option.LANGUAGE,Option.DEFAULT_LANGUAGE).getValue();
+  }
+
+  public String getDateFormat() {
+    return this.options.getOrDefault("DATEFORMAT",Option.DEFAULT_NULL).getValue();
+  }
+  public String getTimeFormat() {
+    return this.options.getOrDefault("TIMEFORMAT",Option.DEFAULT_NULL).getValue();
+  }
+  public String getDateTimeFormat() {
+    return this.options.getOrDefault("DATETIMEFORMAT",Option.DEFAULT_NULL).getValue();
+  }
+  public String getNumberFormat() {
+    return this.options.getOrDefault("NUMBERFORMAT",Option.DEFAULT_NULL).getValue();
+  }
+  public String getDecimalFormat() {
+    return this.options.getOrDefault("DECIMALFORMAT",Option.DEFAULT_NULL).getValue();
+  }
+
   @Getter
   @ToString
   @EqualsAndHashCode(of = "name")
@@ -98,6 +141,17 @@ public static class Option {
 
     public static final String ENCODING = "ENCODING";
 
+    public static final String ZIP = "ZIP";
+
+    public static final String GZIP = "GZIP";
+
+    public static final String BZIP2 = "BZIP2";
+
+    public static final String LANGUAGE = "LANGUAGE";
+
+
+    public static final Option DEFAULT_NULL = new Option(null);
+
     public static final Option DEFAULT_CSV = new Option(CSV);
     public static final Option DEFAULT_HEADER = new Option(HEADER);
 
@@ -107,6 +161,12 @@ public static class Option {
 
     public static final Option DEFAULT_ENCODING = new Option(ENCODING, "utf-8");
 
+    public static final Option DEFAULT_ZIP = new Option(ZIP);
+
+    public static final Option DEFAULT_GZIP = new Option(GZIP);
+
+    public static final Option DEFAULT_LANGUAGE = new Option(LANGUAGE);
+
     private final String name;
     private final String value;
 

src/main/java/com/mageddo/csv2jdbc/Csv2JdbcDriver.java

@@ -45,7 +45,7 @@ public Connection connect(String url, Properties info) throws SQLException {
     final String delegateDriverClassName = getOrDefault(
         params,
         PROP_DELEGATE_DRIVER_CLASSNAME,
-        "org.h2.Driver"
+        info.getProperty(PROP_DELEGATE_DRIVER_CLASSNAME,"org.h2.Driver")
     );
     this.delegate = Reflections.createInstance(delegateDriverClassName);
     final String delegateUrl = toDelegateUrl(url);

src/main/java/com/mageddo/csv2jdbc/Csv2JdbcExecutor.java

@@ -2,17 +2,26 @@
 
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVPrinter;
 
 public class Csv2JdbcExecutor {
 
-  public static final int HEADER_COUNT = 1;
+  //public static final int HEADER_COUNT = 1;
 
   private final Connection connection;
   private final CopyCsvStatement csvStm;
@@ -40,52 +49,79 @@ public int execute() throws SQLException {
   }
 
   private int extractQueryToCsv() throws SQLException {
+    CsvExtractor extractor = new CsvExtractor( this.csvStm.getFile().toString(),
+        this.csvStm.hasHeader(), this.csvStm.getDelimiter(), this.csvStm.getCharset(),
+        this.csvStm.getCompression() );
+    try {
+      CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(),
+          o -> extractor.accept( new FormatterAndCounterResultSet(o, this.csvStm) ) );
+    } catch (Exception e) {
+      throw new SQLException(e);
+    }
+    if( extractor.getFiles().size() == 1 ) {
+      this.csvStm.setFile( Paths.get( extractor.getFiles().iterator().next() ) );
+    }
+    Log.log("status=csvWritten");
+    Log.log("status=linesCount, lines={}", extractor.getRowCount());
+    return extractor.getRowCount();
+  }
+
+  private int extractQueryToCsv1() throws SQLException {
+    final AtomicInteger rowCount = new AtomicInteger();
     try {
       CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(), (rs) -> {
         try (final CSVPrinter printer = this.createCsvPrinter()) {
           Log.log("status=printingRecords");
-          printer.printRecords(rs, true);
+          final FormatterAndCounterResultSet prs = new FormatterAndCounterResultSet(rs, this.csvStm);
+          printer.printRecords(prs, this.csvStm.hasHeader());
+          rowCount.set( prs.getRow() - 1 );
         }
       });
     } catch (Exception e) {
       throw new SQLException(e);
     }
     Log.log("status=csvWritten");
-    try {
+    Log.log("status=linesCount, lines={}", rowCount.get());
+    return rowCount.get();
+    /*try {
       final int lines = Files.countLines(this.csvStm.getFile()) - HEADER_COUNT;
       Log.log("status=linesCount, lines={}", lines);
       return lines;
     } catch (IOException e) {
       throw new SQLException(e);
-    }
+    }/**/
   }
 
   private int extractQueryToCsv0() throws SQLException {
+    final AtomicInteger rowCount = new AtomicInteger(0);
     try {
       final BufferedWriter out = java.nio.file.Files.newBufferedWriter(this.csvStm.getFile());
       CsvTableDaos.streamSelect(this.connection, this.csvStm.getExtractSql(), (rs) -> {
 //        try (final CSVPrinter printer = this.createCsvPrinter()) {
-        final int columns = rs
+        final FormatterAndCounterResultSet prs = new FormatterAndCounterResultSet(rs, this.csvStm);
+        final int columns = prs
             .getMetaData()
             .getColumnCount();
-        while (rs.next()){
+        while (prs.next()){
           for (int i = 1; i <= columns; i++) {
-            out.write(rs.getString(i));
+            out.write( prs.getObject(i).toString() );
             out.write(", ");
           }
           out.write('\n');
+          rowCount.incrementAndGet();
         }
 //          printer.printRecords(rs, true);
 //        }
       });
     } catch (Exception e) {
       throw new SQLException(e);
     }
-    try {
+    return rowCount.get();
+    /*try {
       return Files.countLines(this.csvStm.getFile()) - HEADER_COUNT;
     } catch (IOException e) {
       throw new SQLException(e);
-    }
+    }/**/
   }
 
   private int loadCsvIntoTable() throws SQLException {
@@ -118,10 +154,33 @@ CSVParser createCsvParser() throws IOException {
   }
 
   private CSVPrinter createCsvPrinter() throws IOException {
+    final Appendable appendableOut;
+    if( this.csvStm.isGZIP() ) {
+      Path file = this.csvStm.getFile();
+      file = file.resolveSibling( file.getFileName() + ".gzip" );
+      this.csvStm.setFile(file);
+      final OutputStream outs = java.nio.file.Files.newOutputStream( file );
+      final GZIPOutputStream gzipout = new GZIPOutputStream( outs );
+      appendableOut = new OutputStreamWriter(gzipout, this.csvStm.getCharset() );
+    } else if( this.csvStm.isZIP() ) {
+      final String fileNameInsideZip = this.csvStm.getFile().getFileName().toString();
+      Path file = this.csvStm.getFile();
+      file = file.resolveSibling( file.getFileName() + ".zip" );
+      this.csvStm.setFile(file);
+      final OutputStream outs = java.nio.file.Files.newOutputStream( file );
+      final ZipOutputStream zipout = new ZipOutputStream( outs );
+      final ZipEntry zeCsv = new ZipEntry( fileNameInsideZip );
+      zipout.putNextEntry(zeCsv);
+      appendableOut = new OutputStreamWriter(zipout, this.csvStm.getCharset() );
+    } else {
+      appendableOut = java.nio.file.Files.newBufferedWriter( this.csvStm.getFile(),
+          this.csvStm.getCharset() );
+    }
+
     return this.getCsvFormat()
         .builder()
         .build()
-        .print(this.csvStm.getFile(), this.csvStm.getCharset())
+        .print( appendableOut )
         ;
   }