Add support for delta timestamp_ntz datatype

Add support for TIMESTAMP_NTZ datatype columns in v3 delta tables.

Author: infvg

presto-delta/src/main/java/com/facebook/presto/delta/DeltaTypeUtils.java

@@ -41,6 +41,7 @@
 import io.delta.kernel.types.ShortType;
 import io.delta.kernel.types.StringType;
 import io.delta.kernel.types.StructType;
+import io.delta.kernel.types.TimestampNTZType;
 import io.delta.kernel.types.TimestampType;
 
 import java.math.BigDecimal;
@@ -230,7 +231,7 @@ else if (deltaType instanceof ShortType) {
         else if (deltaType instanceof StringType) {
             return createUnboundedVarcharType();
         }
-        else if (deltaType instanceof TimestampType) {
+        else if (deltaType instanceof TimestampType || deltaType instanceof TimestampNTZType) {
             return TIMESTAMP;
         }
 

presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java

@@ -61,7 +61,8 @@ public abstract class AbstractDeltaDistributedQueryTestBase
             "test-lowercase",
             "test-partitions-lowercase",
             "test-uppercase",
-            "test-partitions-uppercase"
+            "test-partitions-uppercase",
+            "test-typing"
     };
 
     /**

presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.delta;
 
+import com.facebook.presto.testing.MaterializedResult;
 import com.google.common.base.Joiner;
 import org.testng.annotations.Test;
 
@@ -25,6 +26,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static java.lang.String.format;
+import static org.testng.Assert.assertEquals;
 
 /**
  * Integration tests for reading Delta tables.
@@ -247,6 +249,15 @@ public void readPartitionedTableAllDataTypes(String version)
         assertQuery(testQuery, expResultsQuery);
     }
 
+    @Test(dataProvider = "deltaReaderVersions")
+    public void testDeltaTimezoneTypeSupport(String version)
+    {
+        String testQuery = format("select * from \"%s\".\"%s\"",
+                PATH_SCHEMA, goldenTablePathWithPrefix(version, "test-typing"));
+        MaterializedResult expectedResult = getQueryRunner().execute(testQuery);
+        assertEquals(expectedResult.getMaterializedRows().size(), 10);
+    }
+
     /**
      * Expected results for table "data-reader-primitives"
      */

presto-delta/src/test/resources/delta_v1/test-typing/.part-00000-4858a35f-1c46-4eca-a46f-ba7285d14dae-c000.snappy.parquet.crc

@@ -0,0 +1 @@
+{"txnId":"dac93cbf-630e-4a7e-8368-27952a66b0ee","tableSizeBytes":6387,"numFiles":1,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"0a4082b0-2e72-49f1-b617-1b828ad42cd6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"VendorID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tpep_pickup_datetime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tpep_dropoff_datetime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"passenger_count\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trip_distance\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"RatecodeID\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"store_and_fwd_flag\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"PULocationID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DOLocationID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"payment_type\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"fare_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"extra\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"mta_tax\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tip_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tolls_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"improvement_surcharge\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"total_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"congestion_surcharge\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"airport_fee\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1737551907192},"protocol":{"minReaderVersion":1,"minWriterVersion":2},"allFiles":[{"path":"part-00000-4858a35f-1c46-4eca-a46f-ba7285d14dae-c000.snappy.parquet","partitionValues":{},"size":6387,"modificationTime":1737551908307,"dataChange":false,"stats":"{\"numRecords\":10,\"minValues\":{\"VendorID\":1,\"tpep_pickup_datetime\":\"2021-12-31T19:13:04.000+03:00\",\"tpep_dropoff_datetime\":\"2021-12-31T19:22:45.000+03:00\",\"passenger_count\":1.0,\"trip_distance\":0.78,\"RatecodeID\":1.0,\"store_and_fwd_flag\":\"N\",\"PULocationID\":68,\"DOLocationID\":42,\"payment_type\":1,\"fare_amount\":5.0,\"extra\":0.5,\"mta_tax\":0.5,\"tip_amount\":0.0,\"tolls_amount\":0.0,\"improvement_surcharge\":0.3,\"total_amount\":8.8,\"congestion_surcharge\":0.0,\"airport_fee\":0.0},\"maxValues\":{\"VendorID\":2,\"tpep_pickup_datetime\":\"2021-12-31T19:53:21.000+03:00\",\"tpep_dropoff_datetime\":\"2021-12-31T20:14:20.000+03:00\",\"passenger_count\":2.0,\"trip_distance\":10.3,\"RatecodeID\":1.0,\"store_and_fwd_flag\":\"N\",\"PULocationID\":238,\"DOLocationID\":236,\"payment_type\":2,\"fare_amount\":33.0,\"extra\":3.0,\"mta_tax\":0.5,\"tip_amount\":13.0,\"tolls_amount\":6.55,\"improvement_surcharge\":0.3,\"total_amount\":56.35,\"congestion_surcharge\":2.5,\"airport_fee\":0.0},\"nullCount\":{\"VendorID\":0,\"tpep_pickup_datetime\":0,\"tpep_dropoff_datetime\":0,\"passenger_count\":0,\"trip_distance\":0,\"RatecodeID\":0,\"store_and_fwd_flag\":0,\"PULocationID\":0,\"DOLocationID\":0,\"payment_type\":0,\"fare_amount\":0,\"extra\":0,\"mta_tax\":0,\"tip_amount\":0,\"tolls_amount\":0,\"improvement_surcharge\":0,\"total_amount\":0,\"congestion_surcharge\":0,\"airport_fee\":0}}"}]}

presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.crc.crc

@@ -0,0 +1,4 @@
+{"commitInfo":{"timestamp":1737551908361,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"10","numOutputBytes":"6387"},"engineInfo":"Apache-Spark/3.5.4 Delta-Lake/3.3.0","txnId":"dac93cbf-630e-4a7e-8368-27952a66b0ee"}}
+{"metaData":{"id":"0a4082b0-2e72-49f1-b617-1b828ad42cd6","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"VendorID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tpep_pickup_datetime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tpep_dropoff_datetime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"passenger_count\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trip_distance\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"RatecodeID\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"store_and_fwd_flag\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"PULocationID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DOLocationID\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"payment_type\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"fare_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"extra\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"mta_tax\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tip_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"tolls_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"improvement_surcharge\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"total_amount\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"congestion_surcharge\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"airport_fee\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1737551907192}}
+{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
+{"add":{"path":"part-00000-4858a35f-1c46-4eca-a46f-ba7285d14dae-c000.snappy.parquet","partitionValues":{},"size":6387,"modificationTime":1737551908307,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"VendorID\":1,\"tpep_pickup_datetime\":\"2021-12-31T19:13:04.000+03:00\",\"tpep_dropoff_datetime\":\"2021-12-31T19:22:45.000+03:00\",\"passenger_count\":1.0,\"trip_distance\":0.78,\"RatecodeID\":1.0,\"store_and_fwd_flag\":\"N\",\"PULocationID\":68,\"DOLocationID\":42,\"payment_type\":1,\"fare_amount\":5.0,\"extra\":0.5,\"mta_tax\":0.5,\"tip_amount\":0.0,\"tolls_amount\":0.0,\"improvement_surcharge\":0.3,\"total_amount\":8.8,\"congestion_surcharge\":0.0,\"airport_fee\":0.0},\"maxValues\":{\"VendorID\":2,\"tpep_pickup_datetime\":\"2021-12-31T19:53:21.000+03:00\",\"tpep_dropoff_datetime\":\"2021-12-31T20:14:20.000+03:00\",\"passenger_count\":2.0,\"trip_distance\":10.3,\"RatecodeID\":1.0,\"store_and_fwd_flag\":\"N\",\"PULocationID\":238,\"DOLocationID\":236,\"payment_type\":2,\"fare_amount\":33.0,\"extra\":3.0,\"mta_tax\":0.5,\"tip_amount\":13.0,\"tolls_amount\":6.55,\"improvement_surcharge\":0.3,\"total_amount\":56.35,\"congestion_surcharge\":2.5,\"airport_fee\":0.0},\"nullCount\":{\"VendorID\":0,\"tpep_pickup_datetime\":0,\"tpep_dropoff_datetime\":0,\"passenger_count\":0,\"trip_distance\":0,\"RatecodeID\":0,\"store_and_fwd_flag\":0,\"PULocationID\":0,\"DOLocationID\":0,\"payment_type\":0,\"fare_amount\":0,\"extra\":0,\"mta_tax\":0,\"tip_amount\":0,\"tolls_amount\":0,\"improvement_surcharge\":0,\"total_amount\":0,\"congestion_surcharge\":0,\"airport_fee\":0}}"}}