From bd1117e2a446bd47456fb9fd8b37ec99248cd29a Mon Sep 17 00:00:00 2001 From: Hazmi Date: Thu, 23 Jan 2025 14:10:22 +0300 Subject: [PATCH] Add support for delta timestamp_ntz datatype Add support for TIMESTAMP_NTZ datatype columns in v3 delta tables. --- .../facebook/presto/delta/DeltaTypeUtils.java | 3 ++- .../AbstractDeltaDistributedQueryTestBase.java | 3 ++- .../presto/delta/TestDeltaIntegration.java | 13 +++++++++++++ ...e87-961c-e9e18a355eeb-c000.snappy.parquet.crc | Bin 0 -> 52 bytes .../_delta_log/.00000000000000000000.crc.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 40 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 4 ++++ ...b2-4e87-961c-e9e18a355eeb-c000.snappy.parquet | Bin 0 -> 5526 bytes ...2da-a737-4921ed7fc950-c000.snappy.parquet.crc | Bin 0 -> 52 bytes .../_delta_log/.00000000000000000000.crc.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 40 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 4 ++++ ...0d-42da-a737-4921ed7fc950-c000.snappy.parquet | Bin 0 -> 5526 bytes ...c94-a8bb-a437508996d7-c000.snappy.parquet.crc | Bin 0 -> 52 bytes .../_delta_log/.00000000000000000000.crc.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 40 bytes .../_delta_log/00000000000000000000.crc | 1 + .../_delta_log/00000000000000000000.json | 4 ++++ ...a4-4c94-a8bb-a437508996d7-c000.snappy.parquet | Bin 0 -> 5521 bytes 21 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/.part-00000-16ab4bfc-37b2-4e87-961c-e9e18a355eeb-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.crc.crc create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.json.crc create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/00000000000000000000.crc create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/part-00000-16ab4bfc-37b2-4e87-961c-e9e18a355eeb-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/.part-00000-4705e2d5-430d-42da-a737-4921ed7fc950-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.crc.crc create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.json.crc create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/00000000000000000000.crc create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/delta_v1/test-typing/part-00000-4705e2d5-430d-42da-a737-4921ed7fc950-c000.snappy.parquet create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/.part-00000-994a8669-28a4-4c94-a8bb-a437508996d7-c000.snappy.parquet.crc create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/_delta_log/.00000000000000000000.crc.crc create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/_delta_log/.00000000000000000000.json.crc create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/_delta_log/00000000000000000000.crc create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/_delta_log/00000000000000000000.json create mode 100644 presto-delta/src/test/resources/delta_v3/test-typing/part-00000-994a8669-28a4-4c94-a8bb-a437508996d7-c000.snappy.parquet diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaTypeUtils.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaTypeUtils.java index b8d63cbfcf57f..e3c8f1d139227 100644 --- a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaTypeUtils.java +++ b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaTypeUtils.java @@ -41,6 +41,7 @@ import io.delta.kernel.types.ShortType; import io.delta.kernel.types.StringType; import io.delta.kernel.types.StructType; +import io.delta.kernel.types.TimestampNTZType; import io.delta.kernel.types.TimestampType; import java.math.BigDecimal; @@ -230,7 +231,7 @@ else if (deltaType instanceof ShortType) { else if (deltaType instanceof StringType) { return createUnboundedVarcharType(); } - else if (deltaType instanceof TimestampType) { + else if (deltaType instanceof TimestampType || deltaType instanceof TimestampNTZType) { return TIMESTAMP; } diff --git a/presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java b/presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java index ec77098e36312..efb81b24cd801 100644 --- a/presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java +++ b/presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java @@ -61,7 +61,8 @@ public abstract class AbstractDeltaDistributedQueryTestBase "test-lowercase", "test-partitions-lowercase", "test-uppercase", - "test-partitions-uppercase" + "test-partitions-uppercase", + "test-typing" }; /** diff --git a/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java b/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java index 156b6fd06ee61..09cfedf3c4ce9 100644 --- a/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java +++ b/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.delta; +import com.facebook.presto.Session; import com.google.common.base.Joiner; import org.testng.annotations.Test; @@ -24,6 +25,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.common.type.TimeZoneKey.UTC_KEY; import static java.lang.String.format; /** @@ -247,6 +249,17 @@ public void readPartitionedTableAllDataTypes(String version) assertQuery(testQuery, expResultsQuery); } + @Test(dataProvider = "deltaReaderVersions") + public void testDeltaTimezoneTypeSupport(String version) + { + Session session = Session.builder(getSession()) + .setTimeZoneKey(UTC_KEY) + .build(); + String testQuery = format("SELECT tpep_dropoff_datetime FROM \"%s\".\"%s\"", + PATH_SCHEMA, goldenTablePathWithPrefix(version, "test-typing")); + + assertQuery(session, testQuery, "SELECT CAST('2021-12-31 16:53:29' AS TIMESTAMP)"); + } /** * Expected results for table "data-reader-primitives" */ diff --git a/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/.part-00000-16ab4bfc-37b2-4e87-961c-e9e18a355eeb-c000.snappy.parquet.crc b/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/.part-00000-16ab4bfc-37b2-4e87-961c-e9e18a355eeb-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..83724aa6906689d7af90dee92a1341df3288db4a GIT binary patch literal 52 zcmV-40L%Yla$^7h00IDGu?#Gv=d7Xr>}0Zvan)ZOsDv&RyiV4_9CRzN&#V>+!B%+c KpmQA5T2Jvq?HASn literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.crc.crc b/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.crc.crc new file mode 100644 index 0000000000000000000000000000000000000000..c29a2e2a1cf34f26b6da449997793ed9856d6c61 GIT binary patch literal 40 wcmYc;N@ieSU}9)2Pg}3PBl(3`r;l&5Dbt+19|w*ET@(My{@GY{U$-MO04XmJ1poj5 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.json.crc b/presto-delta/src/test/resources/delta_v1/snapshot-data3/test-typing1/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..1c308c0155d14bb94b53a30bcd7901086c2bb6f2 GIT binary patch literal 40 ycmV+@0N4Lxa$^7h00IEMz&nKg$r~TTZp}j1 zo}6>;?|06*=iPhWc5c}f!w83P3tlK)Io}Y%5`3-09)y}YC4^9o%LTv52fJ|q9M6oM zJ0)!fH#b7QR&)Vx`E)x%m)=;5VBt#a$9Fz>>!T)ugziG* z!>1k=!3y={xs$)uID~2iihAvg?k$X7baG3XpC;;ddG^Qe>y|{D=>aquKaSs=9ezph zYFNk1(`OU@dh70Vt>8mTIA{lEyPZ1r_eGC`+wF3<2X2T}jIV$C?smNcng!NIv%K^3 z=&)04mYavVO@2K7PxF#!lcX4=S?2$`c+2PD)?t1cDcTpG{rhU8gPWZwveE9JDE;D8 z5DV=pCH?dCxA&a(({#JrMM>Xz|Nfb!Q*cwa2gMd#B?SW>+WmrvrJ#(EWwsOP1v3>z zGwn1mQ&BWqOJ;i5YG^hN?&=E$Tz!JVL;DE)sVJIh|A3i_qWPgxY2&Ft>+oRg^gP@Q zEmL8hOlvt7HRtPpVARII*Az*$QyHpBi?Gie3EY=OIZHL^Pt@QD|E`8Oc`A1eyB!gcS403ZtT2gwtDEThP<#6xgy zCLY7E6uIUiK?pNK`8P9&C9MuYOqoI4fFQhfp%-^8tM?5F3T~fFy<8#%uhfbKp9K#2 z53}GSa9>fP@&|#@jMM-Py&bqjXezZL%^(JvPa4U-&IvTav&A^(2!|E)Ob?fMDxy~8 zc#m+*m>eZ{2>a}sIXtc=*H(FZq~zWV*B1N$y7^bc`<&p7HsLUe48k-*o4xG$W>F8; zLe%u4;}VIwQY*4tAuLy#NM){zEaH*DSgsQmdS7yhuvBVAmYalSu$i!&h53=?BA!Bw zNUbgr!m|vdj~f36o_OSZFhe2OA@qB_!51x3P|G>m?+WT9M>!8Ay(;B&m$U zEJ;bMvy#D&*E4tzgZBvVqE>`IL*Pp${BbzqC_FuW8NqRK#95I}vNoOU;qi-Fk>E>0 z@cJqe|CB)B2H_InsMLxaKM;;@OpdV?Jc+VPg0YOPV3{C9bAmIF?S!Zm zS#A-QS5}iu76caOdxz025IQ;J*z=UPv zfy7wKv?9y9gk`~G8G=_#y8n*1BV!wCU?p_3ZgUIWCTc~t5yEz94JqNYz$RX}jAWXV zh&%G&eY#9Y<_L*=twEL|9(mN~he@$D5YY62sFGJ;z>1o)R&>9r7hnPi)OJL>qMazP zzQUuhFB65aY+q3|qPlhQ9(z+$yEOmV~AhmRa-W$T^BJ&r|&KN#M!`DFqw9ugl3 zanWeWv65PmB>3>TiqmNIFnoZm9;0&RE&#pI3RPB<+Xu$Jywe$UVhbL2s1l} zSnfWXV}A9^?Kgw!0r=JP2pGcqNA)O&o`|WP b?HiNHeq}wp#lc%^bwB8IGeSGze`NmyP(6>E literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v1/test-typing/.part-00000-4705e2d5-430d-42da-a737-4921ed7fc950-c000.snappy.parquet.crc b/presto-delta/src/test/resources/delta_v1/test-typing/.part-00000-4705e2d5-430d-42da-a737-4921ed7fc950-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2b3b1cafa5263e811502c0f182e2c5b92b04e398 GIT binary patch literal 52 zcmV-40L%Yla$^7h00IC~c|g;o=d7Xr>}0Zvan)aE6((gByiV4_9CRzN&#V>+!B%+c KpmQA5T2Jv$W*7MY literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.crc.crc b/presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.crc.crc new file mode 100644 index 0000000000000000000000000000000000000000..e8cf87488b4a0313c380b70f427ea7b45887114d GIT binary patch literal 40 wcmYc;N@ieSU}EUs*qy1qBl(3`r;qQ#{3y95xv-r(I#`U@KO3v=>vm)Y02+Y~T>t<8 literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.json.crc b/presto-delta/src/test/resources/delta_v1/test-typing/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..3edb1af2ddc9a417e6783e2055c2a42b45396ce4 GIT binary patch literal 40 wcmYc;N@ieSU}E?!@TlwGNol>j347!?U)*KV*fE9Spe|>zo!dPlq3qY!0WusA8vp_oS@I^Pb9_v$2+%C`j@^j6iXww~lrpT-1w`T^A za$df*tUO%~;qQ0uPgQe1q=bD=V2;~|r~bJpxwzdeb$jfF*v0t%`cHRjUC?Z>W|HN- zUxx?XVzb>O;x_Tw=%a>3(I#*)aI?()eesUp#jVx))Dg6AzWn!Oor{~3D00vq4vgJ) zD~OGDm5}~*_QwZq`)Rq|ZzrVhef;p;;wgy4?QyY%RtTY>M7p07Wg(;>WSgBtI>Agt z-b^|T%tYkP_LAuix*D2|fqS|_K~EQ_kVqeaKM{E|=^ro?kvBh4D$OhvXdM=elb*!P z&@yG_36=`93TGyL0ih9*H#@aEK??!LP1)+J7@W4mLNn|?G2>R?%r>~lB8L_zGb0!7 zup-G_u+=CORt7xsR!1BZsCY}D4ntK`oMqyeC~tR`^;J`cn@l`kL$xzZ9Id6=eip+d z6GxU&hj}KBd#U!ckBU={2uaTcUho(4s2 zC0#JpToO>Q&bu2xlX^)ww z6@^1k47DSv23nbHi7Y6BvP0>LN?X`*#3Ti+rH2&LLd(M03J(!N%2xq7pj`0F&!Omm zCoI>5+C@y@y2%#VYswoQaEo_d5CBAh{~%>NAS)<3D6kOR zn~B9RC`4~~a1h*#Q2NaZVo|G05R+CAw;%|wQ|N_VOY410fPyQb@Vu6XKZoJREckvn;s`uBeknmeKH}_1Cuo~abg=k&El=<* zCU|oNj(?IP@Ujz~a!g_lGJ`M)bChd&j%%3X2a9858B3z%lAtUj%V;JDk(}TRq&vZD zd6qkv<@J>~lX;HC{obK;^O%l|MoeN!U4?QIJ5P7w8KC>lq8s;?B*m^W$};XHEHGg? zcpy=h5-rd2K4zJ>SWd$$CfR@2+mW)J_R$iGwA)0k+juR{HiX$Ot->Xo;@J2Lmy%2| z5`IS>zfYGj$t)&OZuk@-Dk;PM08ENa!JuyR#?-tD16ItMwPFV~qW}{?u(~zY9&1Cv zb!8rf1DP0%We19y88h@lv8-mQFn6iJN$;@;o@OGOBL&kaCe27wq$jOqQiaIoSB^z; zY8IQwr~pRCIRkSCe9ILx8MQkDKCoD8iV)n$=3~c?H_`QO;~vAp!ygQ9+58d#7Y>O} zg}7j}WLPP^h!cGBT={9Vau_ziR*q3V^D->h?etQ%kROm4%F^0^zaA^a;4i;QF}RtX zK`eEjtuepy<<6TyHq)$ literal 0 HcmV?d00001 diff --git a/presto-delta/src/test/resources/delta_v3/test-typing/.part-00000-994a8669-28a4-4c94-a8bb-a437508996d7-c000.snappy.parquet.crc b/presto-delta/src/test/resources/delta_v3/test-typing/.part-00000-994a8669-28a4-4c94-a8bb-a437508996d7-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..21ef729a48a5f5ff9a1a5aa0388e9dd89ec7fc60 GIT binary patch literal 52 zcmV-40L%Yla$^7h00IDMzK9{qGZIk|07hX$mh**cAOleO*~k_-yE8}Xs|RGe_L^xW KsO_PCR8*dr++2Rw^-|2byO;GMDj-oG$^!Lv$pdGvUVb}iF6{Y|BMk{lh+zd^5V;j zKPQMA3bIKlTX948#~7|`qO4>7v2H`UZuF1A9Ez0vA^oEk5jTo)vV z7OI|}d*1hR&%G!2p69NYcgHcpVcduxPh7q13u8C@T7x|Z)wjD5LRBso{7pV~*xc~w zRkuSJdj8~r!`G`3mT(aC9^Lxzk+cQWB!v9UXzk<+-5U`4_GA;$S&0Mq-WLOx>WPkI z;XGjrhpKGQxv$RsQRM`(rqgAI%|E_4>J-*ze(Gu1*usrpYo4e!+X-qiUW?ybID1%# z@~z`*&}{>MzjOa~wGczs>9jR3x=tUT`g_#l5O%jT>={@6hsj^=);g$JVl6bwd(#)s zI?c@r^U$!VFMoQ}@Ie}3}gywiS~VfQ;I^n=eH4nH{siH1FGZlP7~P|!n5TrjaaBqL-E)6#}vYL*OJ ziFT1;+Z2asA;K^<#bN7-*-rK})FNlQx+LV3?ZX@Y6CiXVbz{ zfy&~OvD5SLVW_CG;VGU9RFW`E%Lb-KO>x*>yJKu2;K(U^`Wg{u6|;00%7YG*N)d)F zaf#0yDx_=}Ik3aB2X{jypio%uVb+WbZ?&DnZswim=6DtJ4oY~t9Xly8@A+zG&T(_B zhIt2SnR%0&7x-)*aC2e>iyiVZbDEnYKFbR^F9u%m7xId}H&ICLQL_q;!jawN4tjdj zTuRel-Ddq7F!E|XkxwW07W0XeVyH$st2PC8zPmcyY%V6QfrqdemD07m*4M|v5ca}Y zLtZHq)Lg%+Cz4t*XIS1kLr;TXx?m`|q-y!RyFovxrBnj zrN0U73Cb#p&Ya)WxzbeP`{yNGKCTsPG|GFwqG?dg`%F^qv-uM3&IU0QxYCXYOovyGVjEa zXrGeS^O|lX`cxIk?oc?ixF!mxr4;Hwo*F1>x;xk{yX75nchs|mpIh9L(K>dTaW51v z9O>|NLP)+6zybM~AHRg6JzU>$yBVtoSoGxM7GafpMb>q~`rc&CxWc$5)FF{%oU6v9 zy(Pcniq5#ApSr?9zt}aD{s!p(7)7CG=qe;g7jh>5Ao+Fx%P2Ywjf>Y_(WT|dk{x!F zY^_fhx6EwIykfSGfgvxN+0F}~(u!x8d4i!A0=EdJ+$&<<$AI~!j%?sKT$D5);Y-9A z<5et`*-o0uY$s1e^ooqn31h)z3`$|_x7W;0M&re?`43MDKDv zj-coy=+UY!`69DfPl6J=+qgw$O>aEz6)86eWwM^E%z5a77)3lA80CC5Q0NuOEkY^x zij=#AGSomQb0S4NZWv{bQ0S%0EkY^xij?CrP_CPlG3Z6v15psq=23L|)2B5p75bIE6@&mO@-oxZQg1qPzP-4|43DqWgMcN3VjhnO)c#oyU zAl|GDGU8(paYvrMPZI=kJOC-l!#>#^^~ejp05pfq!Jwx1$CSJRO;yb7uVVXEy#QBz zu(~bQ5o<@m^<@!-eVG`vV*84!5!1B;v8-w+&}%8d8Sjw@Xsm0@3q~U`u zEt^l|jKj+mUD^>ks6M;ml0K4arL-bR^4aqhThGd6c(+=)MtOtFwP457%h^V;8|Es@ zD+Te2tW-n104voH8gwqP+tyD=V%x5eU(>w1gnOsX-}+aBN87H{2@>ge6j)~+Ty qHnp{;Hg0H5rW7^)Vq9%+-Iz-CD(m5u4PH?z`#~QyAhZkqGxlE(PmVPJ literal 0 HcmV?d00001