Skip to content

Commit 5c19505

Browse files
yaooqinnyaooqinn
committed
[SPARK-48844][FOLLOWUP][TESTS] Cleanup duplicated data resource files in hive-thriftserver test
### What changes were proposed in this pull request? A follow up of SPARK-48844 to cleanup duplicated data resource files in hive-thriftserver test ### Why are the changes needed? code refactoring ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? new tests ### Was this patch authored or co-authored using generative AI tooling? no Closes apache#47480 from yaooqinn/SPARK-48844-F. Authored-by: Kent Yao <[email protected]> Signed-off-by: Kent Yao <[email protected]>
1 parent bafce5d commit 5c19505

File tree

7 files changed

+193
-48
lines changed

7 files changed

+193
-48
lines changed

‎sql/core/src/test/resources/sql-tests/analyzer-results/sql-on-files.sql.out

+86-20
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
11
-- Automatically generated by SQLQueryTestSuite
2+
-- !query
3+
CREATE DATABASE IF NOT EXISTS sql_on_files
4+
-- !query analysis
5+
CreateNamespace true
6+
+- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files]
7+
8+
9+
-- !query
10+
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1
11+
-- !query analysis
12+
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_parquet`, ErrorIfExists, [1]
13+
+- Project [1 AS 1#x]
14+
+- OneRowRelation
15+
16+
217
-- !query
318
SELECT * FROM parquet.``
419
-- !query analysis
@@ -33,12 +48,25 @@ org.apache.spark.sql.AnalysisException
3348

3449

3550
-- !query
36-
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1
51+
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`
52+
-- !query analysis
53+
Project [1#x]
54+
+- Relation [1#x] parquet
55+
56+
57+
-- !query
58+
DROP TABLE sql_on_files.test_parquet
59+
-- !query analysis
60+
DropTable false, false
61+
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_parquet
62+
63+
64+
-- !query
65+
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1
3766
-- !query analysis
38-
GlobalLimit 1
39-
+- LocalLimit 1
40-
+- Project [fixed_len_dec#x]
41-
+- Relation [fixed_len_dec#x] parquet
67+
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_orc`, ErrorIfExists, [1]
68+
+- Project [1 AS 1#x]
69+
+- OneRowRelation
4270

4371

4472
-- !query
@@ -75,12 +103,25 @@ org.apache.spark.sql.AnalysisException
75103

76104

77105
-- !query
78-
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1
106+
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`
107+
-- !query analysis
108+
Project [1#x]
109+
+- Relation [1#x] orc
110+
111+
112+
-- !query
113+
DROP TABLE sql_on_files.test_orc
79114
-- !query analysis
80-
GlobalLimit 1
81-
+- LocalLimit 1
82-
+- Project [dt#x]
83-
+- Relation [dt#x] orc
115+
DropTable false, false
116+
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_orc
117+
118+
119+
-- !query
120+
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1
121+
-- !query analysis
122+
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_csv`, ErrorIfExists, [1]
123+
+- Project [1 AS 1#x]
124+
+- OneRowRelation
84125

85126

86127
-- !query
@@ -117,12 +158,25 @@ org.apache.spark.sql.AnalysisException
117158

118159

119160
-- !query
120-
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1
161+
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`
162+
-- !query analysis
163+
Project [_c0#x]
164+
+- Relation [_c0#x] csv
165+
166+
167+
-- !query
168+
DROP TABLE sql_on_files.test_csv
169+
-- !query analysis
170+
DropTable false, false
171+
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_csv
172+
173+
174+
-- !query
175+
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1
121176
-- !query analysis
122-
GlobalLimit 1
123-
+- LocalLimit 1
124-
+- Project [_c0#x, _c1#x, _c2#x, _c3#x, _c4#x]
125-
+- Relation [_c0#x,_c1#x,_c2#x,_c3#x,_c4#x] csv
177+
CreateDataSourceTableAsSelectCommand `spark_catalog`.`sql_on_files`.`test_json`, ErrorIfExists, [1]
178+
+- Project [1 AS 1#x]
179+
+- OneRowRelation
126180

127181

128182
-- !query
@@ -159,9 +213,21 @@ org.apache.spark.sql.AnalysisException
159213

160214

161215
-- !query
162-
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1
216+
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`
217+
-- !query analysis
218+
Project [1#xL]
219+
+- Relation [1#xL] json
220+
221+
222+
-- !query
223+
DROP TABLE sql_on_files.test_json
224+
-- !query analysis
225+
DropTable false, false
226+
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), sql_on_files.test_json
227+
228+
229+
-- !query
230+
DROP DATABASE sql_on_files
163231
-- !query analysis
164-
GlobalLimit 1
165-
+- LocalLimit 1
166-
+- Project [id#xL, intervals#x]
167-
+- Relation [id#xL,intervals#x] json
232+
DropNamespace false, false
233+
+- ResolvedNamespace V2SessionCatalog(spark_catalog), [sql_on_files]

‎sql/core/src/test/resources/sql-tests/inputs/sql-on-files.sql

+15-4
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,30 @@
1+
CREATE DATABASE IF NOT EXISTS sql_on_files;
12
-- Parquet
3+
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1;
24
SELECT * FROM parquet.``;
35
SELECT * FROM parquet.`/file/not/found`;
4-
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1;
6+
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`;
7+
DROP TABLE sql_on_files.test_parquet;
58

69
-- ORC
10+
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1;
711
SELECT * FROM orc.``;
812
SELECT * FROM orc.`/file/not/found`;
9-
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1;
13+
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`;
14+
DROP TABLE sql_on_files.test_orc;
1015

1116
-- CSV
17+
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1;
1218
SELECT * FROM csv.``;
1319
SELECT * FROM csv.`/file/not/found`;
14-
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1;
20+
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`;
21+
DROP TABLE sql_on_files.test_csv;
1522

1623
-- JSON
24+
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1;
1725
SELECT * FROM json.``;
1826
SELECT * FROM json.`/file/not/found`;
19-
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1;
27+
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`;
28+
DROP TABLE sql_on_files.test_json;
29+
30+
DROP DATABASE sql_on_files;

‎sql/core/src/test/resources/sql-tests/results/sql-on-files.sql.out

+92-12
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
11
-- Automatically generated by SQLQueryTestSuite
2+
-- !query
3+
CREATE DATABASE IF NOT EXISTS sql_on_files
4+
-- !query schema
5+
struct<>
6+
-- !query output
7+
8+
9+
10+
-- !query
11+
CREATE TABLE sql_on_files.test_parquet USING PARQUET AS SELECT 1
12+
-- !query schema
13+
struct<>
14+
-- !query output
15+
16+
17+
218
-- !query
319
SELECT * FROM parquet.``
420
-- !query schema
@@ -37,11 +53,27 @@ org.apache.spark.sql.AnalysisException
3753

3854

3955
-- !query
40-
SELECT * FROM parquet.`src/test/resources/test-data/dec-in-fixed-len.parquet` LIMIT 1
56+
SELECT * FROM parquet.`${spark.sql.warehouse.dir}/sql_on_files.db/test_parquet`
4157
-- !query schema
42-
struct<fixed_len_dec:decimal(10,2)>
58+
struct<1:int>
4359
-- !query output
44-
0.00
60+
1
61+
62+
63+
-- !query
64+
DROP TABLE sql_on_files.test_parquet
65+
-- !query schema
66+
struct<>
67+
-- !query output
68+
69+
70+
71+
-- !query
72+
CREATE TABLE sql_on_files.test_orc USING ORC AS SELECT 1
73+
-- !query schema
74+
struct<>
75+
-- !query output
76+
4577

4678

4779
-- !query
@@ -82,11 +114,27 @@ org.apache.spark.sql.AnalysisException
82114

83115

84116
-- !query
85-
SELECT * FROM orc.`src/test/resources/test-data/before_1582_date_v2_4.snappy.orc` LIMIT 1
117+
SELECT * FROM orc.`${spark.sql.warehouse.dir}/sql_on_files.db/test_orc`
118+
-- !query schema
119+
struct<1:int>
120+
-- !query output
121+
1
122+
123+
124+
-- !query
125+
DROP TABLE sql_on_files.test_orc
126+
-- !query schema
127+
struct<>
128+
-- !query output
129+
130+
131+
132+
-- !query
133+
CREATE TABLE sql_on_files.test_csv USING CSV AS SELECT 1
86134
-- !query schema
87-
struct<dt:date>
135+
struct<>
88136
-- !query output
89-
1200-01-01
137+
90138

91139

92140
-- !query
@@ -127,11 +175,27 @@ org.apache.spark.sql.AnalysisException
127175

128176

129177
-- !query
130-
SELECT * FROM csv.`src/test/resources/test-data/cars.csv` LIMIT 1
178+
SELECT * FROM csv.`${spark.sql.warehouse.dir}/sql_on_files.db/test_csv`
179+
-- !query schema
180+
struct<_c0:string>
181+
-- !query output
182+
1
183+
184+
185+
-- !query
186+
DROP TABLE sql_on_files.test_csv
131187
-- !query schema
132-
struct<_c0:string,_c1:string,_c2:string,_c3:string,_c4:string>
188+
struct<>
133189
-- !query output
134-
year make model comment blank
190+
191+
192+
193+
-- !query
194+
CREATE TABLE sql_on_files.test_json USING JSON AS SELECT 1
195+
-- !query schema
196+
struct<>
197+
-- !query output
198+
135199

136200

137201
-- !query
@@ -172,8 +236,24 @@ org.apache.spark.sql.AnalysisException
172236

173237

174238
-- !query
175-
SELECT * FROM json.`src/test/resources/test-data/with-map-fields.json` LIMIT 1
239+
SELECT * FROM json.`${spark.sql.warehouse.dir}/sql_on_files.db/test_json`
176240
-- !query schema
177-
struct<id:bigint,intervals:struct<a:struct<endTime:bigint,startTime:bigint>,b:struct<endTime:bigint,startTime:bigint>>>
241+
struct<1:bigint>
242+
-- !query output
243+
1
244+
245+
246+
-- !query
247+
DROP TABLE sql_on_files.test_json
248+
-- !query schema
249+
struct<>
178250
-- !query output
179-
1 {"a":{"endTime":211,"startTime":111},"b":{"endTime":221,"startTime":121}}
251+
252+
253+
254+
-- !query
255+
DROP DATABASE sql_on_files
256+
-- !query schema
257+
struct<>
258+
-- !query output
259+

‎sql/hive-thriftserver/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc

-201 Bytes
Binary file not shown.

‎sql/hive-thriftserver/src/test/resources/test-data/cars.csv

-7
This file was deleted.

‎sql/hive-thriftserver/src/test/resources/test-data/dec-in-fixed-len.parquet

-460 Bytes
Binary file not shown.

‎sql/hive-thriftserver/src/test/resources/test-data/with-map-fields.json

-5
This file was deleted.

0 commit comments

Comments
 (0)