Skip to content

Commit 54bb487

Browse files
author
Dmitriy Fingerman
committed
HIVE-29233: Iceberg: Validate HiveRESTCatalogClient with external RESTCatalogs like Gravitino
1 parent 96cf347 commit 54bb487

File tree

10 files changed

+689
-10
lines changed

10 files changed

+689
-10
lines changed

‎iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
-- SORT_QUERY_RESULTS
2+
-- Mask neededVirtualColumns due to non-strict order
3+
--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
4+
-- Mask random uuid
5+
--! qt:replace:/(\s+'uuid'=')\S+('\s*)/$1#Masked#$2/
6+
-- Mask random uuid
7+
--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
8+
-- Mask a random snapshot id
9+
--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
10+
-- Mask added file size
11+
--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
12+
-- Mask total file size
13+
--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
14+
-- Mask current-snapshot-timestamp-ms
15+
--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
16+
--! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
17+
--! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
18+
-- Mask compaction id as they will be allocated in parallel threads
19+
--! qt:replace:/^[0-9]/#Masked#/
20+
-- Mask removed file size
21+
--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
22+
-- Mask iceberg version
23+
--! qt:replace:/(\S\"iceberg-version\\\":\\\")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))(\\\")/$1#Masked#$3/
24+
25+
set hive.stats.autogather=false;
26+
set metastore.client.impl=org.apache.iceberg.hive.client.HiveRESTCatalogClient;
27+
set metastore.catalog.default=ice01;
28+
set iceberg.catalog.ice01.type=rest;
29+
30+
--! This config is set in the driver setup (see TestIcebergRESTCatalogLlapLocalCliDriver.java)
31+
--! conf.set('iceberg.catalog.ice01.uri', <RESTServer URI>);
32+
33+
create database ice_rest;
34+
use ice_rest;
35+
36+
-----------------------------------------------------------------------------
37+
--! Creating a table without a catalog name in table properties
38+
-----------------------------------------------------------------------------
39+
40+
create table ice_orc1 (
41+
first_name string,
42+
last_name string,
43+
dept_id bigint,
44+
team_id bigint
45+
)
46+
partitioned by (company_id bigint)
47+
stored by iceberg stored as orc;
48+
49+
-----------------------------------------------------------------------------
50+
--! Creating table with a valid catalog name in table properties
51+
-----------------------------------------------------------------------------
52+
53+
create table ice_orc2 (
54+
first_name string,
55+
last_name string,
56+
dept_id bigint,
57+
team_id bigint
58+
)
59+
partitioned by (company_id bigint)
60+
stored by iceberg stored as orc
61+
TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='ice01');
62+
63+
--! Output should contain: 'type' = 'rest'
64+
show create table ice_orc2;
65+
66+
insert into ice_orc2 partition (company_id=100)
67+
VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30);
68+
69+
--! In CI, Testcontainers' .withFileSystemBind() is not able to bind the same host path to the same container path,
70+
--! so as a workaround, the .metadata.json files from container are manually synced in a daemon process,
71+
--! since the sync can take some time, need to wait for it to happen after the insert operation.
72+
! sleep 20;
73+
74+
describe formatted ice_orc2;
75+
select * from ice_orc2;
76+
77+
-----------------------------------------------------------------------------
78+
79+
show tables;
80+
drop table ice_orc1;
81+
drop table ice_orc2;
82+
show tables;
83+
84+
show databases;
85+
drop database ice_rest;
86+
show databases;

‎iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog.q‎ renamed to ‎iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q‎

File renamed without changes.

‎iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
PREHOOK: query: create database ice_rest
2+
PREHOOK: type: CREATEDATABASE
3+
PREHOOK: Output: database:ice_rest
4+
POSTHOOK: query: create database ice_rest
5+
POSTHOOK: type: CREATEDATABASE
6+
POSTHOOK: Output: database:ice_rest
7+
PREHOOK: query: use ice_rest
8+
PREHOOK: type: SWITCHDATABASE
9+
PREHOOK: Input: database:ice_rest
10+
POSTHOOK: query: use ice_rest
11+
POSTHOOK: type: SWITCHDATABASE
12+
POSTHOOK: Input: database:ice_rest
13+
PREHOOK: query: create table ice_orc1 (
14+
first_name string,
15+
last_name string,
16+
dept_id bigint,
17+
team_id bigint
18+
)
19+
partitioned by (company_id bigint)
20+
stored by iceberg stored as orc
21+
PREHOOK: type: CREATETABLE
22+
PREHOOK: Output: database:ice_rest
23+
PREHOOK: Output: ice_rest@ice_orc1
24+
POSTHOOK: query: create table ice_orc1 (
25+
first_name string,
26+
last_name string,
27+
dept_id bigint,
28+
team_id bigint
29+
)
30+
partitioned by (company_id bigint)
31+
stored by iceberg stored as orc
32+
POSTHOOK: type: CREATETABLE
33+
POSTHOOK: Output: database:ice_rest
34+
POSTHOOK: Output: ice_rest@ice_orc1
35+
PREHOOK: query: create table ice_orc2 (
36+
first_name string,
37+
last_name string,
38+
dept_id bigint,
39+
team_id bigint
40+
)
41+
partitioned by (company_id bigint)
42+
stored by iceberg stored as orc
43+
TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='ice01')
44+
PREHOOK: type: CREATETABLE
45+
PREHOOK: Output: database:ice_rest
46+
PREHOOK: Output: ice_rest@ice_orc2
47+
POSTHOOK: query: create table ice_orc2 (
48+
first_name string,
49+
last_name string,
50+
dept_id bigint,
51+
team_id bigint
52+
)
53+
partitioned by (company_id bigint)
54+
stored by iceberg stored as orc
55+
TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='ice01')
56+
POSTHOOK: type: CREATETABLE
57+
POSTHOOK: Output: database:ice_rest
58+
POSTHOOK: Output: ice_rest@ice_orc2
59+
PREHOOK: query: show create table ice_orc2
60+
PREHOOK: type: SHOW_CREATETABLE
61+
PREHOOK: Input: ice_rest@ice_orc2
62+
POSTHOOK: query: show create table ice_orc2
63+
POSTHOOK: type: SHOW_CREATETABLE
64+
POSTHOOK: Input: ice_rest@ice_orc2
65+
CREATE EXTERNAL TABLE `ice_orc2`(
66+
`first_name` string,
67+
`last_name` string,
68+
`dept_id` bigint,
69+
`team_id` bigint,
70+
`company_id` bigint)
71+
PARTITIONED BY (
72+
`company_id` bigint COMMENT 'Transform: identity')
73+
PARTITIONED BY SPEC (
74+
`company_id`)
75+
ROW FORMAT SERDE
76+
'org.apache.iceberg.mr.hive.HiveIcebergSerDe'
77+
STORED BY
78+
'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
79+
WITH SERDEPROPERTIES (
80+
'serialization.format'='1')
81+
LOCATION
82+
#### A masked pattern was here ####
83+
TBLPROPERTIES (
84+
'bucketing_version'='2',
85+
'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"first_name","required":false,"type":"string"},{"id":2,"name":"last_name","required":false,"type":"string"},{"id":3,"name":"dept_id","required":false,"type":"long"},{"id":4,"name":"team_id","required":false,"type":"long"},{"id":5,"name":"company_id","required":false,"type":"long"}]}',
86+
'default-partition-spec'='{"spec-id":0,"fields":[{"name":"company_id","transform":"identity","source-id":5,"field-id":1000}]}',
87+
'format-version'='2',
88+
'iceberg.catalog'='ice01',
89+
'iceberg.orc.files.only'='true',
90+
#### A masked pattern was here ####
91+
'name'='ice_rest.ice_orc2',
92+
'parquet.compression'='zstd',
93+
'serialization.format'='1',
94+
'snapshot-count'='0',
95+
'table_type'='ICEBERG',
96+
'type'='rest',
97+
'uuid'='#Masked#',
98+
'write.delete.mode'='merge-on-read',
99+
'write.format.default'='orc',
100+
'write.merge.mode'='merge-on-read',
101+
'write.update.mode'='merge-on-read')
102+
PREHOOK: query: insert into ice_orc2 partition (company_id=100)
103+
VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30)
104+
PREHOOK: type: QUERY
105+
PREHOOK: Input: _dummy_database@_dummy_table
106+
PREHOOK: Output: ice_rest@ice_orc2@company_id=100
107+
POSTHOOK: query: insert into ice_orc2 partition (company_id=100)
108+
VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30)
109+
POSTHOOK: type: QUERY
110+
POSTHOOK: Input: _dummy_database@_dummy_table
111+
POSTHOOK: Output: ice_rest@ice_orc2@company_id=100
112+
PREHOOK: query: describe formatted ice_orc2
113+
PREHOOK: type: DESCTABLE
114+
PREHOOK: Input: ice_rest@ice_orc2
115+
POSTHOOK: query: describe formatted ice_orc2
116+
POSTHOOK: type: DESCTABLE
117+
POSTHOOK: Input: ice_rest@ice_orc2
118+
# col_name data_type comment
119+
first_name string
120+
last_name string
121+
dept_id bigint
122+
team_id bigint
123+
company_id bigint
124+
125+
# Partition Transform Information
126+
# col_name transform_type
127+
company_id IDENTITY
128+
129+
# Detailed Table Information
130+
Database: ice_rest
131+
#### A masked pattern was here ####
132+
Retention: 0
133+
#### A masked pattern was here ####
134+
Table Type: EXTERNAL_TABLE
135+
Table Parameters:
136+
bucketing_version 2
137+
current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]}
138+
current-snapshot-id #Masked#
139+
current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"3\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"3\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
140+
current-snapshot-timestamp-ms #Masked#
141+
default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000}]}
142+
format-version 2
143+
iceberg.catalog ice01
144+
iceberg.orc.files.only true
145+
#### A masked pattern was here ####
146+
name ice_rest.ice_orc2
147+
numFiles 1
148+
numRows 3
149+
parquet.compression zstd
150+
serialization.format 1
151+
snapshot-count 1
152+
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
153+
table_type ICEBERG
154+
totalSize #Masked#
155+
type rest
156+
uuid #Masked#
157+
write.delete.mode merge-on-read
158+
write.format.default orc
159+
write.merge.mode merge-on-read
160+
write.update.mode merge-on-read
161+
162+
# Storage Information
163+
SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
164+
InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
165+
OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
166+
Compressed: No
167+
Sort Columns: []
168+
Storage Desc Params:
169+
serialization.format 1
170+
PREHOOK: query: select * from ice_orc2
171+
PREHOOK: type: QUERY
172+
PREHOOK: Input: ice_rest@ice_orc2
173+
#### A masked pattern was here ####
174+
POSTHOOK: query: select * from ice_orc2
175+
POSTHOOK: type: QUERY
176+
POSTHOOK: Input: ice_rest@ice_orc2
177+
#### A masked pattern was here ####
178+
fn1 ln1 1 10 100
179+
fn2 ln2 2 20 100
180+
fn3 ln3 3 30 100
181+
PREHOOK: query: show tables
182+
PREHOOK: type: SHOWTABLES
183+
PREHOOK: Input: database:ice_rest
184+
POSTHOOK: query: show tables
185+
POSTHOOK: type: SHOWTABLES
186+
POSTHOOK: Input: database:ice_rest
187+
ice_orc1
188+
ice_orc2
189+
PREHOOK: query: drop table ice_orc1
190+
PREHOOK: type: DROPTABLE
191+
PREHOOK: Input: ice_rest@ice_orc1
192+
PREHOOK: Output: database:ice_rest
193+
PREHOOK: Output: ice_rest@ice_orc1
194+
POSTHOOK: query: drop table ice_orc1
195+
POSTHOOK: type: DROPTABLE
196+
POSTHOOK: Input: ice_rest@ice_orc1
197+
POSTHOOK: Output: database:ice_rest
198+
POSTHOOK: Output: ice_rest@ice_orc1
199+
PREHOOK: query: drop table ice_orc2
200+
PREHOOK: type: DROPTABLE
201+
PREHOOK: Input: ice_rest@ice_orc2
202+
PREHOOK: Output: database:ice_rest
203+
PREHOOK: Output: ice_rest@ice_orc2
204+
POSTHOOK: query: drop table ice_orc2
205+
POSTHOOK: type: DROPTABLE
206+
POSTHOOK: Input: ice_rest@ice_orc2
207+
POSTHOOK: Output: database:ice_rest
208+
POSTHOOK: Output: ice_rest@ice_orc2
209+
PREHOOK: query: show tables
210+
PREHOOK: type: SHOWTABLES
211+
PREHOOK: Input: database:ice_rest
212+
POSTHOOK: query: show tables
213+
POSTHOOK: type: SHOWTABLES
214+
POSTHOOK: Input: database:ice_rest
215+
PREHOOK: query: show databases
216+
PREHOOK: type: SHOWDATABASES
217+
POSTHOOK: query: show databases
218+
POSTHOOK: type: SHOWDATABASES
219+
ice_rest
220+
PREHOOK: query: drop database ice_rest
221+
PREHOOK: type: DROPDATABASE
222+
PREHOOK: Input: database:ice_rest
223+
PREHOOK: Output: database:ice_rest
224+
POSTHOOK: query: drop database ice_rest
225+
POSTHOOK: type: DROPDATABASE
226+
POSTHOOK: Input: database:ice_rest
227+
POSTHOOK: Output: database:ice_rest
228+
PREHOOK: query: show databases
229+
PREHOOK: type: SHOWDATABASES
230+
POSTHOOK: query: show databases
231+
POSTHOOK: type: SHOWDATABASES

‎iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog.q.out‎ renamed to ‎iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out‎

File renamed without changes.

‎itests/qtest-iceberg/pom.xml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,23 @@
475475
<version>${project.version}</version>
476476
<scope>test</scope>
477477
</dependency>
478+
<dependency>
479+
<groupId>org.apache.httpcomponents.client5</groupId>
480+
<artifactId>httpclient5</artifactId>
481+
</dependency>
482+
<dependency>
483+
<groupId>org.apache.httpcomponents.core5</groupId>
484+
<artifactId>httpcore5</artifactId>
485+
</dependency>
486+
<dependency>
487+
<groupId>org.apache.httpcomponents.core5</groupId>
488+
<artifactId>httpcore5-h2</artifactId>
489+
</dependency>
490+
<dependency>
491+
<groupId>org.testcontainers</groupId>
492+
<artifactId>testcontainers</artifactId>
493+
<scope>test</scope>
494+
</dependency>
478495
</dependencies>
479496
<build>
480497
<plugins>
@@ -525,6 +542,30 @@
525542
</additionalClasspathElements>
526543
</configuration>
527544
</plugin>
545+
<plugin>
546+
<groupId>org.apache.maven.plugins</groupId>
547+
<artifactId>maven-dependency-plugin</artifactId>
548+
<executions>
549+
<execution>
550+
<id>copy-h2-driver-for-test</id>
551+
<phase>process-test-resources</phase>
552+
<goals>
553+
<goal>copy</goal>
554+
</goals>
555+
<configuration>
556+
<artifactItems>
557+
<artifactItem>
558+
<groupId>com.h2database</groupId>
559+
<artifactId>h2</artifactId>
560+
<version>2.2.224</version>
561+
<destFileName>h2-driver.jar</destFileName>
562+
</artifactItem>
563+
</artifactItems>
564+
<outputDirectory>${project.build.directory}/test-dependencies</outputDirectory>
565+
</configuration>
566+
</execution>
567+
</executions>
568+
</plugin>
528569
</plugins>
529570
</build>
530571
</project>

0 commit comments

Comments
 (0)