Skip to content

Commit

Permalink
HIVE-28353: Iceberg: Reading *Files Metadata table files if the colum…
Browse files Browse the repository at this point in the history 
…n is of TIMESTAMP type. (#5325). (Ayush Saxena, reviewed by Simhadri Govindappa)
  • Loading branch information
@ayushtkn
ayushtkn committed Jun 29, 2024
1 parent d14f303 commit c5be1fc
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 1 deletion.
2 changes: 1 addition & 1 deletion ...g-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ private Map<String, Integer> buildFieldPositionMap(StructType schema) {
private static Function<Object, Object> converter(Type type) {
switch (type.typeId()) {
case TIMESTAMP:
return timestamp -> DateTimeUtil.timestamptzFromMicros((Long) timestamp);
return timestamp -> DateTimeUtil.timestampFromMicros((Long) timestamp);
case DATE:
return date -> DateTimeUtil.dateFromDays((Integer) date);
case STRUCT:
Expand Down
11 changes: 11 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
-- SORT_QUERY_RESULTS
create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2');
insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp));
select * from ice_ts_4;
delete from ice_ts_4 where id = 2;
select * from ice_ts_4;
select readable_metrics from default.ice_ts_4.FILES;
select readable_metrics from default.ice_ts_4.ALL_FILES;
select readable_metrics from default.ice_ts_4.DATA_FILES;
select readable_metrics from default.ice_ts_4.ALL_DATA_FILES;
select readable_metrics from default.ice_ts_4.DELETE_FILES;
90 changes: 90 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
PREHOOK: query: create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@ice_ts_4
POSTHOOK: query: create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as parquet tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_ts_4
PREHOOK: query: insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp))
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_ts_4
POSTHOOK: query: insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, cast('2023-07-20 00:00:00' as timestamp))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_ts_4
PREHOOK: query: select * from ice_ts_4
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from ice_ts_4
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 2023-07-20 00:00:00
2 2023-07-20 00:00:00
PREHOOK: query: delete from ice_ts_4 where id = 2
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: default@ice_ts_4
POSTHOOK: query: delete from ice_ts_4 where id = 2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: default@ice_ts_4
PREHOOK: query: select * from ice_ts_4
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from ice_ts_4
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 2023-07-20 00:00:00
PREHOOK: query: select readable_metrics from default.ice_ts_4.FILES
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select readable_metrics from default.ice_ts_4.FILES
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
PREHOOK: query: select readable_metrics from default.ice_ts_4.ALL_FILES
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select readable_metrics from default.ice_ts_4.ALL_FILES
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
PREHOOK: query: select readable_metrics from default.ice_ts_4.DATA_FILES
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select readable_metrics from default.ice_ts_4.DATA_FILES
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
PREHOOK: query: select readable_metrics from default.ice_ts_4.ALL_DATA_FILES
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select readable_metrics from default.ice_ts_4.ALL_DATA_FILES
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":46,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":2},"ts":{"column_size":76,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"2023-07-20 00:00:00","upper_bound":"2023-07-20 00:00:00"}}
PREHOOK: query: select readable_metrics from default.ice_ts_4.DELETE_FILES
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_ts_4
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select readable_metrics from default.ice_ts_4.DELETE_FILES
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}

0 comments on commit c5be1fc

Please sign in to comment.