This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f2dd9520882 HIVE-28368: Iceberg: Unable to read PARTITIONS Metadata
table. (#5339). (Ayush Saxena, reviewed by Sourabh Badhya)
f2dd9520882 is described below
commit f2dd9520882ad79575e4db3b57ea5d8c09a4a64e
Author: Ayush Saxena <[email protected]>
AuthorDate: Fri Jul 12 19:06:15 2024 +0530
HIVE-28368: Iceberg: Unable to read PARTITIONS Metadata table. (#5339).
(Ayush Saxena, reviewed by Sourabh Badhya)
---
.../mr/mapreduce/IcebergInternalRecordWrapper.java | 4 +++-
.../test/queries/positive/iceberg_metadata_table.q | 9 ++++++-
.../results/positive/iceberg_metadata_table.q.out | 28 ++++++++++++++++++++++
.../org/apache/hadoop/hive/ql/QOutProcessor.java | 7 +++++-
4 files changed, 45 insertions(+), 3 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
index 254bb91a4e1..ec5a880b078 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
@@ -142,7 +142,9 @@ public class IcebergInternalRecordWrapper implements
Record, StructLike {
private static Function<Object, Object> converter(Type type) {
switch (type.typeId()) {
case TIMESTAMP:
- return timestamp -> DateTimeUtil.timestampFromMicros((Long) timestamp);
+ return timestamp -> ((Types.TimestampType) type).shouldAdjustToUTC() ?
+ DateTimeUtil.timestamptzFromMicros((Long) timestamp) :
+ DateTimeUtil.timestampFromMicros((Long) timestamp);
case DATE:
return date -> DateTimeUtil.dateFromDays((Integer) date);
case STRUCT:
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
index 2d44ad4b551..9389b3fb202 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
@@ -1,4 +1,5 @@
-- SORT_QUERY_RESULTS
+-- MASK_TIMESTAMP
create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as
parquet tblproperties ('format-version'='2');
insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2,
cast('2023-07-20 00:00:00' as timestamp));
select * from ice_ts_4;
@@ -8,4 +9,10 @@ select readable_metrics from default.ice_ts_4.FILES;
select readable_metrics from default.ice_ts_4.ALL_FILES;
select readable_metrics from default.ice_ts_4.DATA_FILES;
select readable_metrics from default.ice_ts_4.ALL_DATA_FILES;
-select readable_metrics from default.ice_ts_4.DELETE_FILES;
\ No newline at end of file
+select readable_metrics from default.ice_ts_4.DELETE_FILES;
+
+-- Test partitions table
+CREATE EXTERNAL TABLE ice_part (`col1` int, `decimalA` decimal(5,2),
`decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2');
+insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 200.12);
+select last_updated_at from default.ice_part.PARTITIONS;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
index 167883a7878..5b5debfcaf4 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
@@ -88,3 +88,31 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_ts_4
POSTHOOK: Output: hdfs://### HDFS PATH ###
{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
+PREHOOK: query: CREATE EXTERNAL TABLE ice_part (`col1` int, `decimalA`
decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_part (`col1` int, `decimalA`
decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32,
200.12)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32,
200.12)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+ ###MaskedTimeStamp###
+ ###MaskedTimeStamp###
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 8a76fdd2da5..3116193576a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -60,6 +60,10 @@ public class QOutProcessor {
private static final PatternReplacementPair MASK_DATA_SIZE = new
PatternReplacementPair(
Pattern.compile(" Data size: [1-9][0-9]*"),
" Data size: ###Masked###");
+ private static final PatternReplacementPair MASK_TIMESTAMP = new
PatternReplacementPair(
+ Pattern.compile(
+ "[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])
(2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9].[0-9]{1,3} [a-zA-Z/]*"),
+ " ###MaskedTimeStamp### ");
private static final PatternReplacementPair MASK_LINEAGE = new
PatternReplacementPair(
Pattern.compile("POSTHOOK: Lineage: .*"),
"POSTHOOK: Lineage: ###Masked###");
@@ -143,7 +147,7 @@ public class QOutProcessor {
};
private enum Mask {
- STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("--
MASK_LINEAGE");
+ STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("--
MASK_LINEAGE"), TIMESTAMP("-- MASK_TIMESTAMP");
private Pattern pattern;
Mask(String pattern) {
@@ -252,6 +256,7 @@ public class QOutProcessor {
maskPattern(result, Mask.STATS, MASK_STATS);
maskPattern(result, Mask.DATASIZE, MASK_DATA_SIZE);
maskPattern(result, Mask.LINEAGE, MASK_LINEAGE);
+ maskPattern(result, Mask.TIMESTAMP, MASK_TIMESTAMP);
for (String prefix : maskIfStartsWith) {
if (result.line.startsWith(prefix)) {