This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new f2dd9520882 HIVE-28368: Iceberg: Unable to read PARTITIONS Metadata 
table. (#5339). (Ayush Saxena, reviewed by Sourabh Badhya)
f2dd9520882 is described below

commit f2dd9520882ad79575e4db3b57ea5d8c09a4a64e
Author: Ayush Saxena <[email protected]>
AuthorDate: Fri Jul 12 19:06:15 2024 +0530

    HIVE-28368: Iceberg: Unable to read PARTITIONS Metadata table. (#5339). 
(Ayush Saxena, reviewed by Sourabh Badhya)
---
 .../mr/mapreduce/IcebergInternalRecordWrapper.java |  4 +++-
 .../test/queries/positive/iceberg_metadata_table.q |  9 ++++++-
 .../results/positive/iceberg_metadata_table.q.out  | 28 ++++++++++++++++++++++
 .../org/apache/hadoop/hive/ql/QOutProcessor.java   |  7 +++++-
 4 files changed, 45 insertions(+), 3 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
index 254bb91a4e1..ec5a880b078 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInternalRecordWrapper.java
@@ -142,7 +142,9 @@ public class IcebergInternalRecordWrapper implements 
Record, StructLike {
   private static Function<Object, Object> converter(Type type) {
     switch (type.typeId()) {
       case TIMESTAMP:
-        return timestamp -> DateTimeUtil.timestampFromMicros((Long) timestamp);
+        return timestamp -> ((Types.TimestampType) type).shouldAdjustToUTC() ?
+            DateTimeUtil.timestamptzFromMicros((Long) timestamp) :
+            DateTimeUtil.timestampFromMicros((Long) timestamp);
       case DATE:
         return date -> DateTimeUtil.dateFromDays((Integer) date);
       case STRUCT:
diff --git 
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
index 2d44ad4b551..9389b3fb202 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_metadata_table.q
@@ -1,4 +1,5 @@
 -- SORT_QUERY_RESULTS
+-- MASK_TIMESTAMP
 create table ice_ts_4 (id int, ts timestamp ) stored by iceberg stored as 
parquet tblproperties ('format-version'='2');
 insert into ice_ts_4 values (1, cast('2023-07-20 00:00:00' as timestamp)), (2, 
cast('2023-07-20 00:00:00' as timestamp));
 select * from ice_ts_4;
@@ -8,4 +9,10 @@ select readable_metrics from default.ice_ts_4.FILES;
 select readable_metrics from default.ice_ts_4.ALL_FILES;
 select readable_metrics from default.ice_ts_4.DATA_FILES;
 select readable_metrics from default.ice_ts_4.ALL_DATA_FILES;
-select readable_metrics from default.ice_ts_4.DELETE_FILES;
\ No newline at end of file
+select readable_metrics from default.ice_ts_4.DELETE_FILES;
+
+-- Test partitions table
+CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` decimal(5,2), 
`decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2');
+insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 200.12);
+select last_updated_at from default.ice_part.PARTITIONS;
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
index 167883a7878..5b5debfcaf4 100644
--- 
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_metadata_table.q.out
@@ -88,3 +88,31 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@ice_ts_4
 POSTHOOK: Output: hdfs://### HDFS PATH ###
 
{"id":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null},"ts":{"column_size":null,"value_count":null,"null_value_count":null,"nan_value_count":null,"lower_bound":null,"upper_bound":null}}
+PREHOOK: query: CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` 
decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_part  (`col1` int, `decimalA` 
decimal(5,2), `decimalC` decimal(5,2)) PARTITIONED BY SPEC
+(decimalC) stored by iceberg tblproperties('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 
200.12)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_part
+POSTHOOK: query: insert into ice_part values(1, 122.91, 102.21), (1, 12.32, 
200.12)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_part
+PREHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_part
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select last_updated_at from default.ice_part.PARTITIONS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_part
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+  ###MaskedTimeStamp### 
+  ###MaskedTimeStamp### 
diff --git 
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java 
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index 8a76fdd2da5..3116193576a 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -60,6 +60,10 @@ public class QOutProcessor {
   private static final PatternReplacementPair MASK_DATA_SIZE = new 
PatternReplacementPair(
       Pattern.compile(" Data size: [1-9][0-9]*"),
       " Data size: ###Masked###");
+    private static final PatternReplacementPair MASK_TIMESTAMP = new 
PatternReplacementPair(
+      Pattern.compile(
+          "[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) 
(2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9].[0-9]{1,3} [a-zA-Z/]*"),
+        "  ###MaskedTimeStamp### ");
   private static final PatternReplacementPair MASK_LINEAGE = new 
PatternReplacementPair(
       Pattern.compile("POSTHOOK: Lineage: .*"),
       "POSTHOOK: Lineage: ###Masked###");
@@ -143,7 +147,7 @@ public class QOutProcessor {
   };
 
   private enum Mask {
-    STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("-- 
MASK_LINEAGE");
+    STATS("-- MASK_STATS"), DATASIZE("-- MASK_DATA_SIZE"), LINEAGE("-- 
MASK_LINEAGE"), TIMESTAMP("-- MASK_TIMESTAMP");
     private Pattern pattern;
 
     Mask(String pattern) {
@@ -252,6 +256,7 @@ public class QOutProcessor {
       maskPattern(result, Mask.STATS, MASK_STATS);
       maskPattern(result, Mask.DATASIZE, MASK_DATA_SIZE);
       maskPattern(result, Mask.LINEAGE,  MASK_LINEAGE);
+      maskPattern(result, Mask.TIMESTAMP, MASK_TIMESTAMP);
 
       for (String prefix : maskIfStartsWith) {
         if (result.line.startsWith(prefix)) {

Reply via email to