This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 28ba6fb564d HIVE-27741: Invalid timezone value in to_utc_timestamp() 
is treated as UTC which can lead to data consistency issues (Zoltan Ratkai, 
reviewed by Shohei Okumiya, Simhadri Govindappa)
28ba6fb564d is described below

commit 28ba6fb564d56a28ecefae53cf48ddbda7bcbc44
Author: Zoltan Ratkai <[email protected]>
AuthorDate: Tue Apr 9 13:20:37 2024 +0200

    HIVE-27741: Invalid timezone value in to_utc_timestamp() is treated as UTC 
which can lead to data consistency issues (Zoltan Ratkai, reviewed by Shohei 
Okumiya, Simhadri Govindappa)
    
    Closes #5014
---
 .../ql/udf/generic/GenericUDFFromUtcTimestamp.java | 41 ++++++++++------------
 .../ql/udf/generic/GenericUDFToUtcTimestamp.java   | 11 ++++--
 .../generic/TestGenericUDFFromUtcTimestamp.java    | 14 ++++++++
 .../clientpositive/udf_from_utc_timestamp.q        |  4 ---
 .../queries/clientpositive/udf_to_utc_timestamp.q  |  4 ---
 .../llap/udf_from_utc_timestamp.q.out              | 12 ++-----
 .../clientpositive/llap/udf_to_utc_timestamp.q.out | 12 ++-----
 7 files changed, 46 insertions(+), 52 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
index 67aec8225e0..5a9b9b76875 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hive.ql.udf.generic;
 
-import java.util.TimeZone;
+import java.time.ZoneId;
 
 import org.apache.hadoop.hive.common.type.Timestamp;
 import org.apache.hadoop.hive.common.type.TimestampTZ;
@@ -45,7 +45,7 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
   private transient PrimitiveObjectInspector[] argumentOIs;
   private transient TimestampConverter timestampConverter;
   private transient TextConverter textConverter;
-  private transient TimeZone tzUTC = TimeZone.getTimeZone("UTC");
+  protected transient ZoneId zoneIdUTC = ZoneId.of("UTC");
 
   @Override
   public ObjectInspector initialize(ObjectInspector[] arguments)
@@ -86,33 +86,33 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
     }
 
     Timestamp inputTs = ((TimestampWritableV2) converted_o0).getTimestamp();
-
     String tzStr = textConverter.convert(o1).toString();
-    TimeZone timezone = TimeZone.getTimeZone(tzStr);
-
-    TimeZone fromTz;
-    TimeZone toTz;
-    if (invert()) {
-      fromTz = timezone;
-      toTz = tzUTC;
-    } else {
-      fromTz = tzUTC;
-      toTz = timezone;
-    }
-
-    // inputTs is the year/month/day/hour/minute/second in the local timezone.
-    // For this UDF we want it in the timezone represented by fromTz
-    TimestampTZ fromTs = TimestampTZUtil.parse(inputTs.toString(), 
fromTz.toZoneId());
+    ZoneId zoneId = ZoneId.of(tzStr, ZoneId.SHORT_IDS);
+    
+    ZoneId fromTz = getFromZoneId(zoneId);
+    ZoneId toTz = getToZoneId(zoneId);
+    
+    // inputTs is the year/month/day/hour/minute/second in the local zoneId.
+    // For this UDF we want it in the zoneId represented by fromTz
+    TimestampTZ fromTs = TimestampTZUtil.parse(inputTs.toString(), fromTz);
     if (fromTs == null) {
       return null;
     }
 
     // Now output this timestamp's millis value to the equivalent toTz.
     Timestamp result = Timestamp.valueOf(
-        
fromTs.getZonedDateTime().withZoneSameInstant(toTz.toZoneId()).toLocalDateTime().toString());
+        
fromTs.getZonedDateTime().withZoneSameInstant(toTz).toLocalDateTime().toString());
     return result;
   }
 
+  protected ZoneId getToZoneId(ZoneId zoneId) {
+    return zoneId;
+  }
+
+  protected ZoneId getFromZoneId(ZoneId zoneId) {
+    return this.zoneIdUTC;
+  }
+
   @Override
   public String getDisplayString(String[] children) {
     StringBuilder sb = new StringBuilder();
@@ -129,7 +129,4 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
     return "from_utc_timestamp";
   }
 
-  protected boolean invert() {
-    return false;
-  }
 }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
index 298514e90a0..4cddadb8e25 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.udf.generic;
 
 import org.apache.hadoop.hive.ql.exec.Description;
 
+import java.time.ZoneId;
+
 @Description(name = "to_utc_timestamp",
              value = "to_utc_timestamp(timestamp, string timezone) - "
                      + "Assumes given timestamp is in given timezone and 
converts to UTC (as of Hive 0.8.0)")
@@ -44,7 +46,12 @@ public class GenericUDFToUtcTimestamp extends
   }
 
   @Override
-  protected boolean invert() {
-    return true;
+  protected ZoneId getToZoneId(ZoneId zoneId) {
+    return this.zoneIdUTC;
+  }
+
+  @Override
+  protected ZoneId getFromZoneId(ZoneId zoneId) {
+    return zoneId;
   }
 }
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
index 59c1c49a277..3a527b76f7e 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
@@ -92,4 +92,18 @@ public class TestGenericUDFFromUtcTimestamp {
         new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
         Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
   }
+
+  @Test
+  public void testInvalidTimestamp() throws Exception {
+    ObjectInspector valueOI = 
PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+    GenericUDFFromUtcTimestamp udf = new GenericUDFFromUtcTimestamp();
+    ObjectInspector[] args2 = {valueOI, valueOI};
+    udf.initialize(args2);
+
+    try {
+      runAndVerify(udf, new Text("2015-03-28 17:00:00"), new 
Text("InvalidTimestamp"), null);
+    }catch (Exception exception){
+      assertEquals("Unknown time-zone ID: InvalidTimestamp", 
exception.getMessage());
+    }
+  }
 }
diff --git a/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q 
b/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
index fc83722d9d2..b68d97c1e7e 100644
--- a/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
+++ b/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
@@ -8,8 +8,6 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
 from_utc_timestamp(cast(null as string), 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', cast(null as string));
 
@@ -18,8 +16,6 @@ from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 
'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 from_utc_timestamp(cast(null as timestamp), 'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string));
 
diff --git a/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q 
b/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
index cca6d7db9c6..584e97f00fd 100644
--- a/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
+++ b/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
@@ -8,8 +8,6 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
 to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
 to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
 to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
 to_utc_timestamp(cast(null as string), 'PST'),
 to_utc_timestamp('2012-02-11 04:30:00', cast(null as string));
 
@@ -18,7 +16,5 @@ to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp), 
'PST'),
 to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
 to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 to_utc_timestamp(cast(null as timestamp), 'PST'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string));
diff --git 
a/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out 
b/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
index aa6a5e73aed..be7f4a8bb55 100644
--- a/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
@@ -39,8 +39,6 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
 from_utc_timestamp(cast(null as string), 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
 PREHOOK: type: QUERY
@@ -51,21 +49,17 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
 from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
 from_utc_timestamp(cast(null as string), 'PST'),
 from_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-2012-02-10 20:30:00    2012-02-11 08:30:00     2012-02-11 12:30:00     
2012-02-11 04:30:00     2012-02-11 04:30:00     2012-02-11 04:30:00     NULL    
NULL
+2012-02-10 20:30:00    2012-02-11 08:30:00     2012-02-11 12:30:00     
2012-02-11 04:30:00     NULL    NULL
 PREHOOK: query: select
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 from_utc_timestamp(cast(null as timestamp), 'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string))
 PREHOOK: type: QUERY
@@ -76,14 +70,12 @@ from_utc_timestamp(cast('2012-02-11 04:30:00' as 
timestamp), 'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 from_utc_timestamp(cast(null as timestamp), 'PST'),
 from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string))
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-2012-02-10 20:30:00    2012-02-11 08:30:00     2012-02-11 12:30:00     
2012-02-11 04:30:00     2012-02-11 04:30:00     2012-02-11 04:30:00     NULL    
NULL
+2012-02-10 20:30:00    2012-02-11 08:30:00     2012-02-11 12:30:00     
2012-02-11 04:30:00     NULL    NULL
 PREHOOK: query: select
 from_utc_timestamp('2012-02-11-04:30:00', 'UTC'),
 from_utc_timestamp('2012-02-11-04:30:00', 'PST')
diff --git a/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out 
b/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
index 62b73a36acf..4f197b9a722 100644
--- a/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
@@ -39,8 +39,6 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
 to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
 to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
 to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
 to_utc_timestamp(cast(null as string), 'PST'),
 to_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
 PREHOOK: type: QUERY
@@ -51,21 +49,17 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
 to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
 to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
 to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
 to_utc_timestamp(cast(null as string), 'PST'),
 to_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-2012-02-11 04:30:00    2012-02-11 04:30:00     2012-02-11 04:30:00     
2012-02-11 04:30:00     2012-02-11 04:30:00     2012-02-11 04:30:00     NULL    
NULL
+2012-02-11 04:30:00    2012-02-11 04:30:00     2012-02-11 04:30:00     
2012-02-11 04:30:00     NULL    NULL
 PREHOOK: query: select
 to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp), 'PST'),
 to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
 to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 to_utc_timestamp(cast(null as timestamp), 'PST'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string))
 PREHOOK: type: QUERY
@@ -76,11 +70,9 @@ to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp), 
'PST'),
 to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
 to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
 to_utc_timestamp(cast(null as timestamp), 'PST'),
 to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as 
string))
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-2012-02-11 04:30:00    2012-02-11 04:30:00     2012-02-11 04:30:00     
2012-02-11 04:30:00     2012-02-11 04:30:00     2012-02-11 04:30:00     NULL    
NULL
+2012-02-11 04:30:00    2012-02-11 04:30:00     2012-02-11 04:30:00     
2012-02-11 04:30:00     NULL    NULL

Reply via email to