This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 28ba6fb564d HIVE-27741: Invalid timezone value in to_utc_timestamp()
is treated as UTC which can lead to data consistency issues (Zoltan Ratkai,
reviewed by Shohei Okumiya, Simhadri Govindappa)
28ba6fb564d is described below
commit 28ba6fb564d56a28ecefae53cf48ddbda7bcbc44
Author: Zoltan Ratkai <[email protected]>
AuthorDate: Tue Apr 9 13:20:37 2024 +0200
HIVE-27741: Invalid timezone value in to_utc_timestamp() is treated as UTC
which can lead to data consistency issues (Zoltan Ratkai, reviewed by Shohei
Okumiya, Simhadri Govindappa)
Closes #5014
---
.../ql/udf/generic/GenericUDFFromUtcTimestamp.java | 41 ++++++++++------------
.../ql/udf/generic/GenericUDFToUtcTimestamp.java | 11 ++++--
.../generic/TestGenericUDFFromUtcTimestamp.java | 14 ++++++++
.../clientpositive/udf_from_utc_timestamp.q | 4 ---
.../queries/clientpositive/udf_to_utc_timestamp.q | 4 ---
.../llap/udf_from_utc_timestamp.q.out | 12 ++-----
.../clientpositive/llap/udf_to_utc_timestamp.q.out | 12 ++-----
7 files changed, 46 insertions(+), 52 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
index 67aec8225e0..5a9b9b76875 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
-import java.util.TimeZone;
+import java.time.ZoneId;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.common.type.TimestampTZ;
@@ -45,7 +45,7 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
private transient PrimitiveObjectInspector[] argumentOIs;
private transient TimestampConverter timestampConverter;
private transient TextConverter textConverter;
- private transient TimeZone tzUTC = TimeZone.getTimeZone("UTC");
+ protected transient ZoneId zoneIdUTC = ZoneId.of("UTC");
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
@@ -86,33 +86,33 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
}
Timestamp inputTs = ((TimestampWritableV2) converted_o0).getTimestamp();
-
String tzStr = textConverter.convert(o1).toString();
- TimeZone timezone = TimeZone.getTimeZone(tzStr);
-
- TimeZone fromTz;
- TimeZone toTz;
- if (invert()) {
- fromTz = timezone;
- toTz = tzUTC;
- } else {
- fromTz = tzUTC;
- toTz = timezone;
- }
-
- // inputTs is the year/month/day/hour/minute/second in the local timezone.
- // For this UDF we want it in the timezone represented by fromTz
- TimestampTZ fromTs = TimestampTZUtil.parse(inputTs.toString(),
fromTz.toZoneId());
+ ZoneId zoneId = ZoneId.of(tzStr, ZoneId.SHORT_IDS);
+
+ ZoneId fromTz = getFromZoneId(zoneId);
+ ZoneId toTz = getToZoneId(zoneId);
+
+ // inputTs is the year/month/day/hour/minute/second in the local zoneId.
+ // For this UDF we want it in the zoneId represented by fromTz
+ TimestampTZ fromTs = TimestampTZUtil.parse(inputTs.toString(), fromTz);
if (fromTs == null) {
return null;
}
// Now output this timestamp's millis value to the equivalent toTz.
Timestamp result = Timestamp.valueOf(
-
fromTs.getZonedDateTime().withZoneSameInstant(toTz.toZoneId()).toLocalDateTime().toString());
+
fromTs.getZonedDateTime().withZoneSameInstant(toTz).toLocalDateTime().toString());
return result;
}
+ protected ZoneId getToZoneId(ZoneId zoneId) {
+ return zoneId;
+ }
+
+ protected ZoneId getFromZoneId(ZoneId zoneId) {
+ return this.zoneIdUTC;
+ }
+
@Override
public String getDisplayString(String[] children) {
StringBuilder sb = new StringBuilder();
@@ -129,7 +129,4 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
return "from_utc_timestamp";
}
- protected boolean invert() {
- return false;
- }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
index 298514e90a0..4cddadb8e25 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUtcTimestamp.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.hadoop.hive.ql.exec.Description;
+import java.time.ZoneId;
+
@Description(name = "to_utc_timestamp",
value = "to_utc_timestamp(timestamp, string timezone) - "
+ "Assumes given timestamp is in given timezone and
converts to UTC (as of Hive 0.8.0)")
@@ -44,7 +46,12 @@ public class GenericUDFToUtcTimestamp extends
}
@Override
- protected boolean invert() {
- return true;
+ protected ZoneId getToZoneId(ZoneId zoneId) {
+ return this.zoneIdUTC;
+ }
+
+ @Override
+ protected ZoneId getFromZoneId(ZoneId zoneId) {
+ return zoneId;
}
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
index 59c1c49a277..3a527b76f7e 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
@@ -92,4 +92,18 @@ public class TestGenericUDFFromUtcTimestamp {
new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
}
+
+ @Test
+ public void testInvalidTimestamp() throws Exception {
+ ObjectInspector valueOI =
PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ GenericUDFFromUtcTimestamp udf = new GenericUDFFromUtcTimestamp();
+ ObjectInspector[] args2 = {valueOI, valueOI};
+ udf.initialize(args2);
+
+ try {
+ runAndVerify(udf, new Text("2015-03-28 17:00:00"), new
Text("InvalidTimestamp"), null);
+ }catch (Exception exception){
+ assertEquals("Unknown time-zone ID: InvalidTimestamp",
exception.getMessage());
+ }
+ }
}
diff --git a/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
b/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
index fc83722d9d2..b68d97c1e7e 100644
--- a/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
+++ b/ql/src/test/queries/clientpositive/udf_from_utc_timestamp.q
@@ -8,8 +8,6 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
from_utc_timestamp(cast(null as string), 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', cast(null as string));
@@ -18,8 +16,6 @@ from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp),
'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
from_utc_timestamp(cast(null as timestamp), 'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string));
diff --git a/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
b/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
index cca6d7db9c6..584e97f00fd 100644
--- a/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
+++ b/ql/src/test/queries/clientpositive/udf_to_utc_timestamp.q
@@ -8,8 +8,6 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
to_utc_timestamp(cast(null as string), 'PST'),
to_utc_timestamp('2012-02-11 04:30:00', cast(null as string));
@@ -18,7 +16,5 @@ to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp),
'PST'),
to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
to_utc_timestamp(cast(null as timestamp), 'PST'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string));
diff --git
a/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
b/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
index aa6a5e73aed..be7f4a8bb55 100644
--- a/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_from_utc_timestamp.q.out
@@ -39,8 +39,6 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
from_utc_timestamp(cast(null as string), 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
PREHOOK: type: QUERY
@@ -51,21 +49,17 @@ from_utc_timestamp('2012-02-11 04:30:00', 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', 'Europe/Moscow'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT+8'),
from_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-from_utc_timestamp('2012-02-11 04:30:00', ''),
-from_utc_timestamp('2012-02-11 04:30:00', '---'),
from_utc_timestamp(cast(null as string), 'PST'),
from_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
-2012-02-10 20:30:00 2012-02-11 08:30:00 2012-02-11 12:30:00
2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00 NULL
NULL
+2012-02-10 20:30:00 2012-02-11 08:30:00 2012-02-11 12:30:00
2012-02-11 04:30:00 NULL NULL
PREHOOK: query: select
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
from_utc_timestamp(cast(null as timestamp), 'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string))
PREHOOK: type: QUERY
@@ -76,14 +70,12 @@ from_utc_timestamp(cast('2012-02-11 04:30:00' as
timestamp), 'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'Europe/Moscow'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT+8'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
from_utc_timestamp(cast(null as timestamp), 'PST'),
from_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
-2012-02-10 20:30:00 2012-02-11 08:30:00 2012-02-11 12:30:00
2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00 NULL
NULL
+2012-02-10 20:30:00 2012-02-11 08:30:00 2012-02-11 12:30:00
2012-02-11 04:30:00 NULL NULL
PREHOOK: query: select
from_utc_timestamp('2012-02-11-04:30:00', 'UTC'),
from_utc_timestamp('2012-02-11-04:30:00', 'PST')
diff --git a/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
b/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
index 62b73a36acf..4f197b9a722 100644
--- a/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_to_utc_timestamp.q.out
@@ -39,8 +39,6 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
to_utc_timestamp(cast(null as string), 'PST'),
to_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
PREHOOK: type: QUERY
@@ -51,21 +49,17 @@ to_utc_timestamp('2012-02-10 20:30:00', 'PST'),
to_utc_timestamp('2012-02-11 08:30:00', 'Europe/Moscow'),
to_utc_timestamp('2012-02-11 12:30:00', 'GMT+8'),
to_utc_timestamp('2012-02-11 04:30:00', 'GMT'),
-to_utc_timestamp('2012-02-11 04:30:00', ''),
-to_utc_timestamp('2012-02-11 04:30:00', '---'),
to_utc_timestamp(cast(null as string), 'PST'),
to_utc_timestamp('2012-02-11 04:30:00', cast(null as string))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
-2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00
2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00 NULL
NULL
+2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00
2012-02-11 04:30:00 NULL NULL
PREHOOK: query: select
to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp), 'PST'),
to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
to_utc_timestamp(cast(null as timestamp), 'PST'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string))
PREHOOK: type: QUERY
@@ -76,11 +70,9 @@ to_utc_timestamp(cast('2012-02-10 20:30:00' as timestamp),
'PST'),
to_utc_timestamp(cast('2012-02-11 08:30:00' as timestamp), 'Europe/Moscow'),
to_utc_timestamp(cast('2012-02-11 12:30:00' as timestamp), 'GMT+8'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), 'GMT'),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), ''),
-to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), '---'),
to_utc_timestamp(cast(null as timestamp), 'PST'),
to_utc_timestamp(cast('2012-02-11 04:30:00' as timestamp), cast(null as
string))
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
#### A masked pattern was here ####
-2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00
2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00 NULL
NULL
+2012-02-11 04:30:00 2012-02-11 04:30:00 2012-02-11 04:30:00
2012-02-11 04:30:00 NULL NULL