This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6d42dc3 HIVE-21729: Arrow serializer sometimes shifts timestamp by
one second (Shubham Chaurasia, reviewed by Sankar Hariappan)
6d42dc3 is described below
commit 6d42dc3ebb2d31c258a5cdd07ea4461dc2d72d87
Author: Shubham Chaurasia <schaura...@cloudera.com>
AuthorDate: Wed May 15 17:20:26 2019 +0530
HIVE-21729: Arrow serializer sometimes shifts timestamp by one second
(Shubham Chaurasia, reviewed by Sankar Hariappan)
Signed-off-by: Sankar Hariappan <sank...@apache.org>
---
.../apache/hadoop/hive/ql/io/arrow/Serializer.java | 7 ++--
.../ql/io/arrow/TestArrowColumnarBatchSerDe.java | 39 ++++++++++++++++++++++
2 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
index c5079e1..5ec800d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
@@ -816,8 +816,11 @@ public class Serializer {
final TimestampColumnVector timestampColumnVector =
(TimestampColumnVector) hiveVector;
// Time = second + sub-second
final long secondInMillis = timestampColumnVector.getTime(j);
- final long secondInMicros = (secondInMillis - secondInMillis %
MILLIS_PER_SECOND) * MICROS_PER_MILLIS;
- final long subSecondInMicros = timestampColumnVector.getNanos(j) /
NS_PER_MICROS;
+ final long nanos = timestampColumnVector.getNanos(j);
+ // nanos should be subtracted from total time(secondInMillis) to obtain
micros
+ // Divide nanos by 1000_000 to bring it to millisecond precision and then
perform the subtraction
+ final long secondInMicros = (secondInMillis - nanos / NS_PER_MILLIS) *
MICROS_PER_MILLIS;
+ final long subSecondInMicros = nanos / NS_PER_MICROS;
if ((secondInMillis > 0 && secondInMicros < 0) || (secondInMillis < 0 &&
secondInMicros > 0)) {
// If the timestamp cannot be represented in long microsecond, set it as
a null value
timeStampMicroTZVector.setNull(i);
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java
index 2e011b5..f1cb6df 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java
@@ -517,6 +517,45 @@ public class TestArrowColumnarBatchSerDe {
initAndSerializeAndDeserialize(schema, rows);
}
+
+ @Test
+ public void testTimestampNanosPrecisionUpTo6Digits() throws SerDeException {
+ String[][] schema = {
+ {"timestamp1", "timestamp"},
+ };
+ //Nanos precise upto 6 digits
+ Object[][] tsRows = new Object[][]{
+ {new TimestampWritableV2(Timestamp.valueOf("1800-04-01
09:01:10.123999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("2050-04-01
09:01:10.999999"))},
+ null
+ };
+ initAndSerializeAndDeserialize(schema, tsRows);
+ }
+
+ @Test
+ public void testPositiveNegativeTSWithNanos() throws SerDeException {
+ String[][] schema = {
+ {"timestamp1", "timestamp"},
+ };
+
+ Object[][] tsRows = new Object[][]{
+ {new TimestampWritableV2(Timestamp.valueOf("1963-04-01
09:01:10.123"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1800-04-01
09:01:10.123999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1750-04-01
09:01:10.123999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1700-04-01
09:01:10.999999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("2050-04-01
09:01:10.999999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1991-06-05
09:01:10.999999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1992-11-04
09:01:10.999999"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1970-01-01 00:00:00"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1964-01-01 00:00:04.78"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1950-01-01 09:23:03.21"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1956-01-01 10:09:03.00"))},
+ {new TimestampWritableV2(Timestamp.valueOf("1947-08-27 10:25:36.26"))},
+ null
+ };
+ initAndSerializeAndDeserialize(schema, tsRows);
+ }
+
@Test
public void testPrimitiveDecimal() throws SerDeException {
String[][] schema = {
