This is an automated email from the ASF dual-hosted git repository.
saihemanth-cloudera pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5a3af40bb15 HIVE-30020: Fix TIMESTAMP WITH LOCAL TIME ZONE data loss
and read failure on ORC tables (#6761)
5a3af40bb15 is described below
commit 5a3af40bb15d986fde8e886897c0e22bc7ebacce
Author: Vikram Ahuja <[email protected]>
AuthorDate: Thu Sep 24 23:08:22 2026 +0530
HIVE-30020: Fix TIMESTAMP WITH LOCAL TIME ZONE data loss and read failure
on ORC tables (#6761)
* HIVE-30020: Fix TIMESTAMP WITH LOCAL TIME ZONE data loss and read failure
on ORC tables
* Retrigger CI
* Retrigger CI again
---
.../apache/hadoop/hive/ql/io/orc/OrcStruct.java | 2 +
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 44 ++++++++-
.../apache/hadoop/hive/ql/io/orc/WriterImpl.java | 15 +++
.../clientpositive/timestampz_with_file_formats.q | 19 ++++
.../llap/timestampz_with_file_formats.q.out | 106 +++++++++++++++++++++
5 files changed, 181 insertions(+), 5 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
index 12291e7f494..700b1bcd97e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
@@ -601,6 +601,8 @@ static ObjectInspector createObjectInspector(int columnId,
TypeInfoFactory.getVarcharTypeInfo(type.getMaximumLength()));
case TIMESTAMP:
return
PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
+ case TIMESTAMP_INSTANT:
+ return
PrimitiveObjectInspectorFactory.writableTimestampTZObjectInspector;
case DATE:
return PrimitiveObjectInspectorFactory.writableDateObjectInspector;
case DECIMAL:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index 190bec88218..6c0685dc5cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -19,12 +19,15 @@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -46,6 +49,7 @@
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -63,6 +67,10 @@ public class RecordReaderImpl extends
org.apache.orc.impl.RecordReaderImpl
private final VectorizedRowBatch batch;
private int rowInBatch;
private long baseRow;
+ // TIMESTAMP WITH LOCAL TIME ZONE is stored on disk as a UTC instant; this
is the session's
+ // display zone (hive.local.time.zone) to render that instant in, not the
JVM default zone,
+ // since SET TIME ZONE can change the former without touching the latter.
+ private final ZoneId localTimeZone;
protected RecordReaderImpl(ReaderImpl fileReader,
Reader.Options options, final Configuration conf) throws IOException {
@@ -75,6 +83,8 @@ protected RecordReaderImpl(ReaderImpl fileReader,
batch = this.schema.createRowBatch();
}
rowInBatch = 0;
+ localTimeZone = conf == null ? ZoneId.systemDefault()
+ : TimestampTZUtil.parseTimeZone(HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE));
}
/**
@@ -466,7 +476,29 @@ static TimestampWritableV2 nextTimestamp(ColumnVector
vector,
}
}
- static OrcStruct nextStruct(ColumnVector vector,
+ TimestampLocalTZWritable nextTimestampLocalTZ(ColumnVector vector,
+ int row,
+ Object previous) {
+ if (vector.isRepeating) {
+ row = 0;
+ }
+ if (vector.noNulls || !vector.isNull[row]) {
+ TimestampLocalTZWritable result;
+ if (previous == null || previous.getClass() !=
TimestampLocalTZWritable.class) {
+ result = new TimestampLocalTZWritable();
+ } else {
+ result = (TimestampLocalTZWritable) previous;
+ }
+ TimestampColumnVector tcv = (TimestampColumnVector) vector;
+ long epochSecond = Math.floorDiv(tcv.time[row], 1000L);
+ result.set(new TimestampTZ(epochSecond, tcv.nanos[row], localTimeZone));
+ return result;
+ } else {
+ return null;
+ }
+ }
+
+ OrcStruct nextStruct(ColumnVector vector,
int row,
TypeDescription schema,
Object previous) {
@@ -494,7 +526,7 @@ static OrcStruct nextStruct(ColumnVector vector,
}
}
- static OrcUnion nextUnion(ColumnVector vector,
+ OrcUnion nextUnion(ColumnVector vector,
int row,
TypeDescription schema,
Object previous) {
@@ -519,7 +551,7 @@ static OrcUnion nextUnion(ColumnVector vector,
}
}
- static ArrayList<Object> nextList(ColumnVector vector,
+ ArrayList<Object> nextList(ColumnVector vector,
int row,
TypeDescription schema,
Object previous) {
@@ -559,7 +591,7 @@ static ArrayList<Object> nextList(ColumnVector vector,
}
}
- static Map<Object,Object> nextMap(ColumnVector vector,
+ Map<Object,Object> nextMap(ColumnVector vector,
int row,
TypeDescription schema,
Object previous) {
@@ -591,7 +623,7 @@ static Map<Object,Object> nextMap(ColumnVector vector,
}
}
- static Object nextValue(ColumnVector vector,
+ Object nextValue(ColumnVector vector,
int row,
TypeDescription schema,
Object previous) {
@@ -624,6 +656,8 @@ static Object nextValue(ColumnVector vector,
return nextDate(vector, row, previous);
case TIMESTAMP:
return nextTimestamp(vector, row, previous);
+ case TIMESTAMP_INSTANT:
+ return nextTimestampLocalTZ(vector, row, previous);
case STRUCT:
return nextStruct(vector, row, schema, previous);
case UNION:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
index 24083b3ab84..780a1efc370 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -60,6 +61,7 @@
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
@@ -211,6 +213,16 @@ static void setColumn(int rowId, ColumnVector column,
.getPrimitiveJavaObject(obj).toSqlTimestamp());
break;
}
+ case TIMESTAMPLOCALTZ: {
+ TimestampColumnVector vector = (TimestampColumnVector) column;
+ vector.setIsUTC(true);
+ vector.setUsingProlepticCalendar(true);
+ TimestampTZ timestampTZ = ((TimestampLocalTZObjectInspector)
inspector)
+ .getPrimitiveJavaObject(obj);
+ vector.time[rowId] = timestampTZ.toEpochMilli();
+ vector.nanos[rowId] = timestampTZ.getNanos();
+ break;
+ }
case DATE: {
DateColumnVector vector = (DateColumnVector) column;
vector.setUsingProlepticCalendar(true);
@@ -230,6 +242,9 @@ static void setColumn(int rowId, ColumnVector column,
}
break;
}
+ default:
+ throw new IllegalArgumentException("Unknown primitive category "
+
+ ((PrimitiveObjectInspector)
inspector).getPrimitiveCategory());
}
break;
case STRUCT: {
diff --git a/ql/src/test/queries/clientpositive/timestampz_with_file_formats.q
b/ql/src/test/queries/clientpositive/timestampz_with_file_formats.q
new file mode 100644
index 00000000000..d42a3ff88a6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/timestampz_with_file_formats.q
@@ -0,0 +1,19 @@
+CREATE TABLE timestampltz_formats (
+ formatid string,
+ tsval timestamp with local time zone
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe';
+
+LOAD DATA LOCAL INPATH '../../data/files/timestamps_mixed_formats.txt'
overwrite into table timestampltz_formats;
+
+SELECT * FROM timestampltz_formats;
+
+CREATE TABLE timestampltz_orc_format (
+ formatid string,
+ tsval timestamp with local time zone
+)
+stored as orc;
+
+insert into timestampltz_orc_format select * from timestampltz_formats;
+
+SELECT * FROM timestampltz_orc_format;
\ No newline at end of file
diff --git
a/ql/src/test/results/clientpositive/llap/timestampz_with_file_formats.q.out
b/ql/src/test/results/clientpositive/llap/timestampz_with_file_formats.q.out
new file mode 100644
index 00000000000..1f9410a0ae3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/timestampz_with_file_formats.q.out
@@ -0,0 +1,106 @@
+PREHOOK: query: CREATE TABLE timestampltz_formats (
+ formatid string,
+ tsval timestamp with local time zone
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@timestampltz_formats
+POSTHOOK: query: CREATE TABLE timestampltz_formats (
+ formatid string,
+ tsval timestamp with local time zone
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@timestampltz_formats
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/timestamps_mixed_formats.txt' overwrite into table
timestampltz_formats
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@timestampltz_formats
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/timestamps_mixed_formats.txt' overwrite into table
timestampltz_formats
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@timestampltz_formats
+PREHOOK: query: SELECT * FROM timestampltz_formats
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestampltz_formats
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM timestampltz_formats
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestampltz_formats
+#### A masked pattern was here ####
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 12:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 12:26:34.12345 US/Pacific
+DEFAULT 2016-05-03 12:26:34.12345 US/Pacific
+DEFAULT 2016-05-03 12:26:34.1 US/Pacific
+DEFAULT 2016-05-03 12:26:34.0 US/Pacific
+DEFAULT 2016-05-03 12:26:34.0 US/Pacific
+DEFAULT 2016-05-03 00:00:00.0 US/Pacific
+FORMAT1 NULL
+FORMAT1 NULL
+FORMAT2 NULL
+FORMAT2 NULL
+PREHOOK: query: CREATE TABLE timestampltz_orc_format (
+ formatid string,
+ tsval timestamp with local time zone
+)
+stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@timestampltz_orc_format
+POSTHOOK: query: CREATE TABLE timestampltz_orc_format (
+ formatid string,
+ tsval timestamp with local time zone
+)
+stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@timestampltz_orc_format
+PREHOOK: query: insert into timestampltz_orc_format select * from
timestampltz_formats
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestampltz_formats
+PREHOOK: Output: default@timestampltz_orc_format
+POSTHOOK: query: insert into timestampltz_orc_format select * from
timestampltz_formats
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestampltz_formats
+POSTHOOK: Output: default@timestampltz_orc_format
+POSTHOOK: Lineage: timestampltz_orc_format.formatid SIMPLE
[(timestampltz_formats)timestampltz_formats.FieldSchema(name:formatid,
type:string, comment:null), ]
+POSTHOOK: Lineage: timestampltz_orc_format.tsval SIMPLE
[(timestampltz_formats)timestampltz_formats.FieldSchema(name:tsval,
type:timestamp with local time zone, comment:null), ]
+PREHOOK: query: SELECT * FROM timestampltz_orc_format
+PREHOOK: type: QUERY
+PREHOOK: Input: default@timestampltz_orc_format
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM timestampltz_orc_format
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@timestampltz_orc_format
+#### A masked pattern was here ####
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 04:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 12:26:34.123456789 US/Pacific
+DEFAULT 2016-05-03 12:26:34.12345 US/Pacific
+DEFAULT 2016-05-03 12:26:34.12345 US/Pacific
+DEFAULT 2016-05-03 12:26:34.1 US/Pacific
+DEFAULT 2016-05-03 12:26:34.0 US/Pacific
+DEFAULT 2016-05-03 12:26:34.0 US/Pacific
+DEFAULT 2016-05-03 00:00:00.0 US/Pacific
+FORMAT1 NULL
+FORMAT1 NULL
+FORMAT2 NULL
+FORMAT2 NULL
