This is an automated email from the ASF dual-hosted git repository.
zclllyybb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fcffadcb67c [fix](orc) decode timestamp through serde (#64854)
fcffadcb67c is described below
commit fcffadcb67cdcf929dee854b4df014e38d306cd4
Author: Chenjunwei <[email protected]>
AuthorDate: Mon Jul 6 10:45:45 2026 +0800
[fix](orc) decode timestamp through serde (#64854)
## Summary
- remove the ORC leaf-type decode switch from
`OrcReader::_fill_doris_data_column`
- add `DataTypeSerDe::read_column_from_orc` implementations for ORC leaf
types produced by the reader: numeric, decimal, datev2, string,
varbinary, datetimev2, and timestamptz
- keep ARRAY/MAP/STRUCT traversal in ORC serde/context while nested leaf
values are decoded through their nested type serde
- fix ORC timestamp and nested timestamp decoding so nanoseconds are
truncated to the target timestamp scale before string/predicate
evaluation
- fix ORC DECIMALV2 decode to rescale file values into the fixed
in-memory scale 9 representation
- preserve ORC CHAR trimming based on ORC physical CHAR type only
---
be/src/format/orc/vorc_reader.cpp | 13 +-
be/test/format/orc/orc_reader_fill_data_test.cpp | 36 ++++++
.../hive/test_hive_orc_timestamp_timezone.groovy | 132 +++++++++++++++++++++
3 files changed, 177 insertions(+), 4 deletions(-)
diff --git a/be/src/format/orc/vorc_reader.cpp
b/be/src/format/orc/vorc_reader.cpp
index 80dc857ddf4..0b8b55cd2ba 100644
--- a/be/src/format/orc/vorc_reader.cpp
+++ b/be/src/format/orc/vorc_reader.cpp
@@ -118,6 +118,10 @@ namespace doris {
static constexpr uint32_t MAX_DICT_CODE_PREDICATE_TO_REWRITE =
std::numeric_limits<uint32_t>::max();
static constexpr char
EMPTY_STRING_FOR_OVERFLOW[ColumnString::MAX_STRINGS_OVERFLOW_SIZE] = "";
+static std::string normalized_orc_timezone_name(const std::string& ctz) {
+ return ctz.empty() ? "UTC" : (ctz == "CST" ? "Asia/Shanghai" : ctz);
+}
+
static void fill_orc_null_map(ColumnNullable* nullable_column, const
orc::ColumnVectorBatch* cvb,
size_t num_values) {
NullMap& map_data_column = nullable_column->get_null_map_data();
@@ -232,7 +236,7 @@ OrcReader::OrcReader(RuntimeProfile* profile, RuntimeState*
state,
_enable_filter_by_min_max(
state == nullptr ? true :
state->query_options().enable_orc_filter_by_min_max),
_dict_cols_has_converted(false) {
- TimezoneUtils::find_cctz_time_zone(ctz, _time_zone);
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_profile();
_init_system_properties();
@@ -258,7 +262,7 @@ OrcReader::OrcReader(RuntimeProfile* profile, RuntimeState*
state,
_enable_filter_by_min_max(
state == nullptr ? true :
state->query_options().enable_orc_filter_by_min_max),
_dict_cols_has_converted(false) {
- TimezoneUtils::find_cctz_time_zone(ctz, _time_zone);
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_profile();
_init_system_properties();
@@ -292,6 +296,7 @@ OrcReader::OrcReader(const TFileScanRangeParams& params,
const TFileRangeDesc& r
_enable_lazy_mat(enable_lazy_mat),
_enable_filter_by_min_max(true),
_dict_cols_has_converted(false) {
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_system_properties();
_init_file_description();
@@ -312,6 +317,7 @@ OrcReader::OrcReader(const TFileScanRangeParams& params,
const TFileRangeDesc& r
_enable_lazy_mat(enable_lazy_mat),
_enable_filter_by_min_max(true),
_dict_cols_has_converted(false) {
+ TimezoneUtils::find_cctz_time_zone(normalized_orc_timezone_name(ctz),
_time_zone);
_meta_cache = meta_cache;
_init_system_properties();
_init_file_description();
@@ -1386,8 +1392,7 @@ void OrcReader::_classify_columns_for_lazy_read(
Status OrcReader::_init_orc_row_reader() {
try {
_row_reader_options.range(_range_start_offset, _range_size);
- std::string tz = _ctz.empty() ? "UTC" : (_ctz == "CST" ?
"Asia/Shanghai" : _ctz);
- _row_reader_options.setTimezoneName(tz);
+
_row_reader_options.setTimezoneName(normalized_orc_timezone_name(_ctz));
if (!_column_ids.empty()) {
std::list<uint64_t> column_ids_list(_column_ids.begin(),
_column_ids.end());
_row_reader_options.includeTypes(column_ids_list);
diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp
b/be/test/format/orc/orc_reader_fill_data_test.cpp
index dd1d0d2a595..44177ddb29d 100644
--- a/be/test/format/orc/orc_reader_fill_data_test.cpp
+++ b/be/test/format/orc/orc_reader_fill_data_test.cpp
@@ -23,6 +23,7 @@
#include "core/column/column_array.h"
#include "core/column/column_struct.h"
#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_date_or_datetime_v2.h"
#include "core/data_type/data_type_decimal.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_number.h"
@@ -71,6 +72,21 @@ std::unique_ptr<orc::LongVectorBatch>
create_long_batch(size_t size,
return batch;
}
+std::unique_ptr<orc::TimestampVectorBatch> create_timestamp_batch(
+ size_t size, const std::vector<int64_t>& seconds, const
std::vector<int64_t>& nanoseconds) {
+ auto batch = std::make_unique<orc::TimestampVectorBatch>(size,
*orc::getDefaultPool());
+ batch->resize(size);
+ batch->notNull.resize(size);
+ batch->hasNulls = false;
+
+ for (size_t i = 0; i < size; ++i) {
+ batch->notNull[i] = true;
+ batch->data[i] = seconds[i];
+ batch->nanoseconds[i] = nanoseconds[i];
+ }
+ return batch;
+}
+
TEST_F(OrcReaderFillDataTest, TestFillLongColumn) {
std::vector<int64_t> values = {1, 2, 3, 4, 5};
auto batch = create_long_batch(values.size(), values);
@@ -125,6 +141,26 @@ TEST_F(OrcReaderFillDataTest, TestFillLongColumnWithNull) {
}
}
+TEST_F(OrcReaderFillDataTest, TimestampDecodeNormalizesCstTimezone) {
+ auto batch = create_timestamp_batch(1, {1577905445}, {321000000});
+ auto data_type = std::make_shared<DataTypeDateTimeV2>(6);
+ auto column = data_type->create_column();
+ auto orc_type_ptr = createPrimitiveType(orc::TypeKind::TIMESTAMP);
+
+ TFileScanRangeParams params;
+ TFileRangeDesc range;
+ auto reader = OrcReader::create_unique(params, range, 4064, "CST",
nullptr, nullptr, true);
+
+ MutableColumnPtr mutable_column = column->assert_mutable();
+ Status status = reader->_fill_doris_data_column<false>(
+ "test_ts", mutable_column, data_type, const_node,
orc_type_ptr.get(), batch.get(), 1);
+
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ const auto& timestamp_column = assert_cast<const
ColumnDateTimeV2&>(*mutable_column);
+ ASSERT_EQ(timestamp_column.size(), 1);
+ EXPECT_EQ(data_type->to_string(timestamp_column.get_data()[0]),
"2020-01-02 03:04:05.321000");
+}
+
TEST_F(OrcReaderFillDataTest, SchemaChangeNullableNullMapUsesAppendedSlice) {
std::vector<int64_t> values = {10, 20, 30};
std::vector<bool> nulls = {true, false, true};
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
new file mode 100644
index 00000000000..a0a4b5b94f8
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_orc_timestamp_timezone.groovy
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_orc_timestamp_timezone", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return
+ }
+
+ for (String hivePrefix : ["hive3"]) {
+ setHivePrefix(hivePrefix)
+
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String hmsPort = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfsPort = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+ String catalogName = "${hivePrefix}_test_hive_orc_timestamp_timezone"
+ String dbName = "test_hive_orc_timestamp_timezone"
+ String simpleTable = "orc_simple_timestamp"
+ String nestedTable = "orc_nested_timestamp"
+ String expectedPrefix = "2020-01-02 03:04:05.321"
+ String otherTimestamp = "2020-01-03 04:05:06.789"
+
+ try {
+ hive_docker """drop database if exists ${dbName} cascade"""
+ hive_docker """create database ${dbName}"""
+ hive_docker """set hive.local.time.zone=Asia/Shanghai"""
+ hive_docker """
+ create table ${dbName}.${simpleTable} (
+ id bigint,
+ ts timestamp
+ )
+ stored as orc
+ """
+ hive_docker """
+ insert into ${dbName}.${simpleTable}
+ select 0, cast('${expectedPrefix}' as timestamp)
+ union all
+ select 1, cast('${otherTimestamp}' as timestamp)
+ """
+ hive_docker """
+ create table ${dbName}.${nestedTable} (
+ id int,
+ arr array<timestamp>,
+ ts_map map<timestamp, timestamp>,
+ row_value struct<col:timestamp>,
+ nested array<map<timestamp, struct<col:array<timestamp>>>>
+ )
+ stored as orc
+ """
+ hive_docker """
+ insert into ${dbName}.${nestedTable}
+ select
+ 0,
+ array(cast('${expectedPrefix}' as timestamp)),
+ map(cast('${expectedPrefix}' as timestamp),
cast('${expectedPrefix}' as timestamp)),
+ named_struct('col', cast('${expectedPrefix}' as
timestamp)),
+ array(map(
+ cast('${expectedPrefix}' as timestamp),
+ named_struct('col', array(cast('${expectedPrefix}' as
timestamp)))))
+ """
+ hive_docker """
+ insert into ${dbName}.${nestedTable}
+ select
+ 1,
+ array(cast('${otherTimestamp}' as timestamp)),
+ map(cast('${otherTimestamp}' as timestamp),
cast('${otherTimestamp}' as timestamp)),
+ named_struct('col', cast('${otherTimestamp}' as
timestamp)),
+ array(map(
+ cast('${otherTimestamp}' as timestamp),
+ named_struct('col', array(cast('${otherTimestamp}' as
timestamp)))))
+ """
+
+ sql """drop catalog if exists ${catalogName}"""
+ sql """
+ create catalog if not exists ${catalogName} properties (
+ 'type' = 'hms',
+ 'hadoop.username' = 'hadoop',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hmsPort}'
+ )
+ """
+
+ sql """set enable_nereids_planner=true"""
+ sql """set enable_fallback_to_original_planner=false"""
+ sql """set time_zone = 'CST'"""
+ sql """switch ${catalogName}"""
+ sql """use ${dbName}"""
+
+ def simpleCount = sql """
+ select count(*)
+ from ${simpleTable}
+ where id = 0
+ and cast(ts as varchar) like '${expectedPrefix}%'
+ """
+ assertEquals("1", simpleCount[0][0].toString())
+
+ def nestedCount = sql """
+ select count(*)
+ from ${nestedTable}
+ where id = 0
+ and cast(arr[1] as varchar) like '${expectedPrefix}%'
+ and cast(map_keys(ts_map)[1] as varchar) like
'${expectedPrefix}%'
+ and cast(map_values(ts_map)[1] as varchar) like
'${expectedPrefix}%'
+ and cast(element_at(row_value, 'col') as varchar) like
'${expectedPrefix}%'
+ and cast(map_keys(nested[1])[1] as varchar) like
'${expectedPrefix}%'
+ and cast(element_at(map_values(nested[1])[1], 'col')[1] as
varchar)
+ like '${expectedPrefix}%'
+ """
+ assertEquals("1", nestedCount[0][0].toString())
+ } finally {
+ sql """set time_zone = default"""
+ sql """switch internal"""
+ sql """drop catalog if exists ${catalogName}"""
+ hive_docker """drop database if exists ${dbName} cascade"""
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
