Gabriel39 commented on code in PR #65446:
URL: https://github.com/apache/doris/pull/65446#discussion_r3567950286
##########
be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp:
##########
@@ -394,14 +402,16 @@ ParquetLeafReader::ParquetLeafReader(
std::shared_ptr<::parquet::internal::RecordReader> record_reader,
ParquetColumnReaderProfile profile, const cctz::time_zone* timezone,
bool enable_strict_mode,
- std::function<Status(MutableColumnPtr&, const DecodedColumnView&)>
decoded_value_appender)
+ std::function<Status(MutableColumnPtr&, const DecodedColumnView&)>
decoded_value_appender,
+ const cctz::time_zone* int96_timezone)
Review Comment:
Correction: Fixed for scanner v2. The configured INT96 timezone is forwarded
into nested value readers, and
NewParquetReaderTest.NestedInt96UsesHiveParquetTimezone covers nested INT96
decoding.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java:
##########
@@ -123,9 +132,24 @@ public void checkProperties() throws DdlException {
throw new DdlException(
"The parameter " + PARTITION_CACHE_TTL_SECOND + " is
wrong, value is " + partitionCacheTtlSecond);
}
+
+ String hiveParquetTimeZone = getHiveParquetTimeZone();
+ if (!hiveParquetTimeZone.isEmpty()) {
+ try {
+
TimeUtils.checkTimeZoneValidAndStandardize(hiveParquetTimeZone);
Review Comment:
Correction: Fixed. Supported offset shorthand is validated and canonicalized
before it is written to BE scan parameters; the FE unit test covers 8:00
becoming +08:00.
##########
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/source/HiveScanNodeTest.java:
##########
@@ -97,6 +98,23 @@ public void
testSelectedPartitionsCarryPartitionPredicateFlag() {
Assert.assertTrue(selectedPartitions.hasPartitionPredicate);
}
+ @Test
+ public void testHiveParquetTimeZoneComesFromCatalogInsteadOfSession()
throws Exception {
+ SessionVariable sv = new SessionVariable();
+ TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
+ HMSExternalTable table = Mockito.mock(HMSExternalTable.class);
+ HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class);
+ Mockito.when(table.getCatalog()).thenReturn(catalog);
+ Mockito.when(catalog.bindBrokerName()).thenReturn("");
+
Mockito.when(catalog.getHiveParquetTimeZone()).thenReturn("Asia/Shanghai");
+ desc.setTable(table);
+ HiveScanNode node = new HiveScanNode(new PlanNodeId(0), desc, false,
sv, null, ScanContext.EMPTY);
+
+ Method method =
FileQueryScanNode.class.getDeclaredMethod("getHiveParquetTimeZone");
+ method.setAccessible(true);
+ Assert.assertEquals("Asia/Shanghai", method.invoke(node));
Review Comment:
Correction: Fixed in 79a3057424a. The test now mocks
HMSExternalTable.DLAType.HIVE, so it exercises the guarded Hive catalog
timezone path.
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -1305,6 +1322,9 @@ Status ParquetReader::_process_page_index_filter(
const auto& page_locations = offset_index.page_locations;
sig_stat.col_schema = col_schema;
+ cached_page_index.ctz = col_schema->parquet_schema.type ==
tparquet::Type::INT96
Review Comment:
Correction: Resolved by narrowing the scope to scanner v2. All legacy v1
Parquet reader and page-index changes have been removed; the affected v1 files
now match origin/master.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
