OIiveirra commented on code in PR #68161:
URL: https://github.com/apache/doris/pull/68161#discussion_r4256884947
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:
##########
@@ -1289,6 +1306,135 @@ public Map<String, String> getScanNodeProperties(
return props;
}
+ static boolean containsTimestampType(DataType type) {
+ if (type instanceof org.apache.paimon.types.TimestampType
+ || type instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ return true;
+ }
+ if (type instanceof org.apache.paimon.types.RowType) {
+ return ((org.apache.paimon.types.RowType)
type).getFields().stream()
+ .anyMatch(field -> containsTimestampType(field.type()));
+ }
+ if (type instanceof org.apache.paimon.types.ArrayType) {
+ return containsTimestampType(((org.apache.paimon.types.ArrayType)
type).getElementType());
+ }
+ if (type instanceof org.apache.paimon.types.MapType) {
+ org.apache.paimon.types.MapType mapType =
(org.apache.paimon.types.MapType) type;
+ return containsTimestampType(mapType.getKeyType()) ||
containsTimestampType(mapType.getValueType());
+ }
+ return false;
+ }
+
+ static boolean filterReadsTimestamp(RowType rowType, ConnectorExpression
filter) {
+ Set<String> names = new HashSet<>();
+ collectFilterColumnNames(filter, names);
+ return rowType.getFields().stream().anyMatch(field ->
+ names.contains(field.name().toLowerCase(Locale.ROOT)) &&
containsTimestampType(field.type()));
+ }
+
+ boolean requiresTimestampPrecisionRepair(ConnectorSession session, Table
table, PaimonTableHandle handle) {
+ if (!containsTimestampType(table.rowType())) {
+ return false;
+ }
+ Supplier<Boolean> lookup = () -> withSchemaHistoryAuthentication(
+ () ->
hasTimestampPrecisionHistory(resolveSchemaDictTable(table, handle)));
+ if (session == null || session.getStatementScope() == null
+ || session.getStatementScope() ==
ConnectorStatementScope.NONE) {
+ return lookup.get();
+ }
+ String memoKey = SCAN_REUSE_NAMESPACE + ":timestamp-history:"
+ + session.getCatalogId() + ":" + session.getQueryId();
+ Map<PaimonTableResolutionKey, Boolean> history =
session.getStatementScope().computeIfAbsent(
+ memoKey, () -> new ConcurrentHashMap<>());
+ return history.computeIfAbsent(new PaimonTableResolutionKey(handle,
effectiveScanOptions(handle)),
+ ignored -> lookup.get());
+ }
+
+ private <T> T withSchemaHistoryAuthentication(Supplier<T> action) {
+ if (context == null) {
+ return action.get();
+ }
+ try {
+ return context.executeAuthenticated(action::get);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to read Paimon timestamp schema
history", e);
+ }
+ }
+
+ private static boolean hasTimestampPrecisionHistory(Table schemaTable) {
+ if (!(schemaTable instanceof FileStoreTable)) {
+ return false;
+ }
+ FileStoreTable fileStoreTable =
PaimonTableDecorators.unwrapToFallbackOrBase((FileStoreTable) schemaTable);
+ if (fileStoreTable instanceof FallbackReadFileStoreTable) {
+ FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable)
fileStoreTable;
+ return hasTimestampPrecisionHistory(pair.wrapped()) ||
hasTimestampPrecisionHistory(pair.other());
+ }
+ RowType currentType = schemaTable.rowType();
+ SchemaManager schemaManager = fileStoreTable.schemaManager();
+ for (Long schemaId : schemaManager.listAllIds()) {
+ TableSchema historicalSchema = schemaManager.schema(schemaId);
+ if (historicalSchema != null
+ &&
hasTimestampPrecisionNarrowing(historicalSchema.logicalRowType(), currentType))
{
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static boolean hasTimestampPrecisionNarrowing(DataType historicalType,
DataType currentType) {
+ return hasTimestampPrecisionNarrowing(historicalType, currentType, 0);
+ }
+
+ private static boolean hasTimestampPrecisionNarrowing(
+ DataType historicalType, DataType currentType, int
minimumHistoricalPrecision) {
+ if (historicalType instanceof org.apache.paimon.types.TimestampType
+ && currentType instanceof
org.apache.paimon.types.TimestampType) {
Review Comment:
Addressed in 40353c25190. The history guard now accepts either TIMESTAMP or
TIMESTAMP_LTZ on each side and compares their precisions using Paimon
DataTypeChecks. Existing recursive traversal and the ORC historical p7-p9
threshold remain unchanged; JNI still requests the target timestamp family at
widened precision, leaving zone conversion to the SDK. Added real historical
Parquet file JNI tests for both directions, covering pre-epoch .600000 and
leading-zero .000001 values in UTC and Asia/Shanghai. Added FE historical-file
planning tests verifying repair=true, retained splits and residual timestamp
predicates, plus equal-precision/widening negatives. Validation: 134 FE tests
and 51 JNI/getter tests passed; connector clean install/package, Checkstyle,
JNI validate and git diff checks passed. No V1 reader, generic CAST or SDK
conversion changes. Full deployed-cluster regression was not rerun for this
incremental fix; community CI will run on the pushed head.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
