OIiveirra commented on code in PR #68161:
URL: https://github.com/apache/doris/pull/68161#discussion_r4291439098
##########
fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java:
##########
@@ -195,11 +197,45 @@ public LocalDateTime getDateTime() {
@Override
public LocalDateTime getTimeStampTz() {
- Timestamp ts = record.getTimestamp(idx, dorisType.getPrecision());
+ Timestamp ts = truncateTimestampPrecision(
+ record.getTimestamp(idx, getPaimonTimestampPrecision()));
// Timestamp's local representation is identical to converting its
epoch instant in UTC.
return ts.toLocalDateTime();
}
+ Timestamp truncateTimestampPrecision(Timestamp timestamp) {
+ if (!(dataType instanceof TimestampType) && !(dataType instanceof
LocalZonedTimestampType)) {
+ return timestamp;
+ }
+ int precision = dorisType.getPrecision();
Review Comment:
Fixed in 0b29f2cbf72e25967b27654b1b323bce974bc280. The scanner now retains
the current Paimon precision in each projected output leaf before widening the
physical read schema, including TIMESTAMP_NS and nested array/map/row leaves.
Historical Parquet p9-to-p7/p8 reader tests compare timestamp-only and mixed
DATETIMEV2 projections; nested getter tests also cover array/map/row values.
All 76 focused tests passed with Maven Checkstyle enabled.
##########
fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java:
##########
@@ -198,6 +204,16 @@ protected void openInternal() throws IOException {
}
private void initReader() throws IOException {
+ Split split = getSplit();
+ QueryAuthSplit authSplit = split instanceof QueryAuthSplit ?
(QueryAuthSplit) split : null;
Review Comment:
Fixed in 0b29f2cbf72e25967b27654b1b323bce974bc280. Authorization discovery
and repair now descend through FallbackSplitImpl containers and reconstruct
them with the original branch flag. Self-wrapped FallbackDataSplit objects
remain leaves. Added real historical-reader tests for a fallback timestamp
policy and an ordinary fallback data split without a policy. The policy still
runs through the SDK after precision repair; filters and masks are retained.
All 76 focused tests and Maven Checkstyle passed.
##########
fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTimestampQueryAuth.java:
##########
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.paimon;
+
+import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericMap;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalMap;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.Timestamp;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateVisitor;
+import org.apache.paimon.predicate.Transform;
+import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeChecks;
+import org.apache.paimon.types.LocalZonedTimestampType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.TimestampType;
+import org.apache.paimon.utils.InternalRowUtils;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/** Keeps SDK authorization and masking after current-precision
materialization. */
+final class PaimonTimestampQueryAuth extends TableQueryAuthResult {
+ private static final long serialVersionUID = 1L;
+ private final RowType currentReadType;
+ private final TableQueryAuthResult original;
+
+ PaimonTimestampQueryAuth(TableQueryAuthResult original, RowType
currentReadType) {
+ super(original.filter(), original.columnMasking());
+ this.original = original;
+ this.currentReadType = currentReadType;
+ }
+
+ @Override
+ public Predicate extractPredicate() {
+ return original.extractPredicate();
+ }
+
+ @Override
+ public Map<String, Transform> extractColumnMasking() {
+ return original.extractColumnMasking();
+ }
+
+ static void addPolicyFields(TableQueryAuthResult auth, RowType tableType,
+ List<DataField> readFields, List<DataField> currentFields) {
+ List<String> projected =
readFields.stream().map(DataField::name).collect(Collectors.toList());
+ Set<String> required = requiredFields(auth, new
LinkedHashSet<>(projected));
+ for (DataField field : tableType.getFields()) {
+ if (required.contains(field.name())) {
+ int index = projected.indexOf(field.name());
+ DataField widened =
field.newType(PaimonJniScanner.createSafeTimestampReadType(field.type()));
+ if (index < 0) {
+ currentFields.add(field);
+ readFields.add(widened);
+ } else if (field.type() instanceof RowType || field.type()
instanceof ArrayType
+ || field.type() instanceof MapType) {
+ // A policy can read a sibling that SQL pruned from a
projected complex column.
+ currentFields.set(index, field);
+ readFields.set(index, widened);
+ }
+ }
+ }
+ }
+
+ static boolean readsTimestamp(TableQueryAuthResult auth, RowType
tableType, List<String> projected) {
+ Set<String> required = requiredFields(auth, new
LinkedHashSet<>(projected));
+ return tableType.getFields().stream().anyMatch(field ->
required.contains(field.name())
+ && PaimonJniScanner.containsTimestampType(field.type()));
+ }
+
+ private static Set<String> requiredFields(TableQueryAuthResult auth,
Set<String> projected) {
+ Set<String> required = new LinkedHashSet<>();
+ Predicate filter = auth.extractPredicate();
+ if (filter != null) {
+ required.addAll(PredicateVisitor.collectFieldNames(filter));
+ }
+ for (Map.Entry<String, Transform> mask :
auth.extractColumnMasking().entrySet()) {
+ if (!projected.contains(mask.getKey())) {
+ continue;
+ }
+ required.add(mask.getKey());
+ for (Object input : mask.getValue().inputs()) {
+ if (input instanceof FieldRef) {
+ required.add(((FieldRef) input).name());
+ }
+ }
+ }
+ return required;
+ }
+
+ static RecordReader<InternalRow> projectOutput(RecordReader<InternalRow>
reader,
+ RowType currentReadType, RowType outputType) {
+ if (currentReadType.equals(outputType)) {
+ return reader;
+ }
+ return reader.transform(row -> repairRow(row, currentReadType,
outputType));
+ }
+
+ @Override
+ public RecordReader<InternalRow> doAuth(RecordReader<InternalRow> reader,
RowType readType) {
+ // Delegate the policy itself to Paimon; neither filters nor masks are
dropped or rewritten.
+ if (readType.equals(currentReadType)) {
+ return original.doAuth(reader, readType);
+ }
+ return original.doAuth(reader.transform(row -> repairRow(row,
readType, currentReadType)), currentReadType);
Review Comment:
Fixed in 0b29f2cbf72e25967b27654b1b323bce974bc280. The authorization
callback derives its precision-repair target from the actual inner readType
field names, leaving audit-log system-field synthesis to the outer SDK wrapper.
Added a historical audit-log reader test selecting rowkind and ts with a
serialized integer id = 7 policy; it verifies row kinds, negative-epoch
precision, and exclusion of the nonmatching id. All 76 focused tests and Maven
Checkstyle passed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
