the-other-tim-brown commented on code in PR #742: URL: https://github.com/apache/incubator-xtable/pull/742#discussion_r2486674646
########## xtable-core/src/main/java/org/apache/xtable/paimon/PaimonPartitionExtractor.java: ########## @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.xtable.paimon; + +import java.util.*; +import java.util.stream.Collectors; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.InternalRowPartitionComputer; + +import org.apache.xtable.model.schema.InternalField; +import org.apache.xtable.model.schema.InternalPartitionField; +import org.apache.xtable.model.schema.InternalSchema; +import org.apache.xtable.model.schema.PartitionTransformType; +import org.apache.xtable.model.stat.PartitionValue; +import org.apache.xtable.model.stat.Range; + +/** Extracts partition spec for Paimon as identity transforms on partition keys. */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class PaimonPartitionExtractor { + + private final PaimonSchemaExtractor paimonSchemaExtractor = PaimonSchemaExtractor.getInstance(); + + private static final PaimonPartitionExtractor INSTANCE = new PaimonPartitionExtractor(); + + public static PaimonPartitionExtractor getInstance() { + return INSTANCE; + } + + public List<InternalPartitionField> toInternalPartitionFields( + List<String> partitionKeys, InternalSchema schema) { + if (partitionKeys == null || partitionKeys.isEmpty()) { + return Collections.emptyList(); + } + return partitionKeys.stream() + .map(key -> toPartitionField(key, schema)) + .collect(Collectors.toList()); + } + + public List<PartitionValue> toPartitionValues(FileStoreTable table, BinaryRow partition) { + InternalRowPartitionComputer partitionComputer = newPartitionComputer(table); + InternalSchema internalSchema = paimonSchemaExtractor.toInternalSchema(table.schema()); + + List<PartitionValue> partitionValues = new ArrayList<>(); + for (Map.Entry<String, String> entry : + partitionComputer.generatePartValues(partition).entrySet()) { + PartitionValue partitionValue = + PartitionValue.builder() + .partitionField(toPartitionField(entry.getKey(), internalSchema)) + .range(Range.scalar(entry.getValue())) + .build(); + partitionValues.add(partitionValue); + } + return partitionValues; + } + + public Optional<String> toPartitionPath(FileStoreTable table, BinaryRow partition) { + InternalRowPartitionComputer partitionComputer = newPartitionComputer(table); + return partitionComputer.generatePartValues(partition).entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .reduce((a, b) -> a + "/" + b); + } + + private InternalPartitionField toPartitionField(String key, InternalSchema schema) { + InternalField sourceField = + findField(schema, key) + .orElseThrow( + () -> new IllegalArgumentException("Partition key not found in schema: " + key)); + return InternalPartitionField.builder() + .sourceField(sourceField) + .transformType(PartitionTransformType.VALUE) Review Comment: There is no documentation at that level for XTable but it is so we can map to the partition types in other systems like Iceberg where you need to specify the [transformation](https://iceberg.apache.org/spec/#partition-transforms) type used. This allows us to properly translate the metadata and allows for partition pruning in the query engines. For the ranges themselves, the [java docs](https://github.com/apache/incubator-xtable/blob/main/xtable-api/src/main/java/org/apache/xtable/model/stat/Range.java#L43) describe the expectations on the values so that we can also ensure each table format interprets the values correctly. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
