This is an automated email from the ASF dual-hosted git repository.
timbrown pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new 50e51814 minor fixes recommended from static analysis tooling
50e51814 is described below
commit 50e518147468e878b3b64c30aa606cedc5ae230f
Author: Timothy Brown <[email protected]>
AuthorDate: Sun Mar 9 10:26:48 2025 -0500
minor fixes recommended from static analysis tooling
---
.../apache/xtable/model/schema/InternalType.java | 14 ++------
.../xtable/conversion/ConversionController.java | 9 +++--
.../xtable/delta/DeltaPartitionExtractor.java | 14 ++++----
.../apache/xtable/hudi/HudiFileStatsExtractor.java | 39 +++++++++++++---------
4 files changed, 40 insertions(+), 36 deletions(-)
diff --git
a/xtable-api/src/main/java/org/apache/xtable/model/schema/InternalType.java
b/xtable-api/src/main/java/org/apache/xtable/model/schema/InternalType.java
index e1b1049d..95f3d6c7 100644
--- a/xtable-api/src/main/java/org/apache/xtable/model/schema/InternalType.java
+++ b/xtable-api/src/main/java/org/apache/xtable/model/schema/InternalType.java
@@ -18,13 +18,13 @@
package org.apache.xtable.model.schema;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Set;
import lombok.Getter;
import lombok.ToString;
+import com.google.common.collect.ImmutableSet;
+
/**
* Collection of supported types
*
@@ -59,13 +59,5 @@ public enum InternalType {
}
public static final Set<InternalType> NON_SCALAR_TYPES =
- Collections.unmodifiableSet(
- new HashSet<InternalType>() {
- {
- add(RECORD);
- add(LIST);
- add(MAP);
- add(UNION);
- }
- });
+ ImmutableSet.<InternalType>builder().add(RECORD, LIST, MAP,
UNION).build();
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionController.java
b/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionController.java
index 1db145ee..9ca4b7a7 100644
---
a/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionController.java
+++
b/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionController.java
@@ -352,11 +352,14 @@ public class ConversionController {
private InstantsForIncrementalSync getMostOutOfSyncCommitAndPendingCommits(
Map<ConversionTarget, TableSyncMetadata> lastSyncMetadataByFormat) {
- Optional<Instant> mostOutOfSyncCommit =
+ Instant mostOutOfSyncCommit =
lastSyncMetadataByFormat.values().stream()
.map(TableSyncMetadata::getLastInstantSynced)
.sorted()
- .findFirst();
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException("No existing commits found
for incremental sync"));
List<Instant> allPendingInstants =
lastSyncMetadataByFormat.values().stream()
.map(TableSyncMetadata::getInstantsToConsiderForNextSync)
@@ -365,7 +368,7 @@ public class ConversionController {
.sorted()
.collect(Collectors.toList());
return InstantsForIncrementalSync.builder()
- .lastSyncInstant(mostOutOfSyncCommit.get())
+ .lastSyncInstant(mostOutOfSyncCommit)
.pendingCommits(allPendingInstants)
.build();
}
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
index 3492857f..172ca26e 100644
---
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
@@ -423,6 +423,8 @@ public class DeltaPartitionExtractor {
private static final Pattern DAY_PATTERN =
Pattern.compile("DAY\\(([^)]+)\\)");
private static final Pattern HOUR_PATTERN =
Pattern.compile("HOUR\\(([^)]+)\\)");
private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\(([^
]+) AS DATE\\)");
+ private static final Pattern DATE_FORMAT_PATTERN =
+ Pattern.compile("DATE_FORMAT\\(([^,]+),[^']+'([^']+)'\\)");
enum GeneratedExprType {
YEAR,
@@ -475,19 +477,17 @@ public class DeltaPartitionExtractor {
.internalPartitionTransformType(PartitionTransformType.DAY)
.build();
} else if (expr.contains("DATE_FORMAT")) {
- if (expr.startsWith("DATE_FORMAT(") && expr.endsWith(")")) {
- int firstParenthesisPos = expr.indexOf("(");
- int commaPos = expr.indexOf(",");
- int lastParenthesisPos = expr.lastIndexOf(")");
+ Matcher matcher = DATE_FORMAT_PATTERN.matcher(expr);
+ if (matcher.find()) {
/*
* from DATE_FORMAT(source_col, 'yyyy-MM-dd-HH') the code below
extracts yyyy-MM-dd-HH.
*/
- String dateFormatExpr =
- expr.substring(commaPos + 1,
lastParenthesisPos).trim().replaceAll("^'|'$", "");
+ String fieldName = matcher.group(1);
+ String dateFormatExpr = matcher.group(2);
return ParsedGeneratedExpr.builder()
.generatedExprType(GeneratedExprType.DATE_FORMAT)
.partitionColumnName(partitionColumnName)
- .sourceColumn(expr.substring(firstParenthesisPos + 1,
commaPos).trim())
+ .sourceColumn(fieldName)
.internalPartitionTransformType(computeInternalPartitionTransform(dateFormatExpr))
.build();
} else {
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
index e47ef72e..82a09493 100644
---
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
@@ -250,21 +250,14 @@ public class HudiFileStatsExtractor {
long nullCount,
long valueCount,
long totalSize) {
- // Special type handling
- if (minValue instanceof Date || maxValue instanceof Date) {
- minValue = minValue == null ? null : dateToDaysSinceEpoch(minValue);
- maxValue = maxValue == null ? null : dateToDaysSinceEpoch(maxValue);
- } else if (field.getSchema().getDataType() == InternalType.ENUM
- && (minValue instanceof ByteBuffer || maxValue instanceof ByteBuffer))
{
- minValue = minValue == null ? null : new String(((ByteBuffer)
minValue).array());
- maxValue = maxValue == null ? null : new String(((ByteBuffer)
maxValue).array());
- } else if (field.getSchema().getDataType() == InternalType.FIXED
- && (minValue instanceof Binary || maxValue instanceof Binary)) {
- minValue = minValue == null ? null : ByteBuffer.wrap(((Binary)
minValue).getBytes());
- maxValue = maxValue == null ? null : ByteBuffer.wrap(((Binary)
maxValue).getBytes());
- }
- boolean isScalar = minValue == null || minValue.compareTo(maxValue) == 0;
- Range range = isScalar ? Range.scalar(minValue) : Range.vector(minValue,
maxValue);
+ Comparable convertedMinValue = convertValue(minValue,
field.getSchema().getDataType());
+ Comparable convertedMaxValue = convertValue(maxValue,
field.getSchema().getDataType());
+ boolean isScalar =
+ convertedMinValue == null ||
convertedMinValue.compareTo(convertedMaxValue) == 0;
+ Range range =
+ isScalar
+ ? Range.scalar(convertedMinValue)
+ : Range.vector(convertedMinValue, convertedMaxValue);
return ColumnStat.builder()
.field(field)
.range(range)
@@ -274,6 +267,22 @@ public class HudiFileStatsExtractor {
.build();
}
+ private static Comparable convertValue(Comparable value, InternalType type) {
+ // Special type handling
+ if (value == null) {
+ return value;
+ }
+ Comparable result = value;
+ if (value instanceof Date) {
+ result = dateToDaysSinceEpoch(value);
+ } else if (type == InternalType.ENUM && (value instanceof ByteBuffer)) {
+ result = new String(((ByteBuffer) value).array());
+ } else if (type == InternalType.FIXED && (value instanceof Binary)) {
+ result = ByteBuffer.wrap(((Binary) value).getBytes());
+ }
+ return result;
+ }
+
private static int dateToDaysSinceEpoch(Object date) {
return (int) ((Date) date).toLocalDate().toEpochDay();
}
