This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 13072be478 [core] fix NPE and ArrayIndexOutOfBoundsException for
PartitionExpire (#6150)
13072be478 is described below
commit 13072be47864460b572853e8fcddb29f61418e5b
Author: LsomeYeah <[email protected]>
AuthorDate: Tue Aug 26 19:27:37 2025 +0800
[core] fix NPE and ArrayIndexOutOfBoundsException for PartitionExpire
(#6150)
---
.../apache/paimon/operation/PartitionExpire.java | 3 ++-
.../paimon/partition/PartitionExpireStrategy.java | 14 ++++++++---
.../partition/PartitionExpireStrategyFactory.java | 6 ++++-
.../PartitionUpdateTimeExpireStrategy.java | 5 ++--
.../PartitionValuesTimeExpireStrategy.java | 2 +-
.../paimon/operation/PartitionExpireTest.java | 29 ++++++++++++++++++++++
.../CustomPartitionExpirationFactory.java | 8 ++++--
7 files changed, 56 insertions(+), 11 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java
b/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java
index 7819063f2e..36d6976152 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java
@@ -185,7 +185,8 @@ public class PartitionExpire {
return expiredPartValues.stream()
.map(values -> String.join(DELIMITER, values))
.sorted()
- .map(s -> s.split(DELIMITER))
+ // Use split(DELIMITER, -1) to preserve trailing empty strings
+ .map(s -> s.split(DELIMITER, -1))
.map(strategy::toPartitionString)
.limit(Math.min(expiredPartValues.size(), maxExpireNum))
.collect(Collectors.toList());
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java
index 7d016c0931..ce021eb1d3 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategy.java
@@ -39,11 +39,13 @@ import java.util.Map;
public abstract class PartitionExpireStrategy {
protected final List<String> partitionKeys;
+ protected final String partitionDefaultName;
private final RowDataToObjectArrayConverter toObjectArrayConverter;
- public PartitionExpireStrategy(RowType partitionType) {
+ public PartitionExpireStrategy(RowType partitionType, String
partitionDefaultName) {
this.toObjectArrayConverter = new
RowDataToObjectArrayConverter(partitionType);
this.partitionKeys = partitionType.getFieldNames();
+ this.partitionDefaultName = partitionDefaultName;
}
public Map<String, String> toPartitionString(Object[] array) {
@@ -57,7 +59,11 @@ public abstract class PartitionExpireStrategy {
public List<String> toPartitionValue(Object[] array) {
List<String> list = new ArrayList<>(partitionKeys.size());
for (int i = 0; i < partitionKeys.size(); i++) {
- list.add(array[i].toString());
+ if (array[i] != null) {
+ list.add(array[i].toString());
+ } else {
+ list.add(partitionDefaultName);
+ }
}
return list;
}
@@ -76,13 +82,13 @@ public abstract class PartitionExpireStrategy {
@Nullable Identifier identifier) {
switch (options.partitionExpireStrategy()) {
case UPDATE_TIME:
- return new PartitionUpdateTimeExpireStrategy(partitionType);
+ return new PartitionUpdateTimeExpireStrategy(options,
partitionType);
case VALUES_TIME:
return new PartitionValuesTimeExpireStrategy(options,
partitionType);
case CUSTOM:
return PartitionExpireStrategyFactory.INSTANCE
.get()
- .create(catalogLoader, identifier, partitionType);
+ .create(catalogLoader, identifier, options,
partitionType);
default:
throw new IllegalArgumentException(
"Unknown partitionExpireStrategy: " +
options.partitionExpireStrategy());
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategyFactory.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategyFactory.java
index 6e89e98725..d871e4cfd2 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategyFactory.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionExpireStrategyFactory.java
@@ -18,6 +18,7 @@
package org.apache.paimon.partition;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.CatalogLoader;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.factories.FactoryUtil;
@@ -30,7 +31,10 @@ import
org.apache.paimon.shade.guava30.com.google.common.base.Suppliers;
public interface PartitionExpireStrategyFactory {
PartitionExpireStrategy create(
- CatalogLoader catalogLoader, Identifier identifier, RowType
partitionType);
+ CatalogLoader catalogLoader,
+ Identifier identifier,
+ CoreOptions options,
+ RowType partitionType);
Supplier<PartitionExpireStrategyFactory> INSTANCE =
Suppliers.memoize(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionUpdateTimeExpireStrategy.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionUpdateTimeExpireStrategy.java
index c2d75e8e74..3cb7a405d2 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionUpdateTimeExpireStrategy.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionUpdateTimeExpireStrategy.java
@@ -18,6 +18,7 @@
package org.apache.paimon.partition;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.operation.FileStoreScan;
import org.apache.paimon.types.RowType;
@@ -33,8 +34,8 @@ import java.util.stream.Collectors;
*/
public class PartitionUpdateTimeExpireStrategy extends PartitionExpireStrategy
{
- public PartitionUpdateTimeExpireStrategy(RowType partitionType) {
- super(partitionType);
+ public PartitionUpdateTimeExpireStrategy(CoreOptions options, RowType
partitionType) {
+ super(partitionType, options.partitionDefaultName());
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
index 51c53282c4..70c55cfb38 100644
---
a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
+++
b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java
@@ -48,7 +48,7 @@ public class PartitionValuesTimeExpireStrategy extends
PartitionExpireStrategy {
private final PartitionTimeExtractor timeExtractor;
public PartitionValuesTimeExpireStrategy(CoreOptions options, RowType
partitionType) {
- super(partitionType);
+ super(partitionType, options.partitionDefaultName());
String timePattern = options.partitionTimestampPattern();
String timeFormatter = options.partitionTimestampFormatter();
this.timeExtractor = new PartitionTimeExtractor(timePattern,
timeFormatter);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java
index 2b45ad4352..4c48671add 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java
@@ -61,6 +61,7 @@ import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -217,6 +218,34 @@ public class PartitionExpireTest {
assertThat(overwriteSnapshotCnt).isEqualTo(3L);
}
+ @Test
+ public void testExpireWithNullOrEmptyPartition() throws Exception {
+ SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(),
path);
+ schemaManager.createTable(
+ new Schema(
+ RowType.of(VarCharType.STRING_TYPE,
VarCharType.STRING_TYPE).getFields(),
+ Arrays.asList("f0", "f1"),
+ emptyList(),
+
Collections.singletonMap(METASTORE_PARTITIONED_TABLE.key(), "true"),
+ ""));
+ newTable();
+ write("20230101", "11");
+ write("20230101", "12");
+ // sub partition is null
+ write("20230101", null);
+ // sub partition is empty string
+ write("20230103", "");
+ write("20230103", "32");
+ write("20230105", "51");
+
+ PartitionExpire expire = newExpire();
+ expire.setLastCheck(date(1));
+ Assertions.assertDoesNotThrow(() -> expire.expire(date(6),
Long.MAX_VALUE));
+
+ // null partition and empty string partition should be expired
+ assertThat(read()).containsExactlyInAnyOrder("20230105:51");
+ }
+
@Test
public void test() throws Exception {
SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(),
path);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/partition/CustomPartitionExpirationFactory.java
b/paimon-core/src/test/java/org/apache/paimon/partition/CustomPartitionExpirationFactory.java
index 4f2b770d55..9fa6369f93 100644
---
a/paimon-core/src/test/java/org/apache/paimon/partition/CustomPartitionExpirationFactory.java
+++
b/paimon-core/src/test/java/org/apache/paimon/partition/CustomPartitionExpirationFactory.java
@@ -18,6 +18,7 @@
package org.apache.paimon.partition;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogLoader;
import org.apache.paimon.catalog.Identifier;
@@ -38,8 +39,11 @@ public class CustomPartitionExpirationFactory implements
PartitionExpireStrategy
@Override
public PartitionExpireStrategy create(
- CatalogLoader catalogLoader, Identifier identifier, RowType
partitionType) {
- return new PartitionExpireStrategy(partitionType) {
+ CatalogLoader catalogLoader,
+ Identifier identifier,
+ CoreOptions options,
+ RowType partitionType) {
+ return new PartitionExpireStrategy(partitionType,
options.partitionDefaultName()) {
@Override
public List<PartitionEntry> selectExpiredPartitions(
FileStoreScan scan, LocalDateTime expirationTime) {
