This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 605d2e4bff54fd675fa322d51851617f773707e8 Author: wangxianghu <[email protected]> AuthorDate: Fri Jul 3 09:51:35 2026 +0800 chore(glue-sync): Ignore EntityNotFoundException when dropping Glue partitions (#19142) (cherry picked from commit 667626cb9b37719d222b7840cdb5ab8f0a41110d) --- .../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 20 +++++++- .../hudi/aws/sync/TestAWSGlueSyncClient.java | 56 ++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java index 884b5c53a25d..10f0b5a1592d 100644 --- a/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java +++ b/hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java @@ -71,6 +71,7 @@ import software.amazon.awssdk.services.glue.model.GetPartitionsRequest; import software.amazon.awssdk.services.glue.model.GetPartitionsResponse; import software.amazon.awssdk.services.glue.model.GetTableRequest; import software.amazon.awssdk.services.glue.model.KeySchemaElement; +import software.amazon.awssdk.services.glue.model.PartitionError; import software.amazon.awssdk.services.glue.model.PartitionIndex; import software.amazon.awssdk.services.glue.model.PartitionIndexDescriptor; import software.amazon.awssdk.services.glue.model.PartitionInput; @@ -137,6 +138,7 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient { private static final int MAX_PARTITIONS_PER_CHANGE_REQUEST = 100; private static final int MAX_PARTITIONS_PER_READ_REQUEST = 1000; private static final int MAX_DELETE_PARTITIONS_PER_REQUEST = 25; + private static final String ENTITY_NOT_FOUND_ERROR_CODE = "EntityNotFoundException"; protected final GlueAsyncClient awsGlue; private static final String GLUE_PARTITION_INDEX_ENABLE = "partition_filtering.enabled"; private static final int PARTITION_INDEX_MAX_NUMBER = 3; @@ -437,8 +439,22 @@ public class AWSGlueCatalogSyncClient extends HoodieSyncClient { BatchDeletePartitionResponse response = future.get(); if (CollectionUtils.nonEmpty(response.errors())) { - throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName) - + " with error(s): " + response.errors()); + // Dropping a partition that no longer exists is a no-op for an idempotent cleanup, so + // ignore EntityNotFoundException errors and only fail on other (e.g. permission/throttling) errors. + Map<Boolean, List<PartitionError>> errorsByIgnorable = response.errors().stream() + .collect(Collectors.partitioningBy( + error -> ENTITY_NOT_FOUND_ERROR_CODE.equals(error.errorDetail().errorCode()))); + List<PartitionError> ignorableErrors = errorsByIgnorable.get(true); + if (!ignorableErrors.isEmpty()) { + log.info("Ignored dropping {} non-existent partition(s) from table {}: {}", ignorableErrors.size(), + tableId(databaseName, tableName), + ignorableErrors.stream().map(PartitionError::partitionValues).collect(Collectors.toList())); + } + List<PartitionError> realErrors = errorsByIgnorable.get(false); + if (!realErrors.isEmpty()) { + throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName) + + " with error(s): " + realErrors); + } } } catch (Exception e) { throw new HoodieGlueSyncException("Fail to drop partitions to " + tableId(databaseName, tableName), e); diff --git a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java index f4822e32f05d..8c3b24c0b677 100644 --- a/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java +++ b/hudi-aws/src/test/java/org/apache/hudi/aws/sync/TestAWSGlueSyncClient.java @@ -647,6 +647,62 @@ class TestAWSGlueSyncClient { assertTrue(ex.getCause().getCause().getMessage().contains("Fail to drop partitions")); } + @Test + void testDropPartitions_IgnoresEntityNotFound() { + String tableName = "tbl"; + List<String> toDrop = List.of("2025/05/19"); + + // Glue reports EntityNotFoundException for a partition that no longer exists; it should be ignored. + ErrorDetail detail = ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build(); + PartitionError pe = PartitionError.builder().partitionValues(Arrays.asList("2025", "05", "19")).errorDetail(detail).build(); + BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder() + .errors(Collections.singletonList(pe)) + .build(); + when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class))) + .thenReturn(CompletableFuture.completedFuture(resp)); + + // should swallow the EntityNotFound error and not throw + awsGlueSyncClient.dropPartitions(tableName, toDrop); + + verify(mockAwsGlue).batchDeletePartition(any(BatchDeletePartitionRequest.class)); + } + + @Test + void testDropPartitions_MixedErrorsStillThrow() { + String tableName = "tbl"; + List<String> toDrop = Arrays.asList("2025/05/19", "2025/05/18"); + + // One ignorable EntityNotFound error and one real error -> should still throw for the real one. + PartitionError ignorable = PartitionError.builder() + .partitionValues(Arrays.asList("2025", "05", "19")) + .errorDetail(ErrorDetail.builder().errorCode(EntityNotFoundException.class.getSimpleName()).build()) + .build(); + PartitionError real = PartitionError.builder() + .partitionValues(Arrays.asList("2025", "05", "18")) + .errorDetail(ErrorDetail.builder().errorCode("InternalServiceException").build()) + .build(); + BatchDeletePartitionResponse resp = BatchDeletePartitionResponse.builder() + .errors(Arrays.asList(ignorable, real)) + .build(); + when(mockAwsGlue.batchDeletePartition(any(BatchDeletePartitionRequest.class))) + .thenReturn(CompletableFuture.completedFuture(resp)); + + HoodieGlueSyncException ex = assertThrows( + HoodieGlueSyncException.class, + () -> awsGlueSyncClient.dropPartitions(tableName, toDrop) + ); + // Walk the full cause chain: the error list is nested a few wrappers deep. + StringBuilder chain = new StringBuilder(); + for (Throwable t = ex; t != null; t = t.getCause()) { + chain.append(t.getMessage()).append('\n'); + } + String messages = chain.toString(); + assertTrue(messages.contains("Fail to drop partitions")); + // Only the real error should be surfaced, not the ignored EntityNotFound one. + assertTrue(messages.contains("InternalServiceException")); + assertFalse(messages.contains(EntityNotFoundException.class.getSimpleName())); + } + @Disabled("Integration test – requires real AWS environment") @Test void testIntegrationTableExists_RealGlueEnvironment() {
