This is an automated email from the ASF dual-hosted git repository.
sarvekshayr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e2ad852dbd2 HDDS-14684. Allow deletion of empty quasi-closed
containers (#9856)
e2ad852dbd2 is described below
commit e2ad852dbd2215a2d231ab4f09ce71c632c64659
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Fri Mar 20 22:01:42 2026 +0530
HDDS-14684. Allow deletion of empty quasi-closed containers (#9856)
---
.../container/AbstractContainerReportHandler.java | 26 +++-
.../hdds/scm/container/ContainerManager.java | 7 +-
.../hdds/scm/container/ContainerManagerImpl.java | 5 +-
.../hdds/scm/container/ContainerStateManager.java | 8 +-
.../scm/container/ContainerStateManagerImpl.java | 14 +-
.../QuasiClosedStuckUnderReplicationHandler.java | 9 ++
.../replication/RatisUnderReplicationHandler.java | 9 ++
.../replication/health/EmptyContainerHandler.java | 74 ++++++++--
.../scm/container/TestContainerManagerImpl.java | 12 +-
.../scm/container/TestContainerReportHandler.java | 101 +++++++++++++-
.../scm/container/TestContainerStateManager.java | 13 +-
.../health/TestEmptyContainerHandler.java | 36 +++++
.../TestReplicationManagerIntegration.java | 154 +++++++++++++++++++++
13 files changed, 424 insertions(+), 44 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
index 6028bbfd90e..57234889dcb 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -322,17 +323,32 @@ private boolean updateContainerState(final
DatanodeDetails datanode,
case DELETING:
// HDDS-11136: If a DELETING container has a non-empty CLOSED replica,
transition the container to CLOSED
// HDDS-12421: If a DELETING or DELETED container has a non-empty
replica, transition the container to CLOSED
- if (replica.getState() == State.CLOSED &&
replica.getBlockCommitSequenceId() <= container.getSequenceId()
+ boolean isReplicaClosed = replica.getState() == State.CLOSED;
+ boolean isReplicaQuasiClosed = replica.getState() == State.QUASI_CLOSED;
+ if ((isReplicaClosed || isReplicaQuasiClosed) &&
replica.getBlockCommitSequenceId() <= container.getSequenceId()
&&
container.getReplicationType().equals(HddsProtos.ReplicationType.RATIS)) {
deleteReplica(containerId, datanode, publisher, "DELETED", true,
detailsForLogging);
- // We should not move back to CLOSED state if replica bcsid <=
container bcsid
+ // We should not move back CLOSED or QUASI_CLOSED if replica bcsId <=
container bcsId
return false;
}
boolean replicaStateAllowed = (replica.getState() != State.INVALID &&
replica.getState() != State.DELETED);
if (!replicaIsEmpty && replicaStateAllowed) {
- getLogger().info("transitionDeletingToClosed due to non-empty CLOSED
replica (keyCount={}) for {}",
- replica.getKeyCount(), detailsForLogging);
- containerManager.transitionDeletingOrDeletedToClosedState(containerId);
+ LifeCycleState targetState;
+ if (replica.getState() == State.CLOSED) {
+ targetState = LifeCycleState.CLOSED;
+ getLogger().info("Resurrecting container {} from {} to CLOSED due to
non-empty CLOSED replica " +
+ "(keyCount={}, BCSID={}) from {}",
+ containerId, container.getState(), replica.getKeyCount(),
replica.getBlockCommitSequenceId(),
+ detailsForLogging);
+ } else {
+ // For OPEN, CLOSING, UNHEALTHY, QUASI_CLOSED replicas, transition
to QUASI_CLOSED state
+ targetState = LifeCycleState.QUASI_CLOSED;
+ getLogger().info("Resurrecting container {} from {} to QUASI_CLOSED
due to non-empty {} replica " +
+ "(keyCount={}, BCSID={}) from {}",
+ containerId, container.getState(), replica.getState(),
replica.getKeyCount(),
+ replica.getBlockCommitSequenceId(), detailsForLogging);
+ }
+ containerManager.transitionDeletingOrDeletedToTargetState(containerId,
targetState);
}
return false;
default:
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index 370c219ac60..36753202ec9 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -134,14 +134,13 @@ void updateContainerState(ContainerID containerID,
throws IOException, InvalidStateTransitionException;
/**
- * Bypasses the container state machine to change a container's state from
DELETING or DELETED to CLOSED. This API was
- * introduced to fix a bug (HDDS-11136), and should be used with care
otherwise.
+ * Bypasses the container state machine to change a container's state from
DELETING/DELETED to CLOSED/QUASI_CLOSED.
*
- * @see <a
href="https://issues.apache.org/jira/browse/HDDS-11136";>HDDS-11136</a>
* @param containerID id of the container to transition
+ * @param targetState the target state (must be CLOSED or QUASI_CLOSED)
* @throws IOException
*/
- void transitionDeletingOrDeletedToClosedState(ContainerID containerID)
throws IOException;
+ void transitionDeletingOrDeletedToTargetState(ContainerID containerID,
LifeCycleState targetState) throws IOException;
/**
* Returns the latest list of replicas for given containerId.
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
index dc701a0be66..f77bf86cec1 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
@@ -295,12 +295,13 @@ public void updateContainerState(final ContainerID cid,
}
@Override
- public void transitionDeletingOrDeletedToClosedState(ContainerID
containerID) throws IOException {
+ public void transitionDeletingOrDeletedToTargetState(ContainerID
containerID, LifeCycleState targetState)
+ throws IOException {
HddsProtos.ContainerID proto = containerID.getProtobuf();
lock.lock();
try {
if (containerExist(containerID)) {
- containerStateManager.transitionDeletingOrDeletedToClosedState(proto);
+ containerStateManager.transitionDeletingOrDeletedToTargetState(proto,
targetState);
} else {
throw new ContainerNotFoundException(containerID);
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index f5a2334b7cd..3809db1cd33 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -172,15 +172,15 @@ void
updateContainerStateWithSequenceId(HddsProtos.ContainerID id,
/**
- * Bypasses the container state machine to change a container's state from
DELETING or DELETED to CLOSED. This API was
- * introduced to fix a bug (HDDS-11136), and should be used with care
otherwise.
+ * Bypasses the container state machine to change a container's state from
DELETING/DELETED to CLOSED/QUASI_CLOSED.
*
- * @see <a
href="https://issues.apache.org/jira/browse/HDDS-11136";>HDDS-11136</a>
* @param id id of the container to transition
+ * @param targetState the target state (must be CLOSED or QUASI_CLOSED)
* @throws IOException
*/
@Replicate
- void transitionDeletingOrDeletedToClosedState(HddsProtos.ContainerID id)
throws IOException;
+ void transitionDeletingOrDeletedToTargetState(HddsProtos.ContainerID id,
LifeCycleState targetState)
+ throws IOException;
/**
*
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
index d971b19c406..dc5afd43a20 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
@@ -186,6 +186,7 @@ private StateMachine<LifeCycleState, LifeCycleEvent>
newStateMachine() {
containerLifecycleSM.addTransition(CLOSING, QUASI_CLOSED, QUASI_CLOSE);
containerLifecycleSM.addTransition(CLOSING, CLOSED, CLOSE);
containerLifecycleSM.addTransition(QUASI_CLOSED, CLOSED, FORCE_CLOSE);
+ containerLifecycleSM.addTransition(QUASI_CLOSED, DELETING, DELETE);
containerLifecycleSM.addTransition(CLOSED, DELETING, DELETE);
containerLifecycleSM.addTransition(DELETING, DELETED, CLEANUP);
@@ -394,7 +395,12 @@ public void updateContainerStateWithSequenceId(final
HddsProtos.ContainerID cont
}
@Override
- public void transitionDeletingOrDeletedToClosedState(HddsProtos.ContainerID
containerID) throws IOException {
+ public void transitionDeletingOrDeletedToTargetState(HddsProtos.ContainerID
containerID,
+ LifeCycleState
targetState) throws IOException {
+ if (targetState != CLOSED && targetState != QUASI_CLOSED) {
+ throw new IllegalArgumentException("Target state must be CLOSED or
QUASI_CLOSED, got: " + targetState);
+ }
+
final ContainerID id = ContainerID.getFromProtobuf(containerID);
try (AutoCloseableLock ignored = writeLock(id)) {
@@ -403,14 +409,14 @@ public void
transitionDeletingOrDeletedToClosedState(HddsProtos.ContainerID cont
final LifeCycleState oldState = oldInfo.getState();
if (oldState != DELETING && oldState != DELETED) {
throw new InvalidContainerStateException("Cannot transition
container " + id + " from " + oldState +
- " back to CLOSED. The container must be in the DELETING or
DELETED state.");
+ " back to " + targetState + ". The container must be in the
DELETING or DELETED state.");
}
ExecutionUtil.create(() -> {
- containers.updateState(id, oldState, CLOSED);
+ containers.updateState(id, oldState, targetState);
transactionBuffer.addToBuffer(containerStore, id,
containers.getContainerInfo(id));
}).onException(() -> {
transactionBuffer.addToBuffer(containerStore, id, oldInfo);
- containers.updateState(id, CLOSED, oldState);
+ containers.updateState(id, targetState, oldState);
}).execute();
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
index 851378c481a..55b503822fd 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
@@ -61,6 +61,15 @@ public int processAndSendCommands(Set<ContainerReplica>
replicas, List<Container
ContainerInfo containerInfo = result.getContainerInfo();
LOG.debug("Handling under replicated QuasiClosed Stuck Ratis container
{}", containerInfo);
+ // Check if container is empty before attempting replication
+ // Empty containers will be deleted by EmptyContainerHandler
+ boolean allReplicasEmpty = !replicas.isEmpty() &&
replicas.stream().allMatch(ContainerReplica::isEmpty);
+ if (allReplicasEmpty) {
+ LOG.info("Skipping replication for empty QUASI_CLOSED stuck container
{}. " +
+ "It will be deleted by EmptyContainerHandler.",
containerInfo.containerID());
+ return 0;
+ }
+
int pendingAdd = 0;
for (ContainerReplicaOp op : pendingOps) {
if (op.getOpType() == ContainerReplicaOp.PendingOpType.ADD) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
index 57eb29033e4..3083aa15c71 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
@@ -89,6 +89,15 @@ public int processAndSendCommands(
ContainerInfo containerInfo = result.getContainerInfo();
LOG.debug("Handling under replicated Ratis container {}", containerInfo);
+ // Check if container is empty before attempting replication
+ // Empty containers will be deleted by EmptyContainerHandler
+ boolean allReplicasEmpty = !replicas.isEmpty() &&
replicas.stream().allMatch(ContainerReplica::isEmpty);
+ if (allReplicasEmpty && containerInfo.getState() ==
LifeCycleState.QUASI_CLOSED) {
+ LOG.info("Skipping replication for empty QUASI_CLOSED container {}. " +
+ "It will be deleted by EmptyContainerHandler.",
containerInfo.containerID());
+ return 0;
+ }
+
RatisContainerReplicaCount withUnhealthy =
new RatisContainerReplicaCount(containerInfo, replicas, pendingOps,
minHealthyForMaintenance, true);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
index 8fa72cbc50f..9eb8ebf4c05 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.container.replication.health;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
@@ -31,7 +32,7 @@
import org.slf4j.LoggerFactory;
/**
- * This handler deletes a container if it's closed and empty (0 key count)
+ * This handler deletes a container if it's closed or quasi-closed and empty
(0 key count)
* and all its replicas are empty.
*/
public class EmptyContainerHandler extends AbstractCheck {
@@ -45,8 +46,8 @@ public EmptyContainerHandler(ReplicationManager
replicationManager) {
}
/**
- * Deletes a container if it's closed and empty (0 key count) and all its
- * replicas are closed and empty.
+ * Deletes a container if it's closed or quasi-closed and empty (0 key
count) and all its
+ * replicas are empty.
* @param request ContainerCheckRequest object representing the container
* @return true if the specified container is empty, otherwise false
*/
@@ -75,6 +76,37 @@ public boolean handle(ContainerCheckRequest request) {
containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE);
}
return true;
+ } else if (isContainerEmptyAndQuasiClosed(containerInfo, replicas)) {
+ request.getReport().incrementAndSample(ContainerHealthState.EMPTY,
containerInfo);
+ if (!request.isReadOnly()) {
+ String originIds = replicas.stream()
+ .map(r -> r.getOriginDatanodeId().toString())
+ .collect(Collectors.joining(", "));
+ long maxReplicaBCSID = replicas.stream()
+ .filter(r -> r.getSequenceId() != null)
+ .mapToLong(ContainerReplica::getSequenceId)
+ .max()
+ .orElse(containerInfo.getSequenceId());
+
+ // Update container bcsId to max replica bcsId before deletion
+ // This ensures resurrection logic uses the correct bcsId for stale
replica detection
+ if (maxReplicaBCSID > containerInfo.getSequenceId()) {
+ LOG.info("Updating bcsId for empty QUASI_CLOSED container {} from {}
to {} before deletion",
+ containerInfo.containerID(), containerInfo.getSequenceId(),
maxReplicaBCSID);
+ containerInfo.updateSequenceId(maxReplicaBCSID);
+ }
+
+ LOG.info("Deleting empty QUASI_CLOSED container {} (container
BCSID={}, max replica BCSID={}) with {} " +
+ "replicas from originIds: [{}]. If resurrected, container will
transition to QUASI_CLOSED",
+ containerInfo.containerID(), containerInfo.getSequenceId(),
maxReplicaBCSID,
+ replicas.size(), originIds);
+ // Update the container's state
+ replicationManager.updateContainerState(
+ containerInfo.containerID(), HddsProtos.LifeCycleEvent.DELETE);
+ // Delete replicas AFTER transitioning to DELETING state
+ deleteContainerReplicas(containerInfo, replicas);
+ }
+ return true;
} else if (containerInfo.getState() == HddsProtos.LifeCycleState.CLOSED
&& containerInfo.getNumberOfKeys() == 0 && replicas.isEmpty()) {
// If the container is empty and has no replicas, it is possible it was
@@ -114,21 +146,45 @@ private boolean isContainerEmptyAndClosed(final
ContainerInfo container,
}
/**
- * Deletes the specified container's replicas if they are closed and empty.
+ * Returns true if the container is empty and QUASI_CLOSED.
+ * For QUASI_CLOSED containers, replicas can be in QUASI_CLOSED, OPEN,
+ * CLOSING, or UNHEALTHY states. We check if all replicas are empty
regardless
+ * of their state.
+ *
+ * @param container Container to check
+ * @param replicas Set of ContainerReplica
+ * @return true if the container is considered empty and quasi-closed, false
otherwise
+ */
+ private boolean isContainerEmptyAndQuasiClosed(final ContainerInfo container,
+ final Set<ContainerReplica> replicas) {
+ return container.getState() == HddsProtos.LifeCycleState.QUASI_CLOSED &&
+ !replicas.isEmpty() &&
+ replicas.stream().allMatch(ContainerReplica::isEmpty);
+ }
+
+ /**
+ * Deletes the specified container's replicas if they are empty.
+ * For CLOSED containers, replicas must also be CLOSED.
+ * For QUASI_CLOSED containers, replicas can be in any state (QUASI_CLOSED,
OPEN, CLOSING, UNHEALTHY),
+ * but delete commands should be sent to replicas in stable states
(QUASI_CLOSED or CLOSED).
*
* @param containerInfo ContainerInfo to delete
* @param replicas Set of ContainerReplica
*/
private void deleteContainerReplicas(final ContainerInfo containerInfo,
final Set<ContainerReplica> replicas) {
- Preconditions.assertSame(HddsProtos.LifeCycleState.CLOSED,
- containerInfo.getState(), "container state");
-
for (ContainerReplica rp : replicas) {
- Preconditions.assertSame(ContainerReplicaProto.State.CLOSED,
- rp.getState(), "replica state");
Preconditions.assertSame(true, rp.isEmpty(), "replica empty");
+ // Only send delete commands to replicas in CLOSED/QUASI_CLOSED states
+ if (rp.getState() != ContainerReplicaProto.State.QUASI_CLOSED
+ && rp.getState() != ContainerReplicaProto.State.CLOSED) {
+ LOG.info("Skipping delete command for replica in {} state for empty
container {} on datanode {}. " +
+ "Will retry after replica transitions to QUASI_CLOSED or CLOSED.",
+ rp.getState(), containerInfo.containerID(),
rp.getDatanodeDetails());
+ continue;
+ }
+
try {
replicationManager.sendDeleteCommand(containerInfo,
rp.getReplicaIndex(), rp.getDatanodeDetails(), false);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java
index daf0b4b4c6a..218a2137e3e 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java
@@ -210,7 +210,7 @@ void testUpdateContainerState() throws Exception {
@ParameterizedTest
@EnumSource(value = HddsProtos.LifeCycleState.class,
names = {"DELETING", "DELETED"})
- void testTransitionDeletingOrDeletedToClosedState(HddsProtos.LifeCycleState
desiredState)
+ void testTransitionDeletingOrDeletedToTargetState(HddsProtos.LifeCycleState
desiredState)
throws IOException, InvalidStateTransitionException {
// Allocate OPEN Ratis and Ec containers, and do a series of state changes
to transition them to DELETING / DELETED
final ContainerInfo container = containerManager.allocateContainer(
@@ -250,8 +250,8 @@ void
testTransitionDeletingOrDeletedToClosedState(HddsProtos.LifeCycleState desi
}
// DELETING / DELETED -> CLOSED
- containerManager.transitionDeletingOrDeletedToClosedState(cid);
- containerManager.transitionDeletingOrDeletedToClosedState(ecCid);
+ containerManager.transitionDeletingOrDeletedToTargetState(cid,
LifeCycleState.CLOSED);
+ containerManager.transitionDeletingOrDeletedToTargetState(ecCid,
LifeCycleState.CLOSED);
// the containers should be back in CLOSED state now
assertEquals(LifeCycleState.CLOSED,
containerManager.getContainer(cid).getState());
assertEquals(LifeCycleState.CLOSED,
containerManager.getContainer(ecCid).getState());
@@ -267,13 +267,15 @@ void
testTransitionContainerToClosedStateAllowOnlyDeletingOrDeletedContainers()
ReplicationFactor.THREE), "admin");
final ContainerID cid = container.containerID();
assertEquals(LifeCycleState.OPEN,
containerManager.getContainer(cid).getState());
- assertThrows(IOException.class, () ->
containerManager.transitionDeletingOrDeletedToClosedState(cid));
+ assertThrows(IOException.class, () ->
+ containerManager.transitionDeletingOrDeletedToTargetState(cid,
LifeCycleState.CLOSED));
// test for EC container
final ContainerInfo ecContainer = containerManager.allocateContainer(new
ECReplicationConfig(3, 2), "admin");
final ContainerID ecCid = ecContainer.containerID();
assertEquals(LifeCycleState.OPEN,
containerManager.getContainer(ecCid).getState());
- assertThrows(IOException.class, () ->
containerManager.transitionDeletingOrDeletedToClosedState(ecCid));
+ assertThrows(IOException.class, () ->
+ containerManager.transitionDeletingOrDeletedToTargetState(ecCid,
LifeCycleState.CLOSED));
}
@Test
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
index 264a429960f..c13deefff2a 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java
@@ -154,10 +154,11 @@ void setup() throws IOException,
InvalidStateTransitionException {
any(ContainerID.class), any(ContainerReplica.class));
doAnswer(invocation -> {
- containerStateManager.transitionDeletingOrDeletedToClosedState(
- ((ContainerID) invocation.getArgument(0)).getProtobuf());
+ containerStateManager.transitionDeletingOrDeletedToTargetState(
+ ((ContainerID) invocation.getArgument(0)).getProtobuf(),
invocation.getArgument(1));
return null;
-
}).when(containerManager).transitionDeletingOrDeletedToClosedState(any(ContainerID.class));
+ }).when(containerManager).transitionDeletingOrDeletedToTargetState(
+ any(ContainerID.class), any(LifeCycleState.class));
}
@AfterEach
@@ -204,7 +205,8 @@ static Stream<Arguments> containerAndReplicaStates() {
continue;
}
if (replicationType == HddsProtos.ReplicationType.RATIS &&
- replicaState.equals(ContainerReplicaProto.State.CLOSED) &&
+ (replicaState.equals(ContainerReplicaProto.State.CLOSED) ||
+ replicaState.equals(ContainerReplicaProto.State.QUASI_CLOSED)) &&
(containerState.equals(HddsProtos.LifeCycleState.DELETED) ||
containerState.equals(HddsProtos.LifeCycleState.DELETING))) {
continue;
@@ -567,7 +569,14 @@ public void
containerShouldTransitionFromDeletingOrDeletedToClosedWhenNonEmptyRe
ContainerReportsProto closedContainerReport =
getContainerReports(validReplica);
containerReportHandler
.onMessage(new ContainerReportFromDatanode(dnWithValidReplica,
closedContainerReport), publisher);
- assertEquals(LifeCycleState.CLOSED,
containerStateManager.getContainer(container.containerID()).getState());
+ // Determine expected state based on replica state
+ LifeCycleState expectedState;
+ if (replicaState == ContainerReplicaProto.State.CLOSED) {
+ expectedState = LifeCycleState.CLOSED;
+ } else {
+ expectedState = LifeCycleState.QUASI_CLOSED;
+ }
+ assertEquals(expectedState,
containerStateManager.getContainer(container.containerID()).getState());
// verify that no delete command is issued for non-empty replica,
regardless of container state
verify(publisher, times(0))
@@ -1179,6 +1188,88 @@ public void testStaleReplicaOfDeletedContainer() throws
NodeNotFoundException, I
assertEquals(1,
containerManager.getContainerReplicas(containerOne.containerID()).size());
}
+ /**
+ * Test resurrection of DELETED container to QUASI_CLOSED state when a
non-empty
+ * QUASI_CLOSED replica with matching bcsId is reported.
+ */
+ @Test
+ public void testDeletedContainerWithStaleQuasiClosedReplicaDoesNotResurrect()
+ throws NodeNotFoundException, IOException {
+ final ContainerReportHandler reportHandler = new
ContainerReportHandler(nodeManager, containerManager);
+ final DatanodeDetails datanodeOne =
nodeManager.getNodes(NodeStatus.inServiceHealthy()).iterator().next();
+ // Create container in DELETED state with bcsId=100
+ final ContainerInfo containerOne = getContainer(LifeCycleState.DELETED);
+ assertEquals(10000L, containerOne.getSequenceId());
+
+ final Set<ContainerID> containerIDSet =
Stream.of(containerOne.containerID()).collect(Collectors.toSet());
+ nodeManager.setContainers(datanodeOne, containerIDSet);
+ containerStateManager.addContainer(containerOne.getProtobuf());
+
+ // Report non-empty QUASI_CLOSED replica with matching bcsId
+ final ContainerReportsProto containerReport = getContainerReportsProto(
+ containerOne.containerID(),
+ ContainerReplicaProto.State.QUASI_CLOSED,
+ datanodeOne.getUuidString(),
+ 200L, // usedBytes
+ 10L, // keyCount (non-empty)
+ 10000L, // bcsId (matches container)
+ 0, // replicaIndex
+ false); // isEmpty=false
+
+ final ContainerReportFromDatanode containerReportFromDatanode =
+ new ContainerReportFromDatanode(datanodeOne, containerReport);
+ reportHandler.onMessage(containerReportFromDatanode, publisher);
+
+ // Container should NOT resurrect
+ final ContainerInfo container =
containerManager.getContainer(containerOne.containerID());
+ assertEquals(LifeCycleState.DELETED, container.getState(),
+ "Container should not resurrect when a stale QUASI_CLOSED replica is
reported");
+
+ // A delete command should be sent for the stale replica
+ verify(publisher, times(1)).fireEvent(eq(SCMEvents.DATANODE_COMMAND),
any(CommandForDatanode.class));
+ }
+
+ /**
+ * Test resurrection of DELETING container to QUASI_CLOSED state when a
non-empty OPEN replica is reported.
+ * OPEN replicas should trigger resurrection to QUASI_CLOSED state.
+ */
+ @Test
+ public void testDeletingContainerResurrectionToQuasiClosedWithOpenReplica()
+ throws NodeNotFoundException, IOException {
+ final ContainerReportHandler reportHandler = new
ContainerReportHandler(nodeManager, containerManager);
+ final DatanodeDetails datanodeOne = nodeManager.getNodes(
+ NodeStatus.inServiceHealthy()).iterator().next();
+ // Create container in DELETING state
+ final ContainerInfo containerOne = getContainer(LifeCycleState.DELETING);
+
+ final Set<ContainerID> containerIDSet =
Stream.of(containerOne.containerID()).collect(Collectors.toSet());
+ nodeManager.setContainers(datanodeOne, containerIDSet);
+ containerStateManager.addContainer(containerOne.getProtobuf());
+
+ // Report non-empty OPEN replica (e.g., stale DN that came back online)
+ final ContainerReportsProto containerReport = getContainerReportsProto(
+ containerOne.containerID(),
+ ContainerReplicaProto.State.OPEN,
+ datanodeOne.getUuidString(),
+ 200L, // usedBytes
+ 10L, // keyCount (non-empty)
+ 10000L, // bcsId
+ 0, // replicaIndex
+ false); // isEmpty=false
+
+ final ContainerReportFromDatanode containerReportFromDatanode =
+ new ContainerReportFromDatanode(datanodeOne, containerReport);
+ reportHandler.onMessage(containerReportFromDatanode, publisher);
+
+ // Container should resurrect to QUASI_CLOSED for OPEN replica
+ final ContainerInfo resurrectedContainer =
containerManager.getContainer(containerOne.containerID());
+ assertEquals(LifeCycleState.QUASI_CLOSED, resurrectedContainer.getState(),
+ "Container should resurrect to QUASI_CLOSED when OPEN replica is
reported");
+
+ // Replica should be updated in SCM
+ assertEquals(1,
containerManager.getContainerReplicas(containerOne.containerID()).size());
+ }
+
@Test
public void testWithNoContainerDataChecksum() throws Exception {
final ContainerReportHandler reportHandler = new
ContainerReportHandler(nodeManager, containerManager);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
index dfb4f38deef..5c3035f28fc 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java
@@ -154,10 +154,11 @@ public void init() throws IOException, TimeoutException,
InvalidStateTransitionE
any(ContainerID.class), any(ContainerReplica.class));
doAnswer(invocation -> {
- containerStateManager.transitionDeletingOrDeletedToClosedState(
- ((ContainerID) invocation.getArgument(0)).getProtobuf());
+ containerStateManager.transitionDeletingOrDeletedToTargetState(
+ ((ContainerID) invocation.getArgument(0)).getProtobuf(),
invocation.getArgument(1));
return null;
-
}).when(containerManager).transitionDeletingOrDeletedToClosedState(any(ContainerID.class));
+ }).when(containerManager).transitionDeletingOrDeletedToTargetState(
+ any(ContainerID.class), any(HddsProtos.LifeCycleState.class));
}
@@ -214,7 +215,7 @@ public void checkReplicationStateMissingReplica()
@ParameterizedTest
@EnumSource(value = HddsProtos.LifeCycleState.class,
names = {"DELETING", "DELETED"})
- public void
testTransitionDeletingOrDeletedToClosedState(HddsProtos.LifeCycleState
lifeCycleState)
+ public void
testTransitionDeletingOrDeletedToTargetState(HddsProtos.LifeCycleState
lifeCycleState)
throws IOException {
HddsProtos.ContainerInfoProto.Builder builder =
HddsProtos.ContainerInfoProto.newBuilder();
builder.setContainerID(1)
@@ -228,7 +229,7 @@ public void
testTransitionDeletingOrDeletedToClosedState(HddsProtos.LifeCycleSta
HddsProtos.ContainerInfoProto container = builder.build();
HddsProtos.ContainerID cid =
HddsProtos.ContainerID.newBuilder().setId(container.getContainerID()).build();
containerStateManager.addContainer(container);
- containerStateManager.transitionDeletingOrDeletedToClosedState(cid);
+ containerStateManager.transitionDeletingOrDeletedToTargetState(cid,
HddsProtos.LifeCycleState.CLOSED);
assertEquals(HddsProtos.LifeCycleState.CLOSED,
containerStateManager.getContainer(ContainerID.getFromProtobuf(cid))
.getState());
}
@@ -254,7 +255,7 @@ public void
testTransitionContainerToClosedStateAllowOnlyDeletingOrDeletedContai
HddsProtos.ContainerID cid =
HddsProtos.ContainerID.newBuilder().setId(container.getContainerID()).build();
containerStateManager.addContainer(container);
try {
- containerStateManager.transitionDeletingOrDeletedToClosedState(cid);
+ containerStateManager.transitionDeletingOrDeletedToTargetState(cid,
HddsProtos.LifeCycleState.CLOSED);
fail("Was expecting an Exception, but did not catch any.");
} catch (IOException e) {
assertInstanceOf(InvalidContainerStateException.class,
e.getCause().getCause());
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java
index 4811a9651c4..0bb3772f0bd 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestEmptyContainerHandler.java
@@ -19,6 +19,7 @@
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSING;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.QUASI_CLOSED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
@@ -284,6 +285,41 @@ public void
testNoUpdateContainerStateWhenReplicaSequenceIdDoesNotMatch()
any(HddsProtos.LifeCycleEvent.class));
}
+ /**
+ * A QUASI_CLOSED container with all empty replicas should be deleted.
+ * Handler should return true and send delete commands to all replicas.
+ */
+ @Test
+ public void testEmptyQuasiClosedRatisContainerReturnsTrue()
+ throws IOException {
+ long keyCount = 0L;
+ long bytesUsed = 0L;
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ ratisReplicationConfig, 1, QUASI_CLOSED, keyCount, bytesUsed);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ ContainerReplicaProto.State.QUASI_CLOSED, keyCount, bytesUsed,
+ 0, 0, 0);
+
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.emptyList())
+ .setReport(new
ReplicationManagerReport(rmConf.getContainerSampleLimit()))
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+
+ ContainerCheckRequest readRequest = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.emptyList())
+ .setReport(new
ReplicationManagerReport(rmConf.getContainerSampleLimit()))
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .setReadOnly(true)
+ .build();
+
+ assertAndVerify(readRequest, true, 0, 1);
+ assertAndVerify(request, true, 3, 1);
+ }
+
/**
* Asserts that handler returns the specified assertion and delete command
* to replicas is sent the specified number of times.
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java
index 5fa918433ee..14f4c849180 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManagerIntegration.java
@@ -29,6 +29,7 @@
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_MAINTENANCE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
+import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
@@ -55,6 +56,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
@@ -362,4 +364,156 @@ public void
testOneDeadMaintenanceNodeAndOneLiveMaintenanceNodeAndOneDecommissio
assertEquals(0, report.getStat(ContainerHealthState.MIS_REPLICATED));
assertEquals(0, report.getStat(ContainerHealthState.OVER_REPLICATED));
}
+
+ /**
+ * Test for empty QUASI_CLOSED container deletion.
+ */
+ @Test
+ public void testEmptyQuasiClosedContainerDeletion() throws Exception {
+ ContainerInfo containerInfo =
containerManager.allocateContainer(RATIS_REPLICATION_CONFIG, "TestOwner");
+ ContainerID cid = containerInfo.containerID();
+ containerManager.updateContainerState(cid,
HddsProtos.LifeCycleEvent.FINALIZE);
+ containerManager.updateContainerState(cid,
HddsProtos.LifeCycleEvent.QUASI_CLOSE);
+
+ // Wait for container to be QUASI_CLOSED
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ContainerInfo info = containerManager.getContainer(cid);
+ return info.getState() == HddsProtos.LifeCycleState.QUASI_CLOSED;
+ } catch (ContainerNotFoundException e) {
+ return false;
+ }
+ }, 100, 5000);
+
+ containerInfo = containerManager.getContainer(cid);
+ assertEquals(HddsProtos.LifeCycleState.QUASI_CLOSED,
containerInfo.getState());
+ assertEquals(0L, containerInfo.getNumberOfKeys());
+
+ // Add empty QUASI_CLOSED replicas
+ List<DatanodeDetails> datanodes = nodeManager.getAllNodes().stream()
+ .limit(3).collect(Collectors.toList());
+
+ for (int i = 0; i < 3; i++) {
+ ContainerReplica replica = ContainerReplica.newBuilder()
+ .setContainerID(cid)
+ .setContainerState(QUASI_CLOSED)
+ .setDatanodeDetails(datanodes.get(i))
+ .setOriginNodeId(datanodes.get(i).getID())
+ .setSequenceId(0L)
+ .setKeyCount(0L)
+ .setBytesUsed(0L)
+ .setEmpty(true)
+ .setReplicaIndex(i)
+ .build();
+ containerManager.updateContainerReplica(cid, replica);
+ }
+
+ Set<ContainerReplica> replicas =
containerManager.getContainerReplicas(cid);
+ assertEquals(3, replicas.size());
+ assertTrue(replicas.stream().allMatch(ContainerReplica::isEmpty));
+
+ replicationManager.getConfig().setInterval(Duration.ofSeconds(1));
+ replicationManager.notifyStatusChanged();
+
+ // QUASI_CLOSED -> DELETING
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ContainerInfo info = containerManager.getContainer(cid);
+ HddsProtos.LifeCycleState state = info.getState();
+ return state == HddsProtos.LifeCycleState.DELETING;
+ } catch (ContainerNotFoundException e) {
+ return false;
+ }
+ }, 1000, 30000);
+
+ containerInfo = containerManager.getContainer(cid);
+ assertEquals(HddsProtos.LifeCycleState.DELETING, containerInfo.getState());
+ }
+
+ /**
+ * Test empty QUASI_CLOSED container deletion with mixed replica states.
+ * Only stable replicas (QUASI_CLOSED/CLOSED) should receive delete commands
initially.
+ * OPEN replicas should be skipped.
+ */
+ @Test
+ public void testEmptyQuasiClosedContainerDeletionWithMixedReplicaStates()
throws Exception {
+ ContainerInfo containerInfo =
containerManager.allocateContainer(RATIS_REPLICATION_CONFIG, "TestOwner");
+ ContainerID cid = containerInfo.containerID();
+ containerManager.updateContainerState(cid,
HddsProtos.LifeCycleEvent.FINALIZE);
+ containerManager.updateContainerState(cid,
HddsProtos.LifeCycleEvent.QUASI_CLOSE);
+
+ // Wait for container to be QUASI_CLOSED
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ContainerInfo info = containerManager.getContainer(cid);
+ return info.getState() == HddsProtos.LifeCycleState.QUASI_CLOSED;
+ } catch (ContainerNotFoundException e) {
+ return false;
+ }
+ }, 100, 5000);
+
+ containerInfo = containerManager.getContainer(cid);
+ assertEquals(HddsProtos.LifeCycleState.QUASI_CLOSED,
containerInfo.getState());
+ assertEquals(0L, containerInfo.getNumberOfKeys());
+ assertEquals(0L, containerInfo.getSequenceId());
+
+ // Add replicas with mixed states: 2 QUASI_CLOSED, 1 OPEN
+ List<DatanodeDetails> datanodes = nodeManager.getAllNodes().stream()
+ .limit(3).collect(Collectors.toList());
+
+ // Add 2 QUASI_CLOSED replicas
+ for (int i = 0; i < 2; i++) {
+ ContainerReplica replica = ContainerReplica.newBuilder()
+ .setContainerID(cid)
+ .setContainerState(QUASI_CLOSED)
+ .setDatanodeDetails(datanodes.get(i))
+ .setOriginNodeId(datanodes.get(i).getID())
+ .setSequenceId(100L)
+ .setKeyCount(0L)
+ .setBytesUsed(0L)
+ .setEmpty(true)
+ .setReplicaIndex(i)
+ .build();
+ containerManager.updateContainerReplica(cid, replica);
+ }
+
+ // Add 1 OPEN replica (will be skipped for delete commands)
+ ContainerReplica openReplica = ContainerReplica.newBuilder()
+ .setContainerID(cid)
+
.setContainerState(StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.OPEN)
+ .setDatanodeDetails(datanodes.get(2))
+ .setOriginNodeId(datanodes.get(2).getID())
+ .setSequenceId(50L) // Lower bcsId (stale)
+ .setKeyCount(0L)
+ .setBytesUsed(0L)
+ .setEmpty(true)
+ .setReplicaIndex(2)
+ .build();
+ containerManager.updateContainerReplica(cid, openReplica);
+
+ Set<ContainerReplica> replicas =
containerManager.getContainerReplicas(cid);
+ assertEquals(3, replicas.size());
+ assertTrue(replicas.stream().allMatch(ContainerReplica::isEmpty), "All
replicas should be empty");
+
+ replicationManager.getConfig().setInterval(Duration.ofSeconds(1));
+ replicationManager.notifyStatusChanged();
+
+ // Container should still transition to DELETING
+ // (Delete commands sent to stable replicas, OPEN replica skipped)
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ContainerInfo info = containerManager.getContainer(cid);
+ HddsProtos.LifeCycleState state = info.getState();
+ return state == HddsProtos.LifeCycleState.DELETING;
+ } catch (ContainerNotFoundException e) {
+ return false;
+ }
+ }, 1000, 30000);
+
+ containerInfo = containerManager.getContainer(cid);
+ assertEquals(HddsProtos.LifeCycleState.DELETING, containerInfo.getState());
+
+ // Verify bcsId was updated to max (100L from QUASI_CLOSED replicas)
+ assertEquals(100L, containerInfo.getSequenceId(), "Container bcsId should
be updated to max replica bcsId");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
