This is an automated email from the ASF dual-hosted git repository.
aryangupta1998 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 524660937a5 HDDS-12865. Handle automatic container cache refresh for
replica verification. (#11044)
524660937a5 is described below
commit 524660937a57b173e29c20220eb18e4abc578912
Author: Aryan Gupta <[email protected]>
AuthorDate: Wed Aug 26 11:34:08 2026 +0530
HDDS-12865. Handle automatic container cache refresh for replica
verification. (#11044)
---
.../debug/replicas/BlockExistenceVerifier.java | 6 +
.../debug/replicas/BlockVerificationResult.java | 15 ++
.../ozone/debug/replicas/ChecksumVerifier.java | 18 ++
.../ozone/debug/replicas/ReplicasVerify.java | 117 +++++++++--
.../ozone/debug/replicas/TestReplicasVerify.java | 227 ++++++++++++++++++++-
5 files changed, 366 insertions(+), 17 deletions(-)
diff --git
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java
index dde79989d02..5cae3453321 100644
---
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java
+++
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -67,6 +68,11 @@ public BlockVerificationResult verifyBlock(DatanodeDetails
datanode, OmKeyLocati
} else {
return BlockVerificationResult.failCheck("Block does not exist on this
replica");
}
+ } catch (StorageContainerException e) {
+ if (e.getResult() == ContainerProtos.Result.NO_SUCH_BLOCK) {
+ return
BlockVerificationResult.failCheckAndRefreshKeyLocation(e.getMessage());
+ }
+ return BlockVerificationResult.failIncomplete(e.getMessage());
} catch (IOException e) {
return BlockVerificationResult.failIncomplete(e.getMessage());
} finally {
diff --git
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java
index c73635ba475..e94e45e2497 100644
---
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java
+++
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java
@@ -28,11 +28,18 @@ public class BlockVerificationResult {
private final boolean completed;
private final boolean pass;
private final List<String> failures;
+ private final boolean refreshKeyLocation;
public BlockVerificationResult(boolean completed, boolean pass, List<String>
failures) {
+ this(completed, pass, failures, false);
+ }
+
+ public BlockVerificationResult(boolean completed, boolean pass, List<String>
failures,
+ boolean refreshKeyLocation) {
this.completed = completed;
this.pass = pass;
this.failures = failures;
+ this.refreshKeyLocation = refreshKeyLocation;
}
public static BlockVerificationResult pass() {
@@ -43,6 +50,10 @@ public static BlockVerificationResult failCheck(String
message) {
return new BlockVerificationResult(true, false,
Collections.singletonList(message));
}
+ public static BlockVerificationResult failCheckAndRefreshKeyLocation(String
message) {
+ return new BlockVerificationResult(true, false,
Collections.singletonList(message), true);
+ }
+
public static BlockVerificationResult failIncomplete(String message) {
return new BlockVerificationResult(false, false,
Collections.singletonList(message));
}
@@ -59,4 +70,8 @@ public List<String> getFailures() {
return failures;
}
+ public boolean shouldRefreshKeyLocation() {
+ return refreshKeyLocation;
+ }
+
}
diff --git
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java
index 96f8218526d..2acc514481a 100644
---
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java
+++
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java
@@ -23,9 +23,11 @@
import org.apache.commons.io.output.NullOutputStream;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
+import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl;
import org.apache.hadoop.ozone.common.OzoneChecksumException;
@@ -68,6 +70,11 @@ public BlockVerificationResult verifyBlock(DatanodeDetails
datanode, OmKeyLocati
return BlockVerificationResult.pass();
} catch (IOException e) {
Throwable cause = e.getCause() != null ? e.getCause() : e;
+ StorageContainerException storageContainerException =
findStorageContainerException(e);
+ if (storageContainerException != null
+ && storageContainerException.getResult() ==
ContainerProtos.Result.NO_SUCH_BLOCK) {
+ return
BlockVerificationResult.failCheckAndRefreshKeyLocation(storageContainerException.getMessage());
+ }
if (cause instanceof OzoneChecksumException) {
return BlockVerificationResult.failCheck(cause.getMessage());
} else {
@@ -75,4 +82,15 @@ public BlockVerificationResult verifyBlock(DatanodeDetails
datanode, OmKeyLocati
}
}
}
+
+ private static StorageContainerException
findStorageContainerException(Throwable throwable) {
+ Throwable current = throwable;
+ while (current != null) {
+ if (current instanceof StorageContainerException) {
+ return (StorageContainerException) current;
+ }
+ current = current.getCause();
+ }
+ return null;
+ }
}
diff --git
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java
index e886f945c34..784e3fdc918 100644
---
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java
+++
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java
@@ -338,14 +338,69 @@ void checkBucket(OzoneClient ozoneClient, OzoneBucket
bucket, ArrayNode keysArra
void processKey(OzoneClient ozoneClient, String volumeName, String
bucketName, String keyName,
ArrayNode keysArray, AtomicBoolean allKeysPassed) throws IOException {
keysProcessed.incrementAndGet();
- OmKeyInfo keyInfo = ozoneClient.getProxy().getKeyInfo(
- volumeName, bucketName, keyName, refreshContainerLocationsFromScm);
+ OmKeyInfo keyInfo;
+ try {
+ keyInfo = ozoneClient.getProxy().getKeyInfo(
+ volumeName, bucketName, keyName, refreshContainerLocationsFromScm);
+ } catch (IOException e) {
+ LOG.warn("Unable to fetch key info from OM for key {}/{}/{}; marking
verification incomplete.",
+ volumeName, bucketName, keyName, e);
+ markKeyFetchFailure(volumeName, bucketName, keyName, e.getMessage(),
keysArray, allKeysPassed);
+ return;
+ }
// Check if key should be processed based on replication config
if (!shouldProcessKeyByReplicationType(keyInfo)) {
return;
}
+ KeyVerificationResult keyVerificationResult = verifyKey(keyInfo,
volumeName, bucketName, keyName);
+ if (keyVerificationResult.shouldRefreshKeyLocation()) {
+ try {
+ OmKeyInfo refreshedKeyInfo = ozoneClient.getProxy().getKeyInfo(
+ volumeName, bucketName, keyName, true);
+ keyVerificationResult = verifyKey(refreshedKeyInfo, volumeName,
bucketName, keyName);
+ } catch (IOException e) {
+ LOG.warn("Unable to refresh key location from OM for key {}/{}/{};
marking verification incomplete.",
+ volumeName, bucketName, keyName, e);
+ keyVerificationResult.markRefreshFailure(e.getMessage());
+ }
+ }
+
+ keyVerificationResult.getKeyNode().put("pass",
keyVerificationResult.passed());
+ if (keyVerificationResult.passed()) {
+ keysPassed.incrementAndGet();
+ } else {
+ keysFailed.incrementAndGet();
+ allKeysPassed.set(false);
+ keyVerificationResult.getFailedVerificationTypes().forEach(failedType ->
failuresByType
+ .computeIfAbsent(failedType, k -> new AtomicInteger(0))
+ .incrementAndGet()
+ );
+ }
+
+ if (!keyVerificationResult.passed() || allResults) {
+ keysArray.add(keyVerificationResult.getKeyNode());
+ }
+ }
+
+ private void markKeyFetchFailure(String volumeName, String bucketName,
String keyName, String message,
+ ArrayNode keysArray, AtomicBoolean allKeysPassed) {
+ ObjectNode keyNode = JsonUtils.createObjectNode(null);
+ keyNode.put("volumeName", volumeName);
+ keyNode.put("bucketName", bucketName);
+ keyNode.put("name", keyName);
+ keyNode.putArray("blocks");
+ keyNode.put("completed", false);
+ keyNode.put("pass", false);
+ keyNode.putArray("failures").addObject().put("message", "Failed to fetch
key info from OM: " + message);
+ keysFailed.incrementAndGet();
+ allKeysPassed.set(false);
+ keysArray.add(keyNode);
+ }
+
+ private KeyVerificationResult verifyKey(
+ OmKeyInfo keyInfo, String volumeName, String bucketName, String keyName)
{
ObjectNode keyNode = JsonUtils.createObjectNode(null);
keyNode.put("volumeName", volumeName);
keyNode.put("bucketName", bucketName);
@@ -353,7 +408,9 @@ void processKey(OzoneClient ozoneClient, String volumeName,
String bucketName, S
ArrayNode blocksArray = keyNode.putArray("blocks");
boolean keyPass = true;
+ boolean shouldRefreshKeyLocation = false;
Set<String> failedVerificationTypes = new HashSet<>();
+ List<ObjectNode> refreshChecks = new ArrayList<>();
for (OmKeyLocationInfo keyLocation :
keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly()) {
long containerID = keyLocation.getContainerID();
@@ -383,6 +440,10 @@ void processKey(OzoneClient ozoneClient, String
volumeName, String bucketName, S
checkNode.put("type", verifier.getType());
checkNode.put("completed", result.isCompleted());
checkNode.put("pass", result.passed());
+ if (result.shouldRefreshKeyLocation()) {
+ shouldRefreshKeyLocation = true;
+ refreshChecks.add(checkNode);
+ }
ArrayNode failuresArray = checkNode.putArray("failures");
for (String failure : result.getFailures()) {
@@ -406,20 +467,48 @@ void processKey(OzoneClient ozoneClient, String
volumeName, String bucketName, S
}
}
- keyNode.put("pass", keyPass);
- if (keyPass) {
- keysPassed.incrementAndGet();
- } else {
- keysFailed.incrementAndGet();
- allKeysPassed.set(false);
- failedVerificationTypes.forEach(failedType -> failuresByType
- .computeIfAbsent(failedType, k -> new AtomicInteger(0))
- .incrementAndGet()
- );
+ return new KeyVerificationResult(
+ keyNode, keyPass, failedVerificationTypes, shouldRefreshKeyLocation,
refreshChecks);
+ }
+
+ private static final class KeyVerificationResult {
+ private final ObjectNode keyNode;
+ private final boolean pass;
+ private final Set<String> failedVerificationTypes;
+ private final boolean refreshKeyLocation;
+ private final List<ObjectNode> refreshChecks;
+
+ private KeyVerificationResult(ObjectNode keyNode, boolean pass,
Set<String> failedVerificationTypes,
+ boolean refreshKeyLocation, List<ObjectNode> refreshChecks) {
+ this.keyNode = keyNode;
+ this.pass = pass;
+ this.failedVerificationTypes = failedVerificationTypes;
+ this.refreshKeyLocation = refreshKeyLocation;
+ this.refreshChecks = refreshChecks;
+ }
+
+ private ObjectNode getKeyNode() {
+ return keyNode;
+ }
+
+ private boolean passed() {
+ return pass;
}
- if (!keyPass || allResults) {
- keysArray.add(keyNode);
+ private Set<String> getFailedVerificationTypes() {
+ return failedVerificationTypes;
+ }
+
+ private boolean shouldRefreshKeyLocation() {
+ return refreshKeyLocation;
+ }
+
+ private void markRefreshFailure(String message) {
+ String refreshFailure = "Failed to refresh key location from OM: " +
message;
+ for (ObjectNode checkNode : refreshChecks) {
+ checkNode.put("completed", false);
+ checkNode.withArray("failures").addObject().put("message",
refreshFailure);
+ }
}
}
diff --git
a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java
b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java
index b42a0cf87b9..a70d7bc7baf 100644
---
a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java
+++
b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java
@@ -18,15 +18,45 @@
package org.apache.hadoop.ozone.debug.replicas;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.shell.ShellReplicationOptions;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;
/**
- * Unit tests for replicas verify command option parsing.
+ * Unit tests for replicas verify command.
*/
-public class TestReplicasVerify {
+class TestReplicasVerify {
@Test
void testRefreshContainerLocationsFromScmOption() throws Exception {
@@ -46,9 +76,200 @@ void testRefreshContainerLocationsFromScmDefault() throws
Exception {
assertThat(isRefreshContainerLocationsFromScmEnabled(command)).isFalse();
}
- private boolean isRefreshContainerLocationsFromScmEnabled(ReplicasVerify
command) throws Exception {
+ @Test
+ void testRetriesWithForcedContainerCacheRefreshOnBlockNotFound() throws
Exception {
+ ReplicasVerify replicasVerify = new ReplicasVerify();
+ ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class);
+ ReplicaVerifier checksumVerifier = mock(ReplicaVerifier.class);
+
+ when(blockExistenceVerifier.getType()).thenReturn("blockExistence");
+ when(checksumVerifier.getType()).thenReturn("checksum");
+ when(blockExistenceVerifier.verifyBlock(any(), any()))
+
.thenReturn(BlockVerificationResult.failCheckAndRefreshKeyLocation("Block not
found"))
+ .thenReturn(BlockVerificationResult.pass());
+ when(checksumVerifier.verifyBlock(any(), any()))
+ .thenReturn(BlockVerificationResult.pass())
+ .thenReturn(BlockVerificationResult.pass());
+ setField(replicasVerify, "replicaVerifiers",
Arrays.asList(blockExistenceVerifier, checksumVerifier));
+ setField(replicasVerify, "replication", emptyReplicationFilter());
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ when(ozoneClient.getProxy()).thenReturn(proxy);
+
+ OmKeyLocationInfo firstLocation = createKeyLocationInfo(1L, 11L);
+ OmKeyLocationInfo refreshedLocation = createKeyLocationInfo(2L, 22L);
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", false))
+ .thenReturn(createKeyInfo(firstLocation));
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", true))
+ .thenReturn(createKeyInfo(refreshedLocation));
+
+ ObjectNode root = JsonUtils.createObjectNode(null);
+ ArrayNode keysArray = root.putArray("keys");
+ AtomicBoolean allKeysPassed = new AtomicBoolean(true);
+
+ replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1",
keysArray, allKeysPassed);
+
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false);
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", true);
+ verify(blockExistenceVerifier, times(2)).verifyBlock(any(), any());
+ verify(checksumVerifier, times(2)).verifyBlock(any(), any());
+ assertTrue(allKeysPassed.get());
+ assertEquals(0, keysArray.size());
+ }
+
+ @Test
+ void testDoesNotRefreshOnNonRefreshableFailure() throws Exception {
+ ReplicasVerify replicasVerify = new ReplicasVerify();
+ ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class);
+ ReplicaVerifier checksumVerifier = mock(ReplicaVerifier.class);
+
+ when(blockExistenceVerifier.getType()).thenReturn("blockExistence");
+ when(checksumVerifier.getType()).thenReturn("checksum");
+ when(blockExistenceVerifier.verifyBlock(any(), any()))
+ .thenReturn(BlockVerificationResult.failCheck("Block does not exist on
this replica"));
+ when(checksumVerifier.verifyBlock(any(), any()))
+ .thenReturn(BlockVerificationResult.pass());
+ setField(replicasVerify, "replicaVerifiers",
Arrays.asList(blockExistenceVerifier, checksumVerifier));
+ setField(replicasVerify, "replication", emptyReplicationFilter());
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ when(ozoneClient.getProxy()).thenReturn(proxy);
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", false))
+ .thenReturn(createKeyInfo(createKeyLocationInfo(1L, 11L)));
+
+ ObjectNode root = JsonUtils.createObjectNode(null);
+ ArrayNode keysArray = root.putArray("keys");
+ AtomicBoolean allKeysPassed = new AtomicBoolean(true);
+
+ replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1",
keysArray, allKeysPassed);
+
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false);
+ verify(proxy, never()).getKeyInfo("vol1", "bucket1", "key1", true);
+ verify(blockExistenceVerifier, times(1)).verifyBlock(any(), any());
+ verify(checksumVerifier, times(1)).verifyBlock(any(), any());
+ assertFalse(allKeysPassed.get());
+ assertEquals(1, keysArray.size());
+ assertFalse(keysArray.get(0).get("pass").asBoolean());
+ }
+
+ @Test
+ void testRefreshFailureMarksKeyIncompleteAndDoesNotAbortRun() throws
Exception {
+ ReplicasVerify replicasVerify = new ReplicasVerify();
+ ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class);
+
+ when(blockExistenceVerifier.getType()).thenReturn("blockExistence");
+ when(blockExistenceVerifier.verifyBlock(any(), any()))
+
.thenReturn(BlockVerificationResult.failCheckAndRefreshKeyLocation("Block not
found"));
+ setField(replicasVerify, "replicaVerifiers",
Collections.singletonList(blockExistenceVerifier));
+ setField(replicasVerify, "replication", emptyReplicationFilter());
+
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ when(ozoneClient.getProxy()).thenReturn(proxy);
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", false))
+ .thenReturn(createKeyInfo(createKeyLocationInfo(1L, 11L)));
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", true))
+ .thenThrow(new IOException("OM refresh failed"));
+
+ ObjectNode root = JsonUtils.createObjectNode(null);
+ ArrayNode keysArray = root.putArray("keys");
+ AtomicBoolean allKeysPassed = new AtomicBoolean(true);
+
+ replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1",
keysArray, allKeysPassed);
+
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false);
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", true);
+ assertFalse(allKeysPassed.get());
+ assertEquals(1, keysArray.size());
+ assertFalse(keysArray.get(0).get("pass").asBoolean());
+ assertFalse(keysArray.get(0).get("blocks").get(0)
+
.get("replicas").get(0).get("checks").get(0).get("completed").asBoolean());
+ assertThat(keysArray.get(0).get("blocks").get(0)
+
.get("replicas").get(0).get("checks").get(0).get("failures").toString())
+ .contains("Failed to refresh key location from OM: OM refresh failed");
+ }
+
+ @Test
+ void testInitialGetKeyInfoFailureMarksKeyIncompleteAndDoesNotAbortRun()
throws Exception {
+ ReplicasVerify replicasVerify = new ReplicasVerify();
+ OzoneClient ozoneClient = mock(OzoneClient.class);
+ ClientProtocol proxy = mock(ClientProtocol.class);
+ when(ozoneClient.getProxy()).thenReturn(proxy);
+ when(proxy.getKeyInfo("vol1", "bucket1", "key1", false))
+ .thenThrow(new IOException("Key not found"));
+
+ ObjectNode root = JsonUtils.createObjectNode(null);
+ ArrayNode keysArray = root.putArray("keys");
+ AtomicBoolean allKeysPassed = new AtomicBoolean(true);
+
+ replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1",
keysArray, allKeysPassed);
+
+ verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false);
+ verify(proxy, never()).getKeyInfo("vol1", "bucket1", "key1", true);
+ assertFalse(allKeysPassed.get());
+ assertEquals(1, keysArray.size());
+ assertFalse(keysArray.get(0).get("completed").asBoolean());
+ assertFalse(keysArray.get(0).get("pass").asBoolean());
+ assertThat(keysArray.get(0).get("failures").toString())
+ .contains("Failed to fetch key info from OM: Key not found");
+ }
+
+ private static boolean
isRefreshContainerLocationsFromScmEnabled(ReplicasVerify command) throws
Exception {
Field field =
ReplicasVerify.class.getDeclaredField("refreshContainerLocationsFromScm");
field.setAccessible(true);
return field.getBoolean(command);
}
+
+ private static OmKeyInfo createKeyInfo(OmKeyLocationInfo keyLocationInfo) {
+ OmKeyLocationInfoGroup latestVersionLocations =
+ new OmKeyLocationInfoGroup(0,
Collections.singletonList(keyLocationInfo));
+ return new OmKeyInfo.Builder()
+ .setVolumeName("vol1")
+ .setBucketName("bucket1")
+ .setKeyName("key1")
+
.setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE))
+
.setOmKeyLocationInfos(Collections.singletonList(latestVersionLocations))
+ .build();
+ }
+
+ private static OmKeyLocationInfo createKeyLocationInfo(long containerId,
long localId) {
+ DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails();
+ Pipeline pipeline = Pipeline.newBuilder()
+ .setId(PipelineID.randomId())
+
.setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE))
+ .setState(Pipeline.PipelineState.OPEN)
+ .setNodes(Collections.singletonList(datanode))
+ .build();
+
+ return new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(containerId, localId))
+ .setPipeline(pipeline)
+ .setLength(1)
+ .setOffset(0)
+ .setCreateVersion(0)
+ .build();
+ }
+
+ private static ShellReplicationOptions emptyReplicationFilter() {
+ ShellReplicationOptions replication = mock(ShellReplicationOptions.class);
+ when(replication.fromParams(any())).thenReturn(Optional.empty());
+ return replication;
+ }
+
+ private static void setField(Object target, String fieldName, Object value)
throws Exception {
+ Class<?> type = target.getClass();
+ while (type != null) {
+ try {
+ Field field = type.getDeclaredField(fieldName);
+ field.setAccessible(true);
+ field.set(target, value);
+ return;
+ } catch (NoSuchFieldException ignored) {
+ type = type.getSuperclass();
+ }
+ }
+ throw new IOException("Unable to set field '" + fieldName + "' on " +
target.getClass().getName());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
