This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8fd042079f4 HDDS-10314. Speed up
TestDatanodeHddsVolumeFailureDetection (#10819)
8fd042079f4 is described below
commit 8fd042079f4008d9b7aaf8cd6b1c7c3a91150d78
Author: Eric C. Ho <[email protected]>
AuthorDate: Wed Jul 29 01:28:16 2026 +0800
HDDS-10314. Speed up TestDatanodeHddsVolumeFailureDetection (#10819)
---
.../TestDatanodeHddsVolumeFailureDetection.java | 437 ++++++++++++---------
1 file changed, 246 insertions(+), 191 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java
index 43f7139b216..5c5baec2849 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/volume/TestDatanodeHddsVolumeFailureDetection.java
@@ -18,14 +18,20 @@
package org.apache.hadoop.ozone.dn.volume;
import static org.apache.commons.io.IOUtils.readFully;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.io.File;
import java.io.IOException;
@@ -34,16 +40,16 @@
import java.nio.file.Paths;
import java.time.Duration;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.ozone.DataTestUtil;
import org.apache.hadoop.ozone.HddsDatanodeService;
@@ -61,175 +67,219 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.dn.DatanodeTestUtils;
-import org.junit.jupiter.params.ParameterizedTest;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.junit.jupiter.params.AfterParameterizedClassInvocation;
+import org.junit.jupiter.params.BeforeParameterizedClassInvocation;
+import org.junit.jupiter.params.Parameter;
+import org.junit.jupiter.params.ParameterizedClass;
import org.junit.jupiter.params.provider.ValueSource;
/**
* This class tests datanode can detect failed volumes.
*/
+@ParameterizedClass
+@ValueSource(booleans = {true, false})
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Execution(ExecutionMode.SAME_THREAD)
class TestDatanodeHddsVolumeFailureDetection {
private static final int KEY_SIZE = 128;
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void corruptChunkFile(boolean schemaV3) throws Exception {
- try (MiniOzoneCluster cluster = newCluster(schemaV3)) {
- try (OzoneClient client = cluster.newClient()) {
- OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
-
- // write a file
- String keyName = UUID.randomUUID().toString();
- long containerId = createKey(bucket, keyName);
-
- // corrupt chunk file by rename file->dir
- HddsDatanodeService dn = cluster.getHddsDatanodes().get(0);
- OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
- MutableVolumeSet volSet = oc.getVolumeSet();
- StorageVolume vol0 = volSet.getVolumesList().get(0);
- HddsVolume volume = assertInstanceOf(HddsVolume.class, vol0);
- Path chunksPath = Paths.get(
- volume.getStorageDir().getPath(),
- volume.getClusterID(),
- Storage.STORAGE_DIR_CURRENT,
- Storage.CONTAINER_DIR + "0",
- String.valueOf(containerId),
- OzoneConsts.STORAGE_DIR_CHUNKS
- );
- File[] chunkFiles = chunksPath.toFile().listFiles();
- assertNotNull(chunkFiles);
-
- try {
- for (File chunkFile : chunkFiles) {
- DatanodeTestUtils.injectDataFileFailure(chunkFile);
- }
-
- // simulate bad volume by removing write permission on root dir
- // refer to HddsVolume.check()
- DatanodeTestUtils.simulateBadVolume(vol0);
-
- // read written file to trigger checkVolumeAsync
- readKeyToTriggerCheckVolumeAsync(bucket, keyName);
-
- // should trigger checkVolumeAsync and
- // a failed volume should be detected
- DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1);
- } finally {
- // restore for cleanup
- DatanodeTestUtils.restoreBadVolume(vol0);
- for (File chunkFile : chunkFiles) {
- DatanodeTestUtils.restoreDataFileFromFailure(chunkFile);
- }
- }
- }
+ @Parameter
+ private boolean schemaV3;
+
+ private MiniOzoneCluster cluster;
+ private HddsDatanodeService currentDatanode;
+ private long currentContainerId;
+
+ @BeforeParameterizedClassInvocation
+ void initCluster() throws Exception {
+ cluster = newCluster(schemaV3);
+ }
+
+ @AfterEach
+ void failCurrentVolume() throws Exception {
+ HddsDatanodeService datanode = currentDatanode;
+ currentDatanode = null;
+ if (datanode == null) {
+ return;
}
+
cluster.getStorageContainerLocationClient().closeContainer(currentContainerId);
+ OzoneContainer container =
datanode.getDatanodeStateMachine().getContainer();
+ MutableVolumeSet volumeSet = container.getVolumeSet();
+ if (!volumeSet.getVolumesList().isEmpty()) {
+ StorageVolume volume = volumeSet.getVolumesList().get(0);
+ volumeSet.failVolume(volume.getStorageDir().getPath());
+ container.handleVolumeFailures();
+ }
+ waitForHandleFailedVolume(volumeSet);
+ GenericTestUtils.waitFor(() -> isFailedVolumeReported(datanode), 100,
10000);
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void corruptContainerFile(boolean schemaV3) throws Exception {
- try (MiniOzoneCluster cluster = newCluster(schemaV3)) {
- // create a container
- ContainerWithPipeline container;
- OzoneConfiguration conf = cluster.getConf();
- try (ScmClient scmClient = new ContainerOperationClient(conf)) {
- container = scmClient.createContainer(ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE, OzoneConsts.OZONE);
- }
+ @AfterParameterizedClassInvocation
+ void shutdown() {
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+
+ @Test
+ void corruptChunkFile() throws Exception {
+ try (OzoneClient client = cluster.newClient()) {
+ OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
+
+ // write a file
+ String keyName = UUID.randomUUID().toString();
+ long containerId = createKey(bucket, keyName);
+ currentContainerId = containerId;
- // corrupt container file by removing write permission on
- // container metadata dir, since container update operation
- // use a create temp & rename way, so we can't just rename
- // container file to simulate corruption
- HddsDatanodeService dn = cluster.getHddsDatanodes().get(0);
+ // corrupt chunk file by rename file->dir
+ HddsDatanodeService dn = getDatanode(containerId);
+ currentDatanode = dn;
OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
MutableVolumeSet volSet = oc.getVolumeSet();
StorageVolume vol0 = volSet.getVolumesList().get(0);
- Container<?> c1 = oc.getContainerSet().getContainer(
- container.getContainerInfo().getContainerID());
- File metadataDir = new File(c1.getContainerFile().getParent());
+ HddsVolume volume = assertInstanceOf(HddsVolume.class, vol0);
+ Path chunksPath = Paths.get(
+ volume.getStorageDir().getPath(),
+ volume.getClusterID(),
+ Storage.STORAGE_DIR_CURRENT,
+ Storage.CONTAINER_DIR + "0",
+ String.valueOf(containerId),
+ OzoneConsts.STORAGE_DIR_CHUNKS
+ );
+ File[] chunkFiles = chunksPath.toFile().listFiles();
+ assertNotNull(chunkFiles);
+
try {
- DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir);
+ for (File chunkFile : chunkFiles) {
+ DatanodeTestUtils.injectDataFileFailure(chunkFile);
+ }
// simulate bad volume by removing write permission on root dir
// refer to HddsVolume.check()
DatanodeTestUtils.simulateBadVolume(vol0);
- // close container to trigger checkVolumeAsync
- assertThrows(IOException.class, c1::close);
+ // read written file to trigger checkVolumeAsync
+ readKeyToTriggerCheckVolumeAsync(bucket, keyName);
- // should trigger CheckVolumeAsync and
+ // should trigger checkVolumeAsync and
// a failed volume should be detected
- DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1);
+ waitForHandleFailedVolume(volSet);
} finally {
// restore for cleanup
DatanodeTestUtils.restoreBadVolume(vol0);
- DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir);
+ for (File chunkFile : chunkFiles) {
+ DatanodeTestUtils.restoreDataFileFromFailure(chunkFile);
+ }
}
}
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- void corruptDbFile(boolean schemaV3) throws Exception {
- try (MiniOzoneCluster cluster = newCluster(schemaV3)) {
- try (OzoneClient client = cluster.newClient()) {
- OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
-
- // write a file, will create container1
- String keyName = UUID.randomUUID().toString();
- long containerId = createKey(bucket, keyName);
-
- // close container1
- HddsDatanodeService dn = cluster.getHddsDatanodes().get(0);
- OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
- Container<?> c1 = oc.getContainerSet().getContainer(containerId);
- c1.close();
-
- // create container2, and container1 is kicked out of cache
- OzoneConfiguration conf = cluster.getConf();
- try (ScmClient scmClient = new ContainerOperationClient(conf)) {
- ContainerWithPipeline c2 = scmClient.createContainer(
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
- OzoneConsts.OZONE);
- assertEquals(c2.getContainerInfo().getState(), LifeCycleState.OPEN);
- }
+ @Test
+ void corruptContainerFile() throws Exception {
+ // create a container
+ ContainerWithPipeline container;
+ OzoneConfiguration conf = cluster.getConf();
+ try (ScmClient scmClient = new ContainerOperationClient(conf)) {
+ container = scmClient.createContainer(
+ RatisReplicationConfig.getInstance(ReplicationFactor.ONE),
+ OzoneConsts.OZONE);
+ }
+ currentContainerId = container.getContainerInfo().getContainerID();
+
+ // corrupt container file by removing write permission on
+ // container metadata dir, since container update operation
+ // use a create temp & rename way, so we can't just rename
+ // container file to simulate corruption
+ HddsDatanodeService dn =
cluster.getHddsDatanode(container.getPipeline().getFirstNode());
+ currentDatanode = dn;
+ OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
+ MutableVolumeSet volSet = oc.getVolumeSet();
+ StorageVolume vol0 = volSet.getVolumesList().get(0);
+ Container<?> c1 = oc.getContainerSet().getContainer(
+ container.getContainerInfo().getContainerID());
+ File metadataDir = new File(c1.getContainerFile().getParent());
+ try {
+ DatanodeTestUtils.injectContainerMetaDirFailure(metadataDir);
+
+ // simulate bad volume by removing write permission on root dir
+ // refer to HddsVolume.check()
+ DatanodeTestUtils.simulateBadVolume(vol0);
+
+ // close container to trigger checkVolumeAsync
+ assertThrows(IOException.class, c1::close);
+
+ // should trigger CheckVolumeAsync and
+ // a failed volume should be detected
+ waitForHandleFailedVolume(volSet);
+ } finally {
+ // restore for cleanup
+ DatanodeTestUtils.restoreBadVolume(vol0);
+ DatanodeTestUtils.restoreContainerMetaDirFromFailure(metadataDir);
+ }
+ }
- // corrupt db by rename dir->file
- File dbDir;
+ @Test
+ void corruptDbFile() throws Exception {
+ try (OzoneClient client = cluster.newClient()) {
+ OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
+
+ // write a file, will create container1
+ String keyName = UUID.randomUUID().toString();
+ long containerId = createKey(bucket, keyName);
+ currentContainerId = containerId;
+
+ // close container1
+ HddsDatanodeService dn = getDatanode(containerId);
+ currentDatanode = dn;
+ OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
+ Container<?> c1 = oc.getContainerSet().getContainer(containerId);
+ c1.close();
+
+ // corrupt db by rename dir->file
+ File dbDir;
+ if (schemaV3) {
+ dbDir = new File(((KeyValueContainerData) (c1.getContainerData()))
+ .getDbFile().getAbsolutePath());
+ } else {
+ File metadataDir = new File(c1.getContainerFile().getParent());
+ dbDir = new File(metadataDir, containerId +
OzoneConsts.DN_CONTAINER_DB);
+ }
+
+ MutableVolumeSet volSet = oc.getVolumeSet();
+ StorageVolume vol0 = volSet.getVolumesList().get(0);
+
+ try {
+ // remove RocksDB from cache
+ KeyValueContainerData containerData = (KeyValueContainerData)
c1.getContainerData();
if (schemaV3) {
- dbDir = new File(((KeyValueContainerData) (c1.getContainerData()))
- .getDbFile().getAbsolutePath());
+ DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath());
} else {
- File metadataDir = new File(c1.getContainerFile().getParent());
- dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB);
+ BlockUtils.removeDB(containerData, cluster.getConf());
}
+ DatanodeTestUtils.injectDataDirFailure(dbDir);
- MutableVolumeSet volSet = oc.getVolumeSet();
- StorageVolume vol0 = volSet.getVolumesList().get(0);
-
- try {
- DatanodeTestUtils.injectDataDirFailure(dbDir);
- if (schemaV3) {
- // remove rocksDB from cache
- DatanodeStoreCache.getInstance().removeDB(dbDir.getAbsolutePath());
- }
-
- // simulate bad volume by removing write permission on root dir
- // refer to HddsVolume.check()
- DatanodeTestUtils.simulateBadVolume(vol0);
-
- readKeyToTriggerCheckVolumeAsync(bucket, keyName);
-
- // should trigger CheckVolumeAsync and
- // a failed volume should be detected
- DatanodeTestUtils.waitForHandleFailedVolume(volSet, 1);
- } finally {
- // restore all
- DatanodeTestUtils.restoreBadVolume(vol0);
- DatanodeTestUtils.restoreDataDirFromFailure(dbDir);
- }
+ // simulate bad volume by removing write permission on root dir
+ // refer to HddsVolume.check()
+ DatanodeTestUtils.simulateBadVolume(vol0);
+
+ readKeyToTriggerCheckVolumeAsync(bucket, keyName);
+
+ // should trigger CheckVolumeAsync and
+ // a failed volume should be detected
+ waitForHandleFailedVolume(volSet);
+ } finally {
+ // restore all
+ DatanodeTestUtils.restoreBadVolume(vol0);
+ DatanodeTestUtils.restoreDataDirFromFailure(dbDir);
}
}
}
@@ -239,65 +289,47 @@ void corruptDbFile(boolean schemaV3) throws Exception {
* test to reach the helper method {@link HddsVolume#checkDbHealth}.
* As a workaround, we test the helper method directly.
* As we test the helper method directly, we cannot test for schemas older
than V3.
- *
- * @param schemaV3
- * @throws Exception
*/
- @ParameterizedTest
- @ValueSource(booleans = {true})
- void corruptDbFileWithoutDbHandleCacheInvalidation(boolean schemaV3) throws
Exception {
- try (MiniOzoneCluster cluster = newCluster(schemaV3)) {
- try (OzoneClient client = cluster.newClient()) {
- OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
-
- // write a file, will create container1
- String keyName = UUID.randomUUID().toString();
- long containerId = createKey(bucket, keyName);
-
- // close container1
- HddsDatanodeService dn = cluster.getHddsDatanodes().get(0);
- OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
- Container<?> c1 = oc.getContainerSet().getContainer(containerId);
- c1.close();
-
- // create container2, and container1 is kicked out of cache
- OzoneConfiguration conf = cluster.getConf();
- try (ScmClient scmClient = new ContainerOperationClient(conf)) {
- ContainerWithPipeline c2 = scmClient.createContainer(
- ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
- OzoneConsts.OZONE);
- assertEquals(c2.getContainerInfo().getState(), LifeCycleState.OPEN);
- }
+ @Test
+ void corruptDbFileWithoutDbHandleCacheInvalidation() throws Exception {
+ assumeTrue(schemaV3);
+ try (OzoneClient client = cluster.newClient()) {
+ OzoneBucket bucket = DataTestUtil.createVolumeAndBucket(client);
+
+ // write a file, will create container1
+ String keyName = UUID.randomUUID().toString();
+ long containerId = createKey(bucket, keyName);
+ currentContainerId = containerId;
+
+ // close container1
+ HddsDatanodeService dn = getDatanode(containerId);
+ currentDatanode = dn;
+ OzoneContainer oc = dn.getDatanodeStateMachine().getContainer();
+ Container<?> c1 = oc.getContainerSet().getContainer(containerId);
+ c1.close();
- // corrupt db by rename dir->file
- File dbDir;
- if (schemaV3) {
- dbDir = new File(((KeyValueContainerData) (c1.getContainerData()))
- .getDbFile().getAbsolutePath());
- } else {
- File metadataDir = new File(c1.getContainerFile().getParent());
- dbDir = new File(metadataDir, "1" + OzoneConsts.DN_CONTAINER_DB);
- }
+ // corrupt db by rename dir->file
+ File dbDir = new File(((KeyValueContainerData) (c1.getContainerData()))
+ .getDbFile().getAbsolutePath());
- MutableVolumeSet volSet = oc.getVolumeSet();
- HddsVolume vol0 = (HddsVolume) volSet.getVolumesList().get(0);
-
- try {
- DatanodeTestUtils.injectDataDirFailure(dbDir);
- // simulate bad volume by removing write permission on root dir
- // refer to HddsVolume.check()
- DatanodeTestUtils.simulateBadVolume(vol0);
-
- // one volume health check got automatically executed when the
cluster started
- // the second health should log the rocksdb failure but return a
healthy-volume status
- assertEquals(VolumeCheckResult.HEALTHY, vol0.checkDbHealth(dbDir));
- // the third health check should log the rocksdb failure and return
a failed-volume status
- assertEquals(VolumeCheckResult.FAILED, vol0.checkDbHealth(dbDir));
- } finally {
- // restore all
- DatanodeTestUtils.restoreBadVolume(vol0);
- DatanodeTestUtils.restoreDataDirFromFailure(dbDir);
- }
+ MutableVolumeSet volSet = oc.getVolumeSet();
+ HddsVolume vol0 = (HddsVolume) volSet.getVolumesList().get(0);
+
+ try {
+ DatanodeTestUtils.injectDataDirFailure(dbDir);
+ // simulate bad volume by removing write permission on root dir
+ // refer to HddsVolume.check()
+ DatanodeTestUtils.simulateBadVolume(vol0);
+
+ // one volume health check got automatically executed when the cluster
started
+ // the second health should log the rocksdb failure but return a
healthy-volume status
+ assertEquals(VolumeCheckResult.HEALTHY, vol0.checkDbHealth(dbDir));
+ // the third health check should log the rocksdb failure and return a
failed-volume status
+ assertEquals(VolumeCheckResult.FAILED, vol0.checkDbHealth(dbDir));
+ } finally {
+ // restore all
+ DatanodeTestUtils.restoreBadVolume(vol0);
+ DatanodeTestUtils.restoreDataDirFromFailure(dbDir);
}
}
}
@@ -309,6 +341,24 @@ private static void
readKeyToTriggerCheckVolumeAsync(OzoneBucket bucket,
}
}
+ private HddsDatanodeService getDatanode(long containerId) throws IOException
{
+ try (ScmClient scmClient = new
ContainerOperationClient(cluster.getConf())) {
+ return
cluster.getHddsDatanode(scmClient.getContainerWithPipeline(containerId)
+ .getPipeline().getFirstNode());
+ }
+ }
+
+ private boolean isFailedVolumeReported(HddsDatanodeService datanode) {
+ DatanodeInfo datanodeInfo =
cluster.getStorageContainerManager().getScmNodeManager()
+ .getNode(datanode.getDatanodeDetails().getID());
+ return datanodeInfo != null && datanodeInfo.getFailedVolumeCount() == 1;
+ }
+
+ private static void waitForHandleFailedVolume(MutableVolumeSet volumeSet)
throws Exception {
+ DatanodeTestUtils.waitForHandleFailedVolume(volumeSet, 1);
+ GenericTestUtils.waitFor(() -> volumeSet.getVolumesList().isEmpty(), 100,
10000);
+ }
+
private static MiniOzoneCluster newCluster(boolean schemaV3)
throws Exception {
OzoneConfiguration ozoneConfig = new OzoneConfiguration();
@@ -319,6 +369,11 @@ private static MiniOzoneCluster newCluster(boolean
schemaV3)
// keep the cache size = 1, so we could trigger io exception on
// reading on-disk db instance
ozoneConfig.setInt(OZONE_CONTAINER_CACHE_SIZE, 1);
+ ozoneConfig.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100,
TimeUnit.MILLISECONDS);
+ ozoneConfig.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 100,
TimeUnit.MILLISECONDS);
+ ozoneConfig.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
TimeUnit.MILLISECONDS);
+ ozoneConfig.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3,
TimeUnit.SECONDS);
+ ozoneConfig.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6,
TimeUnit.SECONDS);
if (!schemaV3) {
ContainerTestUtils.disableSchemaV3(ozoneConfig);
}
@@ -330,7 +385,7 @@ private static MiniOzoneCluster newCluster(boolean schemaV3)
dnConf.setDiskCheckMinGap(Duration.ofSeconds(0));
ozoneConfig.setFromObject(dnConf);
MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(ozoneConfig)
- .setNumDatanodes(1)
+ .setNumDatanodes(4)
.build();
cluster.waitForClusterToBeReady();
cluster.waitForPipelineTobeReady(ReplicationFactor.ONE, 30000);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
