This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-6517-Snapshot by this
push:
new aea5edfe99 HDDS-7845. [Snapshot] Wait for RocksDB checkpoint directory
creation (#4214)
aea5edfe99 is described below
commit aea5edfe99ea92687c604378bec4bfbf1773ba80
Author: Hemant Kumar <[email protected]>
AuthorDate: Tue Jan 31 10:56:00 2023 -0800
HDDS-7845. [Snapshot] Wait for RocksDB checkpoint directory creation (#4214)
---
hadoop-hdds/framework/pom.xml | 5 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 47 +++++++++++++++---
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 3 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 2 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 2 +
.../om/snapshot/TestOzoneSnapshotRestore.java | 57 +++++++---------------
pom.xml | 6 +++
7 files changed, 73 insertions(+), 49 deletions(-)
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index b2a253ad26..4f82a355fd 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -166,7 +166,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>rocksdb-checkpoint-differ</artifactId>
<version>${hdds.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ </dependency>
</dependencies>
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
index f227ab81fd..7bb4694093 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBCheckpointManager.java
@@ -20,28 +20,33 @@
package org.apache.hadoop.hdds.utils.db;
import java.io.Closeable;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.utils.db.RocksDatabase.RocksCheckpoint;
-
+import org.awaitility.core.ConditionTimeoutException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.awaitility.Awaitility.with;
+
/**
* RocksDB Checkpoint Manager, used to create and cleanup checkpoints.
*/
public class RDBCheckpointManager implements Closeable {
-
+ private final RocksDatabase db;
private final RocksCheckpoint checkpoint;
public static final String RDB_CHECKPOINT_DIR_PREFIX = "checkpoint_";
private static final Logger LOG =
LoggerFactory.getLogger(RDBCheckpointManager.class);
private final String checkpointNamePrefix;
+ private static final Duration POLL_DELAY_DURATION = Duration.ZERO;
+ private static final Duration POLL_INTERVAL_DURATION =
Duration.ofMillis(100);
+ private static final Duration POLL_MAX_DURATION = Duration.ofSeconds(5);
/**
* Create a checkpoint manager with a prefix to be added to the
@@ -50,6 +55,7 @@ public class RDBCheckpointManager implements Closeable {
* @param checkpointPrefix prefix string.
*/
public RDBCheckpointManager(RocksDatabase db, String checkpointPrefix) {
+ this.db = db;
this.checkpointNamePrefix = checkpointPrefix;
this.checkpoint = db.createCheckpoint();
}
@@ -76,15 +82,22 @@ public class RDBCheckpointManager implements Closeable {
Path checkpointPath = Paths.get(parentDir, checkpointDir);
Instant start = Instant.now();
+
+ // Flush the DB WAL and mem table.
+ db.flushWal(true);
+ db.flush();
+
checkpoint.createCheckpoint(checkpointPath);
- //Best guesstimate here. Not accurate.
+ // Best guesstimate here. Not accurate.
final long latest = checkpoint.getLatestSequenceNumber();
- Instant end = Instant.now();
+ Instant end = Instant.now();
long duration = Duration.between(start, end).toMillis();
- LOG.info("Created checkpoint at {} in {} milliseconds",
+ LOG.info("Created checkpoint in rocksDB at {} in {} milliseconds",
checkpointPath, duration);
+ waitForCheckpointDirectoryExist(checkpointPath.toFile());
+
return new RocksDBCheckpoint(
checkpointPath,
currentTime,
@@ -96,6 +109,28 @@ public class RDBCheckpointManager implements Closeable {
return null;
}
+ /**
+ * Wait for checkpoint directory to be created for 5 secs with 100 millis
+ * poll interval.
+ */
+ private void waitForCheckpointDirectoryExist(File file) throws IOException {
+ Instant start = Instant.now();
+ try {
+ with().atMost(POLL_MAX_DURATION)
+ .pollDelay(POLL_DELAY_DURATION)
+ .pollInterval(POLL_INTERVAL_DURATION)
+ .await()
+ .until(file::exists);
+ LOG.info("Waited for {} milliseconds for checkpoint directory {}" +
+ " availability.",
+ Duration.between(start, Instant.now()).toMillis(),
+ file.getAbsoluteFile());
+ } catch (ConditionTimeoutException exception) {
+ LOG.info("Checkpoint directory: {} didn't get created in 5 secs.",
+ file.getAbsolutePath());
+ }
+ }
+
/**
* Create RocksDB snapshot by saving a checkpoint to a directory.
*
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 62b06c7c0d..4bb6050697 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -761,7 +761,8 @@ public final class OmKeyInfo extends WithParentObjectId {
.setObjectID(objectID)
.setUpdateID(updateID)
.setParentObjectID(parentObjectID)
- .setFileName(fileName);
+ .setFileName(fileName)
+ .setFile(isFile);
keyLocationVersions.forEach(keyLocationVersion ->
builder.addOmKeyLocationInfoGroup(
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index d5ef479a4d..83ae64aaea 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -445,7 +445,7 @@ Apache License 2.0
org.xerial:sqlite-jdbc
org.yaml:snakeyaml
software.amazon.ion:ion-java
-
+ org.awaitility:awaitility
MIT
=====================
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt
b/hadoop-ozone/dist/src/main/license/jar-report.txt
index c7d55282c0..13de04cf87 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -8,6 +8,7 @@ share/ozone/lib/aopalliance-repackaged.jar
share/ozone/lib/asm.jar
share/ozone/lib/aspectjrt.jar
share/ozone/lib/aspectjweaver.jar
+share/ozone/lib/awaitility.jar
share/ozone/lib/aws-java-sdk-core.jar
share/ozone/lib/aws-java-sdk-kms.jar
share/ozone/lib/aws-java-sdk-s3.jar
@@ -60,6 +61,7 @@ share/ozone/lib/hadoop-hdfs-client.jar
share/ozone/lib/hadoop-hdfs.jar
share/ozone/lib/hadoop-shaded-guava.jar
share/ozone/lib/hadoop-shaded-protobuf_3_7.jar
+share/ozone/lib/hamcrest.jar
share/ozone/lib/hdds-annotation-processing.jar
share/ozone/lib/hdds-client.jar
share/ozone/lib/hdds-common.jar
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
index 799f56f41d..d61efd7d8b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
-import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -39,14 +38,12 @@ import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
-import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.util.ToolRunner;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.rules.Timeout;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -55,7 +52,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
@@ -65,26 +61,18 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
-import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.params.provider.Arguments.arguments;
/**
* Tests Snapshot Restore function.
*/
-
+@Timeout(value = 300)
public class TestOzoneSnapshotRestore {
- private static final String OM_SERVICE_ID = "om-service-test1";
- private static BucketLayout bucketLayout = BucketLayout.LEGACY;
- private static MiniOzoneCluster cluster = null;
- private static String volumeName;
- private static String bucketName;
- private static OzoneManagerProtocol writeClient;
- private static ObjectStore store;
- private static File metaDir;
- private static OzoneManager leaderOzoneManager;
- private static OzoneBucket ozoneBucket;
- @Rule
- public Timeout timeout = new Timeout(500, TimeUnit.SECONDS);
+ private static final String OM_SERVICE_ID = "om-service-test-1";
+ private MiniOzoneCluster cluster;
+ private ObjectStore store;
+ private File metaDir;
+ private OzoneManager leaderOzoneManager;
private OzoneConfiguration clientConf;
private static Stream<Arguments> bucketTypes() {
@@ -102,7 +90,7 @@ public class TestOzoneSnapshotRestore {
}
@BeforeEach
- private void init() throws Exception {
+ public void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
@@ -116,12 +104,6 @@ public class TestOzoneSnapshotRestore {
.build();
cluster.waitForClusterToBeReady();
- // create a volume and a bucket to be used by OzoneFileSystem
- ozoneBucket = TestDataUtil
- .createVolumeAndBucket(cluster, bucketLayout);
- volumeName = ozoneBucket.getVolumeName();
- bucketName = ozoneBucket.getName();
-
leaderOzoneManager = ((MiniOzoneHAClusterImpl) cluster).getOMLeader();
OzoneConfiguration leaderConfig = leaderOzoneManager.getConfiguration();
cluster.setConf(leaderConfig);
@@ -132,7 +114,6 @@ public class TestOzoneSnapshotRestore {
OzoneClient client = cluster.getClient();
store = client.getObjectStore();
- writeClient = store.getClientProxy().getOzoneManagerClient();
KeyManagerImpl keyManager = (KeyManagerImpl) HddsWhiteboxTestUtils
.getInternalState(leaderOzoneManager, "keyManager");
@@ -204,7 +185,7 @@ public class TestOzoneSnapshotRestore {
// Copy key from source to destination path
int res = ToolRunner.run(shell,
new String[]{"-cp", sourcePath, destPath});
- assertEquals(0, res);
+ Assertions.assertEquals(0, res);
} finally {
shell.close();
}
@@ -232,11 +213,11 @@ public class TestOzoneSnapshotRestore {
String snapshotKeyPrefix = createSnapshot(volume, bucket);
int volBucketKeyCount = keyCount(buck, snapshotKeyPrefix + keyPrefix);
- Assert.assertEquals(5, volBucketKeyCount);
+ Assertions.assertEquals(5, volBucketKeyCount);
deleteKeys(buck);
int delKeyCount = keyCount(buck, keyPrefix);
- Assert.assertEquals(0, delKeyCount);
+ Assertions.assertEquals(0, delKeyCount);
String sourcePath = OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket
+ OM_KEY_PREFIX + snapshotKeyPrefix;
@@ -247,8 +228,7 @@ public class TestOzoneSnapshotRestore {
}
int finalKeyCount = keyCount(buck, keyPrefix);
- Assert.assertEquals(5, finalKeyCount);
-
+ Assertions.assertEquals(5, finalKeyCount);
}
@ParameterizedTest
@@ -276,7 +256,7 @@ public class TestOzoneSnapshotRestore {
String snapshotKeyPrefix = createSnapshot(volume, bucket);
int volBucketKeyCount = keyCount(buck, snapshotKeyPrefix + keyPrefix);
- Assert.assertEquals(5, volBucketKeyCount);
+ Assertions.assertEquals(5, volBucketKeyCount);
String sourcePath = OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket
+ OM_KEY_PREFIX + snapshotKeyPrefix;
@@ -287,8 +267,7 @@ public class TestOzoneSnapshotRestore {
}
int finalKeyCount = keyCount(buck2, keyPrefix);
- Assert.assertEquals(5, finalKeyCount);
-
+ Assertions.assertEquals(5, finalKeyCount);
}
@ParameterizedTest
@@ -320,7 +299,7 @@ public class TestOzoneSnapshotRestore {
String snapshotKeyPrefix = createSnapshot(volume, bucket);
int volBucketKeyCount = keyCount(buck, snapshotKeyPrefix + keyPrefix);
- Assert.assertEquals(5, volBucketKeyCount);
+ Assertions.assertEquals(5, volBucketKeyCount);
String sourcePath = OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket
+ OM_KEY_PREFIX + snapshotKeyPrefix;
@@ -331,8 +310,6 @@ public class TestOzoneSnapshotRestore {
}
int finalKeyCount = keyCount(buck2, keyPrefix);
- Assert.assertEquals(5, finalKeyCount);
-
+ Assertions.assertEquals(5, finalKeyCount);
}
-
}
diff --git a/pom.xml b/pom.xml
index e02e40f4d5..9a2bc44173 100644
--- a/pom.xml
+++ b/pom.xml
@@ -284,6 +284,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<aspectj.version>1.9.7</aspectj.version>
<aspectj-plugin.version>1.14.0</aspectj-plugin.version>
<restrict-imports.enforcer.version>2.0.0</restrict-imports.enforcer.version>
+ <awaitility.version>4.2.0</awaitility.version>
</properties>
@@ -1443,6 +1444,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<artifactId>sqlite-jdbc</artifactId>
<version>${sqlite.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
