This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4b8871f1d2 HDDS-10309. Speed up TestSnapshotDeletingService (#6541)
4b8871f1d2 is described below
commit 4b8871f1d277ca5556dda737346ab5cb56bf14d7
Author: Raju Balpande <146973984+raju-balpa...@users.noreply.github.com>
AuthorDate: Wed Apr 17 23:57:27 2024 +0530
HDDS-10309. Speed up TestSnapshotDeletingService (#6541)
---
.../om/snapshot/TestSnapshotDeletingService.java | 89 ++++++++++++++--------
1 file changed, 56 insertions(+), 33 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java
index 1c98ce89af..472abf4209 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingService.java
@@ -44,9 +44,14 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer.OrderAnnotation;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +75,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Test Snapshot Deleting Service.
*/
+
+@Timeout(300)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@TestMethodOrder(OrderAnnotation.class)
public class TestSnapshotDeletingService {
private static final Logger LOG =
@@ -85,8 +94,11 @@ public class TestSnapshotDeletingService {
private static final String VOLUME_NAME = "vol1";
private static final String BUCKET_NAME_ONE = "bucket1";
private static final String BUCKET_NAME_TWO = "bucket2";
+ private static final String BUCKET_NAME_FSO = "bucketfso";
+
+ private boolean runIndividualTest = true;
- @BeforeEach
+ @BeforeAll
public void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
@@ -115,7 +127,7 @@ public class TestSnapshotDeletingService {
client, VOLUME_NAME, BUCKET_NAME_ONE, BucketLayout.DEFAULT);
}
- @AfterEach
+ @AfterAll
public void teardown() {
IOUtils.closeQuietly(client);
if (cluster != null) {
@@ -124,18 +136,21 @@ public class TestSnapshotDeletingService {
}
@Test
+ @Order(2)
public void testSnapshotSplitAndMove() throws Exception {
- SnapshotDeletingService snapshotDeletingService =
- om.getKeyManager().getSnapshotDeletingService();
- Table<String, SnapshotInfo> snapshotInfoTable =
- om.getMetadataManager().getSnapshotInfoTable();
- createSnapshotDataForBucket1();
+ if (runIndividualTest) {
+ SnapshotDeletingService snapshotDeletingService =
+ om.getKeyManager().getSnapshotDeletingService();
+ Table<String, SnapshotInfo> snapshotInfoTable =
+ om.getMetadataManager().getSnapshotInfoTable();
- assertTableRowCount(snapshotInfoTable, 2);
- GenericTestUtils.waitFor(() -> snapshotDeletingService
- .getSuccessfulRunCount() >= 1, 1000, 10000);
+ createSnapshotDataForBucket1();
+ assertTableRowCount(snapshotInfoTable, 2);
+ GenericTestUtils.waitFor(() -> snapshotDeletingService
+ .getSuccessfulRunCount() >= 1, 1000, 10000);
+ }
OmSnapshot bucket1snap3 = om.getOmSnapshotManager()
.getSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap3").get();
@@ -148,12 +163,14 @@ public class TestSnapshotDeletingService {
}
@Test
+ @Order(1)
public void testMultipleSnapshotKeyReclaim() throws Exception {
Table<String, RepeatedOmKeyInfo> deletedTable =
om.getMetadataManager().getDeletedTable();
Table<String, SnapshotInfo> snapshotInfoTable =
om.getMetadataManager().getSnapshotInfoTable();
+ runIndividualTest = false;
createSnapshotDataForBucket1();
@@ -193,10 +210,16 @@ public class TestSnapshotDeletingService {
// verify the cache of purged snapshot
// /vol1/bucket2/bucket2snap1 has been cleaned up from cache map
assertEquals(2, om.getOmSnapshotManager().getSnapshotCacheSize());
+
+ // cleaning up the data
+ client.getProxy().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_ONE,
"bucket1snap1");
+ client.getProxy().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_ONE,
"bucket1snap3");
+ client.getProxy().deleteBucket(VOLUME_NAME, BUCKET_NAME_TWO);
}
@SuppressWarnings("checkstyle:MethodLength")
@Test
+ @Order(3)
public void testSnapshotWithFSO() throws Exception {
Table<String, OmDirectoryInfo> dirTable =
om.getMetadataManager().getDirectoryTable();
@@ -216,7 +239,7 @@ public class TestSnapshotDeletingService {
.build();
OzoneBucket bucket2 = TestDataUtil.createBucket(
- client, VOLUME_NAME, bucketArgs, BUCKET_NAME_TWO);
+ client, VOLUME_NAME, bucketArgs, BUCKET_NAME_FSO);
// Create 10 keys
for (int i = 1; i <= 10; i++) {
@@ -234,12 +257,12 @@ public class TestSnapshotDeletingService {
for (int i = 1; i <= 3; i++) {
String parent = "parent" + i;
client.getProxy().createDirectory(VOLUME_NAME,
- BUCKET_NAME_TWO, parent);
+ BUCKET_NAME_FSO, parent);
for (int j = 1; j <= 3; j++) {
String childFile = "/childFile" + j;
String childDir = "/childDir" + j;
client.getProxy().createDirectory(VOLUME_NAME,
- BUCKET_NAME_TWO, parent + childDir);
+ BUCKET_NAME_FSO, parent + childDir);
TestDataUtil.createKey(bucket2, parent + childFile,
ReplicationFactor.THREE, ReplicationType.RATIS, CONTENT);
}
@@ -251,7 +274,7 @@ public class TestSnapshotDeletingService {
assertTableRowCount(deletedDirTable, 0);
// Create Snapshot1
- client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_FSO,
"snap1");
assertTableRowCount(snapshotInfoTable, 1);
@@ -264,37 +287,37 @@ public class TestSnapshotDeletingService {
// Delete 5 Keys
for (int i = 1; i <= 5; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"key" + i, false);
}
// Rename Keys 3 keys
for (int i = 6; i <= 8; i++) {
- client.getProxy().renameKey(VOLUME_NAME, BUCKET_NAME_TWO, "key" + i,
+ client.getProxy().renameKey(VOLUME_NAME, BUCKET_NAME_FSO, "key" + i,
"renamedKey" + i);
}
// Rename 1 Dir
for (int i = 1; i <= 1; i++) {
- client.getProxy().renameKey(VOLUME_NAME, BUCKET_NAME_TWO, "/parent" + i,
+ client.getProxy().renameKey(VOLUME_NAME, BUCKET_NAME_FSO, "/parent" + i,
"/renamedParent" + i);
}
// Delete 2 Dirs
for (int i = 2; i <= 3; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO, "/parent" + i,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO, "/parent" + i,
true);
}
assertTableRowCount(renamedTable, 4);
// Delete Renamed Keys
for (int i = 6; i <= 8; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"renamedKey" + i, false);
}
// Delete Renamed Dir
for (int i = 1; i <= 1; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"/renamedParent" + i, true);
}
@@ -304,7 +327,7 @@ public class TestSnapshotDeletingService {
assertTableRowCount(renamedTable, 4);
// Create Snapshot2
- client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_FSO,
"snap2");
assertTableRowCount(snapshotInfoTable, 2);
@@ -316,7 +339,7 @@ public class TestSnapshotDeletingService {
// Delete 3 overwritten keys
for (int i = 11; i <= 13; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"key" + i, false);
}
@@ -328,14 +351,14 @@ public class TestSnapshotDeletingService {
// Delete 2 more keys
for (int i = 9; i <= 10; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"key" + i, false);
}
assertTableRowCount(deletedTable, 7);
// Create Snapshot3
- client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getObjectStore().createSnapshot(VOLUME_NAME, BUCKET_NAME_FSO,
"snap3");
assertTableRowCount(snapshotInfoTable, 3);
@@ -344,24 +367,24 @@ public class TestSnapshotDeletingService {
assertTableRowCount(deletedTable, 0);
assertTableRowCount(keyTable, 11);
SnapshotInfo deletedSnap = om.getMetadataManager()
- .getSnapshotInfoTable().get("/vol1/bucket2/snap2");
+ .getSnapshotInfoTable().get("/vol1/bucketfso/snap2");
- client.getObjectStore().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getObjectStore().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_FSO,
"snap2");
assertTableRowCount(snapshotInfoTable, 2);
// Delete 2 overwritten keys
for (int i = 14; i <= 15; i++) {
- client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getProxy().deleteKey(VOLUME_NAME, BUCKET_NAME_FSO,
"key" + i, false);
}
assertTableRowCount(deletedTable, 2);
// Once all the tables are moved, the snapshot is deleted
assertTableRowCount(om.getMetadataManager().getSnapshotInfoTable(), 2);
- verifySnapshotChain(deletedSnap, "/vol1/bucket2/snap3");
+ verifySnapshotChain(deletedSnap, "/vol1/bucketfso/snap3");
OmSnapshot snap3 = om.getOmSnapshotManager()
- .getSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, "snap3").get();
+ .getSnapshot(VOLUME_NAME, BUCKET_NAME_FSO, "snap3").get();
Table<String, OmKeyInfo> snapDeletedDirTable =
snap3.getMetadataManager().getDeletedDirTable();
@@ -380,7 +403,7 @@ public class TestSnapshotDeletingService {
assertTableRowCount(deletedDirTable, 0);
assertTableRowCount(deletedTable, 2);
// Delete Snapshot3 and check entries moved to active DB
- client.getObjectStore().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_TWO,
+ client.getObjectStore().deleteSnapshot(VOLUME_NAME, BUCKET_NAME_FSO,
"snap3");
// Check entries moved to active DB
@@ -390,7 +413,7 @@ public class TestSnapshotDeletingService {
ReferenceCounted<OmSnapshot> rcSnap1 =
om.getOmSnapshotManager().getSnapshot(
- VOLUME_NAME, BUCKET_NAME_TWO, "snap1");
+ VOLUME_NAME, BUCKET_NAME_FSO, "snap1");
OmSnapshot snap1 = rcSnap1.get();
Table<String, OmKeyInfo> snap1KeyTable =
snap1.getMetadataManager().getFileTable();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@ozone.apache.org
For additional commands, e-mail: commits-h...@ozone.apache.org
