This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 23f3e5b2e1 HDDS-11152. OMDoubleBuffer error when handling snapshot's
background operations (#7112)
23f3e5b2e1 is described below
commit 23f3e5b2e1196f4bd1d9d39ab09ed6fb57323bf1
Author: Hemant Kumar <[email protected]>
AuthorDate: Wed Aug 28 16:24:42 2024 -0700
HDDS-11152. OMDoubleBuffer error when handling snapshot's background
operations (#7112)
---
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 12 +--
.../om/snapshot/TestOzoneManagerHASnapshot.java | 104 +++++++++++++++++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 40 ++++++--
.../hadoop/ozone/om/SnapshotChainManager.java | 17 ++--
.../key/OMDirectoriesPurgeRequestWithFSO.java | 5 +-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 7 +-
.../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 8 +-
.../request/snapshot/OMSnapshotPurgeRequest.java | 6 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 5 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 11 +--
.../OMSnapshotMoveDeletedKeysResponse.java | 10 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 7 ++
.../om/service/AbstractKeyDeletingService.java | 7 +-
.../ozone/om/service/KeyDeletingService.java | 6 +-
.../ozone/om/service/SnapshotDeletingService.java | 3 +-
.../service/SnapshotDirectoryCleaningService.java | 6 +-
.../hadoop/ozone/om/snapshot/SnapshotUtils.java | 6 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 16 +---
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 1 -
19 files changed, 183 insertions(+), 94 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
index c123675565..9a6bca29b8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
@@ -204,11 +204,7 @@ public abstract class TestOmSnapshot {
conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths);
conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name());
conf.setBoolean(OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff);
- conf.setBoolean(OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS,
- disableNativeDiff);
- conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths);
- conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name());
- conf.setBoolean(OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff);
+ conf.setBoolean(OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS,
disableNativeDiff);
conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
// Enable filesystem snapshot feature for the test regardless of the
default
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
@@ -1481,10 +1477,8 @@ public abstract class TestOmSnapshot {
String toSnapshotTableKey =
SnapshotInfo.getTableKey(volumeName, bucketName, toSnapName);
- UUID fromSnapshotID = ozoneManager.getOmSnapshotManager()
- .getSnapshotInfo(fromSnapshotTableKey).getSnapshotId();
- UUID toSnapshotID = ozoneManager.getOmSnapshotManager()
- .getSnapshotInfo(toSnapshotTableKey).getSnapshotId();
+ UUID fromSnapshotID = SnapshotUtils.getSnapshotInfo(ozoneManager,
fromSnapshotTableKey).getSnapshotId();
+ UUID toSnapshotID = SnapshotUtils.getSnapshotInfo(ozoneManager,
toSnapshotTableKey).getSnapshotId();
// Construct SnapshotDiffJob table key.
String snapDiffJobKey = fromSnapshotID + DELIMITER + toSnapshotID;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java
index 341b5b78c6..f178d00daa 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java
@@ -21,10 +21,12 @@ package org.apache.hadoop.ozone.om.snapshot;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -34,20 +36,27 @@ import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS;
import static org.apache.ozone.test.LambdaTestUtils.await;
@@ -72,6 +81,8 @@ public class TestOzoneManagerHASnapshot {
public static void staticInit() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
+
conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1,
TimeUnit.SECONDS);
+
conf.setTimeDuration(OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL,
1, TimeUnit.SECONDS);
cluster = MiniOzoneCluster.newHABuilder(conf)
.setOMServiceId("om-service-test")
@@ -265,4 +276,97 @@ public class TestOzoneManagerHASnapshot {
fileKey.write(value);
}
}
+
+ /**
+ * This is to simulate HDDS-11152 scenario. In which a follower's
doubleBuffer is lagging and accumulates purgeKey
+ * and purgeSnapshot in same batch.
+ */
+ @Test
+ public void testKeyAndSnapshotDeletionService() throws IOException,
InterruptedException, TimeoutException {
+ OzoneManager omLeader = cluster.getOMLeader();
+ OzoneManager omFollower;
+
+ if (omLeader != cluster.getOzoneManager(0)) {
+ omFollower = cluster.getOzoneManager(0);
+ } else {
+ omFollower = cluster.getOzoneManager(1);
+ }
+
+ int numKeys = 5;
+ List<String> keys = new ArrayList<>();
+ for (int i = 0; i < numKeys; i++) {
+ String keyName = "key-" + RandomStringUtils.randomNumeric(10);
+ createFileKey(ozoneBucket, keyName);
+ keys.add(keyName);
+ }
+
+ // Stop the key deletion service so that deleted keys get trapped in the
snapshots.
+ omLeader.getKeyManager().getDeletingService().suspend();
+ // Stop the snapshot deletion service so that deleted keys get trapped in
the snapshots.
+ omLeader.getKeyManager().getSnapshotDeletingService().suspend();
+
+ // Delete half of the keys
+ for (int i = 0; i < numKeys / 2; i++) {
+ ozoneBucket.deleteKey(keys.get(i));
+ }
+
+ String snapshotName = "snap-" + RandomStringUtils.randomNumeric(10);
+ createSnapshot(volumeName, bucketName, snapshotName);
+
+ store.deleteSnapshot(volumeName, bucketName, snapshotName);
+
+ // Pause double buffer on follower node to accumulate all the key purge,
snapshot delete and purge transactions.
+
omFollower.getOmRatisServer().getOmStateMachine().getOzoneManagerDoubleBuffer().stopDaemon();
+
+ long keyDeleteServiceCount =
omLeader.getKeyManager().getDeletingService().getRunCount().get();
+ omLeader.getKeyManager().getDeletingService().resume();
+
+ GenericTestUtils.waitFor(
+ () ->
omLeader.getKeyManager().getDeletingService().getRunCount().get() >
keyDeleteServiceCount,
+ 1000, 60000);
+
+ long snapshotDeleteServiceCount =
omLeader.getKeyManager().getSnapshotDeletingService().getRunCount().get();
+ omLeader.getKeyManager().getSnapshotDeletingService().resume();
+
+ GenericTestUtils.waitFor(
+ () ->
omLeader.getKeyManager().getSnapshotDeletingService().getRunCount().get() >
snapshotDeleteServiceCount,
+ 1000, 60000);
+
+ String tableKey = SnapshotInfo.getTableKey(volumeName, bucketName,
snapshotName);
+ checkSnapshotIsPurgedFromDB(omLeader, tableKey);
+
+ // Resume the DoubleBuffer and flush the pending transactions.
+ OzoneManagerDoubleBuffer omDoubleBuffer =
+
omFollower.getOmRatisServer().getOmStateMachine().getOzoneManagerDoubleBuffer();
+ omDoubleBuffer.resume();
+ CompletableFuture.supplyAsync(() -> {
+ omDoubleBuffer.flushTransactions();
+ return null;
+ });
+ omDoubleBuffer.awaitFlush();
+ checkSnapshotIsPurgedFromDB(omFollower, tableKey);
+ }
+
+ private void createSnapshot(String volName, String buckName, String
snapName) throws IOException {
+ store.createSnapshot(volName, buckName, snapName);
+
+ String tableKey = SnapshotInfo.getTableKey(volName, buckName, snapName);
+ SnapshotInfo snapshotInfo =
SnapshotUtils.getSnapshotInfo(cluster.getOMLeader(), tableKey);
+ String fileName =
getSnapshotPath(cluster.getOMLeader().getConfiguration(), snapshotInfo);
+ File snapshotDir = new File(fileName);
+ if (!RDBCheckpointUtils.waitForCheckpointDirectoryExist(snapshotDir)) {
+ throw new IOException("Snapshot directory doesn't exist");
+ }
+ }
+
+ private void checkSnapshotIsPurgedFromDB(OzoneManager ozoneManager, String
snapshotTableKey)
+ throws InterruptedException, TimeoutException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return
ozoneManager.getMetadataManager().getSnapshotInfoTable().get(snapshotTableKey)
== null;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }, 1000, 60000);
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 0d17851ed1..18b2911818 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -98,6 +98,7 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_REPORT_MAX_PAGE_SIZE_DEFAULT;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_SNAPSHOT_ERROR;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager.getSnapshotRootPath;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotActive;
import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
@@ -674,19 +675,38 @@ public final class OmSnapshotManager implements
AutoCloseable {
}
/**
- * Returns true if the snapshot is in given status.
- * @param key DB snapshot table key
- * @param status SnapshotStatus
- * @return true if the snapshot is in given status, false otherwise
+ * Returns OmSnapshot object and skips active check.
+ * This should only be used for API calls initiated by background service
e.g. purgeKeys, purgeSnapshot,
+ * snapshotMoveDeletedKeys, and SetSnapshotProperty.
*/
- public boolean isSnapshotStatus(String key,
- SnapshotInfo.SnapshotStatus status)
- throws IOException {
- return getSnapshotInfo(key).getSnapshotStatus().equals(status);
+ public ReferenceCounted<OmSnapshot> getSnapshot(UUID snapshotId) throws
IOException {
+ return snapshotCache.get(snapshotId);
}
- public SnapshotInfo getSnapshotInfo(String key) throws IOException {
- return SnapshotUtils.getSnapshotInfo(ozoneManager, key);
+ /**
+ * Returns snapshotInfo from cache if it is present in cache, otherwise it
checks RocksDB and return value from there.
+ * #################################################
+ * NOTE: THIS SHOULD BE USED BY SNAPSHOT CACHE ONLY.
+ * #################################################
+ * Sometimes, the follower OM node may be lagging that it gets purgeKeys or
snapshotMoveDeletedKeys from a Snapshot,
+ * and purgeSnapshot for the same Snapshot one after another. And
purgeSnapshot's validateAndUpdateCache gets
+ * executed before doubleBuffer flushes purgeKeys or snapshotMoveDeletedKeys
from that Snapshot.
+ * This should not be a case on the leader node because
SnapshotDeletingService checks that deletedTable and
+ * deletedDirectoryTable in DB don't have entries for the bucket before it
sends a purgeSnapshot on a snapshot.
+ * If that happens, and we just look into the cache, the addToBatch
operation will fail when it tries to open
+ * the DB and purgeKeys from the Snapshot because snapshot is already purged
from the SnapshotInfoTable cache.
+ * Hence, it is needed to look into the table to make sure that snapshot
exists somewhere either in cache or in DB.
+ */
+ private SnapshotInfo getSnapshotInfo(String snapshotKey) throws IOException {
+ SnapshotInfo snapshotInfo =
ozoneManager.getMetadataManager().getSnapshotInfoTable().get(snapshotKey);
+
+ if (snapshotInfo == null) {
+ snapshotInfo =
ozoneManager.getMetadataManager().getSnapshotInfoTable().getSkipCache(snapshotKey);
+ }
+ if (snapshotInfo == null) {
+ throw new OMException("Snapshot '" + snapshotKey + "' is not found.",
INVALID_SNAPSHOT_ERROR);
+ }
+ return snapshotInfo;
}
public static String getSnapshotPrefix(String snapshotName) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
index 60353590e7..b069a174cd 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
@@ -362,13 +362,16 @@ public class SnapshotChainManager {
public synchronized boolean deleteSnapshot(SnapshotInfo snapshotInfo)
throws IOException {
validateSnapshotChain();
- boolean status = deleteSnapshotGlobal(snapshotInfo.getSnapshotId()) &&
- deleteSnapshotPath(snapshotInfo.getSnapshotPath(),
- snapshotInfo.getSnapshotId());
- if (status) {
- snapshotIdToTableKey.remove(snapshotInfo.getSnapshotId());
- }
- return status;
+ return deleteSnapshotGlobal(snapshotInfo.getSnapshotId()) &&
+ deleteSnapshotPath(snapshotInfo.getSnapshotPath(),
snapshotInfo.getSnapshotId());
+ }
+
+ /**
+ * Remove the snapshot from snapshotIdToSnapshotTableKey map.
+ */
+ public synchronized void removeFromSnapshotIdToTable(UUID snapshotId) throws
IOException {
+ validateSnapshotChain();
+ snapshotIdToTableKey.remove(snapshotId);
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
index 2c182a6a5f..cb10c0d2e4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -74,9 +75,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
OMMetrics omMetrics = ozoneManager.getMetrics();
try {
if (fromSnapshot != null) {
- fromSnapshotInfo = ozoneManager.getMetadataManager()
- .getSnapshotInfoTable()
- .get(fromSnapshot);
+ fromSnapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager,
fromSnapshot);
}
for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
index 9ed9218396..5738d7945b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.om.request.key;
import java.io.IOException;
import java.util.ArrayList;
+import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -74,14 +75,12 @@ public class OMKeyPurgeRequest extends OMKeyRequest {
try {
SnapshotInfo fromSnapshotInfo = null;
if (fromSnapshot != null) {
- fromSnapshotInfo = ozoneManager.getMetadataManager()
- .getSnapshotInfoTable().get(fromSnapshot);
+ fromSnapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager,
fromSnapshot);
}
omClientResponse = new OMKeyPurgeResponse(omResponse.build(),
keysToBePurgedList, fromSnapshotInfo, keysToUpdateList);
} catch (IOException ex) {
- omClientResponse = new OMKeyPurgeResponse(
- createErrorOMResponse(omResponse, ex));
+ omClientResponse = new
OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex));
}
return omClientResponse;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java
index 122108ad65..df4781bb0c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.ozone.om.request.snapshot;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.SnapshotChainManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -60,7 +59,6 @@ public class OMSnapshotMoveDeletedKeysRequest extends
OMClientRequest {
@Override
@DisallowedUntilLayoutVersion(FILESYSTEM_SNAPSHOT)
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager,
TermIndex termIndex) {
- OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager();
OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl)
ozoneManager.getMetadataManager();
SnapshotChainManager snapshotChainManager =
@@ -78,8 +76,10 @@ public class OMSnapshotMoveDeletedKeysRequest extends
OMClientRequest {
OzoneManagerProtocolProtos.OMResponse.Builder omResponse =
OmResponseUtil.getOMResponseBuilder(getOmRequest());
try {
- nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot,
- snapshotChainManager, omSnapshotManager);
+ // Check the snapshot exists.
+ SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot.getTableKey());
+
+ nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot,
snapshotChainManager, ozoneManager);
// Get next non-deleted snapshot.
List<SnapshotMoveKeyInfos> nextDBKeysList =
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
index 9b46aeef4c..47304b416a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
@@ -25,7 +25,6 @@ import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.SnapshotChainManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -72,7 +71,6 @@ public class OMSnapshotPurgeRequest extends OMClientRequest {
final long trxnLogIndex = termIndex.getIndex();
- OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager();
OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl)
ozoneManager.getMetadataManager();
SnapshotChainManager snapshotChainManager =
@@ -106,14 +104,12 @@ public class OMSnapshotPurgeRequest extends
OMClientRequest {
}
SnapshotInfo nextSnapshot =
- SnapshotUtils.getNextActiveSnapshot(fromSnapshot,
snapshotChainManager, omSnapshotManager);
+ SnapshotUtils.getNextActiveSnapshot(fromSnapshot,
snapshotChainManager, ozoneManager);
// Step 1: Update the deep clean flag for the next active snapshot
updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager,
trxnLogIndex);
// Step 2: Update the snapshot chain.
updateSnapshotChainAndCache(omMetadataManager, fromSnapshot,
trxnLogIndex);
- // Remove and close snapshot's RocksDB instance from SnapshotCache.
- omSnapshotManager.invalidateCacheEntry(fromSnapshot.getSnapshotId());
// Step 3: Purge the snapshot from SnapshotInfoTable cache.
omMetadataManager.getSnapshotInfoTable()
.addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()),
CacheValue.get(trxnLogIndex));
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
index edb13f8cf9..138e942e2b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
@@ -86,10 +86,7 @@ public class OMDirectoriesPurgeResponseWithFSO extends
OmKeyResponse {
.getOzoneManager().getOmSnapshotManager();
try (ReferenceCounted<OmSnapshot>
- rcFromSnapshotInfo = omSnapshotManager.getSnapshot(
- fromSnapshotInfo.getVolumeName(),
- fromSnapshotInfo.getBucketName(),
- fromSnapshotInfo.getName())) {
+ rcFromSnapshotInfo =
omSnapshotManager.getSnapshot(fromSnapshotInfo.getSnapshotId())) {
OmSnapshot fromSnapshot = rcFromSnapshotInfo.get();
DBStore fromSnapshotStore = fromSnapshot.getMetadataManager()
.getStore();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java
index b16ba95d78..b59c7d18d4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java
@@ -75,18 +75,13 @@ public class OMKeyPurgeResponse extends OmKeyResponse {
if (fromSnapshot != null) {
OmSnapshotManager omSnapshotManager =
- ((OmMetadataManagerImpl) omMetadataManager)
- .getOzoneManager().getOmSnapshotManager();
+ ((OmMetadataManagerImpl)
omMetadataManager).getOzoneManager().getOmSnapshotManager();
try (ReferenceCounted<OmSnapshot> rcOmFromSnapshot =
- omSnapshotManager.getSnapshot(
- fromSnapshot.getVolumeName(),
- fromSnapshot.getBucketName(),
- fromSnapshot.getName())) {
+ omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {
OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get();
- DBStore fromSnapshotStore =
- fromOmSnapshot.getMetadataManager().getStore();
+ DBStore fromSnapshotStore =
fromOmSnapshot.getMetadataManager().getStore();
// Init Batch Operation for snapshot db.
try (BatchOperation writeBatch =
fromSnapshotStore.initBatchOperation()) {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
index 3726faacfd..3b5a7454f9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java
@@ -91,19 +91,13 @@ public class OMSnapshotMoveDeletedKeysResponse extends
OMClientResponse {
.getOzoneManager().getOmSnapshotManager();
try (ReferenceCounted<OmSnapshot> rcOmFromSnapshot =
- omSnapshotManager.getSnapshot(
- fromSnapshot.getVolumeName(),
- fromSnapshot.getBucketName(),
- fromSnapshot.getName())) {
+ omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {
OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get();
if (nextSnapshot != null) {
try (ReferenceCounted<OmSnapshot>
- rcOmNextSnapshot = omSnapshotManager.getSnapshot(
- nextSnapshot.getVolumeName(),
- nextSnapshot.getBucketName(),
- nextSnapshot.getName())) {
+ rcOmNextSnapshot =
omSnapshotManager.getSnapshot(nextSnapshot.getSnapshotId())) {
OmSnapshot nextOmSnapshot = rcOmNextSnapshot.get();
RDBStore nextSnapshotStore =
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
index 139ce468e5..81a020653f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
@@ -90,8 +90,15 @@ public class OMSnapshotPurgeResponse extends
OMClientResponse {
continue;
}
+ // Remove and close snapshot's RocksDB instance from SnapshotCache.
+ ((OmMetadataManagerImpl)
omMetadataManager).getOzoneManager().getOmSnapshotManager()
+ .invalidateCacheEntry(snapshotInfo.getSnapshotId());
+ // Remove the snapshot from snapshotId to snapshotTableKey map.
+ ((OmMetadataManagerImpl) omMetadataManager).getSnapshotChainManager()
+ .removeFromSnapshotIdToTable(snapshotInfo.getSnapshotId());
// Delete Snapshot checkpoint directory.
deleteCheckpointDirectory(omMetadataManager, snapshotInfo);
+ // Delete snapshotInfo from the table.
omMetadataManager.getSnapshotInfoTable().deleteWithBatch(batchOperation,
dbKey);
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index 429e286287..154bd474b6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMMetadataManager;
-import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.SnapshotChainManager;
import org.apache.hadoop.ozone.om.helpers.OMRatisHelper;
@@ -39,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -576,8 +576,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
return cLimit + increment >= maxLimit;
}
- protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo,
- SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager)
+ protected SnapshotInfo getPreviousActiveSnapshot(SnapshotInfo snapInfo,
SnapshotChainManager chainManager)
throws IOException {
SnapshotInfo currSnapInfo = snapInfo;
while (chainManager.hasPreviousPathSnapshot(
@@ -586,7 +585,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
UUID prevPathSnapshot = chainManager.previousPathSnapshot(
currSnapInfo.getSnapshotPath(), currSnapInfo.getSnapshotId());
String tableKey = chainManager.getTableKey(prevPathSnapshot);
- SnapshotInfo prevSnapInfo = omSnapshotManager.getSnapshotInfo(tableKey);
+ SnapshotInfo prevSnapInfo = SnapshotUtils.getSnapshotInfo(ozoneManager,
tableKey);
if (prevSnapInfo.getSnapshotStatus() ==
SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE) {
return prevSnapInfo;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
index c428548287..aa2eb6720a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
@@ -279,13 +279,11 @@ public class KeyDeletingService extends
AbstractKeyDeletingService {
}
String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX;
- SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(
- currSnapInfo, snapChainManager, omSnapshotManager);
+ SnapshotInfo previousSnapshot =
getPreviousActiveSnapshot(currSnapInfo, snapChainManager);
SnapshotInfo previousToPrevSnapshot = null;
if (previousSnapshot != null) {
- previousToPrevSnapshot = getPreviousActiveSnapshot(
- previousSnapshot, snapChainManager, omSnapshotManager);
+ previousToPrevSnapshot =
getPreviousActiveSnapshot(previousSnapshot, snapChainManager);
}
Table<String, OmKeyInfo> previousKeyTable = null;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
index 99e3903447..a98081c63a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java
@@ -207,8 +207,7 @@ public class SnapshotDeletingService extends
AbstractKeyDeletingService {
}
//TODO: [SNAPSHOT] Add lock to deletedTable and Active DB.
- SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(
- snapInfo, chainManager, omSnapshotManager);
+ SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(snapInfo,
chainManager);
Table<String, OmKeyInfo> previousKeyTable = null;
Table<String, OmDirectoryInfo> previousDirTable = null;
OmSnapshot omPreviousSnapshot = null;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
index fe0f6e111e..9746b4421b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDirectoryCleaningService.java
@@ -176,8 +176,7 @@ public class SnapshotDirectoryCleaningService
"unexpected state.");
}
- SnapshotInfo previousSnapshot = getPreviousActiveSnapshot(
- currSnapInfo, snapChainManager, omSnapshotManager);
+ SnapshotInfo previousSnapshot =
getPreviousActiveSnapshot(currSnapInfo, snapChainManager);
SnapshotInfo previousToPrevSnapshot = null;
Table<String, OmKeyInfo> previousKeyTable = null;
@@ -194,8 +193,7 @@ public class SnapshotDirectoryCleaningService
.getKeyTable(bucketInfo.getBucketLayout());
prevRenamedTable = omPreviousSnapshot
.getMetadataManager().getSnapshotRenamedTable();
- previousToPrevSnapshot = getPreviousActiveSnapshot(
- previousSnapshot, snapChainManager, omSnapshotManager);
+ previousToPrevSnapshot =
getPreviousActiveSnapshot(previousSnapshot, snapChainManager);
}
Table<String, OmKeyInfo> previousToPrevKeyTable = null;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
index 2041fa791a..e0f40dabd8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.om.snapshot;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.SnapshotChainManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -143,7 +142,7 @@ public final class SnapshotUtils {
* Get the next non deleted snapshot in the snapshot chain.
*/
public static SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo,
- SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager)
+ SnapshotChainManager chainManager, OzoneManager ozoneManager)
throws IOException {
// If the snapshot is deleted in the previous run, then the in-memory
@@ -162,8 +161,7 @@ public final class SnapshotUtils {
snapInfo.getSnapshotPath(), snapInfo.getSnapshotId());
String tableKey = chainManager.getTableKey(nextPathSnapshot);
- SnapshotInfo nextSnapshotInfo =
- omSnapshotManager.getSnapshotInfo(tableKey);
+ SnapshotInfo nextSnapshotInfo = getSnapshotInfo(ozoneManager,
tableKey);
if (nextSnapshotInfo.getSnapshotStatus().equals(
SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java
index a912f549b3..2cd0de920b 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java
@@ -210,17 +210,8 @@ public class TestOMKeyPurgeRequestAndResponse extends
TestOMKeyRequest {
assertFalse(omMetadataManager.getDeletedTable().isExist(deletedKey));
}
- SnapshotInfo fromSnapshotInfo = new SnapshotInfo.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setName("snap1")
- .build();
-
- ReferenceCounted<OmSnapshot> rcOmSnapshot =
- ozoneManager.getOmSnapshotManager().getSnapshot(
- fromSnapshotInfo.getVolumeName(),
- fromSnapshotInfo.getBucketName(),
- fromSnapshotInfo.getName());
+ ReferenceCounted<OmSnapshot> rcOmSnapshot =
ozoneManager.getOmSnapshotManager()
+ .getSnapshot(snapInfo.getVolumeName(), snapInfo.getBucketName(),
snapInfo.getName());
OmSnapshot omSnapshot = rcOmSnapshot.get();
// The keys should be present in the snapshot's deletedTable
@@ -248,8 +239,7 @@ public class TestOMKeyPurgeRequestAndResponse extends
TestOMKeyRequest {
try (BatchOperation batchOperation =
omMetadataManager.getStore().initBatchOperation()) {
- OMKeyPurgeResponse omKeyPurgeResponse = new OMKeyPurgeResponse(
- omResponse, deletedKeyNames, fromSnapshotInfo, null);
+ OMKeyPurgeResponse omKeyPurgeResponse = new
OMKeyPurgeResponse(omResponse, deletedKeyNames, snapInfo, null);
omKeyPurgeResponse.addToDBBatch(omMetadataManager, batchOperation);
// Do manual commit and see whether addToBatch is successful or not.
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index d07372c4fc..0f2ab61506 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -364,7 +364,6 @@ public class TestSnapshotDiffManager {
omSnapshotManager = mock(OmSnapshotManager.class);
when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager);
- when(omSnapshotManager.isSnapshotStatus(any(), any())).thenReturn(true);
SnapshotCache snapshotCache = new SnapshotCache(mockCacheLoader(), 10,
omMetrics, 0);
when(omSnapshotManager.getActiveSnapshot(anyString(), anyString(),
anyString()))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
