This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 27c1513fd9 HDDS-9983. Changed snapshot list API to return continuation
token rather than using last element from the previous page's response. (#6542)
27c1513fd9 is described below
commit 27c1513fd92346a3e91d3705d057a07b10d7abad
Author: Hemant Kumar <[email protected]>
AuthorDate: Wed May 29 23:05:09 2024 -0700
HDDS-9983. Changed snapshot list API to return continuation token rather
than using last element from the previous page's response. (#6542)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 92 ++++++++++------------
.../ozone/client/protocol/ClientProtocol.java | 9 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 14 ++--
.../ozone/om/protocol/OzoneManagerProtocol.java | 9 ++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 20 ++++-
.../ozone/snapshot/ListSnapshotResponse.java | 52 ++++++++++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 18 ++---
.../src/main/proto/OmClientProtocol.proto | 1 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 40 +++++-----
.../org/apache/hadoop/ozone/om/OzoneManager.java | 7 +-
.../protocolPB/OzoneManagerRequestHandler.java | 18 +++--
.../hadoop/ozone/om/TestOmMetadataManager.java | 61 +++++++-------
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 3 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../ozone/shell/snapshot/ListSnapshotHandler.java | 2 +-
16 files changed, 205 insertions(+), 151 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index e96d0f84a4..e77f5580cc 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -46,6 +47,7 @@ import org.apache.hadoop.ozone.om.helpers.TenantUserList;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.UserGroupInformation;
@@ -612,66 +614,45 @@ public class ObjectStore {
* @param bucketName bucket name
* @param snapshotPrefix snapshot prefix to match
* @param prevSnapshot snapshots will be listed after this snapshot name
- * @return list of snapshots for volume/bucket snapshot path.
+ * @return an iterator of snapshots for volume/bucket snapshot path.
* @throws IOException
*/
- public Iterator<? extends OzoneSnapshot> listSnapshot(
- String volumeName, String bucketName, String snapshotPrefix,
- String prevSnapshot) throws IOException {
- return new SnapshotIterator(
- volumeName, bucketName, snapshotPrefix, prevSnapshot);
- }
-
- /**
- * Create an image of the current compaction log DAG in the OM.
- * @param fileNamePrefix file name prefix of the image file.
- * @param graphType type of node name to use in the graph image.
- * @return message which tells the image name, parent dir and OM leader
- * node information.
- */
- public String printCompactionLogDag(String fileNamePrefix,
- String graphType) throws IOException {
- return proxy.printCompactionLogDag(fileNamePrefix, graphType);
+ public Iterator<OzoneSnapshot> listSnapshot(String volumeName,
+ String bucketName,
+ String snapshotPrefix,
+ String prevSnapshot) throws
IOException {
+ return new SnapshotIterator(volumeName, bucketName, snapshotPrefix,
prevSnapshot);
}
/**
* An Iterator to iterate over {@link OzoneSnapshot} list.
*/
- private class SnapshotIterator implements Iterator<OzoneSnapshot> {
+ private final class SnapshotIterator implements Iterator<OzoneSnapshot> {
- private String volumeName = null;
- private String bucketName = null;
- private String snapshotPrefix = null;
+ private final String volumeName;
+ private final String bucketName;
+ private final String snapshotPrefix;
+ private String lastSnapshot = null;
private Iterator<OzoneSnapshot> currentIterator;
- private OzoneSnapshot currentValue;
- /**
- * Creates an Iterator to iterate over all snapshots after
- * prevSnapshot of specified bucket. If prevSnapshot is null it iterates
- * from the first snapshot. The returned snapshots match snapshot prefix.
- * @param snapshotPrefix snapshot prefix to match
- * @param prevSnapshot snapshots will be listed after this snapshot name
- */
- SnapshotIterator(String volumeName, String bucketName,
- String snapshotPrefix, String prevSnapshot)
- throws IOException {
+ SnapshotIterator(String volumeName,
+ String bucketName,
+ String snapshotPrefix,
+ String prevSnapshot) throws IOException {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.snapshotPrefix = snapshotPrefix;
- this.currentValue = null;
- this.currentIterator = getNextListOfSnapshots(prevSnapshot).iterator();
+ // Initialized the currentIterator and continuationToken.
+ getNextListOfSnapshots(prevSnapshot);
}
@Override
public boolean hasNext() {
- // IMPORTANT: Without this logic, remote iteration will not work.
- // Removing this will break the listSnapshot call if we try to
- // list more than 1000 (ozone.client.list.cache ) snapshots.
- if (!currentIterator.hasNext() && currentValue != null) {
+ if (!currentIterator.hasNext() && StringUtils.isNotEmpty(lastSnapshot)) {
try {
- currentIterator = getNextListOfSnapshots(currentValue.getName())
- .iterator();
+ // fetch the next page if lastSnapshot is not null.
+ getNextListOfSnapshots(lastSnapshot);
} catch (IOException e) {
LOG.error("Error retrieving next batch of list results", e);
}
@@ -682,24 +663,31 @@ public class ObjectStore {
@Override
public OzoneSnapshot next() {
if (hasNext()) {
- currentValue = currentIterator.next();
- return currentValue;
+ return currentIterator.next();
}
throw new NoSuchElementException();
}
- /**
- * Returns the next set of snapshot list using proxy.
- * @param prevSnapshot previous snapshot, this will be excluded from result
- * @return {@code List<OzoneSnapshot>}
- */
- private List<OzoneSnapshot> getNextListOfSnapshots(String prevSnapshot)
- throws IOException {
- return proxy.listSnapshot(volumeName, bucketName, snapshotPrefix,
- prevSnapshot, listCacheSize);
+ private void getNextListOfSnapshots(String startSnapshot) throws
IOException {
+ ListSnapshotResponse response =
+ proxy.listSnapshot(volumeName, bucketName, snapshotPrefix,
startSnapshot, listCacheSize);
+ currentIterator =
response.getSnapshotInfos().stream().map(OzoneSnapshot::fromSnapshotInfo).iterator();
+ lastSnapshot = response.getLastSnapshot();
}
}
+ /**
+ * Create an image of the current compaction log DAG in the OM.
+ * @param fileNamePrefix file name prefix of the image file.
+ * @param graphType type of node name to use in the graph image.
+ * @return message which tells the image name, parent dir and OM leader
+ * node information.
+ */
+ public String printCompactionLogDag(String fileNamePrefix,
+ String graphType) throws IOException {
+ return proxy.printCompactionLogDag(fileNamePrefix, graphType);
+ }
+
/**
* Get the differences between two snapshots.
* @param volumeName Name of the volume to which the snapshot bucket belong
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index f7b84e487d..81a2e7c25c 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -67,6 +67,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleI
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
@@ -1205,12 +1206,12 @@ public interface ClientProtocol {
* @param volumeName volume name
* @param bucketName bucket name
* @param snapshotPrefix snapshot prefix to match
- * @param prevSnapshot start of the list, this snapshot is excluded
- * @param maxListResult max numbet of snapshots to return
- * @return list of snapshots for volume/bucket snapshotpath.
+ * @param prevSnapshot snapshots will be listed after this snapshot name
+ * @param maxListResult max number of snapshots to return
+ * @return list of snapshots for volume/bucket path.
* @throws IOException
*/
- List<OzoneSnapshot> listSnapshot(
+ ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException;
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index d0266c95a2..a1b35d65a5 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -142,6 +142,7 @@ import
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -1121,23 +1122,20 @@ public class RpcClient implements ClientProtocol {
* @param volumeName volume name
* @param bucketName bucket name
* @param snapshotPrefix snapshot prefix to match
- * @param prevSnapshot start of the list, this snapshot is excluded
- * @param maxListResult max numbet of snapshots to return
- * @return list of snapshots for volume/bucket snapshotpath.
+ * @param prevSnapshot snapshots will be listed after this snapshot name
+ * @param maxListResult max number of snapshots to return
+ * @return list of snapshots for volume/bucket path.
* @throws IOException
*/
@Override
- public List<OzoneSnapshot> listSnapshot(
+ public ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
Preconditions.checkArgument(StringUtils.isNotBlank(volumeName),
"volume can't be null or empty.");
Preconditions.checkArgument(StringUtils.isNotBlank(bucketName),
"bucket can't be null or empty.");
- return ozoneManagerClient.listSnapshot(volumeName, bucketName,
- snapshotPrefix, prevSnapshot, maxListResult).stream()
- .map(snapshotInfo -> OzoneSnapshot.fromSnapshotInfo(snapshotInfo))
- .collect(Collectors.toList());
+ return ozoneManagerClient.listSnapshot(volumeName, bucketName,
snapshotPrefix, prevSnapshot, maxListResult);
}
/**
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index ab3f576d44..970eb9d509 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -70,6 +70,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPC
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.KerberosInfo;
@@ -742,12 +743,12 @@ public interface OzoneManagerProtocol
* @param volumeName volume name
* @param bucketName bucket name
* @param snapshotPrefix snapshot prefix to match
- * @param prevSnapshot start of the list, this snapshot is excluded
- * @param maxListResult max numbet of snapshots to return
- * @return list of snapshots for volume/bucket snapshotpath.
+ * @param prevSnapshot snapshots will be listed after this snapshot name
+ * @param maxListResult max number of snapshots to return
+ * @return list of snapshots for volume/bucket path.
* @throws IOException
*/
- default List<SnapshotInfo> listSnapshot(
+ default ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
throw new UnsupportedOperationException("OzoneManager does not require " +
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 87e5079f1d..1208df0f28 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -223,6 +223,7 @@ import
org.apache.hadoop.ozone.security.proto.SecurityProtos.CancelDelegationTok
import
org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import
org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
@@ -1320,7 +1321,7 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
* {@inheritDoc}
*/
@Override
- public List<SnapshotInfo> listSnapshot(
+ public ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
final OzoneManagerProtocolProtos.ListSnapshotRequest.Builder
@@ -1343,11 +1344,24 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
.build();
final OMResponse omResponse = submitRequest(omRequest);
handleError(omResponse);
- List<SnapshotInfo> snapshotInfos = omResponse.getListSnapshotResponse()
+ OzoneManagerProtocolProtos.ListSnapshotResponse response =
omResponse.getListSnapshotResponse();
+ List<SnapshotInfo> snapshotInfos = response
.getSnapshotInfoList().stream()
.map(snapshotInfo -> SnapshotInfo.getFromProtobuf(snapshotInfo))
.collect(Collectors.toList());
- return snapshotInfos;
+
+ String lastSnapshot = null;
+
+ if (response.hasLastSnapshot()) {
+ lastSnapshot = response.getLastSnapshot();
+ } else if (snapshotInfos.size() == maxListResult) {
+ // This is to make sure that the change (HDDS-9983) is forward
compatibility.
+ // Set lastSnapshot only when current list size is equal to maxListResult
+ // and there is possibility of more entries.
+ lastSnapshot = snapshotInfos.get(maxListResult - 1).getName();
+ }
+
+ return new ListSnapshotResponse(snapshotInfos, lastSnapshot);
}
/**
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotResponse.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotResponse.java
new file mode 100644
index 0000000000..5dda8265f6
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotResponse.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.snapshot;
+
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+
+import java.util.List;
+
+/**
+ * POJO for list snapshot info API.
+ */
+public class ListSnapshotResponse {
+ private final List<SnapshotInfo> snapshotInfos;
+ private final String lastSnapshot;
+
+ public ListSnapshotResponse(List<SnapshotInfo> snapshotInfos, String
lastSnapshot) {
+ this.snapshotInfos = snapshotInfos;
+ this.lastSnapshot = lastSnapshot;
+ }
+
+ public List<SnapshotInfo> getSnapshotInfos() {
+ return snapshotInfos;
+ }
+
+ public String getLastSnapshot() {
+ return lastSnapshot;
+ }
+
+ @Override
+ public String toString() {
+ return "ListSnapshotResponse{" +
+ "snapshotInfos: '" + snapshotInfos + '\'' +
+ ", lastSnapshot: '" + lastSnapshot + '\'' +
+ '}';
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index d96d8d0cae..8e22335759 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -4373,7 +4373,6 @@ public abstract class TestOzoneRpcClientAbstract {
}
@Test
- @Flaky("HDDS-9967")
public void testListSnapshot() throws IOException {
String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5);
String volumeB = "vol-b-" + RandomStringUtils.randomNumeric(5);
@@ -4410,11 +4409,10 @@ public abstract class TestOzoneRpcClientAbstract {
snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5));
}
- Iterator<? extends OzoneSnapshot> snapshotIter =
- store.listSnapshot(volumeA, bucketA, null, null);
+ Iterator<OzoneSnapshot> snapshotIter = store.listSnapshot(volumeA,
bucketA, null, null);
int volABucketASnapshotCount = 0;
while (snapshotIter.hasNext()) {
- OzoneSnapshot snapshot = snapshotIter.next();
+ snapshotIter.next();
volABucketASnapshotCount++;
}
assertEquals(20, volABucketASnapshotCount);
@@ -4422,26 +4420,26 @@ public abstract class TestOzoneRpcClientAbstract {
snapshotIter = store.listSnapshot(volumeA, bucketB, null, null);
int volABucketBSnapshotCount = 0;
while (snapshotIter.hasNext()) {
- OzoneSnapshot snapshot = snapshotIter.next();
+ snapshotIter.next();
volABucketBSnapshotCount++;
}
- assertEquals(20, volABucketASnapshotCount);
+ assertEquals(20, volABucketBSnapshotCount);
snapshotIter = store.listSnapshot(volumeB, bucketA, null, null);
int volBBucketASnapshotCount = 0;
while (snapshotIter.hasNext()) {
- OzoneSnapshot snapshot = snapshotIter.next();
+ snapshotIter.next();
volBBucketASnapshotCount++;
}
- assertEquals(20, volABucketASnapshotCount);
+ assertEquals(20, volBBucketASnapshotCount);
snapshotIter = store.listSnapshot(volumeB, bucketB, null, null);
int volBBucketBSnapshotCount = 0;
while (snapshotIter.hasNext()) {
- OzoneSnapshot snapshot = snapshotIter.next();
+ snapshotIter.next();
volBBucketBSnapshotCount++;
}
- assertEquals(20, volABucketASnapshotCount);
+ assertEquals(20, volBBucketBSnapshotCount);
int volABucketASnapshotACount = 0;
snapshotIter = store.listSnapshot(volumeA, bucketA, snapshotPrefixA, null);
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index e4559ad735..48b64b71dd 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1982,6 +1982,7 @@ message CreateSnapshotResponse {
message ListSnapshotResponse {
repeated SnapshotInfo snapshotInfo = 1;
+ optional string lastSnapshot = 2;
}
message SnapshotDiffResponse {
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index f00383cc1b..5a8da0106f 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.storage.proto.
OzoneManagerStorageProtos.PersistedUserVolumeInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -256,11 +257,11 @@ public interface OMMetadataManager extends
DBStoreHAManager {
* @param volumeName volume name
* @param bucketName bucket name
* @param snapshotPrefix snapshot prefix to match
- * @param prevSnapshot start of the list, this snapshot is excluded
- * @param maxListResult max numbet of snapshots to return
+ * @param prevSnapshot snapshots will be listed after this snapshot name
+ * @param maxListResult max number of snapshots to return
* @return list of snapshot
*/
- List<SnapshotInfo> listSnapshot(
+ ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index e35ab118e6..6e25dc1f7f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -93,6 +93,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.storage.proto
.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -1314,7 +1315,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
}
@Override
- public List<SnapshotInfo> listSnapshot(
+ public ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
if (Strings.isNullOrEmpty(volumeName)) {
@@ -1327,8 +1328,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
String bucketNameBytes = getBucketKey(volumeName, bucketName);
if (getBucketTable().get(bucketNameBytes) == null) {
- throw new OMException("Bucket " + bucketName + " not found.",
- BUCKET_NOT_FOUND);
+ throw new OMException("Bucket " + bucketName + " not found.",
BUCKET_NOT_FOUND);
}
String prefix;
@@ -1345,30 +1345,30 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
} else {
// This allows us to seek directly to the first key with the right
prefix.
seek = getOzoneKey(volumeName, bucketName,
- StringUtil.isNotBlank(
- snapshotPrefix) ? snapshotPrefix : OM_KEY_PREFIX);
+ StringUtil.isNotBlank(snapshotPrefix) ? snapshotPrefix :
OM_KEY_PREFIX);
}
List<SnapshotInfo> snapshotInfos = Lists.newArrayList();
+ String lastSnapshot = null;
try (ListIterator.MinHeapIterator snapshotIterator =
- new ListIterator.MinHeapIterator(this, prefix, seek, volumeName,
- bucketName, snapshotInfoTable)) {
- try {
- while (snapshotIterator.hasNext() && maxListResult > 0) {
- SnapshotInfo snapshotInfo =
- (SnapshotInfo) snapshotIterator.next().getValue();
- if (!snapshotInfo.getName().equals(prevSnapshot)) {
- snapshotInfos.add(snapshotInfo);
- maxListResult--;
- }
+ new ListIterator.MinHeapIterator(this, prefix, seek, volumeName,
bucketName, snapshotInfoTable)) {
+ SnapshotInfo snapshotInfo = null;
+ while (snapshotIterator.hasNext() && maxListResult > 0) {
+ snapshotInfo = (SnapshotInfo) snapshotIterator.next().getValue();
+ if (!Objects.equals(snapshotInfo.getName(), prevSnapshot)) {
+ snapshotInfos.add(snapshotInfo);
+ maxListResult--;
}
- } catch (NoSuchElementException e) {
- throw new IOException(e);
- } catch (UncheckedIOException e) {
- throw e.getCause();
}
+ if (snapshotIterator.hasNext() && maxListResult == 0 && snapshotInfo !=
null) {
+ lastSnapshot = snapshotInfo.getName();
+ }
+ } catch (NoSuchElementException e) {
+ throw new IOException(e);
+ } catch (UncheckedIOException e) {
+ throw e.getCause();
}
- return snapshotInfos;
+ return new ListSnapshotResponse(snapshotInfos, lastSnapshot);
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 9e12941a55..954d942a53 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -105,6 +105,7 @@ import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.security.acl.OzoneAuthorizerFactory;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -2991,7 +2992,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
@Override
- public List<SnapshotInfo> listSnapshot(
+ public ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
metrics.incNumSnapshotLists();
@@ -3002,13 +3003,13 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE,
ACLType.LIST, volumeName, bucketName, null);
}
- List<SnapshotInfo> snapshotInfoList =
+ ListSnapshotResponse listSnapshotResponse =
metadataManager.listSnapshot(volumeName, bucketName,
snapshotPrefix, prevSnapshot, maxListResult);
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
OMAction.LIST_SNAPSHOT, auditMap));
- return snapshotInfoList;
+ return listSnapshotResponse;
} catch (Exception ex) {
metrics.incNumSnapshotListFails();
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_SNAPSHOT,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index ca4925d3e5..5b8365e5a7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -158,6 +158,7 @@ import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartInfo;
import static org.apache.hadoop.util.MetricUtil.captureLatencyNs;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.util.ProtobufUtils;
import org.apache.ratis.server.protocol.TermIndex;
@@ -1450,14 +1451,19 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
private OzoneManagerProtocolProtos.ListSnapshotResponse getSnapshots(
OzoneManagerProtocolProtos.ListSnapshotRequest request)
throws IOException {
- List<SnapshotInfo> snapshotInfos = impl.listSnapshot(
+ ListSnapshotResponse implResponse = impl.listSnapshot(
request.getVolumeName(), request.getBucketName(), request.getPrefix(),
request.getPrevSnapshot(), request.getMaxListResult());
- List<OzoneManagerProtocolProtos.SnapshotInfo> snapshotInfoList =
- snapshotInfos.stream().map(SnapshotInfo::getProtobuf)
- .collect(Collectors.toList());
- return OzoneManagerProtocolProtos.ListSnapshotResponse.newBuilder()
- .addAllSnapshotInfo(snapshotInfoList).build();
+
+ List<OzoneManagerProtocolProtos.SnapshotInfo> snapshotInfoList =
implResponse.getSnapshotInfos()
+ .stream().map(SnapshotInfo::getProtobuf).collect(Collectors.toList());
+
+ OzoneManagerProtocolProtos.ListSnapshotResponse.Builder builder =
+
OzoneManagerProtocolProtos.ListSnapshotResponse.newBuilder().addAllSnapshotInfo(snapshotInfoList);
+ if (StringUtils.isNotEmpty(implResponse.getLastSnapshot())) {
+ builder.setLastSnapshot(implResponse.getLastSnapshot());
+ }
+ return builder.build();
}
private TransferLeadershipResponseProto transferLeadership(
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 1e2fb6a60a..5239fa0090 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -40,6 +40,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Expired
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKey;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.util.Time;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.BeforeEach;
@@ -76,6 +77,7 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLU
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.params.provider.Arguments.arguments;
@@ -899,68 +901,59 @@ public class TestOmMetadataManager {
for (int i = 1; i <= 100; i++) {
if (i % 2 == 0) {
snapshotsASnapshotIDMap.put(prefixA + i,
- OMRequestTestUtils.addSnapshotToTable(vol1, bucket1,
- prefixA + i, omMetadataManager));
+ OMRequestTestUtils.addSnapshotToTable(vol1, bucket1, prefixA + i,
omMetadataManager));
if (i % 4 == 0) {
snapshotsASnapshotIDMap.put(prefixA + i,
- OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1,
- prefixA + i, omMetadataManager));
+ OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1,
prefixA + i, omMetadataManager));
}
} else {
- OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1,
- prefixB + i, omMetadataManager);
+ OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1, prefixB + i,
omMetadataManager);
}
}
//Test listing all snapshots.
- List<SnapshotInfo> snapshotInfos = omMetadataManager.listSnapshot(vol1,
- bucket1, null, null, 100);
+ List<SnapshotInfo> snapshotInfos = omMetadataManager.listSnapshot(vol1,
bucket1, null, null, 100)
+ .getSnapshotInfos();
assertEquals(100, snapshotInfos.size());
- snapshotInfos = omMetadataManager.listSnapshot(vol1,
- bucket1, prefixA, null, 50);
+ snapshotInfos = omMetadataManager.listSnapshot(vol1, bucket1, prefixA,
null, 50)
+ .getSnapshotInfos();
assertEquals(50, snapshotInfos.size());
for (SnapshotInfo snapshotInfo : snapshotInfos) {
assertTrue(snapshotInfo.getName().startsWith(prefixA));
}
String startSnapshot = prefixA + 38;
- snapshotInfos = omMetadataManager.listSnapshot(vol1,
- bucket1, prefixA, startSnapshot, 50);
- assertEquals(snapshotsASnapshotIDMap.tailMap(startSnapshot).size() - 1,
- snapshotInfos.size());
+ snapshotInfos = omMetadataManager.listSnapshot(vol1, bucket1, prefixA,
startSnapshot, 50)
+ .getSnapshotInfos();
+
System.out.println(snapshotInfos.stream().map(SnapshotInfo::getName).collect(Collectors.joining(",")));
+ assertEquals(snapshotsASnapshotIDMap.tailMap(startSnapshot).size() - 1,
snapshotInfos.size());
for (SnapshotInfo snapshotInfo : snapshotInfos) {
assertTrue(snapshotInfo.getName().startsWith(prefixA));
- assertEquals(snapshotInfo, snapshotsASnapshotIDMap.get(
- snapshotInfo.getName()));
-
assertThat(snapshotInfo.getName().compareTo(startSnapshot)).isGreaterThan(0);
+ assertEquals(snapshotInfo,
snapshotsASnapshotIDMap.get(snapshotInfo.getName()));
+
assertThat(snapshotInfo.getName().compareTo(startSnapshot)).isGreaterThanOrEqualTo(0);
}
- startSnapshot = null;
+ String lastSnapshot = null;
TreeSet<String> expectedSnapshot = new TreeSet<>();
for (int i = 1; i <= 5; i++) {
- snapshotInfos = omMetadataManager.listSnapshot(
- vol1, bucket1, prefixA, startSnapshot, 10);
+ ListSnapshotResponse listSnapshotResponse =
+ omMetadataManager.listSnapshot(vol1, bucket1, prefixA, lastSnapshot,
10);
+ snapshotInfos = listSnapshotResponse.getSnapshotInfos();
+ lastSnapshot = listSnapshotResponse.getLastSnapshot();
assertEquals(10, snapshotInfos.size());
for (SnapshotInfo snapshotInfo : snapshotInfos) {
expectedSnapshot.add(snapshotInfo.getName());
- assertEquals(snapshotInfo, snapshotsASnapshotIDMap.get(
- snapshotInfo.getName()));
+ assertEquals(snapshotInfo,
snapshotsASnapshotIDMap.get(snapshotInfo.getName()));
assertTrue(snapshotInfo.getName().startsWith(prefixA));
- startSnapshot = snapshotInfo.getName();
}
}
assertEquals(snapshotsASnapshotIDMap.keySet(), expectedSnapshot);
-
- // As now we have iterated all 50 snapshots, calling next time should
- // return empty list.
- snapshotInfos = omMetadataManager.listSnapshot(vol1, bucket1,
- prefixA, startSnapshot, 10);
-
- assertEquals(snapshotInfos.size(), 0);
+ assertNull(lastSnapshot);
}
+
@ParameterizedTest
@MethodSource("listSnapshotWithInvalidPathCases")
public void testListSnapshotWithInvalidPath(String volume,
@@ -1010,15 +1003,15 @@ public class TestOmMetadataManager {
}
//Test listing snapshots only lists snapshots of specified bucket
- List<SnapshotInfo> snapshotInfos1 = omMetadataManager.listSnapshot(vol1,
- bucket1, null, null, Integer.MAX_VALUE);
+ List<SnapshotInfo> snapshotInfos1 = omMetadataManager.listSnapshot(vol1,
bucket1, null, null, Integer.MAX_VALUE)
+ .getSnapshotInfos();
assertEquals(2, snapshotInfos1.size());
for (SnapshotInfo snapshotInfo : snapshotInfos1) {
assertTrue(snapshotInfo.getName().startsWith(snapshotName1));
}
- List<SnapshotInfo> snapshotInfos2 = omMetadataManager.listSnapshot(vol1,
- bucket2, null, null, Integer.MAX_VALUE);
+ List<SnapshotInfo> snapshotInfos2 = omMetadataManager.listSnapshot(vol1,
bucket2, null, null, Integer.MAX_VALUE)
+ .getSnapshotInfos();
assertEquals(5, snapshotInfos2.size());
for (SnapshotInfo snapshotInfo : snapshotInfos2) {
assertTrue(snapshotInfo.getName().startsWith(snapshotName2));
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index af741994b7..109e19d13c 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -819,8 +819,7 @@ public class BasicRootedOzoneClientAdapterImpl
String group = getGroupName(ugi);
List<FileStatusAdapter> res = new ArrayList<>();
- Iterator<? extends OzoneSnapshot> snapshotIter =
- objectStore.listSnapshot(volumeName, bucketName, null, null);
+ Iterator<OzoneSnapshot> snapshotIter =
objectStore.listSnapshot(volumeName, bucketName, null, null);
while (snapshotIter.hasNext()) {
OzoneSnapshot ozoneSnapshot = snapshotIter.next();
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index c675a9ba6a..e70e221559 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -51,6 +51,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.token.Token;
@@ -693,7 +694,7 @@ public class ClientProtocolStub implements ClientProtocol {
}
@Override
- public List<OzoneSnapshot> listSnapshot(
+ public ListSnapshotResponse listSnapshot(
String volumeName, String bucketName, String snapshotPrefix,
String prevSnapshot, int maxListResult) throws IOException {
return null;
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
index bfbaa6b6c3..4744d0dcf3 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
@@ -54,7 +54,7 @@ public class ListSnapshotHandler extends Handler {
String volumeName = snapshotPath.getValue().getVolumeName();
String bucketName = snapshotPath.getValue().getBucketName();
- Iterator<? extends OzoneSnapshot> snapshotInfos = client.getObjectStore()
+ Iterator<OzoneSnapshot> snapshotInfos = client.getObjectStore()
.listSnapshot(volumeName, bucketName, listOptions.getPrefix(),
listOptions.getStartItem());
int counter = printAsJsonArray(snapshotInfos, listOptions.getLimit());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
