This is an automated email from the ASF dual-hosted git repository. dineshc pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new f7ba616 HDDS-2417 Add the list trash command to the client side (#138) f7ba616 is described below commit f7ba61684460dbeff67886c6b7cbfcda7688ec2c Author: Matthew Sharp <56448373+mbsh...@users.noreply.github.com> AuthorDate: Sun Nov 10 20:31:09 2019 -0500 HDDS-2417 Add the list trash command to the client side (#138) --- .../ozone/client/protocol/ClientProtocol.java | 20 +++++++++++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 12 +++++++ .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../ozone/om/protocol/OzoneManagerProtocol.java | 19 ++++++++++ ...OzoneManagerProtocolClientSideTranslatorPB.java | 42 ++++++++++++++++++++++ .../src/main/proto/OzoneManagerProtocol.proto | 21 +++++++++++ .../org/apache/hadoop/ozone/om/OzoneManager.java | 11 ++++++ 7 files changed, 126 insertions(+) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 068c597..77345d9 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; @@ -314,6 +315,25 @@ public interface ClientProtocol { String keyPrefix, String prevKey, int maxListResult) throws IOException; + /** + * List trash allows the user to list the keys that were marked as deleted, + * but not actually deleted by Ozone Manager. This allows a user to recover + * keys within a configurable window. + * @param volumeName - The volume name, which can also be a wild card + * using '*'. + * @param bucketName - The bucket name, which can also be a wild card + * using '*'. + * @param startKeyName - List keys from a specific key name. + * @param keyPrefix - List keys using a specific prefix. + * @param maxKeys - The number of keys to be returned. This must be below + * the cluster level set by admins. + * @return The list of keys that are deleted from the deleted table. + * @throws IOException + */ + List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName, + String startKeyName, String keyPrefix, + int maxKeys) + throws IOException; /** * Get OzoneKey. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index efc9470..302a561 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; @@ -709,6 +710,17 @@ public class RpcClient implements ClientProtocol { } @Override + public List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName, + String startKeyName, String keyPrefix, int maxKeys) throws IOException { + + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + + return ozoneManagerClient.listTrash(volumeName, bucketName, startKeyName, + keyPrefix, maxKeys); + } + + @Override public OzoneKeyDetails getKeyDetails( String volumeName, String bucketName, String keyName) throws IOException { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 8e129c9..ca51187 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -224,6 +224,7 @@ public final class OmUtils { case ListBuckets: case LookupKey: case ListKeys: + case ListTrash: case InfoS3Bucket: case ListS3Buckets: case ServiceList: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 1cdc2fc..9bd5032 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; @@ -527,4 +528,22 @@ public interface OzoneManagerProtocol OzoneManagerProtocolProtos.DBUpdatesRequest dbUpdatesRequest) throws IOException; + /** + * List trash allows the user to list the keys that were marked as deleted, + * but not actually deleted by Ozone Manager. This allows a user to recover + * keys within a configurable window. + * @param volumeName - The volume name, which can also be a wild card + * using '*'. + * @param bucketName - The bucket name, which can also be a wild card + * using '*'. + * @param startKeyName - List keys from a specific key name. + * @param keyPrefix - List keys using a specific prefix. + * @param maxKeys - The number of keys to be returned. This must be below + * the cluster level set by admins. + * @return The list of keys that are deleted from the deleted table. + * @throws IOException + */ + List<RepeatedOmKeyInfo> listTrash(String volumeName, String bucketName, + String startKeyName, String keyPrefix, int maxKeys) throws IOException; + } \ No newline at end of file diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 6924ee2..206bf15 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -56,6 +56,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; @@ -105,6 +106,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBuc import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListVolumeResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupKeyRequest; @@ -1592,4 +1595,43 @@ public final class OzoneManagerProtocolClientSideTranslatorPB } return statusList; } + + @Override + public List<RepeatedOmKeyInfo> listTrash(String volumeName, + String bucketName, String startKeyName, String keyPrefix, int maxKeys) + throws IOException { + + Preconditions.checkArgument(Strings.isNullOrEmpty(volumeName), + "The volume name cannot be null or " + + "empty. Please enter a valid volume name or use '*' as a wild card"); + + Preconditions.checkArgument(Strings.isNullOrEmpty(bucketName), + "The bucket name cannot be null or " + + "empty. Please enter a valid bucket name or use '*' as a wild card"); + + ListTrashRequest trashRequest = ListTrashRequest.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setStartKeyName(startKeyName) + .setKeyPrefix(keyPrefix) + .setMaxKeys(maxKeys) + .build(); + + OMRequest omRequest = createOMRequest(Type.ListTrash) + .setListTrashRequest(trashRequest) + .build(); + + ListTrashResponse trashResponse = + handleError(submitRequest(omRequest)).getListTrashResponse(); + + List<RepeatedOmKeyInfo> deletedKeyList = + new ArrayList<>(trashResponse.getDeletedKeysCount()); + + deletedKeyList.addAll( + trashResponse.getDeletedKeysList().stream() + .map(RepeatedOmKeyInfo::getFromProto) + .collect(Collectors.toList())); + + return deletedKeyList; + } } diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index 569b4f1..62b7190 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -92,6 +92,8 @@ enum Type { PurgeKeys = 81; ListMultipartUploads = 82; + + ListTrash = 91; } message OMRequest { @@ -161,6 +163,8 @@ message OMRequest { optional UpdateGetS3SecretRequest updateGetS3SecretRequest = 82; optional ListMultipartUploadsRequest listMultipartUploadsRequest = 83; + + optional ListTrashRequest listTrashRequest = 91; } message OMResponse { @@ -230,6 +234,8 @@ message OMResponse { optional PurgeKeysResponse purgeKeysResponse = 81; optional ListMultipartUploadsResponse listMultipartUploadsResponse = 82; + + optional ListTrashResponse listTrashResponse = 91; } enum Status { @@ -302,6 +308,21 @@ enum Status { INVALID_PART_ORDER = 56; } +/** + This command acts as a list command for deleted keys that are still present + in the deleted table on Ozone Manager. +*/ +message ListTrashRequest { + required string volumeName = 1; + required string bucketName = 2; + optional string startKeyName = 3; + optional string keyPrefix = 4; + optional int32 maxKeys = 5; +} + +message ListTrashResponse { + repeated RepeatedKeyInfo deletedKeys = 1; +} message VolumeInfo { required string adminName = 1; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 0f9b58d..9061d1b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -78,6 +78,7 @@ import org.apache.hadoop.ozone.om.ha.OMFailoverProxyProvider; import org.apache.hadoop.ozone.om.ha.OMHANodeDetails; import org.apache.hadoop.ozone.om.ha.OMNodeDetails; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.protocol.OzoneManagerServerProtocol; import org.apache.hadoop.ozone.om.ratis.OMRatisSnapshotInfo; @@ -2239,6 +2240,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl } } + // TODO: HDDS-2418 - Complete stub below for server logic + @Override + public List<RepeatedOmKeyInfo> listTrash(String volumeName, + String bucketName, String startKeyName, String keyPrefix, int maxKeys) + throws IOException { + + List<RepeatedOmKeyInfo> deletedKeys = new ArrayList<>(); + return deletedKeys; + } + /** * Sets bucket property from args. * --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org