This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit bddb7387f84678fc47838f8e91a798b3eb0af13a Author: Sadanand Shenoy <[email protected]> AuthorDate: Tue Feb 16 22:07:12 2021 +0530 HDDS-4742. Make trash work with FS Optimised Buckets. (#1915) --- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 ++++ .../hadoop/fs/ozone/TestOzoneFileSystemV1.java | 12 -- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 142 ++++++++++++++++----- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 15 +-- 4 files changed, 135 insertions(+), 59 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java index e9d4cf9..e7e2eb0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java @@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.om.helpers; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.util.StringUtils; import javax.annotation.Nonnull; import java.nio.file.Paths; +import java.util.Map; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -205,4 +207,27 @@ public final class OzoneFSUtils { java.nio.file.Path keyPath = Paths.get(keyName); return keyPath.getNameCount(); } + + + /** + * Returns true if the bucket is FS Optimised. + * @param bucketMetadata + * @return + */ + public static boolean isFSOptimizedBucket( + Map<String, String> bucketMetadata) { + // layout version V1 represents optimized FS path + boolean layoutVersionEnabled = + org.apache.commons.lang3.StringUtils.equalsIgnoreCase( + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1, + bucketMetadata + .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION)); + + boolean fsEnabled = + Boolean.parseBoolean(bucketMetadata + .get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS)); + + return layoutVersionEnabled && fsEnabled; + } + } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index 03846ae..eb7eaca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -383,18 +383,6 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { @Override @Test @Ignore("TODO:HDDS-2939") - public void testTrash() throws Exception { - } - - @Override - @Test - @Ignore("TODO:HDDS-2939") - public void testRenameToTrashEnabled() throws Exception { - } - - @Override - @Test - @Ignore("TODO:HDDS-2939") public void testListStatusWithIntermediateDir() throws Exception { } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java index 6d7a88a..a9408a8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashOzoneFileSystem.java @@ -35,10 +35,11 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.apache.ratis.protocol.ClientId; @@ -154,6 +155,11 @@ public class TrashOzoneFileSystem extends FileSystem { // check whether the src and dst belong to the same bucket & trashroot. OFSPath srcPath = new OFSPath(src); OFSPath dstPath = new OFSPath(dst); + OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(), + srcPath.getBucketName()); + if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) { + return renameV1(srcPath, dstPath); + } Preconditions.checkArgument(srcPath.getBucketName(). equals(dstPath.getBucketName())); Preconditions.checkArgument(srcPath.getTrashRoot(). @@ -163,14 +169,50 @@ public class TrashOzoneFileSystem extends FileSystem { return true; } + private boolean renameV1(OFSPath srcPath, OFSPath dstPath) { + OzoneManagerProtocolProtos.OMRequest omRequest = + getRenameKeyRequest(srcPath, dstPath); + try { + if(omRequest != null) { + submitRequest(omRequest); + return true; + } + return false; + } catch (Exception e){ + LOG.error("couldnt send rename requestV1", e); + return false; + } + } + @Override public boolean delete(Path path, boolean b) throws IOException { ozoneManager.getMetrics().incNumTrashDeletes(); + OFSPath srcPath = new OFSPath(path); + OmBucketInfo bucket = ozoneManager.getBucketInfo(srcPath.getVolumeName(), + srcPath.getBucketName()); + if (OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata())) { + return deleteV1(srcPath); + } DeleteIterator iterator = new DeleteIterator(path, true); iterator.iterate(); return true; } + private boolean deleteV1(OFSPath srcPath) { + OzoneManagerProtocolProtos.OMRequest omRequest = + getDeleteKeyRequest(srcPath); + try { + if(omRequest != null) { + submitRequest(omRequest); + return true; + } + return false; + } catch (Throwable e) { + LOG.error("Couldn't send delete request.", e); + return false; + } + } + @Override public FileStatus[] listStatus(Path path) throws IOException { ozoneManager.getMetrics().incNumTrashListStatus(); @@ -377,6 +419,41 @@ public class TrashOzoneFileSystem extends FileSystem { } } + + private OzoneManagerProtocolProtos.OMRequest + getRenameKeyRequest( + OFSPath src, OFSPath dst) { + String volumeName = src.getVolumeName(); + String bucketName = src.getBucketName(); + String keyName = src.getKeyName(); + + OzoneManagerProtocolProtos.KeyArgs keyArgs = + OzoneManagerProtocolProtos.KeyArgs.newBuilder() + .setKeyName(keyName) + .setVolumeName(volumeName) + .setBucketName(bucketName) + .build(); + String toKeyName = dst.getKeyName(); + OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest = + OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder() + .setKeyArgs(keyArgs) + .setToKeyName(toKeyName) + .build(); + OzoneManagerProtocolProtos.OMRequest omRequest = + null; + try { + omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setClientId(CLIENT_ID.toString()) + .setUserInfo(getUserInfo()) + .setRenameKeyRequest(renameKeyRequest) + .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey) + .build(); + } catch (IOException e) { + LOG.error("Couldn't get userinfo", e); + } + return omRequest; + } + private class RenameIterator extends OzoneListingIterator { private final String srcPath; private final String dstPath; @@ -408,40 +485,37 @@ public class TrashOzoneFileSystem extends FileSystem { } return true; } + } - private OzoneManagerProtocolProtos.OMRequest - getRenameKeyRequest( - OFSPath src, OFSPath dst) { - String volumeName = src.getVolumeName(); - String bucketName = src.getBucketName(); - String keyName = src.getKeyName(); - - OzoneManagerProtocolProtos.KeyArgs keyArgs = - OzoneManagerProtocolProtos.KeyArgs.newBuilder() - .setKeyName(keyName) - .setVolumeName(volumeName) - .setBucketName(bucketName) - .build(); - String toKeyName = dst.getKeyName(); - OzoneManagerProtocolProtos.RenameKeyRequest renameKeyRequest = - OzoneManagerProtocolProtos.RenameKeyRequest.newBuilder() - .setKeyArgs(keyArgs) - .setToKeyName(toKeyName) - .build(); - OzoneManagerProtocolProtos.OMRequest omRequest = - null; - try { - omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() - .setClientId(CLIENT_ID.toString()) - .setUserInfo(getUserInfo()) - .setRenameKeyRequest(renameKeyRequest) - .setCmdType(OzoneManagerProtocolProtos.Type.RenameKey) + private OzoneManagerProtocolProtos.OMRequest getDeleteKeyRequest( + OFSPath srcPath) { + String volume = srcPath.getVolumeName(); + String bucket = srcPath.getBucketName(); + String key = srcPath.getKeyName(); + OzoneManagerProtocolProtos.KeyArgs keyArgs = + OzoneManagerProtocolProtos.KeyArgs.newBuilder() + .setKeyName(key) + .setVolumeName(volume) + .setBucketName(bucket) + .setRecursive(true) .build(); - } catch (IOException e) { - LOG.error("Couldn't get userinfo", e); - } - return omRequest; + OzoneManagerProtocolProtos.DeleteKeyRequest deleteKeyRequest = + OzoneManagerProtocolProtos.DeleteKeyRequest.newBuilder() + .setKeyArgs(keyArgs).build(); + OzoneManagerProtocolProtos.OMRequest omRequest = + null; + try { + omRequest = + OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setClientId(CLIENT_ID.toString()) + .setUserInfo(getUserInfo()) + .setDeleteKeyRequest(deleteKeyRequest) + .setCmdType(OzoneManagerProtocolProtos.Type.DeleteKey) + .build(); + } catch (IOException e) { + LOG.error("Couldn't get userinfo", e); } + return omRequest; } private class DeleteIterator extends OzoneListingIterator { @@ -467,7 +541,7 @@ public class TrashOzoneFileSystem extends FileSystem { for (String keyPath : keyPathList) { OFSPath path = new OFSPath(keyPath); OzoneManagerProtocolProtos.OMRequest omRequest = - getDeleteKeyRequest(path); + getDeleteKeysRequest(path); try { ozoneManager.getMetrics().incNumTrashFilesDeletes(); submitRequest(omRequest); @@ -479,7 +553,7 @@ public class TrashOzoneFileSystem extends FileSystem { } private OzoneManagerProtocolProtos.OMRequest - getDeleteKeyRequest( + getDeleteKeysRequest( OFSPath keyPath) { String volumeName = keyPath.getVolumeName(); String bucketName = keyPath.getBucketName(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 150108c..125934e 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -48,12 +48,12 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; @@ -548,17 +548,6 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter { @Override public boolean isFSOptimizedBucket() { - // layout version V1 represents optimized FS path - boolean layoutVersionEnabled = - StringUtils.equalsIgnoreCase( - OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1, - bucket.getMetadata() - .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION)); - - boolean fsEnabled = - Boolean.parseBoolean(bucket.getMetadata() - .get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS)); - - return layoutVersionEnabled && fsEnabled; + return OzoneFSUtils.isFSOptimizedBucket(bucket.getMetadata()); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
