This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit da0a501ebdfb69baf64b6a661b2450916945bb67 Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Thu Jan 28 15:50:48 2021 +0530 HDDS-4486. Feature Config: Make proper enableFSPaths and OFS optimized flag combinations (#1848) --- .../apache/hadoop/fs/ozone/TestOzoneDirectory.java | 4 +- .../hadoop/fs/ozone/TestOzoneFileInterfacesV1.java | 6 +- .../apache/hadoop/fs/ozone/TestOzoneFileOps.java | 5 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 13 ++- .../hadoop/fs/ozone/TestOzoneFileSystemV1.java | 66 +++--------- .../hadoop/ozone/client/rpc/TestReadRetries.java | 9 +- .../ozone/freon/TestHadoopDirTreeGeneratorV1.java | 4 +- .../apache/hadoop/ozone/om/TestObjectStoreV1.java | 3 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 17 ++- .../om/ratis/utils/OzoneManagerRatisUtils.java | 35 ++++--- .../om/request/bucket/OMBucketCreateRequest.java | 28 +++-- .../ozone/om/request/TestOMRequestUtils.java | 42 ++++++++ .../ozone/om/request/bucket/TestBucketRequest.java | 1 + .../request/bucket/TestOMBucketCreateRequest.java | 2 +- .../bucket/TestOMBucketCreateRequestV1.java | 114 +++++++++++++++++++++ .../file/TestOMDirectoryCreateRequestV1.java | 3 +- .../om/request/file/TestOMFileCreateRequestV1.java | 4 +- .../request/key/TestOMAllocateBlockRequestV1.java | 4 +- .../om/request/key/TestOMKeyCommitRequestV1.java | 4 +- .../om/request/key/TestOMKeyCreateRequestV1.java | 4 +- .../om/request/key/TestOMKeyDeleteRequestV1.java | 4 +- .../response/file/TestOMFileCreateResponseV1.java | 4 +- .../key/TestOMAllocateBlockResponseV1.java | 4 +- .../om/response/key/TestOMKeyCommitResponseV1.java | 4 +- .../om/response/key/TestOMKeyCreateResponseV1.java | 4 +- .../om/response/key/TestOMKeyDeleteResponseV1.java | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 15 ++- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 10 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 5 +- .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 2 +- 32 files changed, 288 insertions(+), 148 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java index 56c6177..22ed13e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneDirectory.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.After; import org.junit.Assert; @@ -149,7 +150,8 @@ public class TestOzoneDirectory { throws IOException, TimeoutException, InterruptedException { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(FS_TRASH_INTERVAL_KEY, 1); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + true, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesV1.java index 93473be..d716457 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfacesV1.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.ozone; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.jetbrains.annotations.NotNull; import org.junit.Ignore; import org.junit.Test; @@ -44,9 +45,8 @@ public class TestOzoneFileInterfacesV1 extends TestOzoneFileInterfaces { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - enableFileSystemPaths); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + enableFileSystemPaths, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); return conf; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java index d097268..12dd51e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.After; import org.junit.Before; @@ -73,8 +74,8 @@ public class TestOzoneFileOps { throws IOException, TimeoutException, InterruptedException { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(FS_TRASH_INTERVAL_KEY, 1); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, false); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + true, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index bf9e09f..685aade 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; @@ -114,6 +115,7 @@ public class TestOzoneFileSystem { } } } + /** * Set a timeout for each test. */ @@ -124,6 +126,8 @@ public class TestOzoneFileSystem { LoggerFactory.getLogger(TestOzoneFileSystem.class); @SuppressWarnings("checkstyle:VisibilityModifier") + protected static boolean isBucketFSOptimized = false; + @SuppressWarnings("checkstyle:VisibilityModifier") protected static boolean enabledFileSystemPaths; @SuppressWarnings("checkstyle:VisibilityModifier") protected static boolean omRatisEnabled; @@ -147,8 +151,13 @@ public class TestOzoneFileSystem { conf.setInt(FS_TRASH_INTERVAL_KEY, 1); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); conf.setBoolean(OZONE_ACL_ENABLED, true); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - enabledFileSystemPaths); + if (isBucketFSOptimized) { + TestOMRequestUtils.configureFSOptimizedPaths(conf, + enabledFileSystemPaths, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); + } else { + conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + enabledFileSystemPaths); + } cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index e574e94..ffeb5a3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -18,26 +18,15 @@ package org.apache.hadoop.fs.ozone; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.junit.Assert; import org.junit.After; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Rule; @@ -51,9 +40,9 @@ import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; @@ -66,49 +55,24 @@ import static org.junit.Assert.fail; @RunWith(Parameterized.class) public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { - @Ignore("TODO:HDDS-2939") - @BeforeClass - public static void init() throws Exception { + @Parameterized.Parameters + public static Collection<Object[]> data() { + return Arrays.asList( + new Object[]{true, true}, + new Object[]{true, false}, + new Object[]{false, true}, + new Object[]{false, false}); + } + @BeforeClass + public static void init() { + isBucketFSOptimized = true; } public TestOzoneFileSystemV1(boolean setDefaultFs, boolean enableOMRatis) { super(setDefaultFs, enableOMRatis); } - @Before - public void setup() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(FS_TRASH_INTERVAL_KEY, 1); - conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, - enabledFileSystemPaths); - if (enabledFileSystemPaths) { - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); - } - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .build(); - cluster.waitForClusterToBeReady(); - - // create a volume and a bucket to be used by OzoneFileSystem - OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster); - volumeName = bucket.getVolumeName(); - bucketName = bucket.getName(); - - String rootPath = String.format("%s://%s.%s/", - OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName); - - // Set the fs.defaultFS and start the filesystem - conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath); - // Set the number of keys to be processed during batch operate. - conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); - - fs = FileSystem.get(conf); - trash = new Trash(conf); - o3fs = (OzoneFileSystem) fs; - } - @After @Override public void cleanup() { @@ -119,10 +83,6 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { LOG.info("Failed to cleanup DB tables.", e); fail("Failed to cleanup DB tables." + e.getMessage()); } - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.closeQuietly(fs); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java index ce2f10b..36146a2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.junit.After; import org.junit.Assert; @@ -102,7 +103,9 @@ public class TestReadRetries { @Parameterized.Parameters public static Collection<Object[]> data() { - return Arrays.asList(new Object[]{"V0"}, new Object[]{"V1"}); + return Arrays.asList( + new Object[]{OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT}, + new Object[]{OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1}); } /** @@ -113,8 +116,8 @@ public class TestReadRetries { public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1); - conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, layoutVersion); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + true, layoutVersion); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .setScmId(SCM_ID) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorV1.java index 99d4f26..01a73bc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorV1.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.freon; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; /** * Test for HadoopDirTreeGenerator layout version V1. @@ -26,7 +27,8 @@ public class TestHadoopDirTreeGeneratorV1 extends TestHadoopDirTreeGenerator { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + true, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); return conf; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java index d343e2c..d09020e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java @@ -88,7 +88,8 @@ public class TestObjectStoreV1 { clusterId = UUID.randomUUID().toString(); scmId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString(); - conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + TestOMRequestUtils.configureFSOptimizedPaths(conf, + true, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); cluster = MiniOzoneCluster.newBuilder(conf) .setClusterId(clusterId) .setScmId(scmId) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 055ab13..604f7d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -657,7 +657,7 @@ public class KeyManagerImpl implements KeyManager { bucketName); OmKeyInfo value = null; try { - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { value = getOmKeyInfoV1(volumeName, bucketName, keyName); } else { value = getOmKeyInfo(volumeName, bucketName, keyName); @@ -1802,7 +1802,7 @@ public class KeyManagerImpl implements KeyManager { String bucketName = args.getBucketName(); String keyName = args.getKeyName(); - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { return getOzoneFileStatusV1(volumeName, bucketName, keyName, args.getSortDatanodes(), clientAddress, false); } @@ -2072,7 +2072,7 @@ public class KeyManagerImpl implements KeyManager { String bucketName = args.getBucketName(); String keyName = args.getKeyName(); OzoneFileStatus fileStatus; - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { fileStatus = getOzoneFileStatusV1(volumeName, bucketName, keyName, args.getSortDatanodes(), clientAddress, false); } else { @@ -2180,7 +2180,7 @@ public class KeyManagerImpl implements KeyManager { return fileStatusList; } - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { return listStatusV1(args, recursive, startKey, numEntries, clientAddress); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 22bbc20..3ac234a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -240,7 +240,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { @Override public Table<String, OmKeyInfo> getKeyTable() { - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { return fileTable; } return keyTable; @@ -253,7 +253,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager { @Override public Table<String, OmKeyInfo> getOpenKeyTable() { - if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { return openFileTable; } return openKeyTable; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 30c1d37..51775ef 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -227,6 +227,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_F import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LAYOUT_VERSION; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME; @@ -1163,7 +1164,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl } // TODO: Temporary workaround for OM upgrade path and will be replaced once - // upgrade HDDS-3698 story reaches consensus. + // upgrade HDDS-3698 story reaches consensus. Instead of cluster level + // configuration, OM needs to check this property on every bucket level. getOMLayoutVersion(); metadataManager.start(configuration); @@ -3708,14 +3710,19 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); } - private void getOMLayoutVersion() { + public String getOMLayoutVersion() { String version = configuration.getTrimmed(OZONE_OM_LAYOUT_VERSION, OZONE_OM_LAYOUT_VERSION_DEFAULT); - boolean omLayoutVersionV1 = - StringUtils.equalsIgnoreCase(version, "V1"); - OzoneManagerRatisUtils.setOmLayoutVersionV1(omLayoutVersionV1); + boolean omLayoutVersionV1 = StringUtils.equalsIgnoreCase(version, + OZONE_OM_LAYOUT_VERSION_V1); LOG.info("Configured {}={} and enabled:{} optimized OM FS operations", OZONE_OM_LAYOUT_VERSION, version, omLayoutVersionV1); + + boolean isBucketFSOptimized = + omLayoutVersionV1 && getEnableFileSystemPaths(); + OzoneManagerRatisUtils.setBucketFSOptimized(isBucketFSOptimized); + + return version; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index c469c2c..26fded2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -92,18 +92,19 @@ public final class OzoneManagerRatisUtils { // TODO: Temporary workaround for OM upgrade path and will be replaced once // upgrade HDDS-3698 story reaches consensus. - private static boolean omLayoutVersionV1 = false; + private static boolean isBucketFSOptimized = false; private OzoneManagerRatisUtils() { } /** - * Sets layout version. + * Sets enabled/disabled file system optimized path property. A true value + * represents enabled, false represents disabled. * - * @param layoutVersionV1 om layout version + * @param enabledFSO enabled/disabled file system optimized */ - public static void setOmLayoutVersionV1(boolean layoutVersionV1) { - OzoneManagerRatisUtils.omLayoutVersionV1 = layoutVersionV1; + public static void setBucketFSOptimized(boolean enabledFSO) { + OzoneManagerRatisUtils.isBucketFSOptimized = enabledFSO; } /** @@ -140,41 +141,41 @@ public final class OzoneManagerRatisUtils { case SetBucketProperty: return new OMBucketSetPropertyRequest(omRequest); case AllocateBlock: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMAllocateBlockRequestV1(omRequest); } return new OMAllocateBlockRequest(omRequest); case CreateKey: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMKeyCreateRequestV1(omRequest); } return new OMKeyCreateRequest(omRequest); case CommitKey: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMKeyCommitRequestV1(omRequest); } return new OMKeyCommitRequest(omRequest); case DeleteKey: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMKeyDeleteRequestV1(omRequest); } return new OMKeyDeleteRequest(omRequest); case DeleteKeys: return new OMKeysDeleteRequest(omRequest); case RenameKey: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMKeyRenameRequestV1(omRequest); } return new OMKeyRenameRequest(omRequest); case RenameKeys: return new OMKeysRenameRequest(omRequest); case CreateDirectory: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMDirectoryCreateRequestV1(omRequest); } return new OMDirectoryCreateRequest(omRequest); case CreateFile: - if (omLayoutVersionV1) { + if (isBucketFSOptimized()) { return new OMFileCreateRequestV1(omRequest); } return new OMFileCreateRequest(omRequest); @@ -352,11 +353,13 @@ public final class OzoneManagerRatisUtils { } /** - * Returns layout version flag represents V1. - * @return + * Returns enabled/disabled file system optimized path property. A true value + * represents FSO path is enabled, false represents disabled. + * + * @return true or false. */ - public static boolean isOmLayoutVersionV1() { - return omLayoutVersionV1; + public static boolean isBucketFSOptimized() { + return isBucketFSOptimized; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index d3b3f61..8b11770 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -27,7 +27,7 @@ import java.util.stream.Collectors; import com.google.common.base.Optional; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; @@ -160,7 +160,7 @@ public class OMBucketCreateRequest extends OMClientRequest { OmBucketInfo omBucketInfo = OmBucketInfo.getFromProtobuf(bucketInfo); // Add layout version V1 to bucket info - addLayoutVersionToBucket(ozoneManager, omBucketInfo); + addFSOptimizedBucketDetails(ozoneManager, omBucketInfo); AuditLogger auditLogger = ozoneManager.getAuditLogger(); OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); @@ -367,20 +367,30 @@ public class OMBucketCreateRequest extends OMClientRequest { } - private void addLayoutVersionToBucket(OzoneManager ozoneManager, - OmBucketInfo omBucketInfo) { + /** + * OM can support FS optimization only if both are flags are TRUE + * (enableFSOptimized=true && enableFSPaths=true) and will write table key + * entries in NEW_FORMAT(prefix separated format using objectID). All the + * other cases, it will + * write table key entries in OLD_FORMAT(existing format). + * + * @param ozoneManager ozone manager + * @param omBucketInfo bucket information + */ + private void addFSOptimizedBucketDetails(OzoneManager ozoneManager, + OmBucketInfo omBucketInfo) { Map<String, String> metadata = omBucketInfo.getMetadata(); if (metadata == null) { metadata = new HashMap<>(); } - OzoneConfiguration configuration = ozoneManager.getConfiguration(); // TODO: Many unit test cases has null config and done a simple null // check now. It can be done later, to avoid massive test code changes. - if (configuration != null) { - String layOutVersion = configuration - .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, - OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT); + if(StringUtils.isNotBlank(ozoneManager.getOMLayoutVersion())){ + String layOutVersion = ozoneManager.getOMLayoutVersion(); metadata.put(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, layOutVersion); + boolean fsPathsEnabled = ozoneManager.getEnableFileSystemPaths(); + metadata.put(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + Boolean.toString(fsPathsEnabled)); omBucketInfo.setMetadata(metadata); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java index 61fd676..eca06f9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMRequestUtils.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.UUID; import com.google.common.base.Optional; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -430,6 +432,25 @@ public final class TestOMRequestUtils { .setClientId(UUID.randomUUID().toString()).build(); } + public static OzoneManagerProtocolProtos.OMRequest createBucketRequestV1( + String bucketName, String volumeName, boolean isVersionEnabled, + OzoneManagerProtocolProtos.StorageTypeProto storageTypeProto) { + OzoneManagerProtocolProtos.BucketInfo bucketInfo = + OzoneManagerProtocolProtos.BucketInfo.newBuilder() + .setBucketName(bucketName) + .setVolumeName(volumeName) + .setIsVersionEnabled(isVersionEnabled) + .setStorageType(storageTypeProto) + .addAllMetadata(getMetadataListV1()).build(); + OzoneManagerProtocolProtos.CreateBucketRequest.Builder req = + OzoneManagerProtocolProtos.CreateBucketRequest.newBuilder(); + req.setBucketInfo(bucketInfo); + return OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCreateBucketRequest(req) + .setCmdType(OzoneManagerProtocolProtos.Type.CreateBucket) + .setClientId(UUID.randomUUID().toString()).build(); + } + public static List< HddsProtos.KeyValue> getMetadataList() { List<HddsProtos.KeyValue> metadataList = new ArrayList<>(); metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key1").setValue( @@ -439,6 +460,20 @@ public final class TestOMRequestUtils { return metadataList; } + public static List< HddsProtos.KeyValue> getMetadataListV1() { + List<HddsProtos.KeyValue> metadataList = new ArrayList<>(); + metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key1").setValue( + "value1").build()); + metadataList.add(HddsProtos.KeyValue.newBuilder().setKey("key2").setValue( + "value2").build()); + metadataList.add(HddsProtos.KeyValue.newBuilder().setKey( + OMConfigKeys.OZONE_OM_LAYOUT_VERSION).setValue( + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1).build()); + metadataList.add(HddsProtos.KeyValue.newBuilder().setKey( + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS).setValue( + "false").build()); + return metadataList; + } /** * Add user to user table. @@ -927,4 +962,11 @@ public final class TestOMRequestUtils { } return parentId; } + + public static void configureFSOptimizedPaths(Configuration conf, + boolean enableFileSystemPaths, String version) { + conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + enableFileSystemPaths); + conf.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, version); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java index 7ae82f8..9381746 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestBucketRequest.java @@ -74,6 +74,7 @@ public class TestBucketRequest { auditLogger = Mockito.mock(AuditLogger.class); when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); + when(ozoneManager.getOMLayoutVersion()).thenReturn(null); } @After diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java index 06e140b..3615ce4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequest.java @@ -186,7 +186,7 @@ public class TestOMBucketCreateRequest extends TestBucketRequest { } - private void verifyRequest(OMRequest modifiedOmRequest, + protected void verifyRequest(OMRequest modifiedOmRequest, OMRequest originalRequest) { OzoneManagerProtocolProtos.BucketInfo original = originalRequest.getCreateBucketRequest().getBucketInfo(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestV1.java new file mode 100644 index 0000000..a3b6ff7 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/bucket/TestOMBucketCreateRequestV1.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.bucket; + +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.StorageTypeProto; +import org.junit.Assert; +import org.junit.Test; + +import java.util.UUID; + +import static org.mockito.Mockito.when; + +/** + * Tests OMBucketCreateRequest class, which handles CreateBucket request. + */ +public class TestOMBucketCreateRequestV1 extends TestOMBucketCreateRequest { + + @Test + public void testValidateAndUpdateCacheWithFSOBucket() throws Exception { + when(ozoneManager.getOMLayoutVersion()).thenReturn( + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + OMBucketCreateRequest omBucketCreateRequest = doPreExecute(volumeName, + bucketName); + + doValidateAndUpdateCache(volumeName, bucketName, + omBucketCreateRequest.getOmRequest()); + } + + private OMBucketCreateRequest doPreExecute(String volumeName, + String bucketName) throws Exception { + addCreateVolumeToTable(volumeName, omMetadataManager); + OMRequest originalRequest = + TestOMRequestUtils.createBucketRequestV1(bucketName, volumeName, + false, StorageTypeProto.SSD); + + OMBucketCreateRequest omBucketCreateRequest = + new OMBucketCreateRequest(originalRequest); + + OMRequest modifiedRequest = omBucketCreateRequest.preExecute(ozoneManager); + verifyRequest(modifiedRequest, originalRequest); + return new OMBucketCreateRequest(modifiedRequest); + } + + private void doValidateAndUpdateCache(String volumeName, String bucketName, + OMRequest modifiedRequest) throws Exception { + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + + // As we have not still called validateAndUpdateCache, get() should + // return null. + + Assert.assertNull(omMetadataManager.getBucketTable().get(bucketKey)); + OMBucketCreateRequest omBucketCreateRequest = + new OMBucketCreateRequest(modifiedRequest); + + + OMClientResponse omClientResponse = + omBucketCreateRequest.validateAndUpdateCache(ozoneManager, 1, + ozoneManagerDoubleBufferHelper); + + // As now after validateAndUpdateCache it should add entry to cache, get + // should return non null value. + OmBucketInfo dbBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + Assert.assertNotNull(omMetadataManager.getBucketTable().get(bucketKey)); + + // verify table data with actual request data. + OmBucketInfo bucketInfoFromProto = OmBucketInfo.getFromProtobuf( + modifiedRequest.getCreateBucketRequest().getBucketInfo()); + + Assert.assertEquals(bucketInfoFromProto.getCreationTime(), + dbBucketInfo.getCreationTime()); + Assert.assertEquals(bucketInfoFromProto.getModificationTime(), + dbBucketInfo.getModificationTime()); + Assert.assertEquals(bucketInfoFromProto.getAcls(), + dbBucketInfo.getAcls()); + Assert.assertEquals(bucketInfoFromProto.getIsVersionEnabled(), + dbBucketInfo.getIsVersionEnabled()); + Assert.assertEquals(bucketInfoFromProto.getStorageType(), + dbBucketInfo.getStorageType()); + Assert.assertEquals(bucketInfoFromProto.getMetadata(), + dbBucketInfo.getMetadata()); + Assert.assertEquals(bucketInfoFromProto.getEncryptionKeyInfo(), + dbBucketInfo.getEncryptionKeyInfo()); + + // verify OMResponse. + verifySuccessCreateBucketResponse(omClientResponse.getOMResponse()); + + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java index f0f0320..454cfbb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java @@ -88,7 +88,8 @@ public class TestOMDirectoryCreateRequestV1 { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.newFolder().getAbsolutePath()); - ozoneConfiguration.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + TestOMRequestUtils.configureFSOptimizedPaths(ozoneConfiguration, + true, OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1); omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java index 046ac90..2631c91 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java @@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.om.request.file; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -183,11 +182,10 @@ public class TestOMFileCreateRequestV1 extends TestOMFileCreateRequest { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java index 4e74979..5d21226 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java @@ -23,7 +23,6 @@ package org.apache.hadoop.ozone.om.request.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -43,11 +42,10 @@ public class TestOMAllocateBlockRequestV1 extends TestOMAllocateBlockRequest { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java index ed1e2bd..379dbcf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java @@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.om.request.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -87,11 +86,10 @@ public class TestOMKeyCommitRequestV1 extends TestOMKeyCommitRequest { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java index 83c640d..b65443d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java @@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.om.request.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -47,11 +46,10 @@ public class TestOMKeyCreateRequestV1 extends TestOMKeyCreateRequest { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java index 7527e78..2c43d51 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java @@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.om.request.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; @@ -61,11 +60,10 @@ public class TestOMKeyDeleteRequestV1 extends TestOMKeyDeleteRequest { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java index bc4345e..e1549e1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om.response.file; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -68,11 +67,10 @@ public class TestOMFileCreateResponseV1 extends TestOMKeyCreateResponse { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java index 92b3efe..1626079 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -73,11 +72,10 @@ public class TestOMAllocateBlockResponseV1 @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java index 4d68a4b..7275f69 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java @@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -95,11 +94,10 @@ public class TestOMKeyCommitResponseV1 extends TestOMKeyCommitResponse { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java index e51a06b..6299639 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -38,11 +37,10 @@ public class TestOMKeyCreateResponseV1 extends TestOMKeyCreateResponse { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java index d35c79e..d46fe72 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java @@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.om.response.key; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; @@ -76,11 +75,10 @@ public class TestOMKeyDeleteResponseV1 extends TestOMKeyDeleteResponse { @Override protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); - config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() // and its not invoked in this test. Hence it is explicitly setting // this configuration to populate prefix tables. - OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + OzoneManagerRatisUtils.setBucketFSOptimized(true); return config; } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index b4153f0..150108c 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -547,7 +547,18 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter { } @Override - public String getBucketLayoutVersion() { - return bucket.getMetadata().get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION); + public boolean isFSOptimizedBucket() { + // layout version V1 represents optimized FS path + boolean layoutVersionEnabled = + StringUtils.equalsIgnoreCase( + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1, + bucket.getMetadata() + .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION)); + + boolean fsEnabled = + Boolean.parseBoolean(bucket.getMetadata() + .get(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS)); + + return layoutVersionEnabled && fsEnabled; } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index 2f7b589..d2b9a6d 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -327,9 +326,7 @@ public class BasicOzoneFileSystem extends FileSystem { return false; } - String layOutVersion = adapter.getBucketLayoutVersion(); - if (layOutVersion != null && - OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1.equals(layOutVersion)) { + if (adapter.isFSOptimizedBucket()) { return renameV1(srcPath, dstPath); } @@ -509,10 +506,7 @@ public class BasicOzoneFileSystem extends FileSystem { statistics.incrementWriteOps(1); LOG.debug("Delete path {} - recursive {}", f, recursive); - String layOutVersion = adapter.getBucketLayoutVersion(); - if (layOutVersion != null && - OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1.equals(layOutVersion)) { - + if (adapter.isFSOptimizedBucket()) { if (f.isRoot()) { LOG.warn("Cannot delete root directory."); return false; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 84cba47..6fdac47 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -59,7 +59,6 @@ import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -1049,8 +1048,8 @@ public class BasicRootedOzoneClientAdapterImpl } @Override - public String getBucketLayoutVersion() { + public boolean isFSOptimizedBucket() { // TODO: Need to refine this part. - return OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT; + return false; } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index 4a4d91b..0258f69 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -77,5 +77,5 @@ public interface OzoneClientAdapter { FileStatusAdapter getFileStatus(String key, URI uri, Path qualifiedPath, String userName) throws IOException; - String getBucketLayoutVersion(); + boolean isFSOptimizedBucket(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
