This is an automated email from the ASF dual-hosted git repository. umamahesh pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new f85ce25 HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G. f85ce25 is described below commit f85ce2570e796e0ca838535340cd3c049634a225 Author: Uma Maheswara Rao G <umamah...@apache.org> AuthorDate: Sat Jun 6 08:11:57 2020 -0700 HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G. (cherry picked from commit 3ca15292c5584ec220b3eeaf76da85d228bcbd8b) --- .../src/main/resources/core-default.xml | 110 +++++++++++++++++++++ .../hadoop/conf/TestCommonConfigurationFields.java | 18 ++++ ...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 9 +- 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 0d583cc..accb1b9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -953,6 +953,116 @@ </property> <property> + <name>fs.viewfs.overload.scheme.target.hdfs.impl</name> + <value>org.apache.hadoop.hdfs.DistributedFileSystem</value> + <description>The DistributedFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are hdfs. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.s3a.impl</name> + <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value> + <description>The S3AFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are s3a.</description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.o3fs.impl</name> + <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value> + <description>The OzoneFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are o3fs.</description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.ftp.impl</name> + <value>org.apache.hadoop.fs.ftp.FTPFileSystem</value> + <description>The FTPFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are ftp. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.webhdfs.impl</name> + <value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value> + <description>The WebHdfsFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are webhdfs. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.swebhdfs.impl</name> + <value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value> + <description>The SWebHdfsFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are swebhdfs. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.file.impl</name> + <value>org.apache.hadoop.fs.LocalFileSystem</value> + <description>The LocalFileSystem for view file system overload scheme when + child file system and ViewFSOverloadScheme's schemes are file. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.abfs.impl</name> + <value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value> + <description>The AzureBlobFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are abfs. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.abfss.impl</name> + <value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value> + <description>The SecureAzureBlobFileSystem for view file system overload + scheme when child file system and ViewFSOverloadScheme's schemes are abfss. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.wasb.impl</name> + <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value> + <description>The NativeAzureFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are wasb. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.swift.impl</name> + <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value> + <description>The SwiftNativeFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are swift. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.oss.impl</name> + <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value> + <description>The AliyunOSSFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are oss. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.http.impl</name> + <value>org.apache.hadoop.fs.http.HttpFileSystem</value> + <description>The HttpFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are http. + </description> +</property> + +<property> + <name>fs.viewfs.overload.scheme.target.https.impl</name> + <value>org.apache.hadoop.fs.http.HttpsFileSystem</value> + <description>The HttpsFileSystem for view file system overload scheme + when child file system and ViewFSOverloadScheme's schemes are https. + </description> +</property> + +<property> <name>fs.AbstractFileSystem.ftp.impl</name> <value>org.apache.hadoop.fs.ftp.FtpFs</value> <description>The FileSystem for Ftp: uris.</description> diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 1ce23a0..3b9947e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -124,6 +124,24 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); + // ViewfsOverloadScheme target fs impl property keys are dynamically + // constructed and they are advanced props. + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfs.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.o3fs.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.oss.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl"); + xmlPropsToSkipCompare. + add("fs.viewfs.overload.scheme.target.swebhdfs.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl"); + xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl"); + // Azure properties are in a different class // - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore // - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java index 6e101a6..f0f3aae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java @@ -382,13 +382,17 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme { * Note: Above links created because to make fs initialization success. * Otherwise will not proceed if no mount links. * - * Don't set fs.viewfs.overload.scheme.target.hdfs.impl property. + * Unset fs.viewfs.overload.scheme.target.hdfs.impl property. * So, OverloadScheme target fs initialization will fail. */ @Test(expected = UnsupportedFileSystemException.class, timeout = 30000) public void testInvalidOverloadSchemeTargetFS() throws Exception { final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER); + String mountTableIfSet = conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH); conf = new Configuration(); + if (mountTableIfSet != null) { + conf.set(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH, mountTableIfSet); + } addMountLinks(defaultFSURI.getAuthority(), new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER, Constants.CONFIG_VIEWFS_LINK_FALLBACK }, @@ -400,6 +404,9 @@ public class TestViewFileSystemOverloadSchemeWithHdfsScheme { defaultFSURI.toString()); conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME), ViewFileSystemOverloadScheme.class.getName()); + conf.unset(String.format( + FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN, + HDFS_SCHEME)); try (FileSystem fs = FileSystem.get(conf)) { fs.createNewFile(new Path("/onRootWhenFallBack")); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org