Repository: hadoop Updated Branches: refs/heads/ozone-0.3 51a79cc71 -> 106be1416
HDDS-651. Rename o3 to o3fs for Filesystem. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/106be141 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/106be141 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/106be141 Branch: refs/heads/ozone-0.3 Commit: 106be1416dfe6e6bbf90d3653edca5eb2ef54d44 Parents: 51a79cc Author: Jitendra Pandey <jiten...@apache.org> Authored: Wed Oct 17 14:18:46 2018 -0700 Committer: Jitendra Pandey <jiten...@apache.org> Committed: Wed Oct 17 14:32:22 2018 -0700 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 13 ----- .../conf/TestCommonConfigurationFields.java | 2 +- .../org/apache/hadoop/ozone/OzoneConsts.java | 5 +- .../dist/src/main/compose/ozonefs/docker-config | 2 +- .../src/main/smoketest/ozonefs/ozonefs.robot | 52 ++++++++++---------- hadoop-ozone/docs/content/OzoneFS.md | 6 +-- .../hadoop/ozone/web/ozShell/Handler.java | 6 +-- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +- .../fs/ozone/TestOzoneFileInterfaces.java | 2 +- 9 files changed, 40 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index f3167f2..c51030b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1772,19 +1772,6 @@ </property> -<!-- Ozone file system properties --> -<property> - <name>fs.o3.impl</name> - <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value> - <description>The implementation class of the Ozone FileSystem.</description> -</property> - -<property> - <name>fs.AbstractFileSystem.o3.impl</name> - <value>org.apache.hadoop.fs.ozone.OzFs</value> - <description>The implementation class of the OzFs AbstractFileSystem.</description> -</property> - <!-- ipc properties --> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index e10617d..50af230 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -102,7 +102,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPrefixToSkipCompare.add("fs.s3a."); // O3 properties are in a different subtree. - xmlPrefixToSkipCompare.add("fs.o3."); + xmlPrefixToSkipCompare.add("fs.o3fs."); //ftp properties are in a different subtree. // - org.apache.hadoop.fs.ftp.FTPFileSystem. http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 8ccc648..b77d621 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -63,7 +63,10 @@ public final class OzoneConsts { public static final String OZONE_USER = "user"; public static final String OZONE_REQUEST = "request"; - public static final String OZONE_URI_SCHEME = "o3"; + // Ozone File System scheme + public static final String OZONE_URI_SCHEME = "o3fs"; + + public static final String OZONE_RPC_SCHEME = "o3"; public static final String OZONE_HTTP_SCHEME = "http"; public static final String OZONE_URI_DELIMITER = "/"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config index 675dcba..5061afa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem +CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem OZONE-SITE.XML_ozone.om.address=ozoneManager OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874 OZONE-SITE.XML_ozone.scm.names=scm http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot index 3720d01..fd34e35 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot @@ -30,83 +30,83 @@ Create volume and bucket Execute ozone sh bucket create http://ozoneManager/fstest2/bucket3 Check volume from ozonefs - ${result} = Execute ozone fs -ls o3://bucket1.fstest/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ Run ozoneFS tests - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} testdir/deep - Execute ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/testdir/deep/ + Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} NOTICE.txt - Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir/deep/PUTFILE.txt + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} PUTFILE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/ Should contain ${result} NOTICE.txt Should contain ${result} PUTFILE.txt - Execute ozone fs -mv o3://bucket1.fstest/testdir/deep/NOTICE.txt o3://bucket1.fstest/testdir/deep/MOVED.TXT + Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} MOVED.TXT Should not contain ${result} NOTICE.txt - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep/subdir1 - Execute ozone fs -cp o3://bucket1.fstest/testdir/deep/MOVED.TXT o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} subdir1/NOTICE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/ Should contain ${result} NOTICE.txt - Execute ozone fs -cat o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt Should not contain ${result} Failed - Execute ozone fs -rm o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} NOTICE.txt - ${result} = Execute ozone fs -rmdir o3://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} subdir1 - Execute ozone fs -touch o3://bucket1.fstest/testdir/TOUCHFILE.txt + Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should contain ${result} TOUCHFILE.txt - Execute ozone fs -rm -r o3://bucket1.fstest/testdir/ + Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' Should not contain ${result} testdir Execute rm -Rf localdir1 Execute mkdir localdir1 Execute cp NOTICE.txt localdir1/LOCAL.txt - Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir1 - Execute ozone fs -copyFromLocal localdir1 o3://bucket1.fstest/testdir1/ - Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir1/NOTICE.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1 + Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/ + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt - ${result} = Execute ozone fs -ls -R o3://bucket1.fstest/testdir1/ + ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/ Should contain ${result} localdir1/LOCAL.txt Should contain ${result} testdir1/NOTICE.txt - Execute ozone fs -mkdir -p o3://bucket2.fstest/testdir2 - Execute ozone fs -mkdir -p o3://bucket3.fstest2/testdir3 + Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2 + Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3 - Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket2.fstest/testdir2/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/ - Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket3.fstest2/testdir3/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/ Execute ozone sh key put o3://ozoneManager/fstest/bucket1/KEY.txt NOTICE.txt - ${result} = Execute ozone fs -ls o3://bucket1.fstest/KEY.txt + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt Should contain ${result} KEY.txt - ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/KEY.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt Should Be Equal As Integers ${rc} 1 Should contain ${result} File exists Execute rm -Rf GET.txt - Execute ozone fs -get o3://bucket1.fstest/KEY.txt GET.txt + Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt Execute ls -l GET.txt - ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3://abcde.pqrs/ + ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/ Should Be Equal As Integers ${rc} 1 Should contain ${result} VOLUME_NOT_FOUND http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/docs/content/OzoneFS.md ---------------------------------------------------------------------- diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md index d0621be..e19c9b9 100644 --- a/hadoop-ozone/docs/content/OzoneFS.md +++ b/hadoop-ozone/docs/content/OzoneFS.md @@ -42,16 +42,16 @@ Please add the following entry to the core-site.xml. {{< highlight xml >}} <property> - <name>fs.o3.impl</name> + <name>fs.o3fs.impl</name> <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value> </property> <property> <name>fs.default.name</name> - <value>o3://localhost:9864/volume/bucket</value> + <value>o3fs://localhost:9864/volume/bucket</value> </property> {{< /highlight >}} -This will make this bucket to be the default file system for HDFS dfs commands and register the o3 file system type.. +This will make this bucket to be the default file system for HDFS dfs commands and register the o3fs file system type.. You also need to add the ozone-filesystem.jar file to the classpath: http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java index a9550c2..5067855 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java @@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.rest.OzoneException; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +81,7 @@ public abstract class Handler implements Callable<Void> { Configuration conf = new OzoneConfiguration(); String scheme = ozoneURI.getScheme(); if (ozoneURI.getScheme() == null || scheme.isEmpty()) { - scheme = OZONE_URI_SCHEME; + scheme = OZONE_RPC_SCHEME; } if (scheme.equals(OZONE_HTTP_SCHEME)) { if (ozoneURI.getHost() != null) { @@ -94,7 +94,7 @@ public abstract class Handler implements Callable<Void> { } else { client = OzoneClientFactory.getRestClient(conf); } - } else if (scheme.equals(OZONE_URI_SCHEME)) { + } else if (scheme.equals(OZONE_RPC_SCHEME)) { if (ozoneURI.getHost() != null) { if (ozoneURI.getPort() == -1) { client = OzoneClientFactory.getRpcClient(ozoneURI.getHost()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index 0b721fb..d52f3b7 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -107,7 +107,7 @@ public class OzoneFileSystem extends FileSystem { if (!matcher.matches()) { throw new IllegalArgumentException("Ozone file system url should be " - + "in the form o3://bucket.volume"); + + "in the form o3fs://bucket.volume"); } String bucketStr = matcher.group(1); String volumeStr = matcher.group(2); http://git-wip-us.apache.org/repos/asf/hadoop/blob/106be141/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java index 2176031..0eb8dce 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -231,7 +231,7 @@ public class TestOzoneFileInterfaces { ozoneFs.pathToKey(new Path("key1/key2"))); assertEquals("key1/key2", - ozoneFs.pathToKey(new Path("o3://test1/key1/key2"))); + ozoneFs.pathToKey(new Path("o3fs://test1/key1/key2"))); } private String getCurrentUser() { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
