This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new cee43e9 HDDS-3878. Make OMHA serviceID optional if one (but only one) is defined in the config (#1149) cee43e9 is described below commit cee43e908f700633a283f8791687523341a8ee27 Author: Elek, Márton <e...@users.noreply.github.com> AuthorDate: Tue Aug 11 15:49:44 2020 +0200 HDDS-3878. Make OMHA serviceID optional if one (but only one) is defined in the config (#1149) --- .../hadoop/hdds/conf/InMemoryConfiguration.java | 58 +++++++ .../hadoop/ozone/client/OzoneClientFactory.java | 8 +- hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 19 +++ .../src/main/compose/ozone-ha/docker-compose.yaml | 93 +++++++++++ .../dist/src/main/compose/ozone-ha/docker-config | 35 +++++ .../dist/src/main/compose/ozone-ha/test.sh | 33 ++++ .../{ozone-shell.robot => ozone-shell-lib.robot} | 25 +-- .../main/smoketest/basic/ozone-shell-single.robot | 27 ++++ .../src/main/smoketest/basic/ozone-shell.robot | 119 +------------- .../hadoop/ozone/shell/TestOzoneShellHA.java | 45 +++--- .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 ++ .../apache/hadoop/ozone/shell/OzoneAddress.java | 103 ++++++++---- .../hadoop/ozone/shell/TestOzoneAddress.java | 6 +- .../shell/TestOzoneAddressClientCreation.java | 172 +++++++++++++++++++++ 14 files changed, 552 insertions(+), 203 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/InMemoryConfiguration.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/InMemoryConfiguration.java new file mode 100644 index 0000000..0bea7af --- /dev/null +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/InMemoryConfiguration.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * In memory, mutable configuration source for testing. + */ +public class InMemoryConfiguration implements MutableConfigurationSource { + + private Map<String, String> configs = new HashMap<>(); + + public InMemoryConfiguration() { + } + + public InMemoryConfiguration(String key, String value) { + set(key, value); + } + + @Override + public String get(String key) { + return configs.get(key); + } + + @Override + public Collection<String> getConfigKeys() { + return configs.keySet(); + } + + @Override + public char[] getPassword(String key) throws IOException { + return configs.get(key).toCharArray(); + } + + @Override + public void set(String key, String value) { + configs.put(key, value); + } +} diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java index 2f7b107..9bf3973 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java @@ -137,13 +137,17 @@ public final class OzoneClientFactory { // configuration, we don't fall back to default ozone.om.address defined // in ozone-default.xml. - if (OmUtils.isServiceIdsDefined(config)) { + String[] serviceIds = config.getTrimmedStrings(OZONE_OM_SERVICE_IDS_KEY); + if (serviceIds.length > 1) { throw new IOException("Following ServiceID's " + config.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY) + " are" + " defined in the configuration. Use the method getRpcClient which " + "takes serviceID and configuration as param"); + } else if (serviceIds.length == 1) { + return getRpcClient(getClientProtocol(config, serviceIds[0]), config); + } else { + return getRpcClient(getClientProtocol(config), config); } - return getRpcClient(getClientProtocol(config), config); } /** diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/.env b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env new file mode 100644 index 0000000..8446b4a --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/.env @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} +OZONE_RUNNER_VERSION=${docker.ozone-runner.version} +HADOOP_OPTS= \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml new file mode 100644 index 0000000..b9f4b60 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields) +x-common-config: + &common-config + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - docker-config + +x-replication: + &replication + OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1} + +services: + datanode: + <<: *common-config + ports: + - 9864 + - 9882 + environment: + <<: *replication + command: ["ozone","datanode"] + om1: + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874 + - 9862 + hostname: om1 + command: ["ozone","om"] + om2: + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874 + - 9862 + hostname: om2 + command: ["ozone","om"] + om3: + <<: *common-config + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + <<: *replication + ports: + - 9874 + - 9862 + hostname: om3 + command: ["ozone","om"] + scm: + <<: *common-config + ports: + - 9876:9876 + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1} + <<: *replication + command: ["ozone","scm"] + s3g: + <<: *common-config + environment: + <<: *replication + ports: + - 9878:9878 + command: ["ozone","s3g"] + recon: + <<: *common-config + ports: + - 9888:9888 + environment: + <<: *replication + command: ["ozone","recon"] diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config new file mode 100644 index 0000000..d378a67 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice + +OZONE-SITE.XML_ozone.om.service.ids=omservice +OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 +OZONE-SITE.XML_ozone.om.address.omservice.om1=om1 +OZONE-SITE.XML_ozone.om.address.omservice.om2=om2 +OZONE-SITE.XML_ozone.om.address.omservice.om3=om3 +OZONE-SITE.XML_ozone.om.ratis.enable=true + +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.client.failover.max.attempts=6 +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds + +no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh new file mode 100755 index 0000000..a14aa9c --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +export SECURITY_ENABLED=false +export OZONE_REPLICATION_FACTOR=3 + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +execute_robot_test scm basic/ozone-shell-single.robot + +stop_docker_env + +generate_report diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot similarity index 93% copy from hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot copy to hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot index 9143f38..ba00061 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot @@ -17,39 +17,16 @@ Documentation Test ozone shell CLI usage Library OperatingSystem Resource ../commonlib.robot -Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab -Test Timeout 2 minute -Suite Setup Generate prefix *** Variables *** ${prefix} generated *** Keywords *** + Generate prefix ${random} = Generate Random String 5 [NUMBERS] Set Suite Variable ${prefix} ${random} -*** Test Cases *** -RpcClient with port - Test ozone shell o3:// om:9862 ${prefix}-rpcwoport - -RpcClient volume acls - Test Volume Acls o3:// om:9862 ${prefix}-rpcwoport2 - -RpcClient bucket acls - Test Bucket Acls o3:// om:9862 ${prefix}-rpcwoport2 - -RpcClient key acls - Test Key Acls o3:// om:9862 ${prefix}-rpcwoport2 - -RpcClient without host - Test ozone shell o3:// ${EMPTY} ${prefix}-rpcwport - -RpcClient without scheme - Test ozone shell ${EMPTY} ${EMPTY} ${prefix}-rpcwoscheme - - -*** Keywords *** Test ozone shell [arguments] ${protocol} ${server} ${volume} ${result} = Execute And Ignore Error ozone sh volume info ${protocol}${server}/${volume} diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-single.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-single.robot new file mode 100644 index 0000000..e08ee09 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-single.robot @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone shell CLI usage +Library OperatingSystem +Resource ../commonlib.robot +Resource ozone-shell-lib.robot +Test Timeout 2 minute +Suite Setup Generate prefix + +*** Test Cases *** + +Test ozone shell + Test ozone shell ${EMPTY} ${EMPTY} ${prefix}-rpcbasic diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot index 9143f38..45b2d35 100644 --- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot +++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot @@ -17,18 +17,11 @@ Documentation Test ozone shell CLI usage Library OperatingSystem Resource ../commonlib.robot +Resource ozone-shell-lib.robot Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab Test Timeout 2 minute Suite Setup Generate prefix -*** Variables *** -${prefix} generated - -*** Keywords *** -Generate prefix - ${random} = Generate Random String 5 [NUMBERS] - Set Suite Variable ${prefix} ${random} - *** Test Cases *** RpcClient with port Test ozone shell o3:// om:9862 ${prefix}-rpcwoport @@ -47,113 +40,3 @@ RpcClient without host RpcClient without scheme Test ozone shell ${EMPTY} ${EMPTY} ${prefix}-rpcwoscheme - - -*** Keywords *** -Test ozone shell - [arguments] ${protocol} ${server} ${volume} - ${result} = Execute And Ignore Error ozone sh volume info ${protocol}${server}/${volume} - Should contain ${result} VOLUME_NOT_FOUND - ${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --quota 100TB - Should not contain ${result} Failed - ${result} = Execute ozone sh volume list ${protocol}${server}/ | jq -r '. | select(.name=="${volume}")' - Should contain ${result} creationTime - ${result} = Execute ozone sh volume list | jq -r '. | select(.name=="${volume}")' - Should contain ${result} creationTime -# TODO: Disable updating the owner, acls should be used to give access to other user. - Execute ozone sh volume update ${protocol}${server}/${volume} --quota 10TB -# ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | jq -r '. | select(.volumeName=="${volume}") | .owner | .name' -# Should Be Equal ${result} bill - ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | jq -r '. | select(.name=="${volume}") | .quota' - Should Be Equal ${result} 10995116277760 - Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 - ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="bb1") | .storageType' - Should Be Equal ${result} DISK - ${result} = Execute ozone sh bucket list ${protocol}${server}/${volume}/ | jq -r '. | select(.name=="bb1") | .volumeName' - Should Be Equal ${result} ${volume} - Run Keyword Test key handling ${protocol} ${server} ${volume} - Execute ozone sh bucket delete ${protocol}${server}/${volume}/bb1 - Execute ozone sh volume delete ${protocol}${server}/${volume} - -Test Volume Acls - [arguments] ${protocol} ${server} ${volume} - Execute ozone sh volume create ${protocol}${server}/${volume} - ${result} = Execute ozone sh volume getacl ${protocol}${server}/${volume} - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \".*\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . - ${result} = Execute ozone sh volume addacl ${protocol}${server}/${volume} -a user:superuser1:rwxy[DEFAULT] - ${result} = Execute ozone sh volume getacl ${protocol}${server}/${volume} - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1*\",\n.*\"aclScope\" : \"DEFAULT\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" . - ${result} = Execute ozone sh volume removeacl ${protocol}${server}/${volume} -a user:superuser1:xy - ${result} = Execute ozone sh volume getacl ${protocol}${server}/${volume} - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"DEFAULT\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" . - ${result} = Execute ozone sh volume setacl ${protocol}${server}/${volume} -al user:superuser1:rwxy,group:superuser1:a,user:testuser/s...@example.com:rwxyc,group:superuser1:a[DEFAULT] - ${result} = Execute ozone sh volume getacl ${protocol}${server}/${volume} - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1*\",\n.*\"aclScope\" : \"DEFAULT\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" . - Should Match Regexp ${result} \"type\" : \"GROUP\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"DEFAULT\",\n.*\"aclList\" : . \"ALL\" . - -Test Bucket Acls - [arguments] ${protocol} ${server} ${volume} - Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1 - ${result} = Execute ozone sh bucket getacl ${protocol}${server}/${volume}/bb1 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \".*\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . - ${result} = Execute ozone sh bucket addacl ${protocol}${server}/${volume}/bb1 -a user:superuser1:rwxy - ${result} = Execute ozone sh bucket getacl ${protocol}${server}/${volume}/bb1 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1*\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" - ${result} = Execute ozone sh bucket removeacl ${protocol}${server}/${volume}/bb1 -a user:superuser1:xy - ${result} = Execute ozone sh bucket getacl ${protocol}${server}/${volume}/bb1 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\" - ${result} = Execute ozone sh bucket setacl ${protocol}${server}/${volume}/bb1 -al user:superuser1:rwxy,group:superuser1:a,user:testuser/s...@example.com:rwxyc,group:superuser1:a[DEFAULT] - ${result} = Execute ozone sh bucket getacl ${protocol}${server}/${volume}/bb1 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1*\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" - Should Match Regexp ${result} \"type\" : \"GROUP\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"DEFAULT\",\n.*\"aclList\" : . \"ALL\" . - - -Test key handling - [arguments] ${protocol} ${server} ${volume} - Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt - Execute rm -f /tmp/NOTICE.txt.1 - Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1 - Execute diff -q /opt/hadoop/NOTICE.txt /tmp/NOTICE.txt.1 - - Execute ozone sh key put -t RATIS ${protocol}${server}/${volume}/bb1/key1_RATIS /opt/hadoop/NOTICE.txt - Execute rm -f /tmp/key1_RATIS - Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1_RATIS /tmp/key1_RATIS - Execute diff -q /opt/hadoop/NOTICE.txt /tmp/key1_RATIS - ${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1_RATIS | jq -r '. | select(.name=="key1_RATIS")' - Should contain ${result} RATIS - Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1_RATIS - - Execute ozone sh key cp ${protocol}${server}/${volume}/bb1 key1 key1-copy - Execute rm -f /tmp/key1-copy - Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1-copy /tmp/key1-copy - Execute diff -q /opt/hadoop/NOTICE.txt /tmp/key1-copy - Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1-copy - - ${result} = Execute And Ignore Error ozone sh key get ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1 - Should Contain ${result} NOTICE.txt.1 exists - ${result} = Execute ozone sh key get --force ${protocol}${server}/${volume}/bb1/key1 /tmp/NOTICE.txt.1 - Should Not Contain ${result} NOTICE.txt.1 exists - ${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | jq -r '. | select(.name=="key1")' - Should contain ${result} creationTime - ${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="key1") | .name' - Should Be Equal ${result} key1 - Execute ozone sh key rename ${protocol}${server}/${volume}/bb1 key1 key2 - ${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | jq -r '.name' - Should Be Equal ${result} key2 - Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key2 - -Test key Acls - [arguments] ${protocol} ${server} ${volume} - Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key2 /opt/hadoop/NOTICE.txt - ${result} = Execute ozone sh key getacl ${protocol}${server}/${volume}/bb1/key2 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \".*\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . - ${result} = Execute ozone sh key addacl ${protocol}${server}/${volume}/bb1/key2 -a user:superuser1:rwxy - ${result} = Execute ozone sh key getacl ${protocol}${server}/${volume}/bb1/key2 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" - ${result} = Execute ozone sh key removeacl ${protocol}${server}/${volume}/bb1/key2 -a user:superuser1:xy - ${result} = Execute ozone sh key getacl ${protocol}${server}/${volume}/bb1/key2 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\" - ${result} = Execute ozone sh key setacl ${protocol}${server}/${volume}/bb1/key2 -al user:superuser1:rwxy,group:superuser1:a,user:testuser/s...@example.com:rwxyc - ${result} = Execute ozone sh key getacl ${protocol}${server}/${volume}/bb1/key2 - Should Match Regexp ${result} \"type\" : \"USER\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"READ\", \"WRITE\", \"READ_ACL\", \"WRITE_ACL\" - Should Match Regexp ${result} \"type\" : \"GROUP\",\n.*\"name\" : \"superuser1\",\n.*\"aclScope\" : \"ACCESS\",\n.*\"aclList\" : . \"ALL\" . diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 17baa06..45258db 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -17,7 +17,14 @@ */ package org.apache.hadoop.ozone.shell; -import com.google.common.base.Strings; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintStream; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -33,9 +40,15 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ToolRunner; + +import com.google.common.base.Strings; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; +import static org.junit.Assert.fail; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Ignore; @@ -51,19 +64,6 @@ import picocli.CommandLine.ParameterException; import picocli.CommandLine.ParseResult; import picocli.CommandLine.RunLast; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; -import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; -import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME; -import static org.junit.Assert.fail; - /** * This class tests Ozone sh shell command. * Inspired by TestS3Shell @@ -346,11 +346,7 @@ public class TestOzoneShellHA { */ @Test public void testOzoneShCmdURIs() { - // Test case 1: ozone sh volume create /volume - // Expectation: Failure. - String[] args = new String[] {"volume", "create", "/volume"}; - executeWithError(ozoneShell, args, - "Service ID or host name must not be omitted"); + // Get leader OM node RPC address from ozone.om.address.omServiceId.omNode String omLeaderNodeId = getLeaderOMNodeId(); @@ -367,7 +363,7 @@ public class TestOzoneShellHA { // TODO: Fix this behavior, then uncomment the execute() below. String setOmAddress = "--set=" + OMConfigKeys.OZONE_OM_ADDRESS_KEY + "=" + omLeaderNodeAddr; - args = new String[] {setOmAddress, + String[] args = new String[] {setOmAddress, "volume", "create", "o3://" + omLeaderNodeAddrWithoutPort + "/volume2"}; //execute(ozoneShell, args); @@ -389,15 +385,14 @@ public class TestOzoneShellHA { executeWithError(ozoneShell, args, "does not use port information"); // Test case 6: ozone sh bucket create /volume/bucket - // Expectation: Failure. - args = new String[] {"bucket", "create", "/volume/bucket"}; - executeWithError(ozoneShell, args, - "Service ID or host name must not be omitted"); + // Expectation: Success. + args = new String[] {"bucket", "create", "/volume/bucket-one"}; + execute(ozoneShell, args); // Test case 7: ozone sh bucket create o3://om1/volume/bucket // Expectation: Success. args = new String[] { - "bucket", "create", "o3://" + omServiceId + "/volume/bucket"}; + "bucket", "create", "o3://" + omServiceId + "/volume/bucket-two"}; execute(ozoneShell, args); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index b9b59ef..1cfff12 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -61,6 +61,7 @@ import io.opentracing.util.GlobalTracer; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.RandomStringUtils; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -320,6 +321,17 @@ public class BaseFreonGenerator { RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, ProtobufRpcEngine.class); String clientId = ClientId.randomId().toString(); + + if (omServiceID == null) { + + //if only one serviceId is configured, use that + final String[] configuredServiceIds = + conf.getTrimmedStrings(OZONE_OM_SERVICE_IDS_KEY); + if (configuredServiceIds.length == 1) { + omServiceID = configuredServiceIds[0]; + } + } + OmTransport transport = OmTransportFactory.create(conf, ugi, omServiceID); return new OzoneManagerProtocolClientSideTranslatorPB(transport, clientId); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java index 1d75328..0ddd657 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java @@ -21,7 +21,10 @@ import java.io.IOException; import java.io.PrintStream; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collection; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.client.OzoneClient; @@ -30,10 +33,10 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; +import com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; - import org.apache.http.client.utils.URIBuilder; /** @@ -42,6 +45,7 @@ import org.apache.http.client.utils.URIBuilder; public class OzoneAddress { private static final int DEFAULT_OZONE_PORT = 50070; + private static final String EMPTY_HOST = "___DEFAULT___"; private URI ozoneURI; @@ -85,7 +89,32 @@ public class OzoneAddress { } - public OzoneClient createClient(OzoneConfiguration conf) + @VisibleForTesting + protected OzoneClient createRpcClient(ConfigurationSource conf) + throws IOException { + return OzoneClientFactory.getRpcClient(conf); + } + + @VisibleForTesting + protected OzoneClient createRpcClientFromHostPort( + String host, + int port, + MutableConfigurationSource conf + ) + throws IOException { + return OzoneClientFactory.getRpcClient(ozoneURI.getHost(), port, conf); + } + + @VisibleForTesting + protected OzoneClient createRpcClientFromServiceId( + String serviceId, + MutableConfigurationSource conf + ) + throws IOException { + return OzoneClientFactory.getRpcClient(serviceId, conf); + } + + public OzoneClient createClient(MutableConfigurationSource conf) throws IOException, OzoneClientException { OzoneClient client; String scheme = ozoneURI.getScheme(); @@ -96,50 +125,62 @@ public class OzoneAddress { throw new UnsupportedOperationException( "REST schema is not supported any more. Please use AWS S3 protocol " + "if you need REST interface."); - } else if (scheme.equals(OZONE_RPC_SCHEME)) { - if (ozoneURI.getHost() != null && !ozoneURI.getAuthority() - .equals(EMPTY_HOST)) { - if (OmUtils.isOmHAServiceId(conf, ozoneURI.getHost())) { - // When host is an HA service ID - if (ozoneURI.getPort() != -1) { - throw new OzoneClientException( - "Port " + ozoneURI.getPort() + " specified in URI but host '" - + ozoneURI.getHost() + "' is a logical (HA) OzoneManager " - + "and does not use port information."); - } - client = OzoneClientFactory.getRpcClient(ozoneURI.getHost(), conf); - } else if (ozoneURI.getPort() == -1) { - client = OzoneClientFactory.getRpcClient(ozoneURI.getHost(), - OmUtils.getOmRpcPort(conf), conf); - } else { - client = OzoneClientFactory - .getRpcClient(ozoneURI.getHost(), ozoneURI.getPort(), conf); + } else if (!scheme.equals(OZONE_RPC_SCHEME)) { + throw new OzoneClientException( + "Invalid URI, unknown protocol scheme: " + scheme + ". Use " + + OZONE_RPC_SCHEME + ":// as the scheme"); + } + + if (ozoneURI.getHost() != null && !ozoneURI.getAuthority() + .equals(EMPTY_HOST)) { + if (OmUtils.isOmHAServiceId(conf, ozoneURI.getHost())) { + // When host is an HA service ID + if (ozoneURI.getPort() != -1) { + throw new OzoneClientException( + "Port " + ozoneURI.getPort() + " specified in URI but host '" + + ozoneURI.getHost() + "' is a logical (HA) OzoneManager " + + "and does not use port information."); } + client = createRpcClient(conf); + } else if (ozoneURI.getPort() == -1) { + client = createRpcClientFromHostPort(ozoneURI.getHost(), + OmUtils.getOmRpcPort(conf), conf); } else { - // When host is not specified - if (OmUtils.isServiceIdsDefined(conf)) { - throw new OzoneClientException("Service ID or host name must not" - + " be omitted when ozone.om.service.ids is defined."); - } - client = OzoneClientFactory.getRpcClient(conf); + client = createRpcClientFromHostPort(ozoneURI.getHost(), + ozoneURI.getPort(), conf); + } + } else {// When host is not specified + + Collection<String> omServiceIds = conf.getTrimmedStringCollection( + OZONE_OM_SERVICE_IDS_KEY); + + if (omServiceIds.size() > 1) { + throw new OzoneClientException("Service ID or host name must not" + + " be omitted when multiple ozone.om.service.ids is defined."); + } else if (omServiceIds.size() == 1) { + client = createRpcClientFromServiceId(omServiceIds.iterator().next(), + conf); + } else { + client = createRpcClient(conf); } - } else { - throw new OzoneClientException( - "Invalid URI, unknown protocol scheme: " + scheme); } + return client; } /** * Create OzoneClient for S3Commands. + * * @param conf * @param omServiceID * @return OzoneClient * @throws IOException * @throws OzoneClientException */ - public OzoneClient createClientForS3Commands(OzoneConfiguration conf, - String omServiceID) + public OzoneClient createClientForS3Commands( + OzoneConfiguration conf, + String omServiceID + ) throws IOException, OzoneClientException { if (omServiceID != null) { // OM HA cluster diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java index 3ab866c..10ba576 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddress.java @@ -39,9 +39,9 @@ public class TestOzoneAddress { @Parameters public static Collection<Object[]> data() { return Arrays.asList(new Object[][] { - {"o3fs://localhost:9878/"}, - {"o3fs://localhost/"}, - {"o3fs:///"}, + {"o3://localhost:9878/"}, + {"o3://localhost/"}, + {"o3:///"}, {"/"}, {""} }); diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddressClientCreation.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddressClientCreation.java new file mode 100644 index 0000000..1c58a7d --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneAddressClientCreation.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.shell; + +import java.io.IOException; + +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientException; +import org.apache.hadoop.hdds.conf.InMemoryConfiguration; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test ozone client creation. + */ +public class TestOzoneAddressClientCreation { + + @Test + public void implicitNonHA() throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("/vol1/bucket1/key1"); + address.createClient(new InMemoryConfiguration()); + Assert.assertTrue(address.simpleCreation); + } + + @Test + public void implicitHAOneServiceId() + throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("/vol1/bucket1/key1"); + address.createClient( + new InMemoryConfiguration(OZONE_OM_SERVICE_IDS_KEY, "service1")); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("service1", address.serviceId); + } + + @Test(expected = OzoneClientException.class) + public void implicitHaMultipleServiceId() + throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("/vol1/bucket1/key1"); + address.createClient( + new InMemoryConfiguration(OZONE_OM_SERVICE_IDS_KEY, + "service1,service2")); + } + + @Test + public void explicitNonHAHostPort() throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("o3://om:9862/vol1/bucket1/key1"); + address.createClient(new InMemoryConfiguration()); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("om", address.host); + Assert.assertEquals(9862, address.port); + } + + @Test + public void explicitHAHostPortWithServiceId() + throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("o3://om:9862/vol1/bucket1/key1"); + address.createClient( + new InMemoryConfiguration(OZONE_OM_SERVICE_IDS_KEY, "service1")); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("om", address.host); + Assert.assertEquals(9862, address.port); + } + + @Test + public void explicitAHostPortWithServiceIds() + throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("o3://om:9862/vol1/bucket1/key1"); + address.createClient( + new InMemoryConfiguration(OZONE_OM_SERVICE_IDS_KEY, + "service1,service2")); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("om", address.host); + Assert.assertEquals(9862, address.port); + } + + @Test + public void explicitNonHAHost() throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("o3://om/vol1/bucket1/key1"); + address.createClient( + new InMemoryConfiguration(OZONE_OM_SERVICE_IDS_KEY, "service1")); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("om", address.host); + } + + @Test + public void explicitHAHostPort() throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("o3://om:1234/vol1/bucket1/key1"); + address.createClient(new InMemoryConfiguration()); + Assert.assertFalse(address.simpleCreation); + Assert.assertEquals("om", address.host); + Assert.assertEquals(1234, address.port); + } + + @Test(expected = OzoneClientException.class) + public void explicitWrongScheme() throws OzoneClientException, IOException { + TestableOzoneAddress address = + new TestableOzoneAddress("ssh://host/vol1/bucket1/key1"); + address.createClient(new InMemoryConfiguration()); + } + + /** + * OzoneAddress with modification to make it easier to test. + */ + @SuppressWarnings("checkstyle") + private static class TestableOzoneAddress extends OzoneAddress { + + private String host; + private int port; + private boolean simpleCreation; + private String serviceId; + + TestableOzoneAddress(String address) throws OzoneClientException { + super(address); + } + + TestableOzoneAddress() throws OzoneClientException { + } + + @Override + protected OzoneClient createRpcClient(ConfigurationSource conf) + throws IOException { + simpleCreation = true; + return null; + } + + @Override + protected OzoneClient createRpcClientFromHostPort( + String hostParam, int portParam, MutableConfigurationSource conf + ) throws IOException { + this.host = hostParam; + this.port = portParam; + return null; + } + + @Override + protected OzoneClient createRpcClientFromServiceId( + String serviceIdParam, MutableConfigurationSource conf + ) throws IOException { + this.serviceId = serviceIdParam; + return null; + } + } + +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org