HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bbc95ee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bbc95ee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bbc95ee Branch: refs/heads/HDDS-4 Commit: 8bbc95ee536caae21f28f9513fe624955aa122a0 Parents: 9128c66 Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Nov 6 16:53:04 2018 -0800 Committer: Xiaoyu Yao <x...@apache.org> Committed: Thu Nov 29 11:57:47 2018 -0800 ---------------------------------------------------------------------- .../acceptance/ozone-secure.robot | 95 ---------------- .../dist/src/main/compose/ozonesecure/.env | 1 - .../compose/ozonesecure/docker-compose.yaml | 22 ++-- .../ozonesecure/docker-image/runner/Dockerfile | 4 +- .../main/smoketest/security/ozone-secure.robot | 111 +++++++++++++++++++ hadoop-ozone/dist/src/main/smoketest/test.sh | 2 + 6 files changed, 126 insertions(+), 109 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot deleted file mode 100644 index 7fc1088..0000000 --- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Smoke test to start cluster with docker-compose environments. -Library OperatingSystem -Suite Setup Startup Ozone Cluster -Suite Teardown Teardown Ozone Cluster - -*** Variables *** -${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" -${version} - -*** Test Cases *** - -Daemons are running - Is daemon running om - Is daemon running scm - Is daemon running datanode - Is daemon running ozone.kdc - -Check if datanode is connected to the scm - Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1 - -Test rest interface - ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"; - Should contain ${result} 201 Created - ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"; - Should contain ${result} 201 Created - ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"; - Should contain ${result} 200 OK - ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"; - Should contain ${result} 200 OK - -Test ozone cli - ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root - Should contain ${result} Client cannot authenticate via - # Authenticate testuser - Execute on 0 datanode kinit -k testuser/datan...@example.com -t /etc/security/keytabs/testuser.keytab - Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root - ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' - Should contain ${result} createdOn - Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB - ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' - Should Be Equal ${result} bill - -*** Keywords *** - -Startup Ozone Cluster - ${rc} ${output} = Run docker compose 0 down - ${rc} ${output} = Run docker compose 0 up -d - Should Be Equal As Integers ${rc} 0 - Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening - -Teardown Ozone Cluster - Run docker compose 0 down - -Is daemon running - [arguments] ${name} - ${result} = Run docker ps - Should contain ${result} _${name}_1 - -Is Daemon started - [arguments] ${name} ${expression} - ${rc} ${result} = Run docker compose 0 logs - Should contain ${result} ${expression} - -Have healthy datanodes - [arguments] ${requirednodes} - ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' - Should Be Equal ${result} ${requirednodes} - -Execute on - [arguments] ${expected_rc} ${componentname} ${command} - ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command} - [return] ${return} - -Run docker compose - [arguments] ${expected_rc} ${command} - Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone - ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command} - Should Be Equal As Integers ${rc} ${expected_rc} - [return] ${rc} ${output} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env index a494004..cac418a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env @@ -15,4 +15,3 @@ # limitations under the License. HDDS_VERSION=${hdds.version} -SRC_VOLUME=../../ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml index fab5ba9..034d0ff 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml @@ -24,7 +24,7 @@ services: buildno: 1 hostname: kdc volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop datanode: build: context: docker-image/runner @@ -32,13 +32,13 @@ services: args: buildno: 1 volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop hostname: datanode ports: - - 9864 + - 9864 command: ["/opt/hadoop/bin/ozone","datanode"] env_file: - - docker-config + - docker-config om: build: context: docker-image/runner @@ -47,13 +47,13 @@ services: buildno: 1 hostname: om volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop ports: - - 9874:9874 + - 9874:9874 environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - - docker-config + - docker-config command: ["/opt/hadoop/bin/ozone","om"] scm: build: @@ -63,11 +63,11 @@ services: buildno: 1 hostname: scm volumes: - - $SRC_VOLUME:/opt/hadoop + - ../..:/opt/hadoop ports: - - 9876:9876 + - 9876:9876 env_file: - - docker-config + - docker-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - command: ["/opt/hadoop/bin/ozone","scm"] + command: ["/opt/hadoop/bin/ozone","scm"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile index efda03e..17eff18 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile @@ -15,8 +15,8 @@ # limitations under the License. FROM openjdk:8-jdk -RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean - +RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean +RUN pip install robotframework RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 RUN chmod +x /usr/local/bin/dumb-init RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot new file mode 100644 index 0000000..457ebaa --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test to start cluster with docker-compose environments. +Library OperatingSystem +Resource ../commonlib.robot + +*** Test Cases *** +Create volume and bucket + ${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root + Should contain ${output} Client cannot authenticate via + # Authenticate testuser + Execute kinit -k testuser/datan...@example.com -t /etc/security/keytabs/testuser.keytab + Execute ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root + Execute ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root + Execute ozone sh bucket create o3://om/fstest/bucket1 + Execute ozone sh bucket create o3://om/fstest/bucket2 + Execute ozone sh bucket create o3://om/fstest2/bucket3 + +Check volume from ozonefs + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ + +Run ozoneFS tests + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} testdir/deep + Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} NOTICE.txt + + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} PUTFILE.txt + + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/ + Should contain ${result} NOTICE.txt + Should contain ${result} PUTFILE.txt + + Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} MOVED.TXT + Should not contain ${result} NOTICE.txt + + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} subdir1/NOTICE.txt + + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/ + Should contain ${result} NOTICE.txt + + Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + Should not contain ${result} Failed + + Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} NOTICE.txt + + ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} subdir1 + + Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} TOUCHFILE.txt + + Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/ + ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should not contain ${result} testdir + + Execute rm -Rf localdir1 + Execute mkdir localdir1 + Execute cp NOTICE.txt localdir1/LOCAL.txt + Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1 + Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/ + Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt + + ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/ + Should contain ${result} localdir1/LOCAL.txt + Should contain ${result} testdir1/NOTICE.txt + + Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2 + Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3 + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/ + Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/ + Execute ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt + ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt + Should contain ${result} KEY.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt + Should Be Equal As Integers ${rc} 1 + Should contain ${result} File exists + Execute rm -Rf GET.txt + Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt + Execute ls -l GET.txt + ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/ + Should Be Equal As Integers ${rc} 1 + Should contain ${result} VOLUME_NOT_FOUND + http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bbc95ee/hadoop-ozone/dist/src/main/smoketest/test.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh index 5819595..53d0876 100755 --- a/hadoop-ozone/dist/src/main/smoketest/test.sh +++ b/hadoop-ozone/dist/src/main/smoketest/test.sh @@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then # # We select the test suites and execute them on multiple type of clusters # + DEFAULT_TESTS=("security") + execute_tests ozonesecure "${DEFAULT_TESTS[@]}" DEFAULT_TESTS=("basic") execute_tests ozone "${DEFAULT_TESTS[@]}" TESTS=("ozonefs") --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
