Repository: hadoop Updated Branches: refs/heads/HDFS-7240 14e268d57 -> eabea3a55
HDFS-13343. Ozone: Provide docker based acceptance testing on pseudo cluster. Contributed by Elek, Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eabea3a5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eabea3a5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eabea3a5 Branch: refs/heads/HDFS-7240 Commit: eabea3a559337b7f9a0d7a6a18f1682fd80f709b Parents: 14e268d Author: Anu Engineer <aengin...@apache.org> Authored: Fri Mar 23 22:39:19 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Fri Mar 23 22:39:19 2018 -0700 ---------------------------------------------------------------------- hadoop-ozone/acceptance-test/README.md | 38 ++++++ hadoop-ozone/acceptance-test/pom.xml | 86 ++++++++++++++ .../acceptance-test/src/test/compose/.env | 17 +++ .../src/test/compose/docker-compose.yaml | 61 ++++++++++ .../src/test/compose/docker-config | 34 ++++++ .../test/robotframework/acceptance/ozone.robot | 116 +++++++++++++++++++ pom.xml | 1 + 7 files changed, 353 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/README.md ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/README.md b/hadoop-ozone/acceptance-test/README.md new file mode 100644 index 0000000..5e642c8 --- /dev/null +++ b/hadoop-ozone/acceptance-test/README.md @@ -0,0 +1,38 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Acceptance test suite for Ozone/Hdsl + +This project contains acceptance tests for ozone/hdsl using docker-compose and [robot framework](http://robotframework.org/). + +## Run + +To run the acceptance tests, please activate the `ozone-acceptance-test` profile and do a full build. + +Typically you need a `mvn install -Phdsl,ozone-acceptance-test,dist -DskipTests` for a build without unit tests but with acceptance test. + +Notes: + + 1. You need a hadoop build in hadoop-dist/target directory. + 2. The `ozone-acceptance-test` could be activated with profile even if the unit tests are disabled. + + +## Development + +You can run manually the robot tests with `robot` cli. (See robotframework docs to install it.) + + 1. Go to the `src/test/robotframework` + 2. Execute `robot -v basedir:${PWD}/../../.. -v VERSION:3.2.0-SNAPSHOT .` + +You can also use select just one test with -t `"*testnamefragment*"` \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/pom.xml b/hadoop-ozone/acceptance-test/pom.xml new file mode 100644 index 0000000..6100beb --- /dev/null +++ b/hadoop-ozone/acceptance-test/pom.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 +http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-project</artifactId> + <version>3.2.0-SNAPSHOT</version> + <relativePath>../../hadoop-project</relativePath> + </parent> + <artifactId>hadoop-ozone-acceptance-test</artifactId> + <version>3.2.0-SNAPSHOT</version> + <description>Apache Hadoop Ozone Acceptance test</description> + <name>Apache Hadoop Ozone acceptance test</name> + <packaging>pom</packaging> + <build> + <plugins> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <executions> + <execution> + <id>copy-docker-compose</id> + <goals> + <goal>copy-resources</goal> + </goals> + <phase>process-test-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/compose + </outputDirectory> + <resources> + <resource> + <directory>src/test/compose</directory> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>ozone-acceptance-test</id> + <build> + <plugins> + <plugin> + <groupId>org.robotframework</groupId> + <artifactId>robotframework-maven-plugin</artifactId> + <version>1.4.7</version> + <executions> + <execution> + <goals> + <goal>run</goal> + </goals> + <configuration> + <variables> + <variable>version:${project.version}</variable> + <variable>basedir:${project.basedir}</variable> + </variables> + <skip>false</skip> + <skipTests>false</skipTests> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/src/test/compose/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/.env b/hadoop-ozone/acceptance-test/src/test/compose/.env new file mode 100644 index 0000000..79f890b --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOPDIR=../../hadoop-dist/target/hadoop-${project.version} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml new file mode 100644 index 0000000..fb4e5d3 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + namenode: + image: elek/hadoop-runner:o3-refactor + hostname: namenode + volumes: + - ${HADOOPDIR}:/opt/hadoop + ports: + - 9870 + environment: + ENSURE_NAMENODE_DIR: /data/namenode + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/hdfs","namenode"] + datanode: + image: elek/hadoop-runner:o3-refactor + volumes: + - ${HADOOPDIR}:/opt/hadoop + ports: + - 9864 + command: ["/opt/hadoop/bin/oz","datanode"] + env_file: + - ./docker-config + ksm: + image: elek/hadoop-runner:o3-refactor + volumes: + - ${HADOOPDIR}:/opt/hadoop + ports: + - 9874 + environment: + ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/oz","ksm"] + scm: + image: elek/hadoop-runner:o3-refactor + volumes: + - ${HADOOPDIR}:/opt/hadoop + ports: + - 9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/oz","scm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/src/test/compose/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/docker-config new file mode 100644 index 0000000..20c1e30 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/compose/docker-config @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 +OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 +HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HdslServerPlugin,org.apache.hadoop.ozone.web.ObjectStoreRestPlugin +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot new file mode 100644 index 0000000..31ddf36 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Smoke test to start cluster with docker-compose environments. +Library OperatingSystem +Suite Setup Startup Ozone Cluster +Suite Teardown Teardown Ozone Cluster + +*** Variables *** +${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" +${version} + +*** Test Cases *** + +Daemons are running without error + Is daemon running without error ksm + Is daemon running without error scm + Is daemon running without error namenode + Is daemon running without error datanode + +Check if datanode is connected to the scm + Wait Until Keyword Succeeds 2min 5sec Have healthy datanodes 1 + +Scale it up to 5 datanodes + Scale datanodes up 5 + Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 5 + +Test rest interface + ${result} = Execute on datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"; + Should contain ${result} 201 Created + ${result} = Execute on datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"; + Should contain ${result} 201 Created + ${result} = Execute on datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"; + Should contain ${result} 200 OK + ${result} = Execute on datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"; + Should contain ${result} 200 OK + +Test oz cli + Execute on datanode oz oz -createVolume http://localhost:9880/hive -user bilbo -quota 100TB -root + ${result} = Execute on datanode oz oz -listVolume http://localhost:9880/ -user bilbo | grep -v Removed | jq '.[] | select(.volumeName=="hive")' + Should contain ${result} createdOn + Execute on datanode oz oz -createBucket http://localhost:9880/hive/bb1 + ${result} Execute on datanode oz oz -listBucket http://localhost:9880/hive/ | grep -v Removed | jq -r '.[] | select(.bucketName=="bb1") | .volumeName' + Should Be Equal ${result} hive + Execute on datanode oz oz -deleteBucket http://localhost:9880/hive/bb1 + Execute on datanode oz oz -deleteVolume http://localhost:9880/hive -user bilbo + + + +Check webui static resources + ${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.0.2/js/bootstrap.min.js + Should contain ${result} 200 + ${result} = Execute on ksm curl -s -I http://localhost:9874/static/bootstrap-3.0.2/js/bootstrap.min.js + Should contain ${result} 200 + +Start freon testing + ${result} = Execute on ksm oz freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10 + Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125 + Should Not Contain ${result} ERROR + +*** Keywords *** + +Startup Ozone Cluster + ${rc} ${output} = Run docker compose down + ${rc} ${output} = Run docker compose up -d + Should Be Equal As Integers ${rc} 0 + Wait Until Keyword Succeeds 1min 5sec Is Daemon started ksm HTTP server of KSM is listening + +Teardown Ozone Cluster + Run docker compose down + +Is daemon running without error + [arguments] ${name} + ${result} = Run docker ps + Should contain ${result} _${name}_1 + ${rc} ${result} = Run docker compose logs ${name} + Should not contain ${result} ERROR + +Is Daemon started + [arguments] ${name} ${expression} + ${rc} ${result} = Run docker compose logs + Should contain ${result} ${expression} + +Have healthy datanodes + [arguments] ${requirednodes} + ${result} = Execute on scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' + Should Be Equal ${result} ${requirednodes} + +Scale datanodes up + [arguments] ${requirednodes} + Run docker compose scale datanode=${requirednodes} + +Execute on + [arguments] ${componentname} ${command} + ${rc} ${return} = Run docker compose exec ${componentname} ${command} + [return] ${return} + +Run docker compose + [arguments] ${command} + Set Environment Variable HADOOPDIR ${basedir}/../../hadoop-dist/target/hadoop-${version} + ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/target/compose/docker-compose.yaml ${command} + Should Be Equal As Integers ${rc} 0 + [return] ${rc} ${output} http://git-wip-us.apache.org/repos/asf/hadoop/blob/eabea3a5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ad306f5..8f8c977 100644 --- a/pom.xml +++ b/pom.xml @@ -721,6 +721,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs <module>hadoop-ozone</module> <module>hadoop-cblock</module> <module>hadoop-hdsl</module> + <module>hadoop-ozone/acceptance-test</module> </modules> </profile> </profiles> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
