HDDS-40. Separating packaging of Ozone/HDDS from the main Hadoop. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b4f24ad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b4f24ad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b4f24ad Branch: refs/heads/HDDS-4 Commit: 4b4f24ad5f2b457ad215d469bf28cf9a799812bc Parents: 50408cf Author: Anu Engineer <aengin...@apache.org> Authored: Fri May 11 13:52:05 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Fri May 11 14:24:36 2018 -0700 ---------------------------------------------------------------------- .gitignore | 5 + dev-support/bin/dist-layout-stitching | 15 -- dev-support/bin/ozone-dist-layout-stitching | 153 +++++++++++++++++++ dev-support/bin/ozone-dist-tar-stitching | 48 ++++++ hadoop-dist/pom.xml | 152 +++++++++++------- hadoop-dist/src/main/compose/ozone/.env | 2 +- .../src/main/compose/ozone/docker-compose.yaml | 8 +- .../hdfs/server/namenode/NameNodeUtils.java | 2 +- hadoop-ozone/acceptance-test/README.md | 22 ++- .../dev-support/bin/robot-all.sh | 18 +++ .../acceptance-test/dev-support/bin/robot.sh | 38 +++++ hadoop-ozone/acceptance-test/pom.xml | 29 +--- .../acceptance-test/src/test/compose/.env | 2 +- .../src/test/compose/docker-compose.yaml | 8 +- .../test/robotframework/acceptance/ozone.robot | 7 +- 15 files changed, 394 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 440708a..3883ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,8 @@ patchprocess/ .history/ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log + +#robotframework outputs +log.html +output.xml +report.html http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index 6557161..584821a 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -146,21 +146,6 @@ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client- run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/ -# HDDS -run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" . - -# Ozone -run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" . -run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" . - run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" . http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/dev-support/bin/ozone-dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching new file mode 100755 index 0000000..1b0b224 --- /dev/null +++ b/dev-support/bin/ozone-dist-layout-stitching @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# project.version +VERSION=$1 + +# project.build.directory +BASEDIR=$2 + +#hdds.version +HDDS_VERSION=$3 + +function run() +{ + declare res + + echo "\$ ${*}" + "${@}" + res=$? + if [[ ${res} != 0 ]]; then + echo + echo "Failed!" + echo + exit "${res}" + fi +} + +function findfileindir() +{ + declare file="$1" + declare dir="${2:-./share}" + declare count + + count=$(find "${dir}" -iname "${file}" | wc -l) + + #shellcheck disable=SC2086 + echo ${count} +} + +function copyifnotexists() +{ + declare src="$1" + declare dest="$2" + + declare srcname + declare destdir + + declare child + declare childpath + + if [[ -f "${src}" ]]; then + srcname=${src##*/} + if [[ "${srcname}" != *.jar || + $(findfileindir "${srcname}") -eq "0" ]]; then + destdir=$(dirname "${dest}") + mkdir -p "${destdir}" + cp -p "${src}" "${dest}" + fi + else + for childpath in "${src}"/*; do + child="${childpath##*/}" + if [[ "${child}" == "doc" || + "${child}" == "webapps" ]]; then + mkdir -p "${dest}/${child}" + cp -r "${src}/${child}"/* "${dest}/${child}" + continue; + fi + copyifnotexists "${src}/${child}" "${dest}/${child}" + done + fi +} + +#Copy all contents as is except the lib. +#for libs check for existence in share directory, if not exist then only copy. +function copy() +{ + declare src="$1" + declare dest="$2" + + declare child + declare childpath + + if [[ -d "${src}" ]]; then + for childpath in "${src}"/*; do + child="${childpath##*/}" + + if [[ "${child}" == "share" ]]; then + copyifnotexists "${src}/${child}" "${dest}/${child}" + else + if [[ -d "${src}/${child}" ]]; then + mkdir -p "${dest}/${child}" + cp -pr "${src}/${child}"/* "${dest}/${child}" + else + cp -pr "${src}/${child}" "${dest}/${child}" + fi + fi + done + fi +} + +# shellcheck disable=SC2164 +ROOT=$(cd "${BASEDIR}"/../..;pwd) +echo +echo "Current directory $(pwd)" +echo +run rm -rf "ozone" +run mkdir "ozone" +run cd "ozone" +run cp -p "${ROOT}/LICENSE.txt" . +run cp -p "${ROOT}/NOTICE.txt" . +run cp -p "${ROOT}/README.txt" . + +# Copy hadoop-common first so that it have always have all dependencies. +# Remaining projects will copy only libraries which are not present already in 'share' directory. +run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" . +run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" . +run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" . + + +# HDDS +run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" . + +# Ozone +run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" . +run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" . + +mkdir -p ./share/hadoop/mapreduce +mkdir -p ./share/hadoop/yarn +echo +echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}" +echo http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/dev-support/bin/ozone-dist-tar-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/ozone-dist-tar-stitching b/dev-support/bin/ozone-dist-tar-stitching new file mode 100755 index 0000000..decfa23 --- /dev/null +++ b/dev-support/bin/ozone-dist-tar-stitching @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# project.version +VERSION=$1 + +# project.build.directory +BASEDIR=$2 + +function run() +{ + declare res + + echo "\$ ${*}" + "${@}" + res=$? + if [[ ${res} != 0 ]]; then + echo + echo "Failed!" + echo + exit "${res}" + fi +} + +#To make the final dist directory easily mountable from docker we don't use +#version name in the directory name. +#To include the version name in the root directory of the tar file +# we create a symbolic link and dereference it during the tar creation +ln -s -f ozone ozone-${VERSION} +run tar -c --dereference -f "ozone-${VERSION}.tar" "ozone" +run gzip -f "ozone-${VERSION}.tar" +echo +echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz" +echo \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 43836eb..999d44c 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -13,8 +13,8 @@ limitations under the License. See accompanying LICENSE file. --> <project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> @@ -168,10 +168,13 @@ </goals> <configuration> <executable>${shell-executable}</executable> - <workingDirectory>${project.build.directory}</workingDirectory> + <workingDirectory>${project.build.directory} + </workingDirectory> <requiresOnline>false</requiresOnline> <arguments> - <argument>${basedir}/../dev-support/bin/dist-layout-stitching</argument> + <argument> + ${basedir}/../dev-support/bin/dist-layout-stitching + </argument> <argument>${project.version}</argument> <argument>${project.build.directory}</argument> <argument>${hdds.version}</argument> @@ -182,14 +185,16 @@ <id>toolshooks</id> <phase>prepare-package</phase> <goals> - <goal>exec</goal> + <goal>exec</goal> </goals> <configuration> <executable>${shell-executable}</executable> <workingDirectory>${basedir}</workingDirectory> <requiresOnline>false</requiresOnline> <arguments> - <argument>${basedir}/../dev-support/bin/dist-tools-hooks-maker</argument> + <argument> + ${basedir}/../dev-support/bin/dist-tools-hooks-maker + </argument> <argument>${project.version}</argument> <argument>${project.build.directory}</argument> <argument>${basedir}/../hadoop-tools</argument> @@ -203,14 +208,16 @@ <goal>exec</goal> </goals> <configuration> - <executable>${shell-executable}</executable> - <workingDirectory>${project.build.directory}</workingDirectory> - <requiresOnline>false</requiresOnline> - <arguments> - <argument>${basedir}/../dev-support/bin/dist-tar-stitching</argument> - <argument>${project.version}</argument> - <argument>${project.build.directory}</argument> - </arguments> + <executable>${shell-executable}</executable> + <workingDirectory>${project.build.directory} + </workingDirectory> + <requiresOnline>false</requiresOnline> + <arguments> + <argument>${basedir}/../dev-support/bin/dist-tar-stitching + </argument> + <argument>${project.version}</argument> + <argument>${project.build.directory}</argument> + </arguments> </configuration> </execution> </executions> @@ -218,14 +225,12 @@ </plugins> </build> </profile> - <profile> <id>hdds</id> <activation> <activeByDefault>false</activeByDefault> </activation> <dependencies> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-ozone-manager</artifactId> @@ -261,41 +266,86 @@ <plugins> <plugin> <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-docker-compose</id> - <goals> - <goal>copy-resources</goal> - </goals> - <phase>prepare-package</phase> - <configuration> - <outputDirectory>${project.build.directory}/compose</outputDirectory> - <resources> - <resource> - <directory>src/main/compose</directory> - <filtering>true</filtering> - </resource> - </resources> - </configuration> - </execution> - <execution> - <id>copy-dockerfile</id> - <goals> - <goal>copy-resources</goal> - </goals> - <phase>prepare-package</phase> - <configuration> - <outputDirectory>${project.build.directory}</outputDirectory> - <resources> - <resource> - <directory>src/main/docker</directory> - <filtering>true</filtering> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> + <executions> + <execution> + <id>copy-docker-compose</id> + <goals> + <goal>copy-resources</goal> + </goals> + <phase>prepare-package</phase> + <configuration> + <outputDirectory>${project.build.directory}/compose + </outputDirectory> + <resources> + <resource> + <directory>src/main/compose</directory> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-dockerfile</id> + <goals> + <goal>copy-resources</goal> + </goals> + <phase>prepare-package</phase> + <configuration> + <outputDirectory>${project.build.directory}</outputDirectory> + <resources> + <resource> + <directory>src/main/docker</directory> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>dist-ozone</id> + <phase>prepare-package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${shell-executable}</executable> + <workingDirectory>${project.build.directory} + </workingDirectory> + <arguments> + <argument> + ${basedir}/../dev-support/bin/ozone-dist-layout-stitching + </argument> + <argument>${project.version}</argument> + <argument>${project.build.directory}</argument> + <argument>${hdds.version}</argument> + </arguments> + </configuration> + </execution> + <execution> + <id>tar-ozone</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${shell-executable}</executable> + <workingDirectory>${project.build.directory} + </workingDirectory> + <arguments> + <argument>${basedir}/../dev-support/bin/ozone-dist-tar-stitching + </argument> + <argument>${hdds.version}</argument> + <argument>${project.build.directory}</argument> + </arguments> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </profile> http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-dist/src/main/compose/ozone/.env ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/.env b/hadoop-dist/src/main/compose/ozone/.env index af20d3e..67eed25 100644 --- a/hadoop-dist/src/main/compose/ozone/.env +++ b/hadoop-dist/src/main/compose/ozone/.env @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION=${project.version} \ No newline at end of file +HDDS_VERSION=${hdds.version} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-dist/src/main/compose/ozone/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml index 13a7db6..faf420c 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml +++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml @@ -20,7 +20,7 @@ services: image: apache/hadoop-runner hostname: namenode volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop + - ../../ozone:/opt/hadoop ports: - 9870:9870 environment: @@ -31,7 +31,7 @@ services: datanode: image: apache/hadoop-runner volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop + - ../../ozone:/opt/hadoop ports: - 9864 command: ["/opt/hadoop/bin/ozone","datanode"] @@ -40,7 +40,7 @@ services: ksm: image: apache/hadoop-runner volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop + - ../../ozone:/opt/hadoop ports: - 9874:9874 environment: @@ -51,7 +51,7 @@ services: scm: image: apache/hadoop-runner volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop + - ../../ozone:/opt/hadoop ports: - 9876:9876 env_file: http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java index 838a8e7..ec1d510 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeUtils.java @@ -112,7 +112,7 @@ public final class NameNodeUtils { } if (port > 0) { - return currentNnAddress; + return currentNnAddress; } else { // the port is missing or 0. Figure out real bind address later. return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/README.md ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/README.md b/hadoop-ozone/acceptance-test/README.md index 3a0ca49..2714e0a 100644 --- a/hadoop-ozone/acceptance-test/README.md +++ b/hadoop-ozone/acceptance-test/README.md @@ -20,19 +20,29 @@ This project contains acceptance tests for ozone/hdds using docker-compose and [ To run the acceptance tests, please activate the `ozone-acceptance-test` profile and do a full build. -Typically you need a `mvn install -Phdds,ozone-acceptance-test,dist -DskipTests` for a build without unit tests but with acceptance test. +``` +mvn clean install -Pdist -Phdds +cd hadoop-ozone/acceptance-test +mvn integration-test -Phdds,ozone-acceptance-test,dist -DskipTests +``` Notes: 1. You need a hadoop build in hadoop-dist/target directory. 2. The `ozone-acceptance-test` could be activated with profile even if the unit tests are disabled. - + 3. This method does not require the robot framework on path as jpython is used. ## Development -You can run manually the robot tests with `robot` cli. (See robotframework docs to install it.) +You can also run manually the robot tests with `robot` cli. + (See robotframework docs to install it: http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#installation-instructions) + +In the dev-support directory we have two wrapper scripts to run robot framework with local robot cli +instead of calling it from maven. - 1. Go to the `src/test/robotframework` - 2. Execute `robot -v basedir:${PWD}/../../.. -v VERSION:3.2.0-SNAPSHOT .` +It's useful during the development of the robot files as any robotframework cli +arguments could be used. -You can also use select just one test with -t `"*testnamefragment*"` \ No newline at end of file + 1. `dev-support/bin/robot.sh` is the simple wrapper. The .robot file should be used as an argument. + 2. `dev-support/bin/robot-all.sh` will call the robot.sh with the main acceptance test directory, + which means all the acceptance tests will be executed. http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh b/hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh new file mode 100755 index 0000000..0e212a2 --- /dev/null +++ b/hadoop-ozone/acceptance-test/dev-support/bin/robot-all.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +$DIR/robot.sh $DIR/../../src/test/robotframework/acceptance \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/dev-support/bin/robot.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/dev-support/bin/robot.sh b/hadoop-ozone/acceptance-test/dev-support/bin/robot.sh new file mode 100755 index 0000000..b651f76 --- /dev/null +++ b/hadoop-ozone/acceptance-test/dev-support/bin/robot.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +#basedir is the directory of the whole hadoop project. Used to calculate the +#exact path to the hadoop-dist project +BASEDIR=${DIR}/../../../.. + +if [ ! "$(which robot)" ] ; then + echo "" + echo "robot is not on your PATH." + echo "" + echo "Please install it according to the documentation:" + echo " http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#installation-instructions"; + echo " (TLDR; most of the time you need: 'pip install robotframework')" + exit -1 +fi + +OZONEDISTDIR="$BASEDIR/hadoop-dist/target/ozone" +if [ ! -d "$OZONEDISTDIR" ]; then + echo "Ozone can't be found in the $OZONEDISTDIR." + echo "You may need a full build with -Phdds and -Pdist profiles" + exit -1 +fi +robot -v basedir:$BASEDIR $@ http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/pom.xml b/hadoop-ozone/acceptance-test/pom.xml index fb6794c..ef45c44 100644 --- a/hadoop-ozone/acceptance-test/pom.xml +++ b/hadoop-ozone/acceptance-test/pom.xml @@ -28,32 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <description>Apache Hadoop Ozone Acceptance Tests</description> <name>Apache Hadoop Ozone Acceptance Tests</name> <packaging>pom</packaging> - <build> - <plugins> - <plugin> - <artifactId>maven-resources-plugin</artifactId> - <executions> - <execution> - <id>copy-docker-compose</id> - <goals> - <goal>copy-resources</goal> - </goals> - <phase>process-test-resources</phase> - <configuration> - <outputDirectory>${project.build.directory}/compose - </outputDirectory> - <resources> - <resource> - <directory>src/test/compose</directory> - <filtering>true</filtering> - </resource> - </resources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> <profiles> <profile> <id>ozone-acceptance-test</id> @@ -70,8 +44,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> </goals> <configuration> <variables> - <variable>version:${project.version}</variable> - <variable>basedir:${project.basedir}</variable> + <variable>basedir:${project.basedir}/../..</variable> </variables> <skip>false</skip> <skipTests>false</skipTests> http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/src/test/compose/.env ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/.env b/hadoop-ozone/acceptance-test/src/test/compose/.env index 79f890b..cf22168 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/.env +++ b/hadoop-ozone/acceptance-test/src/test/compose/.env @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -HADOOPDIR=../../hadoop-dist/target/hadoop-${project.version} \ No newline at end of file +OZONEDIR=../../../hadoop-dist/target/ozone http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml index da63f84..44bd4a0 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml @@ -20,7 +20,7 @@ services: image: apache/hadoop-runner hostname: namenode volumes: - - ${HADOOPDIR}:/opt/hadoop + - ${OZONEDIR}:/opt/hadoop ports: - 9870 environment: @@ -31,7 +31,7 @@ services: datanode: image: apache/hadoop-runner volumes: - - ${HADOOPDIR}:/opt/hadoop + - ${OZONEDIR}:/opt/hadoop ports: - 9864 command: ["/opt/hadoop/bin/ozone","datanode"] @@ -41,7 +41,7 @@ services: image: apache/hadoop-runner hostname: ksm volumes: - - ${HADOOPDIR}:/opt/hadoop + - ${OZONEDIR}:/opt/hadoop ports: - 9874 environment: @@ -52,7 +52,7 @@ services: scm: image: apache/hadoop-runner volumes: - - ${HADOOPDIR}:/opt/hadoop + - ${OZONEDIR}:/opt/hadoop ports: - 9876 env_file: http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4f24ad/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot index 211ec4c..c0e04a8 100644 --- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot +++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone.robot @@ -21,8 +21,7 @@ Suite Teardown Teardown Ozone Cluster *** Variables *** ${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" -${version} - +${basedir} *** Test Cases *** Daemons are running without error @@ -130,8 +129,8 @@ Execute on Run docker compose [arguments] ${command} - Set Environment Variable HADOOPDIR ${basedir}/../../hadoop-dist/target/hadoop-${version} - ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/target/compose/docker-compose.yaml ${command} + Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone + ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/docker-compose.yaml ${command} Log ${output} Should Be Equal As Integers ${rc} 0 [return] ${rc} ${output} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
