This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new a7fe726 HDDS-4000. Split acceptance tests to reduce CI feedback time (#1236) a7fe726 is described below commit a7fe7260f5a72809fe5387725c92942c454ce899 Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Mon Jul 27 13:46:22 2020 +0200 HDDS-4000. Split acceptance tests to reduce CI feedback time (#1236) --- .github/workflows/post-commit.yml | 17 ++++--- hadoop-ozone/dev-support/checks/acceptance.sh | 2 + hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 + .../dist/src/main/compose/ozonesecure/test.sh | 2 + hadoop-ozone/dist/src/main/compose/test-all.sh | 31 +++++++++--- pom.xml | 55 +--------------------- 6 files changed, 42 insertions(+), 67 deletions(-) diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 9588423..e472aea 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -123,6 +123,13 @@ jobs: acceptance: name: acceptance runs-on: ubuntu-18.04 + strategy: + matrix: + suite: + - secure + - unsecure + - misc + fail-fast: false steps: - uses: actions/cache@v2 with: @@ -154,12 +161,13 @@ jobs: cd /mnt/ozone && hadoop-ozone/dev-support/checks/acceptance.sh env: KEEP_IMAGE: false + OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} OZONE_WITH_COVERAGE: true OZONE_VOLUME_OWNER: 1000 - uses: actions/upload-artifact@master if: always() with: - name: acceptance + name: acceptance-${{ matrix.suite }} path: /mnt/ozone/target/acceptance continue-on-error: true - run: | @@ -170,16 +178,11 @@ jobs: integration: name: integration runs-on: ubuntu-18.04 - needs: - - build strategy: matrix: profile: - client - - filesystem - - filesystem-contract - - freon - - hdds-om + - filesystem-hdds - ozone fail-fast: false steps: diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index d95c034..99d8d52 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -28,6 +28,8 @@ fi mkdir -p "$REPORT_DIR" +export OZONE_ACCEPTANCE_SUITE + cd "$DIST_DIR/compose" || exit 1 ./test-all.sh RES=$? diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index c40339e..b5b778f 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +#suite:unsecure + COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index 84de2a9..076b83a 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +#suite:secure + COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh b/hadoop-ozone/dist/src/main/compose/test-all.sh index e7f6f71..da3b80e 100755 --- a/hadoop-ozone/dist/src/main/compose/test-all.sh +++ b/hadoop-ozone/dist/src/main/compose/test-all.sh @@ -31,21 +31,40 @@ if [ "$OZONE_WITH_COVERAGE" ]; then export HADOOP_OPTS="-javaagent:share/coverage/jacoco-agent.jar=output=tcpclient,address=$DOCKER_BRIDGE_IP,includes=org.apache.hadoop.ozone.*:org.apache.hadoop.hdds.*:org.apache.hadoop.fs.ozone.*" fi +if [[ -n "${OZONE_ACCEPTANCE_SUITE}" ]]; then + tests=$(find "$SCRIPT_DIR" -name test.sh | xargs grep -l "^#suite:${OZONE_ACCEPTANCE_SUITE}$" | sort) + + # 'misc' is default suite, add untagged tests, too + if [[ "misc" == "${OZONE_ACCEPTANCE_SUITE}" ]]; then + untagged="$(find "$SCRIPT_DIR" -name test.sh | xargs grep -L "^#suite:")" + if [[ -n "${untagged}" ]]; then + tests=$(echo ${tests} ${untagged} | xargs -n1 | sort) + fi + fi + + if [[ -z "${tests}" ]]; then + echo "No tests found for suite ${OZONE_ACCEPTANCE_SUITE}" + exit 1 + fi +else + tests=$(find "$SCRIPT_DIR" -name test.sh | grep "${OZONE_TEST_SELECTOR:-""}" | sort) +fi + RESULT=0 -IFS=$'\n' # shellcheck disable=SC2044 -for test in $(find "$SCRIPT_DIR" -name test.sh | grep "${OZONE_TEST_SELECTOR:-""}" |sort); do - echo "Executing test in $(dirname "$test")" +for t in ${tests}; do + d="$(dirname "${t}")" + echo "Executing test in ${d}" #required to read the .env file from the right location - cd "$(dirname "$test")" || continue + cd "${d}" || continue ./test.sh ret=$? if [[ $ret -ne 0 ]]; then RESULT=1 - echo "ERROR: Test execution of $(dirname "$test") is FAILED!!!!" + echo "ERROR: Test execution of ${d} is FAILED!!!!" fi - RESULT_DIR="$(dirname "$test")/result" + RESULT_DIR="${d}/result" cp "$RESULT_DIR"/robot-*.xml "$RESULT_DIR"/docker-*.log "$RESULT_DIR"/*.out* "$ALL_RESULT_DIR"/ done diff --git a/pom.xml b/pom.xml index aa62f5d..969e5c6 100644 --- a/pom.xml +++ b/pom.xml @@ -2147,22 +2147,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs </build> </profile> <profile> - <id>freon</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <includes> - <include>org.apache.hadoop.ozone.freon.**</include> - </includes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>client</id> <build> <plugins> @@ -2179,23 +2163,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs </build> </profile> <profile> - <id>filesystem-contract</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <includes> - <include>org.apache.hadoop.fs.ozone.contract.**</include> - </includes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>filesystem</id> + <id>filesystem-hdds</id> <build> <plugins> <plugin> @@ -2204,26 +2172,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs <configuration> <includes> <include>org.apache.hadoop.fs.ozone.**</include> - </includes> - <excludes> - <exclude>org.apache.hadoop.fs.ozone.contract.**</exclude> - </excludes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>hdds-om</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <includes> <include>org.apache.hadoop.hdds.**</include> - <include>org.apache.hadoop.ozone.om.**</include> </includes> </configuration> </plugin> @@ -2243,8 +2192,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs </includes> <excludes> <exclude>org.apache.hadoop.ozone.client.**</exclude> - <exclude>org.apache.hadoop.ozone.freon.**</exclude> - <exclude>org.apache.hadoop.ozone.om.**</exclude> </excludes> </configuration> </plugin> --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org