This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e3d604b9c2 HDDS-9629. Create workflow for running acceptance test
repeatedly (#5548)
e3d604b9c2 is described below
commit e3d604b9c24aabef0290d4853b41bcbbe3e44d4f
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Nov 7 18:49:18 2023 +0100
HDDS-9629. Create workflow for running acceptance test repeatedly (#5548)
---
.github/workflows/repeat-acceptance.yml | 167 ++++++++++++++++++++++++++
hadoop-ozone/dist/src/main/compose/testlib.sh | 23 ++--
2 files changed, 181 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/repeat-acceptance.yml
b/.github/workflows/repeat-acceptance.yml
new file mode 100644
index 0000000000..e19cc2cd26
--- /dev/null
+++ b/.github/workflows/repeat-acceptance.yml
@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow runs some acceptance test(s) multiple times concurrently.
+# Specify the test by either:
+# * "Test Suite", which should be ones of the existing suites from regular CI,
+# e.g. "cert-rotation", or
+# * "Test Filter", which is a regex pattern applied to filter test script's
path
+# (examples: "ozone-csi", "test-vault.sh", "ozone/test-ec.sh",
"test-.*-rotation.sh")
+
+name: repeat-acceptance-test
+on:
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: Git Ref (Branch/Commit_ID/Tag)
+ default: master
+ required: true
+ test-suite:
+ description: Test Suite
+ required: false
+ test-filter:
+ description: Test Filter
+ required: false
+ splits:
+ description: Number of splits
+ default: 10
+ required: true
+ fail-fast:
+ description: Stop after first failure
+ default: false
+ required: true
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+ OZONE_ACCEPTANCE_SUITE: ${{ github.event.inputs.test-suite}}
+ OZONE_TEST_SELECTOR: ${{ github.event.inputs.test-filter }}
+ FAIL_FAST: ${{ github.event.inputs.fail-fast }}
+ JAVA_VERSION: 8
+run-name: ${{ github.event_name == 'workflow_dispatch' &&
format('{0}[{1}]-{2}', inputs.test-suite || inputs.test-filter, inputs.ref,
inputs.splits) || '' }}
+jobs:
+ prepare-job:
+ runs-on: ubuntu-20.04
+ outputs:
+ matrix: ${{steps.generate.outputs.matrix}}
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ ref: ${{ github.event.inputs.ref }}
+ - name: Verify Test Filter
+ run: |
+ cd hadoop-ozone/dist/src/main/compose
+ source testlib.sh
+ find_tests
+ - id: generate
+ name: Generate test matrix
+ run: |
+ splits=()
+ for ((i = 1; i <= ${{ github.event.inputs.splits }}; i++)); do
+ splits+=("$i")
+ done
+ printf -v x "%s," "${splits[@]}"
+ split_matrix="[${x%,}]"
+ echo "matrix=$split_matrix" >> $GITHUB_OUTPUT
+ build:
+ needs:
+ - prepare-job
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v3
+ - name: Cache for npm dependencies
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.pnpm-store
+ **/node_modules
+ key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-
+ - name: Cache for maven dependencies
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2/repository
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ env.JAVA_VERSION
}}
+ restore-keys: |
+ maven-repo-${{ hashFiles('**/pom.xml') }}
+ maven-repo-
+ - name: Setup java
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: ${{ env.JAVA_VERSION }}
+ - name: Run a full build
+ run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc
+ env:
+ GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ - name: Store binaries for tests
+ uses: actions/upload-artifact@v3
+ with:
+ name: ozone-bin
+ path: |
+ hadoop-ozone/dist/target/ozone-*.tar.gz
+ !hadoop-ozone/dist/target/ozone-*-src.tar.gz
+ retention-days: 1
+ - name: Delete temporary build artifacts before caching
+ run: |
+ #Never cache local artifacts
+ rm -rf ~/.m2/repository/org/apache/ozone/hdds*
+ rm -rf ~/.m2/repository/org/apache/ozone/ozone*
+ if: always()
+ acceptance:
+ needs:
+ - prepare-job
+ - build
+ name: Run-Split
+ runs-on: ubuntu-20.04
+ strategy:
+ matrix:
+ split: ${{ fromJson(needs.prepare-job.outputs.matrix) }}
+ fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }}
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ ref: ${{ github.event.inputs.ref }}
+ - name: Download compiled Ozone binaries
+ uses: actions/download-artifact@v3
+ with:
+ name: ozone-bin
+ - name: Untar binaries
+ run: |
+ mkdir -p hadoop-ozone/dist/target
+ tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
+ rm ozone*.tar.gz
+ sudo chmod -R a+rwX hadoop-ozone/dist/target
+ - name: Execute tests
+ run: |
+ pushd hadoop-ozone/dist/target/ozone-*
+ sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
+ popd
+ ./hadoop-ozone/dev-support/checks/acceptance.sh
+ env:
+ KEEP_IMAGE: false
+ OZONE_VOLUME_OWNER: 1000
+ continue-on-error: true
+ - name: Summary of failures
+ run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
+ if: ${{ !cancelled() }}
+ - name: Archive build results
+ uses: actions/upload-artifact@v3
+ if: always()
+ with:
+ name: acceptance-${{ matrix.split }}
+ path: target/acceptance
+ continue-on-error: true
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh
b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 0bf2f88b41..11884de160 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -48,22 +48,27 @@ all_tests_in_immediate_child_dirs() {
## @description applying OZONE_ACCEPTANCE_SUITE or OZONE_TEST_SELECTOR filter.
find_tests(){
if [[ -n "${OZONE_ACCEPTANCE_SUITE}" ]]; then
- tests=$(all_tests_in_immediate_child_dirs | xargs grep -l
"^#suite:${OZONE_ACCEPTANCE_SUITE}$")
+ tests=$(all_tests_in_immediate_child_dirs | xargs grep -l
"^#suite:${OZONE_ACCEPTANCE_SUITE}$" || echo "")
# 'misc' is default suite, add untagged tests, too
if [[ "misc" == "${OZONE_ACCEPTANCE_SUITE}" ]]; then
- untagged="$(all_tests_in_immediate_child_dirs | xargs grep -L
"^#suite:")"
- if [[ -n "${untagged}" ]]; then
- tests=$(echo ${tests} ${untagged} | xargs -n1 | sort)
- fi
- fi
+ untagged="$(all_tests_in_immediate_child_dirs | xargs grep -L
"^#suite:")"
+ if [[ -n "${untagged}" ]]; then
+ tests=$(echo ${tests} ${untagged} | xargs -n1 | sort)
+ fi
+ fi
if [[ -z "${tests}" ]]; then
- echo "No tests found for suite ${OZONE_ACCEPTANCE_SUITE}"
- exit 1
+ echo "No tests found for suite ${OZONE_ACCEPTANCE_SUITE}"
+ exit 1
fi
elif [[ -n "${OZONE_TEST_SELECTOR}" ]]; then
- tests=$(all_tests_in_immediate_child_dirs | grep "${OZONE_TEST_SELECTOR}")
+ tests=$(all_tests_in_immediate_child_dirs | grep "${OZONE_TEST_SELECTOR}"
|| echo "")
+
+ if [[ -z "${tests}" ]]; then
+ echo "No tests found for filter ${OZONE_TEST_SELECTOR}"
+ exit 1
+ fi
else
tests=$(all_tests_in_immediate_child_dirs | xargs grep -L
'^#suite:failing')
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
