This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 041fed250f HDDS-6618. Require successful basic checks for long-running
tests (#5267)
041fed250f is described below
commit 041fed250fe7dc3189a78c45e208854a00a0decb
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Wed Sep 20 01:27:45 2023 +0530
HDDS-6618. Require successful basic checks for long-running tests (#5267)
---
.github/workflows/ci.yml | 47 ++++++++++++++++++++++-
dev-support/ci/categorize_basic_checks.sh | 55 +++++++++++++++++++++++++++
dev-support/ci/selective_ci_checks.sh | 4 +-
hadoop-ozone/dev-support/checks/author.sh | 3 ++
hadoop-ozone/dev-support/checks/bats.sh | 2 +
hadoop-ozone/dev-support/checks/checkstyle.sh | 3 ++
hadoop-ozone/dev-support/checks/docs.sh | 2 +
hadoop-ozone/dev-support/checks/findbugs.sh | 3 ++
hadoop-ozone/dev-support/checks/native.sh | 2 +
hadoop-ozone/dev-support/checks/rat.sh | 3 ++
hadoop-ozone/dev-support/checks/unit.sh | 2 +
11 files changed, 123 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e0cde88398..a2f399e5b9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,8 +26,9 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
- basic-checks: ${{ steps.selective-checks.outputs.basic-checks }}
needs-basic-checks: ${{
steps.selective-checks.outputs.needs-basic-checks }}
+ basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
+ basic-unit: ${{ steps.categorize-basic-checks.outputs.basic-unit }}
needs-build: ${{ steps.selective-checks.outputs.needs-build }}
needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
needs-compose-tests: ${{
steps.selective-checks.outputs.needs-compose-tests }}
@@ -62,6 +63,11 @@ jobs:
- name: Acceptance suites
id: acceptance-suites
run: dev-support/ci/acceptance_suites.sh
+ - name: Categorize Basic Checks
+ id: categorize-basic-checks
+ env:
+ ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks
}}"
+ run: dev-support/ci/categorize_basic_checks.sh
build:
needs:
- build-info
@@ -213,6 +219,39 @@ jobs:
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
+ unit:
+ needs:
+ - build-info
+ - basic
+ runs-on: ubuntu-20.04
+ timeout-minutes: 90
+ if: needs.build-info.outputs.needs-basic-checks == 'true'
+ strategy:
+ matrix:
+ check: ${{ fromJson(needs.build-info.outputs.basic-unit) }}
+ fail-fast: false
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v3
+ - name: Cache for maven dependencies
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2/repository
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile
}}
+ restore-keys: |
+ maven-repo-${{ hashFiles('**/pom.xml') }}-8
+ maven-repo-${{ hashFiles('**/pom.xml') }}
+ maven-repo-
+ - name: Setup java
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'temurin'
+ java-version: 8
+ - name: Execute tests
+ run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh
+ - name: Summary of failures
+ run: cat target/${{ matrix.check }}/summary.txt
+ if: ${{ !cancelled() }}
dependency:
needs:
- build-info
@@ -246,6 +285,7 @@ jobs:
needs:
- build-info
- build
+ - basic
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-compose-tests == 'true'
@@ -286,6 +326,7 @@ jobs:
needs:
- build-info
- build
+ - basic
runs-on: ubuntu-20.04
timeout-minutes: 60
if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
@@ -316,6 +357,8 @@ jobs:
integration:
needs:
- build-info
+ - build
+ - basic
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-integration-tests == 'true'
@@ -376,8 +419,8 @@ jobs:
timeout-minutes: 30
if: github.repository == 'apache/ozone' && github.event_name !=
'pull_request'
needs:
+ - unit
- acceptance
- - basic
- integration
steps:
- name: Checkout project
diff --git a/dev-support/ci/categorize_basic_checks.sh
b/dev-support/ci/categorize_basic_checks.sh
new file mode 100755
index 0000000000..d2beba5ed2
--- /dev/null
+++ b/dev-support/ci/categorize_basic_checks.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# shellcheck source=dev-support/ci/lib/_script_init.sh
+. dev-support/ci/lib/_script_init.sh
+
+# Remove the square brackets
+ALL_BASIC_CHECKS="${ALL_BASIC_CHECKS[@]#\[}"
+ALL_BASIC_CHECKS="${ALL_BASIC_CHECKS[@]%\]}"
+
+# Replace commas with spaces to form a space-delimited list
+SPACE_DELIMITED_ALL_CHECKS=$(echo "$ALL_BASIC_CHECKS" | tr -d '"' | tr ',' ' ')
+
+BASIC_CHECKS=$(grep -lr '^#checks:basic' hadoop-ozone/dev-support/checks \
+ | sort -u | xargs -n1 basename \
+ | cut -f1 -d'.')
+
+UNIT_CHECKS=$(grep -lr '^#checks:unit' hadoop-ozone/dev-support/checks \
+ | sort -u | xargs -n1 basename \
+ | cut -f1 -d'.')
+
+if [[ -n "${SPACE_DELIMITED_ALL_CHECKS}" ]]; then
+ SPACE_DELIMITED_ALL_CHECKS=" ${SPACE_DELIMITED_ALL_CHECKS[*]} " # add
framing blanks
+ for item in ${BASIC_CHECKS[@]}; do
+ if [[ $SPACE_DELIMITED_ALL_CHECKS =~ " $item " ]] ; then # use
$item as regexp
+ basic+=($item)
+ fi
+ done
+ initialization::ga_output basic-checks \
+ "$(initialization::parameters_to_json ${basic[@]})"
+
+ for item in ${UNIT_CHECKS[@]}; do
+ if [[ $SPACE_DELIMITED_ALL_CHECKS =~ " $item " ]] ; then # use $item
as regexp
+ unit+=($item)
+ fi
+ done
+ initialization::ga_output basic-unit \
+ "$(initialization::parameters_to_json ${unit[@]})"
+fi
+
diff --git a/dev-support/ci/selective_ci_checks.sh
b/dev-support/ci/selective_ci_checks.sh
index 89ac6e3739..c2e60b61ea 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -82,7 +82,9 @@ function get_changed_files() {
}
function set_outputs_run_everything_and_exit() {
- BASIC_CHECKS="author bats checkstyle docs findbugs native rat unit"
+ BASIC_CHECKS=$(grep -lr '^#checks:' hadoop-ozone/dev-support/checks \
+ | sort -u | xargs -n1 basename \
+ | cut -f1 -d'.')
compile_needed=true
compose_tests_needed=true
dependency_check_needed=true
diff --git a/hadoop-ozone/dev-support/checks/author.sh
b/hadoop-ozone/dev-support/checks/author.sh
index a4e75155b5..7c95b5d775 100755
--- a/hadoop-ozone/dev-support/checks/author.sh
+++ b/hadoop-ozone/dev-support/checks/author.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+#checks:basic
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
diff --git a/hadoop-ozone/dev-support/checks/bats.sh
b/hadoop-ozone/dev-support/checks/bats.sh
index 407457612c..e4e18957a6 100755
--- a/hadoop-ozone/dev-support/checks/bats.sh
+++ b/hadoop-ozone/dev-support/checks/bats.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#checks:basic
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "${DIR}/../../.." || exit 1
diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh
b/hadoop-ozone/dev-support/checks/checkstyle.sh
index fc9a5b9f85..97ddfa698d 100755
--- a/hadoop-ozone/dev-support/checks/checkstyle.sh
+++ b/hadoop-ozone/dev-support/checks/checkstyle.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+#checks:basic
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
diff --git a/hadoop-ozone/dev-support/checks/docs.sh
b/hadoop-ozone/dev-support/checks/docs.sh
index 11cb21859a..ce80c3f3e5 100755
--- a/hadoop-ozone/dev-support/checks/docs.sh
+++ b/hadoop-ozone/dev-support/checks/docs.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#checks:basic
+
set -u -o pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh
b/hadoop-ozone/dev-support/checks/findbugs.sh
index 3f7f56ec8d..8c61c524a9 100755
--- a/hadoop-ozone/dev-support/checks/findbugs.sh
+++ b/hadoop-ozone/dev-support/checks/findbugs.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+#checks:basic
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
diff --git a/hadoop-ozone/dev-support/checks/native.sh
b/hadoop-ozone/dev-support/checks/native.sh
index a125f58a45..85e9ef8e11 100755
--- a/hadoop-ozone/dev-support/checks/native.sh
+++ b/hadoop-ozone/dev-support/checks/native.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#checks:unit
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CHECK=native
diff --git a/hadoop-ozone/dev-support/checks/rat.sh
b/hadoop-ozone/dev-support/checks/rat.sh
index c87f7d0c3a..32a1034902 100755
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/hadoop-ozone/dev-support/checks/rat.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+#checks:basic
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
diff --git a/hadoop-ozone/dev-support/checks/unit.sh
b/hadoop-ozone/dev-support/checks/unit.sh
index 74ffdb52a2..d310664c38 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#checks:unit
+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CHECK=unit
source "${DIR}/junit.sh" -pl \!:ozone-integration-test,\!:mini-chaos-tests
-DexcludedGroups=native "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
