This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f552cf21a1 HDDS-7540. Separate scheduled CI from push/PR workflows
(#4004)
f552cf21a1 is described below
commit f552cf21a10de7f72fb742262ad8ded04bbabefc
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Nov 29 09:55:49 2022 +0100
HDDS-7540. Separate scheduled CI from push/PR workflows (#4004)
---
.github/workflows/{post-commit.yml => ci.yml} | 11 +-
.github/workflows/post-commit.yml | 409 +-------------------------
.github/workflows/scheduled_ci.yml | 21 ++
3 files changed, 25 insertions(+), 416 deletions(-)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/ci.yml
similarity index 98%
copy from .github/workflows/post-commit.yml
copy to .github/workflows/ci.yml
index d7cddfa4df..82356bafa5 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/ci.yml
@@ -12,19 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-name: build-branch
+name: full-ci
on:
- pull_request:
- types: [opened, ready_for_review, synchronize]
- push:
- schedule:
- - cron: 30 0,12 * * *
+ workflow_call:
env:
FAIL_FAST: ${{ github.event_name == 'pull_request' }}
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
-concurrency:
- group: ci-${{ github.event.pull_request.number || github.sha }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build-info:
runs-on: ubuntu-20.04
diff --git a/.github/workflows/post-commit.yml
b/.github/workflows/post-commit.yml
index d7cddfa4df..c38a0b058d 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -17,414 +17,9 @@ on:
pull_request:
types: [opened, ready_for_review, synchronize]
push:
- schedule:
- - cron: 30 0,12 * * *
-env:
- FAIL_FAST: ${{ github.event_name == 'pull_request' }}
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
concurrency:
group: ci-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
- build-info:
- runs-on: ubuntu-20.04
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- basic-checks: ${{ steps.selective-checks.outputs.basic-checks }}
- needs-basic-checks: ${{
steps.selective-checks.outputs.needs-basic-checks }}
- needs-build: ${{ steps.selective-checks.outputs.needs-build }}
- needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
- needs-compose-tests: ${{
steps.selective-checks.outputs.needs-compose-tests }}
- needs-dependency-check: ${{
steps.selective-checks.outputs.needs-dependency-check }}
- needs-integration-tests: ${{
steps.selective-checks.outputs.needs-integration-tests }}
- needs-kubernetes-tests: ${{
steps.selective-checks.outputs.needs-kubernetes-tests }}
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v3
- with:
- persist-credentials: false
- - name: Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v3
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- if: github.event_name == 'pull_request'
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}"
- PR_DRAFT: "${{ github.event.pull_request.draft }}"
- run: |
- if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
- # Run selective checks
- dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}"
- else
- # Run all checks
- dev-support/ci/selective_ci_checks.sh
- fi
- build:
- needs:
- - build-info
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- if: needs.build-info.outputs.needs-build == 'true'
- strategy:
- matrix:
- java: [ 8 ]
- fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Cache for npm dependencies
- uses: actions/cache@v3
- with:
- path: |
- ~/.pnpm-store
- **/node_modules
- key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
- restore-keys: |
- ${{ runner.os }}-pnpm-
- - name: Cache for maven dependencies
- uses: actions/cache@v3
- with:
- path: ~/.m2/repository
- key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
- restore-keys: |
- maven-repo-${{ hashFiles('**/pom.xml') }}
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: ${{ matrix.java }}
- - name: Run a full build
- run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist -Psrc
- - name: Store binaries for tests
- uses: actions/upload-artifact@v3
- with:
- name: ozone-bin
- path: |
- hadoop-ozone/dist/target/ozone-*.tar.gz
- !hadoop-ozone/dist/target/ozone-*-src.tar.gz
- retention-days: 1
- - name: Store source tarball for compilation
- uses: actions/upload-artifact@v3
- with:
- name: ozone-src
- path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
- retention-days: 1
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/ozone/hdds*
- rm -rf ~/.m2/repository/org/apache/ozone/ozone*
- if: always()
- compile:
- needs:
- - build-info
- - build
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- if: needs.build-info.outputs.needs-compile == 'true'
- strategy:
- matrix:
- java: [ 11 ]
- fail-fast: false
- steps:
- - name: Download Ozone source tarball
- uses: actions/download-artifact@v3
- with:
- name: ozone-src
- - name: Untar sources
- run: |
- tar --strip-components 1 -xzvf ozone*-src.tar.gz
- - name: Cache for maven dependencies
- uses: actions/cache@v3
- with:
- path: ~/.m2/repository
- key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
- restore-keys: |
- maven-repo-${{ hashFiles('**/pom.xml') }}
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: ${{ matrix.java }}
- - name: Compile Ozone using Java ${{ matrix.java }}
- run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx
-Dskip.installnpx
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/ozone/hdds*
- rm -rf ~/.m2/repository/org/apache/ozone/ozone*
- if: always()
- basic:
- needs:
- - build-info
- runs-on: ubuntu-20.04
- timeout-minutes: 90
- if: needs.build-info.outputs.needs-basic-checks == 'true'
- strategy:
- matrix:
- check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
- fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- if: matrix.check != 'bats'
- - name: Checkout project with history
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
- if: matrix.check == 'bats'
- - name: Cache for maven dependencies
- uses: actions/cache@v3
- with:
- path: ~/.m2/repository
- key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }}
- restore-keys: |
- maven-repo-${{ hashFiles('**/pom.xml') }}-8
- maven-repo-${{ hashFiles('**/pom.xml') }}
- maven-repo-
- if: ${{ !contains('author,bats,docs', matrix.check) }}
- - name: Setup java
- uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Execute tests
- run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh
- - name: Summary of failures
- run: cat target/${{ matrix.check }}/summary.txt
- if: ${{ !cancelled() }}
- - name: Archive build results
- uses: actions/upload-artifact@v3
- if: ${{ !cancelled() }}
- with:
- name: ${{ matrix.check }}
- path: target/${{ matrix.check }}
- continue-on-error: true
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/ozone/hdds*
- rm -rf ~/.m2/repository/org/apache/ozone/ozone*
- if: always()
- dependency:
- needs:
- - build-info
- - build
- runs-on: ubuntu-20.04
- timeout-minutes: 5
- if: needs.build-info.outputs.needs-dependency-check == 'true'
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v3
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir dist
- tar -C dist --strip-components 1 -xzf ozone*.tar.gz
- - name: Execute tests
- run: |
- export OZONE_DIST_DIR=`pwd`/dist
- ./hadoop-ozone/dev-support/checks/dependency.sh
- - name: Archive build results
- uses: actions/upload-artifact@v3
- if: always()
- with:
- name: dependency
- path: target/dependency
- continue-on-error: true
- acceptance:
- needs:
- - build-info
- - build
- runs-on: ubuntu-20.04
- timeout-minutes: 150
- if: needs.build-info.outputs.needs-compose-tests == 'true'
- strategy:
- matrix:
- suite:
- - secure
- - unsecure
- - compat
- - HA
- - MR
- - misc
- fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v3
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir -p hadoop-ozone/dist/target
- tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- sudo chmod -R a+rwX hadoop-ozone/dist/target
- - name: Execute tests
- run: |
- pushd hadoop-ozone/dist/target/ozone-*
- sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
- popd
- ./hadoop-ozone/dev-support/checks/acceptance.sh
- env:
- KEEP_IMAGE: false
- OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
- OZONE_WITH_COVERAGE: true
- OZONE_VOLUME_OWNER: 1000
- - name: Archive build results
- uses: actions/upload-artifact@v3
- if: always()
- with:
- name: acceptance-${{ matrix.suite }}
- path: target/acceptance
- continue-on-error: true
- kubernetes:
- needs:
- - build-info
- - build
- runs-on: ubuntu-20.04
- timeout-minutes: 60
- if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v3
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir -p hadoop-ozone/dist/target
- tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- - name: Execute tests
- run: |
- pushd hadoop-ozone/dist/target/ozone-*
- sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
- popd
- ./hadoop-ozone/dev-support/checks/kubernetes.sh
- - name: Archive build results
- uses: actions/upload-artifact@v3
- if: always()
- with:
- name: kubernetes
- path: target/kubernetes
- continue-on-error: true
- integration:
- needs:
- - build-info
- runs-on: ubuntu-20.04
- timeout-minutes: 150
- if: needs.build-info.outputs.needs-integration-tests == 'true'
- strategy:
- matrix:
- profile:
- - client
- - filesystem
- - hdds
- - om
- - ozone
- - scm
- - flaky
- fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Cache for maven dependencies
- uses: actions/cache@v3
- with:
- path: ~/.m2/repository
- key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile
}}
- restore-keys: |
- maven-repo-${{ hashFiles('**/pom.xml') }}-8
- maven-repo-${{ hashFiles('**/pom.xml') }}
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Execute tests
- run: hadoop-ozone/dev-support/checks/integration.sh -P${{
matrix.profile }}
- if: matrix.profile != 'flaky'
- - name: Execute flaky tests
- run: hadoop-ozone/dev-support/checks/integration.sh -P${{
matrix.profile }} -Dsurefire.rerunFailingTestsCount=5
-Dsurefire.fork.timeout=3600
- if: matrix.profile == 'flaky'
- - name: Summary of failures
- run: cat target/${{ github.job }}/summary.txt
- if: always()
- - name: Archive build results
- uses: actions/upload-artifact@v3
- if: always()
- with:
- name: it-${{ matrix.profile }}
- path: target/integration
- continue-on-error: true
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/ozone/hdds*
- rm -rf ~/.m2/repository/org/apache/ozone/ozone*
- if: always()
- coverage:
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- if: github.repository == 'apache/ozone' && github.event_name !=
'pull_request'
- needs:
- - acceptance
- - basic
- - integration
- steps:
- - name: Checkout project
- uses: actions/checkout@v3
- - name: Cache for maven dependencies
- uses: actions/cache@v3
- with:
- path: ~/.m2/repository
- key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }}
- restore-keys: |
- maven-repo-${{ hashFiles('**/pom.xml') }}-8
- maven-repo-${{ hashFiles('**/pom.xml') }}
- maven-repo-
- - name: Download artifacts
- uses: actions/download-artifact@v3
- with:
- path: target/artifacts
- - name: Untar binaries
- run: |
- mkdir -p hadoop-ozone/dist/target
- tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C
hadoop-ozone/dist/target
- - name: Calculate combined coverage
- run: ./hadoop-ozone/dev-support/checks/coverage.sh
- - name: Setup java 11
- uses: actions/setup-java@v3
- with:
- distribution: 'temurin'
- java-version: 11
- - name: Upload coverage to Sonar
- run: ./hadoop-ozone/dev-support/checks/sonar.sh
- env:
- SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Archive build results
- uses: actions/upload-artifact@v3
- with:
- name: coverage
- path: target/coverage
- continue-on-error: true
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/ozone/hdds*
- rm -rf ~/.m2/repository/org/apache/ozone/ozone*
- if: always()
+ CI:
+ uses: ./.github/workflows/ci.yml
diff --git a/.github/workflows/scheduled_ci.yml
b/.github/workflows/scheduled_ci.yml
new file mode 100644
index 0000000000..b89f52a4f1
--- /dev/null
+++ b/.github/workflows/scheduled_ci.yml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: scheduled-ci
+on:
+ schedule:
+ - cron: 30 0,12 * * *
+jobs:
+ CI:
+ uses: ./.github/workflows/ci.yml
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
