This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 05466d3e2 chore(CI): Improve github actions (#1791)
05466d3e2 is described below
commit 05466d3e2becd1a3fe0164bd45febf5b799e83d3
Author: Yingchun Lai <[email protected]>
AuthorDate: Thu Dec 21 21:06:41 2023 +0800
chore(CI): Improve github actions (#1791)
This patch mainly improve the github actions for CI, including:
- Separate Java client CI 'test' step into 'build_server' and
'test_java_client'.
So it's not needed to build server again when retrying to run Java client
tests.
- Encapsulate some actions when build and test servers, including:
- rebuild_thirdparty_if_needed
- build_pegasus
- upload_artifact
- download_artifact
- run_server_tests
---
.github/actions/build_pegasus/action.yaml | 52 +++
.../download_artifact/action.yaml} | 51 +--
.../rebuild_thirdparty_if_needed/action.yaml | 56 +++
.../run_server_tests/action.yaml} | 42 +-
.github/actions/upload_artifact/action.yaml | 38 ++
.github/workflows/lint_and_test_admin-cli.yml | 1 +
.../lint_and_test_admin-cli_always_pass.yml | 1 +
.github/workflows/lint_and_test_collector.yml | 4 +-
.github/workflows/lint_and_test_cpp.yaml | 440 +++------------------
.github/workflows/lint_and_test_go-client.yml | 1 +
...va-client.yml => lint_and_test_java-client.yml} | 71 ++--
.github/workflows/lint_and_test_pegic.yml | 1 +
.../workflows/lint_and_test_pegic_always_pass.yml | 1 +
.github/workflows/test_nodejs-client.yml | 1 +
.github/workflows/test_python-client.yml | 1 +
.github/workflows/test_scala-client.yml | 4 +-
run.sh | 21 +-
17 files changed, 286 insertions(+), 500 deletions(-)
diff --git a/.github/actions/build_pegasus/action.yaml
b/.github/actions/build_pegasus/action.yaml
new file mode 100644
index 000000000..ad640f2fa
--- /dev/null
+++ b/.github/actions/build_pegasus/action.yaml
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Build Pegasus
+runs:
+ using: composite
+ steps:
+ - name: Setup cache
+ uses: actions/cache@v3
+ with:
+ path: |
+ /github/home/.ccache
+ key: ${{ env.ARTIFACT_NAME }}_ccache
+ - name: Free Disk Space (Ubuntu)
+ run: |
+ .github/workflows/free_disk_space.sh
+ shell: bash
+ - name: Compilation
+ run: |
+ ccache -p
+ ccache -z
+ ./run.sh build -j $(nproc) --skip_thirdparty ${BUILD_OPTIONS}
+ ccache -s
+ shell: bash
+ - name: Pack Server
+ run: |
+ ./run.sh pack_server -j
+ rm -rf pegasus-server-*
+ shell: bash
+ - name: Pack Tools
+ run: |
+ ./run.sh pack_tools -j
+ rm -rf pegasus-tools-*
+ shell: bash
+ - name: Clear Build Files
+ run: |
+ find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
+ shell: bash
diff --git a/.github/workflows/lint_and_test_pegic_always_pass.yml
b/.github/actions/download_artifact/action.yaml
similarity index 55%
copy from .github/workflows/lint_and_test_pegic_always_pass.yml
copy to .github/actions/download_artifact/action.yaml
index f4ab49298..35ddb8ae0 100644
--- a/.github/workflows/lint_and_test_pegic_always_pass.yml
+++ b/.github/actions/download_artifact/action.yaml
@@ -14,36 +14,23 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
----
-# workflow name
-name: Golang Lint and Unit Test - pegic
-# on events
-on:
- # run on each pull request
- pull_request:
- types: [ synchronize, opened, reopened ]
- branches:
- - master
- - 'v[0-9]+.*' # release branch
- - ci-test # testing branch for github action
- - '*dev'
- paths-ignore:
- - pegic/**
-
- # for manually triggering workflow
- workflow_dispatch:
-
-# workflow tasks
-jobs:
- lint:
- name: Lint
- runs-on: ubuntu-latest
- steps:
- - run: 'echo "No build required" '
-
- build:
- name: Test
- runs-on: ubuntu-latest
- steps:
- - run: 'echo "No build required" '
+name: Download artifact
+runs:
+ using: composite
+ steps:
+ - name: Unpack prebuilt third-parties
+ run: |
+ unzip /root/thirdparties-bin.zip -d ./thirdparty
+ rm -f /root/thirdparties-bin.zip
+ shell: bash
+ - name: Download tarball
+ uses: actions/download-artifact@v3
+ with:
+ name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
+ path: .
+ - name: Unpack tarball
+ run: |
+ tar -zxvf ${ARTIFACT_NAME}_builder.tar
+ rm -f ${ARTIFACT_NAME}_builder.tar
+ shell: bash
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
new file mode 100644
index 000000000..ff48fff29
--- /dev/null
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Rebuild thirdparty if needed
+runs:
+ using: composite
+ steps:
+ - uses: dorny/paths-filter@v2
+ id: changes
+ with:
+ filters: |
+ thirdparty:
+ - '.github/workflows/thirdparty-regular-push.yml'
+ - 'docker/thirdparties-src/**'
+ - 'docker/thirdparties-bin/**'
+ - 'thirdparty/**'
+ - name: Unpack prebuilt third-parties
+ if: steps.changes.outputs.thirdparty == 'false'
+ run: |
+ rm -f /root/thirdparties-src.zip
+ unzip /root/thirdparties-bin.zip -d ./thirdparty
+ rm -f /root/thirdparties-bin.zip
+ find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ rm -rf ./thirdparty/hadoop-bin/share/doc
+ rm -rf ./thirdparty/zookeeper-bin/docs
+ shell: bash
+ - name: Rebuild third-parties
+ if: steps.changes.outputs.thirdparty == 'true'
+ working-directory: thirdparty
+ # Build third-parties and leave some necessary libraries and source.
+ run: |
+ rm -f /root/thirdparties-src.zip
+ mkdir build
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1
-DUSE_JEMALLOC=${USE_JEMALLOC} -B build/
+ cmake --build build/ -j $(nproc)
+ rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
+ find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ ../scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_zk.sh zookeeper-bin
+ rm -rf hadoop-bin/share/doc
+ rm -rf zookeeper-bin/docs
+ shell: bash
diff --git a/.github/workflows/lint_and_test_pegic_always_pass.yml
b/.github/actions/run_server_tests/action.yaml
similarity index 55%
copy from .github/workflows/lint_and_test_pegic_always_pass.yml
copy to .github/actions/run_server_tests/action.yaml
index f4ab49298..816315d2b 100644
--- a/.github/workflows/lint_and_test_pegic_always_pass.yml
+++ b/.github/actions/run_server_tests/action.yaml
@@ -14,36 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
----
-# workflow name
-name: Golang Lint and Unit Test - pegic
-# on events
-on:
- # run on each pull request
- pull_request:
- types: [ synchronize, opened, reopened ]
- branches:
- - master
- - 'v[0-9]+.*' # release branch
- - ci-test # testing branch for github action
- - '*dev'
- paths-ignore:
- - pegic/**
-
- # for manually triggering workflow
- workflow_dispatch:
-
-# workflow tasks
-jobs:
- lint:
- name: Lint
- runs-on: ubuntu-latest
- steps:
- - run: 'echo "No build required" '
-
- build:
- name: Test
- runs-on: ubuntu-latest
- steps:
- - run: 'echo "No build required" '
+name: Run server tests
+runs:
+ using: composite
+ steps:
+ - name: Unit Testing
+ run: |
+ export
LD_LIBRARY_PATH=$(pwd)/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
+ ulimit -s unlimited
+ ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m
${{ matrix.test_module }}
+ shell: bash
diff --git a/.github/actions/upload_artifact/action.yaml
b/.github/actions/upload_artifact/action.yaml
new file mode 100644
index 000000000..0fb6975d7
--- /dev/null
+++ b/.github/actions/upload_artifact/action.yaml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Upload artifact
+runs:
+ using: composite
+ steps:
+ - name: Tar files
+ run: |
+ mv thirdparty/hadoop-bin ./
+ mv thirdparty/zookeeper-bin ./
+ rm -rf thirdparty
+ # The following operations are tricky, these directories and files
don't exist if not build with '--test'.
+ # When build binaries for client tests, it's not needed to add
'--test'.
+ mkdir -p build/latest/bin
+ mkdir -p build/latest/src/server/test
+ touch build/latest/src/server/test/config.ini
+ tar -zcvhf ${ARTIFACT_NAME}_builder.tar build/latest/output
build/latest/bin build/latest/src/server/test/config.ini hadoop-bin
zookeeper-bin
+ shell: bash
+ - name: Upload tarball
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }}
+ path: ${{ env.ARTIFACT_NAME }}_builder.tar
diff --git a/.github/workflows/lint_and_test_admin-cli.yml
b/.github/workflows/lint_and_test_admin-cli.yml
index ee57ac9d8..788ba8ba3 100644
--- a/.github/workflows/lint_and_test_admin-cli.yml
+++ b/.github/workflows/lint_and_test_admin-cli.yml
@@ -29,6 +29,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
+ - .github/workflows/lint_and_test_admin-cli.yml
- admin-cli/**
# for manually triggering workflow
diff --git a/.github/workflows/lint_and_test_admin-cli_always_pass.yml
b/.github/workflows/lint_and_test_admin-cli_always_pass.yml
index 5189678d0..9c3b18f37 100644
--- a/.github/workflows/lint_and_test_admin-cli_always_pass.yml
+++ b/.github/workflows/lint_and_test_admin-cli_always_pass.yml
@@ -29,6 +29,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths-ignore:
+ - .github/workflows/lint_and_test_admin-cli_always_pass.yml
- admin-cli/**
# for manually triggering workflow
diff --git a/.github/workflows/lint_and_test_collector.yml
b/.github/workflows/lint_and_test_collector.yml
index 3b0a378ae..e257dbc93 100644
--- a/.github/workflows/lint_and_test_collector.yml
+++ b/.github/workflows/lint_and_test_collector.yml
@@ -29,8 +29,8 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
- - '.github/workflows/lint_and_test_collector.yml'
- - 'collector/**'
+ - .github/workflows/lint_and_test_collector.yml
+ - collector/**
# for manually triggering workflow
workflow_dispatch:
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index 572d11b1a..f90a18074 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -14,12 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
----
-# Developer Notes:
-#
-# This config is for github actions. Before merging your changes of this file,
-# it's recommended to create a PR against the ci-test branch to test if it
works
-# as expected.
name: Cpp CI
@@ -33,17 +27,18 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
- - '.github/workflows/lint_and_test_cpp.yaml'
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'CMakeLists.txt'
- - 'run.sh'
- - 'scripts/pack_server.sh'
- - 'scripts/pack_tools.sh'
- - 'cmake_modules/**'
- - 'src/**'
- - 'thirdparty/**'
+ - .github/actions
+ - .github/workflows/lint_and_test_cpp.yaml
+ - .github/workflows/thirdparty-regular-push.yml
+ - docker/thirdparties-bin/**
+ - docker/thirdparties-src/**
+ - CMakeLists.txt
+ - cmake_modules/**
+ - run.sh
+ - scripts/pack_server.sh
+ - scripts/pack_tools.sh
+ - src/**
+ - thirdparty/**
# for manually triggering workflow
workflow_dispatch:
@@ -68,6 +63,8 @@ jobs:
name: IWYU
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
container:
image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
steps:
@@ -75,29 +72,7 @@ jobs:
- name: Free Disk Space (Ubuntu)
run: |
.github/workflows/free_disk_space.sh
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- thirdparty:
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'thirdparty/**'
- - name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
- run: unzip /root/thirdparties-bin.zip -d ./thirdparty
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
- # Build thirdparties and leave some necessary libraries and source
- run: |
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
- cmake --build build/ -j $(nproc)
- rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
- ../scripts/download_hadoop.sh hadoop-bin
- ../scripts/download_zk.sh zookeeper-bin
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
- name: Build IWYU binary
run: |
mkdir iwyu && cd iwyu && git clone
https://github.com/include-what-you-use/include-what-you-use.git
@@ -115,80 +90,17 @@ jobs:
name: Build Release
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
+ ARTIFACT_NAME: release
+ BUILD_OPTIONS: -t release --test
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
steps:
- uses: actions/checkout@v3
- - name: Setup cache
- uses: actions/cache@v3
- with:
- path: |
- /github/home/.ccache
- key: release_ccache
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- thirdparty:
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'thirdparty/**'
- - name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
- # Build thirdparties and leave some necessary libraries and source
- run: |
- rm -f /root/thirdparties-src.zip
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
- cmake --build build/ -j $(nproc)
- rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
- find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- ../scripts/download_hadoop.sh hadoop-bin
- ../scripts/download_zk.sh zookeeper-bin
- rm -rf hadoop-bin/share/doc
- rm -rf zookeeper-bin/docs
- - name: Compilation
- run: |
- ccache -p
- ccache -z
- ./run.sh build --test --skip_thirdparty -j $(nproc) -t release
- ccache -s
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
- - name: Pack Server
- run: |
- ./run.sh pack_server
- rm -rf pegasus-server-*
- - name: Pack Tools
- run: |
- ./run.sh pack_tools
- rm -rf pegasus-tools-*
- - name: Tar files
- run: |
- mv thirdparty/hadoop-bin ./
- mv thirdparty/zookeeper-bin ./
- rm -rf thirdparty
- tar -zcvhf release__builder.tar build/latest/output build/latest/bin
build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
- - name: Upload Artifact
- uses: actions/upload-artifact@v3
- with:
- name: release_artifact_${{ github.sha }}
- path: release__builder.tar
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
test_Release:
name: Test Release
@@ -238,100 +150,31 @@ jobs:
- throttle_test
needs: build_Release
runs-on: ubuntu-latest
+ env:
+ ARTIFACT_NAME: release
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- - name: Download Artifact
- uses: actions/download-artifact@v3
- with:
- name: release_artifact_${{ github.sha }}
- path: .
- - name: Tar files
- run: |
- tar -zxvf release__builder.tar
- rm -f release__builder.tar
- - name: Unit Testing
- run: |
- export
LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
- ulimit -s unlimited
- ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS"
-m ${{ matrix.test_module }}
+ - uses: "./.github/actions/download_artifact"
+ - uses: "./.github/actions/run_server_tests"
build_ASAN:
name: Build ASAN
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
+ ARTIFACT_NAME: release_address
+ BUILD_OPTIONS: --sanitizer address --disable_gperf --test
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
steps:
- uses: actions/checkout@v3
- - name: Setup cache
- uses: actions/cache@v3
- with:
- path: |
- /github/home/.ccache
- key: asan_ccache
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- thirdparty:
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'thirdparty/**'
- - name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
- # Build thirdparties and leave some necessary libraries and source
- run: |
- rm -f /root/thirdparties-src.zip
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
- cmake --build build/ -j $(nproc)
- rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
- find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- ../scripts/download_hadoop.sh hadoop-bin
- ../scripts/download_zk.sh zookeeper-bin
- rm -rf hadoop-bin/share/doc
- rm -rf zookeeper-bin/docs
- - name: Compilation
- run: |
- ccache -p
- ccache -z
- ./run.sh build --test --sanitizer address --skip_thirdparty
--disable_gperf -j $(nproc)
- ccache -s
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
- - name: Tar files
- run: |
- mv thirdparty/hadoop-bin ./
- mv thirdparty/zookeeper-bin ./
- rm -rf thirdparty
- tar -zcvhf release_address_builder.tar build/latest/output
build/latest/bin build/latest/src/server/test/config.ini hadoop-bin
zookeeper-bin
- - name: Upload Artifact
- uses: actions/upload-artifact@v3
- with:
- name: release_address_artifact_${{ github.sha }}
- path: release_address_builder.tar
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
test_ASAN:
name: Test ASAN
@@ -383,29 +226,15 @@ jobs:
# - throttle_test
needs: build_ASAN
runs-on: ubuntu-latest
+ env:
+ ARTIFACT_NAME: release_address
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- - name: Download Artifact
- uses: actions/download-artifact@v3
- with:
- name: release_address_artifact_${{ github.sha }}
- path: .
- - name: Tar files
- run: |
- tar -zxvf release_address_builder.tar
- rm -f release_address_builder.tar
- - name: Unit Testing
- run: |
- export
LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
- ulimit -s unlimited
- ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS"
-m ${{ matrix.test_module }}
+ - uses: "./.github/actions/download_artifact"
+ - uses: "./.github/actions/run_server_tests"
# TODO(yingchun): Build and test UBSAN version would cost a very long time, we
will run these tests
# when we are going to release a stable version. So we disable
them in regular CI
@@ -414,72 +243,17 @@ jobs:
# name: Build UBSAN
# needs: cpp_clang_format_linter
# runs-on: ubuntu-latest
+# env:
+# USE_JEMALLOC: OFF
+# ARTIFACT_NAME: release_undefined
+# BUILD_OPTIONS: --sanitizer undefined --disable_gperf --test
# container:
# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
# steps:
# - uses: actions/checkout@v3
-# - name: Setup cache
-# uses: actions/cache@v3
-# with:
-# path: |
-# /github/home/.ccache
-# key: ubsan_ccache
-# - name: Free Disk Space (Ubuntu)
-# run: |
-# .github/workflows/free_disk_space.sh
-# - uses: dorny/paths-filter@v2
-# id: changes
-# with:
-# filters: |
-# thirdparty:
-# - '.github/workflows/thirdparty-regular-push.yml'
-# - 'docker/thirdparties-src/**'
-# - 'docker/thirdparties-bin/**'
-# - 'thirdparty/**'
-# - name: Unpack prebuilt third-parties
-# if: steps.changes.outputs.thirdparty == 'false'
-# run: |
-# rm -f /root/thirdparties-src.zip
-# unzip /root/thirdparties-bin.zip -d ./thirdparty
-# rm -f /root/thirdparties-bin.zip
-# find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-# rm -rf ./thirdparty/hadoop-bin/share/doc
-# rm -rf ./thirdparty/zookeeper-bin/docs
-# - name: Rebuild third-parties
-# if: steps.changes.outputs.thirdparty == 'true'
-# working-directory: thirdparty
-# # Build thirdparties and leave some necessary libraries and source
-# run: |
-# rm -f /root/thirdparties-src.zip
-# mkdir build
-# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
-# cmake --build build/ -j $(nproc)
-# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
-# find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
-# ../scripts/download_hadoop.sh hadoop-bin
-# ../scripts/download_zk.sh zookeeper-bin
-# rm -rf hadoop-bin/share/doc
-# rm -rf zookeeper-bin/docs
-# - name: Compilation
-# run: |
-# ccache -p
-# ccache -z
-# ./run.sh build --test --sanitizer undefined --skip_thirdparty
--disable_gperf -j $(nproc)
-# ccache -s
-# - name: Clear Build Files
-# run: |
-# find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
-# - name: Tar files
-# run: |
-# mv thirdparty/hadoop-bin ./
-# mv thirdparty/zookeeper-bin ./
-# rm -rf thirdparty
-# tar -zcvhf release_undefined_builder.tar build/latest/output
build/latest/bin build/latest/src/server/test/config.ini hadoop-bin
zookeeper-bin
-# - name: Upload Artifact
-# uses: actions/upload-artifact@v3
-# with:
-# name: release_undefined_artifact_${{ github.sha }}
-# path: release_undefined_builder.tar
+# - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+# - uses: "./.github/actions/build_pegasus"
+# - uses: "./.github/actions/upload_artifact"
#
# test_UBSAN:
# name: Test UBSAN
@@ -527,112 +301,32 @@ jobs:
# - throttle_test
# needs: build_UBSAN
# runs-on: ubuntu-latest
+# env:
+# ARTIFACT_NAME: release_undefined
# container:
# image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
# options: --cap-add=SYS_PTRACE
# steps:
# - uses: actions/checkout@v3
-# - name: Free Disk Space (Ubuntu)
-# run: |
-# .github/workflows/free_disk_space.sh
-# - name: Unpack prebuilt third-parties
-# run: |
-# unzip /root/thirdparties-bin.zip -d ./thirdparty
-# rm -f /root/thirdparties-bin.zip
-# - name: Download Artifact
-# uses: actions/download-artifact@v3
-# with:
-# name: release_undefined_artifact_${{ github.sha }}
-# path: .
-# - name: Tar files
-# run: |
-# tar -zxvf release_undefined_builder.tar
-# rm -f release_undefined_builder.tar
-# - name: Unit Testing
-# run: |
-# export
LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
-# ulimit -s unlimited
-# ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS"
-m ${{ matrix.test_module }}
+# - uses: "./.github/actions/download_artifact"
+# - uses: "./.github/actions/run_server_tests"
build_with_jemalloc:
name: Build with jemalloc
needs: cpp_clang_format_linter
runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: ON
+ ARTIFACT_NAME: release_jemalloc
+ BUILD_OPTIONS: -t release --use_jemalloc --test
container:
image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{
github.base_ref }}
steps:
- uses: actions/checkout@v3
- - name: Setup cache
- uses: actions/cache@v3
- with:
- path: |
- /github/home/.ccache
- key: jemalloc_ccache
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- thirdparty:
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'thirdparty/**'
- - name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
- # Build thirdparties and leave some necessary libraries and source
- run: |
- rm -f /root/thirdparties-src.zip
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1
-DUSE_JEMALLOC=ON -B build/
- cmake --build build/ -j $(nproc)
- rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
- find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- ../scripts/download_hadoop.sh hadoop-bin
- ../scripts/download_zk.sh zookeeper-bin
- rm -rf hadoop-bin/share/doc
- rm -rf zookeeper-bin/docs
- - name: Compilation
- # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not
needed to pack server or tools, for example, the dependencies are static linked.
- run: |
- ccache -p
- ccache -z
- ./run.sh build --test --skip_thirdparty -j $(nproc) -t release
--use_jemalloc
- ccache -s
- - name: Clear Build Files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
- - name: Pack Server
- run: |
- ./run.sh pack_server -j
- rm -rf pegasus-server-*
- - name: Pack Tools
- run: |
- ./run.sh pack_tools -j
- rm -rf pegasus-tools-*
- - name: Tar files
- run: |
- mv thirdparty/hadoop-bin ./
- mv thirdparty/zookeeper-bin ./
- rm -rf thirdparty
- tar -zcvhf release_jemalloc_builder.tar build/latest/output
build/latest/bin build/latest/src/server/test/config.ini hadoop-bin
zookeeper-bin
- - name: Upload Artifact
- uses: actions/upload-artifact@v3
- with:
- name: release_jemalloc_artifact_${{ github.sha }}
- path: release_jemalloc_builder.tar
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not
needed to pack server or tools, for example, the dependencies are static linked.
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
test_with_jemalloc:
name: Test with jemallc
@@ -643,29 +337,15 @@ jobs:
- dsn_utils_tests
needs: build_with_jemalloc
runs-on: ubuntu-latest
+ env:
+ ARTIFACT_NAME: release_jemalloc
container:
image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{
github.base_ref }}
options: --cap-add=SYS_PTRACE
steps:
- uses: actions/checkout@v3
- - name: Unpack prebuilt third-parties
- run: |
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- - name: Download Artifact
- uses: actions/download-artifact@v3
- with:
- name: release_jemalloc_artifact_${{ github.sha }}
- path: .
- - name: Tar files
- run: |
- tar -zxvf release_jemalloc_builder.tar
- rm -f release_jemalloc_builder.tar
- - name: Unit Testing
- run: |
- export
LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
- ulimit -s unlimited
- ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS"
-m ${{ matrix.test_module }}
+ - uses: "./.github/actions/download_artifact"
+ - uses: "./.github/actions/run_server_tests"
build_pegasus_on_macos:
name: macOS
diff --git a/.github/workflows/lint_and_test_go-client.yml
b/.github/workflows/lint_and_test_go-client.yml
index 2b660a022..011ae97c3 100644
--- a/.github/workflows/lint_and_test_go-client.yml
+++ b/.github/workflows/lint_and_test_go-client.yml
@@ -29,6 +29,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
+ - .github/workflows/lint_and_test_go-client.yml
- go-client/**
# for manually triggering workflow
diff --git a/.github/workflows/test_java-client.yml
b/.github/workflows/lint_and_test_java-client.yml
similarity index 54%
rename from .github/workflows/test_java-client.yml
rename to .github/workflows/lint_and_test_java-client.yml
index a3930b963..dfe747bd8 100644
--- a/.github/workflows/test_java-client.yml
+++ b/.github/workflows/lint_and_test_java-client.yml
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
name: Test - java client
on:
@@ -26,6 +25,7 @@ on:
- ci-test # testing branch for github action
- '*dev' # developing branch
paths:
+ - .github/workflows/lint_and_test_java-client.yml
- java-client/**
workflow_dispatch:
@@ -36,7 +36,7 @@ defaults:
jobs:
spotless:
- name: spotless
+ name: Spotless
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@@ -46,10 +46,29 @@ jobs:
- name: spotless
working-directory: ./java-client
run: mvn spotless:check
- test:
- name: test
+
+ build_server:
+ name: Build server
needs: spotless
runs-on: ubuntu-latest
+ env:
+ USE_JEMALLOC: OFF
+ ARTIFACT_NAME: release_for_java_client
+ BUILD_OPTIONS: -t release
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
+ steps:
+ - uses: actions/checkout@v3
+ - uses: "./.github/actions/rebuild_thirdparty_if_needed"
+ - uses: "./.github/actions/build_pegasus"
+ - uses: "./.github/actions/upload_artifact"
+
+ test_java_client:
+ name: Test Java client
+ needs: build_server
+ runs-on: ubuntu-latest
+ env:
+ ARTIFACT_NAME: release_for_java_client
container:
image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
strategy:
@@ -64,52 +83,10 @@ jobs:
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- - name: Free Disk Space (Ubuntu)
- run: |
- .github/workflows/free_disk_space.sh
- - uses: dorny/paths-filter@v2
- id: changes
- with:
- filters: |
- thirdparty:
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - 'thirdparty/**'
- - name: Unpack prebuilt third-parties
- if: steps.changes.outputs.thirdparty == 'false'
- run: |
- rm -f /root/thirdparties-src.zip
- unzip /root/thirdparties-bin.zip -d ./thirdparty
- rm -f /root/thirdparties-bin.zip
- find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- rm -rf ./thirdparty/hadoop-bin/share/doc
- rm -rf ./thirdparty/zookeeper-bin/docs
- # TODO(yingchun): Build a docker image with prebuilt server binaries, we
can use them directly without building server binaries if no server changes.
- - name: Rebuild third-parties
- if: steps.changes.outputs.thirdparty == 'true'
- working-directory: thirdparty
- # Build thirdparties and leave some necessary libraries and source
- run: |
- rm -f /root/thirdparties-src.zip
- mkdir build
- cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
- cmake --build build/ -j $(nproc)
- rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
- find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
- ../scripts/download_hadoop.sh hadoop-bin
- ../scripts/download_zk.sh zookeeper-bin
- rm -rf hadoop-bin/share/doc
- rm -rf zookeeper-bin/docs
- - name: Build Pegasus server
- run: |
- ./run.sh build --skip_thirdparty -j $(nproc) -t release
- - name: Clear build temporary files
- run: |
- find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
+ - uses: "./.github/actions/download_artifact"
- name: Start Pegasus cluster
run: |
apt-get update
diff --git a/.github/workflows/lint_and_test_pegic.yml
b/.github/workflows/lint_and_test_pegic.yml
index 09b85fa92..3788b4412 100644
--- a/.github/workflows/lint_and_test_pegic.yml
+++ b/.github/workflows/lint_and_test_pegic.yml
@@ -29,6 +29,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
+ - .github/workflows/lint_and_test_pegic.yml
- pegic/**
# for manually triggering workflow
diff --git a/.github/workflows/lint_and_test_pegic_always_pass.yml
b/.github/workflows/lint_and_test_pegic_always_pass.yml
index f4ab49298..2a9fb58d9 100644
--- a/.github/workflows/lint_and_test_pegic_always_pass.yml
+++ b/.github/workflows/lint_and_test_pegic_always_pass.yml
@@ -29,6 +29,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths-ignore:
+ - .github/workflows/lint_and_test_pegic_always_pass.yml
- pegic/**
# for manually triggering workflow
diff --git a/.github/workflows/test_nodejs-client.yml
b/.github/workflows/test_nodejs-client.yml
index a3d7005c1..b24065ec9 100644
--- a/.github/workflows/test_nodejs-client.yml
+++ b/.github/workflows/test_nodejs-client.yml
@@ -26,6 +26,7 @@ on:
- ci-test # testing branch for github action
- '*dev' # developing branch
paths:
+ - .github/workflows/test_nodejs-client.yml
- nodejs-client/**
workflow_dispatch:
diff --git a/.github/workflows/test_python-client.yml
b/.github/workflows/test_python-client.yml
index 623c0cfc8..bdef9d221 100644
--- a/.github/workflows/test_python-client.yml
+++ b/.github/workflows/test_python-client.yml
@@ -26,6 +26,7 @@ on:
- ci-test # testing branch for github action
- '*dev' # developing branch
paths:
+ - .github/workflows/test_python-client.yml
- python-client/**
workflow_dispatch:
diff --git a/.github/workflows/test_scala-client.yml
b/.github/workflows/test_scala-client.yml
index 1f9eb044a..047e55042 100644
--- a/.github/workflows/test_scala-client.yml
+++ b/.github/workflows/test_scala-client.yml
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
name: Test - scala client
on:
@@ -24,7 +23,8 @@ on:
- master
- 'v[0-9]+.*' # release branch
paths:
- - scala-client/**
+ - .github/workflows/test_scala-client.yml
+ - scala-client/**
workflow_dispatch:
diff --git a/run.sh b/run.sh
index c73d3e529..dfae368f4 100755
--- a/run.sh
+++ b/run.sh
@@ -455,6 +455,7 @@ function run_test()
mkdir -p $REPORT_DIR
fi
+ # Run all tests if none specified.
if [ "$test_modules" == "" ]; then
test_modules=$(IFS=,; echo "${all_tests[*]}")
fi
@@ -462,7 +463,7 @@ function run_test()
for module in `echo $test_modules | sed 's/,/ /g'`; do
echo "====================== run $module =========================="
- # restart onebox when test pegasus
+ # The tests which need start onebox.
local need_onebox_tests=(
backup_restore_test
base_api_test
@@ -475,11 +476,14 @@ function run_test()
restore_test
throttle_test
)
+ # Restart onebox if needed.
if [[ "${need_onebox_tests[@]}" =~ "${module}" ]]; then
+ # Clean up onebox at first.
run_clear_onebox
- m_count=3
+ master_count=3
+ # Update options if needed, this should be done before starting
onebox to make new options take effect.
if [ "${module}" == "recovery_test" ]; then
- m_count=1
+ master_count=1
opts="meta_state_service_type=meta_state_service_simple,distributed_lock_service_type=distributed_lock_service_simple"
fi
if [ "${module}" == "backup_restore_test" ]; then
@@ -488,24 +492,30 @@ function run_test()
if [ "${module}" == "restore_test" ]; then
opts="cold_backup_disabled=false,cold_backup_checkpoint_reserve_minutes=0,cold_backup_root=onebox"
fi
+ # Append onebox_opts if needed.
[ -z ${onebox_opts} ] || opts="${opts},${onebox_opts}"
- if ! run_start_onebox -m ${m_count} -w -c --opts ${opts}; then
+ # Start onebox.
+ if ! run_start_onebox -m ${master_count} -w -c --opts ${opts}; then
echo "ERROR: unable to continue on testing because starting
onebox failed"
exit 1
fi
# TODO(yingchun): remove it?
sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g"
${BUILD_LATEST_DIR}/src/server/test/config.ini
else
+ # Restart ZK in what ever case.
run_stop_zk
run_start_zk
fi
+
+ # Run server test.
pushd ${BUILD_LATEST_DIR}/bin/${module}
REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts}
./run.sh
if [ $? != 0 ]; then
echo "run test \"$module\" in `pwd` failed"
exit 1
fi
- # clear onebox if needed
+
+ # Clear onebox if needed.
if [[ "${need_onebox_tests[@]}" =~ "${test_modules}" ]]; then
if [ "$clear_flags" == "1" ]; then
run_clear_onebox
@@ -519,6 +529,7 @@ function run_test()
used_time=$((finish_time-start_time))
echo "Test elapsed time: $((used_time/60))m $((used_time%60))s"
+ # TODO(yingchun): make sure if gcov can be ran normally.
if [ "$enable_gcov" == "yes" ]; then
echo "Generating gcov report..."
cd $ROOT
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
