This is an automated email from the ASF dual-hosted git repository. laiyingchun pushed a commit to branch ci_refactor_more in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 5a37c6a49d30e4cbf1819bd56f2e284265f4c336 Author: Yingchun Lai <[email protected]> AuthorDate: Fri Dec 22 16:42:45 2023 +0800 chore(CI): Improve github actions (2nd) --- .github/actions/download_artifact/action.yaml | 5 +- .../rebuild_thirdparty_if_needed/action.yaml | 9 +-- .../action.yaml | 18 ++---- .github/workflows/lint_and_test_cpp.yaml | 66 ++++++++++++++-------- .github/workflows/lint_and_test_java-client.yml | 12 ++-- .github/workflows/regular-build.yml | 27 ++------- 6 files changed, 64 insertions(+), 73 deletions(-) diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/download_artifact/action.yaml index 35ddb8ae0..29d68973c 100644 --- a/.github/actions/download_artifact/action.yaml +++ b/.github/actions/download_artifact/action.yaml @@ -20,10 +20,7 @@ runs: using: composite steps: - name: Unpack prebuilt third-parties - run: | - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - shell: bash + uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml" - name: Download tarball uses: actions/download-artifact@v3 with: diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml b/.github/actions/rebuild_thirdparty_if_needed/action.yaml index ff48fff29..035ae47d5 100644 --- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml +++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml @@ -30,14 +30,7 @@ runs: - 'thirdparty/**' - name: Unpack prebuilt third-parties if: steps.changes.outputs.thirdparty == 'false' - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - shell: bash + uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml" - name: Rebuild third-parties if: steps.changes.outputs.thirdparty == 'true' working-directory: thirdparty diff --git a/.github/actions/download_artifact/action.yaml b/.github/actions/unpack_prebuilt_thirdparties/action.yaml similarity index 71% copy from .github/actions/download_artifact/action.yaml copy to .github/actions/unpack_prebuilt_thirdparties/action.yaml index 35ddb8ae0..0e1b2cdaf 100644 --- a/.github/actions/download_artifact/action.yaml +++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml @@ -15,22 +15,16 @@ # specific language governing permissions and limitations # under the License. -name: Download artifact +name: Unpack prebuilt third-parties runs: using: composite steps: - - name: Unpack prebuilt third-parties + - name: Unpack run: | + rm -f /root/thirdparties-src.zip unzip /root/thirdparties-bin.zip -d ./thirdparty rm -f /root/thirdparties-bin.zip - shell: bash - - name: Download tarball - uses: actions/download-artifact@v3 - with: - name: ${{ env.ARTIFACT_NAME }}_artifact_${{ github.sha }} - path: . - - name: Unpack tarball - run: | - tar -zxvf ${ARTIFACT_NAME}_builder.tar - rm -f ${ARTIFACT_NAME}_builder.tar + find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + + rm -rf ./thirdparty/hadoop-bin/share/doc + rm -rf ./thirdparty/zookeeper-bin/docs shell: bash diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index f90a18074..48abc8e19 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -72,7 +72,8 @@ jobs: - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh - - uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build IWYU binary run: | mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git @@ -97,10 +98,14 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 - - uses: "./.github/actions/rebuild_thirdparty_if_needed" - - uses: "./.github/actions/build_pegasus" - - uses: "./.github/actions/upload_artifact" + - name: Clone code + uses: actions/checkout@v3 + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: Build Pegasus + uses: "./.github/actions/build_pegasus" + - name: Upload artifact + uses: "./.github/actions/upload_artifact" test_Release: name: Test Release @@ -157,8 +162,10 @@ jobs: options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/download_artifact" - - uses: "./.github/actions/run_server_tests" + - name: Download artifact + uses: "./.github/actions/download_artifact" + - name: Run server tests + uses: "./.github/actions/run_server_tests" build_ASAN: name: Build ASAN @@ -172,9 +179,12 @@ jobs: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/rebuild_thirdparty_if_needed" - - uses: "./.github/actions/build_pegasus" - - uses: "./.github/actions/upload_artifact" + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: Build Pegasus + uses: "./.github/actions/build_pegasus" + - name: Upload artifact + uses: "./.github/actions/upload_artifact" test_ASAN: name: Test ASAN @@ -233,8 +243,10 @@ jobs: options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/download_artifact" - - uses: "./.github/actions/run_server_tests" + - name: Download artifact + uses: "./.github/actions/download_artifact" + - name: Run server tests + uses: "./.github/actions/run_server_tests" # TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests # when we are going to release a stable version. So we disable them in regular CI @@ -251,9 +263,12 @@ jobs: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: # - uses: actions/checkout@v3 -# - uses: "./.github/actions/rebuild_thirdparty_if_needed" -# - uses: "./.github/actions/build_pegasus" -# - uses: "./.github/actions/upload_artifact" +# - name: Rebuild thirdparty if needed +# uses: "./.github/actions/rebuild_thirdparty_if_needed" +# - name: Build Pegasus +# uses: "./.github/actions/build_pegasus" +# - name: Upload artifact +# uses: "./.github/actions/upload_artifact" # # test_UBSAN: # name: Test UBSAN @@ -308,8 +323,10 @@ jobs: # options: --cap-add=SYS_PTRACE # steps: # - uses: actions/checkout@v3 -# - uses: "./.github/actions/download_artifact" -# - uses: "./.github/actions/run_server_tests" +# - name: Download artifact +# uses: "./.github/actions/download_artifact" +# - name: Run server tests +# uses: "./.github/actions/run_server_tests" build_with_jemalloc: name: Build with jemalloc @@ -323,10 +340,13 @@ jobs: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked. - - uses: "./.github/actions/build_pegasus" - - uses: "./.github/actions/upload_artifact" + - name: Build Pegasus + uses: "./.github/actions/build_pegasus" + - name: Upload artifact + uses: "./.github/actions/upload_artifact" test_with_jemalloc: name: Test with jemallc @@ -344,8 +364,10 @@ jobs: options: --cap-add=SYS_PTRACE steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/download_artifact" - - uses: "./.github/actions/run_server_tests" + - name: Download artifact + uses: "./.github/actions/download_artifact" + - name: Run server tests + uses: "./.github/actions/run_server_tests" build_pegasus_on_macos: name: macOS diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml index dfe747bd8..119877d0e 100644 --- a/.github/workflows/lint_and_test_java-client.yml +++ b/.github/workflows/lint_and_test_java-client.yml @@ -59,9 +59,12 @@ jobs: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - uses: actions/checkout@v3 - - uses: "./.github/actions/rebuild_thirdparty_if_needed" - - uses: "./.github/actions/build_pegasus" - - uses: "./.github/actions/upload_artifact" + - name: Rebuild thirdparty if needed + uses: "./.github/actions/rebuild_thirdparty_if_needed" + - name: Build Pegasus + uses: "./.github/actions/build_pegasus" + - name: Upload artifact + uses: "./.github/actions/upload_artifact" test_java_client: name: Test Java client @@ -86,7 +89,8 @@ jobs: - uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - uses: "./.github/actions/download_artifact" + - name: Download artifact + uses: "./.github/actions/download_artifact" - name: Start Pegasus cluster run: | apt-get update diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index f536479be..da20c54d8 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -72,30 +72,11 @@ jobs: working-directory: /root/incubator-pegasus steps: - name: Clone Apache Pegasus Source - working-directory: /root - run: | - git clone -b ${{ github.ref_name }} --depth=1 https://github.com/apache/incubator-pegasus.git - - name: Free Disk Space (Ubuntu) - run: | - .github/workflows/free_disk_space.sh + uses: actions/checkout@v3 - name: Unpack prebuilt third-parties - run: | - rm -f /root/thirdparties-src.zip - unzip /root/thirdparties-bin.zip -d ./thirdparty - rm -f /root/thirdparties-bin.zip - find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - rm -rf ./thirdparty/hadoop-bin/share/doc - rm -rf ./thirdparty/zookeeper-bin/docs - - name: Compilation Pegasus - run: | - ./run.sh build --test --compiler ${{ matrix.compiler }} --skip_thirdparty -j $(nproc) - - name: Clear Build Files - run: | - find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf "{}" + - - name: Packaging Server - run: | - ./run.sh pack_server - rm -rf pegasus-server-* + uses: ".github/actions/unpack_prebuilt_thirdparties/action.yaml" + - name: Build Pegasus + uses: "./.github/actions/build_pegasus" build_and_lint_go: name: Build and Lint Golang --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
