This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 223b5c708 chore(java-client-ci): build the server instead of
downloading it from external site (#1778)
223b5c708 is described below
commit 223b5c708ff63244e2f5202ed10e980a256110ea
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Dec 19 11:54:46 2023 +0800
chore(java-client-ci): build the server instead of downloading it from
external site (#1778)
https://github.com/apache/incubator-pegasus/issues/1779
Build Pegasus server binaries in the CI procedure instead of downloading it
from any site.
---
.github/workflows/test_java-client.yml | 75 ++++++++++++++++++++++++++++++++--
java-client/scripts/ci-test.sh | 72 --------------------------------
2 files changed, 72 insertions(+), 75 deletions(-)
diff --git a/.github/workflows/test_java-client.yml
b/.github/workflows/test_java-client.yml
index e6f0c9b31..a3930b963 100644
--- a/.github/workflows/test_java-client.yml
+++ b/.github/workflows/test_java-client.yml
@@ -35,23 +35,92 @@ defaults:
shell: bash
jobs:
+ spotless:
+ name: spotless
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-java@v1
+ with:
+ java-version: 8
+ - name: spotless
+ working-directory: ./java-client
+ run: mvn spotless:check
test:
name: test
+ needs: spotless
runs-on: ubuntu-latest
+ container:
+ image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{
github.base_ref }}
strategy:
+ fail-fast: false
matrix:
java: [ '8', '11']
steps:
+ - uses: actions/checkout@v3
- uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- - uses: actions/checkout@v3
+ - name: Free Disk Space (Ubuntu)
+ run: |
+ .github/workflows/free_disk_space.sh
+ - uses: dorny/paths-filter@v2
+ id: changes
+ with:
+ filters: |
+ thirdparty:
+ - '.github/workflows/thirdparty-regular-push.yml'
+ - 'docker/thirdparties-src/**'
+ - 'docker/thirdparties-bin/**'
+ - 'thirdparty/**'
+ - name: Unpack prebuilt third-parties
+ if: steps.changes.outputs.thirdparty == 'false'
+ run: |
+ rm -f /root/thirdparties-src.zip
+ unzip /root/thirdparties-bin.zip -d ./thirdparty
+ rm -f /root/thirdparties-bin.zip
+ find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ rm -rf ./thirdparty/hadoop-bin/share/doc
+ rm -rf ./thirdparty/zookeeper-bin/docs
+ # TODO(yingchun): Build a docker image with prebuilt server binaries, we
can use them directly without building server binaries if no server changes.
+ - name: Rebuild third-parties
+ if: steps.changes.outputs.thirdparty == 'true'
+ working-directory: thirdparty
+ # Build thirdparties and leave some necessary libraries and source
+ run: |
+ rm -f /root/thirdparties-src.zip
+ mkdir build
+ cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=1 -B build/
+ cmake --build build/ -j $(nproc)
+ rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]*
build/Source/[i-q]* build/Source/[s-z]*
+ find ./ -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
+ ../scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_zk.sh zookeeper-bin
+ rm -rf hadoop-bin/share/doc
+ rm -rf zookeeper-bin/docs
+ - name: Build Pegasus server
+ run: |
+ ./run.sh build --skip_thirdparty -j $(nproc) -t release
+ - name: Clear build temporary files
+ run: |
+ find ./build/latest/src/ -name '*CMakeFiles*' -type d -exec rm -rf
"{}" +
- uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- - name: ci
+ - name: Start Pegasus cluster
+ run: |
+ apt-get update
+ apt-get -y install tree
+ tree ${JAVA_HOME}
+ export
LD_LIBRARY_PATH=$(pwd)/thirdparty/output/lib:${JAVA_HOME}/jre/lib/amd64/server
+ ulimit -s unlimited
+ ./run.sh start_onebox
+ - name: Recompile thrift
+ working-directory: ./java-client/scripts
+ run: ./recompile_thrift.sh
+ - name: Run Java client tests
working-directory: ./java-client
- run: ./scripts/ci-test.sh
+ run: mvn test --no-transfer-progress
diff --git a/java-client/scripts/ci-test.sh b/java-client/scripts/ci-test.sh
deleted file mode 100755
index 935ee9636..000000000
--- a/java-client/scripts/ci-test.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-set -e
-
-SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
-PROJECT_DIR=$(dirname "${SCRIPT_DIR}")
-cd "${PROJECT_DIR}" || exit 1
-
-# lint all scripts, abort if there's any warning.
-function shellcheck_must_pass()
-{
- if [[ $(shellcheck "$1") ]]; then
- echo "shellcheck $1 failed"
- shellcheck "$1"
- exit 1
- fi
-}
-shellcheck_must_pass ./scripts/ci-test.sh
-
-# The new version of pegasus client is not compatible with old version server
which contains old rpc protocol,
-# So we use snapshot version of pegasus-tools, because we don`t have a new
release version, which contains the new version of rpc protocol,
-# TODO: This is a temp url. We will change it later.
-PEGASUS_PKG="pegasus-tools-2.1.SNAPSHOT-dc4c710-glibc2.17-release"
-PEGASUS_PKG_URL="https://github.com/levy5307/pegasus-tools/raw/master/pegasus-tools-2.1.SNAPSHOT-dc4c710-glibc2.17-release.tar.gz";
-
-# check format
-if ! mvn spotless:check
-then
- exit 1
-fi
-
-# start pegasus onebox environment
-if [ ! -f $PEGASUS_PKG.tar.gz ]; then
- wget $PEGASUS_PKG_URL
- tar xvf $PEGASUS_PKG.tar.gz
-fi
-cd $PEGASUS_PKG
-
-./run.sh start_onebox -w
-cd ../
-
-
-pushd scripts
-echo "run recompile_thrift.sh"
-./recompile_thrift.sh
-popd
-mvn spotless:apply
-
-if ! mvn clean test
-then
- cd $PEGASUS_PKG
- ./run.sh list_onebox
- exit 1
-fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
