This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new d91a6e7a4 fix(github): fix redundant `hadoop-bin` while uploading
artifact and build `thirdparties-src` image on each OS version (#2134)
d91a6e7a4 is described below
commit d91a6e7a492d9d53afae4d1f543d93581114ea64
Author: Dan Wang <[email protected]>
AuthorDate: Mon Oct 21 16:48:11 2024 +0800
fix(github): fix redundant `hadoop-bin` while uploading artifact and build
`thirdparties-src` image on each OS version (#2134)
Resolve https://github.com/apache/incubator-pegasus/issues/2136:
- move `hadoop-bin` dir into the specified position at the stage of
rebuilding/unpacking
`thirdparty` instead of uploading artifact;
- build `thirdparties-src` image on Ubuntu instead of EOL CentOS 7 to make
downstream
workflows work.
---
.../rebuild_thirdparty_if_needed/action.yaml | 2 ++
.../unpack_prebuilt_thirdparties/action.yaml | 2 ++
.github/actions/upload_artifact/action.yaml | 2 --
.github/workflows/lint_and_test_cpp.yaml | 2 +-
.github/workflows/thirdparty-regular-push.yml | 34 +++++++++++++---------
admin_tools/download_hadoop.sh | 12 +++++---
admin_tools/download_package.sh | 9 ++----
admin_tools/download_zk.sh | 8 +++--
docker/README.md | 5 +++-
docker/thirdparties-bin/Dockerfile | 6 ++--
docker/thirdparties-src/Dockerfile | 3 +-
11 files changed, 51 insertions(+), 34 deletions(-)
diff --git a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
index 16acc2053..d0ebc3fbf 100644
--- a/.github/actions/rebuild_thirdparty_if_needed/action.yaml
+++ b/.github/actions/rebuild_thirdparty_if_needed/action.yaml
@@ -46,4 +46,6 @@ runs:
../admin_tools/download_zk.sh zookeeper-bin
rm -rf hadoop-bin/share/doc
rm -rf zookeeper-bin/docs
+ mv hadoop-bin ..
+ mv zookeeper-bin ..
shell: bash
diff --git a/.github/actions/unpack_prebuilt_thirdparties/action.yaml
b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
index 0e1b2cdaf..ac6841bdb 100644
--- a/.github/actions/unpack_prebuilt_thirdparties/action.yaml
+++ b/.github/actions/unpack_prebuilt_thirdparties/action.yaml
@@ -27,4 +27,6 @@ runs:
find ./thirdparty -name '*CMakeFiles*' -type d -exec rm -rf "{}" +
rm -rf ./thirdparty/hadoop-bin/share/doc
rm -rf ./thirdparty/zookeeper-bin/docs
+ mv thirdparty/hadoop-bin ./
+ mv thirdparty/zookeeper-bin ./
shell: bash
diff --git a/.github/actions/upload_artifact/action.yaml
b/.github/actions/upload_artifact/action.yaml
index 0fb6975d7..a58606f9e 100644
--- a/.github/actions/upload_artifact/action.yaml
+++ b/.github/actions/upload_artifact/action.yaml
@@ -21,8 +21,6 @@ runs:
steps:
- name: Tar files
run: |
- mv thirdparty/hadoop-bin ./
- mv thirdparty/zookeeper-bin ./
rm -rf thirdparty
# The following operations are tricky, these directories and files
don't exist if not build with '--test'.
# When build binaries for client tests, it's not needed to add
'--test'.
diff --git a/.github/workflows/lint_and_test_cpp.yaml
b/.github/workflows/lint_and_test_cpp.yaml
index bbe257183..c30a82d42 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -27,7 +27,7 @@ on:
- ci-test # testing branch for github action
- '*dev'
paths:
- - .github/actions
+ - .github/actions/**
- .github/workflows/lint_and_test_cpp.yaml
- .github/workflows/thirdparty-regular-push.yml
- build_tools/pack_server.sh
diff --git a/.github/workflows/thirdparty-regular-push.yml
b/.github/workflows/thirdparty-regular-push.yml
index ce8f5d5dc..6ce68aae1 100644
--- a/.github/workflows/thirdparty-regular-push.yml
+++ b/.github/workflows/thirdparty-regular-push.yml
@@ -42,6 +42,15 @@ on:
jobs:
build_push_src_docker_images:
runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ osversion:
+ - ubuntu1804
+ - ubuntu2004
+ - ubuntu2204
+ # TODO(wangdan): disable centos7 temporarily before image
build-env-centos7-* is fixed.
+ # - centos7
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
@@ -60,11 +69,12 @@ jobs:
file: ./docker/thirdparties-src/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-src-${{ github.ref_name }}
+ apache/pegasus:thirdparties-src-${{ matrix.osversion }}-${{
github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
+ OS_VERSION=${{ matrix.osversion }}
+ HADOOP_BIN_PATH=hadoop-bin
+ ZOOKEEPER_BIN_PATH=zookeeper-bin
build_push_bin_docker_images:
runs-on: ubuntu-latest
@@ -82,7 +92,8 @@ jobs:
- ubuntu1804
- ubuntu2004
- ubuntu2204
- - centos7
+ # TODO(wangdan): disable centos7 temporarily before image
build-env-centos7-* is fixed.
+ # - centos7
steps:
# The glibc version on ubuntu1804 and centos7 is lower than the
actions/checkout@v4 required, so
# we need to force to use actions/checkout@v3.
@@ -107,8 +118,8 @@ jobs:
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
+ HADOOP_BIN_PATH=hadoop-bin
+ ZOOKEEPER_BIN_PATH=zookeeper-bin
build_push_bin_jemalloc_docker_images:
runs-on: ubuntu-latest
@@ -126,7 +137,8 @@ jobs:
- ubuntu1804
- ubuntu2004
- ubuntu2204
- - centos7
+ # TODO(wangdan): disable centos7 temporarily before image
build-env-centos7-* is fixed.
+ # - centos7
steps:
# The glibc version on ubuntu1804 and centos7 is lower than the
actions/checkout@v4 required, so
# we need to force to use actions/checkout@v3.
@@ -152,8 +164,8 @@ jobs:
GITHUB_BRANCH=${{ github.ref_name }}
OS_VERSION=${{ matrix.osversion }}
USE_JEMALLOC=ON
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
+ HADOOP_BIN_PATH=hadoop-bin
+ ZOOKEEPER_BIN_PATH=zookeeper-bin
build_push_bin_test_docker_images:
runs-on: ubuntu-latest
@@ -188,8 +200,6 @@ jobs:
ROCKSDB_PORTABLE=1
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
build_push_bin_test_jemalloc_docker_images:
runs-on: ubuntu-latest
@@ -225,5 +235,3 @@ jobs:
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
- - name: Image digest
- run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/admin_tools/download_hadoop.sh b/admin_tools/download_hadoop.sh
index d90fd24f4..6fd101eb9 100755
--- a/admin_tools/download_hadoop.sh
+++ b/admin_tools/download_hadoop.sh
@@ -21,15 +21,19 @@ set -e
CWD=$(cd "$(dirname "$0")" && pwd)
-if [ $# -ge 1 ]; then
- HADOOP_BIN_PATH=$1
+if [ $# -lt 1 ]; then
+ echo "Invalid arguments !"
+ echo "USAGE: $0 <HADOOP_BIN_PATH>"
+ exit 1
fi
+HADOOP_BIN_PATH=$1
+
HADOOP_VERSION="hadoop-3.3.6"
arch_output=$(arch)
if [ "$arch_output"x == "aarch64"x ]; then
- HADOOP_PACKAGE_MD5="369f899194a920e0d1c3c3bc1718b3b5"
- HADOOP_BASE_NAME=${HADOOP_VERSION}-"$(arch)"
+ HADOOP_PACKAGE_MD5="369f899194a920e0d1c3c3bc1718b3b5"
+ HADOOP_BASE_NAME=${HADOOP_VERSION}-"$(arch)"
else
if [ "$arch_output"x != "x86_64"x ]; then
echo "WARNING: unrecognized CPU architecture '$arch_output', use
'x86_64' as default"
diff --git a/admin_tools/download_package.sh b/admin_tools/download_package.sh
index 77e813eee..8c0453269 100755
--- a/admin_tools/download_package.sh
+++ b/admin_tools/download_package.sh
@@ -19,20 +19,15 @@
set -e
-if [ $# -lt 2 ]; then
+if [ $# -lt 3 ]; then
echo "Invalid arguments !"
- echo "USAGE: $0 <PACKAGE_BASE_NAME> <PACKAGE_MD5> [TARGET_PATH]"
+ echo "USAGE: $0 <PACKAGE_BASE_NAME> <PACKAGE_MD5> <TARGET_PATH>"
exit 1
fi
PACKAGE_BASE_NAME=$1
PACKAGE_MD5=$2
-if [ $# -lt 3 ]; then
- echo "TARGET_PATH is not provided, thus do not try to download
${PACKAGE_BASE_NAME}"
- exit 0
-fi
-
TARGET_PATH=$3
if [ -d "${TARGET_PATH}" ]; then
echo "TARGET_PATH ${TARGET_PATH} has existed, thus do not try to download
${PACKAGE_BASE_NAME}"
diff --git a/admin_tools/download_zk.sh b/admin_tools/download_zk.sh
index 41922452a..481c488d7 100755
--- a/admin_tools/download_zk.sh
+++ b/admin_tools/download_zk.sh
@@ -21,10 +21,14 @@ set -e
CWD=$(cd $(dirname $0) && pwd)
-if [ $# -ge 1 ]; then
- ZOOKEEPER_BIN_PATH=$1
+if [ $# -lt 1 ]; then
+ echo "Invalid arguments !"
+ echo "USAGE: $0 <ZOOKEEPER_BIN_PATH>"
+ exit 1
fi
+ZOOKEEPER_BIN_PATH=$1
+
ZOOKEEPER_VERSION=3.7.0
ZOOKEEPER_DIR_NAME=apache-zookeeper-${ZOOKEEPER_VERSION}-bin
ZOOKEEPER_PACKAGE_MD5="8ffa97e7e6b0b2cf1d022e5156a7561a"
diff --git a/docker/README.md b/docker/README.md
index 978c1e484..2d454e1e2 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -51,7 +51,10 @@ It packages the downloaded sources into a zip in the
container, so that
other repos can easily extract third-parties from the container (via `docker
cp`),
without downloading from the cloud object storage.
-- `apache/pegasus:thirdparties-src-<branch>`
+- `apache/pegasus:thirdparties-src-centos7-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu1804-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu2004-<branch>`
+- `apache/pegasus:thirdparties-src-ubuntu2204-<branch>`
## thirdparties-bin
diff --git a/docker/thirdparties-bin/Dockerfile
b/docker/thirdparties-bin/Dockerfile
index 100da6d87..438669f93 100644
--- a/docker/thirdparties-bin/Dockerfile
+++ b/docker/thirdparties-bin/Dockerfile
@@ -17,7 +17,7 @@
ARG GITHUB_BRANCH=master
ARG OS_VERSION=centos7
-FROM apache/pegasus:thirdparties-src-${GITHUB_BRANCH} as builder
+FROM apache/pegasus:thirdparties-src-${OS_VERSION}-${GITHUB_BRANCH} as builder
FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH}
WORKDIR /root
@@ -28,8 +28,8 @@ ARG GITHUB_BRANCH
ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
ARG ROCKSDB_PORTABLE=native
ARG USE_JEMALLOC=OFF
-ARG HADOOP_BIN_PATH
-ARG ZOOKEEPER_BIN_PATH
+ARG HADOOP_BIN_PATH=hadoop-bin
+ARG ZOOKEEPER_BIN_PATH=zookeeper-bin
RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
&& cd incubator-pegasus/thirdparty \
&& unzip /root/thirdparties-src.zip -d . \
diff --git a/docker/thirdparties-src/Dockerfile
b/docker/thirdparties-src/Dockerfile
index 149a5765f..c61a82417 100644
--- a/docker/thirdparties-src/Dockerfile
+++ b/docker/thirdparties-src/Dockerfile
@@ -16,7 +16,8 @@
# under the License.
ARG GITHUB_BRANCH=master
-FROM apache/pegasus:build-env-centos7-${GITHUB_BRANCH} as builder
+ARG OS_VERSION=centos7
+FROM apache/pegasus:build-env-${OS_VERSION}-${GITHUB_BRANCH} as builder
WORKDIR /root
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
