This is an automated email from the ASF dual-hosted git repository.
liuxiaoyu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 5d13cad0 ci: add Rocky 9 support for CI and Docker build environment
(#78)
5d13cad0 is described below
commit 5d13cad04d64c4dee1efe0742e63f4989884415e
Author: liuxiaoyu <[email protected]>
AuthorDate: Thu Mar 26 20:53:27 2026 +0800
ci: add Rocky 9 support for CI and Docker build environment (#78)
- Unify CI scripts from ubuntu/ to common/, supporting both Ubuntu and
Rocky/RHEL
- via auto-detection of package manager (apt-get vs dnf)
- Add Rocky 9 Docker Compose and RPM build script (rocky9/)
- Parameterize singlecluster Dockerfile with ARG BASE_IMAGE for
multi-distro builds
- Add Rocky 9 singlecluster image build job to GitHub Actions workflow
- Cross-platform JAVA_HOME auto-detection in build_pxf.sh
---
.github/workflows/pxf-ci.yml | 402 +++++++++++++++++----
.../{ubuntu => common}/script/build_cloudberrry.sh | 128 +++++--
.../{ubuntu => common}/script/build_pxf.sh | 25 +-
.../{ubuntu => common}/script/entrypoint.sh | 275 +++++++++-----
.../{ubuntu => common}/script/pxf-env.sh | 33 +-
.../{ubuntu => common}/script/pxf-test.sh | 0
.../{ubuntu => common}/script/run_tests.sh | 0
.../{ubuntu => common}/script/start_minio.bash | 0
.../{ubuntu => common}/script/utils.sh | 15 +-
ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml | 46 +++
.../rocky9/script/build_cloudberry_rpm.sh | 110 ++++++
.../ubuntu/script/build_cloudberry_deb.sh | 34 +-
ci/singlecluster/Dockerfile | 67 ++--
13 files changed, 842 insertions(+), 293 deletions(-)
diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml
index 660c47d4..1195d060 100644
--- a/.github/workflows/pxf-ci.yml
+++ b/.github/workflows/pxf-ci.yml
@@ -91,8 +91,54 @@ jobs:
path: workspace/cloudberry-source.tar.gz
retention-days: 7
+ build-cloudberry-rpm:
+ name: Build Cloudberry RPM Package
+ runs-on: ubuntu-latest
+ container:
+ image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
+ options: --user root
+ steps:
+ - name: Checkout Cloudberry source
+ uses: actions/checkout@v4
+ with:
+ repository: apache/cloudberry
+ ref: ${{ env.CLOUDBERRY_VERSION }}
+ path: workspace/cloudberry
+ submodules: true
+
+ - name: Checkout PXF source (for build script)
+ uses: actions/checkout@v4
+ with:
+ path: cloudberry-pxf
+
+ - name: Build Cloudberry RPM
+ run: |
+ export WORKSPACE=$PWD/workspace
+ export CLOUDBERRY_VERSION=99.0.0
+ export CLOUDBERRY_BUILD=1
+ bash
cloudberry-pxf/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh
+
+ - name: Package Cloudberry source
+ run: |
+ cd workspace
+ tar czf cloudberry-source-rocky9.tar.gz cloudberry/
+
+ - name: Upload RPM artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: cloudberry-rpm
+ path: workspace/cloudberry-rpm/*.rpm
+ retention-days: 7
+
+ - name: Upload Cloudberry source artifact (Rocky 9)
+ uses: actions/upload-artifact@v4
+ with:
+ name: cloudberry-source-rocky9
+ path: workspace/cloudberry-source-rocky9.tar.gz
+ retention-days: 7
+
build-docker-images:
- name: Build Docker Images
+ name: Build Docker Images (Ubuntu)
runs-on: ubuntu-latest
steps:
- name: Checkout PXF source
@@ -130,6 +176,28 @@ jobs:
working-directory: ./server
run: make test
+ build-docker-images-rocky9:
+ name: Build Docker Images (Rocky 9)
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout PXF source
+ uses: actions/checkout@v4
+ with:
+ path: cloudberry-pxf
+
+ - name: Build singlecluster Rocky 9 image
+ run: |
+ cd cloudberry-pxf/ci/singlecluster
+ docker build --build-arg
BASE_IMAGE=apache/incubator-cloudberry:cbdb-build-rocky9-latest -t
pxf/singlecluster-rocky9:3 .
+ docker save pxf/singlecluster-rocky9:3 >
/tmp/singlecluster-rocky9-image.tar
+
+ - name: Upload singlecluster Rocky 9 image
+ uses: actions/upload-artifact@v4
+ with:
+ name: singlecluster-rocky9-image
+ path: /tmp/singlecluster-rocky9-image.tar
+ retention-days: 1
+
# Stage 2: Parallel test jobs using matrix strategy
pxf-test:
name: Test PXF - ${{ matrix.test_group }}
@@ -214,14 +282,14 @@ jobs:
docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin
/home/gpadmin/workspace/cloudberry
docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/
docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb
- docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf/ci/docker/pxf-cbdb-dev/ubuntu &&
./script/entrypoint.sh"
+ docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf &&
./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh"
- name: Run Test - ${{ matrix.test_group }}
id: run_test
continue-on-error: true
timeout-minutes: 120
run: |
- docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf/automation && source
../ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh &&
../ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}"
+ docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf/automation && source
../ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh &&
../ci/docker/pxf-cbdb-dev/common/script/run_tests.sh ${{ matrix.test_group }}"
- name: Collect artifacts and generate stats
if: always()
@@ -301,10 +369,182 @@ jobs:
exit 1
fi
+ # Stage 2b: Rocky 9 parallel test jobs
+ pxf-test-rocky9:
+ name: Test PXF Rocky9 - ${{ matrix.test_group }}
+ needs: [build-cloudberry-rpm, build-docker-images-rocky9]
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ test_group:
+ - cli
+ - external-table
+ - fdw
+ - server
+ - sanity
+ - smoke
+ - hdfs
+ - hcatalog
+ - hcfs
+ - hive
+ - hbase
+ - profile
+ - jdbc
+ - proxy
+ - unused
+ - s3
+ - features
+ - gpdb
+ - gpdb_fdw
+ - load
+ - pxf_extension
+ steps:
+ - name: Free disk space
+ run: |
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /opt/ghc
+ sudo rm -rf /usr/local/share/boost
+ sudo rm -rf /usr/local/lib/android
+ sudo rm -rf /opt/hostedtoolcache
+ sudo docker system prune -af
+ df -h
+
+ - name: Checkout PXF source
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 1
+ path: cloudberry-pxf
+ submodules: true
+
+ - name: Download Cloudberry RPM
+ uses: actions/download-artifact@v4
+ with:
+ name: cloudberry-rpm
+ path: /tmp
+
+ - name: Download Cloudberry source (Rocky 9)
+ uses: actions/download-artifact@v4
+ with:
+ name: cloudberry-source-rocky9
+ path: /tmp
+
+ - name: Download singlecluster Rocky 9 image
+ uses: actions/download-artifact@v4
+ with:
+ name: singlecluster-rocky9-image
+ path: /tmp
+
+ - name: Load singlecluster Rocky 9 image
+ run: |
+ docker load < /tmp/singlecluster-rocky9-image.tar
+
+ - name: Prepare Cloudberry source
+ run: |
+ tar xzf /tmp/cloudberry-source-rocky9.tar.gz
+ chmod -R u+rwX,go+rX cloudberry
+
+ - name: Start Services
+ id: start_services
+ run: |
+ cd cloudberry-pxf
+ docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
down -v || true
+ docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
build
+ docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml up
-d
+ docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin
/home/gpadmin/workspace/cloudberry
+ docker cp /tmp/*.rpm pxf-cbdb-dev:/tmp/
+ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm
+ docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf &&
./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh"
+
+ - name: Run Test - ${{ matrix.test_group }}
+ id: run_test
+ continue-on-error: true
+ timeout-minutes: 120
+ run: |
+ docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf/automation && source
../ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh &&
../ci/docker/pxf-cbdb-dev/common/script/run_tests.sh ${{ matrix.test_group }}"
+
+ - name: Collect artifacts and generate stats
+ if: always()
+ id: collect_artifacts
+ run: |
+ mkdir -p artifacts/logs
+ TEST_GROUP="${{ matrix.test_group }}"
+ TEST_RESULT="${{ steps.run_test.outcome }}"
+
+ # Initialize counters
+ TOTAL=0
+ PASSED=0
+ FAILED=0
+ SKIPPED=0
+
+ # Copy test artifacts
+ cp -r cloudberry-pxf/automation/test_artifacts/* artifacts/
2>/dev/null || true
+ docker exec pxf-cbdb-dev bash -c "cp -r /usr/local/pxf/logs/*
/tmp/pxf-logs/ 2>/dev/null || true" || true
+ docker cp pxf-cbdb-dev:/tmp/pxf-logs artifacts/logs/ 2>/dev/null ||
true
+
+ # Parse surefire reports for automation tests
+ if [[ "$TEST_GROUP" != "cli" && "$TEST_GROUP" != "server" ]]; then
+ for xml in
cloudberry-pxf/automation/target/surefire-reports/TEST-*.xml; do
+ if [ -f "$xml" ]; then
+ tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 ||
echo "0")
+ failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head
-1 || echo "0")
+ errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1
|| echo "0")
+ skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1
|| echo "0")
+
+ TOTAL=$((TOTAL + tests))
+ FAILED=$((FAILED + failures + errors))
+ SKIPPED=$((SKIPPED + skipped))
+ fi
+ done
+ PASSED=$((TOTAL - FAILED - SKIPPED))
+ fi
+
+ # Generate stats JSON
+ cat > artifacts/test_stats.json <<EOF
+ {
+ "group": "$TEST_GROUP",
+ "result": "$TEST_RESULT",
+ "total": $TOTAL,
+ "passed": $PASSED,
+ "failed": $FAILED,
+ "skipped": $SKIPPED
+ }
+ EOF
+
+ echo "failed_count=$FAILED" >> $GITHUB_OUTPUT
+ echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT
+ echo "Test stats for $TEST_GROUP (Rocky 9): total=$TOTAL,
passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
+
+ - name: Cleanup containers
+ if: always()
+ run: |
+ cd cloudberry-pxf
+ docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
down -v || true
+
+ - name: Upload test artifacts
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-results-rocky9-${{ matrix.test_group }}
+ path: artifacts/**
+ if-no-files-found: ignore
+ retention-days: 7
+
+ - name: Check test result
+ if: always()
+ run: |
+ FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}"
+ SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0
}}"
+
+ if [ "${{ steps.run_test.outcome }}" == "failure" ] || [
"$FAILED_COUNT" -gt 0 ]; then
+ echo "Test group ${{ matrix.test_group }} (Rocky 9) failed
(Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)"
+ exit 1
+ fi
+
# Stage 3: Summary job
test-summary:
name: Test Summary
- needs: [pxf-test]
+ needs: [pxf-test, pxf-test-rocky9]
if: always()
runs-on: ubuntu-latest
steps:
@@ -318,79 +558,89 @@ jobs:
run: |
echo "## PXF Test Results Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
-
- # Overall counters
- OVERALL_TOTAL=0
- OVERALL_PASSED=0
- OVERALL_FAILED=0
- OVERALL_SKIPPED=0
- GROUPS_PASSED=0
- GROUPS_FAILED=0
- FAILED_GROUP_NAMES=""
-
- # Collect all test stats
- declare -A GROUP_STATS
-
- for dir in all-artifacts/test-results-*; do
- if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then
- group=$(cat "$dir/test_stats.json" | grep -oP
'"group":\s*"\K[^"]+' || basename "$dir" | sed 's/test-results-//')
- result=$(cat "$dir/test_stats.json" | grep -oP
'"result":\s*"\K[^"]+' || echo "unknown")
- total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+'
|| echo "0")
- passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+'
|| echo "0")
- failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+'
|| echo "0")
- skipped=$(cat "$dir/test_stats.json" | grep -oP
'"skipped":\s*\K\d+' || echo "0")
-
- GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped"
-
- OVERALL_TOTAL=$((OVERALL_TOTAL + total))
- OVERALL_PASSED=$((OVERALL_PASSED + passed))
- OVERALL_FAILED=$((OVERALL_FAILED + failed))
- OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped))
-
+
+ generate_os_summary() {
+ local os_label="$1"
+ local pattern="$2"
+
+ # OS-level counters
+ local OVERALL_TOTAL=0
+ local OVERALL_PASSED=0
+ local OVERALL_FAILED=0
+ local OVERALL_SKIPPED=0
+ local GROUPS_PASSED=0
+ local GROUPS_FAILED=0
+ local FAILED_GROUP_NAMES=""
+
+ declare -A GROUP_STATS
+
+ for dir in all-artifacts/${pattern}; do
+ if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then
+ group=$(cat "$dir/test_stats.json" | grep -oP
'"group":\s*"\K[^"]+' || basename "$dir" | sed "s/${pattern%\*}//")
+ result=$(cat "$dir/test_stats.json" | grep -oP
'"result":\s*"\K[^"]+' || echo "unknown")
+ total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+'
|| echo "0")
+ passed=$(cat "$dir/test_stats.json" | grep -oP
'"passed":\s*\K\d+' || echo "0")
+ failed=$(cat "$dir/test_stats.json" | grep -oP
'"failed":\s*\K\d+' || echo "0")
+ skipped=$(cat "$dir/test_stats.json" | grep -oP
'"skipped":\s*\K\d+' || echo "0")
+
+ GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped"
+
+ OVERALL_TOTAL=$((OVERALL_TOTAL + total))
+ OVERALL_PASSED=$((OVERALL_PASSED + passed))
+ OVERALL_FAILED=$((OVERALL_FAILED + failed))
+ OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped))
+
+ if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
+ GROUPS_PASSED=$((GROUPS_PASSED + 1))
+ else
+ GROUPS_FAILED=$((GROUPS_FAILED + 1))
+ FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} "
+ fi
+ fi
+ done
+
+ echo "### ${os_label}" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ if [ $GROUPS_FAILED -eq 0 ]; then
+ echo "All ${GROUPS_PASSED} test groups passed" >>
$GITHUB_STEP_SUMMARY
+ else
+ echo "${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED)) test
groups failed" >> $GITHUB_STEP_SUMMARY
+ fi
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY
+ echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY
+ echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY
+ echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+
+ echo "| Test Group | Status | Passed | Failed | Skipped | Total |"
>> $GITHUB_STEP_SUMMARY
+ echo "|------------|--------|-------:|-------:|--------:|------:|"
>> $GITHUB_STEP_SUMMARY
+
+ for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
+ IFS=',' read -r result total passed failed skipped <<<
"${GROUP_STATS[$group]}"
if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
- GROUPS_PASSED=$((GROUPS_PASSED + 1))
+ status="PASS"
else
- GROUPS_FAILED=$((GROUPS_FAILED + 1))
- FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} "
+ status="FAIL"
fi
+ echo "| $group | $status | $passed | $failed | $skipped | $total
|" >> $GITHUB_STEP_SUMMARY
+ done
+
+ echo "" >> $GITHUB_STEP_SUMMARY
+
+ if [ $GROUPS_FAILED -gt 0 ]; then
+ echo "::error::${os_label}: ${GROUPS_FAILED} test group(s) failed:
${FAILED_GROUP_NAMES}"
+ return 1
fi
- done
-
- # Overall summary
- echo "### Overall Summary" >> $GITHUB_STEP_SUMMARY
- echo "" >> $GITHUB_STEP_SUMMARY
- if [ $GROUPS_FAILED -eq 0 ]; then
- echo "✅ **All ${GROUPS_PASSED} test groups passed**" >>
$GITHUB_STEP_SUMMARY
- else
- echo "❌ **${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED))
test groups failed**" >> $GITHUB_STEP_SUMMARY
- fi
- echo "" >> $GITHUB_STEP_SUMMARY
- echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY
- echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY
- echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY
- echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY
- echo "" >> $GITHUB_STEP_SUMMARY
-
- # Detailed table
- echo "### Test Results by Group" >> $GITHUB_STEP_SUMMARY
- echo "" >> $GITHUB_STEP_SUMMARY
- echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >>
$GITHUB_STEP_SUMMARY
- echo "|------------|--------|-------:|-------:|--------:|------:|" >>
$GITHUB_STEP_SUMMARY
-
- for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
- IFS=',' read -r result total passed failed skipped <<<
"${GROUP_STATS[$group]}"
- if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
- status="✅ PASS"
- else
- status="❌ FAIL"
- fi
- echo "| $group | $status | $passed | $failed | $skipped | $total |"
>> $GITHUB_STEP_SUMMARY
- done
-
- echo "" >> $GITHUB_STEP_SUMMARY
-
- # Check if any group failed
- if [ $GROUPS_FAILED -gt 0 ]; then
- echo "::error::${GROUPS_FAILED} test group(s) failed:
${FAILED_GROUP_NAMES}"
- exit 1
- fi
+ return 0
+ }
+
+ EXIT_CODE=0
+
+ # Ubuntu results (test-results-<group>, excluding
test-results-rocky9-*)
+ generate_os_summary "Ubuntu 22.04" "test-results-[!r]*" || EXIT_CODE=1
+
+ # Rocky 9 results
+ generate_os_summary "Rocky 9" "test-results-rocky9-*" || EXIT_CODE=1
+
+ exit $EXIT_CODE
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh
b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh
similarity index 69%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh
rename to ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh
index e1ae6e26..cd4c4758 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
# --------------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +18,14 @@
# permissions and limitations under the License.
#
# --------------------------------------------------------------------
+# Build Cloudberry from source — works on both Ubuntu and Rocky/RHEL
+
# Install sudo & git
-sudo apt update && sudo apt install -y sudo git
+if command -v apt-get >/dev/null 2>&1; then
+ sudo apt update && sudo apt install -y sudo git
+elif command -v dnf >/dev/null 2>&1; then
+ sudo dnf install -y sudo git
+fi
# Required configuration
## Add Cloudberry environment setup to .bashrc
@@ -57,44 +64,85 @@ EOF
ulimit -a
# Install basic system packages
-sudo apt update
-sudo apt install -y bison \
- bzip2 \
- cmake \
- curl \
- flex \
- gcc \
- g++ \
- iproute2 \
- iputils-ping \
- language-pack-en \
- locales \
- libapr1-dev \
- libbz2-dev \
- libcurl4-gnutls-dev \
- libevent-dev \
- libkrb5-dev \
- libipc-run-perl \
- libldap2-dev \
- libpam0g-dev \
- libprotobuf-dev \
- libreadline-dev \
- libssl-dev \
- libuv1-dev \
- liblz4-dev \
- libxerces-c-dev \
- libxml2-dev \
- libyaml-dev \
- libzstd-dev \
- libperl-dev \
- make \
- pkg-config \
- protobuf-compiler \
- python3-dev \
- python3-pip \
- python3-setuptools \
- rsync \
- libsnappy-dev
+if command -v apt-get >/dev/null 2>&1; then
+ sudo apt update
+ sudo apt install -y bison \
+ bzip2 \
+ cmake \
+ curl \
+ flex \
+ gcc \
+ g++ \
+ iproute2 \
+ iputils-ping \
+ language-pack-en \
+ locales \
+ libapr1-dev \
+ libbz2-dev \
+ libcurl4-gnutls-dev \
+ libevent-dev \
+ libkrb5-dev \
+ libipc-run-perl \
+ libldap2-dev \
+ libpam0g-dev \
+ libprotobuf-dev \
+ libreadline-dev \
+ libssl-dev \
+ libuv1-dev \
+ liblz4-dev \
+ libxerces-c-dev \
+ libxml2-dev \
+ libyaml-dev \
+ libzstd-dev \
+ libperl-dev \
+ make \
+ pkg-config \
+ protobuf-compiler \
+ python3-dev \
+ python3-pip \
+ python3-setuptools \
+ rsync \
+ libsnappy-dev
+elif command -v dnf >/dev/null 2>&1; then
+ sudo dnf install -y \
+ bison \
+ bzip2 \
+ cmake \
+ curl \
+ flex \
+ gcc \
+ gcc-c++ \
+ iproute \
+ iputils \
+ glibc-langpack-en \
+ glibc-locale-source \
+ apr-devel \
+ bzip2-devel \
+ libcurl-devel \
+ libevent-devel \
+ krb5-devel \
+ perl-IPC-Run \
+ openldap-devel \
+ pam-devel \
+ protobuf-devel \
+ readline-devel \
+ openssl-devel \
+ libuv-devel \
+ lz4-devel \
+ xerces-c-devel \
+ libxml2-devel \
+ libyaml-devel \
+ libzstd-devel \
+ perl-devel \
+ make \
+ pkgconfig \
+ protobuf-compiler \
+ python3-devel \
+ python3-pip \
+ python3-setuptools \
+ rsync \
+ snappy-devel
+fi
# Continue as gpadmin user
@@ -149,4 +197,4 @@ source /usr/local/cloudberry-db/cloudberry-env.sh
make create-demo-cluster -C ~/workspace/cloudberry
source ~/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh
psql -P pager=off template1 -c 'SELECT * from gp_segment_configuration'
-psql template1 -c 'SELECT version()'
\ No newline at end of file
+psql template1 -c 'SELECT version()'
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh
b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh
similarity index 80%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh
rename to ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh
index b48cbc9e..08f7c765 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
# --------------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,18 +18,28 @@
# permissions and limitations under the License.
#
# --------------------------------------------------------------------
-case "$(uname -m)" in
- aarch64|arm64) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-arm64} ;;
- x86_64|amd64) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} ;;
- *) JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} ;;
-esac
+# Build and install PXF — works on both Ubuntu and Rocky/RHEL
+
+# Auto-detect Java 11 path
+if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ]; then
+ JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64}
+elif [ -d /usr/lib/jvm/java-11-openjdk-arm64 ]; then
+ JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-arm64}
+else
+ JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk}
+fi
export PATH=$JAVA_HOME/bin:$PATH
export GPHOME=/usr/local/cloudberry-db
source /usr/local/cloudberry-db/cloudberry-env.sh
export PATH=$GPHOME/bin:$PATH
-sudo apt update
-sudo apt install -y openjdk-11-jdk maven
+# Install Java 11 JDK and Maven
+if command -v apt-get >/dev/null 2>&1; then
+ sudo apt update
+ sudo apt install -y openjdk-11-jdk maven
+elif command -v dnf >/dev/null 2>&1; then
+ sudo dnf install -y java-11-openjdk-devel maven
+fi
cd /home/gpadmin/workspace/cloudberry-pxf
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh
b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
similarity index 66%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh
rename to ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
index f3ca1bee..832e5067 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
@@ -26,41 +26,66 @@ die() { log "ERROR $*"; exit 1; }
ROOT_DIR=/home/gpadmin/workspace
REPO_DIR=${ROOT_DIR}/cloudberry-pxf
GPHD_ROOT=${ROOT_DIR}/singlecluster
-PXF_SCRIPTS=${REPO_DIR}/ci/docker/pxf-cbdb-dev/ubuntu/script
-source "${PXF_SCRIPTS}/utils.sh"
+COMMON_SCRIPTS=${REPO_DIR}/ci/docker/pxf-cbdb-dev/common/script
+source "${COMMON_SCRIPTS}/utils.sh"
HADOOP_ROOT=${GPHD_ROOT}/hadoop
HIVE_ROOT=${GPHD_ROOT}/hive
HBASE_ROOT=${GPHD_ROOT}/hbase
ZOOKEEPER_ROOT=${GPHD_ROOT}/zookeeper
-JAVA_11_ARM=/usr/lib/jvm/java-11-openjdk-arm64
-JAVA_11_AMD=/usr/lib/jvm/java-11-openjdk-amd64
-JAVA_8_ARM=/usr/lib/jvm/java-8-openjdk-arm64
-JAVA_8_AMD=/usr/lib/jvm/java-8-openjdk-amd64
+# --------------------------------------------------------------------
+# OS detection: "deb" (Ubuntu/Debian) or "rpm" (Rocky/RHEL/CentOS)
+# --------------------------------------------------------------------
+if command -v apt-get >/dev/null 2>&1; then
+ OS_FAMILY="deb"
+else
+ OS_FAMILY="rpm"
+fi
detect_java_paths() {
- case "$(uname -m)" in
- aarch64|arm64) JAVA_BUILD=${JAVA_11_ARM}; JAVA_HADOOP=${JAVA_8_ARM} ;;
- x86_64|amd64) JAVA_BUILD=${JAVA_11_AMD}; JAVA_HADOOP=${JAVA_8_AMD} ;;
- *) JAVA_BUILD=${JAVA_11_ARM}; JAVA_HADOOP=${JAVA_8_ARM} ;;
- esac
+ if [ "$OS_FAMILY" = "deb" ]; then
+ case "$(uname -m)" in
+ aarch64|arm64) JAVA_BUILD=/usr/lib/jvm/java-11-openjdk-arm64;
JAVA_HADOOP=/usr/lib/jvm/java-8-openjdk-arm64 ;;
+ *) JAVA_BUILD=/usr/lib/jvm/java-11-openjdk-amd64;
JAVA_HADOOP=/usr/lib/jvm/java-8-openjdk-amd64 ;;
+ esac
+ else
+ JAVA_BUILD=/usr/lib/jvm/java-11-openjdk
+ JAVA_HADOOP=/usr/lib/jvm/java-1.8.0-openjdk
+ fi
export JAVA_BUILD JAVA_HADOOP
}
setup_locale_and_packages() {
log "install base packages and locales"
- sudo apt-get update
- sudo apt-get install -y wget lsb-release locales maven unzip openssh-server
iproute2 sudo \
- openjdk-11-jre-headless openjdk-8-jre-headless
- sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8
- sudo update-locale LANG=en_US.UTF-8
+ if [ "$OS_FAMILY" = "deb" ]; then
+ sudo apt-get update
+ sudo apt-get install -y wget lsb-release locales maven unzip
openssh-server iproute2 sudo \
+ openjdk-11-jre-headless openjdk-8-jre-headless
+ sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8
+ sudo update-locale LANG=en_US.UTF-8
+ else
+ sudo dnf install -y wget maven unzip openssh-server iproute sudo \
+ java-11-openjdk-headless java-1.8.0-openjdk-headless \
+ glibc-langpack-en glibc-locale-source
+ sudo localedef -c -i en_US -f UTF-8 en_US.UTF-8 || true
+ sudo localedef -c -i ru_RU -f UTF-8 ru_RU.UTF-8 || true
+ fi
sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251 || true
export LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
}
setup_ssh() {
log "configure ssh"
+ # Rocky 9 / RHEL 9 enforces system-wide crypto-policies that override
sshd_config
+ # settings for algorithm negotiation. The automation test framework uses the
+ # Ganymed SSH-2 (ch.ethz.ssh2) library which only supports older KEX
algorithms
+ # (diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, etc.).
+ # Downgrade to the LEGACY crypto policy so sshd accepts these algorithms.
+ if [ "$OS_FAMILY" = "rpm" ] && command -v update-crypto-policies >/dev/null
2>&1; then
+ log "setting LEGACY crypto policy for SSH compatibility"
+ sudo update-crypto-policies --set LEGACY 2>/dev/null || true
+ fi
sudo ssh-keygen -A
sudo bash -c 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
sudo mkdir -p /etc/ssh/sshd_config.d
@@ -69,7 +94,11 @@ KexAlgorithms
+diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1,di
HostKeyAlgorithms +ssh-rsa,ssh-dss
PubkeyAcceptedAlgorithms +ssh-rsa,ssh-dss
EOF'
- sudo usermod -a -G sudo gpadmin
+ if [ "$OS_FAMILY" = "deb" ]; then
+ sudo usermod -a -G sudo gpadmin
+ else
+ sudo usermod -a -G wheel gpadmin 2>/dev/null || true
+ fi
echo "gpadmin:cbdb@123" | sudo chpasswd
echo "gpadmin ALL=(ALL) NOPASSWD: ALL" | sudo tee -a
/etc/sudoers >/dev/null
echo "root ALL=(ALL) NOPASSWD: ALL" | sudo tee -a
/etc/sudoers >/dev/null
@@ -84,7 +113,21 @@ EOF'
ssh-keyscan -t rsa mdw cdw localhost 2>/dev/null >
/home/gpadmin/.ssh/known_hosts || true
sudo rm -rf /run/nologin
sudo mkdir -p /var/run/sshd && sudo chmod 0755 /var/run/sshd
- sudo /usr/sbin/sshd || die "Failed to start sshd"
+ # Ensure privilege separation user exists (required by Rocky 9 sshd)
+ id sshd &>/dev/null || sudo useradd -r -d /var/empty/sshd -s /sbin/nologin
sshd 2>/dev/null || true
+ sudo mkdir -p /var/empty/sshd && sudo chmod 0755 /var/empty/sshd
+ sudo /usr/sbin/sshd -E /tmp/sshd.log || die "Failed to start sshd, check
/tmp/sshd.log"
+ sleep 1
+ if ! ss -tlnp | grep -q ':22 '; then
+ log "ERROR: sshd is not listening on port 22"
+ cat /tmp/sshd.log 2>/dev/null || true
+ sudo /usr/sbin/sshd -D -e &
+ sleep 1
+ if ! ss -tlnp | grep -q ':22 '; then
+ die "sshd failed to bind to port 22"
+ fi
+ fi
+ log "sshd is running on port 22"
}
relax_pg_hba() {
@@ -101,105 +144,81 @@ EOF
fi
}
-install_cloudberry_from_deb() {
- log "installing Cloudberry from .deb package"
- local deb_file=$(find /tmp -name "apache-cloudberry-db*.deb" 2>/dev/null |
head -1)
- if [ -z "$deb_file" ]; then
- die "No .deb package found in /tmp"
+install_build_deps() {
+ if [ "$OS_FAMILY" = "deb" ]; then
+ sudo apt update && sudo apt install -y sudo git
+ sudo apt update
+ sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2
iputils-ping \
+ language-pack-en locales libapr1-dev libbz2-dev libcurl4-gnutls-dev
libevent-dev \
+ libkrb5-dev libipc-run-perl libldap2-dev libpam0g-dev libprotobuf-dev
libreadline-dev \
+ libssl-dev libuv1-dev liblz4-dev libxerces-c-dev libxml2-dev libyaml-dev
libzstd-dev \
+ libperl-dev make pkg-config protobuf-compiler python3-dev python3-pip
python3-setuptools \
+ rsync libsnappy-dev
+ else
+ sudo dnf install -y sudo git
+ sudo dnf install -y --allowerasing bison bzip2 cmake curl flex gcc gcc-c++
iproute iputils \
+ glibc-langpack-en glibc-locale-source apr-devel bzip2-devel
libcurl-devel libevent-devel \
+ krb5-devel perl-IPC-Run openldap-devel pam-devel protobuf-devel
readline-devel \
+ openssl-devel libuv-devel lz4-devel libxml2-devel libyaml-devel \
+ libzstd-devel perl-devel make pkgconfig protobuf-compiler python3-devel
python3-pip \
+ python3-setuptools rsync snappy-devel
fi
+}
- # Install sudo & git
- sudo apt update && sudo apt install -y sudo git
+install_cloudberry_from_package() {
+ log "installing Cloudberry from package"
+
+ local pkg_file=""
+ if [ "$OS_FAMILY" = "deb" ]; then
+ pkg_file=$(find /tmp -name "apache-cloudberry-db*.deb" 2>/dev/null | head
-1)
+ [ -z "$pkg_file" ] && die "No .deb package found in /tmp"
+ else
+ pkg_file=$(find /tmp -name "apache-cloudberry-db*.rpm" 2>/dev/null | head
-1)
+ [ -z "$pkg_file" ] && die "No .rpm package found in /tmp"
+ fi
+
+ install_build_deps
# Required configuration
- ## Add Cloudberry environment setup to .bashrc
echo -e '\n# Add Cloudberry entries
if [ -f /usr/local/cloudberry-db/cloudberry-env.sh ]; then
source /usr/local/cloudberry-db/cloudberry-env.sh
fi
- ## US English with UTF-8 character encoding
export LANG=en_US.UTF-8
' >> /home/gpadmin/.bashrc
- ## Set up SSH for passwordless access
+
mkdir -p /home/gpadmin/.ssh
if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then
ssh-keygen -t rsa -b 2048 -C 'apache-cloudberry-dev' -f
/home/gpadmin/.ssh/id_rsa -N ""
fi
cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys
- ## Set proper SSH directory permissions
chmod 700 /home/gpadmin/.ssh
chmod 600 /home/gpadmin/.ssh/authorized_keys
chmod 644 /home/gpadmin/.ssh/id_rsa.pub
-# Configure system settings
sudo tee /etc/security/limits.d/90-db-limits.conf << 'EOF'
-## Core dump file size limits for gpadmin
gpadmin soft core unlimited
gpadmin hard core unlimited
-## Open file limits for gpadmin
gpadmin soft nofile 524288
gpadmin hard nofile 524288
-## Process limits for gpadmin
gpadmin soft nproc 131072
gpadmin hard nproc 131072
EOF
- # Verify resource limits
ulimit -a
- # Install basic system packages
- sudo apt update
- sudo apt install -y bison \
- bzip2 \
- cmake \
- curl \
- flex \
- gcc \
- g++ \
- iproute2 \
- iputils-ping \
- language-pack-en \
- locales \
- libapr1-dev \
- libbz2-dev \
- libcurl4-gnutls-dev \
- libevent-dev \
- libkrb5-dev \
- libipc-run-perl \
- libldap2-dev \
- libpam0g-dev \
- libprotobuf-dev \
- libreadline-dev \
- libssl-dev \
- libuv1-dev \
- liblz4-dev \
- libxerces-c-dev \
- libxml2-dev \
- libyaml-dev \
- libzstd-dev \
- libperl-dev \
- make \
- pkg-config \
- protobuf-compiler \
- python3-dev \
- python3-pip \
- python3-setuptools \
- rsync \
- libsnappy-dev
-
-
- # Continue as gpadmin user
-
-
- # Prepare the build environment for Apache Cloudberry
sudo rm -rf /usr/local/cloudberry-db
sudo chmod a+w /usr/local
mkdir -p /usr/local/cloudberry-db
sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db
- sudo dpkg -i "$deb_file" || sudo apt-get install -f -y
- log "Cloudberry installed from $deb_file"
-
+ if [ "$OS_FAMILY" = "deb" ]; then
+ sudo dpkg -i "$pkg_file" || sudo apt-get install -f -y
+ else
+ sudo rpm -Uvh --force "$pkg_file" || sudo dnf install -y "$pkg_file"
+ fi
+ log "Cloudberry installed from $pkg_file"
+
# Initialize and start Cloudberry cluster
source /usr/local/cloudberry-db/cloudberry-env.sh
make create-demo-cluster -C ~/workspace/cloudberry || {
@@ -219,30 +238,30 @@ build_cloudberry() {
rm -rf /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/datadirs
rm -f /tmp/.s.PGSQL.700*
find "${ROOT_DIR}" -not -path '*/.git/*' -exec sudo chown gpadmin:gpadmin {}
+ 2>/dev/null || true
- "${PXF_SCRIPTS}/build_cloudberrry.sh"
+ "${COMMON_SCRIPTS}/build_cloudberrry.sh"
}
setup_cloudberry() {
- # Auto-detect: if deb exists, install it; otherwise build from source
- if [ -f /tmp/apache-cloudberry-db*.deb ]; then
- log "detected .deb package, using fast install"
- install_cloudberry_from_deb
- elif [ "${CLOUDBERRY_USE_DEB:-}" = "true" ]; then
- die "CLOUDBERRY_USE_DEB=true but no .deb found in /tmp"
+ # Auto-detect: if package exists, install it; otherwise build from source
+ if ls /tmp/apache-cloudberry-db*.deb 1>/dev/null 2>&1 || ls
/tmp/apache-cloudberry-db*.rpm 1>/dev/null 2>&1; then
+ log "detected package, using fast install"
+ install_cloudberry_from_package
+ elif [ "${CLOUDBERRY_USE_PKG:-}" = "true" ]; then
+ die "CLOUDBERRY_USE_PKG=true but no .deb/.rpm found in /tmp"
else
- log "no .deb found, building from source (local dev mode)"
+ log "no package found, building from source (local dev mode)"
build_cloudberry
fi
}
build_pxf() {
log "build PXF"
- "${PXF_SCRIPTS}/build_pxf.sh"
+ "${COMMON_SCRIPTS}/build_pxf.sh"
}
configure_pxf() {
log "configure PXF"
- source "${PXF_SCRIPTS}/pxf-env.sh"
+ source "${COMMON_SCRIPTS}/pxf-env.sh"
export PATH="$PXF_HOME/bin:$PATH"
export PXF_JVM_OPTS="-Xmx512m -Xms256m"
export PXF_HOST=localhost
@@ -318,7 +337,7 @@ EOF
# Configure S3 settings
mkdir -p "$PXF_BASE/servers/s3" "$PXF_HOME/servers/s3"
-
+
for s3_site in "$PXF_BASE/servers/s3/s3-site.xml"
"$PXF_BASE/servers/default/s3-site.xml" "$PXF_HOME/servers/s3/s3-site.xml"; do
mkdir -p "$(dirname "$s3_site")"
cat > "$s3_site" <<'EOF'
@@ -364,6 +383,63 @@ EOF
}
+wait_for_datanode() {
+ log "waiting for HDFS DataNode to become available..."
+ local max_attempts=2
+ for _attempt in $(seq 1 ${max_attempts}); do
+ local dn_ready=false
+ # Wait up to 90s (45 tries * 2s) for DataNode to register
+ for _dn_try in $(seq 1 45); do
+ if hdfs dfsadmin -report 2>/dev/null | grep -q "Live datanodes.*[1-9]";
then
+ dn_ready=true
+ break
+ fi
+ sleep 2
+ done
+
+ if [ "${dn_ready}" = "true" ]; then
+ log "HDFS DataNode is available"
+ return 0
+ fi
+
+ # DataNode didn't come up; diagnose and attempt restart
+ log "DataNode not available after 90s (attempt
${_attempt}/${max_attempts})"
+ log "--- DataNode diagnostic info ---"
+ # Check if DataNode process is alive
+ if command -v jps >/dev/null 2>&1; then
+ log "JPS output: $(jps 2>&1)"
+ fi
+ if pgrep -f "proc_datanode" >/dev/null 2>&1 || pgrep -f "datanode"
>/dev/null 2>&1; then
+ log "DataNode process exists but not yet registered with NameNode"
+ else
+ log "DataNode process is NOT running"
+ fi
+ # Show DataNode logs if available
+ local dn_log
+ dn_log=$(ls -t "${GPHD_ROOT}"/storage/logs/*datanode*.log 2>/dev/null |
head -1)
+ if [ -n "${dn_log}" ]; then
+ log "Last 30 lines of DataNode log (${dn_log}):"
+ tail -30 "${dn_log}" 2>/dev/null || true
+ fi
+ log "HDFS report:"
+ hdfs dfsadmin -report 2>&1 | head -20 || true
+ log "--- end diagnostic ---"
+
+ if [ "${_attempt}" -lt "${max_attempts}" ]; then
+ log "Attempting to restart DataNode..."
+ # Stop any zombie DataNode processes
+ pkill -f "proc_datanode" 2>/dev/null || true
+ sleep 2
+ # Restart DataNode via the singlecluster script
+ "${GPHD_ROOT}/bin/hadoop-datanode.sh" start 0 2>&1 || true
+ "${HADOOP_ROOT}/sbin/hadoop-daemon.sh" --config
"${GPHD_ROOT}/storage/hadoop/datanode0/etc/hadoop" start datanode 2>&1 || true
+ log "DataNode restart issued, waiting again..."
+ fi
+ done
+
+ die "HDFS DataNode failed to start after ${max_attempts} attempts. Tez
upload will fail without a running DataNode."
+}
+
prepare_hadoop_stack() {
log "prepare Hadoop/Hive/HBase stack"
export JAVA_HOME="${JAVA_HADOOP}"
@@ -386,12 +462,19 @@ prepare_hadoop_stack() {
if pgrep -f HiveServer2 >/dev/null 2>&1; then
"${GPHD_ROOT}/bin/hive-service.sh" hiveserver2 stop || true
fi
+ log "JAVA_HOME=${JAVA_HOME} JAVA_HADOOP=${JAVA_HADOOP}"
+ log "java check: $(ls -la ${JAVA_HOME}/bin/java 2>&1)"
if [ ! -d "${GPHD_ROOT}/storage/hadoop/dfs/name/current" ]; then
- ${GPHD_ROOT}/bin/init-gphd.sh
+ log "initializing HDFS namenode..."
+ ${GPHD_ROOT}/bin/init-gphd.sh 2>&1 || log "init-gphd.sh failed with exit
code $?"
fi
- if ! ${GPHD_ROOT}/bin/start-gphd.sh; then
+ log "starting HDFS/YARN/HBase via start-gphd.sh..."
+ if ! ${GPHD_ROOT}/bin/start-gphd.sh 2>&1; then
log "start-gphd.sh returned non-zero (services may already be running),
continue"
fi
+ # Wait for HDFS DataNode to be ready before proceeding; Tez upload in
+ # start_hive_services will fail if no DataNode is accepting writes.
+ wait_for_datanode
if ! ${GPHD_ROOT}/bin/start-zookeeper.sh; then
log "start-zookeeper.sh returned non-zero (may already be running)"
fi
@@ -487,7 +570,7 @@ start_hive_services() {
deploy_minio() {
log "deploying MinIO"
- bash "${PXF_SCRIPTS}/start_minio.bash"
+ bash "${COMMON_SCRIPTS}/start_minio.bash"
}
main() {
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh
b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
similarity index 81%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh
rename to ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
index bd785ea1..81b23556 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
@@ -21,22 +21,25 @@
# Centralized environment for Cloudberry + PXF + Hadoop stack
# --------------------------------------------------------------------
-# Architecture-aware Java selections
+# Architecture-aware Java selections (auto-detect OS)
# --------------------------------------------------------------------
-case "$(uname -m)" in
- aarch64|arm64)
- JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-arm64}
- JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-arm64}
- ;;
- x86_64|amd64)
- JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-amd64}
- JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64}
- ;;
- *)
- JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk}
- JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk}
- ;;
-esac
+if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ] || [ -d
/usr/lib/jvm/java-11-openjdk-arm64 ]; then
+ # Debian/Ubuntu: paths include architecture suffix
+ case "$(uname -m)" in
+ aarch64|arm64)
+ JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-arm64}
+ JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-arm64}
+ ;;
+ *)
+ JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk-amd64}
+ JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64}
+ ;;
+ esac
+else
+ # RHEL/Rocky: architecture-independent symlinks
+ JAVA_BUILD=${JAVA_BUILD:-/usr/lib/jvm/java-11-openjdk}
+ JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-1.8.0-openjdk}
+fi
# --------------------------------------------------------------------
# Core paths
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-test.sh
b/ci/docker/pxf-cbdb-dev/common/script/pxf-test.sh
similarity index 100%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-test.sh
rename to ci/docker/pxf-cbdb-dev/common/script/pxf-test.sh
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
similarity index 100%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh
rename to ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/start_minio.bash
b/ci/docker/pxf-cbdb-dev/common/script/start_minio.bash
similarity index 100%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/start_minio.bash
rename to ci/docker/pxf-cbdb-dev/common/script/start_minio.bash
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh
b/ci/docker/pxf-cbdb-dev/common/script/utils.sh
old mode 100644
new mode 100755
similarity index 89%
rename from ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh
rename to ci/docker/pxf-cbdb-dev/common/script/utils.sh
index 786017fc..c055dd25
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/utils.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/utils.sh
@@ -107,13 +107,26 @@ check_pxf() {
fi
}
+# Auto-detect default Java 8 path based on OS
+_detect_java8_default() {
+ if [ -d /usr/lib/jvm/java-8-openjdk-amd64 ]; then
+ echo "/usr/lib/jvm/java-8-openjdk-amd64"
+ elif [ -d /usr/lib/jvm/java-8-openjdk-arm64 ]; then
+ echo "/usr/lib/jvm/java-8-openjdk-arm64"
+ elif [ -d /usr/lib/jvm/java-1.8.0-openjdk ]; then
+ echo "/usr/lib/jvm/java-1.8.0-openjdk"
+ else
+ echo "/usr/lib/jvm/java-8-openjdk"
+ fi
+}
+
health_check() {
log "sanity check Hadoop/Hive/HBase/PXF"
GPHD_ROOT=${GPHD_ROOT:-/home/gpadmin/workspace/singlecluster}
HADOOP_ROOT=${HADOOP_ROOT:-${GPHD_ROOT}/hadoop}
HBASE_ROOT=${HBASE_ROOT:-${GPHD_ROOT}/hbase}
HIVE_ROOT=${HIVE_ROOT:-${GPHD_ROOT}/hive}
- JAVA_HADOOP=${JAVA_HADOOP:-/usr/lib/jvm/java-8-openjdk-amd64}
+ JAVA_HADOOP=${JAVA_HADOOP:-$(_detect_java8_default)}
export JAVA_HOME="${JAVA_HADOOP}"
export
PATH="$JAVA_HOME/bin:$HADOOP_ROOT/bin:$HIVE_ROOT/bin:$HBASE_ROOT/bin:$PATH"
diff --git a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
new file mode 100644
index 00000000..37738078
--- /dev/null
+++ b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml
@@ -0,0 +1,46 @@
+# --------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to You under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the
+# License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# --------------------------------------------------------------------
+services:
+# hadoop
+ singlecluster:
+ build:
+ context: ../../../singlecluster
+ args:
+ BASE_IMAGE: apache/incubator-cloudberry:cbdb-build-rocky9-latest
+ image: pxf/singlecluster-rocky9:3
+ container_name: pxf_singlecluster_rocky9
+ hostname: cdw
+
+ pxf-cbdb-dev:
+ image: pxf/singlecluster-rocky9:3
+ container_name: pxf-cbdb-dev
+ hostname: mdw
+ depends_on:
+ - singlecluster
+ ports:
+ - "2222:22"
+ volumes:
+ - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf
+ - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry
+ command: ["tail", "-f", "/dev/null"]
+
+networks:
+ default:
+ name: pxf-cbdb-ci
diff --git a/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh
b/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh
new file mode 100755
index 00000000..1d3e6888
--- /dev/null
+++ b/ci/docker/pxf-cbdb-dev/rocky9/script/build_cloudberry_rpm.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+# --------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to You under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the
+# License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# --------------------------------------------------------------------
+set -euo pipefail
+
+# Cloudberry RPM Package Build Script for Rocky 9
+CLOUDBERRY_VERSION="${CLOUDBERRY_VERSION:-99.0.0}"
+CLOUDBERRY_BUILD="${CLOUDBERRY_BUILD:-1}"
+INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/cloudberry-db}"
+WORKSPACE="${WORKSPACE:-$HOME/workspace}"
+CLOUDBERRY_SRC="${WORKSPACE}/cloudberry"
+
+echo "=== Cloudberry RPM Package Build ==="
+echo "Version: ${CLOUDBERRY_VERSION}"
+echo "Build: ${CLOUDBERRY_BUILD}"
+echo "Install Prefix: ${INSTALL_PREFIX}"
+echo "Source: ${CLOUDBERRY_SRC}"
+
+# Clean previous installation
+rm -rf "${INSTALL_PREFIX}"
+mkdir -p "${INSTALL_PREFIX}"
+
+# Copy xerces-c shared libraries required by ORCA
+mkdir -p "${INSTALL_PREFIX}/lib"
+cp -v /usr/local/xerces-c/lib/libxerces-c.so \
+ /usr/local/xerces-c/lib/libxerces-c-3.*.so \
+ "${INSTALL_PREFIX}/lib/"
+
+# Build Cloudberry using official build scripts
+export SRC_DIR="${CLOUDBERRY_SRC}"
+export CPPFLAGS="${CPPFLAGS:-} -I/usr/local/xerces-c/include"
+export LDFLAGS="${LDFLAGS:-} -L${INSTALL_PREFIX}/lib"
+mkdir -p "${SRC_DIR}/build-logs"
+cd "${CLOUDBERRY_SRC}"
+./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
+./devops/build/automation/cloudberry/scripts/build-cloudberry.sh
+
+# Copy LICENSE
+cp LICENSE "${INSTALL_PREFIX}/"
+
+# Create RPM build structure
+RPM_BUILD_DIR="${WORKSPACE}/cloudberry-rpm"
+mkdir -p "${RPM_BUILD_DIR}"/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
+RPM_INSTALL_ROOT="${RPM_BUILD_DIR}/BUILDROOT/apache-cloudberry-db-${CLOUDBERRY_VERSION}-${CLOUDBERRY_BUILD}.x86_64"
+mkdir -p "${RPM_INSTALL_ROOT}${INSTALL_PREFIX}"
+
+# Copy installed files
+cp -a "${INSTALL_PREFIX}"/* "${RPM_INSTALL_ROOT}${INSTALL_PREFIX}/"
+
+# Create spec file
+cat > "${RPM_BUILD_DIR}/SPECS/cloudberry-db.spec" << EOF
+Name: apache-cloudberry-db
+Version: ${CLOUDBERRY_VERSION}
+Release: ${CLOUDBERRY_BUILD}%{?dist}
+Summary: Apache Cloudberry Database
+License: Apache-2.0
+Group: Applications/Databases
+AutoReqProv: no
+
+%description
+Apache Cloudberry is a massively parallel processing (MPP) database
+built on PostgreSQL for analytics and data warehousing.
+
+%install
+mkdir -p %{buildroot}${INSTALL_PREFIX}
+cp -a ${RPM_INSTALL_ROOT}${INSTALL_PREFIX}/* %{buildroot}${INSTALL_PREFIX}/
+
+%files
+${INSTALL_PREFIX}
+
+%post
+if ! id -u gpadmin >/dev/null 2>&1; then
+ useradd -m -s /bin/bash gpadmin
+fi
+chown -R gpadmin:gpadmin ${INSTALL_PREFIX}
+echo "Apache Cloudberry Database installed successfully"
+
+%clean
+rm -rf %{buildroot}
+EOF
+
+# Build RPM package
+rpmbuild --define "_topdir ${RPM_BUILD_DIR}" -bb
"${RPM_BUILD_DIR}/SPECS/cloudberry-db.spec"
+
+RPM_FILE=$(find "${RPM_BUILD_DIR}/RPMS" -name "*.rpm" | head -1)
+echo "=== RPM Package Created ==="
+ls -lh "${RPM_FILE}"
+rpm -qpi "${RPM_FILE}"
+
+# Copy RPM to output directory
+cp "${RPM_FILE}" "${RPM_BUILD_DIR}/"
+
+echo "=== Build Complete ==="
diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh
b/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh
index fc117e88..991cb648 100755
--- a/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh
+++ b/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh
@@ -37,36 +37,12 @@ echo "Source: ${CLOUDBERRY_SRC}"
rm -rf "${INSTALL_PREFIX}"
mkdir -p "${INSTALL_PREFIX}"
-# Configure Cloudberry
+# Build Cloudberry using official build scripts
+export SRC_DIR="${CLOUDBERRY_SRC}"
+mkdir -p "${SRC_DIR}/build-logs"
cd "${CLOUDBERRY_SRC}"
-./configure --prefix="${INSTALL_PREFIX}" \
- --disable-external-fts \
- --enable-gpcloud \
- --enable-ic-proxy \
- --enable-mapreduce \
- --enable-orafce \
- --enable-orca \
- --disable-pax \
- --disable-pxf \
- --enable-tap-tests \
- --with-gssapi \
- --with-ldap \
- --with-libxml \
- --with-lz4 \
- --with-pam \
- --with-perl \
- --with-pgport=5432 \
- --with-python \
- --with-pythonsrc-ext \
- --with-ssl=openssl \
- --with-uuid=e2fs \
- --with-includes=/usr/include/xercesc
-
-# Build and install
-make -j$(nproc)
-make -j$(nproc) -C contrib
-make install
-make install -C contrib
+./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
+./devops/build/automation/cloudberry/scripts/build-cloudberry.sh
# Copy LICENSE
cp LICENSE "${INSTALL_PREFIX}/"
diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile
index accf1d51..4d6bb655 100644
--- a/ci/singlecluster/Dockerfile
+++ b/ci/singlecluster/Dockerfile
@@ -17,15 +17,24 @@
# permissions and limitations under the License.
#
# --------------------------------------------------------------------
-FROM apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
-
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN sudo apt-get update && \
- sudo apt-get install -y --no-install-recommends \
- curl ca-certificates \
- openjdk-8-jdk-headless \
- openjdk-11-jdk-headless
+ARG BASE_IMAGE=apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
+FROM ${BASE_IMAGE}
+
+# Install Java 8 & 11: auto-detect OS package manager
+RUN if command -v apt-get >/dev/null 2>&1; then \
+ export DEBIAN_FRONTEND=noninteractive && \
+ sudo apt-get update && \
+ sudo apt-get install -y --no-install-recommends \
+ curl ca-certificates \
+ openjdk-8-jdk-headless \
+ openjdk-11-jdk-headless; \
+ elif command -v dnf >/dev/null 2>&1; then \
+ sudo dnf install -y --allowerasing \
+ curl ca-certificates \
+ java-1.8.0-openjdk-devel \
+ java-11-openjdk-devel && \
+ sudo dnf clean all; \
+ fi
# TODO: update hive to support java 11+
ENV HADOOP_VERSION=3.1.2
@@ -60,36 +69,36 @@ ENV ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper
ENV TEZ_ROOT=$GPHD_ROOT/tez
RUN mkdir -p $HADOOP_ROOT && \
- curl -fSL "$HADOOP_URL" -o hadoop.tar.gz && \
- echo "$HADOOP_SHA512 hadoop.tar.gz" | sha512sum -c && \
- tar xvf hadoop.tar.gz -C $HADOOP_ROOT --strip-components 1
--exclude="share/doc/*" --exclude="*-sources.jar" && \
- rm hadoop.tar.gz && \
+ curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz && \
+ echo "$HADOOP_SHA512 /tmp/hadoop.tar.gz" | sha512sum -c && \
+ tar xvf /tmp/hadoop.tar.gz -C $HADOOP_ROOT --strip-components 1
--exclude="share/doc/*" --exclude="*-sources.jar" && \
+ rm /tmp/hadoop.tar.gz && \
curl -fSL
"https://repo1.maven.org/maven2/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar";
\
-o $HADOOP_ROOT/share/hadoop/common/lib/javax.activation-api-1.2.0.jar
RUN mkdir -p $HIVE_ROOT && \
- curl -fSL $HIVE_URL -o hive.tar.gz && \
- echo "$HIVE_SHA256 hive.tar.gz" | sha256sum -c && \
- tar xvf hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \
- rm hive.tar.gz
+ curl -fSL $HIVE_URL -o /tmp/hive.tar.gz && \
+ echo "$HIVE_SHA256 /tmp/hive.tar.gz" | sha256sum -c && \
+ tar xvf /tmp/hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \
+ rm /tmp/hive.tar.gz
RUN mkdir -p $ZOOKEEPER_ROOT && \
- curl -fSL $ZOOKEEPER_URL -o zookeeper.tar.gz && \
- echo "$ZOOKEEPER_SHA512 zookeeper.tar.gz" | sha512sum -c && \
- tar xvf zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1
--exclude="docs/*" && \
- rm zookeeper.tar.gz
+ curl -fSL $ZOOKEEPER_URL -o /tmp/zookeeper.tar.gz && \
+ echo "$ZOOKEEPER_SHA512 /tmp/zookeeper.tar.gz" | sha512sum -c && \
+ tar xvf /tmp/zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1
--exclude="docs/*" && \
+ rm /tmp/zookeeper.tar.gz
RUN mkdir -p $HBASE_ROOT && \
- curl -fSL "$HBASE_URL" -o hbase.tar.gz && \
- echo "$HBASE_SHA512 hbase.tar.gz" | sha512sum -c && \
- tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1
--exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients"
&& \
- rm hbase.tar.gz
+ curl -fSL "$HBASE_URL" -o /tmp/hbase.tar.gz && \
+ echo "$HBASE_SHA512 /tmp/hbase.tar.gz" | sha512sum -c && \
+ tar xvf /tmp/hbase.tar.gz -C $HBASE_ROOT --strip-components 1
--exclude="docs/*" --exclude="lib/*-tests.jar" --exclude="lib/shaded-clients"
&& \
+ rm /tmp/hbase.tar.gz
RUN mkdir -p $TEZ_ROOT && \
- curl -fSL "$TEZ_URL" -o tez.tar.gz && \
- echo "$TEZ_SHA512 tez.tar.gz" | sha512sum -c && \
- tar xvf tez.tar.gz -C $TEZ_ROOT --strip-components 1 && \
- rm tez.tar.gz
+ curl -fSL "$TEZ_URL" -o /tmp/tez.tar.gz && \
+ echo "$TEZ_SHA512 /tmp/tez.tar.gz" | sha512sum -c && \
+ tar xvf /tmp/tez.tar.gz -C $TEZ_ROOT --strip-components 1 && \
+ rm /tmp/tez.tar.gz
# Install Go (required by PXF). Pick archive based on architecture
(amd64/arm64).
ARG TARGETARCH
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
