woblerr commented on code in PR #108: URL: https://github.com/apache/cloudberry-backup/pull/108#discussion_r3628856086
########## .github/workflows/package-convenience-binaries.yml: ########## @@ -0,0 +1,922 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Apache Cloudberry-backup Convenience Package Build +# -------------------------------------------------------------------- +# Description: +# +# This workflow manually builds convenience portable tarball packages +# from an ASF-approved Apache Cloudberry-backup source release tarball, +# and tests them against Apache Cloudberry built from its official +# source release tarball. +# +# Workflow Overview: +# +# 1. verify-cloudberry-backup-source +# Validates inputs, downloads cloudberry-backup source tarball +# + .asc + .sha512, verifies GPG signature and checksum, uploads +# verified source as a workflow artifact. +# +# 2. verify-cloudberry-source +# Same as above, but for the Cloudberry source release tarball. +# +# 3. build-backup-packages (matrix: amd64 / arm64) +# Downloads the verified cloudberry-backup source artifact, runs +# `make package` on Rocky 8 (glibc 2.28) for maximum run-time +# compatibility, generates .sha512 checksums, uploads per-arch +# portable tar.gz packages. +# +# 4. build-cloudberry (matrix: 5 platforms × amd64 / arm64 = 10) +# Extracts Cloudberry source, runs configure + build inside the +# official Cloudberry build container (~8 min per platform). +# After the build, Cloudberry is already installed at +# /usr/local/cloudberry-db. The job then downloads the matching +# cloudberry-backup package, installs it, creates a gpdemo demo +# cluster, and runs a functional backup + restore smoke test — +# all inside the same container, no separate test job needed. +# Platforms: rocky8, rocky9, rocky10, ubuntu22.04, ubuntu24.04. +# +# Scope: +# - Intended for official Apache Cloudberry-backup source releases +# managed by the release manager. +# - Produces convenience binaries only; detached .asc signatures remain a +# release manager local step. +# -------------------------------------------------------------------- + +name: Apache Cloudberry-backup Convenience Package Build + +on: + workflow_dispatch: + inputs: + # ================================================================ + # cloudberry-backup source release inputs + # ================================================================ + version: + description: '[cloudberry-backup] Release version, e.g. 2.2.0-incubating' + required: true + type: string + source_url: + description: '[cloudberry-backup] Apache source tarball URL from downloads.apache.org' + required: true + type: string + source_asc_url: + description: '[cloudberry-backup] Detached GPG signature URL for the source tarball (.asc)' + required: true + type: string + source_sha512_url: + description: '[cloudberry-backup] SHA-512 checksum URL for the source tarball (.sha512)' + required: true + type: string + + # ================================================================ + # Cloudberry source release inputs (for the test environment) + # ================================================================ + cloudberry_version: + description: '[Cloudberry] Release version, e.g. 2.2.0-incubating' + required: true + type: string + cloudberry_source_url: + description: '[Cloudberry] Apache source tarball URL from downloads.apache.org' + required: true + type: string + cloudberry_source_asc_url: + description: '[Cloudberry] Detached GPG signature URL for the source tarball (.asc)' + required: true + type: string + cloudberry_source_sha512_url: + description: '[Cloudberry] SHA-512 checksum URL for the source tarball (.sha512)' + required: true + type: string + +permissions: + contents: read + +concurrency: + group: backup-package-build-${{ github.ref }}-${{ inputs.version }} + cancel-in-progress: true + +env: + LOG_RETENTION_DAYS: 14 + KEYS_URL: https://downloads.apache.org/incubator/cloudberry/KEYS + +jobs: + # ==================================================================== + # Job 1: Verify the Apache Cloudberry-backup source release + # ==================================================================== + verify-cloudberry-backup-source: + name: Verify cloudberry-backup source + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }} + artifact_name: ${{ steps.validate.outputs.artifact_name }} + packaging_version: ${{ steps.validate.outputs.packaging_version }} + steps: + - name: Validate manual inputs + id: validate + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + run: | + set -euo pipefail + + if [[ -z "${VERSION}" ]]; then + echo "::error::version must not be empty" + exit 1 + fi + + source_tarball_name="apache-cloudberry-backup-${VERSION}-src.tar.gz" + artifact_name="verified-source-release-cbbackup-${VERSION}" + packaging_version="${VERSION%-incubating}" + + if [[ -z "${packaging_version}" ]]; then + echo "::error::Unable to derive packaging version from version=${VERSION}" + exit 1 + fi + + validate_apache_url() { + local value="$1" + local label="$2" + local prefix="https://downloads.apache.org/incubator/cloudberry/"; + if [[ -z "${value}" ]]; then + echo "::error::${label} must not be empty" + exit 1 + fi + if [[ "${value}" != "${prefix}"* ]]; then + echo "::error::${label} must use downloads.apache.org (got: ${value})" + exit 1 + fi + } + + validate_apache_url "${SOURCE_URL}" "source_url" + validate_apache_url "${SOURCE_ASC_URL}" "source_asc_url" + validate_apache_url "${SOURCE_SHA512_URL}" "source_sha512_url" + + if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then + echo "::error::source_url must end with /${source_tarball_name}" + exit 1 + fi + if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then + echo "::error::source_asc_url must end with /${source_tarball_name}.asc" + exit 1 + fi + if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then + echo "::error::source_sha512_url must end with /${source_tarball_name}.sha512" + exit 1 + fi + + echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}" + echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}" + echo "packaging_version=${packaging_version}" >> "${GITHUB_OUTPUT}" + + - name: Download source release and verification files + shell: bash + env: + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + mkdir -p verified-source + + echo "=== Downloading source tarball... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}" \ + "${SOURCE_URL}" + + echo "=== Downloading signature... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "${SOURCE_ASC_URL}" + + echo "=== Downloading checksum... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \ + "${SOURCE_SHA512_URL}" + + echo "=== Downloading KEYS... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/KEYS" \ + "${KEYS_URL}" + + echo "=== All files downloaded successfully. ===" + ls -la verified-source/ + + - name: Verify source release signature and checksum + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + + export GNUPGHOME="${RUNNER_TEMP}/gnupg" + mkdir -p "${GNUPGHOME}" + chmod 700 "${GNUPGHOME}" + + echo "=== Importing project KEYS... ===" + gpg --import verified-source/KEYS + + echo "=== Verifying GPG signature... ===" + gpg --verify \ + "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "verified-source/${SOURCE_TARBALL_NAME}" + + echo "=== Verifying SHA-512 checksum... ===" + ( + cd verified-source + sha512sum -c "${SOURCE_TARBALL_NAME}.sha512" + ) + + echo "=== Verifying tarball integrity (listing contents)... ===" + tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null + echo "=== cloudberry-backup source release verification passed. ===" + + - name: Summarize verified source release + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + SOURCE_URL: ${{ github.event.inputs.source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }} + run: | + { + echo "# Verified cloudberry-backup source release" + echo "- Version: ${VERSION}" + echo "- Packaging version: ${{ steps.validate.outputs.packaging_version }}" + echo "- Source URL: ${SOURCE_URL}" + echo "- Signature URL: ${SOURCE_ASC_URL}" + echo "- Checksum URL: ${SOURCE_SHA512_URL}" + echo "- KEYS URL: ${KEYS_URL}" + echo "- GPG verification: PASS" + echo "- SHA-512 verification: PASS" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload verified source release + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.validate.outputs.artifact_name }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: verified-source/* + + # ==================================================================== + # Job 2: Verify the Apache Cloudberry source release + # ==================================================================== + verify-cloudberry-source: + name: Verify Cloudberry source + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }} + artifact_name: ${{ steps.validate.outputs.artifact_name }} + steps: + - name: Validate manual inputs + id: validate + shell: bash + env: + VERSION: ${{ github.event.inputs.cloudberry_version }} + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + run: | + set -euo pipefail + + if [[ -z "${VERSION}" ]]; then + echo "::error::cloudberry_version must not be empty" + exit 1 + fi + + source_tarball_name="apache-cloudberry-${VERSION}-src.tar.gz" + artifact_name="verified-source-release-cloudberry-${VERSION}" + + validate_apache_url() { + local value="$1" + local label="$2" + local prefix="https://downloads.apache.org/incubator/cloudberry/"; + if [[ -z "${value}" ]]; then + echo "::error::${label} must not be empty" + exit 1 + fi + if [[ "${value}" != "${prefix}"* ]]; then + echo "::error::${label} must use downloads.apache.org (got: ${value})" + exit 1 + fi + } + + validate_apache_url "${SOURCE_URL}" "cloudberry_source_url" + validate_apache_url "${SOURCE_ASC_URL}" "cloudberry_source_asc_url" + validate_apache_url "${SOURCE_SHA512_URL}" "cloudberry_source_sha512_url" + + if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then + echo "::error::cloudberry_source_url must end with /${source_tarball_name}" + exit 1 + fi + if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then + echo "::error::cloudberry_source_asc_url must end with /${source_tarball_name}.asc" + exit 1 + fi + if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then + echo "::error::cloudberry_source_sha512_url must end with /${source_tarball_name}.sha512" + exit 1 + fi + + echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}" + echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}" + + - name: Download source release and verification files + shell: bash + env: + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + mkdir -p verified-source + + echo "=== Downloading Cloudberry source tarball... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}" \ + "${SOURCE_URL}" + + echo "=== Downloading signature... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "${SOURCE_ASC_URL}" + + echo "=== Downloading checksum... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \ + "${SOURCE_SHA512_URL}" + + echo "=== Downloading KEYS... ===" + curl --fail --location --silent --show-error \ + --output "verified-source/KEYS" \ + "${KEYS_URL}" + + ls -la verified-source/ + + - name: Verify source release signature and checksum + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }} + run: | + set -euo pipefail + + export GNUPGHOME="${RUNNER_TEMP}/gnupg" + mkdir -p "${GNUPGHOME}" + chmod 700 "${GNUPGHOME}" + + echo "=== Importing project KEYS... ===" + gpg --import verified-source/KEYS + + echo "=== Verifying GPG signature... ===" + gpg --verify \ + "verified-source/${SOURCE_TARBALL_NAME}.asc" \ + "verified-source/${SOURCE_TARBALL_NAME}" + + echo "=== Verifying SHA-512 checksum... ===" + ( + cd verified-source + sha512sum -c "${SOURCE_TARBALL_NAME}.sha512" + ) + + tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null + echo "=== Cloudberry source release verification passed. ===" + + - name: Summarize verified source release + shell: bash + env: + VERSION: ${{ github.event.inputs.cloudberry_version }} + SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }} + SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }} + SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }} + run: | + { + echo "# Verified Cloudberry source release" + echo "- Version: ${VERSION}" + echo "- Source URL: ${SOURCE_URL}" + echo "- Signature URL: ${SOURCE_ASC_URL}" + echo "- Checksum URL: ${SOURCE_SHA512_URL}" + echo "- KEYS URL: ${KEYS_URL}" + echo "- GPG verification: PASS" + echo "- SHA-512 verification: PASS" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload verified source release + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.validate.outputs.artifact_name }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: verified-source/* + + # ==================================================================== + # Job 3: Build cloudberry-backup portable tarball packages. + # + # Runs BEFORE build-cloudberry so the backup package is ready when + # Cloudberry finishes building. Built on Rocky 8 (glibc 2.28) for + # maximum run-time compatibility across Linux distributions. + # ==================================================================== + build-backup-packages: + name: Build backup ${{ matrix.arch }} + needs: verify-cloudberry-backup-source + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.build_container_image }} + options: >- + --user root + --hostname cdw + strategy: + fail-fast: false + matrix: + include: + - arch: linux-amd64 + runner: ubuntu-24.04 + goarch: amd64 + build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + - arch: linux-arm64 + runner: ubuntu-24.04-arm + goarch: arm64 + build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + + steps: + - name: Initialize build container + shell: bash + run: | + set -euo pipefail + su - gpadmin -c "/tmp/init_system.sh" + + - name: Verify build environment + shell: bash + run: | + set -euo pipefail + + echo "Build host: $(uname -m)" + su - gpadmin -c " + echo \"Go version: \$(go version)\" + echo \"GCC version: \$(gcc --version | head -1 || true)\" + echo \"glibc version: \$(ldd --version 2>&1 | head -1 || true)\" + " + + - name: Download verified cloudberry-backup source + uses: actions/download-artifact@v4 + with: + name: ${{ needs.verify-cloudberry-backup-source.outputs.artifact_name }} + path: ${{ github.workspace }}/verified-source + + - name: Extract source release + id: extract + shell: bash + env: + SOURCE_TARBALL_NAME: ${{ needs.verify-cloudberry-backup-source.outputs.source_tarball_name }} + run: | + set -euo pipefail + tarball_path="${GITHUB_WORKSPACE}/verified-source/${SOURCE_TARBALL_NAME}" + source_root_name="$(tar -tzf "${tarball_path}" | head -1 | cut -d/ -f1 || true)" + + echo "Extracting source: ${source_root_name}" + tar -xzf "${tarball_path}" -C "${GITHUB_WORKSPACE}" + source_dir="${GITHUB_WORKSPACE}/${source_root_name}" + echo "source_dir=${source_dir}" >> "${GITHUB_OUTPUT}" + echo "Source extracted to: ${source_dir}" + ls -la "${source_dir}" + + - name: Build convenience package + id: build + shell: bash + env: + SOURCE_DIR: ${{ steps.extract.outputs.source_dir }} + VERSION: ${{ github.event.inputs.version }} + run: | + set -euo pipefail + + # Give gpadmin ownership so make package can write build/ artifacts + chown -R gpadmin:gpadmin "${SOURCE_DIR}" + + echo "Building package natively on $(uname -m)..." + su - gpadmin -c " + set -euo pipefail + export GOPATH=\$HOME/go + export PATH=\$PATH:/usr/local/go/bin:\$GOPATH/bin + cd '${SOURCE_DIR}' + make package 2>&1 + " + + build_dir="${SOURCE_DIR}/build" + package_file=$(ls -1 "${build_dir}"/*.tar.gz 2>/dev/null | head -1 || true) + if [[ -z "${package_file}" || ! -f "${package_file}" ]]; then + echo "::error::Package file not found in ${build_dir}" + ls -la "${build_dir}" || true + exit 1 + fi + echo "Package built: ${package_file}" + echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}" + echo "Package contents:" + tar -tzf "${package_file}" + + - name: Generate SHA512 checksum + shell: bash + env: + BUILD_DIR: ${{ steps.extract.outputs.source_dir }}/build + run: | + set -euo pipefail + artifact_dir="${GITHUB_WORKSPACE}/package-artifacts" + mkdir -p "${artifact_dir}" + cp "${BUILD_DIR}"/*.tar.gz "${artifact_dir}/" + ( + cd "${artifact_dir}" + for pkg in *.tar.gz; do + sha512sum "${pkg}" > "${pkg}.sha512" + echo "Generated: ${pkg}.sha512" + cat "${pkg}.sha512" + done + ) + + - name: Summarize build + shell: bash + env: + VERSION: ${{ github.event.inputs.version }} + ARCH: ${{ matrix.arch }} + run: | + artifact_dir="${GITHUB_WORKSPACE}/package-artifacts" + { + echo "# Build: ${ARCH}" + echo "- Release version: ${VERSION}" + echo "- Architecture: ${ARCH}" + echo "- Runner: ${{ matrix.runner }}" + echo "- Build container: ${{ matrix.build_container_image }}" + echo "- glibc baseline: Rocky 8 (glibc 2.28) for max compatibility" + echo "- Packages and checksums:" + ( + cd "${artifact_dir}" + for pkg in *.tar.gz; do + echo " - ${pkg}" + echo " - ${pkg}.sha512" + done + ) + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload package artifacts + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.arch }} + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: package-artifacts/ + + # ==================================================================== + # Job 4: Build Apache Cloudberry from verified source (~8 min), + # then test the cloudberry-backup package against it. + # + # Cloudberry is built and installed inside the build container at + # /usr/local/cloudberry-db. After the build, we download the + # cloudberry-backup package artifact, install it, spin up gpdemo, + # and run a functional backup + restore smoke test — all in one job. + # ==================================================================== + build-cloudberry: + name: Build & test ${{ matrix.target_os }}-${{ matrix.arch }} + needs: + - verify-cloudberry-source + - build-backup-packages + runs-on: ${{ matrix.runner }} + timeout-minutes: 75 + container: + image: ${{ matrix.build_container_image }} + options: >- + --user root + --hostname cdw + --shm-size=2gb + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + strategy: + fail-fast: false + matrix: + include: Review Comment: Is this workflow expected to support older source releases? The `make package` target in 2.1.0 does not include the compliance files that the current target adds. If older releases are in scope, we should document the minimum supported version or validate the required package contents before upload. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
