Add convenience package build workflow for cloudberry-backup (#108)
Introduce a manually-triggered GitHub Actions workflow that builds
portable tar.gz packages for Apache cloudberry-backup from its
official source release tarball, and tests them against Apache
Cloudberry built from its own official source release.
Key design decisions:
- cloudberry-backup binaries are built on Rocky 8 (glibc 2.28)
to maximise compatibility across Linux distributions.
- Cloudberry is built per-platform from its source release
tarball (~8 min), avoiding dependence on pre-built Docker
images or external DEB/RPM repositories.
- Build-and-test are fused into one job per platform: after
Cloudberry compiles, gpdemo spins up directly in the same
container, eliminating artifact packaging, transfer, and a
separate test job.
- Only .sha512 checksums are generated; .asc signatures remain
a release manager local step.
diff --git a/.github/workflows/package-convenience-binaries.yml b/.github/workflows/package-convenience-binaries.yml
new file mode 100644
index 0000000..40c50ef
--- /dev/null
+++ b/.github/workflows/package-convenience-binaries.yml
@@ -0,0 +1,924 @@
+# --------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to You under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the
+# License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# --------------------------------------------------------------------
+# GitHub Actions Workflow: Apache Cloudberry-backup Convenience Package Build
+# --------------------------------------------------------------------
+# Description:
+#
+# This workflow manually builds convenience portable tarball packages
+# from an ASF-approved Apache Cloudberry-backup source release tarball,
+# and tests them against Apache Cloudberry built from its official
+# source release tarball.
+#
+# Workflow Overview:
+#
+# 1. verify-cloudberry-backup-source
+# Validates inputs, downloads cloudberry-backup source tarball
+# + .asc + .sha512, verifies GPG signature and checksum, uploads
+# verified source as a workflow artifact.
+#
+# 2. verify-cloudberry-source
+# Same as above, but for the Cloudberry source release tarball.
+#
+# 3. build-backup-packages (matrix: amd64 / arm64)
+# Downloads the verified cloudberry-backup source artifact, runs
+# `make package` on Rocky 8 (glibc 2.28) for maximum run-time
+# compatibility, generates .sha512 checksums, uploads per-arch
+# portable tar.gz packages.
+#
+# 4. build-cloudberry (matrix: 5 platforms × amd64 / arm64 = 10)
+# Extracts Cloudberry source, runs configure + build inside the
+# official Cloudberry build container (~8 min per platform).
+# After the build, Cloudberry is already installed at
+# /usr/local/cloudberry-db. The job then downloads the matching
+# cloudberry-backup package, installs it, creates a gpdemo demo
+# cluster, and runs a functional backup + restore smoke test —
+# all inside the same container, no separate test job needed.
+# Platforms: rocky8, rocky9, rocky10, ubuntu22.04, ubuntu24.04.
+#
+# Scope:
+# - Intended for official Apache Cloudberry-backup source releases
+# managed by the release manager. It's designed for 2.2+ release.
+# - Produces convenience binaries only; detached .asc signatures remain a
+# release manager local step.
+# --------------------------------------------------------------------
+
+name: Apache Cloudberry-backup Convenience Package Build
+
+on:
+ workflow_dispatch:
+ inputs:
+ # ================================================================
+ # cloudberry-backup source release inputs
+ # ================================================================
+ version:
+ description: '[cloudberry-backup] Release version, e.g. 2.2.0-incubating'
+ required: true
+ type: string
+ source_url:
+ description: '[cloudberry-backup] Apache source tarball URL from downloads.apache.org'
+ required: true
+ type: string
+ source_asc_url:
+ description: '[cloudberry-backup] Detached GPG signature URL for the source tarball (.asc)'
+ required: true
+ type: string
+ source_sha512_url:
+ description: '[cloudberry-backup] SHA-512 checksum URL for the source tarball (.sha512)'
+ required: true
+ type: string
+
+ # ================================================================
+ # Cloudberry source release inputs (for the test environment)
+ # ================================================================
+ cloudberry_version:
+ description: '[Cloudberry] Release version, e.g. 2.2.0-incubating'
+ required: true
+ type: string
+ cloudberry_source_url:
+ description: '[Cloudberry] Apache source tarball URL from downloads.apache.org'
+ required: true
+ type: string
+ cloudberry_source_asc_url:
+ description: '[Cloudberry] Detached GPG signature URL for the source tarball (.asc)'
+ required: true
+ type: string
+ cloudberry_source_sha512_url:
+ description: '[Cloudberry] SHA-512 checksum URL for the source tarball (.sha512)'
+ required: true
+ type: string
+
+permissions:
+ contents: read
+
+concurrency:
+ group: backup-package-build-${{ github.ref }}-${{ inputs.version }}
+ cancel-in-progress: true
+
+env:
+ LOG_RETENTION_DAYS: 14
+ KEYS_URL: https://downloads.apache.org/incubator/cloudberry/KEYS
+
+jobs:
+ # ====================================================================
+ # Job 1: Verify the Apache Cloudberry-backup source release
+ # ====================================================================
+ verify-cloudberry-backup-source:
+ name: Verify cloudberry-backup source
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ outputs:
+ source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }}
+ artifact_name: ${{ steps.validate.outputs.artifact_name }}
+ packaging_version: ${{ steps.validate.outputs.packaging_version }}
+ steps:
+ - name: Validate manual inputs
+ id: validate
+ shell: bash
+ env:
+ VERSION: ${{ github.event.inputs.version }}
+ SOURCE_URL: ${{ github.event.inputs.source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }}
+ run: |
+ set -euo pipefail
+
+ if [[ -z "${VERSION}" ]]; then
+ echo "::error::version must not be empty"
+ exit 1
+ fi
+
+ source_tarball_name="apache-cloudberry-backup-${VERSION}-src.tar.gz"
+ artifact_name="verified-source-release-cbbackup-${VERSION}"
+ packaging_version="${VERSION%-incubating}"
+
+ if [[ -z "${packaging_version}" ]]; then
+ echo "::error::Unable to derive packaging version from version=${VERSION}"
+ exit 1
+ fi
+
+ validate_apache_url() {
+ local value="$1"
+ local label="$2"
+ local prefix="https://downloads.apache.org/incubator/cloudberry/"
+ if [[ -z "${value}" ]]; then
+ echo "::error::${label} must not be empty"
+ exit 1
+ fi
+ if [[ "${value}" != "${prefix}"* ]]; then
+ echo "::error::${label} must use downloads.apache.org (got: ${value})"
+ exit 1
+ fi
+ }
+
+ validate_apache_url "${SOURCE_URL}" "source_url"
+ validate_apache_url "${SOURCE_ASC_URL}" "source_asc_url"
+ validate_apache_url "${SOURCE_SHA512_URL}" "source_sha512_url"
+
+ if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then
+ echo "::error::source_url must end with /${source_tarball_name}"
+ exit 1
+ fi
+ if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then
+ echo "::error::source_asc_url must end with /${source_tarball_name}.asc"
+ exit 1
+ fi
+ if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then
+ echo "::error::source_sha512_url must end with /${source_tarball_name}.sha512"
+ exit 1
+ fi
+
+ echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}"
+ echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}"
+ echo "packaging_version=${packaging_version}" >> "${GITHUB_OUTPUT}"
+
+ - name: Download source release and verification files
+ shell: bash
+ env:
+ SOURCE_URL: ${{ github.event.inputs.source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }}
+ SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+ mkdir -p verified-source
+
+ echo "=== Downloading source tarball... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}" \
+ "${SOURCE_URL}"
+
+ echo "=== Downloading signature... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \
+ "${SOURCE_ASC_URL}"
+
+ echo "=== Downloading checksum... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \
+ "${SOURCE_SHA512_URL}"
+
+ echo "=== Downloading KEYS... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/KEYS" \
+ "${KEYS_URL}"
+
+ echo "=== All files downloaded successfully. ==="
+ ls -la verified-source/
+
+ - name: Verify source release signature and checksum
+ shell: bash
+ env:
+ SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+
+ export GNUPGHOME="${RUNNER_TEMP}/gnupg"
+ mkdir -p "${GNUPGHOME}"
+ chmod 700 "${GNUPGHOME}"
+
+ echo "=== Importing project KEYS... ==="
+ gpg --import verified-source/KEYS
+
+ echo "=== Verifying GPG signature... ==="
+ gpg --verify \
+ "verified-source/${SOURCE_TARBALL_NAME}.asc" \
+ "verified-source/${SOURCE_TARBALL_NAME}"
+
+ echo "=== Verifying SHA-512 checksum... ==="
+ (
+ cd verified-source
+ sha512sum -c "${SOURCE_TARBALL_NAME}.sha512"
+ )
+
+ echo "=== Verifying tarball integrity (listing contents)... ==="
+ tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null
+ echo "=== cloudberry-backup source release verification passed. ==="
+
+ - name: Summarize verified source release
+ shell: bash
+ env:
+ VERSION: ${{ github.event.inputs.version }}
+ SOURCE_URL: ${{ github.event.inputs.source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.source_sha512_url }}
+ run: |
+ {
+ echo "# Verified cloudberry-backup source release"
+ echo "- Version: ${VERSION}"
+ echo "- Packaging version: ${{ steps.validate.outputs.packaging_version }}"
+ echo "- Source URL: ${SOURCE_URL}"
+ echo "- Signature URL: ${SOURCE_ASC_URL}"
+ echo "- Checksum URL: ${SOURCE_SHA512_URL}"
+ echo "- KEYS URL: ${KEYS_URL}"
+ echo "- GPG verification: PASS"
+ echo "- SHA-512 verification: PASS"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Upload verified source release
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ steps.validate.outputs.artifact_name }}
+ retention-days: ${{ env.LOG_RETENTION_DAYS }}
+ if-no-files-found: error
+ path: verified-source/*
+
+ # ====================================================================
+ # Job 2: Verify the Apache Cloudberry source release
+ # ====================================================================
+ verify-cloudberry-source:
+ name: Verify Cloudberry source
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ outputs:
+ source_tarball_name: ${{ steps.validate.outputs.source_tarball_name }}
+ artifact_name: ${{ steps.validate.outputs.artifact_name }}
+ steps:
+ - name: Validate manual inputs
+ id: validate
+ shell: bash
+ env:
+ VERSION: ${{ github.event.inputs.cloudberry_version }}
+ SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }}
+ run: |
+ set -euo pipefail
+
+ if [[ -z "${VERSION}" ]]; then
+ echo "::error::cloudberry_version must not be empty"
+ exit 1
+ fi
+
+ source_tarball_name="apache-cloudberry-${VERSION}-src.tar.gz"
+ artifact_name="verified-source-release-cloudberry-${VERSION}"
+
+ validate_apache_url() {
+ local value="$1"
+ local label="$2"
+ local prefix="https://downloads.apache.org/incubator/cloudberry/"
+ if [[ -z "${value}" ]]; then
+ echo "::error::${label} must not be empty"
+ exit 1
+ fi
+ if [[ "${value}" != "${prefix}"* ]]; then
+ echo "::error::${label} must use downloads.apache.org (got: ${value})"
+ exit 1
+ fi
+ }
+
+ validate_apache_url "${SOURCE_URL}" "cloudberry_source_url"
+ validate_apache_url "${SOURCE_ASC_URL}" "cloudberry_source_asc_url"
+ validate_apache_url "${SOURCE_SHA512_URL}" "cloudberry_source_sha512_url"
+
+ if [[ "${SOURCE_URL}" != */"${source_tarball_name}" ]]; then
+ echo "::error::cloudberry_source_url must end with /${source_tarball_name}"
+ exit 1
+ fi
+ if [[ "${SOURCE_ASC_URL}" != */"${source_tarball_name}.asc" ]]; then
+ echo "::error::cloudberry_source_asc_url must end with /${source_tarball_name}.asc"
+ exit 1
+ fi
+ if [[ "${SOURCE_SHA512_URL}" != */"${source_tarball_name}.sha512" ]]; then
+ echo "::error::cloudberry_source_sha512_url must end with /${source_tarball_name}.sha512"
+ exit 1
+ fi
+
+ echo "source_tarball_name=${source_tarball_name}" >> "${GITHUB_OUTPUT}"
+ echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}"
+
+ - name: Download source release and verification files
+ shell: bash
+ env:
+ SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }}
+ SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+ mkdir -p verified-source
+
+ echo "=== Downloading Cloudberry source tarball... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}" \
+ "${SOURCE_URL}"
+
+ echo "=== Downloading signature... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}.asc" \
+ "${SOURCE_ASC_URL}"
+
+ echo "=== Downloading checksum... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/${SOURCE_TARBALL_NAME}.sha512" \
+ "${SOURCE_SHA512_URL}"
+
+ echo "=== Downloading KEYS... ==="
+ curl --fail --location --silent --show-error \
+ --output "verified-source/KEYS" \
+ "${KEYS_URL}"
+
+ ls -la verified-source/
+
+ - name: Verify source release signature and checksum
+ shell: bash
+ env:
+ SOURCE_TARBALL_NAME: ${{ steps.validate.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+
+ export GNUPGHOME="${RUNNER_TEMP}/gnupg"
+ mkdir -p "${GNUPGHOME}"
+ chmod 700 "${GNUPGHOME}"
+
+ echo "=== Importing project KEYS... ==="
+ gpg --import verified-source/KEYS
+
+ echo "=== Verifying GPG signature... ==="
+ gpg --verify \
+ "verified-source/${SOURCE_TARBALL_NAME}.asc" \
+ "verified-source/${SOURCE_TARBALL_NAME}"
+
+ echo "=== Verifying SHA-512 checksum... ==="
+ (
+ cd verified-source
+ sha512sum -c "${SOURCE_TARBALL_NAME}.sha512"
+ )
+
+ tar -tzf "verified-source/${SOURCE_TARBALL_NAME}" >/dev/null
+ echo "=== Cloudberry source release verification passed. ==="
+
+ - name: Summarize verified source release
+ shell: bash
+ env:
+ VERSION: ${{ github.event.inputs.cloudberry_version }}
+ SOURCE_URL: ${{ github.event.inputs.cloudberry_source_url }}
+ SOURCE_ASC_URL: ${{ github.event.inputs.cloudberry_source_asc_url }}
+ SOURCE_SHA512_URL: ${{ github.event.inputs.cloudberry_source_sha512_url }}
+ run: |
+ {
+ echo "# Verified Cloudberry source release"
+ echo "- Version: ${VERSION}"
+ echo "- Source URL: ${SOURCE_URL}"
+ echo "- Signature URL: ${SOURCE_ASC_URL}"
+ echo "- Checksum URL: ${SOURCE_SHA512_URL}"
+ echo "- KEYS URL: ${KEYS_URL}"
+ echo "- GPG verification: PASS"
+ echo "- SHA-512 verification: PASS"
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Upload verified source release
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ steps.validate.outputs.artifact_name }}
+ retention-days: ${{ env.LOG_RETENTION_DAYS }}
+ if-no-files-found: error
+ path: verified-source/*
+
+ # ====================================================================
+ # Job 3: Build cloudberry-backup portable tarball packages.
+ #
+ # Runs BEFORE build-cloudberry so the backup package is ready when
+ # Cloudberry finishes building. Built on Rocky 8 (glibc 2.28) for
+ # maximum run-time compatibility across Linux distributions.
+ # ====================================================================
+ build-backup-packages:
+ name: Build backup ${{ matrix.arch }}
+ needs: verify-cloudberry-backup-source
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 30
+ container:
+ image: ${{ matrix.build_container_image }}
+ options: >-
+ --user root
+ --hostname cdw
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - arch: linux-amd64
+ runner: ubuntu-24.04
+ goarch: amd64
+ build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest
+ - arch: linux-arm64
+ runner: ubuntu-24.04-arm
+ goarch: arm64
+ build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest
+
+ steps:
+ - name: Initialize build container
+ shell: bash
+ run: |
+ set -euo pipefail
+ su - gpadmin -c "/tmp/init_system.sh"
+
+ - name: Verify build environment
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ echo "Build host: $(uname -m)"
+ su - gpadmin -c "
+ echo \"Go version: \$(go version)\"
+ echo \"GCC version: \$(gcc --version | head -1 || true)\"
+ echo \"glibc version: \$(ldd --version 2>&1 | head -1 || true)\"
+ "
+
+ - name: Download verified cloudberry-backup source
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ needs.verify-cloudberry-backup-source.outputs.artifact_name }}
+ path: ${{ github.workspace }}/verified-source
+
+ - name: Extract source release
+ id: extract
+ shell: bash
+ env:
+ SOURCE_TARBALL_NAME: ${{ needs.verify-cloudberry-backup-source.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+ tarball_path="${GITHUB_WORKSPACE}/verified-source/${SOURCE_TARBALL_NAME}"
+ source_root_name="$(tar -tzf "${tarball_path}" | head -1 | cut -d/ -f1 || true)"
+
+ echo "Extracting source: ${source_root_name}"
+ tar -xzf "${tarball_path}" -C "${GITHUB_WORKSPACE}"
+ source_dir="${GITHUB_WORKSPACE}/${source_root_name}"
+ echo "source_dir=${source_dir}" >> "${GITHUB_OUTPUT}"
+ echo "Source extracted to: ${source_dir}"
+ ls -la "${source_dir}"
+
+ - name: Build convenience package
+ id: build
+ shell: bash
+ env:
+ SOURCE_DIR: ${{ steps.extract.outputs.source_dir }}
+ VERSION: ${{ github.event.inputs.version }}
+ run: |
+ set -euo pipefail
+
+ # Give gpadmin ownership so make package can write build/ artifacts
+ chown -R gpadmin:gpadmin "${SOURCE_DIR}"
+
+ echo "Building package natively on $(uname -m)..."
+ su - gpadmin -c "
+ set -euo pipefail
+ export GOPATH=\$HOME/go
+ export PATH=\$PATH:/usr/local/go/bin:\$GOPATH/bin
+ cd '${SOURCE_DIR}'
+ make package 2>&1
+ "
+
+ build_dir="${SOURCE_DIR}/build"
+ package_file=$(ls -1 "${build_dir}"/*.tar.gz 2>/dev/null | head -1 || true)
+ if [[ -z "${package_file}" || ! -f "${package_file}" ]]; then
+ echo "::error::Package file not found in ${build_dir}"
+ ls -la "${build_dir}" || true
+ exit 1
+ fi
+ echo "Package built: ${package_file}"
+ echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}"
+ echo "Package contents:"
+ tar -tzf "${package_file}"
+
+ - name: Generate SHA512 checksum
+ shell: bash
+ env:
+ BUILD_DIR: ${{ steps.extract.outputs.source_dir }}/build
+ run: |
+ set -euo pipefail
+ artifact_dir="${GITHUB_WORKSPACE}/package-artifacts"
+ mkdir -p "${artifact_dir}"
+ cp "${BUILD_DIR}"/*.tar.gz "${artifact_dir}/"
+ (
+ cd "${artifact_dir}"
+ for pkg in *.tar.gz; do
+ sha512sum "${pkg}" > "${pkg}.sha512"
+ echo "Generated: ${pkg}.sha512"
+ cat "${pkg}.sha512"
+ done
+ )
+
+ - name: Summarize build
+ shell: bash
+ env:
+ VERSION: ${{ github.event.inputs.version }}
+ ARCH: ${{ matrix.arch }}
+ run: |
+ artifact_dir="${GITHUB_WORKSPACE}/package-artifacts"
+ {
+ echo "# Build: ${ARCH}"
+ echo "- Release version: ${VERSION}"
+ echo "- Architecture: ${ARCH}"
+ echo "- Runner: ${{ matrix.runner }}"
+ echo "- Build container: ${{ matrix.build_container_image }}"
+ echo "- glibc baseline: Rocky 8 (glibc 2.28) for max compatibility"
+ echo "- Packages and checksums:"
+ (
+ cd "${artifact_dir}"
+ for pkg in *.tar.gz; do
+ echo " - ${pkg}"
+ echo " - ${pkg}.sha512"
+ done
+ )
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ - name: Upload package artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: packages-${{ matrix.arch }}
+ retention-days: ${{ env.LOG_RETENTION_DAYS }}
+ if-no-files-found: error
+ path: package-artifacts/
+
+ # ====================================================================
+ # Job 4: Build Apache Cloudberry from verified source (~8 min),
+ # then test the cloudberry-backup package against it.
+ #
+ # Cloudberry is built and installed inside the build container at
+ # /usr/local/cloudberry-db. After the build, we download the
+ # cloudberry-backup package artifact, install it, spin up gpdemo,
+ # and run a functional backup + restore smoke test — all in one job.
+ # ====================================================================
+ build-cloudberry:
+ name: Build & test ${{ matrix.target_os }}-${{ matrix.arch }}
+ needs:
+ - verify-cloudberry-source
+ - build-backup-packages
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 75
+ container:
+ image: ${{ matrix.build_container_image }}
+ options: >-
+ --user root
+ --hostname cdw
+ --shm-size=2gb
+ -v /usr/share:/host_usr_share
+ -v /usr/local:/host_usr_local
+ -v /opt:/host_opt
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ # Rocky 8
+ - {target_os: rocky8, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest}
+ - {target_os: rocky8, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky8-latest}
+ # Rocky 9
+ - {target_os: rocky9, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest}
+ - {target_os: rocky9, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest}
+ # Rocky 10
+ - {target_os: rocky10, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest}
+ - {target_os: rocky10, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-rocky10-latest}
+ # Ubuntu 22.04
+ - {target_os: ubuntu22.04, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest}
+ - {target_os: ubuntu22.04, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest}
+ # Ubuntu 24.04
+ - {target_os: ubuntu24.04, arch: amd64, backup_arch: linux-amd64, runner: ubuntu-24.04, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest}
+ - {target_os: ubuntu24.04, arch: arm64, backup_arch: linux-arm64, runner: ubuntu-24.04-arm, build_container_image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest}
+
+ steps:
+ # ----------------------------------------------------------------
+ # Phase A: Build Cloudberry from verified source
+ # ----------------------------------------------------------------
+
+ - name: Free disk space
+ shell: bash
+ run: |
+ set -euo pipefail
+ rm -rf /host_opt/hostedtoolcache || true
+ rm -rf /host_usr_local/lib/android || true
+ rm -rf /host_usr_share/dotnet || true
+ rm -rf /host_opt/ghc || true
+ rm -rf /host_usr_local/.ghcup || true
+ rm -rf /host_usr_share/swift || true
+ rm -rf /host_usr_local/share/powershell || true
+ rm -rf /host_usr_local/share/chromium || true
+ rm -rf /host_usr_share/miniconda || true
+ rm -rf /host_opt/az || true
+ rm -rf /host_usr_share/sbt || true
+ df -h /
+
+ - name: Initialize build container
+ shell: bash
+ run: |
+ set -euo pipefail
+ su - gpadmin -c "/tmp/init_system.sh"
+
+ - name: Download verified Cloudberry source
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ needs.verify-cloudberry-source.outputs.artifact_name }}
+ path: ${{ github.workspace }}/verified-source
+
+ - name: Extract Cloudberry source
+ id: extract
+ shell: bash
+ env:
+ SOURCE_TARBALL_NAME: ${{ needs.verify-cloudberry-source.outputs.source_tarball_name }}
+ run: |
+ set -euo pipefail
+ tarball_path="${GITHUB_WORKSPACE}/verified-source/${SOURCE_TARBALL_NAME}"
+ source_root_name="$(tar -tzf "${tarball_path}" | head -1 | cut -d/ -f1 || true)"
+
+ tar -xzf "${tarball_path}" -C "${GITHUB_WORKSPACE}"
+ mkdir -p "${GITHUB_WORKSPACE}/cloudberry"
+ mv "${GITHUB_WORKSPACE}/${source_root_name}"/* "${GITHUB_WORKSPACE}/cloudberry/"
+ mv "${GITHUB_WORKSPACE}/${source_root_name}"/.[!.]* "${GITHUB_WORKSPACE}/cloudberry/" 2>/dev/null || true
+ rmdir "${GITHUB_WORKSPACE}/${source_root_name}"
+
+ source_dir="${GITHUB_WORKSPACE}/cloudberry"
+ chown -R gpadmin:gpadmin "${GITHUB_WORKSPACE}"
+ echo "source_dir=${source_dir}" >> "${GITHUB_OUTPUT}"
+ echo "Cloudberry source extracted to: ${source_dir}"
+
+ - name: Configure Cloudberry
+ shell: bash
+ env:
+ SOURCE_DIR: ${{ steps.extract.outputs.source_dir }}
+ run: |
+ set -euo pipefail
+ export BUILD_DESTINATION="/usr/local/cloudberry-db"
+
+ mkdir -p "${SOURCE_DIR}/build-logs"
+ chown -R gpadmin:gpadmin "${SOURCE_DIR}/build-logs"
+ chmod +x "${SOURCE_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
+
+ echo "=== Configuring Cloudberry... ==="
+ if ! su - gpadmin -c "cd ${SOURCE_DIR} && SRC_DIR=${SOURCE_DIR} BUILD_USER=github-actions BUILD_DESTINATION=${BUILD_DESTINATION} ${SOURCE_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then
+ echo "::error::Cloudberry configure script failed"
+ exit 1
+ fi
+ echo "=== Cloudberry configuration complete. ==="
+
+ - name: Build Cloudberry
+ shell: bash
+ env:
+ SOURCE_DIR: ${{ steps.extract.outputs.source_dir }}
+ run: |
+ set -euo pipefail
+ export BUILD_DESTINATION="/usr/local/cloudberry-db"
+
+ chmod +x "${SOURCE_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh
+
+ echo "=== Building Cloudberry... ==="
+ if ! su - gpadmin -c "cd ${SOURCE_DIR} && SRC_DIR=${SOURCE_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SOURCE_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then
+ echo "::error::Cloudberry build script failed"
+ exit 1
+ fi
+ echo "=== Cloudberry build complete. ==="
+
+ - name: Verify Cloudberry installation
+ shell: bash
+ run: |
+ set -euo pipefail
+ echo "Verifying Cloudberry installation..."
+ if [[ ! -f /usr/local/cloudberry-db/cloudberry-env.sh ]]; then
+ echo "::error::Cloudberry installation not found at /usr/local/cloudberry-db"
+ exit 1
+ fi
+ ls -la /usr/local/cloudberry-db/bin/
+ echo "Cloudberry installation verified."
+
+ # ----------------------------------------------------------------
+ # Phase B: Download & test the cloudberry-backup package
+ # ----------------------------------------------------------------
+
+ - name: Download backup package
+ uses: actions/download-artifact@v4
+ with:
+ name: packages-${{ matrix.backup_arch }}
+ path: ${{ github.workspace }}/package-artifacts
+
+ - name: Verify backup package checksum
+ id: verify-package
+ shell: bash
+ run: |
+ set -euo pipefail
+ artifact_dir="${GITHUB_WORKSPACE}/package-artifacts"
+
+ package_file=$(ls "${artifact_dir}"/*.tar.gz 2>/dev/null | head -1 || true)
+ if [[ -z "${package_file}" || ! -f "${package_file}" ]]; then
+ echo "::error::No tar.gz package found in ${artifact_dir}"
+ ls -la "${artifact_dir}" || true
+ exit 1
+ fi
+
+ checksum_file="${package_file}.sha512"
+ if [[ ! -f "${checksum_file}" ]]; then
+ echo "::error::Checksum file not found: ${checksum_file}"
+ exit 1
+ fi
+
+ ( cd "${artifact_dir}" && sha512sum -c "$(basename "${checksum_file}")" )
+ echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}"
+ echo "Checksum verification passed."
+
+ - name: Extract and install package
+ shell: bash
+ env:
+ PACKAGE_FILE: ${{ steps.verify-package.outputs.package_file }}
+ run: |
+ set -euo pipefail
+
+ work_dir="${GITHUB_WORKSPACE}/pkg-test"
+ mkdir -p "${work_dir}"
+
+ echo "=== Extracting package ==="
+ tar -xzf "${PACKAGE_FILE}" -C "${work_dir}"
+
+ extracted_dir=$(ls -1d "${work_dir}"/apache-cloudberry-backup-incubating-*/ 2>/dev/null | head -1 || true)
+ if [[ -z "${extracted_dir}" ]]; then
+ echo "::error::Extracted package directory not found"
+ ls -la "${work_dir}"
+ exit 1
+ fi
+
+ echo "Package extracted to: ${extracted_dir}"
+ echo "Extracted contents:"
+ ls -lhR "${extracted_dir}"
+
+ echo "=== Installing via install.sh ==="
+ chmod +x "${extracted_dir}/install.sh"
+ sed -i 's/sudo //g' "${extracted_dir}/install.sh"
+ export GPHOME="/usr/local/cloudberry-db"
+ "${extracted_dir}/install.sh" 2>&1
+ echo "Installation complete."
+
+ - name: Verify installed commands
+ shell: bash
+ run: |
+ set -euo pipefail
+ GPHOME="/usr/local/cloudberry-db"
+ expected="gpbackup gprestore gpbackup_helper gpbackup_s3_plugin gpbackman gpbackup_exporter"
+
+ echo "=== Verifying installed binaries ==="
+ for binary in ${expected}; do
+ bin_path="${GPHOME}/bin/${binary}"
+ if [[ ! -f "${bin_path}" ]]; then
+ echo "::warning::${binary} not found — may be absent in older releases"
+ continue
+ fi
+ version_output=$("${bin_path}" --version 2>&1) || {
+ echo "::warning::${binary} --version failed"
+ continue
+ }
+ echo " ${binary}: ${version_output}"
+ done
+ echo "Command verification complete."
+
+ - name: Run backup/restore functional test
+ shell: bash
+ env:
+ TARGET_OS: ${{ matrix.target_os }}
+ ARCH: ${{ matrix.arch }}
+ run: |
+ set -euo pipefail
+
+ work_dir="${GITHUB_WORKSPACE}/functional-test"
+ mkdir -p "${work_dir}"
+ chown -R gpadmin:gpadmin "${GITHUB_WORKSPACE}"
+
+ cat <<'SCRIPT' > /tmp/run_backup_restore_test.sh
+ #!/bin/bash
+ set -euo pipefail
+
+ GPHOME="/usr/local/cloudberry-db"
+
+ source "${GPHOME}/cloudberry-env.sh"
+
+ echo "=== Creating demo cluster ==="
+ cd "${HOME}"
+ gpdemo
+ source "${HOME}/gpdemo-env.sh"
+
+ echo "=== Verifying cluster is running ==="
+ psql -d postgres -c "SELECT 1 AS cluster_ok;"
+ psql -d postgres -c "SELECT version();"
+
+ echo "=== Preparing test database ==="
+ test_db="package_test_db"
+ psql -d postgres -c "DROP DATABASE IF EXISTS ${test_db}"
+ createdb "${test_db}"
+ psql -d "${test_db}" -c "CREATE TABLE test_tbl (id int, name text) DISTRIBUTED RANDOMLY;"
+ psql -d "${test_db}" -c "INSERT INTO test_tbl VALUES (1, 'hello'), (2, 'world');"
+ psql -d "${test_db}" -c "SELECT count(*) AS row_count FROM test_tbl;"
+
+ echo "=== Running gpbackup ==="
+ backup_dir="/tmp/backup_restore_test"
+ rm -rf "${backup_dir}"
+ mkdir -p "${backup_dir}"
+
+ backup_log="${backup_dir}/gpbackup_output.log"
+ gpbackup --dbname "${test_db}" --backup-dir "${backup_dir}" 2>&1 | tee "${backup_log}"
+
+ timestamp=$(grep -E "Backup Timestamp[[:space:]]*=" "${backup_log}" | grep -Eo "[[:digit:]]{14}" | head -1 || true)
+
+ if [[ -z "${timestamp}" ]]; then
+ latest_gpbackup_log=$(ls -1t "${HOME}/gpAdminLogs"/gpbackup_*.log 2>/dev/null | head -1 || true)
+ if [[ -n "${latest_gpbackup_log}" ]]; then
+ timestamp=$(grep -E "Backup Timestamp[[:space:]]*=" "${latest_gpbackup_log}" | grep -Eo "[[:digit:]]{14}" | head -1 || true)
+ fi
+ fi
+
+ if [[ -z "${timestamp}" ]]; then
+ echo "ERROR: Could not parse backup timestamp from gpbackup logs"
+ echo "=== Backup log ==="
+ cat "${backup_log}" || true
+ echo "=== Backup directory contents ==="
+ find "${backup_dir}" -type f | sort
+ exit 1
+ fi
+
+ echo "Backup timestamp: ${timestamp}"
+
+ echo "=== Dropping test database ==="
+ dropdb "${test_db}"
+
+ echo "=== Running gprestore ==="
+ gprestore --timestamp "${timestamp}" --backup-dir "${backup_dir}" --create-db --on-error-continue 2>&1
+
+ echo "=== Verification: connecting to restored database ==="
+ row_count=$(psql -d "${test_db}" -t -c "SELECT count(*) FROM test_tbl" | xargs)
+ if [[ "${row_count}" != "2" ]]; then
+ echo "ERROR: Expected 2 rows in restored test_tbl, got ${row_count}"
+ exit 1
+ fi
+ echo "Restore verified: ${row_count} rows in test_tbl"
+
+ echo "=== All functional tests passed ==="
+ SCRIPT
+
+ chmod +x /tmp/run_backup_restore_test.sh
+
+ set +e
+ su - gpadmin -c "/tmp/run_backup_restore_test.sh"
+ test_status=$?
+ set -e
+
+ {
+ echo "## Cloudberry-backup package functional test"
+ echo "- Target: ${{ matrix.target_os }}/${{ matrix.arch }}"
+ echo "- Package: ${{ steps.verify-package.outputs.package_file }}"
+ if [[ ${test_status} -eq 0 ]]; then
+ echo "- Result: PASS"
+ else
+ echo "- Result: FAIL"
+ fi
+ } >> "${GITHUB_STEP_SUMMARY}"
+
+ exit ${test_status}
