blob: e915c49291f46ac08b307787030d919688ae86ca [file]
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: CI
on:
workflow_call:
# Called from ci-main.yml and ci-pr.yml, which define different concurrency groups.
secrets:
DEVELOCITY_ACCESS_KEY:
required: false
jobs:
build-checks:
name: Gradle Build Checks
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: &setup-java-vars
java-version: '21'
distribution: 'temurin'
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run Checks
env: &gradle_env_vars
GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }}
DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }}
DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} # zizmor: ignore[secrets-outside-env]
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # zizmor: ignore[secrets-outside-env]
run: |
./gradlew check sourceTarball distTar distZip publishToMavenLocal \
-x :polaris-runtime-service:test \
-x :polaris-admin:test \
-PnoIntegrationTests \
--continue
- name: Verify configuration reference is up to date
run: make config-doc-verify
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: failure()
with:
name: upload-${{ github.job }}-artifacts
path: &test-archive-path |
**/build/test-results/**
**/build/reports/tests/**
#
# The following are not strictly "Gradle build checks", but run quite fast.
#
# Adding separate jobs for these requires a new GH runner instance, which requires even more resources.
#
- name: Verify Copilot instructions
run: .github/scripts/check-copilot-instructions.sh
- name: Set up Node 24
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '24'
- name: Renovate Configuration Validation
run: npx --yes --package renovate renovate-config-validator --strict --no-global .github/renovate.json5
# Intentionally unpinned to always use the latest allowlist from the ASF.
- uses: apache/infrastructure-actions/allowlist-check@main # zizmor: ignore[unpinned-uses]
runtime-service-tests:
name: Runtime Service Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run Quarkus runtime tests
env: *gradle_env_vars
run: ./gradlew :polaris-runtime-service:test --continue
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: failure()
with:
name: upload-${{ github.job }}-artifacts
path: *test-archive-path
runtime-service-int-tests:
name: Runtime Service Integration Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run Quarkus runtime tests
env: *gradle_env_vars
run: ./gradlew :polaris-runtime-service:intTest --continue
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: failure()
with:
name: upload-${{ github.job }}-artifacts
path: *test-archive-path
admin-tool-tests:
name: Admin Tool Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run Quarkus admin tests
env: *gradle_env_vars
run: ./gradlew :polaris-admin:test --continue
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: failure()
with:
name: upload-${{ github.job }}-artifacts
path: *test-archive-path
integration-tests:
name: Other Integration Tests
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Run integration tests
env: *gradle_env_vars
run: |
./gradlew \
intTest \
-x :polaris-runtime-service:intTest \
--continue
- name: Save partial Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-save
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: failure()
with:
name: upload-${{ github.job }}-artifacts
path: *test-archive-path
docker-image-build:
name: Polaris Docker Image Build
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Image build
env: *gradle_env_vars
run: |
./gradlew \
:polaris-server:assemble \
-Dquarkus.container-image.build=true
- name: Save image to tar
run: |
docker save apache/polaris:latest | zstd -T0 > polaris-image.tar.zst
- name: Upload image artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: polaris-image
path: polaris-image.tar.zst
retention-days: 1
- name: Upload server app artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: polaris-server-app
path: runtime/server/build/quarkus-app
retention-days: 1
docker-image-scan:
name: Polaris Docker Image Scan
if: github.repository == 'apache/polaris'
runs-on: ubuntu-latest
needs:
- docker-image-build
timeout-minutes: 20
permissions:
contents: read
security-events: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Download image artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: polaris-image
- name: Load image
run: zstd -d -c polaris-image.tar.zst | docker load
- name: Cache Trivy vulnerability DB
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ github.workspace }}/.cache/trivy
key: trivy-db-${{ runner.os }}
# lhotari/sandboxed-trivy-action is the ASF-approved fork of
# aquasecurity/trivy-action; it runs Trivy inside a hardened container.
- name: Run Trivy scan (table, logged)
uses: lhotari/sandboxed-trivy-action@f01374b6cc3bf7264ab238293e94f6db7ada6dd0 # v1.0.2
with:
scan-type: image
scan-ref: apache/polaris:latest
format: table
severity: HIGH,CRITICAL
ignore-unfixed: 'true'
exit-code: '0'
- name: Run Trivy scan (SARIF)
uses: lhotari/sandboxed-trivy-action@f01374b6cc3bf7264ab238293e94f6db7ada6dd0 # v1.0.2
with:
scan-type: image
scan-ref: apache/polaris:latest
format: sarif
output: trivy-results.sarif
severity: HIGH,CRITICAL
ignore-unfixed: 'true'
exit-code: '0'
- name: Upload SARIF to GitHub Code Scanning
if: ${{ startsWith(github.ref, 'refs/heads/') }}
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
with:
sarif_file: trivy-results.sarif
category: trivy-image
- name: Upload SARIF as workflow artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: trivy-results-sarif
path: trivy-results.sarif
retention-days: 7
helm-tests:
name: Helm Tests (Helm ${{ matrix.helm-version }} - K8s ${{ matrix.kubernetes-version }})
runs-on: ubuntu-latest
needs:
- docker-image-build
timeout-minutes: 60
permissions:
contents: read
strategy:
matrix:
include:
- helm-version: 'v3.20.0'
kubernetes-version: 'v1.33.8'
- helm-version: 'v4.0.5'
kubernetes-version: 'v1.34.4'
- helm-version: 'v4.1.1'
kubernetes-version: 'v1.35.1'
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Set up Helm
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
with:
version: ${{ matrix.helm-version }}
- name: Set up chart-testing
run: |
CT_VERSION=3.14.0
ARCH=$(uname -m)
case "$ARCH" in
x86_64|amd64) ARCH=amd64 ;;
aarch64|arm64) ARCH=arm64 ;;
esac
curl -sSLo ct.tar.gz "https://github.com/helm/chart-testing/releases/download/v${CT_VERSION}/chart-testing_${CT_VERSION}_linux_${ARCH}.tar.gz"
sudo tar -xzf ct.tar.gz -C /usr/local/bin ct
mkdir -p .ct
tar -xzf ct.tar.gz --strip-components=1 -C .ct etc
rm ct.tar.gz
pip install yamllint yamale
- name: Set up helm-docs
run: |
go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Install Helm plugins
run: make helm-install-plugins
- name: Verify Helm schema is up to date
run: make helm-schema-verify
- name: Verify Helm documentation is up to date
run: make helm-doc-verify
- name: Run 'helm template' validation
run: |
cd helm/polaris
for f in values.yaml ci/*.yaml; do
echo "::group::helm template $f"
helm template --debug --namespace polaris-ns --values $f .
echo "::endgroup::"
done
- name: Run Helm unit tests
run: make helm-unittest
- name: Run chart-testing (lint)
env:
DEFAULT_BRANCH: ${{ github.base_ref || github.event.repository.default_branch }}
run: ct lint --target-branch "${DEFAULT_BRANCH}" --debug --charts ./helm/polaris --validate-maintainers=false
- name: Set up Minikube
uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21
with:
kubernetes-version: ${{ matrix.kubernetes-version }}
- name: Download image artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: polaris-image
- name: Load image into Minikube
run: |
zstd -d -c polaris-image.tar.zst | minikube image load -
APP_VERSION=$(grep '^appVersion:' helm/polaris/Chart.yaml | awk '{print $2}')
minikube image tag apache/polaris:latest apache/polaris:${APP_VERSION}
echo "Available images:"
minikube image ls
- name: Install fixtures
run: |
kubectl create namespace polaris-ns
kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures
kubectl wait --namespace polaris-ns --for=condition=ready pod \
--selector=app.kubernetes.io/name=postgres --timeout=120s
kubectl wait --namespace polaris-ns --for=condition=ready pod \
--selector=app.kubernetes.io/name=mongodb --timeout=120s
- name: Run chart-testing (install)
env:
DEFAULT_BRANCH: ${{ github.base_ref || github.event.repository.default_branch }}
run: |
ct install --target-branch "${DEFAULT_BRANCH}" \
--namespace polaris-ns \
--debug --charts ./helm/polaris
python-client:
name: Polaris Python Client Tests
runs-on: ubuntu-latest
permissions:
contents: read
needs:
- docker-image-build
strategy:
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Download server-app artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: polaris-server-app
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Lint
run: make client-lint
- name: License Compliance Check
run: make client-license-check
- name: Generated Client Tests
run: make client-unit-test
- name: Integration Tests
run: make run-client-integration-test
- name: Run Polaris Client help manual
run: .venv/bin/polaris --help
regtest:
name: Regression Tests (${{ matrix.s3-backend }})
runs-on: ubuntu-latest
permissions:
contents: read
needs:
- docker-image-build
strategy:
fail-fast: false
matrix:
# Matrix job for future extensibility
s3-backend: ["rustfs"]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Free disk space
uses: ./.github/actions/free-disk-space
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Fix permissions
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
- name: Download image artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: polaris-image
- name: Load image
run: zstd -d -c polaris-image.tar.zst | docker load
- name: Regression Test
env:
AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} # zizmor: ignore[secrets-outside-env]
AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} # zizmor: ignore[secrets-outside-env]
S3_TEST_BACKEND: ${{ matrix.s3-backend }}
run: docker compose --profile ${{ matrix.s3-backend }} -f regtests/docker-compose.yml up --build --exit-code-from regtest
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Build Spark plugin artifacts
env: *gradle_env_vars
run: |
./gradlew \
:publishToMavenLocal \
:polaris-core:publishToMavenLocal \
:polaris-spark-3.5_2.12:publishToMavenLocal \
:polaris-spark-3.5_2.12:createPolarisSparkJar
- name: Spark Plugin Regression Test
# NOTE: the regression test runs with spark 3.5.6 and scala 2.12 in Java 17. We also have integration
# tests runs with the existing gradle.yml, which only runs on Java 21. Since spark Java compatibility
# for 3.5 is 8, 11, and 17, we should run spark client with those compatible java versions.
# TODO: add separate spark client CI and run with Java 8, 11 and 17.
run: docker compose -f plugins/spark/v3.5/regtests/docker-compose.yml up --build --exit-code-from regtest
markdown-link-check:
name: "Markdown Link Check"
if: github.repository == 'apache/polaris'
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Markdown link check
uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225 # v1.1.2
with:
use-quiet-mode: 'yes'
config-file: '.github/workflows/check-md-link-config.json'
folder-path: 'regtests, .github, build-logic, polaris-core, runtime, persistence, spec, getting-started, helm'
file-path: 'CHAT_BYLAWS.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md, README.md, SECURITY.md'
site:
name: "Site Generation"
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Checkout Versioned Docs
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: "versioned-docs"
path: site/content/releases
persist-credentials: false
# Do not let this step fail when the versioned-docs branch does not exist, as on most forks
continue-on-error: true
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Install docker-compose
run: |
sudo curl --fail-with-body -SL \
https://github.com/docker/compose/releases/download/v2.29.2/docker-compose-linux-x86_64 \
-o /usr/local/bin/docker-compose
sudo chmod 755 /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
- name: Verify release-doc link policy
run: site/bin/check-release-doc-links.sh
- name: Build Apache Site
run: site/bin/create-static-site.sh
test-site-guides:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with:
java-version: '21'
distribution: 'temurin'
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Setup test environment
uses: ./.github/actions/setup-test-env
- name: Test site testing
working-directory: site/it
run: |
echo "::group::Setup"
python3 -m pip install --upgrade pip
python3 -m venv .venv
. .venv/bin/activate
UV_VERSION="$(cat pyproject.toml | grep -A1 tool.uv | grep -v tool.uv | grep required-version | sed 's/required-version *= *"\([^"]*\)".*/\1/')"
pip install "uv$UV_VERSION"
uv sync --all-packages
echo "::endgroup::"
echo "::group::pytest"
pytest
echo "::endgroup::"
- name: Free disk space
uses: ./.github/actions/free-disk-space
- name: Prepare Gradle build cache
uses: ./.github/actions/ci-incr-build-cache-prepare
- name: Guide testing
working-directory: site
run: it/markdown-testing.py
- name: Archive test results
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: upload-guides-test-artifacts
path: |
site/it/build/tests/**
store-gradle-cache:
name: Store Gradle Cache
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
contents: read
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs:
- build-checks
- runtime-service-tests
- runtime-service-int-tests
- admin-tool-tests
- integration-tests
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up JDK 21
uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
with: *setup-java-vars
- name: Collect partial Gradle build caches
uses: ./.github/actions/ci-incr-build-cache-prepare
with:
cache-read-only: false
required-checks:
# Do not rename this job, this is referenced in .asf.yaml!
name: "Required Checks"
needs:
- build-checks
- runtime-service-tests
- runtime-service-int-tests
- admin-tool-tests
- integration-tests
- docker-image-build
- docker-image-scan
- helm-tests
- python-client
- regtest
- markdown-link-check
- site
- test-site-guides
# Always run this job, even if a "needed" job fails. Without this one, GitHub will not run
# this job and that yields "Success" to the branch protection rule, which is wrong.
# Potential results for each "needed" job are: `success`, `failure`, `cancelled`, `skipped`.
# We consider `success` and `skipped` as "ok", `failure` and `cancelled` as "not ok".
if: ${{ always() }}
runs-on: ubuntu-24.04
permissions: {}
steps:
- env:
NEEDS_RESULTS: ${{ join(needs.*.result, ',') }}
run: |
if [[ "${NEEDS_RESULTS}" =~ .*(failure|cancelled).* ]]; then
echo "At least one check failed"
exit 1
else
echo "All checks passed"
fi