blob: ace3a7e15b984887d55b8a99a3a71af91b33cf70 [file] [log] [blame]
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: full-ci
on:
workflow_call:
inputs:
ratis_args:
type: string
description: Version overrides from custom Ratis build
default: ''
required: false
ref:
type: string
description: Ozone ref (branch, tag or commit SHA)
default: ''
required: false
env:
FAIL_FAST: ${{ github.event_name == 'pull_request' }}
# Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
TEST_JAVA_VERSION: 17 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
jobs:
build-info:
runs-on: ubuntu-20.04
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }}
needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }}
needs-native-check: ${{ steps.categorize-basic-checks.outputs.needs-native-check }}
basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }}
needs-build: ${{ steps.selective-checks.outputs.needs-build }}
needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }}
needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }}
needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }}
needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }}
sha: ${{ steps.get-sha.outputs.sha }}
steps:
- name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)"
uses: actions/checkout@v4
with:
persist-credentials: false
if: github.event_name == 'push'
- name: "Checkout ${{ github.sha }} with its parent (pull request)"
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
if: github.event_name == 'pull_request'
- name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)"
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
persist-credentials: false
if: github.event_name == 'workflow_dispatch'
- name: Get SHA of ${{ inputs.ref || github.ref }}
id: get-sha
run: |
if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
sha="$(git rev-parse --verify HEAD)"
else
sha="${GITHUB_SHA}"
fi
echo "sha=$sha" >> $GITHUB_OUTPUT
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}"
PR_DRAFT: "${{ github.event.pull_request.draft }}"
run: |
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
# Run selective checks
dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}"
else
# Run all checks
dev-support/ci/selective_ci_checks.sh
fi
- name: Acceptance suites
id: acceptance-suites
run: dev-support/ci/acceptance_suites.sh
- name: Categorize Basic Checks
id: categorize-basic-checks
env:
ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks }}"
run: dev-support/ci/categorize_basic_checks.sh
build:
needs:
- build-info
runs-on: ubuntu-20.04
timeout-minutes: 60
if: needs.build-info.outputs.needs-build == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Cache for npm dependencies
uses: actions/cache@v4
with:
path: |
~/.pnpm-store
**/node_modules
key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Run a full build
run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Store binaries for tests
uses: actions/upload-artifact@v4
with:
name: ozone-bin
path: |
hadoop-ozone/dist/target/ozone-*.tar.gz
!hadoop-ozone/dist/target/ozone-*-src.tar.gz
retention-days: 1
- name: Store source tarball for compilation
uses: actions/upload-artifact@v4
if: needs.build-info.outputs.needs-compile == 'true'
with:
name: ozone-src
path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
retention-days: 1
- name: Store Maven repo for tests
uses: actions/upload-artifact@v4
with:
name: ozone-repo
path: |
~/.m2/repository/org/apache/ozone
retention-days: 1
compile:
needs:
- build-info
- build
- basic
- dependency
- license
timeout-minutes: 45
if: needs.build-info.outputs.needs-compile == 'true'
strategy:
matrix:
java: [ 11, 17, 21 ]
include:
- os: ubuntu-20.04
- java: 8
os: macos-13
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Download Ozone source tarball
uses: actions/download-artifact@v4
with:
name: ozone-src
- name: Untar sources
run: |
tar --strip-components 1 -xzvf ozone*-src.tar.gz
- name: Workaround for HADOOP-19011
run: |
git init
git config user.name 'Github Actions'
git config user.email 'noreply@github.com'
git commit --allow-empty -a -m 'workaround for HADOOP-19011'
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Compile Ozone using Java ${{ matrix.java }}
run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnpx -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
env:
OZONE_WITH_COVERAGE: false
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
basic:
needs:
- build-info
runs-on: ubuntu-20.04
timeout-minutes: 30
if: needs.build-info.outputs.needs-basic-check == 'true'
strategy:
matrix:
check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
if: matrix.check != 'bats'
- name: Checkout project with history
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ needs.build-info.outputs.sha }}
if: matrix.check == 'bats'
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
if: ${{ !contains('author,bats,docs', matrix.check) }}
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java 8
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 8
- name: Execute tests
run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }}
continue-on-error: true
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: ${{ matrix.check }}
path: target/${{ matrix.check }}
continue-on-error: true
native:
needs:
- build-info
- basic
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-native-check == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Execute tests
run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }}
continue-on-error: true
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: ${{ github.job }}
path: target/${{ github.job }}
continue-on-error: true
dependency:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 5
if: needs.build-info.outputs.needs-dependency-check == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v4
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir dist
tar -C dist --strip-components 1 -xzf ozone*.tar.gz
- name: Execute tests
run: |
export OZONE_DIST_DIR=`pwd`/dist
./hadoop-ozone/dev-support/checks/dependency.sh
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: dependency
path: target/dependency
continue-on-error: true
license:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 15
if: needs.build-info.outputs.needs-dependency-check == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ozone repo
id: download-ozone-repo
uses: actions/download-artifact@v4
with:
name: ozone-repo
path: |
~/.m2/repository/org/apache/ozone
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Execute tests
run: |
hadoop-ozone/dev-support/checks/${{ github.job }}.sh
continue-on-error: true
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: ${{ github.job }}
path: target/${{ github.job }}
continue-on-error: true
repro:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 30
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ozone repo
id: download-ozone-repo
uses: actions/download-artifact@v4
with:
name: ozone-repo
path: |
~/.m2/repository/org/apache/ozone
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Execute tests
run: |
hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
continue-on-error: true
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Install diffoscope
run: |
sudo apt update -q
sudo apt install -y diffoscope
if: ${{ failure() }}
- name: Check artifact differences
run: |
hadoop-ozone/dev-support/checks/_diffoscope.sh
if: ${{ failure() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: ${{ github.job }}
path: target/${{ github.job }}
continue-on-error: true
acceptance:
needs:
- build-info
- build
- basic
- dependency
- license
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-compose-tests == 'true'
strategy:
matrix:
suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }}
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v4
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
rm ozone*.tar.gz
- name: Execute tests
run: |
pushd hadoop-ozone/dist/target/ozone-*
sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
popd
./hadoop-ozone/dev-support/checks/acceptance.sh
env:
KEEP_IMAGE: false
OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
OZONE_VOLUME_OWNER: 1000
continue-on-error: true
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: acceptance-${{ matrix.suite }}
path: target/acceptance
continue-on-error: true
kubernetes:
needs:
- build-info
- build
- basic
- dependency
- license
runs-on: ubuntu-20.04
timeout-minutes: 60
if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v4
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- name: Execute tests
run: |
pushd hadoop-ozone/dist/target/ozone-*
sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
popd
./hadoop-ozone/dev-support/checks/kubernetes.sh
continue-on-error: true
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: kubernetes
path: target/kubernetes
continue-on-error: true
integration:
needs:
- build-info
- basic
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-integration-tests == 'true'
strategy:
matrix:
profile:
- client
- container
- filesystem
- hdds
- om
- ozone
- recon
- shell
- snapshot
- flaky
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.sha }}
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Execute tests
continue-on-error: true
run: |
args="${{ inputs.ratis_args }}"
if [[ "${{ matrix.profile }}" == "flaky" ]]; then
args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
fi
hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Summary of failures
run: |
if [[ -s "target/${{ github.job }}/summary.md" ]]; then
cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
fi
hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: it-${{ matrix.profile }}
path: target/integration
continue-on-error: true
coverage:
runs-on: ubuntu-20.04
timeout-minutes: 30
if: github.event_name == 'push'
needs:
- build-info
- acceptance
- integration
- native
steps:
- name: Checkout project
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ needs.build-info.outputs.sha }}
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: target/artifacts
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Calculate combined coverage
run: ./hadoop-ozone/dev-support/checks/coverage.sh
- name: Upload coverage to Sonar
run: ./hadoop-ozone/dev-support/checks/sonar.sh
if: github.repository == 'apache/ozone'
env:
SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Archive build results
uses: actions/upload-artifact@v4
with:
name: coverage
path: target/coverage
continue-on-error: true