| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| name: full-ci |
| on: |
| workflow_call: |
| inputs: |
| ratis_args: |
| type: string |
| description: Version overrides from custom Ratis build |
| default: '' |
| required: false |
| ref: |
| type: string |
| description: Ozone ref (branch, tag or commit SHA) |
| default: '' |
| required: false |
| secrets: |
| DEVELOCITY_ACCESS_KEY: |
| description: 'Token for submitting build scan to Develocity' |
| required: false |
| OZONE_WEBSITE_BUILD: |
| description: 'Token for publishing to apache/ozone-site' |
| required: false |
| SONARCLOUD_TOKEN: |
| description: 'Token for submitting coverage data to SonarCloud' |
| required: false |
| |
| env: |
| # BUILD_ARGS and TEST_JAVA_VERSION are duplicated in populate-cache.yml, please keep in sync |
| BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true -Drocks_tools_native" |
| TEST_JAVA_VERSION: 25 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image |
| # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml and populate-cache.yml, please keep in sync |
| MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml |
| MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 |
| OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }} |
| |
| permissions: { } |
| |
| jobs: |
| build-info: |
| runs-on: ubuntu-slim |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| acceptance-suites: ${{ steps.acceptance-suites.outputs.suites }} |
| integration-suites: ${{ steps.integration-suites.outputs.suites }} |
| needs-basic-check: ${{ steps.categorize-basic-checks.outputs.needs-basic-check }} |
| basic-checks: ${{ steps.categorize-basic-checks.outputs.basic-checks }} |
| needs-build: ${{ steps.selective-checks.outputs.needs-build }} |
| needs-compile: ${{ steps.selective-checks.outputs.needs-compile }} |
| needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }} |
| needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} |
| needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} |
| sha: ${{ steps.get-sha.outputs.sha }} |
| # `env` context cannot be used when calling reusable workflow, so we need to convert these to `outputs` |
| build-args: ${{ env.BUILD_ARGS }} |
| java-version: ${{ env.TEST_JAVA_VERSION }} |
| with-coverage: ${{ env.OZONE_WITH_COVERAGE }} |
| steps: |
| - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)" |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| with: |
| persist-credentials: false |
| if: github.event_name == 'push' |
| - name: "Checkout ${{ github.sha }} with its parent (pull request)" |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| if: github.event_name == 'pull_request' |
| - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)" |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| with: |
| ref: ${{ inputs.ref }} |
| persist-credentials: false |
| if: github.event_name == 'workflow_dispatch' |
| - name: Get SHA of ${{ inputs.ref || github.ref }} |
| id: get-sha |
| run: | |
| if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then |
| sha="$(git rev-parse --verify HEAD)" |
| else |
| sha="${GITHUB_SHA}" |
| fi |
| echo "sha=$sha" >> $GITHUB_OUTPUT |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}" |
| PR_DRAFT: "${{ github.event.pull_request.draft }}" |
| run: | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then |
| # Run selective checks |
| dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}" |
| else |
| # Run all checks |
| dev-support/ci/selective_ci_checks.sh |
| fi |
| - name: Acceptance suites |
| id: acceptance-suites |
| run: dev-support/ci/acceptance_suites.sh |
| - name: Integration suites |
| id: integration-suites |
| run: dev-support/ci/integration_suites.sh |
| - name: Categorize Basic Checks |
| id: categorize-basic-checks |
| env: |
| ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks }}" |
| run: dev-support/ci/categorize_basic_checks.sh |
| |
| build: |
| needs: |
| - build-info |
| if: needs.build-info.outputs.needs-build == 'true' || needs.build-info.outputs.needs-integration-tests == 'true' |
| uses: ./.github/workflows/check.yml |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-npm-cache: true |
| ratis-args: ${{ inputs.ratis_args }} |
| script: build |
| script-args: ${{ needs.build-info.outputs.build-args }} |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 60 |
| with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| |
| generate-config-doc: |
| needs: |
| - build-info |
| - build |
| if: | |
| needs.build-info.outputs.needs-build == 'true' && |
| (github.repository != 'apache/ozone' || github.event_name == 'pull_request' || |
| (github.event_name == 'push' && github.ref_name == 'master')) |
| uses: ./.github/workflows/generate-config-doc.yml |
| with: |
| sha: ${{ needs.build-info.outputs.sha }} |
| |
| update-ozone-site-config-doc: |
| needs: |
| - generate-config-doc |
| if: | |
| github.event_name == 'push' && |
| github.repository == 'apache/ozone' && |
| github.ref_name == 'master' |
| uses: ./.github/workflows/update-ozone-site-config-doc.yml |
| permissions: |
| contents: read |
| secrets: |
| OZONE_WEBSITE_BUILD: ${{ secrets.OZONE_WEBSITE_BUILD }} |
| |
| compile: |
| needs: |
| - build-info |
| - build |
| - basic |
| - dependency |
| - license |
| if: needs.build-info.outputs.needs-compile == 'true' |
| strategy: |
| matrix: |
| java: [ 17, 25 ] |
| include: |
| - os: ubuntu-24.04 |
| - java: 21 |
| os: macos-15 |
| fail-fast: false |
| uses: ./.github/workflows/check.yml |
| with: |
| java-version: ${{ matrix.java }} |
| needs-ozone-source-tarball: true |
| ratis-args: ${{ inputs.ratis_args }} |
| runner: ${{ matrix.os }} |
| script: compile |
| script-args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings -Dmaven.compiler.release=${{ matrix.java }}" |
| sha: ${{ needs.build-info.outputs.sha }} |
| split: ${{ matrix.java }} |
| timeout-minutes: 45 |
| with-coverage: false |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| |
| basic: |
| needs: |
| - build-info |
| if: needs.build-info.outputs.needs-basic-check == 'true' |
| uses: ./.github/workflows/check.yml |
| with: |
| checkout-fetch-depth: ${{ matrix.check != 'bats' && 1 || 0 }} |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-maven-cache: ${{ !contains('author,bats', matrix.check) }} |
| ratis-args: ${{ inputs.ratis_args }} |
| script: ${{ matrix.check }} |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 30 |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| strategy: |
| matrix: |
| check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} |
| fail-fast: false |
| |
| dependency: |
| needs: |
| - build-info |
| - build |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-ozone-binary-tarball: true |
| script: dependency |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 5 |
| |
| license: |
| needs: |
| - build-info |
| - build |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-ozone-repo: true |
| script: license |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 15 |
| |
| javadoc: |
| needs: |
| - build-info |
| - build |
| - basic |
| if: needs.build-info.outputs.needs-compile == 'true' |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-ozone-repo: true |
| script: javadoc |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 30 |
| |
| repro: |
| needs: |
| - build-info |
| - build |
| - basic |
| - dependency |
| - license |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| needs-ozone-repo: true |
| ratis-args: ${{ inputs.ratis_args }} |
| script: repro |
| script-args: ${{ needs.build-info.outputs.build-args }} |
| post-failure: hadoop-ozone/dev-support/checks/_diffoscope.sh |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 30 |
| with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} |
| |
| acceptance: |
| needs: |
| - build-info |
| - build |
| - basic |
| - dependency |
| - license |
| if: needs.build-info.outputs.needs-compose-tests == 'true' |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: 11 # Hadoop may not work with newer Java |
| needs-ozone-binary-tarball: true |
| ratis-args: ${{ inputs.ratis_args }} |
| script: acceptance |
| script-args: ${{ matrix.suite }} |
| sha: ${{ needs.build-info.outputs.sha }} |
| split: ${{ matrix.suite }} |
| timeout-minutes: 150 |
| with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} |
| strategy: |
| matrix: |
| suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }} |
| fail-fast: false |
| |
| kubernetes: |
| needs: |
| - build-info |
| - build |
| - basic |
| - dependency |
| - license |
| if: needs.build-info.outputs.needs-kubernetes-tests == 'true' |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| needs-ozone-binary-tarball: true |
| ratis-args: ${{ inputs.ratis_args }} |
| script: kubernetes |
| sha: ${{ needs.build-info.outputs.sha }} |
| timeout-minutes: 60 |
| with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} |
| |
| integration: |
| needs: |
| - build-info |
| - build |
| - basic |
| - dependency |
| - license |
| if: needs.build-info.outputs.needs-integration-tests == 'true' |
| uses: ./.github/workflows/check.yml |
| secrets: |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| with: |
| java-version: ${{ needs.build-info.outputs.java-version }} |
| pre-script: sudo hostname localhost |
| ratis-args: ${{ inputs.ratis_args }} |
| script: integration |
| script-args: -Ptest-${{ matrix.profile }} -Phadoop-native-lib -Drocks_tools_native |
| sha: ${{ needs.build-info.outputs.sha }} |
| split: ${{ matrix.profile }} |
| timeout-minutes: 90 |
| with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }} |
| strategy: |
| matrix: |
| profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }} |
| fail-fast: false |
| |
| coverage: |
| runs-on: ubuntu-24.04 |
| timeout-minutes: 30 |
| if: github.event_name == 'push' |
| needs: |
| - build-info |
| - acceptance |
| - integration |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| with: |
| fetch-depth: 0 |
| persist-credentials: false |
| ref: ${{ needs.build-info.outputs.sha }} |
| - name: Cache for maven dependencies |
| uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 |
| with: |
| path: | |
| ~/.m2/repository/*/*/* |
| !~/.m2/repository/org/apache/ozone |
| key: maven-repo-${{ hashFiles('**/pom.xml') }} |
| restore-keys: | |
| maven-repo- |
| - name: Download artifacts |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| with: |
| path: target/artifacts |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target |
| - name: Setup java ${{ env.TEST_JAVA_VERSION }} |
| uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0 |
| with: |
| distribution: 'temurin' |
| java-version: ${{ env.TEST_JAVA_VERSION }} |
| - name: Calculate combined coverage |
| run: ./hadoop-ozone/dev-support/checks/coverage.sh |
| - name: Upload coverage to Sonar |
| run: ./hadoop-ozone/dev-support/checks/sonar.sh |
| if: github.repository == 'apache/ozone' |
| env: |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} |
| - name: Archive build results |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| with: |
| name: coverage |
| path: target/coverage |
| continue-on-error: true |
