| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| name: build-branch |
| on: |
| pull_request: |
| types: [opened, ready_for_review, synchronize] |
| push: |
| schedule: |
| - cron: 30 0,12 * * * |
| env: |
| FAIL_FAST: ${{ github.event_name == 'pull_request' }} |
| MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 |
| concurrency: |
| group: ci-${{ github.event.pull_request.number || github.sha }} |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} |
| jobs: |
| build-info: |
| runs-on: ubuntu-20.04 |
| env: |
| GITHUB_CONTEXT: ${{ toJson(github) }} |
| outputs: |
| basic-checks: ${{ steps.selective-checks.outputs.basic-checks }} |
| needs-basic-checks: ${{ steps.selective-checks.outputs.needs-basic-checks }} |
| needs-build: ${{ steps.selective-checks.outputs.needs-build }} |
| needs-compile: ${{ steps.selective-checks.outputs.needs-compile }} |
| needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }} |
| needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }} |
| needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }} |
| needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }} |
| steps: |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" |
| uses: actions/checkout@v2 |
| with: |
| persist-credentials: false |
| - name: Fetch incoming commit ${{ github.sha }} with its parent |
| uses: actions/checkout@v2 |
| with: |
| ref: ${{ github.sha }} |
| fetch-depth: 2 |
| persist-credentials: false |
| if: github.event_name == 'pull_request' |
| - name: Selective checks |
| id: selective-checks |
| env: |
| PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}" |
| PR_DRAFT: "${{ github.event.pull_request.draft }}" |
| run: | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then |
| # Run selective checks |
| dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}" |
| else |
| # Run all checks |
| dev-support/ci/selective_ci_checks.sh |
| fi |
| build: |
| needs: |
| - build-info |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 30 |
| if: needs.build-info.outputs.needs-build == 'true' |
| strategy: |
| matrix: |
| java: [ 8 ] |
| fail-fast: false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for npm dependencies |
| uses: actions/cache@v2 |
| with: |
| path: | |
| ~/.pnpm-store |
| **/node_modules |
| key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} |
| restore-keys: | |
| ${{ runner.os }}-pnpm- |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Setup java |
| uses: actions/setup-java@v1 |
| with: |
| java-version: ${{ matrix.java }} |
| - name: Run a full build |
| run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist -Psrc |
| - name: Store binaries for tests |
| uses: actions/upload-artifact@v2 |
| with: |
| name: ozone-bin |
| path: | |
| hadoop-ozone/dist/target/ozone-*.tar.gz |
| !hadoop-ozone/dist/target/ozone-*-src.tar.gz |
| retention-days: 1 |
| - name: Store source tarball for compilation |
| uses: actions/upload-artifact@v2 |
| with: |
| name: ozone-src |
| path: hadoop-ozone/dist/target/ozone-*-src.tar.gz |
| retention-days: 1 |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| compile: |
| needs: |
| - build-info |
| - build |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 30 |
| if: needs.build-info.outputs.needs-compile == 'true' |
| strategy: |
| matrix: |
| java: [ 11 ] |
| fail-fast: false |
| steps: |
| - name: Download Ozone source tarball |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-src |
| - name: Untar sources |
| run: | |
| tar --strip-components 1 -xzvf ozone*-src.tar.gz |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Setup java |
| uses: actions/setup-java@v1 |
| with: |
| java-version: ${{ matrix.java }} |
| - name: Compile Ozone using Java ${{ matrix.java }} |
| run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| basic: |
| needs: |
| - build-info |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 90 |
| if: needs.build-info.outputs.needs-basic-checks == 'true' |
| strategy: |
| matrix: |
| check: ${{ fromJson(needs.build-info.outputs.basic-checks) }} |
| fail-fast: false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| if: matrix.check != 'bats' |
| - name: Checkout project with history |
| uses: actions/checkout@v2 |
| with: |
| fetch-depth: 0 |
| if: matrix.check == 'bats' |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| if: ${{ !contains('author,bats,docs', matrix.check) }} |
| - name: Setup java |
| uses: actions/setup-java@v1 |
| with: |
| java-version: 8 |
| - name: Execute tests |
| run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh |
| - name: Summary of failures |
| run: cat target/${{ matrix.check }}/summary.txt |
| if: ${{ !cancelled() }} |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: ${{ !cancelled() }} |
| with: |
| name: ${{ matrix.check }} |
| path: target/${{ matrix.check }} |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| dependency: |
| needs: |
| - build-info |
| - build |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 5 |
| if: needs.build-info.outputs.needs-dependency-check == 'true' |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir dist |
| tar -C dist --strip-components 1 -xzf ozone*.tar.gz |
| - name: Execute tests |
| run: | |
| export OZONE_DIST_DIR=`pwd`/dist |
| ./hadoop-ozone/dev-support/checks/dependency.sh |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: dependency |
| path: target/dependency |
| continue-on-error: true |
| acceptance: |
| needs: |
| - build-info |
| - build |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 150 |
| if: needs.build-info.outputs.needs-compose-tests == 'true' |
| strategy: |
| matrix: |
| suite: |
| - secure |
| - unsecure |
| - compat |
| - HA |
| - MR |
| - misc |
| fail-fast: false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target |
| sudo chmod -R a+rwX hadoop-ozone/dist/target |
| - name: Execute tests |
| run: | |
| pushd hadoop-ozone/dist/target/ozone-* |
| sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| popd |
| ./hadoop-ozone/dev-support/checks/acceptance.sh |
| env: |
| KEEP_IMAGE: false |
| OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} |
| OZONE_WITH_COVERAGE: true |
| OZONE_VOLUME_OWNER: 1000 |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: acceptance-${{ matrix.suite }} |
| path: target/acceptance |
| continue-on-error: true |
| kubernetes: |
| needs: |
| - build-info |
| - build |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 60 |
| if: needs.build-info.outputs.needs-kubernetes-tests == 'true' |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target |
| - name: Execute tests |
| run: | |
| pushd hadoop-ozone/dist/target/ozone-* |
| sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| popd |
| ./hadoop-ozone/dev-support/checks/kubernetes.sh |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: kubernetes |
| path: target/kubernetes |
| continue-on-error: true |
| integration: |
| needs: |
| - build-info |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 150 |
| if: needs.build-info.outputs.needs-integration-tests == 'true' |
| strategy: |
| matrix: |
| profile: |
| - client |
| - filesystem |
| - hdds |
| - om |
| - ozone |
| - scm |
| - flaky |
| fail-fast: false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Execute tests |
| run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} |
| if: matrix.profile != 'flaky' |
| - name: Execute flaky tests |
| run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600 |
| if: matrix.profile == 'flaky' |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: it-${{ matrix.profile }} |
| path: target/integration |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| coverage: |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 30 |
| if: github.repository == 'apache/ozone' && github.event_name != 'pull_request' |
| needs: |
| - acceptance |
| - basic |
| - integration |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Download artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| path: target/artifacts |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target |
| - name: Calculate combined coverage |
| run: ./hadoop-ozone/dev-support/checks/coverage.sh |
| - name: Setup java 11 |
| uses: actions/setup-java@v1 |
| with: |
| java-version: 11 |
| - name: Upload coverage to Sonar |
| run: ./hadoop-ozone/dev-support/checks/sonar.sh |
| env: |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage |
| path: target/coverage |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |