| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| name: build-branch |
| on: |
| pull_request: |
| types: [opened, reopened, ready_for_review, synchronize] |
| push: |
| schedule: |
| - cron: 30 0,12 * * * |
| env: |
| FAIL_FAST: ${{ github.event_name == 'pull_request' }} |
| MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 |
| concurrency: |
| group: ci-${{ github.event.pull_request.number || github.sha }} |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} |
| jobs: |
| compile: |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 30 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| strategy: |
| matrix: |
| java: [ 8, 11 ] |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for npm dependencies |
| uses: actions/cache@v2 |
| with: |
| path: | |
| ~/.pnpm-store |
| **/node_modules |
| key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} |
| restore-keys: | |
| ${{ runner.os }}-pnpm- |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Setup java |
| uses: actions/setup-java@v1 |
| with: |
| java-version: ${{ matrix.java }} |
| - name: Run a full build |
| run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist |
| - name: Store binaries for tests |
| uses: actions/upload-artifact@v2 |
| if: matrix.java == '8' |
| with: |
| name: ozone-bin |
| path: hadoop-ozone/dist/target/ozone*.tar.gz |
| retention-days: 1 |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| basic: |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 40 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| strategy: |
| matrix: |
| check: |
| - author |
| - bats |
| - checkstyle |
| - docs |
| - findbugs |
| - rat |
| - unit |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| if: ${{ !contains('author,bats,docs', matrix.check) }} |
| - name: Execute tests |
| run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh |
| - name: Summary of failures |
| run: cat target/${{ matrix.check }}/summary.txt |
| if: ${{ !cancelled() }} |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: ${{ !cancelled() }} |
| with: |
| name: ${{ matrix.check }} |
| path: target/${{ matrix.check }} |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| dependency: |
| needs: compile |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 5 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir dist |
| tar -C dist --strip-components 1 -xzf ozone*.tar.gz |
| - name: Execute tests |
| run: | |
| export OZONE_DIST_DIR=`pwd`/dist |
| ./hadoop-ozone/dev-support/checks/dependency.sh |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: dependency |
| path: target/dependency |
| continue-on-error: true |
| acceptance: |
| needs: compile |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 120 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| strategy: |
| matrix: |
| suite: |
| - secure |
| - unsecure |
| - misc |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: ozone |
| - name: Move Ozone to /mnt |
| run: | |
| sudo chmod 777 /mnt |
| mv ozone /mnt/ |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p /mnt/ozone/hadoop-ozone/dist/target |
| tar xzvf ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target |
| sudo chmod -R a+rwX /mnt/ozone/hadoop-ozone/dist/target |
| - name: Execute tests |
| run: | |
| cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| cd /mnt/ozone && hadoop-ozone/dev-support/checks/acceptance.sh |
| env: |
| KEEP_IMAGE: false |
| OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} |
| OZONE_WITH_COVERAGE: true |
| OZONE_VOLUME_OWNER: 1000 |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: acceptance-${{ matrix.suite }} |
| path: /mnt/ozone/target/acceptance |
| continue-on-error: true |
| kubernetes: |
| needs: compile |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 60 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: ozone |
| - name: Move Ozone to /mnt |
| run: | |
| sudo chmod 777 /mnt |
| mv ozone /mnt/ |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p /mnt/ozone/hadoop-ozone/dist/target |
| tar xzvf ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target |
| - name: Execute tests |
| run: | |
| cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| cd /mnt/ozone && hadoop-ozone/dev-support/checks/kubernetes.sh |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: kubernetes |
| path: /mnt/ozone/target/kubernetes |
| continue-on-error: true |
| integration: |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 150 |
| if: github.event_name != 'pull_request' || github.event.pull_request.draft == false |
| strategy: |
| matrix: |
| profile: |
| - client |
| - filesystem-hdds |
| - ozone |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Setup link to SSD |
| run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: mnt/ozone |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Execute tests |
| run: mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} |
| - name: Summary of failures |
| run: cat mnt/ozone/target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: it-${{ matrix.profile }} |
| path: mnt/ozone/target/integration |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |
| coverage: |
| runs-on: ubuntu-18.04 |
| timeout-minutes: 30 |
| if: github.repository == 'apache/ozone' && github.event_name != 'pull_request' |
| needs: |
| - acceptance |
| - basic |
| - integration |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }} |
| restore-keys: | |
| maven-repo-${{ hashFiles('**/pom.xml') }}-8 |
| maven-repo-${{ hashFiles('**/pom.xml') }} |
| maven-repo- |
| - name: Download artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| path: target/artifacts |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target |
| - name: Calculate combined coverage |
| run: ./hadoop-ozone/dev-support/checks/coverage.sh |
| - name: Setup java 11 |
| uses: actions/setup-java@v1 |
| with: |
| java-version: 11 |
| - name: Upload coverage to Sonar |
| run: ./hadoop-ozone/dev-support/checks/sonar.sh |
| env: |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage |
| path: target/coverage |
| continue-on-error: true |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/ozone/hdds* |
| rm -rf ~/.m2/repository/org/apache/ozone/ozone* |
| if: always() |