| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| name: build-branch |
| on: |
| pull_request: |
| push: |
| schedule: |
| - cron: 30 0,12 * * * |
| env: |
| FAIL_FAST: ${{ github.event_name == 'pull_request' }} |
| jobs: |
| compile: |
| runs-on: ubuntu-18.04 |
| strategy: |
| matrix: |
| java: [ 8, 11 ] |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Cache for npm dependencies |
| uses: actions/cache@v2 |
| with: |
| path: | |
| ~/.pnpm-store |
| **/node_modules |
| key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }} |
| restore-keys: | |
| ${{ runner.os }}-pnpm- |
| - name: Cache for maven dependencies |
| uses: actions/cache@v2 |
| with: |
| path: ~/.m2/repository |
| key: maven-repo-${{ hashFiles('**/pom.xml') }} |
| restore-keys: maven-repo- |
| - name: Setup java |
| uses: actions/setup-java@v1 |
| with: |
| java-version: ${{ matrix.java }} |
| - name: Run a full build |
| run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist |
| - name: Store binaries for tests |
| uses: actions/upload-artifact@v2 |
| if: matrix.java == '8' |
| with: |
| name: ozone-bin |
| path: hadoop-ozone/dist/target/hadoop-ozone*.tar.gz |
| retention-days: 1 |
| - name: Delete temporary build artifacts before caching |
| run: | |
| #Never cache local artifacts |
| rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds* |
| rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone* |
| if: always() |
| bats: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Install bats |
| run: | |
| cd /tmp |
| curl -LSs https://github.com/bats-core/bats-core/archive/v1.2.1.tar.gz | tar xzf - |
| cd bats-core-1.2.1 |
| sudo ./install.sh /usr/local |
| - name: Execute tests |
| run: ./hadoop-ozone/dev-support/checks/${{ github.job }}.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: ${{ github.job }} |
| path: target/${{ github.job }} |
| continue-on-error: true |
| rat: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Execute tests |
| uses: ./.github/buildenv |
| with: |
| args: ./hadoop-ozone/dev-support/checks/rat.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: rat |
| path: target/rat |
| continue-on-error: true |
| author: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Execute tests |
| run: hadoop-ozone/dev-support/checks/author.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: author |
| path: target/author |
| continue-on-error: true |
| unit: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Execute tests |
| uses: ./.github/buildenv |
| with: |
| args: ./hadoop-ozone/dev-support/checks/unit.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: unit |
| path: target/unit |
| continue-on-error: true |
| checkstyle: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Execute tests |
| uses: ./.github/buildenv |
| with: |
| args: ./hadoop-ozone/dev-support/checks/checkstyle.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: checkstyle |
| path: target/checkstyle |
| continue-on-error: true |
| findbugs: |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Execute tests |
| uses: ./.github/buildenv |
| with: |
| args: ./hadoop-ozone/dev-support/checks/findbugs.sh |
| - name: Summary of failures |
| run: cat target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: findbugs |
| path: target/findbugs |
| continue-on-error: true |
| acceptance: |
| needs: compile |
| runs-on: ubuntu-18.04 |
| strategy: |
| matrix: |
| suite: |
| - secure |
| - unsecure |
| - misc |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: ozone |
| - name: Move Ozone to /mnt |
| run: | |
| sudo chmod 777 /mnt |
| mv ozone /mnt/ |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p /mnt/ozone/hadoop-ozone/dist/target |
| tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target |
| sudo chmod -R a+rwX /mnt/ozone/hadoop-ozone/dist/target |
| - name: Install robotframework |
| run: sudo pip install robotframework |
| - name: Execute tests |
| run: | |
| cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| cd /mnt/ozone && hadoop-ozone/dev-support/checks/acceptance.sh |
| env: |
| KEEP_IMAGE: false |
| OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} |
| OZONE_WITH_COVERAGE: true |
| OZONE_VOLUME_OWNER: 1000 |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: acceptance-${{ matrix.suite }} |
| path: /mnt/ozone/target/acceptance |
| continue-on-error: true |
| integration: |
| runs-on: ubuntu-18.04 |
| strategy: |
| matrix: |
| profile: |
| - client |
| - filesystem-hdds |
| - ozone |
| fail-fast: ${{ github.event_name == 'pull_request' }} |
| steps: |
| - name: Setup link to SSD |
| run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: mnt/ozone |
| - name: Execute tests |
| uses: ./mnt/ozone/.github/buildenv |
| with: |
| args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} |
| - name: Summary of failures |
| run: cat mnt/ozone/target/${{ github.job }}/summary.txt |
| if: always() |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: it-${{ matrix.profile }} |
| path: mnt/ozone/target/integration |
| continue-on-error: true |
| coverage: |
| runs-on: ubuntu-18.04 |
| if: github.repository == 'apache/ozone' && github.event_name != 'pull_request' |
| needs: |
| - acceptance |
| - integration |
| - unit |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| - name: Download artifacts |
| uses: actions/download-artifact@v2 |
| with: |
| path: target/artifacts |
| - name: Untar binaries |
| run: | |
| mkdir -p hadoop-ozone/dist/target |
| tar xzvf target/artifacts/ozone-bin/hadoop-ozone*.tar.gz -C hadoop-ozone/dist/target |
| - name: Calculate combined coverage |
| run: ./hadoop-ozone/dev-support/checks/coverage.sh |
| - name: Setup java 11 |
| uses: actions/setup-java@v1 |
| with: |
| java-version: 11 |
| - name: Upload coverage to Sonar |
| run: ./hadoop-ozone/dev-support/checks/sonar.sh |
| env: |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| with: |
| name: coverage |
| path: target/coverage |
| continue-on-error: true |
| kubernetes: |
| needs: compile |
| runs-on: ubuntu-18.04 |
| steps: |
| - name: Checkout project |
| uses: actions/checkout@v2 |
| with: |
| path: ozone |
| - name: Move Ozone to /mnt |
| run: | |
| sudo chmod 777 /mnt |
| mv ozone /mnt/ |
| - name: Download compiled Ozone binaries |
| uses: actions/download-artifact@v2 |
| with: |
| name: ozone-bin |
| - name: Untar binaries |
| run: | |
| mkdir -p /mnt/ozone/hadoop-ozone/dist/target |
| tar xzvf hadoop-ozone*.tar.gz -C /mnt/ozone/hadoop-ozone/dist/target |
| - name: Install robotframework |
| run: sudo pip install robotframework |
| - name: Install k3s |
| run: curl -sfL https://get.k3s.io | sh - |
| - name: Copy Kubernetes config file |
| run: | |
| sudo mkdir ~/.kube |
| sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config |
| sudo chown $(id -u) ~/.kube/config |
| - name: Install flekszible |
| run: | |
| cd /tmp |
| wget https://github.com/elek/flekszible/releases/download/v1.8.1/flekszible_1.8.1_Linux_x86_64.tar.gz -O - | tar -zx |
| chmod +x flekszible |
| sudo mv flekszible /usr/bin/flekszible |
| - name: Execute tests |
| run: | |
| cd /mnt/ozone/hadoop-ozone/dist/target/ozone-* && sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws |
| cd /mnt/ozone && hadoop-ozone/dev-support/checks/kubernetes.sh |
| - name: Archive build results |
| uses: actions/upload-artifact@v2 |
| if: always() |
| with: |
| name: kubernetes |
| path: /mnt/ozone/target/kubernetes |
| continue-on-error: true |