| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| name: Build and Test |
| |
| on: |
| workflow_call: |
| inputs: |
| java: |
| required: false |
| type: string |
| default: 17 |
| branch: |
| required: false |
| type: string |
| description: Branch to run the build against |
| default: trunk |
| os: |
| required: false |
| type: string |
| description: Operating system to run the build on |
| default: ubuntu_24 |
| jobs: |
| required: false |
| type: string |
| description: >- |
| Jobs to run, and should be in JSON Array format. |
| Candidates: "build-only", "build-and-test". |
| default: '[ "build-and-test" ]' |
| |
| # Default to minimal permissions for workflow. |
| permissions: |
| packages: read |
| |
| concurrency: |
| group: >- |
| build-and-test |
| ${{ github.workflow }} |
| ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }} |
| ${{ inputs.java }} |
| ${{ inputs.branch }} |
| ${{ inputs.os }} |
| ${{ inputs.jobs }} |
| cancel-in-progress: true |
| |
| env: |
| MAVEN_ARGS: |
| --batch-mode |
| --no-transfer-progress |
| -Pyarn-ui |
| -Pnative |
| -Drequire.test.libhadoop |
| -Drequire.fuse |
| -Drequire.openssl |
| -Drequire.snappy |
| -Drequire.valgrind |
| -Dmaven.test.failure.ignore=false |
| -Dsurefire.rerunFailingTestsCount=2 |
| |
| jobs: |
| # TODO: use common composite actions in .github/actions/ |
| precondition: |
| name: Preparation |
| runs-on: ubuntu-24.04 |
| outputs: |
| build_image_url: ${{ steps.variables.outputs.build_image_url }} |
| steps: |
| - name: Set up Outputs |
| id: variables |
| # Security: passing inputs.{os, branch} through workflow (above) inputs removes |
| # ability to do shell injection below. |
| # See: https://securitylab.github.com/resources/github-actions-untrusted-input/ |
| run: | |
| # Convert to lowercase to meet Docker repo name requirement |
| REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') |
| echo "build_image_url=ghcr.io/${REPO_OWNER}/gha-build-${{ inputs.os }}:${{ inputs.branch }}-${{ github.run_id }}" >> $GITHUB_OUTPUT |
| build-image: |
| name: Build Image ${{ inputs.os }}-${{ inputs.branch }} |
| runs-on: ubuntu-24.04 |
| needs: [ precondition ] |
| # Security: this does not leak write access for our image repository to |
| # forked repos. |
| # |
| # We have `packages: write` permissions for our GITHUB_TOKEN below. However: |
| # |
| # - For `pull_request`, GitHub downgrades GITHUB_TOKEN permissions to |
| # read-only. |
| # - For `push` triggers on a fork, the GITHUB_TOKEN retains write |
| # permissions, but the `push` is happening in the context of the fork, not |
| # the upstream repo. |
| # - `pull_request_target` events (not used here) use the base repo's |
| # default branch. This prevents an attacker from adding code in a pull |
| # request which prints / leaks secrets, etc.. Also the GITHUB_TOKEN we |
| # use to grant access to writing to the image repo. is scoped to the |
| # repository it runs on. Even if a fork were to gain access to a |
| # GITHUB_TOKEN with write privileges, it doesn't grant access for other |
| # repositories to write. |
| permissions: |
| packages: write |
| outputs: |
| uid: ${{ steps.variables.outputs.uid }} |
| steps: |
| - uses: actions/checkout@v6 |
| with: |
| fetch-depth: 0 |
| - name: Login to GitHub Container Registry |
| uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 |
| with: |
| registry: ghcr.io |
| username: ${{ github.actor }} |
| password: ${{ secrets.GITHUB_TOKEN }} |
| - name: Set up Docker Buildx |
| uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 |
| - name: Build and push ${{ inputs.os }} base build image for ${{ inputs.branch }} |
| id: docker_build_base |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 |
| with: |
| context: ./dev-support/docker/ |
| file: ./dev-support/docker/Dockerfile_${{ inputs.os }} |
| push: true |
| tags: ${{ needs.precondition.outputs.build_image_url }}-base |
| cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{ inputs.os }}-image-cache:${{ inputs.branch }} |
| - name: User-specific Dockerfile |
| run: | |
| USER_ID=$(id -u "${USER}") |
| GROUP_ID=$(id -g "${USER}") |
| cat > /tmp/Dockerfile.gha <<UserSpecificDocker |
| FROM ${{ needs.precondition.outputs.build_image_url }}-base |
| RUN rm -f /var/log/faillog /var/log/lastlog |
| RUN groupadd --non-unique -g ${GROUP_ID} ${USER} |
| RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER} |
| RUN echo "${USER} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}" |
| UserSpecificDocker |
| - name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch }} |
| id: docker_build_gha |
| uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 |
| with: |
| context: ./dev-support/docker/ |
| file: /tmp/Dockerfile.gha |
| push: true |
| tags: ${{ needs.precondition.outputs.build_image_url }} |
| - name: Set up Outputs |
| id: variables |
| run: | |
| echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT |
| build-only: |
| name: Build Only (Java ${{ inputs.java }}) ${{ inputs.os }}-${{ inputs.branch }} |
| if: (!cancelled()) && contains(fromJSON(inputs.jobs), 'build-only') |
| runs-on: ubuntu-24.04 |
| needs: [ precondition, build-image ] |
| container: |
| image: ${{ needs.precondition.outputs.build_image_url }} |
| options: --user ${{ needs.build-image.outputs.uid }} |
| steps: |
| - uses: actions/checkout@v6 |
| with: |
| fetch-depth: 0 |
| - name: Setup JDK ${{ inputs.java }} |
| uses: actions/setup-java@v5 |
| with: |
| distribution: zulu |
| java-version: ${{ inputs.java }} |
| cache: 'maven' |
| check-latest: false |
| - name: Build |
| shell: bash |
| run: ./mvnw $MAVEN_ARGS clean install -DskipTests |
| build-and-test: |
| if: (!cancelled()) && contains(fromJSON(inputs.jobs), 'build-and-test') |
| name: Test ${{ matrix.comment }} (Java ${{ inputs.java }}) ${{ inputs.os }}-${{ inputs.branch }} |
| runs-on: ubuntu-24.04 |
| needs: [ precondition, build-image ] |
| container: |
| image: ${{ needs.precondition.outputs.build_image_url }} |
| options: --user ${{ needs.build-image.outputs.uid }} |
| strategy: |
| fail-fast: false |
| max-parallel: 8 |
| matrix: |
| include: |
| - comment: hdfs - slow |
| modules: |
| -pl :hadoop-hdfs |
| -Dgroups=slow |
| - comment: hdfs - other |
| modules: |
| -pl :hadoop-hdfs |
| -DexcludedGroups=slow |
| - comment: yarn-server-rm |
| modules: |
| -pl :hadoop-yarn-server-resourcemanager |
| - comment: mr |
| modules: |
| -pl :hadoop-mapreduce-client-core |
| -pl :hadoop-mapreduce-client-common |
| -pl :hadoop-mapreduce-client-shuffle |
| -pl :hadoop-mapreduce-client-jobclient |
| -pl :hadoop-mapreduce-client-app |
| -pl :hadoop-mapreduce-client-hs |
| -pl :hadoop-mapreduce-client-hs-plugins |
| -pl :hadoop-mapreduce-client-nativetask |
| -pl :hadoop-mapreduce-client-uploader |
| -pl :hadoop-mapreduce-examples |
| - comment: common |
| modules: |
| -pl :hadoop-common |
| -pl :hadoop-yarn-server-nodemanager |
| -pl :hadoop-yarn-client |
| -pl :hadoop-distcp |
| - comment: hdfs-rbf |
| modules: |
| -pl :hadoop-hdfs-rbf |
| -pl :hadoop-yarn-applications-distributedshell |
| -pl :hadoop-yarn-services-core |
| - comment: other |
| modules: |
| -pl :hadoop-minikdc |
| -pl :hadoop-auth |
| -pl :hadoop-auth-examples |
| -pl :hadoop-nfs |
| -pl :hadoop-kms |
| -pl :hadoop-registry |
| -pl :hadoop-hdfs-client |
| -pl :hadoop-hdfs-native-client |
| -pl :hadoop-hdfs-httpfs |
| -pl :hadoop-hdfs-nfs |
| -pl :hadoop-yarn-api |
| -pl :hadoop-yarn-common |
| -pl :hadoop-yarn-server-common |
| -pl :hadoop-yarn-server-applicationhistoryservice |
| -pl :hadoop-yarn-server-timelineservice |
| -pl :hadoop-yarn-server-web-proxy |
| -pl :hadoop-yarn-server-tests |
| -pl :hadoop-yarn-server-sharedcachemanager |
| -pl :hadoop-yarn-server-timeline-pluginstorage |
| -pl :hadoop-yarn-server-timelineservice-hbase-common |
| -pl :hadoop-yarn-server-timelineservice-hbase-client |
| -pl :hadoop-yarn-server-timelineservice-hbase-server-2 |
| -pl :hadoop-yarn-server-timelineservice-hbase-tests |
| -pl :hadoop-yarn-server-router |
| -pl :hadoop-yarn-server-timelineservice-documentstore |
| -pl :hadoop-yarn-server-globalpolicygenerator |
| -pl :hadoop-yarn-applications-unmanaged-am-launcher |
| -pl :hadoop-yarn-services-api |
| -pl :hadoop-yarn-registry |
| -pl org.apache.hadoop.applications.mawo:hadoop-yarn-applications-mawo-core |
| -pl :hadoop-yarn-csi |
| -pl :hadoop-minicluster |
| -pl :hadoop-federation-balance |
| -pl :hadoop-streaming |
| -pl :hadoop-client |
| -pl :hadoop-dynamometer-workload |
| -pl :hadoop-dynamometer-infra |
| -pl :hadoop-dynamometer-blockgen |
| -pl :hadoop-dynamometer-dist |
| -pl :hadoop-archives |
| -pl :hadoop-archive-logs |
| -pl :hadoop-rumen |
| -pl :hadoop-gridmix |
| -pl :hadoop-datajoin |
| -pl :hadoop-extras |
| -pl :hadoop-aws |
| -pl :hadoop-kafka |
| -pl :hadoop-aliyun |
| -pl :hadoop-sls |
| -pl :hadoop-resourceestimator |
| -pl :hadoop-azure |
| -pl :hadoop-azure-datalake |
| -pl :hadoop-fs2img |
| -pl :hadoop-tools-dist |
| -pl :hadoop-benchmark |
| -pl :hadoop-compat-bench |
| -pl :hadoop-client-api |
| -pl :hadoop-client-runtime |
| -pl :hadoop-client-minicluster |
| -pl :hadoop-client-integration-tests |
| -pl :hadoop-cos |
| -pl :hadoop-tos |
| -pl :hadoop-huaweicloud |
| -pl :hadoop-cloud-storage |
| -pl :hadoop-dist |
| # skipped modules |
| # -pl :hadoop-yarn-applications-catalog-webapp |
| steps: |
| - name: Checkout Hadoop repository |
| uses: actions/checkout@v6 |
| # In order to fetch changed files |
| with: |
| fetch-depth: 0 |
| - name: Setup JDK ${{ inputs.java }} |
| uses: actions/setup-java@v5 |
| with: |
| distribution: zulu |
| java-version: ${{ inputs.java }} |
| cache: 'maven' |
| check-latest: false |
| - name: Build (Java ${{ inputs.java }}) ${{ inputs.os }}-${{ inputs.branch }} |
| shell: bash |
| run: ./mvnw $MAVEN_ARGS ${{ matrix.modules }} clean install -am -DskipTests |
| - name: Test (Java ${{ inputs.java }}) ${{ inputs.os }}-${{ inputs.branch }} |
| shell: bash |
| run: ./mvnw $MAVEN_ARGS ${{ matrix.modules }} test -Dsurefire.excludesFile=$PWD/.github/gha-tests/exclude-tests.txt |
| - name: Upload test logs |
| if: failure() |
| uses: actions/upload-artifact@v7 |
| with: |
| name: unit-test-logs-${{ matrix.comment }}-java${{ inputs.java }})-${{ inputs.os }}-${{ inputs.branch }} |
| path: | |
| **/target/*.log |
| **/target/*.xml |
