| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| |
| name: Main (Build/Test/Publish) |
| |
| on: |
| workflow_call: |
| inputs: |
| spark: |
| description: The Spark version of Spark image. |
| required: true |
| type: string |
| default: 3.3.0 |
| scala: |
| description: The Scala version of Spark image. |
| required: true |
| type: string |
| default: 2.12 |
| java: |
| description: The Java version of Spark image. |
| required: true |
| type: string |
| default: 11 |
| build: |
| description: Build the image or not. |
| required: false |
| type: boolean |
| default: true |
| publish: |
| description: Publish the image or not. |
| required: false |
| type: boolean |
| default: false |
| repository: |
| description: The registry to be published/tested. (Available only in publish/test workflow) |
| required: false |
| type: string |
| default: ghcr.io/apache/spark-docker |
| image-type: |
| description: The image type of the image (all, python, scala, r). |
| required: false |
| type: string |
| default: python |
| image-tag: |
| type: string |
| description: The image tag to be tested. (Available only in test workflow) |
| required: false |
| default: latest |
| |
| jobs: |
| main: |
| runs-on: ubuntu-latest |
| # Due to the multi-platform images cannot be exported with the `docker` export type, |
| # https://github.com/docker/buildx/issues/59 |
| # So, the local registry (push) is used here rather than local build (load): |
| # https://github.com/docker/build-push-action/blob/master/docs/advanced/local-registry.md |
| services: |
| registry: |
| image: registry:2 |
| ports: |
| - 5000:5000 |
| steps: |
| - name: Checkout Spark Docker repository |
| uses: actions/checkout@v3 |
| |
| - name: Free up disk space |
| shell: 'script -q -e -c "bash {0}"' |
| run: | |
| chmod +x tools/ci_runner_cleaner/free_disk_space_container.sh |
| tools/ci_runner_cleaner/free_disk_space_container.sh |
| chmod +x tools/ci_runner_cleaner/free_disk_space.sh |
| tools/ci_runner_cleaner/free_disk_space.sh |
| |
| - name: Prepare - Generate tags |
| run: | |
| case "${{ inputs.image-type }}" in |
| all) SUFFIX=python3-r-ubuntu |
| ;; |
| python) SUFFIX=python3-ubuntu |
| ;; |
| r) SUFFIX=r-ubuntu |
| ;; |
| scala) SUFFIX=ubuntu |
| ;; |
| esac |
| BASE_IMGAE_TAG=${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }}-ubuntu |
| TAG=scala${{ inputs.scala }}-java${{ inputs.java }}-$SUFFIX |
| |
| IMAGE_NAME=spark |
| IMAGE_PATH=${{ inputs.spark }}/$TAG |
| BASE_IMAGE_PATH=${{ inputs.spark }}/scala${{ inputs.scala }}-java${{ inputs.java }}-ubuntu |
| if [ "${{ inputs.build }}" == "true" ]; then |
| # Use the local registry to build and test |
| REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') |
| TEST_REPO=localhost:5000/$REPO_OWNER/spark-docker |
| UNIQUE_IMAGE_TAG=${{ inputs.spark }}-$TAG |
| else |
| # Use specified {repository}/spark:{image-tag} image to test |
| TEST_REPO=${{ inputs.repository }} |
| UNIQUE_IMAGE_TAG=${{ inputs.image-tag }} |
| fi |
| |
| # We can't use the real image for build because we haven't publish the image yet. |
| # The base image for build, it's something like localhost:5000/$REPO_OWNER/spark-docker/spark:3.3.0-scala2.12-java11-ubuntu |
| BASE_IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$BASE_IMGAE_TAG |
| IMAGE_URL=$TEST_REPO/$IMAGE_NAME:$UNIQUE_IMAGE_TAG |
| |
| PUBLISH_REPO=${{ inputs.repository }} |
| PUBLISH_IMAGE_URL=`tools/manifest.py tags -i ${PUBLISH_REPO}/${IMAGE_NAME} -p ${{ inputs.spark }}/${TAG}` |
| |
| # Unique image tag in each version: 3.3.0-scala2.12-java11-python3-ubuntu |
| echo "UNIQUE_IMAGE_TAG=${UNIQUE_IMAGE_TAG}" >> $GITHUB_ENV |
| # Test repo: ghcr.io/apache/spark-docker |
| echo "TEST_REPO=${TEST_REPO}" >> $GITHUB_ENV |
| # Image name: spark |
| echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV |
| # Base Image Dockerfile: 3.3.0/scala2.12-java11-ubuntu |
| echo "BASE_IMAGE_PATH=${BASE_IMAGE_PATH}" >> $GITHUB_ENV |
| # Image dockerfile path: 3.3.0/scala2.12-java11-python3-ubuntu |
| echo "IMAGE_PATH=${IMAGE_PATH}" >> $GITHUB_ENV |
| # Base Image URL: spark:3.3.0-scala2.12-java11-ubuntu |
| echo "BASE_IMAGE_URL=${BASE_IMAGE_URL}" >> $GITHUB_ENV |
| # Image URL: ghcr.io/apache/spark-docker/spark:3.3.0-scala2.12-java11-python3-ubuntu |
| echo "IMAGE_URL=${IMAGE_URL}" >> $GITHUB_ENV |
| |
| echo "PUBLISH_REPO=${PUBLISH_REPO}" >> $GITHUB_ENV |
| echo "PUBLISH_IMAGE_URL=${PUBLISH_IMAGE_URL}" >> $GITHUB_ENV |
| |
| - name: Prepare - Print Image tags |
| run: | |
| echo "UNIQUE_IMAGE_TAG: "${UNIQUE_IMAGE_TAG} |
| echo "TEST_REPO: "${TEST_REPO} |
| echo "IMAGE_NAME: "${IMAGE_NAME} |
| echo "IMAGE_PATH: "${IMAGE_PATH} |
| echo "IMAGE_URL: "${IMAGE_URL} |
| |
| echo "BASE_IMAGE_PATH: "${BASE_IMAGE_PATH} |
| echo "BASE_IMAGE_URL: "${BASE_IMAGE_URL} |
| |
| echo "PUBLISH_REPO:"${PUBLISH_REPO} |
| echo "PUBLISH_IMAGE_URL:"${PUBLISH_IMAGE_URL} |
| |
| - name: Build - Set up QEMU |
| if: ${{ inputs.build }} |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a |
| with: |
| ## Temporary due to bug in qemu: https://github.com/docker/setup-qemu-action/issues/198 |
| image: tonistiigi/binfmt:qemu-v7.0.0-28 |
| |
| - name: Build - Set up Docker Buildx |
| if: ${{ inputs.build }} |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd |
| with: |
| # This required by local registry |
| driver-opts: network=host |
| |
| - name: Build - Build the base image |
| # Don't need to build the base image when publish |
| if: ${{ inputs.build && !inputs.publish }} |
| uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f |
| with: |
| context: ${{ env.BASE_IMAGE_PATH }} |
| tags: ${{ env.BASE_IMAGE_URL }} |
| platforms: linux/amd64,linux/arm64 |
| push: true |
| |
| - name: Build - Use the test image repo when build |
| # Don't need to build the base image when publish |
| if: ${{ inputs.build && !inputs.publish }} |
| working-directory: ${{ env.IMAGE_PATH }} |
| run: | |
| sed -i "s@FROM spark@FROM $TEST_REPO/spark@g" ./Dockerfile |
| |
| - name: Build - Use real image repo when publish |
| # Don't need to build the base image when publish |
| if: ${{ inputs.publish }} |
| working-directory: ${{ env.IMAGE_PATH }} |
| run: | |
| sed -i "s@FROM spark@FROM $PUBLISH_REPO/spark@g" ./Dockerfile |
| |
| - name: Build - Build and push test image |
| if: ${{ inputs.build }} |
| uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f |
| with: |
| context: ${{ env.IMAGE_PATH }} |
| tags: ${{ env.IMAGE_URL }} |
| platforms: linux/amd64,linux/arm64 |
| push: true |
| |
| - name : Test - Run spark application for standalone cluster on docker |
| run: testing/run_tests.sh --image-url $IMAGE_URL --scala-version ${{ inputs.scala }} --spark-version ${{ inputs.spark }} |
| |
| - name: Test - Checkout Spark repository for Spark 3.3.0 (with fetch-depth 0) |
| if: inputs.spark == '3.3.0' |
| uses: actions/checkout@v3 |
| with: |
| fetch-depth: 0 |
| repository: apache/spark |
| ref: v${{ inputs.spark }} |
| path: ${{ github.workspace }}/spark |
| |
| - name: Test - Checkout Spark repository |
| if: inputs.spark != '3.3.0' |
| uses: actions/checkout@v3 |
| with: |
| repository: apache/spark |
| ref: v${{ inputs.spark }} |
| path: ${{ github.workspace }}/spark |
| |
| - name: Test - Cherry pick commits |
| # Apache Spark enable resource limited k8s IT since v3.3.1, cherry-pick patches for old release |
| # https://github.com/apache/spark/pull/36087#issuecomment-1251756266 |
| if: inputs.spark == '3.3.0' |
| working-directory: ${{ github.workspace }}/spark |
| run: | |
| # SPARK-38802: Add driverRequestCores/executorRequestCores supported |
| # https://github.com/apache/spark/commit/83963828b54bffe99527a004057272bc584cbc26 |
| git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' cherry-pick 83963828b54bffe99527a004057272bc584cbc26 |
| # SPARK-38803: Lower minio cpu to 250m |
| # https://github.com/apache/spark/commit/5ea2b386eb866e20540660cdb6ed43792cb29969 |
| git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' cherry-pick 5ea2b386eb866e20540660cdb6ed43792cb29969 |
| |
| - name: Test - Install Java ${{ inputs.java }} |
| uses: actions/setup-java@v3 |
| with: |
| # This is required after v2, now just keep same distribution with v1 |
| # https://github.com/actions/setup-java/releases/tag/v2.0.0 |
| distribution: 'zulu' |
| java-version: ${{ inputs.java }} |
| |
| - name: Test - Cache Scala, SBT and Maven |
| uses: actions/cache@v3 |
| with: |
| path: | |
| build/apache-maven-* |
| build/scala-* |
| build/*.jar |
| ~/.sbt |
| key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }} |
| |
| - name: Test - Cache Coursier local repository |
| uses: actions/cache@v3 |
| with: |
| path: ~/.cache/coursier |
| key: build-${{ inputs.spark }}-scala${{ inputs.scala }}-java${{ inputs.java }}-coursier |
| |
| - name: Install R |
| run: | |
| sudo apt update |
| sudo apt-get install r-base |
| |
| - name: Test - Start minikube |
| uses: medyagh/setup-minikube@e9e035a86bbc3caea26a450bd4dbf9d0c453682e # v0.0.21 |
| with: |
| cpus: 2 |
| memory: 6144m |
| |
| - name: Test - Print K8S pods and nodes info |
| run: | |
| kubectl get pods -A |
| kubectl describe node |
| |
| - name: Test - Run Spark on K8S integration test (With driver cpu 0.5, executor cpu 0.2 limited) |
| working-directory: ${{ github.workspace }}/spark |
| run: | |
| kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true |
| minikube image load ${{ env.IMAGE_URL }} |
| |
| sed -i '/<id>volcano<\/id>/,/<\/profile>/ s|<activeByDefault>true</activeByDefault>|<activeByDefault>false</activeByDefault>|' resource-managers/kubernetes/integration-tests/pom.xml |
| |
| eval $(minikube docker-env) |
| OPTS="-Pkubernetes -Pkubernetes-integration-tests " |
| OPTS+="-Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 " |
| OPTS+="-Dspark.kubernetes.test.deployMode=minikube " |
| OPTS+="-Dspark.kubernetes.test.imageRepo=${TEST_REPO} -Dspark.kubernetes.test.imageTag=${UNIQUE_IMAGE_TAG} " |
| OPTS+="-Dspark.kubernetes.test.jvmImage=${IMAGE_NAME} " |
| OPTS+="-Dspark.kubernetes.test.pythonImage=${IMAGE_NAME} " |
| OPTS+="-Dspark.kubernetes.test.rImage=${IMAGE_NAME} " |
| |
| if [ "${{ inputs.image-type }}" = "all" ]; then |
| # Prepare test jar for client tests |
| CONTAINER_TMP_NAME=spark-example-image |
| docker create -ti --name $CONTAINER_TMP_NAME ${{ env.IMAGE_URL }} bash |
| docker cp $CONTAINER_TMP_NAME:/opt/spark/examples/jars/spark-examples_${{ inputs.scala }}-${{ inputs.spark }}.jar . |
| docker rm -f $CONTAINER_TMP_NAME |
| # Prepare PV test |
| PVC_TMP_DIR=$(mktemp -d) |
| export PVC_TESTS_HOST_PATH=$PVC_TMP_DIR |
| export PVC_TESTS_VM_PATH=$PVC_TMP_DIR |
| minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0 --uid=185 & |
| # Run all K8s test for all in one image |
| build/sbt $OPTS 'kubernetes-integration-tests/testOnly' |
| else |
| # Run basic test for Scala/PySpark/SparkR image |
| build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run SparkPi"' |
| |
| # Run basic test for PySpark image |
| if [ "${{ inputs.image-type }}" = "python" ]; then |
| build/sbt $OPTS 'kubernetes-integration-tests/testOnly -- -z "Run PySpark"' |
| fi |
| |
| # Run basic test for SparkR image |
| if [ "${{ inputs.image-type }}" = "r" ]; then |
| OPTS+="-Psparkr -Dtest.include.tags=r " |
| build/sbt $OPTS 'kubernetes-integration-tests/testOnly' |
| fi |
| fi |
| |
| - name: Test - Upload Spark on K8S integration tests log files |
| if: failure() |
| uses: actions/upload-artifact@v4 |
| with: |
| name: spark-on-kubernetes-it-log |
| path: "**/target/integration-tests.log" |
| |
| - name: Publish - Login to GitHub Container Registry |
| if: ${{ inputs.publish }} |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 |
| with: |
| registry: ghcr.io |
| username: ${{ github.actor }} |
| password: ${{ secrets.GITHUB_TOKEN }} |
| |
| - name: Publish - Login to Dockerhub Registry |
| if: ${{ inputs.publish }} |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 |
| with: |
| username: ${{ secrets.DOCKERHUB_USER }} |
| password: ${{ secrets.DOCKERHUB_TOKEN }} |
| |
| - name: Publish - Push Image |
| if: ${{ inputs.publish }} |
| uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f |
| with: |
| context: ${{ env.IMAGE_PATH }} |
| push: true |
| tags: ${{ env.PUBLISH_IMAGE_URL }} |
| platforms: linux/amd64,linux/arm64 |
