.github/workflows/ci.yaml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: CI

on:
  push:
    branches:
      - main
  pull_request:
  schedule:
    # Schedule daily builds at 00:00 UTC, because pydolphinscheduler rely on dolphinscheduler's API server, and
    # pydolphinscheduler can not aware of the changes of dolphinscheduler's API server code, so we need to add
    # schedule job to need passive inspection the latest code in apache/dolphinscheduler still work on.
    - cron: '0 0 * * *'

concurrency:
  group: ci-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
env:
  DEPENDENCES: pip setuptools wheel tox
  CODECOV_TOKEN: 7b597b03-7c9f-4175-b3ac-e74034ea87a6

jobs:
  license:
    runs-on: ubuntu-latest
    steps:
        - uses: apache/skywalking-eyes/header@main
  dead-link:
    runs-on: ubuntu-latest
    needs: license
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
      - run: sudo npm install -g markdown-link-check@3.10.0
      - run: |
          for file in $(find . -name "*.md"); do
            markdown-link-check -c .dlc.json -q "$file"
          done
  lint:
    timeout-minutes: 15
    runs-on: ubuntu-latest
    needs: license
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}
      - name: Run All Lint Check
        run: |
          python -m tox -vv -e lint
  pytest:
    timeout-minutes: 15
    needs: lint
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also:
        # https://github.com/actions/setup-python/issues/160#issuecomment-724485470
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        # FIXME: temp change os to ubuntu-20.04 to fix python can not found error https://github.com/actions/setup-python/issues/162#issuecomment-1325307787
        os: [ubuntu-20.04, macOS-latest, windows-latest]
        exclude:
          # Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments
          - os: windows-latest
            python-version: '3.10'
          - os: windows-latest
            python-version: 3.11
          - os: windows-latest
            python-version: 3.12
          # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
          # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
          - os: macos-latest
            python-version: 3.9
          - os: macos-latest
            python-version: 3.8
        include:
          - os: macos-13
            python-version: 3.9
          - os: macos-13
            python-version: 3.8
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}
      - name: Run All Tests
        run: |
          python -m tox -vv -e code-test
      - uses: codecov/codecov-action@v3
        # Codecov have a 100-upload limit per commit, and there are 3 * 6 files upload each time run pytest,
        # We should not run upload in schedule GitHub event, because the sixth day we do not change our code
        # and the upload limit will be reached 3 * 6 * 6. For more detail can see:
        # https://community.codecov.com/t/ci-failure-due-to-too-many-uploads-to-this-commit/2587/7
        if: ${{ github.event_name != 'schedule' }}
        with:
          token: ${{ env.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella
          fail_ci_if_error: true
          verbose: true
  doc-build:
    timeout-minutes: 15
    needs: lint
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        env-list: [doc-build, doc-build-multi]
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}
      - name: Run Build Docs Tests ${{ matrix.env-list }}
        run: |
          python -m tox -vv -e ${{ matrix.env-list }}
  local-ci:
    timeout-minutes: 15
    needs:
      - pytest
      - doc-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}
      - name: Run Tests Build Docs
        run: |
          python -m tox -vv -e local-ci
  commit-msg:
    needs: license
    runs-on: ubuntu-latest
    outputs:
      commit_message: ${{ steps.set-commit-message.outputs.commit_message }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - id: set-commit-message
        run: echo "commit_message=$(git show -s --format=%B | tr '\n' ' ')" >> $GITHUB_OUTPUT
  integrate-test:
    needs: commit-msg
    runs-on: ubuntu-latest
    if: ${{ github.event_name == 'schedule' || contains(needs.commit-msg.outputs.commit_message, '[run-it]') }}
    timeout-minutes: 30
    steps:
      - name: Checkout Dolphinscheduler Python SDK
        uses: actions/checkout@v3
        with:
          path: src
      - name: Checkout Dolphinscheduler
        uses: actions/checkout@v3
        with:
          repository: apache/dolphinscheduler
          path: dolphinscheduler
          submodules: true
          # Temporary add to make https://github.com/apache/dolphinscheduler-sdk-python/issues/12 work
          # ref: refs/pull/12918/head
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-maven-
      # Switch to project root directory to run mvnw command
      - name: Build Image
        working-directory: dolphinscheduler

        run: |
          ./mvnw -B clean install \
          -Dmaven.test.skip \
          -Dmaven.javadoc.skip \
          -Dcheckstyle.skip=true \
          -Pdocker,release -Ddocker.tag=ci \
          -pl dolphinscheduler-standalone-server -am
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install Dependences
        run: |
          python -m pip install --upgrade ${{ env.DEPENDENCES }}
      - name: Run Integrate Tests
        working-directory: src
        run: |
          python -m tox -vv -e integrate-test
  result:
    name: CI
    if: always()
    needs:
      - dead-link
      - local-ci
      - integrate-test
    runs-on: ubuntu-latest
    steps:
      - name: Status
        run: |
          if [[ ${{ needs.dead-link.result }} != 'success' ]] || \
            [[ ${{ needs.local-ci.result }} != 'success' ]] || \
            ([[ ${{ needs.integrate-test.result }} != 'skipped' ]] && [[ ${{ needs.integrate-test.result }} != 'success' ]]); then
            echo "CI Failed!"
            exit -1
          fi