| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| name: Python API |
| |
| on: |
| push: |
| branches: |
| - dev |
| paths: |
| - 'dolphinscheduler-python/**' |
| pull_request: |
| |
| concurrency: |
| group: py-${{ github.event.pull_request.number || github.ref }} |
| cancel-in-progress: true |
| |
| defaults: |
| run: |
| working-directory: dolphinscheduler-python/pydolphinscheduler |
| |
| # We have to update setuptools wheel to package with package_data, LICENSE, NOTICE |
| env: |
| DEPENDENCES: pip setuptools wheel tox |
| |
| jobs: |
| paths-filter: |
| name: Python-Path-Filter |
| runs-on: ubuntu-latest |
| outputs: |
| not-docs: ${{ steps.filter.outputs.not-docs }} |
| py-change: ${{ steps.filter.outputs.py-change }} |
| steps: |
| - uses: actions/checkout@v2 |
| - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 |
| id: filter |
| with: |
| filters: | |
| not-docs: |
| - '!(docs/**)' |
| py-change: |
| - 'dolphinscheduler-python/pydolphinscheduler/**' |
| lint: |
| name: Lint |
| if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }} |
| timeout-minutes: 15 |
| needs: paths-filter |
| runs-on: ubuntu-latest |
| steps: |
| - uses: actions/checkout@v2 |
| - name: Set up Python 3.7 |
| uses: actions/setup-python@v4 |
| with: |
| python-version: 3.7 |
| - name: Install Dependences |
| run: | |
| python -m pip install --upgrade ${{ env.DEPENDENCES }} |
| - name: Run All Lint Check |
| run: | |
| python -m tox -vv -e lint |
| pytest: |
| name: Pytest |
| timeout-minutes: 15 |
| needs: lint |
| runs-on: ${{ matrix.os }} |
| strategy: |
| fail-fast: false |
| matrix: |
| # YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also: |
| # https://github.com/actions/setup-python/issues/160#issuecomment-724485470 |
| python-version: [3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev] |
| os: [ubuntu-latest, macOS-latest, windows-latest] |
| # Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments |
| exclude: |
| - os: windows-latest |
| python-version: '3.10' |
| - os: windows-latest |
| python-version: 3.11-dev |
| steps: |
| - uses: actions/checkout@v2 |
| - name: Set up Python ${{ matrix.python-version }} |
| uses: actions/setup-python@v4 |
| with: |
| python-version: ${{ matrix.python-version }} |
| - name: Install Dependences |
| run: | |
| python -m pip install --upgrade ${{ env.DEPENDENCES }} |
| - name: Run All Tests |
| run: | |
| python -m tox -vv -e code-test |
| doc-build: |
| name: Docs Build Test |
| timeout-minutes: 15 |
| needs: lint |
| runs-on: ubuntu-latest |
| strategy: |
| fail-fast: false |
| matrix: |
| env-list: [doc-build, doc-build-multi] |
| steps: |
| - uses: actions/checkout@v2 |
| - name: Set up Python 3.7 |
| uses: actions/setup-python@v4 |
| with: |
| python-version: 3.7 |
| - name: Install Dependences |
| run: | |
| python -m pip install --upgrade ${{ env.DEPENDENCES }} |
| - name: Run Build Docs Tests ${{ matrix.env-list }} |
| run: | |
| python -m tox -vv -e ${{ matrix.env-list }} |
| local-ci: |
| name: Local CI |
| timeout-minutes: 15 |
| needs: |
| - pytest |
| - doc-build |
| runs-on: ubuntu-latest |
| steps: |
| - uses: actions/checkout@v2 |
| - name: Set up Python 3.7 |
| uses: actions/setup-python@v4 |
| with: |
| python-version: 3.7 |
| - name: Install Dependences |
| run: | |
| python -m pip install --upgrade ${{ env.DEPENDENCES }} |
| - name: Run Tests Build Docs |
| run: | |
| python -m tox -vv -e local-ci |
| integrate-test: |
| name: Integrate Test |
| if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }} |
| runs-on: ubuntu-latest |
| needs: paths-filter |
| timeout-minutes: 30 |
| steps: |
| - uses: actions/checkout@v2 |
| with: |
| submodules: true |
| - name: Sanity Check |
| uses: ./.github/actions/sanity-check |
| with: |
| token: ${{ secrets.GITHUB_TOKEN }} |
| - name: Cache local Maven repository |
| uses: actions/cache@v3 |
| with: |
| path: ~/.m2/repository |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} |
| restore-keys: ${{ runner.os }}-maven- |
| # Switch to project root directory to run mvnw command |
| - name: Build Image |
| working-directory: ./ |
| run: | |
| ./mvnw -B clean install \ |
| -Dmaven.test.skip \ |
| -Dmaven.javadoc.skip \ |
| -Dcheckstyle.skip=true \ |
| -Pdocker,release -Ddocker.tag=ci \ |
| -pl dolphinscheduler-standalone-server -am |
| - name: Set up Python 3.7 |
| uses: actions/setup-python@v4 |
| with: |
| python-version: 3.7 |
| - name: Install Dependences |
| run: | |
| python -m pip install --upgrade ${{ env.DEPENDENCES }} |
| - name: Run Integrate Tests |
| run: | |
| python -m tox -vv -e integrate-test |
| result: |
| name: Python |
| runs-on: ubuntu-latest |
| timeout-minutes: 30 |
| needs: [ paths-filter, local-ci, integrate-test ] |
| if: always() |
| steps: |
| - name: Status |
| # We need change CWD to current directory to avoid global default working directory not exists |
| working-directory: ./ |
| run: | |
| if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then |
| echo "Only document change, skip both python unit and integrate test!" |
| exit 0 |
| fi |
| if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then |
| echo "No python code change, and integrate test pass!" |
| exit 0 |
| fi |
| if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then |
| echo "py-ci Failed!" |
| exit -1 |
| fi |