.github/workflows/python-ci.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: "Python CI"

on:
  push:
    branches:
    - 'main'
  pull_request:
    paths:
    - '**'                              # Include all files and directories in the repository by default.
    - '!.github/workflows/**'           # Exclude all workflow files
    - '.github/workflows/python-ci.yml' # except the current file.
    - '!.github/ISSUE_TEMPLATE/**'      # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
    - '!.gitignore'
    - '!.asf.yml'
    - '!mkdocs/**'
    - '!.gitattributes'
    - '!README.md'
    - '!CONTRIBUTING.md'
    - '!LICENSE'
    - '!NOTICE'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint-and-test:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        python: ['3.9', '3.10', '3.11', '3.12']

    steps:
    - uses: actions/checkout@v4
    - name: Install poetry
      run: make install-poetry
    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
        cache: poetry
        cache-dependency-path: ./poetry.lock
    - name: Install system dependencies
      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
    - name: Install
      run: make install-dependencies
    - name: Run linters
      run: make lint
    - name: Run unit tests with coverage
      run: COVERAGE=1 make test
    - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it!
      run: COVERAGE_FAIL_UNDER=85 make coverage-report

  integration-test:
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        python: ['3.9', '3.10', '3.11', '3.12']

    steps:
    - uses: actions/checkout@v4
    - name: Install system dependencies
      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
    - name: Install
      run: make install

    - name: Run integration tests with coverage
      run: COVERAGE=1 make test-integration
    - name: Show debug logs
      if: ${{ failure() }}
      run: docker compose -f dev/docker-compose.yml logs

    - name: Run s3 integration tests with coverage
      run: COVERAGE=1 make test-s3
    - name: Show debug logs
      if: ${{ failure() }}
      run: docker compose -f dev/docker-compose.yml logs

    - name: Run adls integration tests with coverage
      run: COVERAGE=1 make test-adls
    - name: Show debug logs
      if: ${{ failure() }}
      run: docker compose -f dev/docker-compose-azurite.yml logs

    - name: Run gcs integration tests with coverage
      run: COVERAGE=1 make test-gcs
    - name: Show debug logs
      if: ${{ failure() }}
      run: docker compose -f dev/docker-compose-gcs-server.yml logs

    - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
      run: COVERAGE_FAIL_UNDER=75 make coverage-report