.github/workflows/build.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Reusable workflow for running building
# This ensures the same tests run for both debug (PRs) and release (main/tags) builds

name: Build

on:
  workflow_call:
    inputs:
      build_mode:
        description: 'Build mode: debug or release'
        required: true
        type: string
      run_wheels:
        description: 'Whether to build distribution wheels'
        required: false
        type: boolean
        default: false

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  UV_LOCKED: true

jobs:
  # ============================================
  # Linting Jobs
  # ============================================
  lint-rust:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
        with:
          toolchain: "nightly"
          components: rustfmt

      - name: Cache Cargo
        uses: Swatinem/rust-cache@v2

      - name: Check formatting
        run: cargo +nightly fmt --all -- --check

  lint-python:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Install dependencies
        run: uv sync --dev --no-install-package datafusion

      - name: Run Ruff
        run: |
          uv run --no-project ruff check --output-format=github python/
          uv run --no-project ruff format --check python/

      - name: Run codespell
        run: |
          uv run --no-project codespell --toml pyproject.toml

  lint-toml:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Install taplo
        uses: taiki-e/install-action@v2
        with:
          tool: taplo-cli

      # if you encounter an error, try running 'taplo format' to fix the formatting automatically.
      - name: Check Cargo.toml formatting
        run: taplo format --check

  check-crates-patch:
    if: inputs.build_mode == 'release' && startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Ensure [patch.crates-io] is empty
        run: python3 dev/check_crates_patch.py

  generate-license:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Install cargo-license
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-license

      - name: Generate license file
        run: uv run --no-project python ./dev/create_license.py

      - uses: actions/upload-artifact@v7
        with:
          name: python-wheel-license
          path: LICENSE.txt

  # ============================================
  # Build - Linux x86_64
  # ============================================
  build-manylinux-x86_64:
    needs: [generate-license, lint-rust, lint-python]
    name: Linux x86_64 (${{ matrix.python-tag }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-tag: ["abi3", "3.14t"]
    steps:
      - uses: actions/checkout@v6

      - run: rm LICENSE.txt
      - name: Download LICENSE.txt
        uses: actions/download-artifact@v8
        with:
          name: python-wheel-license
          path: .

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8

      - name: Cache Cargo
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Add extra swap for release build
        if: inputs.build_mode == 'release'
        run: |
          set -euxo pipefail
          sudo swapoff -a || true
          sudo rm -f /swapfile
          sudo fallocate -l 8G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=8192
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
          free -h
          swapon --show

      - name: Build wheel
        uses: ./.github/actions/build-wheel
        with:
          target: x86_64-unknown-linux-gnu
          python-tag: ${{ matrix.python-tag }}
          build-mode: ${{ inputs.build_mode }}
          features: "protoc,substrait"
          manylinux: "2_28"

      # FFI test wheel only needs to be built once per platform; gate to abi3.
      - name: Build FFI test library
        if: matrix.python-tag == 'abi3'
        uses: PyO3/maturin-action@v1
        with:
          target: x86_64-unknown-linux-gnu
          manylinux: "2_28"
          working-directory: examples/datafusion-ffi-example
          args: --out dist
          rustup-components: rust-std

      - name: Archive wheels
        uses: actions/upload-artifact@v7
        with:
          name: dist-manylinux-x86_64-${{ matrix.python-tag }}
          path: dist/*

      - name: Archive FFI test wheel
        if: matrix.python-tag == 'abi3'
        uses: actions/upload-artifact@v7
        with:
          name: test-ffi-manylinux-x86_64
          path: examples/datafusion-ffi-example/dist/*

  # ============================================
  # Build - Linux ARM64
  # ============================================
  build-manylinux-aarch64:
    needs: [generate-license, lint-rust, lint-python]
    name: Linux arm64 (${{ matrix.python-tag }})
    runs-on: ubuntu-24.04-arm
    strategy:
      fail-fast: false
      matrix:
        python-tag: ["abi3", "3.14t"]
    steps:
      - uses: actions/checkout@v6

      - run: rm LICENSE.txt
      - name: Download LICENSE.txt
        uses: actions/download-artifact@v8
        with:
          name: python-wheel-license
          path: .

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8

      - name: Cache Cargo
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Add extra swap for release build
        if: inputs.build_mode == 'release'
        run: |
          set -euxo pipefail
          sudo swapoff -a || true
          sudo rm -f /swapfile
          sudo fallocate -l 8G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=8192
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
          free -h
          swapon --show

      - name: Build wheel
        uses: ./.github/actions/build-wheel
        with:
          target: aarch64-unknown-linux-gnu
          python-tag: ${{ matrix.python-tag }}
          build-mode: ${{ inputs.build_mode }}
          features: "protoc,substrait"
          manylinux: "2_28"

      - name: Archive wheels
        uses: actions/upload-artifact@v7
        if: inputs.build_mode == 'release'
        with:
          name: dist-manylinux-aarch64-${{ matrix.python-tag }}
          path: dist/*

  # ============================================
  # Build - macOS arm64 / Windows
  # ============================================
  build-python-mac-win:
    needs: [generate-license, lint-rust, lint-python]
    name: ${{ matrix.os == 'macos-latest' && 'macOS arm64' || 'Windows x86_64' }} (${{ matrix.python-tag }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest]
        python-tag: ["abi3", "3.14t"]
    steps:
      - uses: actions/checkout@v6

      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8

      - run: rm LICENSE.txt
      - name: Download LICENSE.txt
        uses: actions/download-artifact@v8
        with:
          name: python-wheel-license
          path: .

      - name: Cache Cargo
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}

      - name: Setup Python (free-threaded)
        if: matrix.python-tag != 'abi3'
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-tag }}
          freethreaded: true

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "27.4"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install dependencies
        if: matrix.python-tag == 'abi3'
        run: uv sync --dev --no-install-package datafusion

      # Clippy is interpreter-agnostic; run once per OS (against the abi3 entry)
      # so the matrix doesn't pay the cost three times.
      - name: Run Clippy
        if: matrix.os != 'windows-latest' && matrix.python-tag == 'abi3'
        run: cargo clippy --no-deps --all-targets --features substrait -- -D warnings

      - name: Build wheel
        uses: ./.github/actions/build-wheel
        with:
          python-tag: ${{ matrix.python-tag }}
          build-mode: ${{ inputs.build_mode }}
          features: "substrait"
          out-dir: "target/wheels"

      - name: List Windows wheels
        if: matrix.os == 'windows-latest'
        run: dir target\wheels\
        # since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell
        # so we specify it explicitly
        shell: powershell

      - name: List Mac wheels
        if: matrix.os != 'windows-latest'
        run: find target/wheels/

      - name: Archive wheels
        uses: actions/upload-artifact@v7
        if: inputs.build_mode == 'release'
        with:
          name: dist-${{ matrix.os }}-${{ matrix.python-tag }}
          path: target/wheels/*

  # ============================================
  # Build - macOS x86_64 (release only)
  # ============================================
  build-macos-x86_64:
    if: inputs.build_mode == 'release'
    needs: [generate-license, lint-rust, lint-python]
    name: macOS x86_64 (${{ matrix.python-tag }})
    runs-on: macos-15-intel
    strategy:
      fail-fast: false
      matrix:
        python-tag: ["abi3", "3.14t"]
    steps:
      - uses: actions/checkout@v6

      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8

      - run: rm LICENSE.txt
      - name: Download LICENSE.txt
        uses: actions/download-artifact@v8
        with:
          name: python-wheel-license
          path: .

      - name: Cache Cargo
        uses: Swatinem/rust-cache@v2
        with:
          key: ${{ inputs.build_mode }}-${{ matrix.python-tag }}

      - name: Setup Python (free-threaded)
        if: matrix.python-tag != 'abi3'
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-tag }}
          freethreaded: true

      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "27.4"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install dependencies
        if: matrix.python-tag == 'abi3'
        run: uv sync --dev --no-install-package datafusion

      - name: Build wheel
        uses: ./.github/actions/build-wheel
        with:
          python-tag: ${{ matrix.python-tag }}
          build-mode: ${{ inputs.build_mode }}
          features: "substrait"
          out-dir: "target/wheels"

      - name: List Mac wheels
        run: find target/wheels/

      - name: Archive wheels
        uses: actions/upload-artifact@v7
        with:
          name: dist-macos-aarch64-${{ matrix.python-tag }}
          path: target/wheels/*

  # ============================================
  # Build - Source Distribution
  # ============================================

  build-sdist:
    needs: [generate-license]
    name: Source distribution
    if: inputs.build_mode == 'release'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - run: rm LICENSE.txt
      - name: Download LICENSE.txt
        uses: actions/download-artifact@v8
        with:
          name: python-wheel-license
          path: .
      - run: cat LICENSE.txt
      - name: Build sdist
        uses: PyO3/maturin-action@v1
        with:
          rust-toolchain: stable
          manylinux: auto
          rustup-components: rust-std rustfmt
          args: --release --sdist --out dist --features protoc,substrait
      - name: Assert sdist build does not generate wheels
        run: |
          if [ "$(ls -A target/wheels)" ]; then
            echo "Error: Sdist build generated wheels"
            exit 1
          else
            echo "Directory is clean"
          fi
        shell: bash

  # ============================================
  # Build - Source Distribution
  # ============================================

  merge-build-artifacts:
    runs-on: ubuntu-latest
    name: Merge build artifacts
    if: inputs.build_mode == 'release'
    needs:
      - build-python-mac-win
      - build-macos-x86_64
      - build-manylinux-x86_64
      - build-manylinux-aarch64
      - build-sdist
    steps:
      - name: Merge Build Artifacts
        uses: actions/upload-artifact/merge@v7
        with:
          name: dist
          pattern: dist-*

  # ============================================
  # Build - Documentation
  # ============================================
  # Documentation build job that runs after wheels are built
  build-docs:
    name: Build docs
    runs-on: ubuntu-latest
    needs: [build-manylinux-x86_64]  # Only need the Linux wheel for docs
    # Only run docs on main branch pushes, tags, or PRs
    if: github.event_name == 'push' || github.event_name == 'pull_request'
    steps:
      - name: Set target branch
        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
        id: target-branch
        run: |
          set -x
          if test '${{ github.ref }}' = 'refs/heads/main'; then
            echo "value=asf-staging" >> "$GITHUB_OUTPUT"
          elif test '${{ github.ref_type }}' = 'tag'; then
            echo "value=asf-site" >> "$GITHUB_OUTPUT"
          else
            echo "Unsupported input: ${{ github.ref }} / ${{ github.ref_type }}"
            exit 1
          fi

      - name: Checkout docs sources
        uses: actions/checkout@v6

      - name: Checkout docs target branch
        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ steps.target-branch.outputs.value }}
          path: docs-target

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"

      - name: Install dependencies
        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          enable-cache: true

      # Download the Linux wheel built in the previous job.
      # Docs only need the abi3 wheel — interpreter doesn't matter for sphinx.
      - name: Download pre-built Linux wheel
        uses: actions/download-artifact@v8
        with:
          name: dist-manylinux-x86_64-abi3
          path: wheels/

      # Install from the pre-built wheels
      - name: Install from pre-built wheels
        run: |
          set -x
          uv venv
          # Install documentation dependencies
          uv sync --dev --no-install-package datafusion --group docs
          # Install all pre-built wheels
          WHEELS=$(find wheels/ -name "*.whl")
          if [ -n "$WHEELS" ]; then
            echo "Installing wheels:"
            echo "$WHEELS"
            uv pip install wheels/*.whl
          else
            echo "ERROR: No wheels found!"
            exit 1
          fi

      - name: Build docs
        run: |
          set -x
          cd docs
          # build.sh downloads the example data, registers the Jupyter kernel
          # myst-nb needs, symlinks the data next to each executed page, and
          # runs sphinx. Using it here keeps CI identical to a local build.
          uv run --no-project bash ./build.sh

      - name: Copy & push the generated HTML
        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
        run: |
          set -x
          cd docs-target
          # delete anything but: 1) '.'; 2) '..'; 3) .git/
          find ./ | grep -vE "^./$|^../$|^./.git" | xargs rm -rf
          cp ../.asf.yaml .
          cp -r ../docs/build/html/* .
          git status --porcelain
          if [ "$(git status --porcelain)" != "" ]; then
            git config user.name "github-actions[bot]"
            git config user.email "github-actions[bot]@users.noreply.github.com"
            git add --all
            git commit -m 'Publish built docs triggered by ${{ github.sha }}'
            git push || git push --force
          fi