.github/workflows/ci.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: CI

on:
  push:
    branches:
      - main
      - release/**
    paths-ignore:
      - '**.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/PULL_REQUEST_TEMPLATE.md'
      - '.asf.yaml'
      - '.licenserc.yaml'
      - '.commitlintrc.yaml'
      - 'LICENSE'
      - 'NOTICE'
  pull_request:
    branches:
      - main
    paths-ignore:
      - '**.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/PULL_REQUEST_TEMPLATE.md'
      - '.asf.yaml'
      - '.licenserc.yaml'
      - '.commitlintrc.yaml'
      - 'LICENSE'
      - 'NOTICE'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  rust-tests:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-24.04 ]
    runs-on: ${{ matrix.os }}
    container:
      image: xd009642/tarpaulin:0.32.7
      options: --security-opt seccomp=unconfined
    steps:
      - uses: actions/checkout@v7
      - name: Install system dependencies
        # librocksdb-sys generates its bindings with bindgen at build time, which
        # needs libclang. The hosted runners ship it; this container does not.
        run: |
          apt-get update
          apt-get install -y protobuf-compiler clang libclang-dev
      - name: Cache dependencies
        uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          # Only cache cargo registry and git, not target directory
          # Tarpaulin needs clean builds for accurate coverage
          cache-targets: false
          # Reduce cache size by cleaning old artifacts
          cache-all-crates: false
      - name: Rust unit tests with coverage report
        run: make coverage-xml
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        with:
          name: cov-report-rust-tests-${{ runner.os }}
          path: ./cov-reports
          if-no-files-found: 'error'

  # hudi-core without `spill-rocksdb` must keep building and passing. That build
  # is the reason the feature exists — it needs no libclang and no C++ toolchain,
  # so a consumer that cannot take a native build (or does not want one in a
  # Python wheel) has a configuration to use. Nothing else in CI compiles it, so
  # without this job it would break silently.
  rust-tests-no-spill-backend:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - name: Cache dependencies
        uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          key: no-spill-backend
      # Deliberately no clang/libclang: if a change makes the default-off build
      # pull rocksdb back in, this job fails to compile rather than quietly
      # reintroducing the native dependency.
      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - name: Clippy without the spill backend
        run: cargo clippy -p hudi-core --lib --no-default-features -- -D warnings
      - name: Unit tests without the spill backend
        run: cargo test -p hudi-core --lib --no-default-features --no-fail-fast
      # The umbrella crate is where a consumer actually opts out, and it reaches
      # hudi-core twice: directly, and through hudi-datafusion. Either edge taking
      # default features hands the tier back without the steps above noticing,
      # since they only build hudi-core. Both shapes are built here, where there
      # is no libclang to build rocksdb with.
      - name: Check the umbrella crate without the spill backend
        run: cargo check -p hudi --no-default-features
      - name: Check the umbrella crate without the spill backend, with datafusion
        run: cargo check -p hudi --no-default-features --features datafusion
      # The steps above prove the opt-out COMPILES. This proves it actually drops
      # the dependency, and unlike them it does not rest on libclang being absent,
      # so it keeps working if this job ever gains a C++ toolchain.
      - name: Assert the opt-out drops rocksdb from the dependency graph
        run: |
          # A graph assertion has two ways to look like it passed: the tier
          # really is gone, or cargo tree never ran. `-i` exits non-zero with an
          # empty stdout in BOTH cases, so neither the output nor the status
          # alone tells them apart, and discarding stderr makes a broken command
          # indistinguishable from a working opt-out. Each check below therefore
          # pins the specific reason rather than inferring from emptiness.

          # Under defaults the tier must be reachable. If this ever stops
          # holding, the absence checks below are asserting nothing.
          for args in "" "--features datafusion"; do
            if ! out=$(cargo tree -p hudi $args -i rocksdb 2>&1) \
               || ! printf '%s\n' "$out" | grep -q '^rocksdb'; then
              echo "cargo tree did not find rocksdb under default features: '$args'"
              echo "so the absence checks below cannot tell an opt-out from a broken command"
              printf '%s\n' "$out"
              exit 1
            fi
          done

          # With the opt-out it must be gone — and gone because the package is
          # absent from the graph, which cargo says in those words, not because
          # the command failed for some other reason.
          for args in "--no-default-features" "--no-default-features --features datafusion"; do
            if out=$(cargo tree -p hudi $args -i rocksdb 2>&1); then
              echo "rocksdb is still reachable from hudi with: $args"
              printf '%s\n' "$out"
              exit 1
            elif ! printf '%s\n' "$out" | grep -q 'did not match any packages'; then
              echo "cargo tree failed for a reason other than rocksdb being absent: '$args'"
              printf '%s\n' "$out"
              exit 1
            fi
          done

  # Every other job builds with the rust-toolchain.toml dev pin; this job
  # compiles with the MSRV that Cargo.toml declares (read at run time so it
  # follows bumps), keeping the declared floor buildable once the two diverge.
  msrv-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - name: Install system dependencies
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler clang libclang-dev
      - name: Install the MSRV toolchain
        run: |
          msrv=$(grep -m1 '^rust-version' Cargo.toml | cut -d'"' -f2)
          if [ -z "$msrv" ]; then
            echo "could not read rust-version from Cargo.toml"
            exit 1
          fi
          rustup toolchain install "$msrv" --profile minimal
          echo "MSRV=$msrv" >> "$GITHUB_ENV"
      - name: Cache dependencies
        uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          key: msrv
      - name: Check the workspace on the MSRV
        run: cargo +"$MSRV" check --workspace --all-targets --all-features

  rust-tests-windows:
    runs-on: windows-2025
    # disable until https://github.com/apache/hudi-rs/issues/445 is resolved
    if: false    
    steps:
      - uses: actions/checkout@v7
      - name: Run unit tests on Windows with no coverage report
        run: cargo test --no-fail-fast --all-targets --all-features --workspace

  python-tests:
    strategy:
      fail-fast: false
      matrix:
        # TODO: add windows
        os: [ ubuntu-24.04, ubuntu-22.04-arm, macos-15 ]
        python-version: [ '3.10', '3.13' ]
        exclude:
          - os: ubuntu-24.04
            python-version: '3.10'
          - os: ubuntu-22.04-arm
            python-version: '3.13'
          - os: macos-15
            python-version: '3.10'
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v7

      - name: Install protobuf compiler (Linux)
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler

      - name: Install protobuf compiler (macOS)
        if: runner.os == 'macOS'
        run: brew install protobuf

      # Cache Rust dependencies and build artifacts
      - name: Cache Rust dependencies
        uses: swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          # Cache based on Python version and OS for better hit rate
          shared-key: "python-binding-${{ matrix.os }}-py${{ matrix.python-version }}"
          # Save cache even on failure to speed up retries
          save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'pull_request' }}

      - name: install uv and set the python version
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.7.19"
          python-version: ${{ matrix.python-version }}
          enable-cache: true
          cache-dependency-glob: "python/pyproject.toml"

      - name: Setup Python venv and build
        run: |
          make setup-venv
          source .venv/bin/activate
          make build
          make develop
          
      - name: Python unit tests
        # Only compute coverage for one matrix combination to save time
        if: ${{ !((matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13')) }}
        run: |
          source .venv/bin/activate
          pytest -v
          
      - name: Python unit tests with coverage report
        # Only compute coverage for ubuntu-24.04 with Python 3.13
        if: ${{ (matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13') }}
        run: |
          source .venv/bin/activate
          coverage run --include 'python/hudi/*' -m pytest -v
          coverage xml --data-file=.coverage -o ./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml
          
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        if: ${{ (matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13') }}
        with:
          name: cov-report-python-tests-${{ join(matrix.*, '-') }}
          path: ./cov-reports
          if-no-files-found: 'error'
          
      - name: Upload wheel files
        uses: actions/upload-artifact@v7
        with:
          name: python-artifact-${{ join(matrix.*, '-') }}
          path: ./target/wheels/*.whl
          if-no-files-found: 'error'

  integration-tests:
    strategy:
        fail-fast: false
        matrix:
          app-path: [ 'datafusion', 'hudi-file-group-api/cpp', 'hudi-table-api/rust' , 'hudi-table-api/python' ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
      - name: Pre-pull base images
        run: |
          docker pull quay.io/minio/minio:latest
      - name: Prepare cache directories with correct permissions
        run: |
          mkdir -p .cargo/registry .cargo/git target python/target .uv-cache
          sudo chown -R $(id -u):$(id -g) .cargo target python/target .uv-cache
      - name: Cache integration build artifacts (cargo/target/uv)
        uses: actions/cache@v6
        with:
          path: |
            target
            .cargo/registry
            .cargo/git
            python/target
            .uv-cache
          key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock', 'python/Cargo.toml', 'demo/infra/runner/Dockerfile') }}
          restore-keys: |
            integration-${{ runner.os }}-
      - name: Integration test - ${{ matrix.app-path }}
        run: |
          cd demo
          ./ci_run.sh ${{ matrix.app-path }}
      - name: Fix cache ownership before save
        if: always()
        run: |
          sudo chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE/.cargo $GITHUB_WORKSPACE/target $GITHUB_WORKSPACE/python/target $GITHUB_WORKSPACE/.uv-cache || true

  manylinux-wheel-build:
    name: Build the manylinux wheel
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: ubuntu-latest
            target: x86_64-unknown-linux-gnu
          - runner: ubuntu-22.04-arm
            target: aarch64-unknown-linux-gnu
    runs-on: ${{ matrix.runner }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v7
        with:
          python-version: '3.13'
      # Builds the wheel in the same container, running the same dependency
      # script, that the release workflow publishes from. Regular CI runs on the
      # host, whose apt protoc is new enough for lance-encoding's prost-build and
      # which carries libclang for librocksdb-sys, so a container-only break (see
      # #750) stays invisible until a release tag is pushed - by which point the
      # version is burnt on crates.io and cannot be reused.
      - name: Build wheel
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          command: build
          args: --release -m python/Cargo.toml --out dist
          # Pin the image rather than relying on the default, so this job and the
          # release steps are provably the same environment.
          manylinux: '2014'
          before-script-linux: |
            source .github/scripts/manylinux-build-deps.sh
      # A wheel that compiles is not necessarily a wheel that loads: the
      # bindings rocksdb generates come from the container's older libclang, and
      # bad bindings tend to surface on import or first use rather than at
      # compile time. Each runner installs the wheel it just built.
      - name: Import the built wheel
        run: |
          python -m pip install --upgrade pip
          python -m pip install dist/*.whl
          python -c "import hudi; print(hudi.__file__)"

  publish-coverage:
    name: Publish coverage reports to codecov.io
    runs-on: ubuntu-latest
    needs: [ rust-tests, python-tests ]
    if: github.event.pull_request.user.login != 'dependabot[bot]'
    steps:
      - uses: actions/checkout@v7
      - uses: actions/download-artifact@v8
        with:
          pattern: cov-report-*
          merge-multiple: true
          path: ./cov-reports
      - name: Upload coverage reports to codecov.io
        uses: codecov/codecov-action@v7
        with:
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
          directory: ./cov-reports