.github/workflows/ci-python.yml

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

name: CI / Python

on:
  workflow_call:

permissions:
  contents: read

env:
  JDK_VERSION: 8
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true


jobs:
  # Lint + test on selected versions only (3.6 / 3.7 low end, 3.10 / 3.11 / 3.12 / 3.13 current), not every Python version.
  test:
    name: Tests / Python ${{ matrix.python-version }}
    env:
      # Native planning, reading, writing and REST commits run in Native CI.
      PYTEST_ADDOPTS: "-m 'not native_plan'"
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:${{ matrix.python-version }}-slim"
    strategy:
      fail-fast: false
      matrix:
        python-version: [ '3.6.15', '3.7', '3.10', '3.11', '3.12', '3.13' ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v5
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v5
        with:
          path: ~/.m2/repository
          key: ci-maven-python-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ci-maven-python-${{ runner.os }}-${{ runner.arch }}-

      - name: Set up Maven
        uses: stCarolas/setup-maven@v5
        with:
          maven-version: 3.8.8

      - name: Install system dependencies
        shell: bash
        run: |
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            # The Python 3.6 image uses EOL Debian Bullseye.
            sed -i '/security.debian.org\/debian-security/d' /etc/apt/sources.list
            sed -i 's|deb.debian.org/debian|archive.debian.org/debian|g' /etc/apt/sources.list
            echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive
          fi
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            pkg-config \
            libssl-dev \
            libclang-dev \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Verify Python distributions
        run: |
          # Build tools and legacy Flake8 require incompatible importlib-metadata versions.
          python -m venv "${RUNNER_TEMP}/paimon-packaging"
          "${RUNNER_TEMP}/paimon-packaging/bin/python" -m pip install --upgrade pip setuptools wheel packaging tomli build
          "${RUNNER_TEMP}/paimon-packaging/bin/python" paimon-python/dev/test_packaging.py

      - name: Verify core package dependencies can be installed
        shell: bash
        run: |
          cd paimon-python
          python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || {
            echo "ERROR: Failed to install core PyPaimon dependencies"
            rm -rf "$TEMP_DIR"
            exit 1
          }
          rm -rf "$TEMP_DIR"
          echo "✓ Core PyPaimon dependencies resolve on this Python"

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B -ntp install -DskipTests \
            -pl paimon-core,paimon-lance,paimon-vortex/paimon-vortex-format,paimon-full-text,paimon-lumina,paimon-vector -am

      - name: Install Python dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          if python -c "import sys; sys.exit(0 if sys.version_info < (3, 8) else 1)"; then
            # Old pip has no --group; read the same TOML with the compatibility bridge.
            python -m pip install tomli
            python paimon-python/dev/install.py legacy
          else
            python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
            dependency_group=dev
            if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
              dependency_group=torch
            fi
            python -m pip install -e ./paimon-python --group "./paimon-python/pyproject.toml:${dependency_group}"
          fi
          python -m pip check

      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -e pytest_torch

  rust-plan:
    name: Native CI
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:3.11-slim"
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          apt-get update && apt-get install -y --no-install-recommends \
            build-essential git curl cmake pkg-config libssl-dev libclang-dev \
            && rm -rf /var/lib/apt/lists/*

      - name: Install Rust toolchain
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - name: Build paimon-rust from main
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade "git+https://github.com/apache/paimon-rust.git@main#subdirectory=bindings/python"

      - name: Install Python test dependencies
        run: |
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:dev
          python -m pip check

      - name: Verify native wheel provenance
        run: |
          python - <<'PY'
          import importlib.metadata as metadata
          import json

          import pypaimon_rust.datafusion

          distribution = metadata.distribution('pypaimon-rust')
          source = json.loads(distribution.read_text('direct_url.json'))
          print('pypaimon-rust', distribution.version,
                'commit', source['vcs_info']['commit_id'])
          PY

      - name: Run Python tests with native planning, reading, writing, updating and committing
        shell: bash
        env:
          PYPAIMON_TEST_NATIVE_PLAN: '1'
          PYPAIMON_TEST_NATIVE_READ: '1'
          PYPAIMON_TEST_NATIVE_WRITE: '1'
          PYPAIMON_TEST_NATIVE_UPDATE: '1'
          PYPAIMON_TEST_NATIVE_COMMIT: '1'
        run: |
          bash paimon-python/dev/lint-python.sh -i pytest

  torch:
    name: PyTorch / Python 3.10
    timeout-minutes: 60
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Python version
        run: python --version

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:torch
          python -m pip check
      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -i pytest_torch

  # Check core installation across Python versions, then basic Ray compatibility on 3.10.
  compatibility:
    name: Package and Ray compatibility
    timeout-minutes: 60
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          sudo apt-get update && sudo apt-get install -y --no-install-recommends \
            build-essential git curl cmake \
            && sudo rm -rf /var/lib/apt/lists/*

      - name: Verify core installation on Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - name: Install core package (3.8)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.8"

      - name: Verify core installation on Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Install core package (3.9)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.9"

      - name: Verify core installation on Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install core package (3.10)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.10"

      - name: Verify core installation on Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install core package (3.11)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.11"

      - name: Verify core installation on Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install core package (3.12)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.12"

      - name: Verify core installation on Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - name: Install core package (3.13)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.13"

      - name: Setup Python 3.10 for Ray test
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install base Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          # Use only test tools here: the dev group's Ray >=2.50 floor is for full-feature tests.
          python -m pip install './paimon-python[ray,oss,lumina,vindex,full-text,theta-sketch]' \
            --group ./paimon-python/pyproject.toml:test
      - name: Test Ray version compatibility
        run: |
          cd paimon-python
          echo "=========================================="
          echo "Testing Ray version compatibility"
          echo "=========================================="
          for ray_version in 2.44.0 2.48.0 2.50.1 2.53.0; do
            echo "Testing Ray version: $ray_version"
            python -m pip install --no-cache-dir -q ray==$ray_version
            python -c "import ray; print(f'Ray version: {ray.__version__}')"
            python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
            tests="pypaimon/tests/ray_data_test.py::RayDataTest pypaimon/tests/ray_range_join_test.py::RayRangeJoinTest pypaimon/tests/ray_vector_search_test.py::test_raw_search_matches_exact_top_k"
            tests="$tests pypaimon/tests/ray_batch_vector_search_test.py::test_raw_batch_shares_split_tasks"
            if [ "$ray_version" = "2.50.1" ]; then
              tests="$tests pypaimon/tests/ray_update_by_row_id_test.py::RayUpdateByRowIdTest::test_empty_dataset_after_transform_is_noop"
            fi
            if [ "$ray_version" = "2.44.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_legacy_map_batches_cardinality_is_unknown"
            fi
            if [ "$ray_version" = "2.53.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_metadata_respects_transform_cardinality"
            fi
            python -m pytest $tests -v --tb=short || {
              echo "Tests failed for Ray $ray_version"; python -m pip uninstall -y ray; exit 1;
            }
            python -m pip uninstall -y ray
          done
        env:
          PYTHONPATH: ${{ github.workspace }}/paimon-python