.github/workflows/test.yml - datafusion-python - Git at Google

 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.

 # Reusable workflow for running tests.
 # Single matrix covers both GIL (abi3 wheel) and free-threaded
 # (per-interpreter wheels) builds.

 name: Test

 on:
   workflow_call:

 env:
   UV_LOCKED: true

 jobs:
   test-matrix:
     runs-on: ubuntu-latest
     # Backstop: a hung multiprocessing worker (e.g. during a pickle regression)
     # should not block CI longer than this.
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         include:
           # GIL builds — all share the same abi3 wheel.
           - { python-version: "3.10", wheel-tag: "abi3", freethreaded: false }
           - { python-version: "3.11", wheel-tag: "abi3", freethreaded: false }
           - { python-version: "3.12", wheel-tag: "abi3", freethreaded: false }
           - { python-version: "3.13", wheel-tag: "abi3", freethreaded: false }
           - { python-version: "3.14", wheel-tag: "abi3", freethreaded: false }
           # Free-threaded builds — one wheel per interpreter.
           - { python-version: "3.14t", wheel-tag: "3.14t", freethreaded: true }
     steps:
       - uses: actions/checkout@v7

       - name: Setup Python
         id: setup-python
         uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
           freethreaded: ${{ matrix.freethreaded }}

       - name: Cache Cargo
         uses: actions/cache@v6
         with:
           path: ~/.cargo
           key: cargo-cache-stable-${{ hashFiles('Cargo.lock') }}

       - name: Install dependencies
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
         with:
           enable-cache: true

       - name: Download pre-built Linux wheel
         uses: actions/download-artifact@v8
         with:
           name: dist-manylinux-x86_64-${{ matrix.wheel-tag }}
           path: wheels/

       # The example wheels are abi3 and build.yml builds them under its abi3
       # entry only, so there is nothing to download for a free-threaded entry.
       - name: Download pre-built example test wheels
         if: matrix.wheel-tag == 'abi3'
         uses: actions/download-artifact@v8
         with:
           name: test-example-wheels-x86_64
           path: wheels/

       - name: Install from pre-built wheels
         run: |
           set -x
           # Create the venv with the setup-python interpreter, then point
           # every uv command explicitly at the venv's own interpreter.
           # uv's interpreter discovery skips free-threaded builds unless
           # asked by exact path, so plain `uv sync` (even with an activated
           # venv or --active) re-picks the system 3.12 and recreates .venv.
           # Targeting .venv/bin/python keeps sync and pip install in the
           # same 3.14t environment as the cp314t wheel.
           uv venv --python "${{ steps.setup-python.outputs.python-path }}"
           VENV_PY="$PWD/.venv/bin/python"
           uv sync --python "$VENV_PY" --dev --no-install-package datafusion
           # Search recursively: the example artifact bundles more than one
           # project, so upload-artifact keeps a `<project>/dist/` prefix and
           # the wheels are not all at the top of wheels/.
           WHEELS=$(find wheels/ -name "*.whl")
           if [ -n "$WHEELS" ]; then
             echo "Installing wheels:"
             echo "$WHEELS"
             # shellcheck disable=SC2086  # intentional split on newlines
             uv pip install --python "$VENV_PY" $WHEELS
           else
             echo "ERROR: No wheels found!"
             exit 1
           fi

       - name: Run tests
         env:
           RUST_BACKTRACE: 1
           # On free-threaded interpreters, fail loud if any C extension
           # re-enables the GIL implicitly.
           PYTHON_GIL: ${{ matrix.freethreaded && '0' || '' }}
         run: |
           git submodule update --init
           # Use the .venv interpreter directly; uv discovery would skip the
           # free-threaded build and re-pick the system 3.12 (see install step).
           uv run --python "$PWD/.venv/bin/python" --no-project pytest -v --import-mode=importlib

       # Gated to the abi3 entries because that is where the example wheels
       # exist -- see the download step above -- and not in order to run these
       # once. Every entry that has wheels runs them, which is the point rather
       # than a cost to trim: the wheels are abi3, so the extension binaries
       # are identical across those entries and the interpreter underneath is
       # the only variable. That is what catches a build claiming the stable
       # ABI without honouring it, and what exercises cloudpickle's
       # version-sensitive bytecode handling in the distributed example's
       # worker round trip.
       #
       # The free-threaded entries are the gap this leaves, and the one worth
       # closing: nothing here exercises the capsule protocol without the GIL.
       # It needs per-interpreter example wheels in build.yml first.
       - name: Example library tests
         if: matrix.wheel-tag == 'abi3'
         run: |
           cd examples/datafusion-ffi-example
           uv run --no-project pytest python/tests/_test*.py
           cd ../datafusion-ffi-query-planner-example
           uv run --no-project pytest python/tests/_test*.py
           # The distributed example. Its tests spawn worker processes with
           # `sys.executable`, so they need the same interpreter the wheels
           # were installed into -- which `uv run` gives them.
           cd ../distributed/storage-library
           uv run --no-project pytest python/tests/_test*.py
           cd ../udf-library
           uv run --no-project pytest python/tests/_test*.py
           cd ../engine-library
           uv run --no-project pytest python/tests/_test*.py

       - name: Run tpchgen-cli to create 1 Gb dataset
         if: matrix.wheel-tag == 'abi3'
         run: |
           mkdir examples/tpch/data
           cd examples/tpch/data
           uv pip install tpchgen-cli
           uv run --no-project tpchgen-cli -s 1 --format=parquet

       - name: Run TPC-H examples
         if: matrix.wheel-tag == 'abi3'
         run: |
           cd examples/tpch
           uv run --no-project pytest _tests.py

       # The script the distributed example's README advertises, run exactly as
       # a reader would copy-paste it -- so that entry point cannot rot while
       # its test suite stays green. It generates its own `lineitem` rather than
       # reading the dataset above, which is why `tpchgen-cli` is installed here
       # too: the step is meant to stand on its own if the TPC-H steps move or
       # go away. The scale factor keeps generation and the worker round trip to
       # seconds; correctness on the full shape is the pytest suites' job.
       - name: Run distributed TPC-H example
         if: matrix.wheel-tag == 'abi3'
         run: |
           uv pip install tpchgen-cli
           uv run --no-project python examples/distributed/run_tpch.py --scale 0.1
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	# Reusable workflow for running tests.
	# Single matrix covers both GIL (abi3 wheel) and free-threaded
	# (per-interpreter wheels) builds.

	name: Test

	on:
	workflow_call:

	env:
	UV_LOCKED: true

	jobs:
	test-matrix:
	runs-on: ubuntu-latest
	# Backstop: a hung multiprocessing worker (e.g. during a pickle regression)
	# should not block CI longer than this.
	timeout-minutes: 30
	strategy:
	fail-fast: false
	matrix:
	include:
	# GIL builds — all share the same abi3 wheel.
	- { python-version: "3.10", wheel-tag: "abi3", freethreaded: false }
	- { python-version: "3.11", wheel-tag: "abi3", freethreaded: false }
	- { python-version: "3.12", wheel-tag: "abi3", freethreaded: false }
	- { python-version: "3.13", wheel-tag: "abi3", freethreaded: false }
	- { python-version: "3.14", wheel-tag: "abi3", freethreaded: false }
	# Free-threaded builds — one wheel per interpreter.
	- { python-version: "3.14t", wheel-tag: "3.14t", freethreaded: true }
	steps:
	- uses: actions/checkout@v7

	- name: Setup Python
	id: setup-python
	uses: actions/setup-python@v7
	with:
	python-version: ${{ matrix.python-version }}
	freethreaded: ${{ matrix.freethreaded }}

	- name: Cache Cargo
	uses: actions/cache@v6
	with:
	path: ~/.cargo
	key: cargo-cache-stable-${{ hashFiles('Cargo.lock') }}

	- name: Install dependencies
	uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
	with:
	enable-cache: true

	- name: Download pre-built Linux wheel
	uses: actions/download-artifact@v8
	with:
	name: dist-manylinux-x86_64-${{ matrix.wheel-tag }}
	path: wheels/

	# The example wheels are abi3 and build.yml builds them under its abi3
	# entry only, so there is nothing to download for a free-threaded entry.
	- name: Download pre-built example test wheels
	if: matrix.wheel-tag == 'abi3'
	uses: actions/download-artifact@v8
	with:
	name: test-example-wheels-x86_64
	path: wheels/

	- name: Install from pre-built wheels
	run: \|
	set -x
	# Create the venv with the setup-python interpreter, then point
	# every uv command explicitly at the venv's own interpreter.
	# uv's interpreter discovery skips free-threaded builds unless
	# asked by exact path, so plain `uv sync` (even with an activated
	# venv or --active) re-picks the system 3.12 and recreates .venv.
	# Targeting .venv/bin/python keeps sync and pip install in the
	# same 3.14t environment as the cp314t wheel.
	uv venv --python "${{ steps.setup-python.outputs.python-path }}"
	VENV_PY="$PWD/.venv/bin/python"
	uv sync --python "$VENV_PY" --dev --no-install-package datafusion
	# Search recursively: the example artifact bundles more than one
	# project, so upload-artifact keeps a `<project>/dist/` prefix and
	# the wheels are not all at the top of wheels/.
	WHEELS=$(find wheels/ -name "*.whl")
	if [ -n "$WHEELS" ]; then
	echo "Installing wheels:"
	echo "$WHEELS"
	# shellcheck disable=SC2086 # intentional split on newlines
	uv pip install --python "$VENV_PY" $WHEELS
	else
	echo "ERROR: No wheels found!"
	exit 1
	fi

	- name: Run tests
	env:
	RUST_BACKTRACE: 1
	# On free-threaded interpreters, fail loud if any C extension
	# re-enables the GIL implicitly.
	PYTHON_GIL: ${{ matrix.freethreaded && '0' \|\| '' }}
	run: \|
	git submodule update --init
	# Use the .venv interpreter directly; uv discovery would skip the
	# free-threaded build and re-pick the system 3.12 (see install step).
	uv run --python "$PWD/.venv/bin/python" --no-project pytest -v --import-mode=importlib

	# Gated to the abi3 entries because that is where the example wheels
	# exist -- see the download step above -- and not in order to run these
	# once. Every entry that has wheels runs them, which is the point rather
	# than a cost to trim: the wheels are abi3, so the extension binaries
	# are identical across those entries and the interpreter underneath is
	# the only variable. That is what catches a build claiming the stable
	# ABI without honouring it, and what exercises cloudpickle's
	# version-sensitive bytecode handling in the distributed example's
	# worker round trip.
	#
	# The free-threaded entries are the gap this leaves, and the one worth
	# closing: nothing here exercises the capsule protocol without the GIL.
	# It needs per-interpreter example wheels in build.yml first.
	- name: Example library tests
	if: matrix.wheel-tag == 'abi3'
	run: \|
	cd examples/datafusion-ffi-example
	uv run --no-project pytest python/tests/_test*.py
	cd ../datafusion-ffi-query-planner-example
	uv run --no-project pytest python/tests/_test*.py
	# The distributed example. Its tests spawn worker processes with
	# `sys.executable`, so they need the same interpreter the wheels
	# were installed into -- which `uv run` gives them.
	cd ../distributed/storage-library
	uv run --no-project pytest python/tests/_test*.py
	cd ../udf-library
	uv run --no-project pytest python/tests/_test*.py
	cd ../engine-library
	uv run --no-project pytest python/tests/_test*.py

	- name: Run tpchgen-cli to create 1 Gb dataset
	if: matrix.wheel-tag == 'abi3'
	run: \|
	mkdir examples/tpch/data
	cd examples/tpch/data
	uv pip install tpchgen-cli
	uv run --no-project tpchgen-cli -s 1 --format=parquet

	- name: Run TPC-H examples
	if: matrix.wheel-tag == 'abi3'
	run: \|
	cd examples/tpch
	uv run --no-project pytest _tests.py

	# The script the distributed example's README advertises, run exactly as
	# a reader would copy-paste it -- so that entry point cannot rot while
	# its test suite stays green. It generates its own `lineitem` rather than
	# reading the dataset above, which is why `tpchgen-cli` is installed here
	# too: the step is meant to stand on its own if the TPC-H steps move or
	# go away. The scale factor keeps generation and the worker round trip to
	# seconds; correctness on the full shape is the pytest suites' job.
	- name: Run distributed TPC-H example
	if: matrix.wheel-tag == 'abi3'
	run: \|
	uv pip install tpchgen-cli
	uv run --no-project python examples/distributed/run_tpch.py --scale 0.1