.github/workflows/extended.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Each test command below is implemented with `cargo xtask` so developers can
# run the same suite locally. Append `--explain` to print the underlying
# `cargo test` invocation without running it. Job setup, the clean working tree
# check, and `cargo clean` remain in this workflow.
#
# See `xtask/README.md` for details about DataFusion's `xtask` tooling.
# Track progress at https://github.com/apache/datafusion/issues/24487.

name: Datafusion extended tests

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# Run these required checks in the merge queue before merging into main.
# On PR updates, skip the jobs so their required checks allow queue entry
# without running the expensive suites. Do not add pull_request path filters:
# a skipped workflow would leave required checks pending.
on:
  pull_request:
  merge_group:
  push:
    branches:
      # Release branches retain post-push coverage. main is covered by the
      # merge queue, so do not repeat the tests after merging.
      - 'branch-*'
  workflow_dispatch:
    inputs:
      pr_head_sha:
        description: 'PR head SHA'
        type: string

permissions:
  contents: read
  checks: write

jobs:
  # Run extended tests (with feature 'extended_tests')
  linux-test-extended:
    name: cargo test 'extended_tests' (amd64)
    # Long-running job: run in the merge queue, on release pushes, or manually.
    if: github.event_name != 'pull_request'
    runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    # note: do not use amd/rust container to preserve disk space
    steps:
      - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3  # v2.5.0
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
          submodules: true
          fetch-depth: 1
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6  # v2.0.0
      - parallel:
          - name: Install Rust
            run: |
              curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
              source $HOME/.cargo/env
              rustup toolchain install
          - name: Install Protobuf Compiler
            run: |
              sudo apt-get update
              sudo apt-get install -y protobuf-compiler
      # For debugging, test binaries can be large.
      - name: Show available disk space
        run: |
          df -h
      - name: Run tests (excluding doctests)
        run: cargo xtask ci step test extended
      - parallel:
          - name: Verify Working Directory Clean
            run: git diff --exit-code
          - name: Cleanup
            run: cargo clean

  # Execute Criterion benchmarks once to catch panics without measuring performance.
  linux-test-microbenchmarks:
    name: microbenchmark tests (amd64)
    # Long-running job: run in the merge queue, on release pushes, or manually.
    if: github.event_name != 'pull_request'
    runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    container:
      image: amd64/rust
    steps:
      - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3  # v2.5.0
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Test microbenchmarks
        run: cargo xtask ci step test microbenchmarks

  # Check answers are correct when hash values collide
  hash-collisions:
    name: cargo test hash collisions (amd64)
    # Long-running job: run in the merge queue, on release pushes, or manually.
    if: github.event_name != 'pull_request'
    runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    container:
      image: amd64/rust
    steps:
      - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3  # v2.5.0
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Run tests
        run: |
          cargo xtask ci step test hash-collisions
          cd datafusion
          cargo clean

  sqllogictest-sqlite:
    name: "Run sqllogictests with the sqlite test suite"
    # Long-running job: run in the merge queue, on release pushes, or manually.
    if: github.event_name != 'pull_request'
    runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    container:
      image: amd64/rust
    steps:
      - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3  # v2.5.0
      - parallel:
          - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
            with:
              ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
              submodules: true
              fetch-depth: 1
          # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
          - name: Install protobuf compiler
            run: |
              apt-get update && apt-get install -y protobuf-compiler
      - name: Run sqllogictest
        run: cargo xtask ci step test sqlite