.github/workflows/extended.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Datafusion extended tests

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
#
# These jobs are not run as part of PR checks as they are time-consuming
# and should not fail often.
#
# We still run them as they provide important coverage to ensure correctness
# in the (very rare) event of a hash failure or sqlite library query failure.
on:
  push:
    branches:
      - main
      # support extended test suite for release candidate branches,
      # it is not expected to have many changes in these branches,
      # so running extended tests is not a burden
      - 'branch-*'
  # Also run for changes to some critical areas that are most likely
  # to trigger errors in extended tests
  pull_request:
    branches: [ '**' ]
    paths:
      - 'datafusion/physical*/**/*.rs'
      - 'datafusion/expr*/**/*.rs'
      - 'datafusion/optimizer/**/*.rs'
      - 'datafusion/sql/**/*.rs'
      - 'datafusion-testing'
  workflow_dispatch:
    inputs:
      pr_head_sha:
        description: 'PR head SHA'
        type: string

permissions:
  contents: read
  checks: write

jobs:

  # Check crate compiles and base cargo check passes
  linux-build-lib:
    name: linux build test
    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    # note: do not use amd/rust container to preserve disk space
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0  # v2.1.0
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
          submodules: true
          fetch-depth: 1
      - name: Install Rust
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          source $HOME/.cargo/env
          rustup toolchain install
      - name: Install Protobuf Compiler
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler
      - name: Prepare cargo build
        run: |
          cargo check --profile ci --all-targets
          cargo clean

  # Run extended tests (with feature 'extended_tests')
  linux-test-extended:
    name: cargo test 'extended_tests' (amd64)
    needs: [linux-build-lib]
    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    # note: do not use amd/rust container to preserve disk space
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0  # v2.1.0
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
          submodules: true
          fetch-depth: 1
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be  # v1.3.1
      - name: Install Rust
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          source $HOME/.cargo/env
          rustup toolchain install
      - name: Install Protobuf Compiler
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler
      # For debugging, test binaries can be large.
      - name: Show available disk space
        run: |
          df -h
      - name: Run tests (excluding doctests)
        env:
          RUST_BACKTRACE: 1
        run: |
          cargo test \
            --profile ci \
            --exclude datafusion-examples \
            --exclude datafusion-benchmarks \
            --exclude datafusion-cli \
            --workspace \
            --lib \
            --tests \
            --bins \
            --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption
      - name: Verify Working Directory Clean
        run: git diff --exit-code
      - name: Cleanup
        run: cargo clean

  # Check answers are correct when hash values collide
  hash-collisions:
    name: cargo test hash collisions (amd64)
    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    container:
      image: amd64/rust
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0  # v2.1.0
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
          submodules: true
          fetch-depth: 1
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          rust-version: stable
      - name: Run tests
        run: |
          cd datafusion
          cargo test  --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro
          cargo clean

  sqllogictest-sqlite:
    name: "Run sqllogictests with the sqlite test suite"
    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
    container:
      image: amd64/rust
    steps:
      - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0  # v2.1.0
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
          submodules: true
          fetch-depth: 1
      # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
      - name: Install protobuf compiler
        run: |
          apt-get update && apt-get install -y protobuf-compiler
      - name: Run sqllogictest
        run: |
          cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite