.github/workflows/benchmarks.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the 'License'); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Benchmarks

on:
  schedule:
    # Avoid the high-load top of the hour for scheduled workflows.
    - cron: '17 0 * * *'
  workflow_dispatch:
    inputs:
      seatunnel_ref:
        description: 'SeaTunnel branch, tag, or commit SHA to benchmark.'
        required: false
        type: string
        default: 'dev'
      benchmarks:
        description: 'Select a benchmark suite or class.'
        required: true
        type: choice
        options:
          - 'benchmarks_core'
          - 'SeaTunnelRowBenchmark'
          - 'IntermediateQueueBenchmark'
          - 'DebeziumJsonFormatBenchmark'
          - 'ProtoStuffSerializerBenchmark'
          - 'SeaTunnelPipelineBenchmark'
          - 'CheckpointingTimeBenchmark'
          - 'CheckpointStorageBenchmark'
          - 'IMapJobStorageBenchmark'
          - 'IMapDagStorageBenchmark'
          - 'IMapWalStorageBenchmark'
        default: 'benchmarks_core'
      custom_benchmarks:
        description: 'Exact JMH selector; overrides the choice. .* may exceed 240 minutes.'
        required: false
        type: string
      pr_number:
        description: 'Optional trusted PR to compare. The workflow executes its code.'
        required: false
        type: string

permissions:
  contents: read

concurrency:
  group: benchmarks-${{ github.event.inputs.seatunnel_ref || 'dev' }}-${{ github.event.inputs.pr_number || 'no-pr' }}
  cancel-in-progress: false

jobs:
  benchmark:
    name: Run benchmarks with Java ${{ matrix.java-version }}
    runs-on: ubuntu-24.04
    timeout-minutes: 240

    strategy:
      fail-fast: false
      matrix:
        java-version: [ '8', '11' ]

    env:
      BENCHMARK_SUITE: ${{ github.event_name == 'schedule' && 'benchmarks_core' || '' }}
      BENCHMARKS: ${{ github.event.inputs.benchmarks || 'benchmarks_core' }}
      CUSTOM_BENCHMARKS: ${{ github.event.inputs.custom_benchmarks || '' }}
      JAVA_VERSION: ${{ matrix.java-version }}
      PR_NUMBER: ${{ github.event.inputs.pr_number || '' }}
      SEATUNNEL_REF: ${{ github.event.inputs.seatunnel_ref || 'dev' }}
    steps:
      - name: Checkout benchmark driver
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.workflow_sha }}
          path: benchmark-driver

      - name: Checkout baseline
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.inputs.seatunnel_ref || 'dev' }}
          path: baseline

      - name: Checkout PR
        if: env.PR_NUMBER != ''
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: refs/pull/${{ github.event.inputs.pr_number }}/head
          path: candidate

      - name: Set up JDK ${{ matrix.java-version }} with Maven cache
        if: env.PR_NUMBER == ''
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: ${{ matrix.java-version }}
          cache: maven

      - name: Set up JDK ${{ matrix.java-version }} without Maven cache
        if: env.PR_NUMBER != ''
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: ${{ matrix.java-version }}

      - name: Build baseline benchmark module
        working-directory: baseline
        run: ./mvnw -Pbenchmark -pl seatunnel-benchmarks -am -DskipTests package

      - name: Build PR benchmark module
        if: env.PR_NUMBER != ''
        working-directory: candidate
        run: ./mvnw -Pbenchmark -pl seatunnel-benchmarks -am -DskipTests package

      - name: Run benchmarks and build report
        run: bash benchmark-driver/tools/benchmarks/run_benchmarks.sh

      - name: Upload benchmark result
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: seatunnel-benchmarks-java${{ matrix.java-version }}-${{ github.run_id }}
          retention-days: 90
          if-no-files-found: warn
          path: benchmark-artifacts/**