.github/workflows/cpp_extra.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: C++ Extra

on:
  push:
    branches:
      - '**'
      - '!dependabot/**'
    paths:
      - '.dockerignore'
      - '.github/workflows/check_labels.yml'
      - '.github/workflows/cpp_extra.yml'
      - '.github/workflows/cpp_windows.yml'
      - '.github/workflows/report_ci.yml'
      - 'ci/conda_env_*'
      - 'ci/docker/**'
      - 'ci/scripts/ccache_setup.sh'
      - 'ci/scripts/cpp_*'
      - 'ci/scripts/install_azurite.sh'
      - 'ci/scripts/install_gcs_testbench.sh'
      - 'ci/scripts/install_minio.sh'
      - 'ci/scripts/msys2_*'
      - 'ci/scripts/util_*'
      - 'cpp/**'
      - 'compose.yaml'
      - 'dev/archery/archery/**'
      - 'format/Flight.proto'
      - 'testing'
    tags:
      - '**'
  pull_request:
    paths:
      - '.dockerignore'
      - '.github/workflows/check_labels.yml'
      - '.github/workflows/cpp_extra.yml'
      - '.github/workflows/cpp_windows.yml'
      - '.github/workflows/report_ci.yml'
      - 'ci/conda_env_*'
      - 'ci/docker/**'
      - 'ci/scripts/ccache_setup.sh'
      - 'ci/scripts/cpp_*'
      - 'ci/scripts/install_azurite.sh'
      - 'ci/scripts/install_gcs_testbench.sh'
      - 'ci/scripts/install_minio.sh'
      - 'ci/scripts/msys2_*'
      - 'ci/scripts/util_*'
      - 'cpp/**'
      - 'compose.yaml'
      - 'dev/archery/archery/**'
      - 'format/Flight.proto'
      - 'testing'
    types:
      - labeled
      - opened
      - reopened
      - synchronize
  schedule:
    - cron: |
        0 0 * * *

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

permissions:
  actions: read
  contents: read
  pull-requests: read

jobs:
  check-labels:
    if: github.event_name != 'schedule' || github.repository == 'apache/arrow'
    uses: ./.github/workflows/check_labels.yml
    secrets: inherit
    with:
      parent-workflow: cpp_extra

  docker:
    needs: check-labels
    name: ${{ matrix.title }}
    runs-on: ${{ matrix.runs-on }}
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
    timeout-minutes: 75
    strategy:
      fail-fast: false
      matrix:
        include:
          - image: alpine-linux-cpp
            runs-on: ubuntu-latest
            title: AMD64 Alpine Linux
          - image: conda-cpp
            run-options: >-
              -e ARROW_USE_MESON=ON
            runs-on: ubuntu-latest
            title: AMD64 Ubuntu Meson
          # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved
          - continue-on-error: true
            envs:
              - DEBIAN=13
            image: debian-cpp
            run-options: >-
              -e CMAKE_CXX_STANDARD=23
            runs-on: ubuntu-latest
            title: AMD64 Debian C++23
    env:
      ARCHERY_DEBUG: 1
      ARROW_ENABLE_TIMING_TESTS: OFF
      DOCKER_VOLUME_PREFIX: ".docker/"
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Cache Docker Volumes
        uses: actions/cache@v5
        with:
          path: .docker
          key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }}
          restore-keys: extra-${{ matrix.image }}-
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: 3
      - name: Setup Archery
        run: python3 -m pip install -e dev/archery[docker]
      - name: Execute Docker Build
        continue-on-error: ${{ matrix.continue-on-error || false }}
        env:
          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
          ENVS: ${{ toJSON(matrix.envs) }}
        run: |
          # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
          sudo sysctl -w vm.mmap_rnd_bits=28
          source ci/scripts/util_enable_core_dumps.sh
          if [ "${ENVS}" != "null" ]; then
            echo "${ENVS}" | jq -r '.[]' | while read env; do
              echo "${env}" >> .env
            done
          fi
          archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }}
      - name: Docker Push
        if: >-
          success() &&
          github.event_name == 'push' &&
          github.repository == 'apache/arrow' &&
          github.ref_name == 'main'
        env:
          ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
        continue-on-error: true
        run: archery docker push ${{ matrix.image }}

  msvc-arm64:
    needs: check-labels
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
    name: ARM64 Windows 11 MSVC
    uses: ./.github/workflows/cpp_windows.yml
    with:
      arch: arm64
      os: windows-11-arm
      simd-level: NONE

  jni-linux:
    needs: check-labels
    name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }}
    runs-on: ${{ matrix.platform.runs-on }}
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
    timeout-minutes: 240
    permissions:
      # This is for using GitHub Packages for vcpkg cache
      packages: write
    strategy:
      fail-fast: false
      matrix:
        platform:
          - arch: "amd64"
            runs-on: ubuntu-latest
          - arch: "arm64v8"
            runs-on: ubuntu-24.04-arm
    env:
      ARCH: ${{ matrix.platform.arch }}
      REPO: ghcr.io/${{ github.repository }}-dev
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Free up disk space
        run: |
          ci/scripts/util_free_space.sh
      - name: Cache Docker Volumes
        uses: actions/cache@v5
        with:
          path: .docker
          key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }}
          restore-keys: jni-${{ matrix.platform.runs-on }}-
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: 3
      - name: Setup Archery
        run: python3 -m pip install -e dev/archery[docker]
      - name: Execute Docker Build
        env:
          ARCHERY_DOCKER_USER: ${{ github.actor }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
        run: |
          source ci/scripts/util_enable_core_dumps.sh
          archery docker run cpp-jni
      - name: Docker Push
        if: >-
          success() &&
          github.event_name == 'push' &&
          github.ref_name == 'main'
        env:
          ARCHERY_DOCKER_USER: ${{ github.actor }}
          ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
        continue-on-error: true
        run: archery docker push cpp-jni

  jni-macos:
    needs: check-labels
    name: JNI macOS
    runs-on: macos-14
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
    timeout-minutes: 45
    env:
      MACOSX_DEPLOYMENT_TARGET: "14.0"
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Install dependencies
        run: |
          brew bundle --file=cpp/Brewfile
          # We want to link aws-sdk-cpp statically but Homebrew's
          # aws-sdk-cpp provides only shared library. If we have
          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
          brew uninstall aws-sdk-cpp
          # We want to use bundled RE2 for static linking. If
          # Homebrew's RE2 is installed, its header file may be used.
          # We uninstall Homebrew's RE2 to ensure using bundled RE2.
          brew uninstall grpc || : # gRPC depends on RE2
          brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too
          brew uninstall re2
          # We want to use bundled Protobuf for static linking. If
          # Homebrew's Protobuf is installed, its library file may be
          # used on test  We uninstall Homebrew's Protobuf to ensure using
          # bundled Protobuf.
          brew uninstall protobuf
      - name: Prepare ccache
        run: |
          echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
      - name: Cache ccache
        uses: actions/cache@v5
        with:
          path: ccache
          key: jni-macos-${{ hashFiles('cpp/**') }}
          restore-keys: jni-macos-
      - name: CMake
        run: |
          cmake \
            -S cpp \
            -B cpp.build \
            --preset=ninja-release-jni-macos \
            -DARROW_BUILD_TESTS=ON \
            -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
      - name: Build
        run: |
          cmake --build cpp.build
      - name: Install
        run: |
          cmake --install cpp.build
      - name: Test
        env:
          ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
          PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
        run: |
          # MinIO is required
          exclude_tests="arrow-s3fs-test"
          # unstable
          exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
          exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
          ctest \
            --exclude-regex "${exclude_tests}" \
            --label-regex unittest \
            --output-on-failure \
            --parallel "$(sysctl -n hw.ncpu)" \
            --test-dir "cpp.build" \
            --timeout 300
      - name: Build example
        run: |
          cmake \
            -S cpp/examples/minimal_build/ \
            -B cpp/examples/minimal_build.build \
            -GNinja \
            -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
          cmake --build cpp/examples/minimal_build.build
          cd cpp/examples/minimal_build
          ../minimal_build.build/arrow-example

  odbc:
    needs: check-labels
    name: ODBC
    runs-on: windows-2022
    if: >-
      needs.check-labels.outputs.force == 'true' ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
    timeout-minutes: 240
    permissions:
      packages: write
    env:
      ARROW_BUILD_SHARED: ON
      ARROW_BUILD_STATIC: OFF
      ARROW_BUILD_TESTS: ON
      ARROW_BUILD_TYPE: release
      ARROW_DEPENDENCY_SOURCE: VCPKG
      ARROW_FLIGHT_SQL_ODBC: ON
      ARROW_FLIGHT_SQL_ODBC_INSTALLER: ON
      ARROW_HOME: /usr
      CMAKE_GENERATOR: Ninja
      CMAKE_INSTALL_PREFIX: /usr
      VCPKG_BINARY_SOURCES: 'clear;nugettimeout,600;nuget,GitHub,readwrite'
      VCPKG_DEFAULT_TRIPLET: x64-windows
    steps:
      - name: Disable Crash Dialogs
        run: |
          reg add `
            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
            /v DontShowUI `
            /t REG_DWORD `
            /d 1 `
            /f
      - name: Checkout Arrow
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Download Timezone Database
        shell: bash
        run: ci/scripts/download_tz_database.sh
      - name: Install ccache
        shell: bash
        run: |
          ci/scripts/install_ccache.sh 4.12.1 /usr
      - name: Setup ccache
        shell: bash
        run: |
          ci/scripts/ccache_setup.sh
      - name: ccache info
        id: ccache-info
        shell: bash
        run: |
          echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT
      - name: Cache ccache
        uses: actions/cache@v5
        with:
          path: ${{ steps.ccache-info.outputs.cache-dir }}
          key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }}
          restore-keys: cpp-odbc-ccache-windows-x64-
      - name: Checkout vcpkg
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          path: vcpkg
          repository: microsoft/vcpkg
      - name: Bootstrap vcpkg
        run: |
          vcpkg\bootstrap-vcpkg.bat
          $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString()
          Write-Output ${VCPKG_ROOT} | `
            Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append
          Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | `
            Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append
      - name: Setup NuGet credentials for vcpkg caching
        shell: bash
        run: |
          $(vcpkg fetch nuget | tail -n 1) \
            sources add \
            -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \
            -storepasswordincleartext \
            -name "GitHub" \
            -username "$GITHUB_REPOSITORY_OWNER" \
            -password "${{ secrets.GITHUB_TOKEN }}"
          $(vcpkg fetch nuget | tail -n 1) \
            setapikey "${{ secrets.GITHUB_TOKEN }}" \
            -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
      - name: Build
        shell: cmd
        run: |
          set VCPKG_ROOT_KEEP=%VCPKG_ROOT%
          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
          set VCPKG_ROOT=%VCPKG_ROOT_KEEP%
          bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
      - name: Register Flight SQL ODBC Driver
        shell: cmd
        run: |
          call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll
      # GH-48270 TODO: Resolve segementation fault during Arrow library unload
      # GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI
      # GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved.

      - name: Install WiX Toolset
        shell: pwsh
        run: |
          Invoke-WebRequest -Uri https://github.com/wixtoolset/wix/releases/download/v6.0.0/wix-cli-x64.msi -OutFile wix-cli-x64.msi
          Start-Process -FilePath wix-cli-x64.msi -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
          echo "C:\Program Files\WiX Toolset v6.0\bin\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      - name: Build MSI ODBC installer
        shell: pwsh
        run: |
          # Verify WiX version
          wix --version
          cd build/cpp
          cpack
      - name: Upload the artifacts to the job
        uses: actions/upload-artifact@v6
        with:
          name: flight-sql-odbc-msi-installer
          path: build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi
          if-no-files-found: error
      # Upload ODBC installer as nightly release in scheduled runs
      - name: Prepare ODBC installer for sync
        if: github.event_name == 'schedule'
        run: |
          mkdir odbc-installer
          Move-Item "build/cpp/Apache Arrow Flight SQL ODBC-*-win64.msi" odbc-installer/
          tree odbc-installer /f
      - name: Sync to Remote
        if: github.event_name == 'schedule'
        uses: ./.github/actions/sync-nightlies
        with:
          upload: true
          switches: -avzh --update --delete --progress
          local_path: odbc-installer
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/odbc
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
          remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }}

  report-extra-cpp:
    if: github.event_name == 'schedule' && always()
    needs:
      - docker
      - jni-linux
      - jni-macos
      - msvc-arm64
      - odbc
    uses: ./.github/workflows/report_ci.yml
    secrets: inherit