GH-1244: Move integration tests to separate workflow (#1245)
## What's Changed
Move the `integration` job from `test.yml` into its own
`integration.yml` workflow.
The new workflow uses top-level `paths` filters, so integration tests
run only when changes affect relevant code or build inputs.
This should also help to reduce the amount of runner time. Topic
initially triggered in the [mailing
list](https://lists.apache.org/thread/drf0o5kzg1zfmok7gc09k8qz8hh9ymvh)
Closes #1244
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 0000000..3872d03
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Integration
+
+on:
+ push:
+ branches:
+ - '**'
+ - '!dependabot/**'
+ tags:
+ - '**'
+ paths:
+ - '.github/workflows/integration.yml'
+ - '**/pom.xml'
+ - 'c/**'
+ - 'ci/scripts/**'
+ - 'compose.yaml'
+ - 'flight/**'
+ - 'format/**'
+ - 'testing/data/**'
+ - 'vector/**'
+ pull_request:
+ paths:
+ - '.github/workflows/integration.yml'
+ - '**/pom.xml'
+ - 'c/**'
+ - 'ci/scripts/**'
+ - 'compose.yaml'
+ - 'flight/**'
+ - 'format/**'
+ - 'testing/data/**'
+ - 'vector/**'
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+env:
+ DOCKER_VOLUME_PREFIX: ".docker/"
+
+jobs:
+ integration:
+ name: AMD64 integration
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - name: Checkout Arrow
+ uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+ repository: apache/arrow
+ submodules: recursive
+ - name: Checkout Arrow Rust
+ uses: actions/checkout@v7
+ with:
+ repository: apache/arrow-rs
+ path: rust
+ - name: Checkout Arrow nanoarrow
+ uses: actions/checkout@v7
+ with:
+ repository: apache/arrow-nanoarrow
+ path: nanoarrow
+ - name: Checkout Arrow .NET
+ uses: actions/checkout@v7
+ with:
+ repository: apache/arrow-dotnet
+ path: dotnet
+ - name: Checkout Arrow Go
+ uses: actions/checkout@v7
+ with:
+ repository: apache/arrow-go
+ path: go
+ - name: Checkout Arrow Java
+ uses: actions/checkout@v7
+ with:
+ path: java
+ - name: Checkout Arrow JavaScript
+ uses: actions/checkout@v7
+ with:
+ repository: apache/arrow-js
+ path: js
+ - name: Free up disk space
+ run: |
+ ci/scripts/util_free_space.sh
+ - name: Cache Docker Volumes
+ uses: actions/cache@v6
+ with:
+ path: .docker
+ key: integration-conda-${{ hashFiles('cpp/**') }}
+ restore-keys: integration-conda-
+ - name: Setup Python
+ uses: actions/setup-python@v6
+ with:
+ python-version: 3.12
+ - name: Setup Archery
+ run: pip install -e dev/archery[docker]
+ - name: Execute Docker Build
+ run: |
+ source ci/scripts/util_enable_core_dumps.sh
+ archery docker run \
+ -e ARCHERY_DEFAULT_BRANCH=main \
+ -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \
+ -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \
+ -e ARCHERY_INTEGRATION_WITH_GO=1 \
+ -e ARCHERY_INTEGRATION_WITH_JAVA=1 \
+ -e ARCHERY_INTEGRATION_WITH_JS=1 \
+ -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \
+ -e ARCHERY_INTEGRATION_WITH_RUST=1 \
+ conda-integration
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 853b7ce..653b16f 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -147,72 +147,3 @@
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
run: ci/scripts/test.sh . build jni
-
- integration:
- name: AMD64 integration
- runs-on: ubuntu-latest
- timeout-minutes: 60
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v7
- with:
- fetch-depth: 0
- repository: apache/arrow
- submodules: recursive
- - name: Checkout Arrow Rust
- uses: actions/checkout@v7
- with:
- repository: apache/arrow-rs
- path: rust
- - name: Checkout Arrow nanoarrow
- uses: actions/checkout@v7
- with:
- repository: apache/arrow-nanoarrow
- path: nanoarrow
- - name: Checkout Arrow .NET
- uses: actions/checkout@v7
- with:
- repository: apache/arrow-dotnet
- path: dotnet
- - name: Checkout Arrow Go
- uses: actions/checkout@v7
- with:
- repository: apache/arrow-go
- path: go
- - name: Checkout Arrow Java
- uses: actions/checkout@v7
- with:
- path: java
- - name: Checkout Arrow JavaScript
- uses: actions/checkout@v7
- with:
- repository: apache/arrow-js
- path: js
- - name: Free up disk space
- run: |
- ci/scripts/util_free_space.sh
- - name: Cache Docker Volumes
- uses: actions/cache@v6
- with:
- path: .docker
- key: integration-conda-${{ hashFiles('cpp/**') }}
- restore-keys: integration-conda-
- - name: Setup Python
- uses: actions/setup-python@v7
- with:
- python-version: 3.12
- - name: Setup Archery
- run: pip install -e dev/archery[docker]
- - name: Execute Docker Build
- run: |
- source ci/scripts/util_enable_core_dumps.sh
- archery docker run \
- -e ARCHERY_DEFAULT_BRANCH=main \
- -e ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS=java \
- -e ARCHERY_INTEGRATION_WITH_DOTNET=1 \
- -e ARCHERY_INTEGRATION_WITH_GO=1 \
- -e ARCHERY_INTEGRATION_WITH_JAVA=1 \
- -e ARCHERY_INTEGRATION_WITH_JS=1 \
- -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \
- -e ARCHERY_INTEGRATION_WITH_RUST=1 \
- conda-integration
