| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| name: Python |
| |
| on: |
| push: |
| branches: |
| - '**' |
| - '!dependabot/**' |
| tags: |
| - '**' |
| paths: |
| - '.dockerignore' |
| - '.github/workflows/python.yml' |
| - 'ci/**' |
| - 'cpp/**' |
| - 'compose.yaml' |
| - 'docs/source/python/**' |
| - 'python/**' |
| pull_request: |
| paths: |
| - '.dockerignore' |
| - '.github/workflows/python.yml' |
| - 'ci/**' |
| - 'cpp/**' |
| - 'compose.yaml' |
| - 'docs/source/python/**' |
| - 'python/**' |
| |
| concurrency: |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} |
| cancel-in-progress: true |
| |
| permissions: |
| contents: read |
| |
| env: |
| ARCHERY_DEBUG: 1 |
| DOCKER_VOLUME_PREFIX: ".docker/" |
| |
| jobs: |
| |
| docker: |
| name: ${{ matrix.title }} |
| runs-on: ubuntu-latest |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} |
| timeout-minutes: 60 |
| strategy: |
| fail-fast: false |
| max-parallel: 20 |
| matrix: |
| name: |
| - conda-python-docs |
| - conda-python-3.12-nopandas |
| - conda-python-3.11-pandas-2.0.3 |
| - conda-python-3.14-pandas-latest |
| - conda-python-3.13-no-numpy |
| include: |
| - name: conda-python-docs |
| image: conda-python-docs |
| title: AMD64 Conda Python 3.12 Sphinx & Numpydoc |
| python: "3.12" |
| - name: conda-python-3.12-nopandas |
| image: conda-python |
| title: AMD64 Conda Python 3.12 Without Pandas |
| python: "3.12" |
| - name: conda-python-3.11-pandas-2.0.3 |
| image: conda-python-pandas |
| title: AMD64 Conda Python 3.11 Pandas 2.0.3 |
| python: "3.11" |
| pandas: "2.0.3" |
| numpy: "1.23.2" |
| - name: conda-python-3.14-pandas-latest |
| image: conda-python-pandas |
| title: AMD64 Conda Python 3.14 Pandas latest |
| python: "3.14" |
| pandas: latest |
| - name: conda-python-3.13-no-numpy |
| image: conda-python-no-numpy |
| title: AMD64 Conda Python 3.13 without NumPy |
| python: "3.13" |
| env: |
| PYTHON: ${{ matrix.python || 3.11 }} |
| UBUNTU: ${{ matrix.ubuntu || 22.04 }} |
| PANDAS: ${{ matrix.pandas || 'latest' }} |
| NUMPY: ${{ matrix.numpy || 'latest' }} |
| steps: |
| - name: Checkout Arrow |
| uses: actions/checkout@v7 |
| with: |
| persist-credentials: false |
| fetch-depth: 0 |
| submodules: recursive |
| - name: Restore Docker Volumes |
| uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: .docker |
| key: python-${{ matrix.name }} |
| - name: Setup Python |
| uses: actions/setup-python@v7 |
| with: |
| python-version: 3.12 |
| - name: Setup Archery |
| run: pip install -e dev/archery[docker] |
| - name: Execute Docker Build |
| env: |
| ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} |
| run: | |
| source ci/scripts/util_enable_core_dumps.sh |
| archery docker run ${{ matrix.image }} |
| - name: Save Docker Volumes |
| if: ${{ !cancelled() }} |
| continue-on-error: true |
| uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: .docker |
| key: python-${{ matrix.name }} |
| include-hidden-files: true |
| - name: Docker Push |
| if: >- |
| success() && |
| github.event_name == 'push' && |
| github.repository == 'apache/arrow' && |
| github.ref_name == 'main' |
| env: |
| ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} |
| ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} |
| continue-on-error: true |
| run: archery docker push ${{ matrix.image }} |
| |
| macos: |
| name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3 |
| runs-on: macos-${{ matrix.macos-version }} |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} |
| timeout-minutes: 60 |
| strategy: |
| fail-fast: false |
| max-parallel: 20 |
| matrix: |
| include: |
| - architecture: AMD64 |
| macos-version: "15-intel" |
| large-memory-tests: "OFF" |
| - architecture: ARM64 |
| macos-version: "14" |
| large-memory-tests: "ON" |
| env: |
| ARROW_HOME: /tmp/local |
| ARROW_AZURE: ON |
| ARROW_DATASET: ON |
| ARROW_FLIGHT: ON |
| ARROW_GANDIVA: ON |
| ARROW_GCS: OFF |
| ARROW_HDFS: ON |
| ARROW_JEMALLOC: ON |
| ARROW_ORC: ON |
| ARROW_PARQUET: ON |
| PARQUET_REQUIRE_ENCRYPTION: ON |
| ARROW_PYTHON: ON |
| ARROW_S3: ON |
| ARROW_SUBSTRAIT: ON |
| ARROW_WITH_ZLIB: ON |
| ARROW_WITH_LZ4: ON |
| ARROW_WITH_BZ2: ON |
| ARROW_WITH_ZSTD: ON |
| ARROW_WITH_SNAPPY: ON |
| ARROW_WITH_BROTLI: ON |
| ARROW_BUILD_TESTS: OFF |
| PYARROW_TEST_LARGE_MEMORY: ${{ matrix.large-memory-tests }} |
| PYTEST_ARGS: "-n auto --durations=40" |
| # Current oldest supported version according to https://endoflife.date/macos |
| MACOSX_DEPLOYMENT_TARGET: 12.0 |
| steps: |
| - name: Checkout Arrow |
| uses: actions/checkout@v7 |
| with: |
| persist-credentials: false |
| fetch-depth: 0 |
| submodules: recursive |
| - name: Setup Python |
| uses: actions/setup-python@v7 |
| with: |
| python-version: '3.11' |
| - name: Install Dependencies |
| shell: bash |
| run: | |
| # Workaround for https://github.com/grpc/grpc/issues/41755 |
| # Remove once the runner ships a newer Homebrew. |
| brew update |
| brew bundle --file=cpp/Brewfile |
| python -m pip install \ |
| -r python/requirements-build.txt \ |
| -r python/requirements-test.txt |
| - name: Install MinIO |
| run: | |
| $(brew --prefix bash)/bin/bash \ |
| ci/scripts/install_minio.sh latest /usr/local |
| - name: Setup ccache |
| shell: bash |
| run: ci/scripts/ccache_setup.sh |
| - name: ccache info |
| id: ccache-info |
| shell: bash |
| run: echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT |
| - name: Restore ccache |
| uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ steps.ccache-info.outputs.cache-dir }} |
| key: python-ccache-macos-${{ matrix.macos-version }} |
| - name: Build |
| shell: bash |
| run: | |
| if [ "${{ matrix.macos-version }}" = "15-intel" ]; then |
| # This is a workaround. |
| # |
| # Homebrew uses /usr/local as prefix. So packages |
| # installed by Homebrew also use /usr/local/include. We |
| # want to include headers for packages installed by |
| # Homebrew as system headers to ignore warnings in them. |
| # But "-isystem /usr/local/include" isn't used by CMake |
| # because /usr/local/include is marked as the default |
| # include path. So we disable -Werror to avoid build error |
| # by warnings from packages installed by Homebrew. |
| export BUILD_WARNING_LEVEL=PRODUCTION |
| fi |
| |
| python -m pip install wheel |
| ci/scripts/cpp_build.sh $(pwd) $(pwd)/build |
| ci/scripts/python_build.sh $(pwd) $(pwd)/build |
| - name: Save ccache |
| if: ${{ !cancelled() }} |
| continue-on-error: true |
| uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ steps.ccache-info.outputs.cache-dir }} |
| key: python-ccache-macos-${{ matrix.macos-version }} |
| - name: Test |
| shell: bash |
| run: ci/scripts/python_test.sh $(pwd) $(pwd)/build |
| |
| windows: |
| name: AMD64 Windows 2022 Python 3.13 |
| runs-on: windows-2022 |
| if: ${{ !contains(github.event.pull_request.title, 'WIP') }} |
| timeout-minutes: 60 |
| env: |
| CCACHE_DIR: C:/ccache |
| PYTHON_CMD: "py -3.13" |
| steps: |
| - name: Disable Crash Dialogs |
| run: | |
| reg add ` |
| "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` |
| /v DontShowUI ` |
| /t REG_DWORD ` |
| /d 1 ` |
| /f |
| - name: Checkout Arrow |
| uses: actions/checkout@v7 |
| with: |
| persist-credentials: false |
| fetch-depth: 0 |
| submodules: recursive |
| - name: Setup Python |
| uses: actions/setup-python@v7 |
| with: |
| python-version: 3.13 |
| - name: Install ccache |
| shell: bash |
| run: | |
| ci/scripts/install_ccache.sh 4.13.6 /usr |
| - name: Setup ccache |
| shell: bash |
| run: | |
| ci/scripts/ccache_setup.sh |
| - name: Generate path variables |
| id: path-info |
| shell: bash |
| run: | |
| echo "usr-windows-dir="$(cygpath --absolute --windows /usr)"" >> $GITHUB_OUTPUT |
| - name: Restore ccache |
| uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ env.CCACHE_DIR }} |
| key: python-ccache-windows |
| - name: Build Arrow C++ and PyArrow |
| shell: cmd |
| env: |
| USR_WINDOWS_DIR: ${{ steps.path-info.outputs.usr-windows-dir }} |
| run: | |
| call "ci\scripts\python_build.bat" %cd% "%USR_WINDOWS_DIR%" |
| - name: Save ccache |
| if: ${{ !cancelled() }} |
| continue-on-error: true |
| uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53 |
| with: |
| path: ${{ env.CCACHE_DIR }} |
| key: python-ccache-windows |
| - name: Test PyArrow |
| shell: cmd |
| run: | |
| call "ci\scripts\python_test.bat" %cd% |
