| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Usage |
| # ----- |
| # |
| # The docker compose file is parametrized using environment variables, the |
| # defaults are set in .env file. |
| # |
| # Example: |
| # $ ARCH=arm64v8 docker compose build ubuntu-cpp |
| # $ ARCH=arm64v8 docker compose run ubuntu-cpp |
| # |
| # |
| # Coredumps |
| # --------- |
| # |
| # In order to enable coredumps for the C++ tests run by CTest either with |
| # command `make unittest` or `ctest --output-on-failure` the correct coredump |
| # patterns must be set. |
| # The kernel settings are coming from the host, so while it can be enabled from |
| # a running container using --privileged option the change will affect all other |
| # containers, so prefer setting it explicitly, directly on the host. |
| # WARNING: setting this will affect the host machine. |
| # |
| # Linux host: |
| # $ sudo sysctl -w kernel.core_pattern=/tmp/core.%e.%p |
| # |
| # macOS host running Docker for Mac (won't persist between restarts): |
| # $ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty |
| # # echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern |
| # |
| # The setup attempts to generate coredumps by default, but the correct paths |
| # above must be set. In order to disable the coredump generation set |
| # ULIMIT_CORE environment variable to 0 before running docker compose |
| # (or by setting it in .env file): |
| # |
| # ULIMIT_CORE=0 docker compose run --rm conda-cpp |
| # |
| # See more in cpp/build-support/run-test.sh::print_coredumps |
| |
| x-ccache: &ccache |
| CCACHE_COMPILERCHECK: content |
| CCACHE_COMPRESS: 1 |
| CCACHE_COMPRESSLEVEL: 6 |
| CCACHE_MAXSIZE: 1G |
| CCACHE_DIR: /ccache |
| |
| x-common: &common |
| GITHUB_ACTIONS: |
| |
| x-cpp: &cpp |
| ARROW_RUNTIME_SIMD_LEVEL: |
| ARROW_SIMD_LEVEL: |
| |
| x-sccache: &sccache |
| AWS_ACCESS_KEY_ID: |
| AWS_SECRET_ACCESS_KEY: |
| SCCACHE_BUCKET: |
| SCCACHE_REGION: |
| SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache} |
| |
| x-vcpkg-build-secrets: &vcpkg-build-secrets |
| - github_repository_owner |
| - github_token |
| - vcpkg_binary_sources |
| |
| # CPU/memory limit presets to pass to Docker. |
| # |
| # Usage: archery docker run --resource-limit=github <image> |
| # |
| # Note that exporting ARCHERY_DOCKER_BIN="sudo docker" is likely required, |
| # unless Docker is configured with cgroups v2 (else Docker will silently |
| # ignore the limits). |
| x-limit-presets: |
| # These values emulate GitHub Actions: |
| # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners |
| github: |
| # Note we use cpuset and not cpus since Ninja only detects and limits |
| # parallelism given the former |
| cpuset_cpus: [0, 1] |
| memory: 7g |
| |
| x-with-gpus: |
| - ubuntu-cuda-cpp |
| - ubuntu-cuda-python |
| |
| x-hierarchy: |
| # This section is used by the archery tool to enable building nested images, |
| # so it is enough to call: |
| # archery docker run debian-ruby |
| # instead of a sequence of docker compose commands: |
| # docker compose build debian-cpp |
| # docker compose build debian-c-glib |
| # docker compose build debian-ruby |
| # docker compose run --rm debian-ruby |
| # |
| # Each node must be either a string scalar of a list containing the |
| # descendant images if any. Archery checks that all node has a corresponding |
| # service entry, so any new image/service must be listed here. |
| - almalinux-verify-rc |
| - alpine-linux-cpp |
| - alpine-linux-r |
| - conda: |
| - conda-cpp: |
| - conda-integration |
| - conda-cpp-valgrind |
| - conda-python: |
| - conda-python-pandas: |
| - conda-python-docs |
| - conda-python-cpython-debug |
| - conda-python-dask |
| - conda-python-emscripten |
| - conda-python-hdfs |
| - conda-python-no-numpy |
| - conda-python-spark |
| - conda-verify-rc |
| - conan |
| - cpp-jni |
| - debian-cpp: |
| - debian-c-glib: |
| - debian-ruby |
| - debian-python: |
| - debian-docs |
| - fedora-cpp: |
| - fedora-python |
| - python-sdist |
| - ubuntu-cpp: |
| - ubuntu-cpp-static |
| - ubuntu-c-glib: |
| - ubuntu-ruby |
| - ubuntu-python |
| - ubuntu-python-sdist-test |
| - ubuntu-python-313-freethreading |
| - ubuntu-r |
| - ubuntu-r-only-r |
| - ubuntu-cpp-bundled |
| - ubuntu-cpp-bundled-offline |
| - ubuntu-cpp-minimal |
| - ubuntu-cuda-cpp: |
| - ubuntu-cuda-python |
| - ubuntu-cpp-sanitizer |
| - ubuntu-cpp-thread-sanitizer |
| - ubuntu-cpp-emscripten |
| - ubuntu-r-valgrind |
| - ubuntu-verify-rc |
| - r |
| - r-revdepcheck |
| - r-san |
| # helper services |
| - impala |
| - postgres |
| - python-wheel-manylinux-2-28 |
| - python-wheel-musllinux-1-2 |
| - python-wheel-manylinux-test-imports |
| - python-free-threaded-wheel-manylinux-test-imports |
| - python-wheel-musllinux-test-imports |
| - python-free-threaded-wheel-musllinux-test-imports |
| - python-wheel-manylinux-test-unittests |
| - python-free-threaded-wheel-manylinux-test-unittests |
| - python-wheel-musllinux-test-unittests |
| - python-free-threaded-wheel-musllinux-test-unittests |
| - python-wheel-windows-vs2022-base: |
| - python-wheel-windows-vs2022 |
| - python-free-threaded-wheel-windows-vs2022 |
| - python-wheel-windows-test-base: |
| - python-wheel-windows-test |
| - python-free-threaded-wheel-windows-test |
| |
| volumes: |
| almalinux-ccache: |
| name: ${ARCH}-almalinux-ccache |
| alpine-linux-ccache: |
| name: ${ARCH}-alpine-linux-ccache |
| conda-ccache: |
| name: ${ARCH}-conda-ccache |
| cpp-jni-ccache: |
| name: ${ARCH}-cpp-jni-ccache |
| debian-ccache: |
| name: ${ARCH}-debian-${DEBIAN}-ccache |
| fedora-ccache: |
| name: ${ARCH}-fedora-${FEDORA}-ccache |
| maven-cache: |
| name: maven-cache |
| python-wheel-manylinux-2-28-ccache: |
| name: python-wheel-manylinux-2-28-ccache |
| python-wheel-musllinux-1-2-ccache: |
| name: python-wheel-musllinux-1-2-ccache |
| ubuntu-ccache: |
| name: ${ARCH}-ubuntu-${UBUNTU}-ccache |
| |
| services: |
| |
| ################################# C++ ####################################### |
| # Release build: |
| # docker compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|... |
| # Shared only: |
| # docker compose run -e ARROW_BUILD_STATIC=OFF conda-cpp|debian-cpp|... |
| # Static only: |
| # docker compose run \ |
| # -e ARROW_BUILD_SHARED=OFF \ |
| # -e ARROW_TEST_LINKAGE=static \ |
| # conda-cpp|debian-cpp|... |
| |
| alpine-linux-cpp: |
| # Usage: |
| # docker compose build alpine-linux-cpp |
| # docker compose run --rm alpine-linux-cpp |
| # Parameters: |
| # ALPINE_LINUX: 3.22 |
| # ARCH: amd64, arm64v8, ... |
| image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp |
| args: |
| arch: ${ARCH} |
| shm_size: &shm-size 2G |
| ulimits: &ulimits |
| core: ${ULIMIT_CORE} |
| environment: |
| <<: [*common, *ccache, *cpp] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &alpine-linux-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}alpine-linux-ccache:/ccache:delegated |
| command: >- |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build" |
| |
| conda: |
| # Base image for conda builds. |
| # |
| # Usage: |
| # docker compose build conda |
| # docker compose run --rm conda |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda |
| build: |
| context: . |
| dockerfile: ci/docker/conda.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda |
| args: |
| arch: ${ARCH} |
| volumes: |
| - .:/arrow:delegated |
| |
| conda-cpp: |
| # C++ build in conda environment, including the doxygen docs. |
| # |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose run --rm conda-cpp |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/conda-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-cpp |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_BUILD_BENCHMARKS: "ON" |
| ARROW_BUILD_EXAMPLES: "ON" |
| ARROW_BUILD_OPENMP_BENCHMARKS: "ON" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_EXTRA_ERROR_CONTEXT: "ON" |
| ARROW_MIMALLOC: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| volumes: &conda-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| command: &conda-cpp-command |
| [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build"] |
| |
| conda-cpp-valgrind: |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose run --rm conda-cpp-valgrind |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/conda-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-cpp |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| # Shrink test runtime by enabling minimal optimizations |
| ARROW_C_FLAGS_DEBUG: "-g1 -Og" |
| ARROW_CXX_FLAGS_DEBUG: "-g1 -Og" |
| # GH-39973: Do not use debug memory pool for valgrind |
| ARROW_DEBUG_MEMORY_POOL: "none" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_MIMALLOC: "OFF" |
| ARROW_RUNTIME_SIMD_LEVEL: "AVX2" # AVX512 not supported by Valgrind (ARROW-9851) |
| ARROW_TEST_MEMCHECK: "ON" |
| BUILD_WARNING_LEVEL: "PRODUCTION" |
| ARROW_CTEST_TIMEOUT: 500 |
| volumes: *conda-volumes |
| command: *conda-cpp-command |
| |
| debian-cpp: |
| # Usage: |
| # docker compose build debian-cpp |
| # docker compose run --rm debian-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 12, experimental |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| args: |
| arch: ${ARCH} |
| gcc: ${GCC} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &debian-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated |
| command: &cpp-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build" |
| |
| ubuntu-cpp: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose run --rm ubuntu-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, s390x, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| clang_tools: ${CLANG_TOOLS} |
| cmake: ${CMAKE} |
| gcc: ${GCC} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| cap_add: |
| - SYS_ADMIN |
| devices: |
| - "/dev/fuse:/dev/fuse" |
| security_opt: |
| - apparmor:unconfined |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &ubuntu-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated |
| command: *cpp-command |
| |
| ubuntu-cpp-static: |
| # Usage: |
| # docker compose build ubuntu-cpp-static |
| # docker compose run --rm ubuntu-cpp-static |
| # Parameters: |
| # ARCH: amd64, arm64v8, s390x, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| clang_tools: ${CLANG_TOOLS} |
| cmake: ${CMAKE} |
| gcc: ${GCC} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| cap_add: |
| - SYS_ADMIN |
| devices: |
| - "/dev/fuse:/dev/fuse" |
| security_opt: |
| - apparmor:unconfined |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_HOME: /arrow |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| LIBARROW_MINIMAL: "false" |
| ARROW_MIMALLOC: "ON" |
| volumes: *ubuntu-volumes |
| command: &cpp-static-command |
| /bin/bash -c " |
| cd /arrow && |
| r/inst/build_arrow_static.sh" |
| |
| ubuntu-cpp-bundled: |
| # Arrow build with BUNDLED dependencies |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| cmake: ${CMAKE} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| CMAKE_GENERATOR: "Unix Makefiles" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| |
| ubuntu-cpp-bundled-offline: |
| # Arrow build with BUNDLED dependencies with downloaded dependencies. |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *cpp] |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| ARROW_OFFLINE: ON |
| # Apache ORC always uses external orc-format. |
| ARROW_ORC: OFF |
| CMAKE_GENERATOR: "Unix Makefiles" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| |
| ubuntu-cpp-minimal: |
| # Arrow build with minimal components/dependencies |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_ACERO: "OFF" |
| ARROW_AZURE: "OFF" |
| ARROW_COMPUTE: "OFF" |
| ARROW_CSV: "OFF" |
| ARROW_DATASET: "OFF" |
| ARROW_FILESYSTEM: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_BROTLI: "OFF" |
| ARROW_WITH_BZ2: "OFF" |
| ARROW_WITH_LZ4: "OFF" |
| ARROW_WITH_SNAPPY: "OFF" |
| ARROW_WITH_ZLIB: "OFF" |
| ARROW_WITH_ZSTD: "OFF" |
| PARQUET_BUILD_EXAMPLES: "OFF" |
| PARQUET_BUILD_EXECUTABLES: "OFF" |
| PARQUET_REQUIRE_ENCRYPTION: "OFF" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| |
| ubuntu-cuda-cpp: |
| # Usage: |
| # docker compose build ubuntu-cuda-cpp |
| # docker compose run --rm ubuntu-cuda-cpp |
| # Parameters: |
| # ARCH: amd64 |
| # UBUNTU: 22.04, 24.04 |
| # CUDA: <depends on your nvidia driver, should match system CUDA> |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| args: |
| arch: ${ARCH} |
| base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} |
| clang_tools: ${CLANG_TOOLS} |
| cmake: ${CMAKE} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_ACERO: "OFF" |
| ARROW_AZURE: "OFF" |
| ARROW_COMPUTE: "OFF" |
| ARROW_CSV: "OFF" |
| ARROW_CUDA: "ON" |
| ARROW_DATASET: "OFF" |
| ARROW_ENABLE_TIMING_TESTS: "OFF" |
| ARROW_FILESYSTEM: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_JSON: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| deploy: &cuda-deploy |
| resources: |
| reservations: |
| devices: |
| - driver: nvidia |
| capabilities: [gpu] |
| |
| ubuntu-cpp-sanitizer: |
| # Usage: |
| # docker compose build ubuntu-cpp-sanitizer |
| # docker compose run --rm ubuntu-cpp-sanitizer |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, ... |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| cap_add: |
| # For LeakSanitizer |
| - SYS_PTRACE |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| cmake: ${CMAKE} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache, *cpp] |
| CC: clang-${CLANG_TOOLS} |
| CXX: clang++-${CLANG_TOOLS} |
| # Avoid creating huge static libraries |
| ARROW_BUILD_STATIC: "OFF" |
| # GH-39973: Do not use debug memory pool for ASAN |
| ARROW_DEBUG_MEMORY_POOL: "none" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| # GH-33920: Disable Flight SQL to reduce build time. |
| # We'll be able to re-enable this with Ubuntu 24.04 because |
| # Ubuntu 24.04 will ship ProtoBuf 3.15.0 or later that is required |
| # by Flight SQL. |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_FUZZING: "ON" # Check fuzz regressions |
| ARROW_JEMALLOC: "OFF" |
| ARROW_MIMALLOC: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_USE_ASAN: "ON" |
| ARROW_USE_UBSAN: "ON" |
| # 1 GB isn't enough for a single sanitizer build |
| CCACHE_MAXSIZE: 2G |
| Protobuf_SOURCE: "AUTO" |
| command: *cpp-command |
| |
| ubuntu-cpp-thread-sanitizer: |
| # Usage: |
| # docker compose build ubuntu-cpp-thread-sanitizer |
| # docker compose run --rm ubuntu-cpp-thread-sanitizer |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| CC: clang-${CLANG_TOOLS} |
| CXX: clang++-${CLANG_TOOLS} |
| ARROW_BUILD_STATIC: "OFF" |
| ARROW_CTEST_TIMEOUT: 500 |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_MIMALLOC: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_USE_TSAN: "ON" |
| command: *cpp-command |
| |
| ubuntu-cpp-emscripten: |
| # Usage: |
| # docker compose build ubuntu-cpp-emscripten |
| # docker compose run --rm ubuntu-cpp-emscripten |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| cmake: ${CMAKE} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_EMSCRIPTEN: "ON" |
| UBUNTU: |
| command: *cpp-command |
| |
| fedora-cpp: |
| # Usage: |
| # docker compose build fedora-cpp |
| # docker compose run --rm fedora-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # FEDORA: 42 |
| image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| args: |
| arch: ${ARCH} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &fedora-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated |
| command: *cpp-command |
| |
| conan: |
| # Base service for Conan. |
| # |
| # Usage: |
| # docker compose run --rm conan |
| # Parameters: |
| # CONAN_BASE: gcc11-ubuntu16.04, ... |
| # CONAN_VERSION: 2.12.1, ... |
| # See https://github.com/conan-io/conan-docker-tools#readme for |
| # available images. |
| image: conanio/${CONAN_BASE}:${CONAN_VERSION} |
| user: root:root |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *sccache] |
| volumes: |
| - .:/arrow:delegated |
| command: >- |
| /bin/bash -c " |
| sudo /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin && |
| /arrow/ci/scripts/conan_setup.sh && |
| /arrow/ci/scripts/conan_build.sh /arrow /build" |
| |
| cpp-jni: |
| # Test for the build configuration for JNI. |
| # |
| # Usage: |
| # docker compose run --rm cpp-jni |
| # Parameters: |
| # ARCH: amd64, arm64v8 |
| # ARCH_ALIAS: x86_64, aarch64 |
| # ARCH_SHORT: amd64, arm64 |
| image: ${REPO}:${ARCH}-cpp-jni-${VCPKG} |
| build: |
| args: |
| arch: ${ARCH} |
| arch_short: ${ARCH_SHORT} |
| # See available versions at: |
| # https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags |
| # https://quay.io/repository/pypa/manylinux_2_28_aarch64?tab=tags |
| base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2025.10.09-1 |
| vcpkg: ${VCPKG} |
| context: . |
| dockerfile: ci/docker/cpp-jni.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-cpp-jni-${VCPKG} |
| secrets: *vcpkg-build-secrets |
| environment: |
| <<: [*common, *ccache] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}cpp-jni-ccache:/ccache:delegated |
| command: *cpp-command |
| |
| ############################### C GLib ###################################### |
| |
| debian-c-glib: |
| # Usage: |
| # docker compose build debian-cpp |
| # docker compose build debian-c-glib |
| # docker compose run --rm debian-c-glib |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 12 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-c-glib.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| BUILD_DOCS_C_GLIB: "ON" |
| volumes: *debian-volumes |
| command: &c-glib-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_test.sh /arrow /build" |
| |
| ubuntu-c-glib: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-c-glib |
| # docker compose run --rm ubuntu-c-glib |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-c-glib.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *c-glib-command |
| |
| ############################### Ruby ######################################## |
| # Until Ruby is the only dependent implementation on top of C Glib we can |
| # test C Glib and Ruby in one pass. This is an optimization to avoid |
| # redundant (one for C GLib and one for Ruby doing the same work twice) |
| # builds on CI services. |
| |
| debian-ruby: |
| # Usage: |
| # docker compose build debian-cpp |
| # docker compose build debian-c-glib |
| # docker compose build debian-ruby |
| # docker compose run --rm debian-ruby |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 12 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-ruby.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-ruby |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| BUILD_DOCS_C_GLIB: "ON" |
| volumes: *debian-volumes |
| command: &ruby-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_test.sh /arrow /build && |
| /arrow/ci/scripts/ruby_test.sh /arrow /build" |
| |
| ubuntu-ruby: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-c-glib |
| # docker compose build ubuntu-ruby |
| # docker compose run --rm ubuntu-ruby |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-ruby.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *ruby-command |
| |
| ############################### Python ###################################### |
| |
| conda-python: |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose run --rm conda-python |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| # PYTHON: 3.10, 3.11, 3.12, 3.13 |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| PYTEST_ARGS: # inherit |
| volumes: *conda-volumes |
| command: &python-conda-command |
| [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| conda-python-emscripten: |
| # Usage: |
| # docker compose build conda-python-emscripten |
| # docker compose run --rm conda-python-emscripten |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04 |
| image: ${REPO}:${ARCH}-conda-python-emscripten |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-emscripten.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| pyodide_version: "0.26.0" |
| chrome_version: "134" |
| selenium_version: "4.15.2" |
| required_python_min: "(3,12)" |
| python: ${PYTHON} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache, *sccache, *cpp] |
| command: [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build_emscripten.sh /arrow /build && |
| /arrow/ci/scripts/python_test_emscripten.sh /build /pyodide"] |
| |
| ubuntu-cuda-python: |
| # Usage: |
| # docker compose build ubuntu-cuda-cpp |
| # docker compose build ubuntu-cuda-python |
| # docker compose run --rm ubuntu-cuda-python |
| # Parameters: |
| # ARCH: amd64 |
| # CUDA: <depends on your nvidia driver, should match system CUDA> |
| # UBUNTU: 22.04, 24.04 |
| # NUMBA: master, latest, <version> |
| # NUMBA_CUDA: master, latest, <version> |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| numba: ${NUMBA} |
| numba_cuda: ${NUMBA_CUDA} |
| cuda: ${CUDA} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_ACERO: "OFF" |
| ARROW_AZURE: "OFF" |
| ARROW_CUDA: "ON" |
| ARROW_DATASET: "OFF" |
| ARROW_ENABLE_TIMING_TESTS: "OFF" |
| ARROW_FILESYSTEM: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| SETUPTOOLS_SCM_PRETEND_VERSION: |
| volumes: *ubuntu-volumes |
| deploy: *cuda-deploy |
| command: &python-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/python_test.sh /arrow" |
| |
| debian-python: |
| # Usage: |
| # docker compose build debian-cpp |
| # docker compose build debian-python |
| # docker compose run --rm debian-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 12 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *debian-volumes |
| command: *python-command |
| |
| ubuntu-python: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-python |
| # docker compose run --rm ubuntu-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *python-command |
| |
| fedora-python: |
| # Usage: |
| # docker compose build fedora-cpp |
| # docker compose build fedora-python |
| # docker compose run --rm fedora-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # FEDORA: 42 |
| image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-dnf-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *fedora-volumes |
| command: *python-command |
| |
| ############################ Python sdist ################################### |
| |
| python-sdist: |
| # Usage: |
| # docker compose build python-sdist |
| # docker compose run --rm python-sdist |
| # Parameters: |
| # PYARROW_VERSION: The pyarrow version for sdist such as "3.0.0" |
| image: ${REPO}:python-sdist |
| build: |
| context: . |
| dockerfile: ci/docker/python-sdist.dockerfile |
| cache_from: |
| - ${REPO}:python-sdist |
| environment: |
| PYARROW_VERSION: ${PYARROW_VERSION:-} |
| volumes: |
| - .:/arrow:delegated |
| command: /arrow/ci/scripts/python_sdist_build.sh /arrow |
| |
| ubuntu-python-sdist-test: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-python-sdist-test |
| # docker compose run --rm ubuntu-python-sdist-test |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| # Bundled build of OpenTelemetry needs a git client |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| PYARROW_VERSION: ${PYARROW_VERSION:-} |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| apt remove -y git && |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_sdist_test.sh /arrow" |
| |
| ############################ Python free-threading ########################## |
| |
| ubuntu-python-313-freethreading: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-python-313-freethreading |
| # docker compose run --rm ubuntu-python-313-freethreading |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-313-freethreading.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| # Bundled build of OpenTelemetry needs a git client |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| volumes: *ubuntu-volumes |
| command: *python-command |
| |
| ############################ Python wheels ################################## |
| |
| # See available versions at: |
| # https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags |
| python-wheel-manylinux-2-28: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} |
| build: |
| args: |
| arch: ${ARCH} |
| arch_short: ${ARCH_SHORT} |
| base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2025-06-04-496f7e1 |
| manylinux: 2_28 |
| python: ${PYTHON} |
| python_abi_tag: ${PYTHON_ABI_TAG} |
| vcpkg: ${VCPKG} |
| context: . |
| dockerfile: ci/docker/python-wheel-manylinux.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} |
| secrets: *vcpkg-build-secrets |
| environment: |
| <<: [*common, *ccache] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated |
| command: /arrow/ci/scripts/python_wheel_xlinux_build.sh |
| |
| # See available versions at: |
| # https://quay.io/repository/pypa/musllinux_1_2_x86_64?tab=tags |
| python-wheel-musllinux-1-2: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} |
| build: |
| args: |
| arch: ${ARCH} |
| arch_short: ${ARCH_SHORT} |
| base: quay.io/pypa/musllinux_1_2_${ARCH_ALIAS}:2025-06-04-496f7e1 |
| musllinux: 1_2 |
| python: ${PYTHON} |
| python_abi_tag: ${PYTHON_ABI_TAG} |
| vcpkg: ${VCPKG} |
| context: . |
| dockerfile: ci/docker/python-wheel-musllinux.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-1-2-vcpkg-${VCPKG} |
| secrets: *vcpkg-build-secrets |
| environment: |
| <<: [ *common, *ccache ] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}python-wheel-musllinux-1-2-ccache:/ccache:delegated |
| command: /arrow/ci/scripts/python_wheel_xlinux_build.sh |
| |
| python-wheel-musllinux-test-imports: |
| image: python:${PYTHON_IMAGE_TAG}-alpine${ALPINE_LINUX} |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "ON" |
| CHECK_UNITTESTS: "OFF" |
| CHECK_WHEEL_CONTENT: "ON" |
| command: > |
| /bin/sh -c " |
| apk update && |
| apk add --no-cache bash && |
| /arrow/ci/scripts/python_wheel_unix_test.sh /arrow" |
| |
| # TODO: Remove this when the official Docker Python image supports the free-threaded build. |
| # See https://github.com/docker-library/python/issues/947 for more info. |
| python-free-threaded-wheel-musllinux-test-imports: |
| image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports |
| build: |
| args: |
| base: "${ARCH}/alpine:${ALPINE_LINUX}" |
| python_version: ${PYTHON} |
| build_date: "20251014" # python-build-standalone release date |
| arch: ${ARCH_ALIAS} |
| context: . |
| dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-imports |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "ON" |
| CHECK_UNITTESTS: "OFF" |
| PYTHON: ${PYTHON} |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-musllinux-test-unittests: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-test |
| build: |
| args: |
| alpine_linux: ${ALPINE_LINUX} |
| python_image_tag: ${PYTHON_IMAGE_TAG} |
| context: . |
| dockerfile: ci/docker/python-wheel-musllinux-test.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-musllinux-test |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "OFF" |
| CHECK_UNITTESTS: "ON" |
| CHECK_WHEEL_CONTENT: "OFF" |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| # TODO: Remove this when the official Docker Python image supports the free-threaded build. |
| # See https://github.com/docker-library/python/issues/947 for more info. |
| python-free-threaded-wheel-musllinux-test-unittests: |
| image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests |
| build: |
| args: |
| base: "${ARCH}/alpine:${ALPINE_LINUX}" |
| python_version: ${PYTHON} |
| build_date: "20251014" # python-build-standalone release date |
| arch: ${ARCH_ALIAS} |
| context: . |
| dockerfile: ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-musllinux-test-unittests |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "OFF" |
| CHECK_UNITTESTS: "ON" |
| PYTHON: ${PYTHON} |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-manylinux-test-imports: |
| image: ${ARCH}/python:${PYTHON_IMAGE_TAG} |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "ON" |
| CHECK_UNITTESTS: "OFF" |
| CHECK_WHEEL_CONTENT: "ON" |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| # TODO: Remove this when the official Docker Python image supports the free-threaded build. |
| # See https://github.com/docker-library/python/issues/947 for more info. |
| python-free-threaded-wheel-manylinux-test-imports: |
| image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports |
| build: |
| args: |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| python_version: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-imports |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "ON" |
| CHECK_UNITTESTS: "OFF" |
| PYTHON: ${PYTHON} |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-manylinux-test-unittests: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test |
| build: |
| args: |
| arch: ${ARCH} |
| python: ${PYTHON} |
| python_image_tag: ${PYTHON_IMAGE_TAG} |
| context: . |
| dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "OFF" |
| CHECK_UNITTESTS: "ON" |
| CHECK_WHEEL_CONTENT: "OFF" |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| # TODO: Remove this when the official Docker Python image supports the free-threaded build. |
| # See https://github.com/docker-library/python/issues/947 for more info. |
| python-free-threaded-wheel-manylinux-test-unittests: |
| image: ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests |
| build: |
| args: |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| python_version: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON_IMAGE_TAG}-free-threaded-wheel-manylinux-test-unittests |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "OFF" |
| CHECK_UNITTESTS: "ON" |
| PYTHON: ${PYTHON} |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-windows-vs2022-base: |
| image: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| build: |
| args: |
| vcpkg: ${VCPKG} |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-vs2022-base.dockerfile |
| # This should make the pushed images reusable, but the image gets rebuilt. |
| # Uncomment if no local cache is available. |
| # cache_from: |
| # - mcr.microsoft.com/windows/servercore:ltsc2022 |
| # - ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| volumes: &python-wheel-windows-vs2022-volumes |
| - type: bind |
| source: . |
| target: "C:/arrow" |
| |
| python-wheel-windows-vs2022: |
| image: ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| build: |
| args: |
| base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| python: ${PYTHON} |
| python_variant: default |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile |
| # This should make the pushed images reusable, but the image gets rebuilt. |
| # Uncomment if no local cache is available. |
| # cache_from: |
| # - mcr.microsoft.com/windows/servercore:ltsc2022 |
| # - ${REPO}:python-${PYTHON}-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| volumes: *python-wheel-windows-vs2022-volumes |
| command: arrow\\ci\\scripts\\python_wheel_windows_build.bat |
| |
| python-free-threaded-wheel-windows-vs2022: |
| image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| build: |
| args: |
| base: ${REPO}:python-wheel-windows-vs2022-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| python: ${PYTHON} |
| python_variant: freethreaded |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-vs2022.dockerfile |
| # This should make the pushed images reusable, but the image gets rebuilt. |
| # Uncomment if no local cache is available. |
| # cache_from: |
| # - mcr.microsoft.com/windows/servercore:ltsc2022 |
| # - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2022-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| volumes: *python-wheel-windows-vs2022-volumes |
| command: arrow\\ci\\scripts\\python_wheel_windows_build.bat |
| |
| python-wheel-windows-test-base: |
| image: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} |
| build: |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-test-vs2022-base.dockerfile |
| volumes: *python-wheel-windows-vs2022-volumes |
| |
| python-wheel-windows-test: |
| image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} |
| build: |
| args: |
| base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} |
| python: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-test-vs2022.dockerfile |
| volumes: *python-wheel-windows-vs2022-volumes |
| command: arrow\\ci\\scripts\\python_wheel_windows_test.bat |
| |
| python-free-threaded-wheel-windows-test: |
| image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2022-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} |
| build: |
| args: |
| base: ${REPO}:python-wheel-windows-test-vs2022-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION} |
| python: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile |
| volumes: *python-wheel-windows-vs2022-volumes |
| command: arrow\\ci\\scripts\\python_wheel_windows_test.bat |
| |
| ############################## Integration ################################# |
| |
| conda-python-pandas: |
| # Possible $PANDAS parameters: |
| # - `latest`: latest release |
| # - `master`: git master branch, use `docker compose run --no-cache` |
| # - `<version>`: specific version available on conda-forge |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-pandas |
| # docker compose run --rm conda-python-pandas |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-pandas.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| numpy: ${NUMPY} |
| pandas: ${PANDAS} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| PARQUET_REQUIRE_ENCRYPTION: # inherit |
| PYTEST_ARGS: # inherit |
| HYPOTHESIS_PROFILE: # inherit |
| PYARROW_TEST_HYPOTHESIS: # inherit |
| PANDAS_FUTURE_INFER_STRING: # inherit |
| volumes: *conda-volumes |
| command: *python-conda-command |
| |
| conda-python-no-numpy: |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-no-numpy |
| # docker compose run --rm conda-python-no-numpy |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-no-numpy |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| PARQUET_REQUIRE_ENCRYPTION: # inherit |
| HYPOTHESIS_PROFILE: # inherit |
| PYARROW_TEST_HYPOTHESIS: # inherit |
| volumes: *conda-volumes |
| command: |
| [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| mamba uninstall -y numpy && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| conda-python-docs: |
| # Usage: |
| # archery docker run conda-python-docs |
| # |
| # Only a single rule is enabled for now to check undocumented arguments. |
| # We should extend the list of enabled rules after adding this build to |
| # the CI pipeline. |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| cap_add: |
| - SYS_ADMIN |
| environment: |
| <<: [*common, *ccache] |
| ARROW_SUBSTRAIT: "ON" |
| LC_ALL: "C.UTF-8" |
| LANG: "C.UTF-8" |
| BUILD_DOCS_CPP: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| PYTEST_ARGS: "--doctest-modules --doctest-cython" |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| pip install -e /arrow/dev/archery[numpydoc] && |
| archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| conda-python-dask: |
| # Possible $DASK parameters: |
| # - `latest`: latest release |
| # - `master`: git master branch, use `docker compose run --no-cache` |
| # - `<version>`: specific version available on conda-forge |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-dask |
| # docker compose run --rm conda-python-dask |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-dask.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| dask: ${DASK} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_dask.sh"] |
| |
| conda-python-cpython-debug: |
| # Usage: |
| # docker compose build conda |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-cpython-debug |
| # docker compose run --rm conda-python-cpython-debug |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-cpython-debug.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| PYTEST_ARGS: # inherit |
| volumes: *conda-volumes |
| command: *python-conda-command |
| |
| ################################## R ######################################## |
| |
| ubuntu-r: |
| # Usage: |
| # docker compose build ubuntu-cpp |
| # docker compose build ubuntu-r |
| # docker compose run ubuntu-r |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-r.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} |
| args: |
| arch: ${ARCH} |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| gcc: ${GCC} |
| r: ${R} |
| r_duckdb_dev: ${R_DUCKDB_DEV:-} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| tz: ${TZ} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_R_CXXFLAGS: '-Werror' |
| ARROW_FLIGHT: 'ON' |
| LIBARROW_BUILD: 'false' |
| NOT_CRAN: 'true' |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| ARROW_SOURCE_HOME: '/arrow' |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/r_test.sh /arrow" |
| |
| ubuntu-r-only-r: |
| environment: |
| <<: *common |
| ARROW_DEPENDENCY_SOURCE: '' |
| ARROW_SOURCE_HOME: '/arrow' |
| FORCE_BUNDLED_BUILD: 'true' |
| LIBARROW_BINARY: 'false' |
| LIBARROW_BUILD: 'true' |
| extends: ubuntu-r |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_test.sh /arrow" |
| |
| r: |
| # This lets you test building/installing the arrow R package |
| # (including building the C++ library) on any Docker image that contains R |
| # |
| # Usage: |
| # R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker compose build r |
| # R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker compose run r |
| image: ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} |
| args: |
| base: ${R_ORG}/${R_IMAGE}:${R_TAG} |
| cmake: ${CMAKE} |
| r_dev: ${ARROW_R_DEV} |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| r_custom_ccache: ${R_CUSTOM_CCACHE} |
| r_update_clang: ${R_UPDATE_CLANG} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *sccache] |
| LIBARROW_BINARY: "false" |
| ARROW_SOURCE_HOME: "/arrow" |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| # To test for CRAN release, delete ^^ these two env vars so we download the Apache release |
| ARROW_USE_PKG_CONFIG: "false" |
| volumes: |
| - .:/arrow:delegated |
| command: > |
| /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow" |
| |
| r-san: |
| image: ${REPO}:r-${R_ORG}-${R_IMAGE} |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-${R_ORG}-${R_IMAGE} |
| args: |
| base: ${R_ORG}/${R_IMAGE}:${R_TAG} |
| cmake: ${CMAKE} |
| r_dev: ${ARROW_R_DEV} |
| r_bin: R |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| shm_size: *shm-size |
| # These environment variables are *not* set by CRAN, so we should be careful about adding |
| # to them. I.E. if we have to add something here to have CI pass, it will likely fail on CRAN. |
| environment: |
| <<: *common |
| LIBARROW_DOWNLOAD: "false" |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| ARROW_SOURCE_HOME: "/arrow" |
| ARROW_USE_PKG_CONFIG: "false" |
| volumes: |
| - .:/arrow:delegated |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_sanitize.sh /arrow" |
| |
| ubuntu-r-valgrind: |
| # Only amd64 supported |
| # Usage: |
| # docker compose build ubuntu-r-valgrind |
| # docker compose run ubuntu-r-valgrind |
| image: ${REPO}:amd64-ubuntu-r-valgrind |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:amd64-ubuntu-r-valgrind |
| args: |
| base: rhub/valgrind:latest |
| cmake: ${CMAKE} |
| r_bin: R |
| tz: ${TZ} |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| # AVX512 not supported by Valgrind (similar to ARROW-9851) some runners support AVX512 and some do not |
| # so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner. |
| EXTRA_CMAKE_FLAGS: "-DARROW_RUNTIME_SIMD_LEVEL=AVX2" |
| ARROW_SOURCE_HOME: "/arrow" |
| LIBARROW_DOWNLOAD: "false" |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_valgrind.sh /arrow" |
| |
| r-revdepcheck: |
| # Usage: |
| # docker compose build r-revdepcheck |
| # docker compose run r-revdepcheck |
| image: ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck |
| args: |
| base: rstudio/r-base:4.2-focal |
| r_dev: ${ARROW_R_DEV} |
| tz: ${TZ} |
| shm_size: *shm-size |
| environment: |
| <<: *common |
| N_JOBS: |
| ARROW_REVDEP_WORKERS: |
| ARROW_R_DEV: "true" |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow" |
| |
| alpine-linux-r: |
| # Usage: |
| # docker compose build alpine-linux-r |
| # docker compose run alpine-linux-r |
| # Tests R package installation on musl (Alpine Linux) for CRAN checks. |
| # R package builds C++ from source (bundled RE2 2023-03-01 supports musl). |
| # Parameters: |
| # ALPINE_LINUX: 3.22 |
| # R: 4.5 (Alpine's R version) |
| # ARCH: amd64, arm64v8, ... |
| image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-r |
| build: |
| context: . |
| dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-r.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-r |
| args: |
| arch: ${ARCH} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *sccache] |
| LIBARROW_BINARY: "false" |
| ARROW_SOURCE_HOME: "/arrow" |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| ARROW_USE_PKG_CONFIG: "false" |
| SKIP_VIGNETTES: "true" |
| NOT_CRAN: "false" |
| volumes: |
| - .:/arrow:delegated |
| command: /arrow/ci/scripts/r_test.sh /arrow |
| |
| ############################## Integration ################################## |
| |
| conda-integration: |
| # Usage: |
| # docker compose build conda-cpp |
| # docker compose build conda-integration |
| # docker compose run conda-integration |
| image: ${REPO}:${ARCH}-conda-integration |
| build: |
| context: . |
| dockerfile: ci/docker/conda-integration.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-integration |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| # Use a newer JDK as it seems to improve stability |
| jdk: 17 |
| maven: ${MAVEN} |
| node: ${NODE} |
| volumes: *conda-volumes |
| environment: |
| <<: [*common, *ccache] |
| ARCHERY_INTEGRATION_WITH_GO: 0 |
| ARCHERY_INTEGRATION_WITH_NANOARROW: 0 |
| ARCHERY_INTEGRATION_WITH_RUST: 0 |
| # Tell Archery where Arrow binaries are located |
| ARROW_CPP_EXE_PATH: /build/cpp/debug |
| ARROW_NANOARROW_PATH: /build/nanoarrow |
| ARROW_RUST_EXE_PATH: /build/rust/debug |
| command: |
| ["git config --global --add safe.directory /arrow/go && |
| /arrow/ci/scripts/integration_arrow_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_arrow.sh /arrow /build"] |
| |
| ################################ Docs ####################################### |
| |
| debian-docs: |
| # Usage: |
| # docker compose build debian-cpp |
| # docker compose build debian-python |
| # docker compose build debian-docs |
| # docker compose run --rm debian-docs |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-docs |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-docs.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-docs |
| args: |
| r: ${R} |
| node: ${NODE} |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 |
| # This is for Chromium used by Mermaid. Chromium uses namespace |
| # isolation for security by default. |
| cap_add: |
| - SYS_ADMIN |
| environment: |
| <<: [*common, *ccache] |
| ARROW_AZURE: "ON" |
| ARROW_CUDA: "ON" |
| ARROW_CXX_FLAGS_DEBUG: "-g1" |
| ARROW_C_FLAGS_DEBUG: "-g1" |
| ARROW_HOME: "/tmp/local" |
| ARROW_SUBSTRAIT: "ON" |
| BUILD_DOCS_C_GLIB: "ON" |
| BUILD_DOCS_CPP: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| BUILD_DOCS_R: "ON" |
| volumes: *debian-volumes |
| command: > |
| /bin/bash -c " |
| sudo mkdir -p /build /ccache && |
| sudo chown -R `id --user --name`: /build /ccache && |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/r_build.sh /arrow /build" |
| |
| ######################### Integration Tests ################################# |
| |
| postgres: |
| # required for the impala service |
| image: postgres |
| ports: |
| - 5432:5432 |
| environment: |
| POSTGRES_PASSWORD: postgres |
| |
| impala: |
| # required for the hdfs tests |
| image: ibisproject/impala:latest |
| hostname: impala |
| links: |
| - postgres:postgres |
| environment: |
| PGPASSWORD: postgres |
| ports: |
| # HDFS |
| - 9020:9020 |
| - 50070:50070 |
| - 50075:50075 |
| - 8020:8020 |
| - 8042:8042 |
| # Hive |
| - 9083:9083 |
| # Impala |
| - 21000:21000 |
| - 21050:21050 |
| - 25000:25000 |
| - 25010:25010 |
| - 25020:25020 |
| |
| conda-python-hdfs: |
| # Usage: |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-hdfs |
| # docker compose run conda-python-hdfs |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-hdfs.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| jdk: ${JDK} |
| maven: ${MAVEN} |
| hdfs: ${HDFS} |
| links: |
| - impala:impala |
| environment: |
| <<: [*common, *ccache] |
| ARROW_ENGINE: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_HDFS: "ON" |
| ARROW_HDFS_TEST_HOST: impala |
| ARROW_HDFS_TEST_PORT: 8020 |
| ARROW_HDFS_TEST_USER: hdfs |
| ARROW_S3: "OFF" |
| shm_size: *shm-size |
| volumes: &conda-maven-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_hdfs.sh /arrow /build"] |
| |
| conda-python-spark: |
| # Usage: |
| # docker compose build conda-cpp |
| # docker compose build conda-python |
| # docker compose build conda-python-spark |
| # docker compose run conda-python-spark |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-spark.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| jdk: ${JDK} |
| maven: ${MAVEN} |
| spark: ${SPARK} |
| numpy: ${NUMPY} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *conda-maven-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_spark.sh /arrow /spark"] |
| |
| ################################# Source Verification ##################################### |
| |
| conda-verify-rc: |
| image: ubuntu:${UBUNTU} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| DEBIAN_FRONTEND: "noninteractive" |
| DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| USE_CONDA: 1 |
| command: > |
| /bin/bash -c " |
| apt update -y && apt install -y curl git gnupg tzdata wget && |
| git config --global --add safe.directory /arrow && |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
| |
| almalinux-verify-rc: |
| # Usage: |
| # docker compose build almalinux-verify-rc |
| # docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 almalinux-verify-rc |
| # Parameters: |
| # ALMALINUX: 8 |
| image: ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc |
| build: |
| context: . |
| dockerfile: ci/docker/almalinux-${ALMALINUX}-verify-rc.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}almalinux-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| command: > |
| /bin/bash -c " |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
| |
| ubuntu-verify-rc: |
| # Usage: |
| # docker compose build ubuntu-verify-rc |
| # docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 ubuntu-verify-rc |
| # Parameters: |
| # UBUNTU: 22.04, 24.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-verify-rc.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc |
| args: |
| arch: ${ARCH} |
| cmake: ${CMAKE} |
| repo: ${REPO} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| command: > |
| /bin/bash -c " |
| git config --global --add safe.directory /arrow && |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
| |
| secrets: |
| github_repository_owner: |
| environment: GITHUB_REPOSITORY_OWNER |
| github_token: |
| environment: GITHUB_TOKEN |
| vcpkg_binary_sources: |
| environment: VCPKG_BINARY_SOURCES |
