| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # Usage |
| # ----- |
| # |
| # The docker compose file is parametrized using environment variables, the |
| # defaults are set in .env file. |
| # |
| # Example: |
| # $ ARCH=arm64v8 docker-compose build ubuntu-cpp |
| # $ ARCH=arm64v8 docker-compose run ubuntu-cpp |
| # |
| # |
| # Coredumps |
| # --------- |
| # |
| # In order to enable coredumps for the C++ tests run by CTest either with |
| # command `make unittest` or `ctest --output-on-failure` the correct coredump |
| # patterns must be set. |
| # The kernel settings are coming from the host, so while it can be enabled from |
| # a running container using --priviled option the change will affect all other |
| # containers, so prefer setting it explicitly, directly on the host. |
| # WARNING: setting this will affect the host machine. |
| # |
| # Linux host: |
| # $ sudo sysctl -w kernel.core_pattern=core.%e.%p |
| # |
| # macOS host running Docker for Mac (won't persist between restarts): |
| # $ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty |
| # # echo "core.%e.%p" > /proc/sys/kernel/core_pattern |
| # |
| # The setup attempts to generate coredumps by default, but the correct paths |
| # above must be set. In order to disable the coredump generation set |
| # ULIMIT_CORE environment variable to 0 before running docker-compose |
| # (or by setting it in .env file): |
| # |
| # ULIMIT_CORE=0 docker-compose run --rm conda-cpp |
| # |
| # See more in cpp/build-support/run-test.sh::print_coredumps |
| |
| version: '3.5' |
| |
| x-common: &common |
| GITHUB_ACTIONS: |
| |
| x-ccache: &ccache |
| CCACHE_COMPILERCHECK: content |
| CCACHE_COMPRESS: 1 |
| CCACHE_COMPRESSLEVEL: 6 |
| CCACHE_MAXSIZE: 1G |
| CCACHE_DIR: /ccache |
| |
| x-sccache: &sccache |
| AWS_ACCESS_KEY_ID: |
| AWS_SECRET_ACCESS_KEY: |
| SCCACHE_BUCKET: |
| SCCACHE_REGION: |
| SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache} |
| |
| # CPU/memory limit presets to pass to Docker. |
| # |
| # Usage: archery docker run --resource-limit=github <image> |
| # |
| # Note that exporting ARCHERY_DOCKER_BIN="sudo docker" is likely required, |
| # unless Docker is configured with cgroups v2 (else Docker will silently |
| # ignore the limits). |
| x-limit-presets: |
| # These values emulate GitHub Actions: |
| # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners |
| github: |
| # Note we use cpuset and not cpus since Ninja only detects and limits |
| # parallelism given the former |
| cpuset_cpus: [0, 1] |
| memory: 7g |
| |
| x-with-gpus: |
| - ubuntu-cuda-cpp |
| - ubuntu-cuda-python |
| |
| x-hierarchy: |
| # This section is used by the archery tool to enable building nested images, |
| # so it is enough to call: |
| # archery run debian-ruby |
| # instead of a sequence of docker-compose commands: |
| # docker-compose build debian-cpp |
| # docker-compose build debian-c-glib |
| # docker-compose build debian-ruby |
| # docker-compose run --rm debian-ruby |
| # |
| # Each node must be either a string scalar of a list containing the |
| # descendant images if any. Archery checks that all node has a corresponding |
| # service entry, so any new image/service must be listed here. |
| - almalinux-verify-rc |
| - alpine-linux-cpp |
| - centos-cpp-static |
| - conda: |
| - conda-cpp: |
| - conda-integration |
| - conda-cpp-valgrind |
| - conda-python: |
| - conda-python-pandas: |
| - conda-python-docs |
| - conda-python-dask |
| - conda-python-hdfs |
| - conda-python-java-integration |
| - conda-python-jpype |
| - conda-python-spark |
| - conda-python-substrait |
| - conda-verify-rc |
| - conan |
| - debian-cpp: |
| - debian-c-glib: |
| - debian-ruby |
| - debian-python |
| - debian-go: |
| - debian-go-cgo |
| - debian-go-cgo-python |
| - debian-java |
| - debian-js |
| - fedora-cpp: |
| - fedora-python |
| - oracle-java |
| - python-sdist |
| - ubuntu-cpp: |
| - ubuntu-cpp-static |
| - ubuntu-c-glib: |
| - ubuntu-ruby |
| - ubuntu-lint |
| - ubuntu-python: |
| - ubuntu-docs |
| - ubuntu-python-sdist-test |
| - ubuntu-r |
| - ubuntu-r-only-r |
| - ubuntu-cpp-bundled |
| - ubuntu-cpp-minimal |
| - ubuntu-cuda-cpp: |
| - ubuntu-cuda-python |
| - ubuntu-csharp |
| - ubuntu-cpp-sanitizer |
| - ubuntu-cpp-thread-sanitizer |
| - ubuntu-r-sanitizer |
| - ubuntu-r-valgrind |
| - ubuntu-swift |
| - ubuntu-verify-rc |
| - fedora-r-clang-sanitizer |
| - r |
| - r-revdepcheck |
| # helper services |
| - impala |
| - postgres |
| - python-wheel-manylinux-2014: |
| - java-jni-manylinux-2014 |
| - python-wheel-manylinux-2-28 |
| - python-wheel-manylinux-test-imports |
| - python-wheel-manylinux-test-unittests |
| - python-wheel-windows-vs2017 |
| - python-wheel-windows-test |
| |
| volumes: |
| almalinux-ccache: |
| name: ${ARCH}-almalinux-ccache |
| alpine-linux-ccache: |
| name: ${ARCH}-alpine-linux-ccache |
| conda-ccache: |
| name: ${ARCH}-conda-ccache |
| debian-ccache: |
| name: ${ARCH}-debian-${DEBIAN}-ccache |
| debian-rust: |
| name: ${ARCH}-debian-${DEBIAN}-rust |
| fedora-ccache: |
| name: ${ARCH}-fedora-${FEDORA}-ccache |
| maven-cache: |
| name: maven-cache |
| python-wheel-manylinux2014-ccache: |
| name: python-wheel-manylinux2014-ccache |
| python-wheel-manylinux-2-28-ccache: |
| name: python-wheel-manylinux-2-28-ccache |
| python-wheel-windows-clcache: |
| name: python-wheel-windows-clcache |
| ubuntu-ccache: |
| name: ${ARCH}-ubuntu-${UBUNTU}-ccache |
| |
| services: |
| |
| ################################# C++ ####################################### |
| # Release build: |
| # docker-compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|... |
| # Shared only: |
| # docker-compose run -e ARROW_BUILD_STATIC=OFF conda-cpp|debian-cpp|... |
| # Static only: |
| # docker-compose run \ |
| # -e ARROW_BUILD_SHARED=OFF \ |
| # -e ARROW_TEST_LINKAGE=static \ |
| # conda-cpp|debian-cpp|... |
| |
| alpine-linux-cpp: |
| # Usage: |
| # docker-compose build alpine-linux-cpp |
| # docker-compose run --rm alpine-linux-cpp |
| # Parameters: |
| # ALPINE_LINUX: 3.16 |
| # ARCH: amd64, arm64v8, ... |
| image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp |
| args: |
| arch: ${ARCH} |
| shm_size: &shm-size 2G |
| ulimits: &ulimits |
| core: ${ULIMIT_CORE} |
| environment: |
| <<: [*common, *ccache] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &alpine-linux-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}alpine-linux-ccache:/ccache:delegated |
| command: >- |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build" |
| |
| conda: |
| # Base image for conda builds. |
| # |
| # Usage: |
| # docker-compose build conda |
| # docker-compose run --rm conda |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda |
| build: |
| context: . |
| dockerfile: ci/docker/conda.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda |
| args: |
| arch: ${ARCH} |
| volumes: |
| - .:/arrow:delegated |
| |
| conda-cpp: |
| # C++ build in conda environment, including the doxygen docs. |
| # |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose run --rm conda-cpp |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/conda-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-cpp |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_BUILD_BENCHMARKS: "ON" |
| ARROW_BUILD_EXAMPLES: "ON" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_EXTRA_ERROR_CONTEXT: "ON" |
| ARROW_MIMALLOC: "ON" |
| ARROW_USE_LD_GOLD: "ON" |
| ARROW_USE_PRECOMPILED_HEADERS: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| volumes: &conda-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| command: &conda-cpp-command |
| [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build"] |
| |
| conda-cpp-valgrind: |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose run --rm conda-cpp-valgrind |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| image: ${REPO}:${ARCH}-conda-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/conda-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-cpp |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| # Shrink test runtime by enabling minimal optimizations |
| ARROW_C_FLAGS_DEBUG: "-g1 -Og" |
| ARROW_CXX_FLAGS_DEBUG: "-g1 -Og" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_RUNTIME_SIMD_LEVEL: "AVX2" # AVX512 not supported by Valgrind (ARROW-9851) |
| ARROW_TEST_MEMCHECK: "ON" |
| ARROW_USE_LD_GOLD: "ON" |
| BUILD_WARNING_LEVEL: "PRODUCTION" |
| ARROW_CTEST_TIMEOUT: 500 |
| volumes: *conda-volumes |
| command: *conda-cpp-command |
| |
| debian-cpp: |
| # Usage: |
| # docker-compose build debian-cpp |
| # docker-compose run --rm debian-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 11 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| args: |
| arch: ${ARCH} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &debian-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated |
| # integration_skyhook.sh is a no-op unless skyhook is on. |
| command: &cpp-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_skyhook.sh /build && |
| /arrow/ci/scripts/cpp_test.sh /arrow /build" |
| |
| ubuntu-cpp: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose run --rm ubuntu-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, s390x, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| gcc_version: ${GCC_VERSION} |
| shm_size: *shm-size |
| cap_add: |
| - SYS_ADMIN |
| devices: |
| - "/dev/fuse:/dev/fuse" |
| security_opt: |
| - apparmor:unconfined |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| volumes: &ubuntu-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated |
| command: *cpp-command |
| |
| ubuntu-cpp-static: |
| # Usage: |
| # docker-compose build ubuntu-cpp-static |
| # docker-compose run --rm ubuntu-cpp-static |
| # Parameters: |
| # ARCH: amd64, arm64v8, s390x, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| gcc_version: ${GCC_VERSION} |
| shm_size: *shm-size |
| cap_add: |
| - SYS_ADMIN |
| devices: |
| - "/dev/fuse:/dev/fuse" |
| security_opt: |
| - apparmor:unconfined |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_HOME: /arrow |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| LIBARROW_MINIMAL: "false" |
| ARROW_MIMALLOC: "ON" |
| volumes: *ubuntu-volumes |
| command: &cpp-static-command |
| /bin/bash -c " |
| cd /arrow && |
| r/inst/build_arrow_static.sh" |
| |
| centos-cpp-static: |
| image: ${REPO}:centos-7-cpp-static |
| build: |
| context: . |
| dockerfile: ci/docker/centos-7-cpp.dockerfile |
| cache_from: |
| - ${REPO}:centos-7-cpp-static |
| shm_size: *shm-size |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| ARROW_HOME: /arrow |
| LIBARROW_MINIMAL: "false" |
| command: /bin/bash -c " |
| scl enable devtoolset-8 '/arrow/r/inst/build_arrow_static.sh'" |
| |
| ubuntu-cpp-bundled: |
| # Arrow build with BUNDLED dependencies |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_DEPENDENCY_SOURCE: BUNDLED |
| CMAKE_GENERATOR: "Unix Makefiles" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| |
| ubuntu-cpp-minimal: |
| # Arrow build with minimal components/dependencies |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal |
| args: |
| arch: ${ARCH} |
| base: "${ARCH}/ubuntu:${UBUNTU}" |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_COMPUTE: "OFF" |
| ARROW_CSV: "OFF" |
| ARROW_DATASET: "OFF" |
| ARROW_FILESYSTEM: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_BROTLI: "OFF" |
| ARROW_WITH_BZ2: "OFF" |
| ARROW_WITH_LZ4: "OFF" |
| ARROW_WITH_SNAPPY: "OFF" |
| ARROW_WITH_ZLIB: "OFF" |
| ARROW_WITH_ZSTD: "OFF" |
| PARQUET_BUILD_EXAMPLES: "OFF" |
| PARQUET_BUILD_EXECUTABLES: "OFF" |
| PARQUET_REQUIRE_ENCRYPTION: "OFF" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| |
| ubuntu-cuda-cpp: |
| # Usage: |
| # docker-compose build cuda-cpp |
| # docker-compose run --rm cuda-cpp |
| # Parameters: |
| # ARCH: amd64 |
| # CUDA: <depends on your nvidia driver, should match system CUDA> |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| args: |
| arch: ${ARCH} |
| base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU} |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_COMPUTE: "OFF" |
| ARROW_CSV: "OFF" |
| ARROW_CUDA: "ON" |
| ARROW_DATASET: "OFF" |
| ARROW_ENABLE_TIMING_TESTS: "OFF" |
| ARROW_FILESYSTEM: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "OFF" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_JSON: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| volumes: *ubuntu-volumes |
| command: *cpp-command |
| deploy: &cuda-deploy |
| resources: |
| reservations: |
| devices: |
| - driver: nvidia |
| capabilities: [gpu] |
| |
| ubuntu-cpp-sanitizer: |
| # Usage: |
| # docker-compose build ubuntu-cpp-sanitizer |
| # docker-compose run --rm ubuntu-cpp-sanitizer |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 22.04, ... |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| cap_add: |
| # For LeakSanitizer |
| - SYS_PTRACE |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache] |
| CC: clang-${CLANG_TOOLS} |
| CXX: clang++-${CLANG_TOOLS} |
| # Avoid creating huge static libraries |
| ARROW_BUILD_STATIC: "OFF" |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| # GH-33920: Disable Flight SQL to reduce build time. |
| # We'll be able to re-enable this with Ubuntu 24.04 because |
| # Ubuntu 24.04 will ship ProtoBuf 3.15.0 or later that is required |
| # by Flight SQL. |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_FUZZING: "ON" # Check fuzz regressions |
| ARROW_JEMALLOC: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_USE_ASAN: "ON" |
| ARROW_USE_UBSAN: "ON" |
| # 1 GB isn't enough for a single sanitizer build |
| CCACHE_MAXSIZE: 2G |
| Protobuf_SOURCE: "AUTO" |
| command: *cpp-command |
| |
| ubuntu-cpp-thread-sanitizer: |
| # Usage: |
| # docker-compose build ubuntu-cpp-thread-sanitizer |
| # docker-compose run --rm ubuntu-cpp-thread-sanitizer |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| args: |
| arch: ${ARCH} |
| clang_tools: ${CLANG_TOOLS} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| environment: |
| <<: [*common, *ccache, *sccache] |
| CC: clang-${CLANG_TOOLS} |
| CXX: clang++-${CLANG_TOOLS} |
| ARROW_BUILD_STATIC: "OFF" |
| ARROW_CTEST_TIMEOUT: 500 |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_DATASET: "ON" |
| ARROW_JEMALLOC: "OFF" |
| ARROW_ORC: "OFF" |
| ARROW_S3: "OFF" |
| ARROW_USE_TSAN: "ON" |
| command: *cpp-command |
| |
| fedora-cpp: |
| # Usage: |
| # docker-compose build fedora-cpp |
| # docker-compose run --rm fedora-cpp |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # FEDORA: 33 |
| image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| build: |
| context: . |
| dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| args: |
| arch: ${ARCH} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_ENABLE_TIMING_TESTS: # inherit |
| ARROW_MIMALLOC: "ON" |
| Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 |
| volumes: &fedora-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated |
| command: *cpp-command |
| |
| conan: |
| # Base service for Conan. |
| # |
| # Usage: |
| # docker-compose run --rm conan |
| # Parameters: |
| # CONAN: gcc11, gcc11-armv7, ... |
| # See https://github.com/conan-io/conan-docker-tools#readme for |
| # available images. |
| image: conanio/${CONAN} |
| user: root:root |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *sccache] |
| volumes: |
| - .:/arrow:delegated |
| command: >- |
| /bin/bash -c " |
| /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin && |
| /arrow/ci/scripts/conan_setup.sh && |
| /arrow/ci/scripts/conan_build.sh /arrow /build" |
| |
| ############################### C GLib ###################################### |
| |
| debian-c-glib: |
| # Usage: |
| # docker-compose build debian-cpp |
| # docker-compose build debian-c-glib |
| # docker-compose run --rm debian-c-glib |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 11 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-c-glib.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| BUILD_DOCS_C_GLIB: "ON" |
| volumes: *debian-volumes |
| command: &c-glib-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_test.sh /arrow /build" |
| |
| ubuntu-c-glib: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-c-glib |
| # docker-compose run --rm ubuntu-c-glib |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-c-glib.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *c-glib-command |
| |
| ############################### Ruby ######################################## |
| # Until Ruby is the only dependent implementation on top of C Glib we can |
| # test C Glib and Ruby in one pass. This is an optimization to avoid |
| # redundant (one for C GLib and one for Ruby doing the same work twice) |
| # builds on CI services. |
| |
| debian-ruby: |
| # Usage: |
| # docker-compose build debian-cpp |
| # docker-compose build debian-c-glib |
| # docker-compose build debian-ruby |
| # docker-compose run --rm debian-ruby |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 11 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-ruby.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-ruby |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| BUILD_DOCS_C_GLIB: "ON" |
| volumes: *debian-volumes |
| command: &ruby-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_test.sh /arrow /build && |
| /arrow/ci/scripts/ruby_test.sh /arrow /build" |
| |
| ubuntu-ruby: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-c-glib |
| # docker-compose build ubuntu-ruby |
| # docker-compose run --rm ubuntu-ruby |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-ruby.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib |
| shm_size: *shm-size |
| ulimits: *ulimits |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *ruby-command |
| |
| ############################### Python ###################################### |
| |
| conda-python: |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose run --rm conda-python |
| # Parameters: |
| # ARCH: amd64, arm32v7 |
| # PYTHON: 3.8, 3.9, 3.10, 3.11 |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| volumes: *conda-volumes |
| command: &python-conda-command |
| [" |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| ubuntu-cuda-python: |
| # Usage: |
| # docker-compose build cuda-cpp |
| # docker-compose build cuda-python |
| # docker-compose run --rm cuda-python |
| # Parameters: |
| # ARCH: amd64 |
| # CUDA: <depends on your nvidia driver, should match system CUDA> |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp |
| numba: ${NUMBA} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_BUILD_UTILITIES: "OFF" |
| ARROW_COMPUTE: "ON" |
| ARROW_CSV: "ON" |
| ARROW_CUDA: "ON" |
| ARROW_DATASET: "ON" |
| ARROW_ENABLE_TIMING_TESTS: "OFF" |
| ARROW_FILESYSTEM: "ON" |
| ARROW_GANDIVA: "OFF" |
| ARROW_GCS: "OFF" |
| ARROW_HDFS: "ON" |
| ARROW_JEMALLOC: "ON" |
| ARROW_JSON: "ON" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "ON" |
| ARROW_S3: "OFF" |
| ARROW_SUBSTRAIT: "OFF" |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| SETUPTOOLS_SCM_PRETEND_VERSION: |
| volumes: *ubuntu-volumes |
| deploy: *cuda-deploy |
| command: &python-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/python_test.sh /arrow" |
| |
| debian-python: |
| # Usage: |
| # docker-compose build debian-cpp |
| # docker-compose build debian-python |
| # docker-compose run --rm debian-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # DEBIAN: 11 |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *debian-volumes |
| command: *python-command |
| |
| ubuntu-python: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-python |
| # docker-compose run --rm ubuntu-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: *python-command |
| |
| ubuntu-swift: |
| # Usage: |
| # docker-compose build ubuntu-swift |
| # docker-compose run --rm ubuntu-swift |
| # Parameters: |
| image: ubuntu-swift |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-swift.dockerfile |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c "/arrow/ci/scripts/swift_test.sh /arrow" |
| |
| |
| fedora-python: |
| # Usage: |
| # docker-compose build fedora-cpp |
| # docker-compose build fedora-python |
| # docker-compose run --rm fedora-python |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # FEDORA: 33 |
| image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-dnf-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 |
| volumes: *fedora-volumes |
| command: *python-command |
| |
| ############################ Python sdist ################################### |
| |
| python-sdist: |
| # Usage: |
| # docker-compose build python-sdist |
| # docker-compose run --rm python-sdist |
| # Parameters: |
| # PYARROW_VERSION: The pyarrow version for sdist such as "3.0.0" |
| image: ${REPO}:python-sdist |
| build: |
| context: . |
| dockerfile: ci/docker/python-sdist.dockerfile |
| cache_from: |
| - ${REPO}:python-sdist |
| environment: |
| PYARROW_VERSION: ${PYARROW_VERSION:-} |
| volumes: |
| - .:/arrow:delegated |
| command: /arrow/ci/scripts/python_sdist_build.sh /arrow |
| |
| ubuntu-python-sdist-test: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-python-sdist-test |
| # docker-compose run --rm ubuntu-python-sdist-test |
| # Parameters: |
| # ARCH: amd64, arm64v8, ... |
| # PYARROW_VERSION: The test target pyarrow version such as "3.0.0" |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-python-3.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| # Bundled build of OpenTelemetry needs a git client |
| ARROW_WITH_OPENTELEMETRY: "OFF" |
| PYARROW_VERSION: ${PYARROW_VERSION:-} |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| apt remove -y git && |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_sdist_test.sh /arrow" |
| |
| ############################ Python wheels ################################## |
| |
| # See available versions at: |
| # https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags |
| python-wheel-manylinux-2014: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} |
| build: |
| args: |
| arch: ${ARCH} |
| arch_short: ${ARCH_SHORT} |
| base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779 |
| vcpkg: ${VCPKG} |
| python: ${PYTHON} |
| manylinux: 2014 |
| context: . |
| dockerfile: ci/docker/python-wheel-manylinux.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} |
| environment: |
| <<: [*common, *ccache] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated |
| command: /arrow/ci/scripts/python_wheel_manylinux_build.sh |
| |
| # See available versions at: |
| # https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags |
| python-wheel-manylinux-2-28: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} |
| build: |
| args: |
| arch: ${ARCH} |
| arch_short: ${ARCH_SHORT} |
| base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-01-14-103cb93 |
| vcpkg: ${VCPKG} |
| python: ${PYTHON} |
| manylinux: 2_28 |
| context: . |
| dockerfile: ci/docker/python-wheel-manylinux.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG} |
| environment: |
| <<: [*common, *ccache] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated |
| command: /arrow/ci/scripts/python_wheel_manylinux_build.sh |
| |
| python-wheel-manylinux-test-imports: |
| image: ${ARCH}/python:${PYTHON} |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "ON" |
| CHECK_UNITTESTS: "OFF" |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-manylinux-test-unittests: |
| image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test |
| build: |
| args: |
| arch: ${ARCH} |
| python: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test |
| shm_size: 2G |
| volumes: |
| - .:/arrow:delegated |
| environment: |
| <<: *common |
| CHECK_IMPORTS: "OFF" |
| CHECK_UNITTESTS: "ON" |
| command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow |
| |
| python-wheel-windows-vs2017: |
| image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| build: |
| args: |
| vcpkg: ${VCPKG} |
| python: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-vs2017.dockerfile |
| # This should make the pushed images reusable, but the image gets rebuilt. |
| # Uncomment if no local cache is available. |
| # cache_from: |
| # - abrarov/msvc-2017:2.11.0 |
| # - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| volumes: |
| - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache" |
| - type: bind |
| source: . |
| target: "C:/arrow" |
| command: arrow\\ci\\scripts\\python_wheel_windows_build.bat |
| |
| python-wheel-windows-test: |
| image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2017-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} |
| build: |
| args: |
| python: ${PYTHON} |
| context: . |
| dockerfile: ci/docker/python-wheel-windows-test-vs2017.dockerfile |
| volumes: |
| - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache" |
| - type: bind |
| source: . |
| target: "C:/arrow" |
| command: arrow\\ci\\scripts\\python_wheel_windows_test.bat |
| |
| java-jni-manylinux-2014: |
| image: ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG} |
| build: |
| args: |
| base: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} |
| java: 1.8.0 |
| context: . |
| dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG} |
| environment: |
| <<: [*common, *ccache] |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated |
| - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated |
| command: |
| ["pip install -e /arrow/dev/archery && \ |
| /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \ |
| source /opt/rh/rh-maven35/enable && \ |
| /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \ |
| /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"] |
| |
| ############################## Integration ################################# |
| |
| conda-python-pandas: |
| # Possible $PANDAS parameters: |
| # - `latest`: latest release |
| # - `master`: git master branch, use `docker-compose run --no-cache` |
| # - `<version>`: specific version available on conda-forge |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-pandas |
| # docker-compose run --rm conda-python-pandas |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-pandas.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| numpy: ${NUMPY} |
| pandas: ${PANDAS} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| PARQUET_REQUIRE_ENCRYPTION: # inherit |
| PYTEST_ARGS: # inherit |
| HYPOTHESIS_PROFILE: # inherit |
| PYARROW_TEST_HYPOTHESIS: # inherit |
| volumes: *conda-volumes |
| command: *python-conda-command |
| |
| conda-python-docs: |
| # Usage: |
| # archery docker run conda-python-docs |
| # |
| # Only a single rule is enabled for now to check undocumented arguments. |
| # We should extend the list of enabled rules after adding this build to |
| # the CI pipeline. |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS} |
| environment: |
| <<: [*common, *ccache] |
| ARROW_SUBSTRAIT: "ON" |
| LC_ALL: "C.UTF-8" |
| LANG: "C.UTF-8" |
| BUILD_DOCS_CPP: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| # GH-31506/GH-33609: Remove --disable-warnings once |
| # https://github.com/lgpage/pytest-cython/issues/24 is resolved |
| # and a new version that includes the fix is released. |
| PYTEST_ARGS: "--doctest-modules --doctest-cython --disable-warnings" |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| pip install -e /arrow/dev/archery[numpydoc] && |
| archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| conda-python-dask: |
| # Possible $DASK parameters: |
| # - `latest`: latest release |
| # - `master`: git master branch, use `docker-compose run --no-cache` |
| # - `<version>`: specific version available on conda-forge |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-dask |
| # docker-compose run --rm conda-python-dask |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-dask.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| dask: ${DASK} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_dask.sh"] |
| |
| conda-python-substrait: |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-substrait |
| # docker-compose run --rm conda-python-substrait |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-substrait.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| ARROW_SUBSTRAIT: "ON" |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_substrait.sh"] |
| |
| conda-python-jpype: |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-jpype |
| # docker-compose run --rm conda-python-jpype |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-jpype.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_GANDIVA: "OFF" |
| volumes: *conda-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/java_build.sh /arrow /build && |
| /arrow/ci/scripts/python_test.sh /arrow"] |
| |
| conda-python-java-integration: |
| # Usage: |
| # docker-compose build conda |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-java-integration |
| # docker-compose run --rm conda-python-java-integration |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-jpype.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| llvm: ${LLVM} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| ARROW_DATASET: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_GANDIVA: "OFF" |
| ARROW_JAVA_CDATA: "ON" |
| ARROW_ORC: "OFF" |
| ARROW_PARQUET: "OFF" |
| JAVA_JNI_CMAKE_ARGS: >- |
| -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF |
| -DARROW_JAVA_JNI_ENABLE_C=ON |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated |
| - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated |
| command: |
| [ "/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) && |
| /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java && |
| /arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ] |
| |
| ################################## R ######################################## |
| |
| ubuntu-r: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-r |
| # docker-compose run ubuntu-r |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-r.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R} |
| args: |
| arch: ${ARCH} |
| r: ${R} |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| gcc_version: ${GCC_VERSION} |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| r_duckdb_dev: ${R_DUCKDB_DEV:-} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_R_CXXFLAGS: '-Werror' |
| ARROW_FLIGHT: 'ON' |
| LIBARROW_BUILD: 'false' |
| NOT_CRAN: 'true' |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| ARROW_SOURCE_HOME: '/arrow' |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/r_test.sh /arrow" |
| |
| ubuntu-r-only-r: |
| environment: |
| <<: *common |
| ARROW_DEPENDENCY_SOURCE: '' |
| ARROW_SOURCE_HOME: '/arrow' |
| FORCE_BUNDLED_BUILD: 'true' |
| LIBARROW_BUILD: 'true' |
| extends: ubuntu-r |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_test.sh /arrow" |
| |
| r: |
| # This lets you test building/installing the arrow R package |
| # (including building the C++ library) on any Docker image that contains R |
| # |
| # Usage: |
| # R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose build r |
| # R_ORG=rhub R_IMAGE=ubuntu-gcc-release R_TAG=latest docker-compose run r |
| image: ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG} |
| args: |
| base: ${R_ORG}/${R_IMAGE}:${R_TAG} |
| r_dev: ${ARROW_R_DEV} |
| devtoolset_version: ${DEVTOOLSET_VERSION} |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| r_custom_ccache: ${R_CUSTOM_CCACHE} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *sccache] |
| LIBARROW_DOWNLOAD: "false" |
| ARROW_SOURCE_HOME: "/arrow" |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| # To test for CRAN release, delete ^^ these two env vars so we download the Apache release |
| ARROW_USE_PKG_CONFIG: "false" |
| devtoolset_version: ${DEVTOOLSET_VERSION} |
| volumes: |
| - .:/arrow:delegated |
| command: > |
| /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow" |
| |
| ubuntu-r-sanitizer: |
| # Only 20.04 and amd64 supported |
| # Usage: |
| # docker-compose build ubuntu-r-sanitizer |
| # docker-compose run ubuntu-r-sanitizer |
| image: ${REPO}:amd64-ubuntu-20.04-r-sanitizer |
| cap_add: |
| # LeakSanitizer and gdb requires ptrace(2) |
| - SYS_PTRACE |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:amd64-ubuntu-20.04-r-sanitizer |
| args: |
| base: wch1/r-debug:latest |
| r_bin: RDsan |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| environment: |
| <<: [*common, *ccache] |
| ARROW_SOURCE_HOME: "/arrow" |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_sanitize.sh /arrow" |
| |
| fedora-r-clang-sanitizer: |
| image: ${REPO}:r-rhub-fedora-clang-devel-latest |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-rhub-fedora-clang-devel-latest |
| args: |
| base: rhub/fedora-clang-devel-san |
| r_dev: ${ARROW_R_DEV} |
| devtoolset_version: ${DEVTOOLSET_VERSION} |
| r_bin: RDsan |
| tz: ${TZ} |
| r_prune_deps: ${R_PRUNE_DEPS} |
| shm_size: *shm-size |
| environment: |
| <<: *common |
| LIBARROW_DOWNLOAD: "false" |
| ARROW_SOURCE_HOME: "/arrow" |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| # To test for CRAN release, delete ^^ these two env vars so we download the Apache release |
| ARROW_USE_PKG_CONFIG: "false" |
| volumes: |
| - .:/arrow:delegated |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_sanitize.sh /arrow" |
| |
| ubuntu-r-valgrind: |
| # Only 20.04 and amd64 supported |
| # Usage: |
| # docker-compose build ubuntu-r-valgrind |
| # docker-compose run ubuntu-r-valgrind |
| image: ${REPO}:amd64-ubuntu-20.04-r-valgrind |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:amd64-ubuntu-20.04-r-valgrind |
| args: |
| base: wch1/r-debug:latest |
| r_bin: RDvalgrind |
| tz: ${TZ} |
| environment: |
| <<: [*common, *ccache, *sccache] |
| ARROW_R_DEV: ${ARROW_R_DEV} |
| # AVX512 not supported by Valgrind (similar to ARROW-9851) some runners support AVX512 and some do not |
| # so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner. |
| EXTRA_CMAKE_FLAGS: "-DARROW_RUNTIME_SIMD_LEVEL=AVX2" |
| ARROW_SOURCE_HOME: "/arrow" |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| /arrow/ci/scripts/r_valgrind.sh /arrow" |
| |
| r-revdepcheck: |
| # Usage: |
| # docker-compose build r-revdepcheck |
| # docker-compose run r-revdepcheck |
| image: ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck |
| build: |
| context: . |
| dockerfile: ci/docker/linux-r.dockerfile |
| cache_from: |
| - ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck |
| args: |
| base: rstudio/r-base:4.2-focal |
| r_dev: ${ARROW_R_DEV} |
| tz: ${TZ} |
| shm_size: *shm-size |
| environment: |
| <<: *common |
| N_JOBS: |
| ARROW_REVDEP_WORKERS: |
| ARROW_R_DEV: "true" |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow" |
| |
| ################################# Go ######################################## |
| |
| debian-go: |
| # Usage: |
| # docker-compose build debian-go |
| # docker-compose run debian-go |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO} |
| build: |
| context: . |
| dockerfile: ci/docker/debian-${DEBIAN}-go.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO} |
| args: |
| arch: ${ARCH} |
| go: ${GO} |
| staticcheck: ${STATICCHECK} |
| shm_size: *shm-size |
| volumes: *debian-volumes |
| command: &go-command > |
| /bin/bash -c " |
| git config --global --add safe.directory /arrow && |
| /arrow/ci/scripts/go_build.sh /arrow && |
| /arrow/ci/scripts/go_test.sh /arrow" |
| |
| debian-go-cgo: |
| # Usage: |
| # docker-compose build debian-go-cgo |
| # docker-compose run debian-go-cgo |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo |
| build: |
| context: . |
| dockerfile: ci/docker/debian-go-cgo.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO} |
| shm_size: *shm-size |
| volumes: *debian-volumes |
| environment: |
| <<: *common |
| ARROW_GO_TESTCGO: "1" |
| command: *go-command |
| |
| debian-go-cgo-python: |
| # Usage: |
| # docker-compose build debian-go-cgo-python |
| # docker-compose run debian-go-cgo-python |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python |
| build: |
| context: . |
| dockerfile: ci/docker/debian-${DEBIAN}-go-cgo-python.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python |
| args: |
| base: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO} |
| shm_size: *shm-size |
| volumes: *debian-volumes |
| command: &go-cgo-python-command > |
| /bin/bash -c " |
| git config --global --add safe.directory /arrow && |
| /arrow/ci/scripts/go_cgo_python_test.sh /arrow" |
| |
| ############################# JavaScript #################################### |
| |
| debian-js: |
| # Usage: |
| # docker-compose build debian-js |
| # docker-compose run debian-js |
| image: ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE} |
| build: |
| context: . |
| dockerfile: ci/docker/debian-${DEBIAN}-js.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE} |
| args: |
| arch: ${ARCH} |
| node: ${NODE} |
| shm_size: *shm-size |
| volumes: *debian-volumes |
| environment: |
| <<: *common |
| BUILD_DOCS_JS: "ON" |
| command: &js-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/js_build.sh /arrow /build && |
| /arrow/ci/scripts/js_test.sh /arrow" |
| |
| #################################### C# ##################################### |
| |
| ubuntu-csharp: |
| # Usage: |
| # docker-compose build ubuntu-csharp |
| # docker-compose run ubuntu-csharp |
| image: ${REPO}:${ARCH}-ubuntu-22.04-csharp-${DOTNET} |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-22.04-csharp.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-22.04-csharp-${DOTNET} |
| args: |
| arch: ${ARCH} |
| dotnet: ${DOTNET} |
| platform: jammy |
| shm_size: *shm-size |
| volumes: *ubuntu-volumes |
| command: &csharp-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/csharp_build.sh /arrow && |
| /arrow/ci/scripts/csharp_test.sh /arrow && |
| /arrow/ci/scripts/csharp_pack.sh /arrow" |
| |
| ################################ Java ####################################### |
| |
| debian-java: |
| # Usage: |
| # docker-compose build debian-java |
| # docker-compose run debian-java |
| # Parameters: |
| # MAVEN: 3.5.4, 3.6.2 |
| # JDK: 8, 11 |
| image: ${ARCH}/maven:${MAVEN}-jdk-${JDK} |
| shm_size: *shm-size |
| volumes: &java-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated |
| command: &java-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/java_build.sh /arrow /build && |
| /arrow/ci/scripts/java_test.sh /arrow /build" |
| |
| oracle-java: |
| # Usage: |
| # docker-compose build oracle-java |
| # docker-compose run oracle-java |
| # Parameters: |
| # MAVEN: 3.8.5 |
| # JDK: 17 |
| image: ${ARCH}/maven:${MAVEN}-openjdk-${JDK} |
| shm_size: *shm-size |
| volumes: *java-volumes |
| command: *java-command |
| |
| ############################## Integration ################################## |
| |
| conda-integration: |
| # Usage: |
| # docker-compose build conda-cpp |
| # docker-compose build conda-integration |
| # docker-compose run conda-integration |
| image: ${REPO}:${ARCH}-conda-integration |
| build: |
| context: . |
| dockerfile: ci/docker/conda-integration.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-integration |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| jdk: ${JDK} |
| # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should |
| # be set to ${MAVEN} |
| maven: 3.5 |
| node: ${NODE} |
| go: ${GO} |
| volumes: *conda-volumes |
| environment: |
| <<: [*common, *ccache] |
| # tell archery where the arrow binaries are located |
| ARROW_CPP_EXE_PATH: /build/cpp/debug |
| ARCHERY_INTEGRATION_WITH_RUST: 0 |
| command: |
| ["/arrow/ci/scripts/rust_build.sh /arrow /build && |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/csharp_build.sh /arrow /build && |
| /arrow/ci/scripts/go_build.sh /arrow && |
| /arrow/ci/scripts/java_build.sh /arrow /build && |
| /arrow/ci/scripts/js_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_arrow.sh /arrow /build"] |
| |
| ################################ Docs ####################################### |
| |
| ubuntu-docs: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-python |
| # docker-compose build ubuntu-docs |
| # docker-compose run --rm ubuntu-docs |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-docs |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-docs.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-docs |
| args: |
| r: ${R} |
| jdk: ${JDK} |
| node: ${NODE} |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 |
| environment: |
| <<: [*common, *ccache] |
| ARROW_CUDA: "ON" |
| ARROW_CXX_FLAGS_DEBUG: "-g1" |
| ARROW_C_FLAGS_DEBUG: "-g1" |
| ARROW_JAVA_SKIP_GIT_PLUGIN: |
| ARROW_SUBSTRAIT: "ON" |
| BUILD_DOCS_C_GLIB: "ON" |
| BUILD_DOCS_CPP: "ON" |
| BUILD_DOCS_JAVA: "ON" |
| BUILD_DOCS_JS: "ON" |
| BUILD_DOCS_PYTHON: "ON" |
| BUILD_DOCS_R: "ON" |
| Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 |
| volumes: *ubuntu-volumes |
| command: &docs-command > |
| /bin/bash -c " |
| /arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/c_glib_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/r_build.sh /arrow /build && |
| /arrow/ci/scripts/js_build.sh /arrow /build && |
| /arrow/ci/scripts/java_build.sh /arrow /build" |
| |
| ################################# Tools ##################################### |
| |
| ubuntu-lint: |
| # Usage: |
| # docker-compose build ubuntu-cpp |
| # docker-compose build ubuntu-lint |
| # docker-compose run ubuntu-lint |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint |
| build: |
| context: . |
| dockerfile: ci/docker/linux-apt-lint.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint |
| args: |
| base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp |
| clang_tools: ${CLANG_TOOLS} |
| environment: |
| <<: [*common, *ccache] |
| volumes: *ubuntu-volumes |
| command: > |
| /bin/bash -c " |
| git config --global --add safe.directory /arrow && |
| archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc" |
| |
| ######################### Integration Tests ################################# |
| |
| postgres: |
| # required for the impala service |
| image: postgres |
| ports: |
| - 5432:5432 |
| environment: |
| POSTGRES_PASSWORD: postgres |
| |
| impala: |
| # required for the hdfs tests |
| image: ibisproject/impala:latest |
| hostname: impala |
| links: |
| - postgres:postgres |
| environment: |
| PGPASSWORD: postgres |
| ports: |
| # HDFS |
| - 9020:9020 |
| - 50070:50070 |
| - 50075:50075 |
| - 8020:8020 |
| - 8042:8042 |
| # Hive |
| - 9083:9083 |
| # Impala |
| - 21000:21000 |
| - 21050:21050 |
| - 25000:25000 |
| - 25010:25010 |
| - 25020:25020 |
| |
| conda-python-hdfs: |
| # Usage: |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-hdfs |
| # docker-compose run conda-python-hdfs |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-hdfs.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| jdk: ${JDK} |
| # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should |
| # be set to ${MAVEN} |
| maven: 3.5 |
| hdfs: ${HDFS} |
| links: |
| - impala:impala |
| environment: |
| <<: [*common, *ccache] |
| ARROW_ENGINE: "OFF" |
| ARROW_FLIGHT: "OFF" |
| ARROW_FLIGHT_SQL: "OFF" |
| ARROW_HDFS: "ON" |
| ARROW_HDFS_TEST_HOST: impala |
| ARROW_HDFS_TEST_PORT: 8020 |
| ARROW_HDFS_TEST_USER: hdfs |
| ARROW_S3: "OFF" |
| CMAKE_UNITY_BUILD: "ON" |
| shm_size: *shm-size |
| volumes: &conda-maven-volumes |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_hdfs.sh /arrow /build"] |
| |
| conda-python-spark: |
| # Usage: |
| # docker-compose build conda-cpp |
| # docker-compose build conda-python |
| # docker-compose build conda-python-spark |
| # docker-compose run conda-python-spark |
| image: ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} |
| build: |
| context: . |
| dockerfile: ci/docker/conda-python-spark.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK} |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| python: ${PYTHON} |
| jdk: ${JDK} |
| # conda-forge doesn't have 3.5.4 so pinning explicitly, but this should |
| # be set to ${MAVEN} |
| maven: 3.5 |
| spark: ${SPARK} |
| numpy: ${NUMPY} |
| shm_size: *shm-size |
| environment: |
| <<: [*common, *ccache] |
| volumes: *conda-maven-volumes |
| command: |
| ["/arrow/ci/scripts/cpp_build.sh /arrow /build && |
| /arrow/ci/scripts/python_build.sh /arrow /build && |
| /arrow/ci/scripts/java_build.sh /arrow /build && |
| /arrow/ci/scripts/integration_spark.sh /arrow /spark ${TEST_PYARROW_ONLY:-false}"] |
| |
| ################################# Source Verification ##################################### |
| |
| conda-verify-rc: |
| image: ubuntu:${UBUNTU} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| DEBIAN_FRONTEND: "noninteractive" |
| DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| USE_CONDA: 1 |
| command: > |
| /bin/bash -c " |
| apt update -y && apt install -y curl git gnupg tzdata wget && |
| git config --global --add safe.directory /arrow && |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
| |
| almalinux-verify-rc: |
| # Usage: |
| # docker-compose build almalinux-verify-rc |
| # docker-compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 almalinux-verify-rc |
| # Parameters: |
| # ALMALINUX: 8 |
| image: ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc |
| build: |
| context: . |
| dockerfile: ci/docker/almalinux-${ALMALINUX}-verify-rc.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}almalinux-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| command: > |
| /bin/bash -c " |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
| |
| ubuntu-verify-rc: |
| # Usage: |
| # docker-compose build ubuntu-verify-rc |
| # docker-compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 ubuntu-verify-rc |
| # Parameters: |
| # UBUNTU: 20.04, 22.04 |
| image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc |
| build: |
| context: . |
| dockerfile: ci/docker/ubuntu-${UBUNTU}-verify-rc.dockerfile |
| cache_from: |
| - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc |
| args: |
| repo: ${REPO} |
| arch: ${ARCH} |
| volumes: |
| - .:/arrow:delegated |
| - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated |
| shm_size: '1gb' |
| environment: |
| <<: [*common, *ccache] |
| CMAKE_GENERATOR: Ninja |
| TEST_APT: 0 # would require docker-in-docker |
| TEST_YUM: 0 |
| command: > |
| /bin/bash -c " |
| git config --global --add safe.directory /arrow && |
| /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}" |
