| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| {% import 'macros.jinja' as macros with context %} | |
| {% set is_fork = macros.is_fork %} | |
| {{ macros.github_header() }} | |
| jobs: | |
| source: | |
| # This job will change the version to either the custom_version param or YMD format. | |
| # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) | |
| name: Source Package | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} | |
| steps: | |
| {{ macros.github_checkout_arrow()|indent }} | |
| {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} | |
| - name: Save Version | |
| id: save-version | |
| shell: bash | |
| run: | | |
| echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| install-r: true | |
| - name: Build R source package | |
| shell: bash | |
| run: | | |
| cd arrow/r | |
| # Copy in the Arrow C++ source | |
| make sync-cpp | |
| R CMD build --no-build-vignettes . | |
| - name: Upload package artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: r-pkg__src__contrib | |
| path: arrow/r/arrow_*.tar.gz | |
| macos-cpp: | |
| name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} | |
| runs-on: {{ '${{ matrix.platform.runs_on }}' }} | |
| needs: source | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - { runs_on: macos-15-intel, arch: "x86_64" } | |
| - { runs_on: macos-14, arch: "arm64" } | |
| env: | |
| PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} | |
| PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip | |
| steps: | |
| {{ macros.github_checkout_arrow(action_v="3")|indent }} | |
| {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} | |
| - name: Install Deps | |
| run: | | |
| brew install sccache ninja | |
| brew install openssl@3.0 | |
| - name: Build libarrow | |
| shell: bash | |
| env: | |
| {{ macros.github_set_sccache_envvars()|indent(8) }} | |
| MACOSX_DEPLOYMENT_TARGET: "11.6" | |
| ARROW_S3: ON | |
| ARROW_GCS: ON | |
| ARROW_DEPENDENCY_SOURCE: BUNDLED | |
| CMAKE_GENERATOR: Ninja | |
| LIBARROW_MINIMAL: false | |
| run: | | |
| sccache --start-server | |
| export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3.0)" | |
| cd arrow | |
| r/inst/build_arrow_static.sh | |
| - name: Bundle libarrow | |
| shell: bash | |
| run: | | |
| cd arrow/r/libarrow/dist | |
| zip -r $PKG_FILE lib/ include/ | |
| - name: Create Checksum | |
| shell: bash | |
| run: | | |
| cd arrow/r/libarrow/dist | |
| shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 | |
| mv ${PKG_FILE}{,.sha512} ../../../../ | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: {{ '${{ env.PKG_ID }}' }} | |
| path: | | |
| {{ '${{ env.PKG_FILE }}' }} | |
| {{ '${{ env.PKG_FILE }}' }}.sha512 | |
| linux-cpp: | |
| name: C++ Binary Linux | |
| runs-on: ubuntu-latest | |
| needs: source | |
| strategy: | |
| fail-fast: false | |
| env: | |
| PKG_ID: r-libarrow-linux-x86_64 | |
| PKG_FILE: r-libarrow-linux-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip | |
| steps: | |
| {{ macros.github_checkout_arrow()|indent }} | |
| {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} | |
| {{ macros.github_install_archery()|indent }} | |
| - name: Build libarrow | |
| shell: bash | |
| env: | |
| UBUNTU: "22.04" | |
| {{ macros.github_set_sccache_envvars()|indent(8) }} | |
| run: | | |
| source arrow/ci/scripts/util_enable_core_dumps.sh | |
| archery docker run ubuntu-cpp-static | |
| - name: Bundle libarrow | |
| shell: bash | |
| run: | | |
| # These files were created by the docker user so we have to chown them | |
| sudo chown -R $USER:$USER arrow/r/libarrow | |
| cd arrow/r/libarrow/dist | |
| zip -r $PKG_FILE lib/ include/ | |
| - name: Create Checksum | |
| shell: bash | |
| run: | | |
| cd arrow/r/libarrow/dist | |
| shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 | |
| mv ${PKG_FILE}{,.sha512} ../../../../ | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: {{ '${{ env.PKG_ID }}' }} | |
| path: | | |
| {{ '${{ env.PKG_FILE }}' }} | |
| {{ '${{ env.PKG_FILE }}' }}.sha512 | |
| windows-cpp: | |
| name: C++ Binary Windows RTools (40 only) | |
| needs: source | |
| runs-on: windows-latest | |
| env: | |
| PKG_ID: r-libarrow-windows-x86_64 | |
| PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip | |
| steps: | |
| - run: git config --global core.autocrlf false | |
| {{ macros.github_checkout_arrow()|indent }} | |
| {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| rtools-version: 40 | |
| r-version: "4.0" | |
| Ncpus: 2 | |
| - name: Install sccache | |
| shell: bash | |
| run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache | |
| - name: Build Arrow C++ with rtools40 | |
| shell: bash | |
| env: | |
| ARROW_HOME: "arrow" | |
| MINGW_ARCH: "ucrt64" | |
| RTOOLS_VERSION: 40 | |
| {{ macros.github_set_sccache_envvars()|indent(8) }} | |
| run: arrow/ci/scripts/r_windows_build.sh | |
| - name: Create Checksum | |
| shell: bash | |
| run: | | |
| cd build | |
| sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 | |
| mv ${PKG_FILE}{,.sha512} ../ | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: {{ '${{ env.PKG_ID }}' }} | |
| path: | | |
| {{ '${{ env.PKG_FILE }}' }} | |
| {{ '${{ env.PKG_FILE }}' }}.sha512 | |
| r-packages: | |
| needs: [source, windows-cpp, macos-cpp] | |
| name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} | |
| runs-on: {{ '${{ matrix.platform.runs_on }}' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - { runs_on: 'windows-latest', name: "Windows"} | |
| - { runs_on: macos-15-intel, name: "macOS x86_64"} | |
| - { runs_on: macos-14, name: "macOS arm64" } | |
| r_version: [oldrel, release] | |
| steps: | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: {{ '${{ matrix.r_version }}' }} | |
| {{ macros.github_setup_local_r_repo(false, true, true)|indent }} | |
| - name: Prepare Dependency Installation | |
| shell: bash | |
| run: | | |
| tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION | |
| - name: Install dependencies | |
| uses: r-lib/actions/setup-r-dependencies@v2 | |
| with: | |
| working-directory: 'arrow' | |
| extra-packages: cpp11 | |
| - name: Set CRAN like openssl | |
| if: contains(matrix.platform.name, 'macOS') | |
| # The -E forwards the GITHUB_* envvars | |
| shell: sudo -E Rscript {0} | |
| run: | | |
| # get the mac-recipes version of openssl from CRAN | |
| source("https://mac.R-project.org/bin/install.R") | |
| install.libs("openssl") | |
| # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. | |
| if (Sys.info()[["machine"]] == "arm64"){ | |
| cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) | |
| } else { | |
| cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) | |
| } | |
| - name: Build Binary | |
| id: build | |
| shell: Rscript {0} | |
| env: | |
| NOT_CRAN: "false" # actions/setup-r sets this implicitly | |
| ARROW_R_DEV: "true" | |
| LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated | |
| LIBARROW_BUILD: "false" | |
| ARROW_R_ENFORCE_CHECKSUM: "true" | |
| ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" | |
| run: | | |
| on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" | |
| # Build | |
| Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) | |
| INSTALL_opts <- "--build" | |
| if (!on_windows) { | |
| # Windows doesn't support the --strip arg | |
| INSTALL_opts <- c(INSTALL_opts, "--strip") | |
| } | |
| # always remove arrow (mainly for self-hosted runners) | |
| try(remove.packages("arrow"), silent = TRUE) | |
| cat("Install arrow from dev repo.\n") | |
| install.packages( | |
| "arrow", | |
| type = "source", | |
| # The sub is necessary to prevent an error on windows. | |
| repos = sub("file://", "file:", getOption("arrow.dev_repo")),, | |
| INSTALL_opts = INSTALL_opts | |
| ) | |
| # Test | |
| library(arrow) | |
| arrow_info() | |
| read_parquet(system.file("v0.7.1.parquet", package = "arrow")) | |
| # encode contrib.url for artifact name | |
| cmd <- paste0( | |
| "path=", | |
| gsub( | |
| "/", "__", | |
| contrib.url("", type = "binary") | |
| ), | |
| "\n" | |
| ) | |
| cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: r-pkg{{ '${{ steps.build.outputs.path }}' }} | |
| path: arrow_* | |
| test-linux-binary: | |
| needs: [source, linux-cpp] | |
| name: Test binary {{ '${{ matrix.config.image }}' }} | |
| runs-on: ubuntu-latest | |
| container: {{ '${{ matrix.config.image }}' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| # If libarrow_binary is unset, we're testing that we're automatically | |
| # choosing a binary on this OS. If libarrow_binary is TRUE, we're on | |
| # an OS that is not in the allowlist, so we have to opt-in to use the | |
| # binary. Other env vars used in r_docker_configure.sh can be added | |
| # here and wired up in the later steps. | |
| - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE"} | |
| # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) | |
| # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE"} | |
| - {image: "rhub/ubuntu-release"} # currently ubuntu-22.04 | |
| - {image: "rstudio/r-base:4.1-jammy"} | |
| - {image: "rstudio/r-base:4.2-jammy"} | |
| - {image: "rstudio/r-base:4.3-noble"} | |
| steps: | |
| # Get the arrow checkout just for the docker config scripts | |
| # Don't need submodules for this (hence false arg to macro): they fail on | |
| # actions/checkout for some reason in this context | |
| {{ macros.github_checkout_arrow(1, false, '3')|indent }} | |
| - name: Install system requirements | |
| env: | |
| ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh | |
| shell: bash | |
| run: | | |
| # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) | |
| echo "${RPREFIX}/bin" >> $GITHUB_PATH | |
| export PATH="${RPREFIX}/bin:${PATH}" | |
| cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh | |
| {{ macros.github_setup_local_r_repo(true, false)|indent }} | |
| - name: Install arrow from our repo | |
| env: | |
| ARROW_R_DEV: "TRUE" | |
| LIBARROW_BUILD: "FALSE" | |
| LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} | |
| ARROW_R_ENFORCE_CHECKSUM: "true" | |
| ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" | |
| shell: bash | |
| run: | | |
| Rscript -e ' | |
| {{ macros.github_test_r_src_pkg()|indent(8) }} | |
| ' | |
| - name: Upload binary artifact | |
| if: matrix.config.devtoolset | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: r-pkg_centos7 | |
| path: arrow_* | |
| test-centos-binary: | |
| # arrow binary package not on ppm currently see #37922 | |
| if: false | |
| needs: test-linux-binary | |
| runs-on: ubuntu-latest | |
| container: "rstudio/r-base:4.2-centos7" | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: r-pkg_centos7 | |
| - name: Install DTS Package | |
| shell: Rscript {0} | |
| run: | | |
| pkg <- list.files(pattern = "arrow_*") | |
| if(length(pkg) > 1) { | |
| pkg <- pkg[[1]] | |
| warning("Multiple packages found! Using first one.") | |
| } | |
| # Install dependencies from RSPM | |
| install.packages("arrow", repos = "https://packagemanager.rstudio.com/all/__linux__/centos7/latest") | |
| remove.packages("arrow") | |
| install.packages(pkg) | |
| library(arrow) | |
| read_parquet(system.file("v0.7.1.parquet", package = "arrow")) | |
| print(arrow_info()) | |
| test-source: | |
| needs: source | |
| name: Test {{ '${{ matrix.platform.name }}' }} source build | |
| runs-on: {{ '${{ matrix.platform.runs_on }}' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - {runs_on: "ubuntu-latest", name: "Linux"} | |
| - {runs_on: "macos-15-intel" , name: "macOS"} | |
| steps: | |
| - name: Install R | |
| uses: r-lib/actions/setup-r@v2 | |
| {{ macros.github_setup_local_r_repo(false, false)|indent }} | |
| {{ macros.github_checkout_arrow(action_v="3")|indent }} | |
| - name: Install sccache | |
| if: matrix.platform.name == 'Linux' | |
| shell: bash | |
| run: | | |
| arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin | |
| - name: Install R package system dependencies (Linux) | |
| if: matrix.platform.name == 'Linux' | |
| run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev | |
| - name: Install R package system dependencies (macOS) | |
| if: matrix.platform.name == 'macOS' | |
| run: brew install sccache openssl curl | |
| - name: Remove arrow/ | |
| run: | | |
| rm -rf arrow/ | |
| - name: Enable parallel build | |
| run: | | |
| cores=`nproc || sysctl -n hw.logicalcpu` | |
| echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV | |
| - name: Install arrow source package | |
| env: | |
| # Test source build so be sure not to download a binary | |
| LIBARROW_BINARY: "FALSE" | |
| {{ macros.github_set_sccache_envvars()|indent(8) }} | |
| shell: Rscript {0} | |
| run: | | |
| {{ macros.github_test_r_src_pkg()|indent(8) }} | |
| - name: Retry with verbosity if that failed | |
| if: failure() | |
| env: | |
| LIBARROW_BINARY: "FALSE" | |
| ARROW_R_DEV: "TRUE" | |
| CMAKE_FIND_DEBUG_MODE: "ON" | |
| {{ macros.github_set_sccache_envvars()|indent(8) }} | |
| shell: Rscript {0} | |
| run: | | |
| {{ macros.github_test_r_src_pkg()|indent(8) }} | |
| upload-binaries: | |
| # Only upload binaries if all tests pass. | |
| needs: [r-packages, test-source, test-linux-binary] | |
| name: Upload artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| {{ macros.github_checkout_arrow()|indent }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Install R | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| install-r: true | |
| - name: Move libarrow artifacts | |
| run: | | |
| mkdir -p binaries/ | |
| mv artifacts/r-libarrow-*/* binaries/ | |
| - name: Rename artifacts | |
| shell: Rscript {0} | |
| run: | | |
| file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) | |
| new_names <- file.path("binaries", sub("/", "__", file_paths)) | |
| dir.create("binaries", showWarnings = FALSE) | |
| file.copy(file.path("artifacts", file_paths), new_names) | |
| {{ macros.github_upload_releases("binaries/r-*")|indent }} |