# Licensed to the Apache Software Foundation (ASF) under one | |
# or more contributor license agreements. See the NOTICE file | |
# distributed with this work for additional information | |
# regarding copyright ownership. The ASF licenses this file | |
# to you under the Apache License, Version 2.0 (the | |
# "License"); you may not use this file except in compliance | |
# with the License. You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, | |
# software distributed under the License is distributed on an | |
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
# KIND, either express or implied. See the License for the | |
# specific language governing permissions and limitations | |
# under the License. | |
name: Upload R Nightly builds | |
# This workflow downloads the (nightly) binaries created in crossbow and uploads them | |
# to nightlies.apache.org. Due to authorization requirements, this upload can't be done | |
# from the crossbow repository. | |
on: | |
workflow_dispatch: | |
inputs: | |
prefix: | |
description: Job prefix to use. | |
required: false | |
default: '' | |
keep: | |
description: Number of versions to keep. | |
required: false | |
default: 14 | |
schedule: | |
#Crossbow packaging runs at 0 8 * * * | |
- cron: '0 14 * * *' | |
permissions: | |
contents: read | |
jobs: | |
upload: | |
if: github.repository == 'apache/arrow' | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout Arrow | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 1 | |
path: arrow | |
repository: apache/arrow | |
ref: main | |
submodules: recursive | |
- name: Checkout Crossbow | |
uses: actions/checkout@v3 | |
with: | |
fetch-depth: 0 | |
path: crossbow | |
repository: ursacomputing/crossbow | |
ref: master | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
cache: 'pip' | |
python-version: 3.8 | |
- name: Install Archery | |
shell: bash | |
run: pip install -e arrow/dev/archery[all] | |
- run: mkdir -p binaries | |
- name: Download Artifacts | |
env: | |
PREFIX: ${{ github.event.inputs.prefix || ''}} | |
run: | | |
if [ -z $PREFIX ]; then | |
PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 | |
fi | |
echo $PREFIX | |
archery crossbow download-artifacts -f r-binary-packages -t binaries $PREFIX | |
if [ -n "$(ls -A binaries/*/*/)" ]; then | |
echo "Found files!" | |
else | |
echo "No files found. Stopping upload." | |
exit 1 | |
fi | |
- name: Cache Repo | |
uses: actions/cache@v3 | |
with: | |
path: repo | |
key: r-nightly-${{ github.run_id }} | |
restore-keys: r-nightly- | |
- name: Sync from Remote | |
uses: ./arrow/.github/actions/sync-nightlies | |
with: | |
switches: -avzh --update --delete --progress | |
local_path: repo | |
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r | |
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} | |
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} | |
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} | |
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} | |
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} | |
- run: tree repo | |
- name: Build Repository | |
shell: Rscript {0} | |
run: | | |
# folder that we sync to nightlies.apache.org | |
repo_root <- "repo" | |
# The binaries are in a nested dir | |
# so we need to find the correct path. | |
art_path <- list.files("binaries", | |
recursive = TRUE, | |
include.dirs = TRUE, | |
pattern = "r-binary-packages$", | |
full.names = TRUE | |
) | |
current_path <- list.files(art_path, full.names = TRUE, recursive = TRUE) | |
files <- sub("r-(pkg|lib)", repo_root, current_path) | |
# decode contrib.url from artifact name: | |
# bin__windows__contrib__4.1 -> bin/windows/contrib/4.1 | |
new_paths <- gsub("__", "/", files) | |
# strip superfluous nested dirs | |
new_paths <- sub(art_path, ".", new_paths) | |
dirs <- dirname(new_paths) | |
sapply(dirs, dir.create, recursive = TRUE, showWarnings = FALSE) | |
# overwrite allows us to "force push" a new version with the same name | |
copy_result <- file.copy(current_path, new_paths, overwrite = TRUE) | |
if (!all(copy_result)) { | |
stop("There was an issue while copying the files!") | |
} | |
- name: Prune Repository | |
shell: bash | |
env: | |
KEEP: ${{ github.event.inputs.keep || 14 }} | |
run: | | |
prune() { | |
# list files | retain $KEEP newest files | delete everything else | |
ls -t $1/arrow* | tail -n +$((KEEP + 1)) | xargs --no-run-if-empty rm | |
} | |
# find leaf sub dirs | |
repo_dirs=$(find repo -type d -links 2) | |
# We want to retain $keep (14) versions of each pkg/lib so we call | |
# prune on each leaf dir and not on repo/. | |
for dir in ${repo_dirs[@]}; do | |
prune $dir | |
done | |
- name: Update Repository Index | |
shell: Rscript {0} | |
run: | | |
# folder that we sync to nightlies.apache.org | |
repo_root <- "repo" | |
tools::write_PACKAGES(file.path(repo_root, "src/contrib"), | |
type = "source", | |
verbose = TRUE, | |
latestOnly = FALSE | |
) | |
repo_dirs <- list.dirs(repo_root) | |
# find dirs with binary R packages: e.g. */contrib/4.1 | |
pkg_dirs <- grep(".+contrib\\/\\d.+", repo_dirs, value = TRUE) | |
for (dir in pkg_dirs) { | |
on_win <- grepl("windows", dir) | |
tools::write_PACKAGES(dir, | |
type = ifelse(on_win, "win.binary", "mac.binary"), | |
verbose = TRUE, | |
latestOnly = FALSE | |
) | |
} | |
- name: Show repo contents | |
run: tree repo | |
- name: Sync to Remote | |
uses: ./arrow/.github/actions/sync-nightlies | |
with: | |
upload: true | |
switches: -avzh --update --delete --progress | |
local_path: repo | |
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/r | |
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} | |
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} | |
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} | |
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} | |
remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} |