chore(ci): Add Python packaging tools to images + fix s390x build (#206)
s390x was failing because it was running `docker compose build` and
there are now some compose jobs that aren't intended to run on s390x
(e.g., documentation, coverage). Python build tools aren't essential yet
but will be at some point and updating the images is a bit of a pain.
diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml
index fe824d7..39e24e7 100644
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -76,7 +76,7 @@
NANOARROW_PLATFORM: ${{ matrix.config.platform }}
DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
run: |
- docker compose build
+ docker compose build verify
- name: Push
if: ${{ github.event_name != 'pull_request' && github.repository == 'apache/arrow-nanoarrow'}}
@@ -86,3 +86,54 @@
DOCKER_DEFAULT_PLATFORM: "linux/${{ matrix.config.arch }}"
run: |
docker compose push
+
+ # Build some arch-agnostic images for non-verify docker bits
+ build-docker-manifest:
+ runs-on: ubuntu-latest
+ needs: build-docker
+ steps:
+ - name: Build
+ run: |
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu \
+ ghcr.io/apache/arrow-nanoarrow:ubuntu-arm64 --arch arm64
+
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:centos7 \
+ ghcr.io/apache/arrow-nanoarrow:centos7-arm64 --arch arm64
+
+ docker manifest create \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-amd64 \
+ ghcr.io/apache/arrow-nanoarrow:alpine-arm64 \
+ ghcr.io/apache/arrow-nanoarrow:alpine-s390x
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-amd64 --arch amd64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-arm64 --arch arm64
+ docker manifest annotate \
+ ghcr.io/apache/arrow-nanoarrow:alpine \
+ ghcr.io/apache/arrow-nanoarrow:alpine-s390x --arch s390x
+
+ - name: Push
+ if: ${{ github.event_name != 'pull_request' && github.repository == 'apache/arrow-nanoarrow'}}
+ run: |
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:ubuntu
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:centos7
+ docker manifest push ghcr.io/apache/arrow-nanoarrow:alpine
diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
index 0e736eb..6bf5a6f 100644
--- a/ci/docker/alpine.dockerfile
+++ b/ci/docker/alpine.dockerfile
@@ -19,7 +19,13 @@
FROM --platform=linux/${NANOARROW_ARCH} alpine:latest
-RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl
+RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl py3-pip python3-dev
+
+RUN pip3 install build Cython pytest
+
+# There's a missing define that numpy's build needs on s390x and there is no wheel
+RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true
+RUN pip3 install numpy
# For Arrow C++
RUN curl -L https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz | tar -zxf - && \
@@ -33,6 +39,8 @@
cmake --install . --prefix=../arrow
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN rm -f ~/.R/Makevars
ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/archlinux.dockerfile b/ci/docker/archlinux.dockerfile
index 810d00a..9446e9a 100644
--- a/ci/docker/archlinux.dockerfile
+++ b/ci/docker/archlinux.dockerfile
@@ -17,7 +17,11 @@
FROM archlinux:latest
-RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow
+RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow python-pip
+
+RUN pip3 install build Cython numpy pytest pyarrow
# For R
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/centos7.dockerfile b/ci/docker/centos7.dockerfile
index 0009dc6..b76f56e 100644
--- a/ci/docker/centos7.dockerfile
+++ b/ci/docker/centos7.dockerfile
@@ -20,7 +20,8 @@
FROM --platform=linux/${NANOARROW_ARCH} centos:7
RUN yum install -y epel-release
-RUN yum install -y git gnupg curl R gcc-c++ cmake3
+RUN yum install -y git gnupg curl R gcc-c++ cmake3 python3-devel
+RUN pip3 install build Cython numpy pytest
RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
ENV LC_ALL en_US.UTF-8
@@ -37,7 +38,9 @@
make install
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN rm -f ~/.R/Makevars
ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
ENV CMAKE_BIN cmake3
diff --git a/ci/docker/fedora.dockerfile b/ci/docker/fedora.dockerfile
index a03de1a..a42455a 100644
--- a/ci/docker/fedora.dockerfile
+++ b/ci/docker/fedora.dockerfile
@@ -19,7 +19,12 @@
FROM --platform=linux/${NANOARROW_ARCH} fedora:latest
-RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en
+RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en \
+ python3-pip python3-devel
+
+RUN pip3 install build Cython numpy pytest pyarrow
# For R. Note that arrow is not installed (takes too long).
+RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile
index 788b0b6..93d2bc1 100644
--- a/ci/docker/ubuntu.dockerfile
+++ b/ci/docker/ubuntu.dockerfile
@@ -20,7 +20,7 @@
FROM --platform=linux/${NANOARROW_ARCH} ubuntu:latest
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
- locales git cmake r-base gnupg curl valgrind python3-pip doxygen pandoc lcov \
+ locales git cmake r-base gnupg curl valgrind python3-pip python3-venv doxygen pandoc lcov \
libxml2-dev libfontconfig1-dev libfreetype6-dev libfribidi-dev libharfbuzz-dev \
libjpeg-dev libpng-dev libtiff-dev
@@ -33,13 +33,15 @@
apt-get update && \
apt-get install -y -V libarrow-dev
-# For documentation build
-RUN pip3 install pydata-sphinx-theme sphinx breathe
+# For documentation build + Python build
+RUN pip3 install pydata-sphinx-theme sphinx breathe build Cython numpy pytest pyarrow
# For R. Note that we install arrow here so that the integration tests for R run
# in at least one test image.
+RUN mkdir ~/.R && echo "MAKEFLAGS += -j$(nproc)" > ~/.R/Makevars
RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "pkgdown", "covr"), repos = "https://cloud.r-project.org")'
# Required for this to work on MacOS/arm64
-RUN mkdir ~/.R && echo "CXX17FLAGS += -fPIC" > ~/.R/Makevars
+RUN echo "CXX17FLAGS += -fPIC" >> ~/.R/Makevars
RUN ARROW_USE_PKG_CONFIG=false ARROW_R_DEV=true R -e 'install.packages("arrow", repos = "https://cloud.r-project.org"); library(arrow)'
+RUN rm -f ~/.R/Makevars
