ARROW-12262: [Doc] Enable S3 and Flight in docs build
Also use Ubuntu 20.04 by default, instead of 18.04.
Closes #9933 from pitrou/ARROW-12262-ubuntu-docs
Authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>
diff --git a/.env b/.env
index cd6b57e..c4eb6d2 100644
--- a/.env
+++ b/.env
@@ -42,7 +42,7 @@
REPO=apache/arrow-dev
CUDA=9.1
DEBIAN=10
-UBUNTU=18.04
+UBUNTU=20.04
FEDORA=33
PYTHON=3.6
LLVM=11
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index 5f25deb..0bcf346 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -59,7 +59,7 @@
- image: conda-cpp
title: AMD64 Conda C++
- image: ubuntu-cpp-sanitizer
- title: AMD64 Ubuntu 18.04 C++ ASAN UBSAN
+ title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN
steps:
- name: Checkout Arrow
uses: actions/checkout@v2
diff --git a/ci/docker/linux-apt-docs.dockerfile b/ci/docker/linux-apt-docs.dockerfile
index 604a05a..46c31bb 100644
--- a/ci/docker/linux-apt-docs.dockerfile
+++ b/ci/docker/linux-apt-docs.dockerfile
@@ -18,9 +18,10 @@
ARG base
FROM ${base}
-ARG r=3.6
+ARG r=4.0
ARG jdk=8
+# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/
RUN apt-get update -y && \
apt-get install -y \
dirmngr \
@@ -29,8 +30,8 @@
apt-key adv \
--keyserver keyserver.ubuntu.com \
--recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
- add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran35/' && \
- apt-get install -y \
+ add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \
+ apt-get install -y --no-install-recommends \
autoconf-archive \
automake \
curl \
@@ -43,6 +44,7 @@
libgirepository1.0-dev \
libglib2.0-doc \
libharfbuzz-dev \
+ libtiff-dev \
libtool \
libxml2-dev \
ninja-build \
@@ -72,7 +74,6 @@
rm -rf /var/lib/apt/lists/* && \
npm install -g yarn
-# Sphinx is pinned because of ARROW-9693
RUN pip install \
meson \
breathe \
@@ -96,8 +97,11 @@
RUN /arrow/ci/scripts/r_deps.sh /arrow && \
R -e "install.packages('pkgdown')"
-ENV ARROW_PYTHON=ON \
+ENV ARROW_FLIGHT=ON \
+ ARROW_PYTHON=ON \
+ ARROW_S3=ON \
ARROW_BUILD_STATIC=OFF \
ARROW_BUILD_TESTS=OFF \
ARROW_BUILD_UTILITIES=OFF \
ARROW_USE_GLOG=OFF \
+ CMAKE_UNITY_BUILD=ON \
diff --git a/ci/docker/ubuntu-20.10-cpp.dockerfile b/ci/docker/ubuntu-20.10-cpp.dockerfile
new file mode 100644
index 0000000..80eb072
--- /dev/null
+++ b/ci/docker/ubuntu-20.10-cpp.dockerfile
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base=amd64/ubuntu:20.10
+FROM ${base}
+ARG arch
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN echo "debconf debconf/frontend select Noninteractive" | \
+ debconf-set-selections
+
+# Installs LLVM toolchain, for Gandiva and testing other compilers
+#
+# Note that this is installed before the base packages to improve iteration
+# while debugging package list with docker build.
+ARG clang_tools
+ARG llvm
+RUN if [ "${llvm}" -gt "10" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \
+ /etc/apt/sources.list.d/llvm.list && \
+ if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+ echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \
+ /etc/apt/sources.list.d/clang-tools.list; \
+ fi \
+ fi && \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ clang-${clang_tools} \
+ clang-${llvm} \
+ clang-format-${clang_tools} \
+ clang-tidy-${clang_tools} \
+ llvm-${llvm}-dev && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists*
+
+# Installs C++ toolchain and dependencies
+RUN apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ autoconf \
+ ca-certificates \
+ ccache \
+ cmake \
+ g++ \
+ gcc \
+ gdb \
+ git \
+ libbenchmark-dev \
+ libboost-filesystem-dev \
+ libboost-system-dev \
+ libbrotli-dev \
+ libbz2-dev \
+ libgflags-dev \
+ libcurl4-openssl-dev \
+ libgoogle-glog-dev \
+ libgrpc++-dev \
+ liblz4-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libre2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libthrift-dev \
+ libutf8proc-dev \
+ libzstd-dev \
+ make \
+ ninja-build \
+ pkg-config \
+ protobuf-compiler \
+ protobuf-compiler-grpc \
+ rapidjson-dev \
+ tzdata \
+ wget && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists*
+
+COPY ci/scripts/install_minio.sh \
+ /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
+# Prioritize system packages and local installation
+# The following dependencies will be downloaded due to missing/invalid packages
+# provided by the distribution:
+# - libc-ares-dev does not install CMake config files
+# - flatbuffer is not packaged
+# - libgtest-dev only provide sources
+# - libprotobuf-dev only provide sources
+ENV ARROW_BUILD_TESTS=ON \
+ ARROW_DEPENDENCY_SOURCE=SYSTEM \
+ ARROW_DATASET=ON \
+ ARROW_FLIGHT=OFF \
+ ARROW_GANDIVA=ON \
+ ARROW_HDFS=ON \
+ ARROW_HOME=/usr/local \
+ ARROW_INSTALL_NAME_RPATH=OFF \
+ ARROW_NO_DEPRECATED_API=ON \
+ ARROW_ORC=ON \
+ ARROW_PARQUET=ON \
+ ARROW_PLASMA=ON \
+ ARROW_S3=ON \
+ ARROW_USE_ASAN=OFF \
+ ARROW_USE_CCACHE=ON \
+ ARROW_USE_UBSAN=OFF \
+ ARROW_WITH_BROTLI=ON \
+ ARROW_WITH_BZ2=ON \
+ ARROW_WITH_LZ4=ON \
+ ARROW_WITH_SNAPPY=ON \
+ ARROW_WITH_ZLIB=ON \
+ ARROW_WITH_ZSTD=ON \
+ AWSSDK_SOURCE=BUNDLED \
+ GTest_SOURCE=BUNDLED \
+ ORC_SOURCE=BUNDLED \
+ PARQUET_BUILD_EXAMPLES=ON \
+ PARQUET_BUILD_EXECUTABLES=ON \
+ PATH=/usr/lib/ccache/:$PATH \
+ PYTHON=python3
diff --git a/ci/scripts/docs_build.sh b/ci/scripts/docs_build.sh
index a0d926a..e6ee768 100755
--- a/ci/scripts/docs_build.sh
+++ b/ci/scripts/docs_build.sh
@@ -27,8 +27,10 @@
export CFLAGS="-DARROW_NO_DEPRECATED_API"
export CXXFLAGS="-DARROW_NO_DEPRECATED_API"
-# Prose and Python
-sphinx-build -b html ${arrow_dir}/docs/source ${build_dir}
+ncpus=$(python3 -c "import os; print(os.cpu_count())")
+
+# Sphinx docs
+sphinx-build -b html -j ${ncpus} ${arrow_dir}/docs/source ${build_dir}
# C++ - original doxygen
# rsync -a ${arrow_dir}/cpp/apidoc/ ${build_dir}/cpp
diff --git a/dev/release/post-09-docs.sh b/dev/release/post-09-docs.sh
index 51e7454..c9f75b4 100755
--- a/dev/release/post-09-docs.sh
+++ b/dev/release/post-09-docs.sh
@@ -46,6 +46,7 @@
archery docker run \
-v "${ARROW_SITE_DIR}/docs:/build/docs" \
-e ARROW_DOCS_VERSION="${version}" \
+ -e UBUNTU=20.10 \
ubuntu-docs
: ${PUSH:=1}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index dcf4941..0080b38 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -674,6 +674,8 @@
params:
target: "ubuntu-bionic"
task_namespace: "apt"
+ env:
+ UBUNTU: 18.04
upload_extensions:
- .ddeb
- .deb
@@ -1474,6 +1476,8 @@
ci: azure
template: docker-tests/azure.linux.yml
params:
+ env:
+ UBUNTU: 18.04
run: ubuntu-r-sanitizer
test-debian-10-go-1.15:
@@ -1484,12 +1488,12 @@
GO: 1.15
run: debian-go
- test-ubuntu-18.04-docs:
+ test-ubuntu-20.10-docs:
ci: azure
template: docker-tests/azure.linux.yml
params:
env:
- UBUNTU: 18.04
+ UBUNTU: "20.10"
run: ubuntu-docs
############################## vcpkg tests ##################################
diff --git a/docker-compose.yml b/docker-compose.yml
index 539d5ad..b22b1f6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -366,6 +366,7 @@
ARROW_FUZZING: "ON" # Check fuzz regressions
ARROW_JEMALLOC: "OFF"
ARROW_ORC: "OFF"
+ ARROW_S3: "OFF"
ARROW_USE_ASAN: "ON"
ARROW_USE_UBSAN: "ON"
# utf8proc 2.1.0 in Ubuntu Bionic has test failures
diff --git a/docs/source/cpp/examples/row_columnar_conversion.rst b/docs/source/cpp/examples/row_columnar_conversion.rst
index 02fd61b..3f45864 100644
--- a/docs/source/cpp/examples/row_columnar_conversion.rst
+++ b/docs/source/cpp/examples/row_columnar_conversion.rst
@@ -24,4 +24,4 @@
The following example converts an array of structs to a :class:`arrow::Table`
instance, and then converts it back to the original array of structs.
-.. literalinclude:: ../../../../cpp/examples/arrow/row-wise-conversion-example.cc
+.. literalinclude:: ../../../../cpp/examples/arrow/row_wise_conversion_example.cc
diff --git a/docs/source/status.rst b/docs/source/status.rst
index acf5af9..176d35e 100644
--- a/docs/source/status.rst
+++ b/docs/source/status.rst
@@ -126,7 +126,7 @@
+-----------------------------+-------+-------+-------+------------+-------+-------+-------+
| Sparse tensors | ✓ | | | | | | |
+-----------------------------+-------+-------+-------+------------+-------+-------+-------+
-| Buffer compression | ✓ | ✓ (3) | ✓ | | | | ✓ |
+| Buffer compression | ✓ | ✓ (3) | ✓ | | | | ✓ |
+-----------------------------+-------+-------+-------+------------+-------+-------+-------+
| Endianness conversion | ✓ (2) | | | | | | |
+-----------------------------+-------+-------+-------+------------+-------+-------+-------+
