dev/release/verify-apt.sh

#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

set -exu

if [ $# -lt 2 ]; then
  echo "Usage: $0 VERSION rc"
  echo "       $0 VERSION release"
  echo "       $0 VERSION local"
  echo " e.g.: $0 0.13.0 rc                # Verify 0.13.0 RC"
  echo " e.g.: $0 0.13.0 release           # Verify 0.13.0"
  echo " e.g.: $0 0.13.0-dev20210203 local # Verify 0.13.0-dev20210203 on local"
  exit 1
fi

VERSION="$1"
TYPE="$2"

SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TOP_SOURCE_DIR="${SOURCE_DIR}/../.."
local_prefix="${TOP_SOURCE_DIR}/dev/tasks/linux-packages"


echo "::group::Prepare repository"

export DEBIAN_FRONTEND=noninteractive

retry()
{
  local n_retries=0
  local max_n_retries=3
  while ! "$@"; do
    n_retries=$((n_retries + 1))
    if [ ${n_retries} -eq ${max_n_retries} ]; then
      echo "Failed: $@"
      return 1
    fi
    echo "Retry: $@"
  done
}

APT_INSTALL="retry apt install -y -V --no-install-recommends"

apt update
${APT_INSTALL} \
  base-files \
  ca-certificates \
  curl \
  lsb-release

code_name="$(lsb_release --codename --short)"
distribution="$(lsb_release --id --short | tr 'A-Z' 'a-z')"
artifactory_base_url="https://packages.apache.org/artifactory/arrow/${distribution}"
if [ "${TYPE}" = "rc" ]; then
  suffix=${TYPE%-release}
  artifactory_base_url+="-${suffix}"
fi

workaround_missing_packages=()
case "${distribution}-${code_name}" in
  debian-*)
    sed \
      -i"" \
      -e "s/ main$/ main contrib non-free/g" \
      /etc/apt/sources.list.d/debian.sources
    ;;
esac

if [ "${TYPE}" = "local" ]; then
  case "${VERSION}" in
    *-dev*)
      package_version="$(echo "${VERSION}" | sed -E -e 's/-(dev.*)$/~\1/g')"
      ;;
    *-rc*)
      package_version="$(echo "${VERSION}" | sed -e 's/-rc.*$//g')"
      ;;
    *)
      package_version="${VERSION}"
      ;;
  esac
  package_version+="-1"
  apt_source_path="${local_prefix}/apt/repositories"
  apt_source_path+="/${distribution}/pool/${code_name}/main"
  apt_source_path+="/a/apache-arrow-apt-source"
  apt_source_path+="/apache-arrow-apt-source_${package_version}_all.deb"
  find "${local_prefix}/apt/repositories/"
  ${APT_INSTALL} "${apt_source_path}"
else
  package_version="${VERSION}-1"
  apt_source_base_name="apache-arrow-apt-source-latest-${code_name}.deb"
  curl \
    --output "${apt_source_base_name}" \
    "${artifactory_base_url}/${apt_source_base_name}"
  ${APT_INSTALL} "./${apt_source_base_name}"
fi

if [ "${TYPE}" = "local" ]; then
  sed \
    -i.bak \
    -e "s,^URIs: .*$,URIs: file://${local_prefix}/apt/repositories/${distribution},g" \
    /etc/apt/sources.list.d/apache-arrow.sources
  keys="${local_prefix}/KEYS"
  if [ -f "${keys}" ]; then
    gpg \
      --no-default-keyring \
      --keyring /tmp/apache-arrow-apt-source.kbx \
      --import "${keys}"
    gpg \
      --no-default-keyring \
      --keyring /tmp/apache-arrow-apt-source.kbx \
      --armor \
      --export > /usr/share/keyrings/apache-arrow-apt-source.asc
  fi
elif [ "${TYPE}" = "rc" ]; then
  sed \
    -i.bak \
    -e "s,^URIs: \\(.*\\)/,URIs: \\1-${suffix}/,g" \
    /etc/apt/sources.list.d/apache-arrow.sources
fi

apt update

echo "::endgroup::"


echo "::group::Test Apache Arrow C++"
mkdir -p build
${APT_INSTALL} libarrow-dev=${package_version}
required_packages=()
required_packages+=(cmake)
required_packages+=(g++)
required_packages+=(git)
required_packages+=(make)
required_packages+=(pkg-config)
required_packages+=(${workaround_missing_packages[@]})
${APT_INSTALL} ${required_packages[@]}
# cmake version 3.31.6 -> 3.31.6
cmake_version=$(cmake --version | head -n1 | sed -e 's/^cmake version //')
# 3.31.6 -> 3.31
cmake_version_major_minor=${cmake_version%.*}
# 3.31 -> 3
cmake_version_major=${cmake_version_major_minor%.*}
# 3.31 -> 31
cmake_version_minor=${cmake_version_major_minor#*.}
if [ "${cmake_version_major}" -gt "3" ] || \
   [ "${cmake_version_major}" -eq "3" -a "${cmake_version_minor}" -ge "25" ]; then
  cp -a "${TOP_SOURCE_DIR}/cpp/examples/minimal_build" build/
  pushd build/minimal_build
  cmake -S . -B build_shared
  make -C build_shared -j$(nproc)
  build_shared/arrow-example
  cmake -S . -B build_static -DARROW_LINK_SHARED=OFF
  make -C build_static -j$(nproc)
  build_static/arrow-example
  mkdir -p build_pkg_config
  c++ \
    example.cc \
    -o build_pkg_config/arrow-example \
    $(pkg-config --cflags --libs arrow) \
    -std=c++20
  build_pkg_config/arrow-example
  popd
fi
echo "::endgroup::"


echo "::group::Test Apache Arrow GLib"
export G_DEBUG=fatal-warnings

${APT_INSTALL} libarrow-glib-dev=${package_version}
${APT_INSTALL} libarrow-glib-doc=${package_version}

${APT_INSTALL} valac
cp -a "${TOP_SOURCE_DIR}/c_glib/example/vala" build/
pushd build/vala
valac --pkg arrow-glib --pkg posix build.vala
./build
popd


${APT_INSTALL} ruby-dev rubygems-integration
MAKEFLAGS="-j$(nproc)" gem install gobject-introspection
ruby -r gi -e "p GI.load('Arrow')"
echo "::endgroup::"


echo "::group::Test Apache Arrow Dataset"
${APT_INSTALL} libarrow-dataset-glib-dev=${package_version}
${APT_INSTALL} libarrow-dataset-glib-doc=${package_version}
ruby -r gi -e "p GI.load('ArrowDataset')"
echo "::endgroup::"


echo "::group::Test Apache Arrow Flight"
${APT_INSTALL} libarrow-flight-glib-dev=${package_version}
${APT_INSTALL} libarrow-flight-glib-doc=${package_version}
ruby -r gi -e "p GI.load('ArrowFlight')"
echo "::endgroup::"

echo "::group::Test Apache Arrow Flight SQL"
${APT_INSTALL} libarrow-flight-sql-glib-dev=${package_version}
${APT_INSTALL} libarrow-flight-sql-glib-doc=${package_version}
ruby -r gi -e "p GI.load('ArrowFlightSQL')"
echo "::endgroup::"


echo "::group::Test Gandiva"
${APT_INSTALL} libgandiva-glib-dev=${package_version}
${APT_INSTALL} libgandiva-glib-doc=${package_version}
ruby -r gi -e "p GI.load('Gandiva')"
echo "::endgroup::"


echo "::group::Test Apache Parquet"
${APT_INSTALL} libparquet-glib-dev=${package_version}
${APT_INSTALL} libparquet-glib-doc=${package_version}
ruby -r gi -e "p GI.load('Parquet')"
echo "::endgroup::"

echo "::group::Prepare downgrade test"
can_downgrade=false
if [ "${distribution}" = 'debian' ] && [[ "$(cat /etc/debian_version)" =~ /sid$ ]]; then
  # Skip downgrade test on Debian testing.
  # Debian testing and unstable use "${testing_code_name}/"sid" as /etc/debian_version content.
  :
elif [ -f /etc/apt/sources.list.d/apache-arrow.sources.bak ]; then
  mv /etc/apt/sources.list.d/apache-arrow.sources \
     /etc/apt/sources.list.d/apache-arrow-current.sources
  mv /etc/apt/sources.list.d/apache-arrow.sources{.bak,}
  cp -a /usr/share/keyrings/apache-arrow-apt-source{,-current}.asc
  sed \
    -i.bak \
    -e 's/\.asc$/-current.asc/g' \
    /etc/apt/sources.list.d/apache-arrow-current.sources
  if curl \
       --fail \
       --output "apache-arrow-apt-source-latest.deb" \
       https://packages.apache.org/artifactory/arrow/${distribution}/apache-arrow-apt-source-latest-${code_name}.deb; then
    ${APT_INSTALL} --allow-downgrades ./apache-arrow-apt-source-latest.deb
    apt update
    can_downgrade=true
  fi
fi

# 22.0.0.dev54 -> 22
# 22.0.0 -> 22
current_major_version=${package_version%%.*}
# 22 -> 21
previous_major_version=$[current_major_version -1]
# 22 -> 21
previous_package_version="${previous_major_version}.0.0-1"
echo "::endgroup::"

if [ "${can_downgrade}" != "true" ]; then
  exit 0
fi

echo "::group::Downgrade Gandiva"
${APT_INSTALL} --allow-downgrades \
  libarrow-acero-dev=${previous_package_version} \
  libarrow-compute-dev=${previous_package_version} \
  libarrow-dev=${previous_package_version} \
  libarrow-glib-dev=${previous_package_version} \
  libgandiva-dev=${previous_package_version} \
  libgandiva-glib-dev=${previous_package_version} \
  libparquet-dev=${previous_package_version}
echo "::endgroup::"

echo "::group::Downgrade Apache Arrow Flight SQL"
${APT_INSTALL} --allow-downgrades \
  libarrow-acero-dev=${previous_package_version} \
  libarrow-compute-dev=${previous_package_version} \
  libarrow-dev=${previous_package_version} \
  libarrow-flight-dev=${previous_package_version} \
  libarrow-flight-glib-dev=${previous_package_version} \
  libarrow-flight-sql-dev=${previous_package_version} \
  libarrow-flight-sql-glib-dev=${previous_package_version} \
  libarrow-glib-dev=${previous_package_version}
echo "::endgroup::"

echo "::group::Downgrade Apache Arrow Dataset"
${APT_INSTALL} --allow-downgrades \
  libarrow-acero-dev=${previous_package_version} \
  libarrow-compute-dev=${previous_package_version} \
  libarrow-dataset-dev=${previous_package_version} \
  libarrow-dataset-glib-dev=${previous_package_version} \
  libarrow-dev=${previous_package_version} \
  libarrow-glib-dev=${previous_package_version} \
  libparquet-dev=${previous_package_version} \
  libparquet-glib-dev=${previous_package_version}
echo "::endgroup::"


echo "::group::Prepare upgrade test"
mv /etc/apt/sources.list.d/apache-arrow-current.sources \
   /etc/apt/sources.list.d/apache-arrow.sources
apt update
echo "::endgroup::"

echo "::group::Upgrade Gandiva"
${APT_INSTALL} libgandiva-glib-dev=${package_version}
echo "::endgroup::"

echo "::group::Upgrade Apache Arrow Flight SQL"
${APT_INSTALL} libarrow-flight-sql-glib-dev=${package_version}
echo "::endgroup::"

echo "::group::Upgrade Apache Arrow Dataset"
${APT_INSTALL} libarrow-dataset-dev=${package_version}
echo "::endgroup::"