ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)
Because system jemalloc can't be used with dlopen(). If system
jemalloc can't used with dlopen(), our shared libraried can't be
loaded as bindings of script languages such as Ruby:
+ ruby -r gi -e 'p GI.load('\''Arrow'\'')'
(null)-WARNING **: Failed to load shared library 'libarrow-glib.so.900' referenced by the typelib:
/lib64/libjemalloc.so.2: cannot allocate memory in static TLS block
This is caused because system jemalloc isn't built with
--disable-initial-exec-tls.
See also:
* https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=951704
* https://github.com/jemalloc/jemalloc/issues/1237
Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
index 846d9c7..121abdb 100755
--- a/dev/release/verify-apt.sh
+++ b/dev/release/verify-apt.sh
@@ -158,6 +158,8 @@
echo "::group::Test Apache Arrow GLib"
+export G_DEBUG=fatal-warnings
+
${APT_INSTALL} libarrow-glib-dev=${package_version}
${APT_INSTALL} libarrow-glib-doc=${package_version}
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index 889dddd..93bbcfe 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -46,12 +46,14 @@
cmake_package=cmake
cmake_command=cmake
+have_arrow_libs=no
have_flight=yes
have_gandiva=yes
have_glib=yes
have_parquet=yes
have_python=yes
-have_arrow_libs=no
+have_ruby=yes
+ruby_devel_packages=(ruby-devel)
install_command="dnf install -y --enablerepo=crb"
uninstall_command="dnf remove -y"
clean_command="dnf clean"
@@ -63,6 +65,7 @@
almalinux-8)
distribution_prefix="almalinux"
have_arrow_libs=yes
+ ruby_devel_packages+=(redhat-rpm-config)
install_command="dnf install -y --enablerepo=powertools"
info_command="dnf info --enablerepo=powertools"
;;
@@ -73,12 +76,13 @@
distribution_prefix="amazon-linux"
cmake_package=cmake3
cmake_command=cmake3
- have_flight=no
- have_gandiva=no
- have_python=no
if [ "$(arch)" != "aarch64" ]; then
have_arrow_libs=yes
fi
+ have_flight=no
+ have_gandiva=no
+ have_python=no
+ have_ruby=no
install_command="yum install -y"
uninstall_command="yum remove -y"
clean_command="yum clean"
@@ -89,10 +93,11 @@
distribution_prefix="centos"
cmake_package=cmake3
cmake_command=cmake3
+ have_arrow_libs=yes
have_flight=no
have_gandiva=no
have_python=no
- have_arrow_libs=yes
+ have_ruby=no
install_command="yum install -y"
uninstall_command="yum remove -y"
clean_command="yum clean"
@@ -101,6 +106,7 @@
centos-8)
distribution_prefix="centos"
repository_version+="-stream"
+ ruby_devel_packages+=(redhat-rpm-config)
install_command="dnf install -y --enablerepo=powertools"
info_command="dnf info --enablerepo=powertools"
;;
@@ -206,6 +212,8 @@
if [ "${have_glib}" = "yes" ]; then
echo "::group::Test Apache Arrow GLib"
+ export G_DEBUG=fatal-warnings
+
${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
@@ -215,6 +223,12 @@
valac --pkg arrow-glib --pkg posix build.vala
./build
popd
+
+ if [ "${have_ruby}" = "yes" ]; then
+ ${install_command} "${ruby_devel_packages[@]}"
+ gem install gobject-introspection
+ ruby -r gi -e "p GI.load('Arrow')"
+ fi
echo "::endgroup::"
fi
@@ -222,11 +236,17 @@
echo "::group::Test Apache Arrow Flight"
${install_command} --enablerepo=epel arrow-flight-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-flight-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('ArrowFlight')"
+ fi
echo "::endgroup::"
echo "::group::Test Apache Arrow Flight SQL"
${install_command} --enablerepo=epel arrow-flight-sql-glib-devel-${package_version}
${install_command} --enablerepo=epel arrow-flight-sql-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('ArrowFlightSQL')"
+ fi
echo "::endgroup::"
fi
@@ -240,6 +260,9 @@
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel plasma-glib-devel-${package_version}
${install_command} --enablerepo=epel plasma-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Plasma')"
+ fi
else
${install_command} --enablerepo=epel plasma-devel-${package_version}
fi
@@ -250,6 +273,9 @@
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel gandiva-glib-devel-${package_version}
${install_command} --enablerepo=epel gandiva-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Gandiva')"
+ fi
else
${install_command} --enablerepo=epel gandiva-devel-${package_version}
fi
@@ -261,6 +287,9 @@
if [ "${have_glib}" = "yes" ]; then
${install_command} --enablerepo=epel parquet-glib-devel-${package_version}
${install_command} --enablerepo=epel parquet-glib-doc-${package_version}
+ if [ "${have_ruby}" = "yes" ]; then
+ ruby -r gi -e "p GI.load('Parquet')"
+ fi
else
${install_command} --enablerepo=epel parquet-devel-${package_version}
fi
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
index 42a7327..152ac08 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
@@ -53,7 +53,6 @@
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
index 1562f1d..440a07f 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
@@ -53,7 +53,6 @@
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
index 64ebb3b..b897fa7 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -48,7 +48,6 @@
libglib2.0-doc \
libgoogle-glog-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index 9efc560..5abee7c 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -50,7 +50,6 @@
libgmock-dev \
libgoogle-glog-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
index e6f6575..7e95c37 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -51,7 +51,6 @@
libgoogle-glog-dev \
libgrpc++-dev \
libgtest-dev \
- libjemalloc-dev \
liblz4-dev \
libprotoc-dev \
libprotobuf-dev \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
index 8de71bd..e0a6378 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control.in
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -18,7 +18,6 @@
libgoogle-glog-dev,
@USE_SYSTEM_GRPC@ libgrpc++-dev,
libgtest-dev,
- libjemalloc-dev,
liblz4-dev,
@USE_SYSTEM_GRPC@ libprotoc-dev,
@USE_SYSTEM_GRPC@ libprotobuf-dev,
@@ -147,7 +146,6 @@
libbz2-dev,
@USE_SYSTEM_C_ARES@ libc-ares-dev,
@USE_SYSTEM_GRPC@ libgrpc++-dev,
- libjemalloc-dev,
liblz4-dev,
libre2-dev,
libsnappy-dev,
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
index b55c76d..40a834c 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
+++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
@@ -4,7 +4,6 @@
usr/lib/*/cmake/arrow/ArrowTargets*.cmake
usr/lib/*/cmake/arrow/Find*Alt.cmake
usr/lib/*/cmake/arrow/FindArrow.cmake
-usr/lib/*/cmake/arrow/Findjemalloc.cmake
usr/lib/*/cmake/arrow/Find[BTuz]*.cmake
usr/lib/*/cmake/arrow/arrow-config.cmake
usr/lib/*/libarrow.a
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
index 9178111..2f3b694 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -62,8 +62,7 @@
-DCUDAToolkit_ROOT=/usr \
-DPARQUET_REQUIRE_ENCRYPTION=ON \
-DPythonInterp_FIND_VERSION=ON \
- -DPythonInterp_FIND_VERSION_MAJOR=3 \
- -Djemalloc_SOURCE=SYSTEM
+ -DPythonInterp_FIND_VERSION_MAJOR=3
override_dh_auto_build:
dh_auto_build \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
index 0a37bab..a75fd02 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -40,7 +40,6 @@
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
index 8561779..df63f7a 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
@@ -39,7 +39,6 @@
git \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
index fa1ded6..68d604d 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
@@ -37,7 +37,6 @@
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
lz4-devel \
make \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index 78f974f..098342b 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -105,7 +105,6 @@
%if %{use_glog}
BuildRequires: glog-devel
%endif
-BuildRequires: jemalloc-devel
%if %{use_gcs}
BuildRequires: json-devel
%endif
@@ -194,7 +193,6 @@
-DPythonInterp_FIND_VERSION=ON \
-DPythonInterp_FIND_VERSION_MAJOR=3 \
%endif
- -Djemalloc_SOURCE=SYSTEM \
-G"Unix Makefiles"
%arrow_cmake_build
cd -
@@ -243,7 +241,6 @@
%if %{use_glog}
Requires: glog
%endif
-Requires: jemalloc
%if %{have_zstd}
Requires: libzstd
%endif
@@ -275,7 +272,6 @@
%if %{use_flight}
Requires: c-ares-devel
%endif
-Requires: jemalloc-devel
%if %{use_gcs}
Requires: json-devel
%endif
@@ -323,7 +319,6 @@
%if %{use_flight}
%{_libdir}/cmake/arrow/Findc-aresAlt.cmake
%endif
-%{_libdir}/cmake/arrow/Findjemalloc.cmake
%{_libdir}/cmake/arrow/Findlz4Alt.cmake
%if %{have_re2}
%{_libdir}/cmake/arrow/Findre2Alt.cmake
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
index 93d4a78..9c93e2f 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
@@ -39,7 +39,6 @@
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libzstd-devel \
lz4-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
index ca9e44d..b29cc45 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
@@ -40,7 +40,6 @@
glog-devel \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
index 2bc0364..38e6ae4 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
@@ -39,7 +39,6 @@
git \
gobject-introspection-devel \
gtk-doc \
- jemalloc-devel \
json-devel \
libarchive \
libzstd-devel \
