HADOOP-19661. Migrate CentOS 8 to Rocky Linux 8 in build env Dockerfile
diff --git a/BUILDING.txt b/BUILDING.txt
index 4dfb48f..dfbf95f 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -31,7 +31,7 @@
$ ./start-build-env.sh [OS platform]
- - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
+ - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
Default is 'ubuntu_20'.
Note: Currently only default ('ubuntu_20') is supported on arm machine
@@ -461,7 +461,7 @@
----------------------------------------------------------------------------------
-Building on CentOS 8
+Building on Rocky Linux 8
----------------------------------------------------------------------------------
@@ -482,7 +482,7 @@
$ sudo make install
$ cd ..
-* Install libraries provided by CentOS 8.
+* Install libraries provided by Rocky Linux 8.
$ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
* Install GCC 9.3.0
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 06095cc..2a2b52b 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -84,10 +84,10 @@
stage ('setup sources') {
steps {
- dir("${WORKSPACE}/centos-8") {
+ dir("${WORKSPACE}/rockylinux-8") {
sh '''#!/usr/bin/env bash
- cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
+ cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
'''
}
@@ -111,12 +111,12 @@
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
- // break the Hadoop build on Centos 8.
- stage ('precommit-run Centos 8') {
+ // break the Hadoop build on Rocky Linux 8.
+ stage ('precommit-run Rocky Linux 8') {
environment {
- SOURCEDIR = "${WORKSPACE}/centos-8/src"
- PATCHDIR = "${WORKSPACE}/centos-8/out"
- DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
+ SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
+ PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
+ DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8"
IS_OPTIONAL = 1
}
@@ -136,7 +136,7 @@
failure {
sh '''#!/usr/bin/env bash
- cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
+ cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}
diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8
deleted file mode 100644
index ee0c8e8..0000000
--- a/dev-support/docker/Dockerfile_centos_8
+++ /dev/null
@@ -1,118 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM centos:8
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
- && chmod a+r pkg-resolver/*.json
-
-######
-# Centos 8 has reached its EOL and the packages
-# are no longer available on mirror.centos.org site.
-# Please see https://www.centos.org/centos-linux-eol/
-######
-RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
-
-######
-# Install packages from yum
-######
-# hadolint ignore=DL3008,SC2046
-RUN yum update -y \
- && yum install -y python3 \
- && yum install -y $(pkg-resolver/resolve.py centos:8)
-
-####
-# Install EPEL
-####
-RUN pkg-resolver/install-epel.sh centos:8
-
-RUN dnf --enablerepo=powertools install -y \
- doxygen \
- snappy-devel \
- yasm
-
-RUN dnf install -y \
- bouncycastle \
- gcc-toolset-9-gcc \
- gcc-toolset-9-gcc-c++ \
- libpmem-devel
-
-# Set GCC 9 as the default C/C++ compiler
-RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc
-SHELL ["/bin/bash", "--login", "-c"]
-
-######
-# Set the environment variables needed for CMake
-# to find and use GCC 9 for compilation
-######
-ENV GCC_HOME "/opt/rh/gcc-toolset-9"
-ENV CC "${GCC_HOME}/root/usr/bin/gcc"
-ENV CXX "${GCC_HOME}/root/usr/bin/g++"
-ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
-ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
-ENV SHLVL 1
-ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
-ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
-ENV MODULESHOME "/usr/share/Modules"
-ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
-ENV PCP_DIR "${GCC_HOME}/root"
-ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
-ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
-ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
-
-# TODO: Set locale
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:${MAVEN_HOME}/bin"
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
-
-#######
-# Set env vars for SpotBugs
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf 3.21.12
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-######
-# Install packages
-######
-RUN pkg-resolver/install-maven.sh centos:8
-RUN pkg-resolver/install-cmake.sh centos:8
-RUN pkg-resolver/install-boost.sh centos:8
-RUN pkg-resolver/install-spotbugs.sh centos:8
-RUN pkg-resolver/install-protobuf.sh centos:8
-RUN pkg-resolver/install-zstandard.sh centos:8
-RUN pkg-resolver/install-common-pkgs.sh
diff --git a/dev-support/docker/Dockerfile_rockylinux_8 b/dev-support/docker/Dockerfile_rockylinux_8
new file mode 100644
index 0000000..63c22c5
--- /dev/null
+++ b/dev-support/docker/Dockerfile_rockylinux_8
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM rockylinux:8
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+ && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from yum
+######
+# hadolint ignore=DL3008,SC2046
+RUN yum update -y \
+ && yum install -y python3 \
+ && yum install -y $(pkg-resolver/resolve.py rockylinux:8)
+
+####
+# Install EPEL
+####
+RUN pkg-resolver/install-epel.sh rockylinux:8
+
+RUN dnf --enablerepo=powertools install -y \
+ doxygen \
+ snappy-devel \
+ yasm
+
+RUN dnf install -y \
+ bouncycastle \
+ gcc-toolset-9-gcc \
+ gcc-toolset-9-gcc-c++ \
+ libpmem-devel
+
+# Set GCC 9 as the default C/C++ compiler
+RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+######
+# Set the environment variables needed for CMake
+# to find and use GCC 9 for compilation
+######
+ENV GCC_HOME="/opt/rh/gcc-toolset-9"
+ENV CC="${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX="${GCC_HOME}/root/usr/bin/g++"
+ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
+ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
+ENV SHLVL=1
+ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
+ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
+ENV MODULESHOME="/usr/share/Modules"
+ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR="${GCC_HOME}/root"
+ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
+ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH="${GCC_HOME}/root/usr/share/info"
+
+# TODO: Set locale
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME=/opt/maven
+ENV PATH="${PATH}:${MAVEN_HOME}/bin"
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+#######
+# Set env vars for SpotBugs
+#######
+ENV SPOTBUGS_HOME=/opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf 3.21.12
+#######
+ENV PROTOBUF_HOME=/opt/protobuf
+ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"
+
+# Skip gpg verification when downloading Yetus via yetus-wrapper
+ENV HADOOP_SKIP_YETUS_VERIFICATION=true
+
+######
+# Install packages
+######
+RUN pkg-resolver/install-maven.sh rockylinux:8
+RUN pkg-resolver/install-cmake.sh rockylinux:8
+RUN pkg-resolver/install-boost.sh rockylinux:8
+RUN pkg-resolver/install-spotbugs.sh rockylinux:8
+RUN pkg-resolver/install-protobuf.sh rockylinux:8
+RUN pkg-resolver/install-zstandard.sh rockylinux:8
+RUN pkg-resolver/install-common-pkgs.sh
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
+
+# Add a welcome message and environment checks.
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
+RUN chmod 755 /root/hadoop_env_checks.sh
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh
index 875dce3..b6eabff 100644
--- a/dev-support/docker/pkg-resolver/install-epel.sh
+++ b/dev-support/docker/pkg-resolver/install-epel.sh
@@ -40,7 +40,7 @@
if [ "$version_to_install" == "8" ]; then
mkdir -p /tmp/epel &&
- curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
+ curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
-o /tmp/epel/epel-release-latest-8.noarch.rpm &&
rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
else
diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json
index efa6b17..28b8733 100644
--- a/dev-support/docker/pkg-resolver/packages.json
+++ b/dev-support/docker/pkg-resolver/packages.json
@@ -4,7 +4,7 @@
"ubuntu:focal": "ant",
"ubuntu:focal::arch64": "ant",
"ubuntu:noble": "ant",
- "centos:8": "ant"
+ "rockylinux:8": "ant"
},
"apt-utils": {
"debian:10": "apt-utils",
@@ -17,7 +17,7 @@
"ubuntu:focal": "automake",
"ubuntu:noble": "automake",
"ubuntu:focal::arch64": "automake",
- "centos:8": "automake"
+ "rockylinux:8": "automake"
},
"bats": {
"debian:10": "bats",
@@ -48,7 +48,7 @@
"bzip2",
"libbz2-dev"
],
- "centos:8": [
+ "rockylinux:8": [
"bzip2",
"bzip2-devel"
]
@@ -58,7 +58,7 @@
"ubuntu:focal": "clang",
"ubuntu:noble": "clang",
"ubuntu:focal::arch64": "clang",
- "centos:8": "clang"
+ "rockylinux:8": "clang"
},
"cmake": {
"ubuntu:focal": "cmake",
@@ -82,7 +82,7 @@
"curl",
"libcurl4-openssl-dev"
],
- "centos:8": [
+ "rockylinux:8": [
"curl",
"libcurl-devel"
]
@@ -94,7 +94,7 @@
"ubuntu:focal::arch64": "doxygen"
},
"dnf": {
- "centos:8": "dnf"
+ "rockylinux:8": "dnf"
},
"fuse": {
"debian:10": [
@@ -113,7 +113,7 @@
"fuse",
"libfuse-dev"
],
- "centos:8": [
+ "rockylinux:8": [
"fuse",
"fuse-libs",
"fuse-devel"
@@ -144,7 +144,7 @@
"ubuntu:focal": "git",
"ubuntu:noble": "git",
"ubuntu:focal::arch64": "git",
- "centos:8": "git"
+ "rockylinux:8": "git"
},
"gnupg-agent": {
"debian:10": "gnupg-agent",
@@ -169,17 +169,17 @@
"ubuntu:focal": "libtool",
"ubuntu:noble": "libtool",
"ubuntu:focal::arch64": "libtool",
- "centos:8": "libtool"
+ "rockylinux:8": "libtool"
},
"openssl": {
"debian:10": "libssl-dev",
"ubuntu:focal": "libssl-dev",
"ubuntu:noble": "libssl-dev",
"ubuntu:focal::arch64": "libssl-dev",
- "centos:8": "openssl-devel"
+ "rockylinux:8": "openssl-devel"
},
"perl": {
- "centos:7": [
+ "rockylinux:7": [
"perl-CPAN",
"perl-devel"
]
@@ -207,7 +207,7 @@
"ubuntu:focal": "libsasl2-dev",
"ubuntu:noble": "libsasl2-dev",
"ubuntu:focal::arch64": "libsasl2-dev",
- "centos:8": "cyrus-sasl-devel"
+ "rockylinux:8": "cyrus-sasl-devel"
},
"snappy": {
"debian:10": "libsnappy-dev",
@@ -232,7 +232,7 @@
"libzstd-dev",
"zlib1g-dev"
],
- "centos:8": [
+ "rockylinux:8": [
"zlib-devel",
"lz4-devel"
]
@@ -244,7 +244,7 @@
"ubuntu:focal::arch64": "locales"
},
"libtirpc-devel": {
- "centos:8": "libtirpc-devel",
+ "rockylinux:8": "libtirpc-devel",
"ubuntu:noble": "libtirpc-dev"
},
"make": {
@@ -252,7 +252,7 @@
"ubuntu:focal": "make",
"ubuntu:noble": "make",
"ubuntu:focal::arch64": "make",
- "centos:8": "make"
+ "rockylinux:8": "make"
},
"maven": {
"debian:10": "maven",
@@ -285,14 +285,14 @@
"ubuntu:focal": "pinentry-curses",
"ubuntu:noble": "pinentry-curses",
"ubuntu:focal::arch64": "pinentry-curses",
- "centos:8": "pinentry-curses"
+ "rockylinux:8": "pinentry-curses"
},
"pkg-config": {
"debian:10": "pkg-config",
"ubuntu:focal": "pkg-config",
"ubuntu:noble": "pkg-config",
"ubuntu:focal::arch64": "pkg-config",
- "centos:8": "pkg-config"
+ "rockylinux:8": "pkg-config"
},
"python": {
"debian:10": [
@@ -324,7 +324,7 @@
"python3-setuptools",
"python3-wheel"
],
- "centos:8": [
+ "rockylinux:8": [
"python3",
"python3-pip",
"python3-setuptools",
@@ -336,7 +336,7 @@
"ubuntu:focal": "rsync",
"ubuntu:noble": "rsync",
"ubuntu:focal::arch64": "rsync",
- "centos:8": "rsync"
+ "rockylinux:8": "rsync"
},
"shellcheck": {
"debian:10": "shellcheck",
@@ -345,7 +345,7 @@
"ubuntu:focal::arch64": "shellcheck"
},
"shasum": {
- "centos:8": "perl-Digest-SHA"
+ "rockylinux:8": "perl-Digest-SHA"
},
"software-properties-common": {
"debian:10": "software-properties-common",
@@ -358,14 +358,14 @@
"ubuntu:focal": "sudo",
"ubuntu:noble": "sudo",
"ubuntu:focal::arch64": "sudo",
- "centos:8": "sudo"
+ "rockylinux:8": "sudo"
},
"valgrind": {
"debian:10": "valgrind",
"ubuntu:focal": "valgrind",
"ubuntu:noble": "valgrind",
"ubuntu:focal::arch64": "valgrind",
- "centos:8": "valgrind"
+ "rockylinux:8": "valgrind"
},
"yasm": {
"debian:10": "yasm",
diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json
index 47842ca..eebf7fc 100644
--- a/dev-support/docker/pkg-resolver/platforms.json
+++ b/dev-support/docker/pkg-resolver/platforms.json
@@ -2,6 +2,6 @@
"ubuntu:focal",
"ubuntu:focal::arch64",
"ubuntu:noble",
- "centos:8",
+ "rockylinux:8",
"debian:10"
-]
\ No newline at end of file
+]
diff --git a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
deleted file mode 100644
index 905ac50..0000000
--- a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
- echo "ERROR: Need at least 1 argument, $# were provided"
- exit 1
-fi
-
-if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
- cd /etc/yum.repos.d/ || exit &&
- sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
- sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
- yum update -y &&
- cd /root || exit
-else
- echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments"
- exit 1
-fi
diff --git a/start-build-env.sh b/start-build-env.sh
index 3c2a74b..4f5f859 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -93,7 +93,7 @@
RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} -d "${DOCKER_HOME_DIR}"
RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}"
-ENV HOME "${DOCKER_HOME_DIR}"
+ENV HOME="${DOCKER_HOME_DIR}"
UserSpecificDocker
