This is an automated email from the ASF dual-hosted git repository. slfan1989 pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 48bdbb75be81065810dd87111dd95bafa1777f3b Author: Cheng Pan <[email protected]> AuthorDate: Fri Aug 29 05:56:46 2025 +0000 HADOOP-19661. Migrate CentOS 8 to Rocky Linux 8 in build env Dockerfile Closes #7900 Signed-off-by: Shilun Fan <[email protected]> Signed-off-by: Chris Nauroth <[email protected]> --- BUILDING.txt | 6 +- dev-support/Jenkinsfile | 14 ++-- dev-support/docker/Dockerfile_centos_8 | 82 ++++++++++++---------- dev-support/docker/pkg-resolver/install-epel.sh | 2 +- dev-support/docker/pkg-resolver/packages.json | 44 ++++++------ dev-support/docker/pkg-resolver/platforms.json | 4 +- .../pkg-resolver/set-vault-as-baseurl-centos.sh | 33 --------- start-build-env.sh | 2 +- 8 files changed, 81 insertions(+), 106 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 8795880c552..9a7af891db6 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -31,7 +31,7 @@ On Linux / Mac: $ ./start-build-env.sh [OS platform] - - [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10]. + - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10]. Default is 'ubuntu_20'. Note: Currently only default ('ubuntu_20') is supported on arm machine @@ -505,7 +505,7 @@ path. This is necessary at least for Homebrewed OpenSSL. ---------------------------------------------------------------------------------- -Building on CentOS 8 +Building on Rocky Linux 8 ---------------------------------------------------------------------------------- @@ -526,7 +526,7 @@ Building on CentOS 8 $ sudo make install $ cd .. -* Install libraries provided by CentOS 8. +* Install libraries provided by Rocky Linux 8. $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel * Install GCC 9.3.0 diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 06095cc8a4f..9ed78f1ff32 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -84,10 +84,10 @@ pipeline { stage ('setup sources') { steps { - dir("${WORKSPACE}/centos-8") { + dir("${WORKSPACE}/rockylinux-8") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8 ''' } @@ -111,11 +111,11 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Centos 8. - stage ('precommit-run Centos 8') { + // break the Hadoop build on Rocky Linux 8. + stage ('precommit-run Rocky Linux 8') { environment { - SOURCEDIR = "${WORKSPACE}/centos-8/src" - PATCHDIR = "${WORKSPACE}/centos-8/out" + SOURCEDIR = "${WORKSPACE}/rockylinux-8/src" + PATCHDIR = "${WORKSPACE}/rockylinux-8/out" DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" IS_OPTIONAL = 1 } @@ -136,7 +136,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index ee0c8e88f74..63c22c5e3e1 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM centos:8 +FROM rockylinux:8 WORKDIR /root @@ -30,25 +30,18 @@ COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ && chmod a+r pkg-resolver/*.json -###### -# Centos 8 has reached its EOL and the packages -# are no longer available on mirror.centos.org site. -# Please see https://www.centos.org/centos-linux-eol/ -###### -RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8 - ###### # Install packages from yum ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ && yum install -y python3 \ - && yum install -y $(pkg-resolver/resolve.py centos:8) + && yum install -y $(pkg-resolver/resolve.py rockylinux:8) #### # Install EPEL #### -RUN pkg-resolver/install-epel.sh centos:8 +RUN pkg-resolver/install-epel.sh rockylinux:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ @@ -69,50 +62,65 @@ SHELL ["/bin/bash", "--login", "-c"] # Set the environment variables needed for CMake # to find and use GCC 9 for compilation ###### -ENV GCC_HOME "/opt/rh/gcc-toolset-9" -ENV CC "${GCC_HOME}/root/usr/bin/gcc" -ENV CXX "${GCC_HOME}/root/usr/bin/g++" -ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD" -ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl" -ENV SHLVL 1 -ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" -ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" -ENV MODULESHOME "/usr/share/Modules" -ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" -ENV PCP_DIR "${GCC_HOME}/root" -ENV MANPATH "${GCC_HOME}/root/usr/share/man::" -ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" -ENV INFOPATH "${GCC_HOME}/root/usr/share/info" +ENV GCC_HOME="/opt/rh/gcc-toolset-9" +ENV CC="${GCC_HOME}/root/usr/bin/gcc" +ENV CXX="${GCC_HOME}/root/usr/bin/g++" +ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD" +ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl" +ENV SHLVL=1 +ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" +ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" +ENV MODULESHOME="/usr/share/Modules" +ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" +ENV PCP_DIR="${GCC_HOME}/root" +ENV MANPATH="${GCC_HOME}/root/usr/share/man::" +ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig" +ENV INFOPATH="${GCC_HOME}/root/usr/share/info" # TODO: Set locale ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /opt/maven -ENV PATH "${PATH}:${MAVEN_HOME}/bin" +ENV MAVEN_HOME=/opt/maven +ENV PATH="${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 +ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ####### # Set env vars for SpotBugs ####### -ENV SPOTBUGS_HOME /opt/spotbugs +ENV SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:${PROTOBUF_HOME}/bin" + +# Skip gpg verification when downloading Yetus via yetus-wrapper +ENV HADOOP_SKIP_YETUS_VERIFICATION=true ###### # Install packages ###### -RUN pkg-resolver/install-maven.sh centos:8 -RUN pkg-resolver/install-cmake.sh centos:8 -RUN pkg-resolver/install-boost.sh centos:8 -RUN pkg-resolver/install-spotbugs.sh centos:8 -RUN pkg-resolver/install-protobuf.sh centos:8 -RUN pkg-resolver/install-zstandard.sh centos:8 +RUN pkg-resolver/install-maven.sh rockylinux:8 +RUN pkg-resolver/install-cmake.sh rockylinux:8 +RUN pkg-resolver/install-boost.sh rockylinux:8 +RUN pkg-resolver/install-spotbugs.sh rockylinux:8 +RUN pkg-resolver/install-protobuf.sh rockylinux:8 +RUN pkg-resolver/install-zstandard.sh rockylinux:8 RUN pkg-resolver/install-common-pkgs.sh + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### + +# Add a welcome message and environment checks. +COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh +RUN chmod 755 /root/hadoop_env_checks.sh +# hadolint ignore=SC2016 +RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh index 875dce3a9ae..b6eabffca02 100644 --- a/dev-support/docker/pkg-resolver/install-epel.sh +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -40,7 +40,7 @@ fi if [ "$version_to_install" == "8" ]; then mkdir -p /tmp/epel && - curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ -o /tmp/epel/epel-release-latest-8.noarch.rpm && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm else diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index efa6b1726ce..28b8733e04b 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -4,7 +4,7 @@ "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "ubuntu:noble": "ant", - "centos:8": "ant" + "rockylinux:8": "ant" }, "apt-utils": { "debian:10": "apt-utils", @@ -17,7 +17,7 @@ "ubuntu:focal": "automake", "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", - "centos:8": "automake" + "rockylinux:8": "automake" }, "bats": { "debian:10": "bats", @@ -48,7 +48,7 @@ "bzip2", "libbz2-dev" ], - "centos:8": [ + "rockylinux:8": [ "bzip2", "bzip2-devel" ] @@ -58,7 +58,7 @@ "ubuntu:focal": "clang", "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", - "centos:8": "clang" + "rockylinux:8": "clang" }, "cmake": { "ubuntu:focal": "cmake", @@ -82,7 +82,7 @@ "curl", "libcurl4-openssl-dev" ], - "centos:8": [ + "rockylinux:8": [ "curl", "libcurl-devel" ] @@ -94,7 +94,7 @@ "ubuntu:focal::arch64": "doxygen" }, "dnf": { - "centos:8": "dnf" + "rockylinux:8": "dnf" }, "fuse": { "debian:10": [ @@ -113,7 +113,7 @@ "fuse", "libfuse-dev" ], - "centos:8": [ + "rockylinux:8": [ "fuse", "fuse-libs", "fuse-devel" @@ -144,7 +144,7 @@ "ubuntu:focal": "git", "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", - "centos:8": "git" + "rockylinux:8": "git" }, "gnupg-agent": { "debian:10": "gnupg-agent", @@ -169,17 +169,17 @@ "ubuntu:focal": "libtool", "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", - "centos:8": "libtool" + "rockylinux:8": "libtool" }, "openssl": { "debian:10": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", - "centos:8": "openssl-devel" + "rockylinux:8": "openssl-devel" }, "perl": { - "centos:7": [ + "rockylinux:7": [ "perl-CPAN", "perl-devel" ] @@ -207,7 +207,7 @@ "ubuntu:focal": "libsasl2-dev", "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", - "centos:8": "cyrus-sasl-devel" + "rockylinux:8": "cyrus-sasl-devel" }, "snappy": { "debian:10": "libsnappy-dev", @@ -232,7 +232,7 @@ "libzstd-dev", "zlib1g-dev" ], - "centos:8": [ + "rockylinux:8": [ "zlib-devel", "lz4-devel" ] @@ -244,7 +244,7 @@ "ubuntu:focal::arch64": "locales" }, "libtirpc-devel": { - "centos:8": "libtirpc-devel", + "rockylinux:8": "libtirpc-devel", "ubuntu:noble": "libtirpc-dev" }, "make": { @@ -252,7 +252,7 @@ "ubuntu:focal": "make", "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", - "centos:8": "make" + "rockylinux:8": "make" }, "maven": { "debian:10": "maven", @@ -285,14 +285,14 @@ "ubuntu:focal": "pinentry-curses", "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", - "centos:8": "pinentry-curses" + "rockylinux:8": "pinentry-curses" }, "pkg-config": { "debian:10": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", - "centos:8": "pkg-config" + "rockylinux:8": "pkg-config" }, "python": { "debian:10": [ @@ -324,7 +324,7 @@ "python3-setuptools", "python3-wheel" ], - "centos:8": [ + "rockylinux:8": [ "python3", "python3-pip", "python3-setuptools", @@ -336,7 +336,7 @@ "ubuntu:focal": "rsync", "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", - "centos:8": "rsync" + "rockylinux:8": "rsync" }, "shellcheck": { "debian:10": "shellcheck", @@ -345,7 +345,7 @@ "ubuntu:focal::arch64": "shellcheck" }, "shasum": { - "centos:8": "perl-Digest-SHA" + "rockylinux:8": "perl-Digest-SHA" }, "software-properties-common": { "debian:10": "software-properties-common", @@ -358,14 +358,14 @@ "ubuntu:focal": "sudo", "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", - "centos:8": "sudo" + "rockylinux:8": "sudo" }, "valgrind": { "debian:10": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", - "centos:8": "valgrind" + "rockylinux:8": "valgrind" }, "yasm": { "debian:10": "yasm", diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 47842cac9b4..eebf7fc948f 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -2,6 +2,6 @@ "ubuntu:focal", "ubuntu:focal::arch64", "ubuntu:noble", - "centos:8", + "rockylinux:8", "debian:10" -] \ No newline at end of file +] diff --git a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh deleted file mode 100644 index 905ac5077de..00000000000 --- a/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ $# -lt 1 ]; then - echo "ERROR: Need at least 1 argument, $# were provided" - exit 1 -fi - -if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then - cd /etc/yum.repos.d/ || exit && - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && - sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && - yum update -y && - cd /root || exit -else - echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments" - exit 1 -fi diff --git a/start-build-env.sh b/start-build-env.sh index de82d7e64ab..b97eca8ad74 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -92,7 +92,7 @@ RUN rm -f /var/log/faillog /var/log/lastlog RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} -d "${DOCKER_HOME_DIR}" RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}" -ENV HOME "${DOCKER_HOME_DIR}" +ENV HOME="${DOCKER_HOME_DIR}" UserSpecificDocker --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
