This is an automated email from the ASF dual-hosted git repository. slfan1989 pushed a commit to branch revert-8105-branch-3.4-docker in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit a4ced1e9247ab1beb27ce42343938d1ce5805d64 Author: slfan1989 <[email protected]> AuthorDate: Thu Dec 4 05:00:03 2025 +0800 Revert "Cherry-pick necessary mirror PRs from trunk branch to Branch-3.4 bran…" This reverts commit adb0b001617d7b6df09b19cbbf70d367832a70b1. --- BUILDING.txt | 10 +- dev-support/Jenkinsfile | 84 ++++++-- dev-support/docker/Dockerfile | 5 - dev-support/docker/Dockerfile_aarch64 | 5 - dev-support/docker/Dockerfile_centos_7 | 108 ++++++++++ ...Dockerfile_rockylinux_8 => Dockerfile_centos_8} | 82 ++++---- .../{Dockerfile_debian_11 => Dockerfile_debian_10} | 40 ++-- dev-support/docker/Dockerfile_ubuntu_24 | 106 ---------- dev-support/docker/Dockerfile_windows_10 | 8 +- dev-support/docker/pkg-resolver/install-epel.sh | 2 +- .../docker/pkg-resolver/install-pkg-resolver.sh | 2 +- dev-support/docker/pkg-resolver/packages.json | 233 ++++++++++----------- dev-support/docker/pkg-resolver/platforms.json | 8 +- ...-resolver.sh => set-vault-as-baseurl-centos.sh} | 22 +- hadoop-project/pom.xml | 10 - start-build-env.sh | 26 +-- 16 files changed, 365 insertions(+), 386 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 73c50c1f67f..173cc0bcad4 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -29,11 +29,7 @@ This requires a recent version of docker (1.4.1 and higher are known to work). On Linux / Mac: Install Docker and run this command: - $ ./start-build-env.sh [OS platform] - - - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10]. - Default is 'ubuntu_20'. - Note: Currently only default ('ubuntu_20') is supported on arm machine + $ ./start-build-env.sh The prompt which is then presented is located at a mounted version of the source tree and all required tools for testing and building have been installed and configured. @@ -505,7 +501,7 @@ path. This is necessary at least for Homebrewed OpenSSL. ---------------------------------------------------------------------------------- -Building on Rocky Linux 8 +Building on CentOS 8 ---------------------------------------------------------------------------------- @@ -526,7 +522,7 @@ Building on Rocky Linux 8 $ sudo make install $ cd .. -* Install libraries provided by Rocky Linux 8. +* Install libraries provided by CentOS 8. $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel * Install GCC 9.3.0 diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 672061ea09a..51225268b65 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -83,18 +83,24 @@ pipeline { // optional stages after the first one. stage ('setup sources') { steps { + dir("${WORKSPACE}/centos-7") { + sh '''#!/usr/bin/env bash + + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7 + ''' + } - dir("${WORKSPACE}/rockylinux-8") { + dir("${WORKSPACE}/centos-8") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8 ''' } - dir("${WORKSPACE}/debian-11") { + dir("${WORKSPACE}/debian-10") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 ''' } @@ -111,12 +117,58 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Rocky Linux 8. - stage ('precommit-run Rocky Linux 8') { + // break the Hadoop build on Centos 7. + stage ('precommit-run Centos 7') { + environment { + SOURCEDIR = "${WORKSPACE}/centos-7/src" + PATCHDIR = "${WORKSPACE}/centos-7/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" + IS_OPTIONAL = 1 + } + + steps { + withCredentials(getGithubCreds()) { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' + } + } + + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash + + cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } + + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } + } + } + + // This is an optional stage which runs only when there's a change in + // C++/C++ build/platform. + // This stage serves as a means of cross platform validation, which is + // really needed to ensure that any C++ related/platform change doesn't + // break the Hadoop build on Centos 8. + stage ('precommit-run Centos 8') { environment { - SOURCEDIR = "${WORKSPACE}/rockylinux-8/src" - PATCHDIR = "${WORKSPACE}/rockylinux-8/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8" + SOURCEDIR = "${WORKSPACE}/centos-8/src" + PATCHDIR = "${WORKSPACE}/centos-8/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" IS_OPTIONAL = 1 } @@ -136,7 +188,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } @@ -157,12 +209,12 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Debian 11. - stage ('precommit-run Debian 11') { + // break the Hadoop build on Debian 10. + stage ('precommit-run Debian 10') { environment { - SOURCEDIR = "${WORKSPACE}/debian-11/src" - PATCHDIR = "${WORKSPACE}/debian-11/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11" + SOURCEDIR = "${WORKSPACE}/debian-10/src" + PATCHDIR = "${WORKSPACE}/debian-10/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" IS_OPTIONAL = 1 } @@ -182,7 +234,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 6beef860a9b..3b71e622a57 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -44,14 +44,9 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ && apt-get -q install -y --no-install-recommends python3 \ - && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ - && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ - && apt-get -q update \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ - && update-java-alternatives -s java-1.8.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index b9249f5aec9..26ad10edd6f 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -44,15 +44,10 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ && apt-get -q install -y --no-install-recommends python3 \ && apt-get -q install -y --no-install-recommends phantomjs \ - && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ - && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ - && apt-get -q update \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \ && apt-get clean \ - && update-java-alternatives -s java-1.8.0-openjdk-arm64 \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 new file mode 100644 index 00000000000..b97e59969a7 --- /dev/null +++ b/dev-support/docker/Dockerfile_centos_7 @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for installing the necessary dependencies for building Hadoop. +# See BUILDING.txt. + +FROM centos:7 + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json + +###### +# Centos 7 has reached its EOL and the packages +# are no longer available on mirror.centos.org site. +# Please see https://www.centos.org/centos-linux-eol/ +###### +RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7 + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 +RUN yum update -y \ + && yum groupinstall -y "Development Tools" \ + && yum install -y \ + centos-release-scl \ + python3 + +# Apply the script again because centos-release-scl creates new YUM repo files +RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7 + +# hadolint ignore=DL3008,SC2046 +RUN yum install -y $(pkg-resolver/resolve.py centos:7) + +# Set GCC 9 as the default C/C++ compiler +RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +###### +# Set the environment variables needed for CMake +# to find and use GCC 9 for compilation +###### +ENV GCC_HOME "/opt/rh/devtoolset-9" +ENV CC "${GCC_HOME}/root/usr/bin/gcc" +ENV CXX "${GCC_HOME}/root/usr/bin/g++" +ENV SHLVL 1 +ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" +ENV PCP_DIR "${GCC_HOME}/root" +ENV MANPATH "${GCC_HOME}/root/usr/share/man:" +ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" +ENV INFOPATH "${GCC_HOME}/root/usr/share/info" + +# TODO: Set locale + +###### +# Set env vars required to build Hadoop +###### +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" +# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 + +####### +# Set env vars for SpotBugs +####### +ENV SPOTBUGS_HOME /opt/spotbugs + +####### +# Set env vars for Google Protobuf 3.21.12 +####### +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:7 +RUN pkg-resolver/install-cmake.sh centos:7 +RUN pkg-resolver/install-zstandard.sh centos:7 +RUN pkg-resolver/install-yasm.sh centos:7 +RUN pkg-resolver/install-protobuf.sh centos:7 +RUN pkg-resolver/install-boost.sh centos:7 +RUN pkg-resolver/install-spotbugs.sh centos:7 +RUN pkg-resolver/install-nodejs.sh centos:7 +RUN pkg-resolver/install-git.sh centos:7 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile_rockylinux_8 b/dev-support/docker/Dockerfile_centos_8 similarity index 55% rename from dev-support/docker/Dockerfile_rockylinux_8 rename to dev-support/docker/Dockerfile_centos_8 index 63c22c5e3e1..ee0c8e88f74 100644 --- a/dev-support/docker/Dockerfile_rockylinux_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM rockylinux:8 +FROM centos:8 WORKDIR /root @@ -30,18 +30,25 @@ COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ && chmod a+r pkg-resolver/*.json +###### +# Centos 8 has reached its EOL and the packages +# are no longer available on mirror.centos.org site. +# Please see https://www.centos.org/centos-linux-eol/ +###### +RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8 + ###### # Install packages from yum ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ && yum install -y python3 \ - && yum install -y $(pkg-resolver/resolve.py rockylinux:8) + && yum install -y $(pkg-resolver/resolve.py centos:8) #### # Install EPEL #### -RUN pkg-resolver/install-epel.sh rockylinux:8 +RUN pkg-resolver/install-epel.sh centos:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ @@ -62,65 +69,50 @@ SHELL ["/bin/bash", "--login", "-c"] # Set the environment variables needed for CMake # to find and use GCC 9 for compilation ###### -ENV GCC_HOME="/opt/rh/gcc-toolset-9" -ENV CC="${GCC_HOME}/root/usr/bin/gcc" -ENV CXX="${GCC_HOME}/root/usr/bin/g++" -ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD" -ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl" -ENV SHLVL=1 -ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" -ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" -ENV MODULESHOME="/usr/share/Modules" -ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" -ENV PCP_DIR="${GCC_HOME}/root" -ENV MANPATH="${GCC_HOME}/root/usr/share/man::" -ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig" -ENV INFOPATH="${GCC_HOME}/root/usr/share/info" +ENV GCC_HOME "/opt/rh/gcc-toolset-9" +ENV CC "${GCC_HOME}/root/usr/bin/gcc" +ENV CXX "${GCC_HOME}/root/usr/bin/g++" +ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD" +ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl" +ENV SHLVL 1 +ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles" +ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1" +ENV MODULESHOME "/usr/share/Modules" +ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" +ENV PCP_DIR "${GCC_HOME}/root" +ENV MANPATH "${GCC_HOME}/root/usr/share/man::" +ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" +ENV INFOPATH "${GCC_HOME}/root/usr/share/info" # TODO: Set locale ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME=/opt/maven -ENV PATH="${PATH}:${MAVEN_HOME}/bin" +ENV MAVEN_HOME /opt/maven +ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ####### # Set env vars for SpotBugs ####### -ENV SPOTBUGS_HOME=/opt/spotbugs +ENV SPOTBUGS_HOME /opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME=/opt/protobuf -ENV PATH="${PATH}:${PROTOBUF_HOME}/bin" - -# Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION=true +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" ###### # Install packages ###### -RUN pkg-resolver/install-maven.sh rockylinux:8 -RUN pkg-resolver/install-cmake.sh rockylinux:8 -RUN pkg-resolver/install-boost.sh rockylinux:8 -RUN pkg-resolver/install-spotbugs.sh rockylinux:8 -RUN pkg-resolver/install-protobuf.sh rockylinux:8 -RUN pkg-resolver/install-zstandard.sh rockylinux:8 +RUN pkg-resolver/install-maven.sh centos:8 +RUN pkg-resolver/install-cmake.sh centos:8 +RUN pkg-resolver/install-boost.sh centos:8 +RUN pkg-resolver/install-spotbugs.sh centos:8 +RUN pkg-resolver/install-protobuf.sh centos:8 +RUN pkg-resolver/install-zstandard.sh centos:8 RUN pkg-resolver/install-common-pkgs.sh - -### -# Everything past this point is either not needed for testing or breaks Yetus. -# So tell Yetus not to read the rest of the file: -# YETUS CUT HERE -### - -# Add a welcome message and environment checks. -COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh -RUN chmod 755 /root/hadoop_env_checks.sh -# hadolint ignore=SC2016 -RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/Dockerfile_debian_11 b/dev-support/docker/Dockerfile_debian_10 similarity index 71% rename from dev-support/docker/Dockerfile_debian_11 rename to dev-support/docker/Dockerfile_debian_10 index 2dc47eee6ac..71446b27f68 100644 --- a/dev-support/docker/Dockerfile_debian_11 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM debian:11 +FROM debian:10 WORKDIR /root @@ -29,24 +29,26 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras -ENV DEBIAN_FRONTEND=noninteractive -ENV DEBCONF_TERSE=true +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true ###### # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-pkg-resolver.sh -RUN pkg-resolver/install-pkg-resolver.sh debian:11 +RUN pkg-resolver/install-pkg-resolver.sh debian:10 ###### # Install packages from apt ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ + && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \ + && apt-get -q update \ + && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \ && apt-get clean \ - && update-java-alternatives -s java-1.11.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* # TODO : Set locale @@ -54,38 +56,38 @@ RUN apt-get -q update \ ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME=/usr +ENV MAVEN_HOME /usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 ####### # Set env vars for SpotBugs 4.2.2 ####### -ENV SPOTBUGS_HOME=/opt/spotbugs +ENV SPOTBUGS_HOME /opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME=/opt/protobuf -ENV PATH="${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS="-Xms256m -Xmx3072m" +ENV MAVEN_OPTS -Xms256m -Xmx3072m # Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION=true +ENV HADOOP_SKIP_YETUS_VERIFICATION true #### # Install packages #### -RUN pkg-resolver/install-cmake.sh debian:11 -RUN pkg-resolver/install-spotbugs.sh debian:11 -RUN pkg-resolver/install-boost.sh debian:11 -RUN pkg-resolver/install-protobuf.sh debian:11 -RUN pkg-resolver/install-hadolint.sh debian:11 -RUN pkg-resolver/install-intel-isa-l.sh debian:11 +RUN pkg-resolver/install-cmake.sh debian:10 +RUN pkg-resolver/install-spotbugs.sh debian:10 +RUN pkg-resolver/install-boost.sh debian:10 +RUN pkg-resolver/install-protobuf.sh debian:10 +RUN pkg-resolver/install-hadolint.sh debian:10 +RUN pkg-resolver/install-intel-isa-l.sh debian:10 ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/Dockerfile_ubuntu_24 b/dev-support/docker/Dockerfile_ubuntu_24 deleted file mode 100644 index 28ab04c3e09..00000000000 --- a/dev-support/docker/Dockerfile_ubuntu_24 +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Dockerfile for installing the necessary dependencies for building Hadoop. -# See BUILDING.txt. - -FROM ubuntu:noble - -WORKDIR /root - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -##### -# Disable suggests/recommends -##### -RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras -RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras - -ENV DEBIAN_FRONTEND noninteractive -ENV DEBCONF_TERSE true - -###### -# Platform package dependency resolver -###### -COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ - && chmod a+r pkg-resolver/*.json - -###### -# Install packages from apt -###### -# hadolint ignore=DL3008,SC2046 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends wget apt-transport-https gpg gpg-agent gawk ca-certificates \ - && apt-get -q install -y --no-install-recommends python3 \ - && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \ - && wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \ - && apt-get -q update \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:noble) \ - && apt-get clean \ - && update-java-alternatives -s java-1.17.0-openjdk-amd64 \ - && rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 - -###### -# Set env vars required to build Hadoop -###### -ENV MAVEN_HOME /usr -# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64 - -####### -# Set env vars for SpotBugs 4.2.2 -####### -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Set env vars for Google Protobuf 3.21.12 -####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - -### -# Avoid out of memory errors in builds -### -ENV MAVEN_OPTS -Xms256m -Xmx3072m - -# Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION true - -#### -# Install packages -#### -RUN pkg-resolver/install-spotbugs.sh ubuntu:noble -RUN pkg-resolver/install-boost.sh ubuntu:noble -RUN pkg-resolver/install-protobuf.sh ubuntu:noble -RUN pkg-resolver/install-hadolint.sh ubuntu:noble -RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble - -### -# Everything past this point is either not needed for testing or breaks Yetus. -# So tell Yetus not to read the rest of the file: -# YETUS CUT HERE -### - -# Add a welcome message and environment checks. -COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh -RUN chmod 755 /root/hadoop_env_checks.sh -# hadolint ignore=SC2016 -RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index 7d32e6a84ff..105529c5d65 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -79,11 +79,11 @@ RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:T RUN powershell mkdir "C:\LibOpenSSL" RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL" -# Install libxxhash 0.8.3 needed for rsync 3.2.7. -RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.3-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst -RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar +# Install libxxhash 0.8.1 needed for rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar RUN powershell mkdir "C:\LibXXHash" -RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar -C "C:\LibXXHash" +RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash" # Install libzstd 1.5.4 needed for rsync 3.2.7. RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh index b6eabffca02..875dce3a9ae 100644 --- a/dev-support/docker/pkg-resolver/install-epel.sh +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -40,7 +40,7 @@ fi if [ "$version_to_install" == "8" ]; then mkdir -p /tmp/epel && - curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ -o /tmp/epel/epel-release-latest-8.noarch.rpm && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm else diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh index 81dd0002f40..70e94b3792d 100644 --- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh +++ b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh @@ -24,7 +24,7 @@ fi chmod a+x pkg-resolver/*.sh pkg-resolver/*.py chmod a+r pkg-resolver/*.json -if [ "$1" == "debian:11" ]; then +if [ "$1" == "debian:10" ]; then apt-get -q update apt-get -q install -y --no-install-recommends python3 \ python3-pip \ diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index b46d10e8200..00d6ba36de2 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,39 +1,39 @@ { "ant": { - "debian:11": "ant", + "debian:10": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", - "ubuntu:noble": "ant", - "rockylinux:8": "ant" + "centos:7": "ant", + "centos:8": "ant" }, "apt-utils": { - "debian:11": "apt-utils", + "debian:10": "apt-utils", "ubuntu:focal": "apt-utils", - "ubuntu:noble": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { - "debian:11": "automake", + "debian:10": "automake", "ubuntu:focal": "automake", - "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", - "rockylinux:8": "automake" + "centos:7": "automake", + "centos:8": "automake" + }, + "autoconf": { + "centos:7": "autoconf" }, "bats": { - "debian:11": "bats", + "debian:10": "bats", "ubuntu:focal": "bats", - "ubuntu:noble": "bats", - "ubuntu:focal::arch64": "bats", - "rockylinux:8": "bats" + "ubuntu:focal::arch64": "bats" }, "build-essential": { - "debian:11": "build-essential", + "debian:10": "build-essential", "ubuntu:focal": "build-essential", - "ubuntu:noble": "build-essential", - "ubuntu:focal::arch64": "build-essential" + "ubuntu:focal::arch64": "build-essential", + "centos:7": "build-essential" }, "bzip2": { - "debian:11": [ + "debian:10": [ "bzip2", "libbz2-dev" ], @@ -41,33 +41,32 @@ "bzip2", "libbz2-dev" ], - "ubuntu:noble": [ + "ubuntu:focal::arch64": [ "bzip2", "libbz2-dev" ], - "ubuntu:focal::arch64": [ + "centos:7": [ "bzip2", - "libbz2-dev" + "bzip2-devel" ], - "rockylinux:8": [ + "centos:8": [ "bzip2", "bzip2-devel" ] }, "clang": { - "debian:11": "clang", + "debian:10": "clang", "ubuntu:focal": "clang", - "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", - "rockylinux:8": "clang" + "centos:7": "clang", + "centos:8": "clang" }, "cmake": { "ubuntu:focal": "cmake", - "ubuntu:noble": "cmake", "ubuntu:focal::arch64": "cmake" }, "curl": { - "debian:11": [ + "debian:10": [ "curl", "libcurl4-openssl-dev" ], @@ -75,30 +74,30 @@ "curl", "libcurl4-openssl-dev" ], - "ubuntu:noble": [ + "ubuntu:focal::arch64": [ "curl", "libcurl4-openssl-dev" ], - "ubuntu:focal::arch64": [ + "centos:7": [ "curl", - "libcurl4-openssl-dev" + "libcurl-devel" ], - "rockylinux:8": [ + "centos:8": [ "curl", "libcurl-devel" ] }, "doxygen": { - "debian:11": "doxygen", + "debian:10": "doxygen", "ubuntu:focal": "doxygen", - "ubuntu:noble": "doxygen", - "ubuntu:focal::arch64": "doxygen" + "ubuntu:focal::arch64": "doxygen", + "centos:7": "doxygen" }, "dnf": { - "rockylinux:8": "dnf" + "centos:8": "dnf" }, "fuse": { - "debian:11": [ + "debian:10": [ "fuse", "libfuse-dev" ], @@ -106,22 +105,23 @@ "fuse", "libfuse-dev" ], - "ubuntu:noble": [ + "ubuntu:focal::arch64": [ "fuse", "libfuse-dev" ], - "ubuntu:focal::arch64": [ + "centos:7": [ "fuse", - "libfuse-dev" + "fuse-libs", + "fuse-devel" ], - "rockylinux:8": [ + "centos:8": [ "fuse", "fuse-libs", "fuse-devel" ] }, "gcc": { - "debian:11": { + "debian:10": { "bullseye": [ "gcc", "g++" @@ -131,62 +131,61 @@ "gcc", "g++" ], - "ubuntu:noble": [ - "gcc", - "g++" - ], "ubuntu:focal::arch64": [ "gcc", "g++" + ], + "centos:7": [ + "centos-release-scl", + "devtoolset-9" ] }, + "gettext": { + "centos:7": "gettext-devel" + }, "git": { - "debian:11": "git", + "debian:10": "git", "ubuntu:focal": "git", - "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", - "rockylinux:8": "git" + "centos:8": "git" }, "gnupg-agent": { - "debian:11": "gnupg-agent", + "debian:10": "gnupg-agent", "ubuntu:focal": "gnupg-agent", - "ubuntu:noble": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { - "debian:11": "hugo", + "debian:10": "hugo", "ubuntu:focal": "hugo", - "ubuntu:noble": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { - "debian:11": "libbcprov-java", + "debian:10": "libbcprov-java", "ubuntu:focal": "libbcprov-java", - "ubuntu:noble": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { - "debian:11": "libtool", + "debian:10": "libtool", "ubuntu:focal": "libtool", - "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", - "rockylinux:8": "libtool" + "centos:7": "libtool", + "centos:8": "libtool" }, "openssl": { - "debian:11": "libssl-dev", + "debian:10": "libssl-dev", "ubuntu:focal": "libssl-dev", - "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", - "rockylinux:8": "openssl-devel" + "centos:7": "openssl-devel", + "centos:8": "openssl-devel" }, "perl": { - "rockylinux:8": [ + "centos:7": [ "perl-CPAN", "perl-devel" ] }, "protocol-buffers": { - "debian:11": [ + "debian:10": [ "libprotobuf-dev", "libprotoc-dev" ], @@ -194,33 +193,26 @@ "libprotobuf-dev", "libprotoc-dev" ], - "ubuntu:noble": [ - "libprotobuf-dev", - "libprotoc-dev" - ], "ubuntu:focal::arch64": [ "libprotobuf-dev", "libprotoc-dev" ] }, - "procps": { - "debian:11": "procps" - }, "sasl": { - "debian:11": "libsasl2-dev", + "debian:10": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", - "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", - "rockylinux:8": "cyrus-sasl-devel" + "centos:7": "cyrus-sasl-devel", + "centos:8": "cyrus-sasl-devel" }, "snappy": { - "debian:11": "libsnappy-dev", + "debian:10": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", - "ubuntu:noble": "libsnappy-dev", - "ubuntu:focal::arch64": "libsnappy-dev" + "ubuntu:focal::arch64": "libsnappy-dev", + "centos:7": "snappy-devel" }, "zlib": { - "debian:11": [ + "debian:10": [ "libzstd-dev", "zlib1g-dev" ], @@ -228,81 +220,71 @@ "libzstd-dev", "zlib1g-dev" ], - "ubuntu:noble": [ - "libzstd-dev", - "zlib1g-dev" - ], "ubuntu:focal::arch64": [ "libzstd-dev", "zlib1g-dev" ], - "rockylinux:8": [ + "centos:7": [ + "zlib-devel", + "lz4-devel" + ], + "centos:8": [ "zlib-devel", "lz4-devel" ] }, "locales": { - "debian:11": "locales", + "debian:10": "locales", "ubuntu:focal": "locales", - "ubuntu:noble": "locales", "ubuntu:focal::arch64": "locales" }, "libtirpc-devel": { - "rockylinux:8": "libtirpc-devel", - "ubuntu:noble": "libtirpc-dev" + "centos:7": "libtirpc-devel", + "centos:8": "libtirpc-devel" + }, + "libpmem": { + "centos:7": "libpmem-devel" }, "make": { - "debian:11": "make", + "debian:10": "make", "ubuntu:focal": "make", - "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", - "rockylinux:8": "make" + "centos:7": "make", + "centos:8": "make" }, "maven": { - "debian:11": "maven", + "debian:10": "maven", "ubuntu:focal": "maven", - "ubuntu:noble": "maven", "ubuntu:focal::arch64": "maven" }, "java": { - "debian:11": [ - "openjdk-11-jdk", - "openjdk-17-jdk" - ], + "debian:10": "openjdk-11-jdk", "ubuntu:focal": [ - "temurin-24-jdk", "openjdk-8-jdk", "openjdk-11-jdk", "openjdk-17-jdk" ], - "ubuntu:noble": [ - "temurin-24-jdk", - "openjdk-11-jdk", - "openjdk-17-jdk" - ], "ubuntu:focal::arch64": [ - "temurin-24-jdk", "openjdk-8-jdk", "openjdk-11-jdk", "openjdk-17-jdk" ] }, "pinentry-curses": { - "debian:11": "pinentry-curses", + "debian:10": "pinentry-curses", "ubuntu:focal": "pinentry-curses", - "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", - "rockylinux:8": "pinentry-curses" + "centos:7": "pinentry-curses", + "centos:8": "pinentry-curses" }, "pkg-config": { - "debian:11": "pkg-config", + "debian:10": "pkg-config", "ubuntu:focal": "pkg-config", - "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", - "rockylinux:8": "pkg-config" + "centos:8": "pkg-config" }, "python": { - "debian:11": [ + "debian:10": [ "python3", "python3-pip", "python3-pkg-resources", @@ -316,22 +298,21 @@ "python3-setuptools", "python3-wheel" ], - "ubuntu:noble": [ + "ubuntu:focal::arch64": [ + "python2.7", "python3", "python3-pip", "python3-pkg-resources", "python3-setuptools", "python3-wheel" ], - "ubuntu:focal::arch64": [ - "python2.7", + "centos:7": [ "python3", "python3-pip", - "python3-pkg-resources", "python3-setuptools", "python3-wheel" ], - "rockylinux:8": [ + "centos:8": [ "python3", "python3-pip", "python3-setuptools", @@ -339,45 +320,43 @@ ] }, "rsync": { - "debian:11": "rsync", + "debian:10": "rsync", "ubuntu:focal": "rsync", - "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", - "rockylinux:8": "rsync" + "centos:7": "rsync", + "centos:8": "rsync" }, "shellcheck": { - "debian:11": "shellcheck", + "debian:10": "shellcheck", "ubuntu:focal": "shellcheck", - "ubuntu:noble": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, "shasum": { - "rockylinux:8": "perl-Digest-SHA" + "centos:7": "perl-Digest-SHA", + "centos:8": "perl-Digest-SHA" }, "software-properties-common": { - "debian:11": "software-properties-common", + "debian:10": "software-properties-common", "ubuntu:focal": "software-properties-common", - "ubuntu:noble": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { - "debian:11": "sudo", + "debian:10": "sudo", "ubuntu:focal": "sudo", - "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", - "rockylinux:8": "sudo" + "centos:7": "sudo", + "centos:8": "sudo" }, "valgrind": { - "debian:11": "valgrind", + "debian:10": "valgrind", "ubuntu:focal": "valgrind", - "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", - "rockylinux:8": "valgrind" + "centos:7": "valgrind", + "centos:8": "valgrind" }, "yasm": { - "debian:11": "yasm", + "debian:10": "yasm", "ubuntu:focal": "yasm", - "ubuntu:noble": "yasm", "ubuntu:focal::arch64": "yasm" } } diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index 00f6c129b20..93e2a93df42 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -1,7 +1,7 @@ [ "ubuntu:focal", "ubuntu:focal::arch64", - "ubuntu:noble", - "rockylinux:8", - "debian:11" -] + "centos:7", + "centos:8", + "debian:10" +] \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh similarity index 61% copy from dev-support/docker/pkg-resolver/install-pkg-resolver.sh copy to dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh index 81dd0002f40..905ac5077de 100644 --- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh +++ b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh @@ -17,23 +17,17 @@ # limitations under the License. if [ $# -lt 1 ]; then - echo "ERROR: No platform specified, please specify one" + echo "ERROR: Need at least 1 argument, $# were provided" exit 1 fi -chmod a+x pkg-resolver/*.sh pkg-resolver/*.py -chmod a+r pkg-resolver/*.json - -if [ "$1" == "debian:11" ]; then - apt-get -q update - apt-get -q install -y --no-install-recommends python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel - pip3 install pylint==2.6.0 python-dateutil==2.8.1 +if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then + cd /etc/yum.repos.d/ || exit && + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && + sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && + yum update -y && + cd /root || exit else - # Need to add the code for the rest of the platforms - HADOOP-17920 - echo "ERROR: The given platform $1 is not yet supported or is invalid" + echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments" exit 1 fi diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 0a4cda6ffc3..251d86965e3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -2786,16 +2786,6 @@ </dependencies> </dependencyManagement> </profile> - <profile> - <id>jdk17+</id> - <activation> - <jdk>[17,)</jdk> - </activation> - <properties> - <javac.version>17</javac.version> - <maven.compiler.release>${javac.version}</maven.compiler.release> - </properties> - </profile> </profiles> <repositories> diff --git a/start-build-env.sh b/start-build-env.sh index b97eca8ad74..375a027a06b 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -19,29 +19,12 @@ set -e # exit on error cd "$(dirname "$0")" # connect to root -OS_PLATFORM="${1:-}" -[ "$#" -gt 0 ] && shift - -DEFAULT_OS_PLATFORM="ubuntu_20" - -OS_PLATFORM_SUFFIX="" - -if [[ -n ${OS_PLATFORM} ]] && [[ "${OS_PLATFORM}" != "${DEFAULT_OS_PLATFORM}" ]]; then - # ubuntu_20 (default) platform does not have suffix in Dockerfile. - OS_PLATFORM_SUFFIX="_${OS_PLATFORM}" -fi - DOCKER_DIR=dev-support/docker DOCKER_FILE="${DOCKER_DIR}/Dockerfile" CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1) -if [[ "$CPU_ARCH" == "aarch64" || "$CPU_ARCH" == "arm64" ]]; then - DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_aarch64" -fi - -if [ ! -e "${DOCKER_FILE}" ] ; then - echo "'${OS_PLATFORM}' environment not available yet for '${CPU_ARCH}'" - exit 1 +if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then + DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64" fi docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR @@ -92,7 +75,7 @@ RUN rm -f /var/log/faillog /var/log/lastlog RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} -d "${DOCKER_HOME_DIR}" RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}" -ENV HOME="${DOCKER_HOME_DIR}" +ENV HOME "${DOCKER_HOME_DIR}" UserSpecificDocker @@ -110,5 +93,4 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \ -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ -u "${USER_ID}" \ - --name "hadoop-build${OS_PLATFORM_SUFFIX}" \ - "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" "$@" + "hadoop-build-${USER_ID}" "$@" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
