This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new d9a136b7c50 Revert "Cherry-pick necessary mirror PRs from trunk branch
to Branch-3.4 bran…" (#8119)
d9a136b7c50 is described below
commit d9a136b7c5089aaf12b72b9f85c146495f66d4d3
Author: slfan1989 <[email protected]>
AuthorDate: Thu Dec 4 05:01:16 2025 +0800
Revert "Cherry-pick necessary mirror PRs from trunk branch to Branch-3.4
bran…" (#8119)
This reverts commit adb0b001617d7b6df09b19cbbf70d367832a70b1.
---
BUILDING.txt | 10 +-
dev-support/Jenkinsfile | 84 ++++++--
dev-support/docker/Dockerfile | 5 -
dev-support/docker/Dockerfile_aarch64 | 5 -
dev-support/docker/Dockerfile_centos_7 | 108 ++++++++++
...Dockerfile_rockylinux_8 => Dockerfile_centos_8} | 82 ++++----
.../{Dockerfile_debian_11 => Dockerfile_debian_10} | 40 ++--
dev-support/docker/Dockerfile_ubuntu_24 | 106 ----------
dev-support/docker/Dockerfile_windows_10 | 8 +-
dev-support/docker/pkg-resolver/install-epel.sh | 2 +-
.../docker/pkg-resolver/install-pkg-resolver.sh | 2 +-
dev-support/docker/pkg-resolver/packages.json | 233 ++++++++++-----------
dev-support/docker/pkg-resolver/platforms.json | 8 +-
...-resolver.sh => set-vault-as-baseurl-centos.sh} | 22 +-
hadoop-project/pom.xml | 10 -
start-build-env.sh | 26 +--
16 files changed, 365 insertions(+), 386 deletions(-)
diff --git a/BUILDING.txt b/BUILDING.txt
index 73c50c1f67f..173cc0bcad4 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -29,11 +29,7 @@ This requires a recent version of docker (1.4.1 and higher
are known to work).
On Linux / Mac:
Install Docker and run this command:
- $ ./start-build-env.sh [OS platform]
-
- - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24,
windows_10].
- Default is 'ubuntu_20'.
- Note: Currently only default ('ubuntu_20') is supported on arm machine
+ $ ./start-build-env.sh
The prompt which is then presented is located at a mounted version of the
source tree
and all required tools for testing and building have been installed and
configured.
@@ -505,7 +501,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
----------------------------------------------------------------------------------
-Building on Rocky Linux 8
+Building on CentOS 8
----------------------------------------------------------------------------------
@@ -526,7 +522,7 @@ Building on Rocky Linux 8
$ sudo make install
$ cd ..
-* Install libraries provided by Rocky Linux 8.
+* Install libraries provided by CentOS 8.
$ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel
openssl-devel cyrus-sasl-devel libpmem-devel
* Install GCC 9.3.0
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 672061ea09a..51225268b65 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -83,18 +83,24 @@ pipeline {
// optional stages after the first one.
stage ('setup sources') {
steps {
+ dir("${WORKSPACE}/centos-7") {
+ sh '''#!/usr/bin/env bash
+
+ cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7
+ '''
+ }
- dir("${WORKSPACE}/rockylinux-8") {
+ dir("${WORKSPACE}/centos-8") {
sh '''#!/usr/bin/env bash
- cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
+ cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
'''
}
- dir("${WORKSPACE}/debian-11") {
+ dir("${WORKSPACE}/debian-10") {
sh '''#!/usr/bin/env bash
- cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11
+ cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
'''
}
@@ -111,12 +117,58 @@ pipeline {
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
- // break the Hadoop build on Rocky Linux 8.
- stage ('precommit-run Rocky Linux 8') {
+ // break the Hadoop build on Centos 7.
+ stage ('precommit-run Centos 7') {
+ environment {
+ SOURCEDIR = "${WORKSPACE}/centos-7/src"
+ PATCHDIR = "${WORKSPACE}/centos-7/out"
+ DOCKERFILE =
"${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7"
+ IS_OPTIONAL = 1
+ }
+
+ steps {
+ withCredentials(getGithubCreds()) {
+ sh '''#!/usr/bin/env bash
+
+ chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+ "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
+ '''
+ }
+ }
+
+ post {
+ // Since this is an optional platform, we want to copy the
artifacts
+ // and archive it only if the build fails, to help with
debugging.
+ failure {
+ sh '''#!/usr/bin/env bash
+
+ cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}"
+ '''
+ archiveArtifacts "out/**"
+ }
+
+ cleanup() {
+ script {
+ sh '''#!/usr/bin/env bash
+
+ chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+ "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+ '''
+ }
+ }
+ }
+ }
+
+ // This is an optional stage which runs only when there's a change in
+ // C++/C++ build/platform.
+ // This stage serves as a means of cross platform validation, which is
+ // really needed to ensure that any C++ related/platform change doesn't
+ // break the Hadoop build on Centos 8.
+ stage ('precommit-run Centos 8') {
environment {
- SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
- PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
- DOCKERFILE =
"${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8"
+ SOURCEDIR = "${WORKSPACE}/centos-8/src"
+ PATCHDIR = "${WORKSPACE}/centos-8/out"
+ DOCKERFILE =
"${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
IS_OPTIONAL = 1
}
@@ -136,7 +188,7 @@ pipeline {
failure {
sh '''#!/usr/bin/env bash
- cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
+ cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}
@@ -157,12 +209,12 @@ pipeline {
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
- // break the Hadoop build on Debian 11.
- stage ('precommit-run Debian 11') {
+ // break the Hadoop build on Debian 10.
+ stage ('precommit-run Debian 10') {
environment {
- SOURCEDIR = "${WORKSPACE}/debian-11/src"
- PATCHDIR = "${WORKSPACE}/debian-11/out"
- DOCKERFILE =
"${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11"
+ SOURCEDIR = "${WORKSPACE}/debian-10/src"
+ PATCHDIR = "${WORKSPACE}/debian-10/out"
+ DOCKERFILE =
"${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
IS_OPTIONAL = 1
}
@@ -182,7 +234,7 @@ pipeline {
failure {
sh '''#!/usr/bin/env bash
- cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}"
+ cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
'''
archiveArtifacts "out/**"
}
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 6beef860a9b..3b71e622a57 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -44,14 +44,9 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
######
# hadolint ignore=DL3008,SC2046
RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends wget apt-transport-https
gpg gpg-agent gawk ca-certificates \
&& apt-get -q install -y --no-install-recommends python3 \
- && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F=
'/^VERSION_CODENAME/{print$2}' /etc/os-release) main" >
/etc/apt/sources.list.d/adoptium.list \
- && wget -q -O -
https://packages.adoptium.net/artifactory/api/gpg/key/public >
/etc/apt/trusted.gpg.d/adoptium.asc \
- && apt-get -q update \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
ubuntu:focal) \
&& apt-get clean \
- && update-java-alternatives -s java-1.8.0-openjdk-amd64 \
&& rm -rf /var/lib/apt/lists/*
RUN locale-gen en_US.UTF-8
diff --git a/dev-support/docker/Dockerfile_aarch64
b/dev-support/docker/Dockerfile_aarch64
index b9249f5aec9..26ad10edd6f 100644
--- a/dev-support/docker/Dockerfile_aarch64
+++ b/dev-support/docker/Dockerfile_aarch64
@@ -44,15 +44,10 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
######
# hadolint ignore=DL3008,SC2046
RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends wget apt-transport-https
gpg gpg-agent gawk ca-certificates \
&& apt-get -q install -y --no-install-recommends python3 \
&& apt-get -q install -y --no-install-recommends phantomjs \
- && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F=
'/^VERSION_CODENAME/{print$2}' /etc/os-release) main" >
/etc/apt/sources.list.d/adoptium.list \
- && wget -q -O -
https://packages.adoptium.net/artifactory/api/gpg/key/public >
/etc/apt/trusted.gpg.d/adoptium.asc \
- && apt-get -q update \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
ubuntu:focal::arch64) \
&& apt-get clean \
- && update-java-alternatives -s java-1.8.0-openjdk-arm64 \
&& rm -rf /var/lib/apt/lists/*
RUN locale-gen en_US.UTF-8
diff --git a/dev-support/docker/Dockerfile_centos_7
b/dev-support/docker/Dockerfile_centos_7
new file mode 100644
index 00000000000..b97e59969a7
--- /dev/null
+++ b/dev-support/docker/Dockerfile_centos_7
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM centos:7
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+ && chmod a+r pkg-resolver/*.json
+
+######
+# Centos 7 has reached its EOL and the packages
+# are no longer available on mirror.centos.org site.
+# Please see https://www.centos.org/centos-linux-eol/
+######
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
+######
+# Install packages from yum
+######
+# hadolint ignore=DL3008,SC2046
+RUN yum update -y \
+ && yum groupinstall -y "Development Tools" \
+ && yum install -y \
+ centos-release-scl \
+ python3
+
+# Apply the script again because centos-release-scl creates new YUM repo files
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
+# hadolint ignore=DL3008,SC2046
+RUN yum install -y $(pkg-resolver/resolve.py centos:7)
+
+# Set GCC 9 as the default C/C++ compiler
+RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+######
+# Set the environment variables needed for CMake
+# to find and use GCC 9 for compilation
+######
+ENV GCC_HOME "/opt/rh/devtoolset-9"
+ENV CC "${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX "${GCC_HOME}/root/usr/bin/g++"
+ENV SHLVL 1
+ENV LD_LIBRARY_PATH
"${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR "${GCC_HOME}/root"
+ENV MANPATH "${GCC_HOME}/root/usr/share/man:"
+ENV PATH
"${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
+
+# TODO: Set locale
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /opt/maven
+ENV PATH "${PATH}:${MAVEN_HOME}/bin"
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+
+#######
+# Set env vars for SpotBugs
+#######
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf 3.21.12
+#######
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install packages
+######
+RUN pkg-resolver/install-maven.sh centos:7
+RUN pkg-resolver/install-cmake.sh centos:7
+RUN pkg-resolver/install-zstandard.sh centos:7
+RUN pkg-resolver/install-yasm.sh centos:7
+RUN pkg-resolver/install-protobuf.sh centos:7
+RUN pkg-resolver/install-boost.sh centos:7
+RUN pkg-resolver/install-spotbugs.sh centos:7
+RUN pkg-resolver/install-nodejs.sh centos:7
+RUN pkg-resolver/install-git.sh centos:7
+RUN pkg-resolver/install-common-pkgs.sh
diff --git a/dev-support/docker/Dockerfile_rockylinux_8
b/dev-support/docker/Dockerfile_centos_8
similarity index 55%
rename from dev-support/docker/Dockerfile_rockylinux_8
rename to dev-support/docker/Dockerfile_centos_8
index 63c22c5e3e1..ee0c8e88f74 100644
--- a/dev-support/docker/Dockerfile_rockylinux_8
+++ b/dev-support/docker/Dockerfile_centos_8
@@ -17,7 +17,7 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
-FROM rockylinux:8
+FROM centos:8
WORKDIR /root
@@ -30,18 +30,25 @@ COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
&& chmod a+r pkg-resolver/*.json
+######
+# Centos 8 has reached its EOL and the packages
+# are no longer available on mirror.centos.org site.
+# Please see https://www.centos.org/centos-linux-eol/
+######
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
+
######
# Install packages from yum
######
# hadolint ignore=DL3008,SC2046
RUN yum update -y \
&& yum install -y python3 \
- && yum install -y $(pkg-resolver/resolve.py rockylinux:8)
+ && yum install -y $(pkg-resolver/resolve.py centos:8)
####
# Install EPEL
####
-RUN pkg-resolver/install-epel.sh rockylinux:8
+RUN pkg-resolver/install-epel.sh centos:8
RUN dnf --enablerepo=powertools install -y \
doxygen \
@@ -62,65 +69,50 @@ SHELL ["/bin/bash", "--login", "-c"]
# Set the environment variables needed for CMake
# to find and use GCC 9 for compilation
######
-ENV GCC_HOME="/opt/rh/gcc-toolset-9"
-ENV CC="${GCC_HOME}/root/usr/bin/gcc"
-ENV CXX="${GCC_HOME}/root/usr/bin/g++"
-ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
-ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
-ENV SHLVL=1
-ENV
MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
-ENV
MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
-ENV MODULESHOME="/usr/share/Modules"
-ENV
LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
-ENV PCP_DIR="${GCC_HOME}/root"
-ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
-ENV
PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
-ENV INFOPATH="${GCC_HOME}/root/usr/share/info"
+ENV GCC_HOME "/opt/rh/gcc-toolset-9"
+ENV CC "${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX "${GCC_HOME}/root/usr/bin/g++"
+ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
+ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
+ENV SHLVL 1
+ENV MODULEPATH
"/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
+ENV MODULEPATH_modshare
"/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
+ENV MODULESHOME "/usr/share/Modules"
+ENV LD_LIBRARY_PATH
"${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR "${GCC_HOME}/root"
+ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
+ENV PATH
"${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
# TODO: Set locale
######
# Set env vars required to build Hadoop
######
-ENV MAVEN_HOME=/opt/maven
-ENV PATH="${PATH}:${MAVEN_HOME}/bin"
+ENV MAVEN_HOME /opt/maven
+ENV PATH "${PATH}:${MAVEN_HOME}/bin"
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
#######
# Set env vars for SpotBugs
#######
-ENV SPOTBUGS_HOME=/opt/spotbugs
+ENV SPOTBUGS_HOME /opt/spotbugs
#######
# Set env vars for Google Protobuf 3.21.12
#######
-ENV PROTOBUF_HOME=/opt/protobuf
-ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"
-
-# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION=true
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
######
# Install packages
######
-RUN pkg-resolver/install-maven.sh rockylinux:8
-RUN pkg-resolver/install-cmake.sh rockylinux:8
-RUN pkg-resolver/install-boost.sh rockylinux:8
-RUN pkg-resolver/install-spotbugs.sh rockylinux:8
-RUN pkg-resolver/install-protobuf.sh rockylinux:8
-RUN pkg-resolver/install-zstandard.sh rockylinux:8
+RUN pkg-resolver/install-maven.sh centos:8
+RUN pkg-resolver/install-cmake.sh centos:8
+RUN pkg-resolver/install-boost.sh centos:8
+RUN pkg-resolver/install-spotbugs.sh centos:8
+RUN pkg-resolver/install-protobuf.sh centos:8
+RUN pkg-resolver/install-zstandard.sh centos:8
RUN pkg-resolver/install-common-pkgs.sh
-
-###
-# Everything past this point is either not needed for testing or breaks Yetus.
-# So tell Yetus not to read the rest of the file:
-# YETUS CUT HERE
-###
-
-# Add a welcome message and environment checks.
-COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
-RUN chmod 755 /root/hadoop_env_checks.sh
-# hadolint ignore=SC2016
-RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
diff --git a/dev-support/docker/Dockerfile_debian_11
b/dev-support/docker/Dockerfile_debian_10
similarity index 71%
rename from dev-support/docker/Dockerfile_debian_11
rename to dev-support/docker/Dockerfile_debian_10
index 2dc47eee6ac..71446b27f68 100644
--- a/dev-support/docker/Dockerfile_debian_11
+++ b/dev-support/docker/Dockerfile_debian_10
@@ -17,7 +17,7 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
-FROM debian:11
+FROM debian:10
WORKDIR /root
@@ -29,24 +29,26 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
-ENV DEBIAN_FRONTEND=noninteractive
-ENV DEBCONF_TERSE=true
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
######
# Platform package dependency resolver
######
COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/install-pkg-resolver.sh
-RUN pkg-resolver/install-pkg-resolver.sh debian:11
+RUN pkg-resolver/install-pkg-resolver.sh debian:10
######
# Install packages from apt
######
# hadolint ignore=DL3008,SC2046
RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
debian:11) \
+ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
debian:10) \
+ && echo 'deb http://deb.debian.org/debian bullseye main' >>
/etc/apt/sources.list \
+ && apt-get -q update \
+ && apt-get -q install -y --no-install-recommends -t bullseye
$(pkg-resolver/resolve.py --release=bullseye debian:10) \
&& apt-get clean \
- && update-java-alternatives -s java-1.11.0-openjdk-amd64 \
&& rm -rf /var/lib/apt/lists/*
# TODO : Set locale
@@ -54,38 +56,38 @@ RUN apt-get -q update \
######
# Set env vars required to build Hadoop
######
-ENV MAVEN_HOME=/usr
+ENV MAVEN_HOME /usr
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
#######
# Set env vars for SpotBugs 4.2.2
#######
-ENV SPOTBUGS_HOME=/opt/spotbugs
+ENV SPOTBUGS_HOME /opt/spotbugs
#######
# Set env vars for Google Protobuf 3.21.12
#######
-ENV PROTOBUF_HOME=/opt/protobuf
-ENV PATH="${PATH}:/opt/protobuf/bin"
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
###
# Avoid out of memory errors in builds
###
-ENV MAVEN_OPTS="-Xms256m -Xmx3072m"
+ENV MAVEN_OPTS -Xms256m -Xmx3072m
# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION=true
+ENV HADOOP_SKIP_YETUS_VERIFICATION true
####
# Install packages
####
-RUN pkg-resolver/install-cmake.sh debian:11
-RUN pkg-resolver/install-spotbugs.sh debian:11
-RUN pkg-resolver/install-boost.sh debian:11
-RUN pkg-resolver/install-protobuf.sh debian:11
-RUN pkg-resolver/install-hadolint.sh debian:11
-RUN pkg-resolver/install-intel-isa-l.sh debian:11
+RUN pkg-resolver/install-cmake.sh debian:10
+RUN pkg-resolver/install-spotbugs.sh debian:10
+RUN pkg-resolver/install-boost.sh debian:10
+RUN pkg-resolver/install-protobuf.sh debian:10
+RUN pkg-resolver/install-hadolint.sh debian:10
+RUN pkg-resolver/install-intel-isa-l.sh debian:10
###
# Everything past this point is either not needed for testing or breaks Yetus.
diff --git a/dev-support/docker/Dockerfile_ubuntu_24
b/dev-support/docker/Dockerfile_ubuntu_24
deleted file mode 100644
index 28ab04c3e09..00000000000
--- a/dev-support/docker/Dockerfile_ubuntu_24
+++ /dev/null
@@ -1,106 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM ubuntu:noble
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-#####
-# Disable suggests/recommends
-#####
-RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
-RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
- && chmod a+r pkg-resolver/*.json
-
-######
-# Install packages from apt
-######
-# hadolint ignore=DL3008,SC2046
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends wget apt-transport-https
gpg gpg-agent gawk ca-certificates \
- && apt-get -q install -y --no-install-recommends python3 \
- && echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F=
'/^VERSION_CODENAME/{print$2}' /etc/os-release) main" >
/etc/apt/sources.list.d/adoptium.list \
- && wget -q -O -
https://packages.adoptium.net/artifactory/api/gpg/key/public >
/etc/apt/trusted.gpg.d/adoptium.asc \
- && apt-get -q update \
- && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py
ubuntu:noble) \
- && apt-get clean \
- && update-java-alternatives -s java-1.17.0-openjdk-amd64 \
- && rm -rf /var/lib/apt/lists/*
-
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV PYTHONIOENCODING=utf-8
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /usr
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
-
-#######
-# Set env vars for SpotBugs 4.2.2
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf 3.21.12
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
-
-# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
-
-####
-# Install packages
-####
-RUN pkg-resolver/install-spotbugs.sh ubuntu:noble
-RUN pkg-resolver/install-boost.sh ubuntu:noble
-RUN pkg-resolver/install-protobuf.sh ubuntu:noble
-RUN pkg-resolver/install-hadolint.sh ubuntu:noble
-RUN pkg-resolver/install-intel-isa-l.sh ubuntu:noble
-
-###
-# Everything past this point is either not needed for testing or breaks Yetus.
-# So tell Yetus not to read the rest of the file:
-# YETUS CUT HERE
-###
-
-# Add a welcome message and environment checks.
-COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
-RUN chmod 755 /root/hadoop_env_checks.sh
-# hadolint ignore=SC2016
-RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
diff --git a/dev-support/docker/Dockerfile_windows_10
b/dev-support/docker/Dockerfile_windows_10
index 7d32e6a84ff..105529c5d65 100644
--- a/dev-support/docker/Dockerfile_windows_10
+++ b/dev-support/docker/Dockerfile_windows_10
@@ -79,11 +79,11 @@ RUN powershell zstd -d
$Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:T
RUN powershell mkdir "C:\LibOpenSSL"
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C
"C:\LibOpenSSL"
-# Install libxxhash 0.8.3 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri
https://repo.msys2.org/msys/x86_64/libxxhash-0.8.3-1-x86_64.pkg.tar.zst
-OutFile $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst -o
$Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar
+# Install libxxhash 0.8.1 needed for rsync 3.2.7.
+RUN powershell Invoke-WebRequest -Uri
https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst
-OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
+RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o
$Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
RUN powershell mkdir "C:\LibXXHash"
-RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar -C
"C:\LibXXHash"
+RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C
"C:\LibXXHash"
# Install libzstd 1.5.4 needed for rsync 3.2.7.
RUN powershell Invoke-WebRequest -Uri
https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile
$Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
diff --git a/dev-support/docker/pkg-resolver/install-epel.sh
b/dev-support/docker/pkg-resolver/install-epel.sh
index b6eabffca02..875dce3a9ae 100644
--- a/dev-support/docker/pkg-resolver/install-epel.sh
+++ b/dev-support/docker/pkg-resolver/install-epel.sh
@@ -40,7 +40,7 @@ fi
if [ "$version_to_install" == "8" ]; then
mkdir -p /tmp/epel &&
- curl -L -s -S
https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
+ curl -L -s -S
https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
\
-o /tmp/epel/epel-release-latest-8.noarch.rpm &&
rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
else
diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
index 81dd0002f40..70e94b3792d 100644
--- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
+++ b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
@@ -24,7 +24,7 @@ fi
chmod a+x pkg-resolver/*.sh pkg-resolver/*.py
chmod a+r pkg-resolver/*.json
-if [ "$1" == "debian:11" ]; then
+if [ "$1" == "debian:10" ]; then
apt-get -q update
apt-get -q install -y --no-install-recommends python3 \
python3-pip \
diff --git a/dev-support/docker/pkg-resolver/packages.json
b/dev-support/docker/pkg-resolver/packages.json
index b46d10e8200..00d6ba36de2 100644
--- a/dev-support/docker/pkg-resolver/packages.json
+++ b/dev-support/docker/pkg-resolver/packages.json
@@ -1,39 +1,39 @@
{
"ant": {
- "debian:11": "ant",
+ "debian:10": "ant",
"ubuntu:focal": "ant",
"ubuntu:focal::arch64": "ant",
- "ubuntu:noble": "ant",
- "rockylinux:8": "ant"
+ "centos:7": "ant",
+ "centos:8": "ant"
},
"apt-utils": {
- "debian:11": "apt-utils",
+ "debian:10": "apt-utils",
"ubuntu:focal": "apt-utils",
- "ubuntu:noble": "apt-utils",
"ubuntu:focal::arch64": "apt-utils"
},
"automake": {
- "debian:11": "automake",
+ "debian:10": "automake",
"ubuntu:focal": "automake",
- "ubuntu:noble": "automake",
"ubuntu:focal::arch64": "automake",
- "rockylinux:8": "automake"
+ "centos:7": "automake",
+ "centos:8": "automake"
+ },
+ "autoconf": {
+ "centos:7": "autoconf"
},
"bats": {
- "debian:11": "bats",
+ "debian:10": "bats",
"ubuntu:focal": "bats",
- "ubuntu:noble": "bats",
- "ubuntu:focal::arch64": "bats",
- "rockylinux:8": "bats"
+ "ubuntu:focal::arch64": "bats"
},
"build-essential": {
- "debian:11": "build-essential",
+ "debian:10": "build-essential",
"ubuntu:focal": "build-essential",
- "ubuntu:noble": "build-essential",
- "ubuntu:focal::arch64": "build-essential"
+ "ubuntu:focal::arch64": "build-essential",
+ "centos:7": "build-essential"
},
"bzip2": {
- "debian:11": [
+ "debian:10": [
"bzip2",
"libbz2-dev"
],
@@ -41,33 +41,32 @@
"bzip2",
"libbz2-dev"
],
- "ubuntu:noble": [
+ "ubuntu:focal::arch64": [
"bzip2",
"libbz2-dev"
],
- "ubuntu:focal::arch64": [
+ "centos:7": [
"bzip2",
- "libbz2-dev"
+ "bzip2-devel"
],
- "rockylinux:8": [
+ "centos:8": [
"bzip2",
"bzip2-devel"
]
},
"clang": {
- "debian:11": "clang",
+ "debian:10": "clang",
"ubuntu:focal": "clang",
- "ubuntu:noble": "clang",
"ubuntu:focal::arch64": "clang",
- "rockylinux:8": "clang"
+ "centos:7": "clang",
+ "centos:8": "clang"
},
"cmake": {
"ubuntu:focal": "cmake",
- "ubuntu:noble": "cmake",
"ubuntu:focal::arch64": "cmake"
},
"curl": {
- "debian:11": [
+ "debian:10": [
"curl",
"libcurl4-openssl-dev"
],
@@ -75,30 +74,30 @@
"curl",
"libcurl4-openssl-dev"
],
- "ubuntu:noble": [
+ "ubuntu:focal::arch64": [
"curl",
"libcurl4-openssl-dev"
],
- "ubuntu:focal::arch64": [
+ "centos:7": [
"curl",
- "libcurl4-openssl-dev"
+ "libcurl-devel"
],
- "rockylinux:8": [
+ "centos:8": [
"curl",
"libcurl-devel"
]
},
"doxygen": {
- "debian:11": "doxygen",
+ "debian:10": "doxygen",
"ubuntu:focal": "doxygen",
- "ubuntu:noble": "doxygen",
- "ubuntu:focal::arch64": "doxygen"
+ "ubuntu:focal::arch64": "doxygen",
+ "centos:7": "doxygen"
},
"dnf": {
- "rockylinux:8": "dnf"
+ "centos:8": "dnf"
},
"fuse": {
- "debian:11": [
+ "debian:10": [
"fuse",
"libfuse-dev"
],
@@ -106,22 +105,23 @@
"fuse",
"libfuse-dev"
],
- "ubuntu:noble": [
+ "ubuntu:focal::arch64": [
"fuse",
"libfuse-dev"
],
- "ubuntu:focal::arch64": [
+ "centos:7": [
"fuse",
- "libfuse-dev"
+ "fuse-libs",
+ "fuse-devel"
],
- "rockylinux:8": [
+ "centos:8": [
"fuse",
"fuse-libs",
"fuse-devel"
]
},
"gcc": {
- "debian:11": {
+ "debian:10": {
"bullseye": [
"gcc",
"g++"
@@ -131,62 +131,61 @@
"gcc",
"g++"
],
- "ubuntu:noble": [
- "gcc",
- "g++"
- ],
"ubuntu:focal::arch64": [
"gcc",
"g++"
+ ],
+ "centos:7": [
+ "centos-release-scl",
+ "devtoolset-9"
]
},
+ "gettext": {
+ "centos:7": "gettext-devel"
+ },
"git": {
- "debian:11": "git",
+ "debian:10": "git",
"ubuntu:focal": "git",
- "ubuntu:noble": "git",
"ubuntu:focal::arch64": "git",
- "rockylinux:8": "git"
+ "centos:8": "git"
},
"gnupg-agent": {
- "debian:11": "gnupg-agent",
+ "debian:10": "gnupg-agent",
"ubuntu:focal": "gnupg-agent",
- "ubuntu:noble": "gnupg-agent",
"ubuntu:focal::arch64": "gnupg-agent"
},
"hugo": {
- "debian:11": "hugo",
+ "debian:10": "hugo",
"ubuntu:focal": "hugo",
- "ubuntu:noble": "hugo",
"ubuntu:focal::arch64": "hugo"
},
"libbcprov-java": {
- "debian:11": "libbcprov-java",
+ "debian:10": "libbcprov-java",
"ubuntu:focal": "libbcprov-java",
- "ubuntu:noble": "libbcprov-java",
"ubuntu:focal::arch64": "libbcprov-java"
},
"libtool": {
- "debian:11": "libtool",
+ "debian:10": "libtool",
"ubuntu:focal": "libtool",
- "ubuntu:noble": "libtool",
"ubuntu:focal::arch64": "libtool",
- "rockylinux:8": "libtool"
+ "centos:7": "libtool",
+ "centos:8": "libtool"
},
"openssl": {
- "debian:11": "libssl-dev",
+ "debian:10": "libssl-dev",
"ubuntu:focal": "libssl-dev",
- "ubuntu:noble": "libssl-dev",
"ubuntu:focal::arch64": "libssl-dev",
- "rockylinux:8": "openssl-devel"
+ "centos:7": "openssl-devel",
+ "centos:8": "openssl-devel"
},
"perl": {
- "rockylinux:8": [
+ "centos:7": [
"perl-CPAN",
"perl-devel"
]
},
"protocol-buffers": {
- "debian:11": [
+ "debian:10": [
"libprotobuf-dev",
"libprotoc-dev"
],
@@ -194,33 +193,26 @@
"libprotobuf-dev",
"libprotoc-dev"
],
- "ubuntu:noble": [
- "libprotobuf-dev",
- "libprotoc-dev"
- ],
"ubuntu:focal::arch64": [
"libprotobuf-dev",
"libprotoc-dev"
]
},
- "procps": {
- "debian:11": "procps"
- },
"sasl": {
- "debian:11": "libsasl2-dev",
+ "debian:10": "libsasl2-dev",
"ubuntu:focal": "libsasl2-dev",
- "ubuntu:noble": "libsasl2-dev",
"ubuntu:focal::arch64": "libsasl2-dev",
- "rockylinux:8": "cyrus-sasl-devel"
+ "centos:7": "cyrus-sasl-devel",
+ "centos:8": "cyrus-sasl-devel"
},
"snappy": {
- "debian:11": "libsnappy-dev",
+ "debian:10": "libsnappy-dev",
"ubuntu:focal": "libsnappy-dev",
- "ubuntu:noble": "libsnappy-dev",
- "ubuntu:focal::arch64": "libsnappy-dev"
+ "ubuntu:focal::arch64": "libsnappy-dev",
+ "centos:7": "snappy-devel"
},
"zlib": {
- "debian:11": [
+ "debian:10": [
"libzstd-dev",
"zlib1g-dev"
],
@@ -228,81 +220,71 @@
"libzstd-dev",
"zlib1g-dev"
],
- "ubuntu:noble": [
- "libzstd-dev",
- "zlib1g-dev"
- ],
"ubuntu:focal::arch64": [
"libzstd-dev",
"zlib1g-dev"
],
- "rockylinux:8": [
+ "centos:7": [
+ "zlib-devel",
+ "lz4-devel"
+ ],
+ "centos:8": [
"zlib-devel",
"lz4-devel"
]
},
"locales": {
- "debian:11": "locales",
+ "debian:10": "locales",
"ubuntu:focal": "locales",
- "ubuntu:noble": "locales",
"ubuntu:focal::arch64": "locales"
},
"libtirpc-devel": {
- "rockylinux:8": "libtirpc-devel",
- "ubuntu:noble": "libtirpc-dev"
+ "centos:7": "libtirpc-devel",
+ "centos:8": "libtirpc-devel"
+ },
+ "libpmem": {
+ "centos:7": "libpmem-devel"
},
"make": {
- "debian:11": "make",
+ "debian:10": "make",
"ubuntu:focal": "make",
- "ubuntu:noble": "make",
"ubuntu:focal::arch64": "make",
- "rockylinux:8": "make"
+ "centos:7": "make",
+ "centos:8": "make"
},
"maven": {
- "debian:11": "maven",
+ "debian:10": "maven",
"ubuntu:focal": "maven",
- "ubuntu:noble": "maven",
"ubuntu:focal::arch64": "maven"
},
"java": {
- "debian:11": [
- "openjdk-11-jdk",
- "openjdk-17-jdk"
- ],
+ "debian:10": "openjdk-11-jdk",
"ubuntu:focal": [
- "temurin-24-jdk",
"openjdk-8-jdk",
"openjdk-11-jdk",
"openjdk-17-jdk"
],
- "ubuntu:noble": [
- "temurin-24-jdk",
- "openjdk-11-jdk",
- "openjdk-17-jdk"
- ],
"ubuntu:focal::arch64": [
- "temurin-24-jdk",
"openjdk-8-jdk",
"openjdk-11-jdk",
"openjdk-17-jdk"
]
},
"pinentry-curses": {
- "debian:11": "pinentry-curses",
+ "debian:10": "pinentry-curses",
"ubuntu:focal": "pinentry-curses",
- "ubuntu:noble": "pinentry-curses",
"ubuntu:focal::arch64": "pinentry-curses",
- "rockylinux:8": "pinentry-curses"
+ "centos:7": "pinentry-curses",
+ "centos:8": "pinentry-curses"
},
"pkg-config": {
- "debian:11": "pkg-config",
+ "debian:10": "pkg-config",
"ubuntu:focal": "pkg-config",
- "ubuntu:noble": "pkg-config",
"ubuntu:focal::arch64": "pkg-config",
- "rockylinux:8": "pkg-config"
+ "centos:8": "pkg-config"
},
"python": {
- "debian:11": [
+ "debian:10": [
"python3",
"python3-pip",
"python3-pkg-resources",
@@ -316,22 +298,21 @@
"python3-setuptools",
"python3-wheel"
],
- "ubuntu:noble": [
+ "ubuntu:focal::arch64": [
+ "python2.7",
"python3",
"python3-pip",
"python3-pkg-resources",
"python3-setuptools",
"python3-wheel"
],
- "ubuntu:focal::arch64": [
- "python2.7",
+ "centos:7": [
"python3",
"python3-pip",
- "python3-pkg-resources",
"python3-setuptools",
"python3-wheel"
],
- "rockylinux:8": [
+ "centos:8": [
"python3",
"python3-pip",
"python3-setuptools",
@@ -339,45 +320,43 @@
]
},
"rsync": {
- "debian:11": "rsync",
+ "debian:10": "rsync",
"ubuntu:focal": "rsync",
- "ubuntu:noble": "rsync",
"ubuntu:focal::arch64": "rsync",
- "rockylinux:8": "rsync"
+ "centos:7": "rsync",
+ "centos:8": "rsync"
},
"shellcheck": {
- "debian:11": "shellcheck",
+ "debian:10": "shellcheck",
"ubuntu:focal": "shellcheck",
- "ubuntu:noble": "shellcheck",
"ubuntu:focal::arch64": "shellcheck"
},
"shasum": {
- "rockylinux:8": "perl-Digest-SHA"
+ "centos:7": "perl-Digest-SHA",
+ "centos:8": "perl-Digest-SHA"
},
"software-properties-common": {
- "debian:11": "software-properties-common",
+ "debian:10": "software-properties-common",
"ubuntu:focal": "software-properties-common",
- "ubuntu:noble": "software-properties-common",
"ubuntu:focal::arch64": "software-properties-common"
},
"sudo": {
- "debian:11": "sudo",
+ "debian:10": "sudo",
"ubuntu:focal": "sudo",
- "ubuntu:noble": "sudo",
"ubuntu:focal::arch64": "sudo",
- "rockylinux:8": "sudo"
+ "centos:7": "sudo",
+ "centos:8": "sudo"
},
"valgrind": {
- "debian:11": "valgrind",
+ "debian:10": "valgrind",
"ubuntu:focal": "valgrind",
- "ubuntu:noble": "valgrind",
"ubuntu:focal::arch64": "valgrind",
- "rockylinux:8": "valgrind"
+ "centos:7": "valgrind",
+ "centos:8": "valgrind"
},
"yasm": {
- "debian:11": "yasm",
+ "debian:10": "yasm",
"ubuntu:focal": "yasm",
- "ubuntu:noble": "yasm",
"ubuntu:focal::arch64": "yasm"
}
}
diff --git a/dev-support/docker/pkg-resolver/platforms.json
b/dev-support/docker/pkg-resolver/platforms.json
index 00f6c129b20..93e2a93df42 100644
--- a/dev-support/docker/pkg-resolver/platforms.json
+++ b/dev-support/docker/pkg-resolver/platforms.json
@@ -1,7 +1,7 @@
[
"ubuntu:focal",
"ubuntu:focal::arch64",
- "ubuntu:noble",
- "rockylinux:8",
- "debian:11"
-]
+ "centos:7",
+ "centos:8",
+ "debian:10"
+]
\ No newline at end of file
diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
similarity index 61%
copy from dev-support/docker/pkg-resolver/install-pkg-resolver.sh
copy to dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
index 81dd0002f40..905ac5077de 100644
--- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh
+++ b/dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh
@@ -17,23 +17,17 @@
# limitations under the License.
if [ $# -lt 1 ]; then
- echo "ERROR: No platform specified, please specify one"
+ echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
-chmod a+x pkg-resolver/*.sh pkg-resolver/*.py
-chmod a+r pkg-resolver/*.json
-
-if [ "$1" == "debian:11" ]; then
- apt-get -q update
- apt-get -q install -y --no-install-recommends python3 \
- python3-pip \
- python3-pkg-resources \
- python3-setuptools \
- python3-wheel
- pip3 install pylint==2.6.0 python-dateutil==2.8.1
+if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
+ cd /etc/yum.repos.d/ || exit &&
+ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
+ sed -i 's|#
*baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g'
/etc/yum.repos.d/CentOS-* &&
+ yum update -y &&
+ cd /root || exit
else
- # Need to add the code for the rest of the platforms - HADOOP-17920
- echo "ERROR: The given platform $1 is not yet supported or is invalid"
+ echo "ERROR: Setting the archived baseurl is only supported for centos 7 and
8 environments"
exit 1
fi
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 0a4cda6ffc3..251d86965e3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -2786,16 +2786,6 @@
</dependencies>
</dependencyManagement>
</profile>
- <profile>
- <id>jdk17+</id>
- <activation>
- <jdk>[17,)</jdk>
- </activation>
- <properties>
- <javac.version>17</javac.version>
- <maven.compiler.release>${javac.version}</maven.compiler.release>
- </properties>
- </profile>
</profiles>
<repositories>
diff --git a/start-build-env.sh b/start-build-env.sh
index b97eca8ad74..375a027a06b 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -19,29 +19,12 @@ set -e # exit on error
cd "$(dirname "$0")" # connect to root
-OS_PLATFORM="${1:-}"
-[ "$#" -gt 0 ] && shift
-
-DEFAULT_OS_PLATFORM="ubuntu_20"
-
-OS_PLATFORM_SUFFIX=""
-
-if [[ -n ${OS_PLATFORM} ]] && [[ "${OS_PLATFORM}" != "${DEFAULT_OS_PLATFORM}"
]]; then
- # ubuntu_20 (default) platform does not have suffix in Dockerfile.
- OS_PLATFORM_SUFFIX="_${OS_PLATFORM}"
-fi
-
DOCKER_DIR=dev-support/docker
DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-if [[ "$CPU_ARCH" == "aarch64" || "$CPU_ARCH" == "arm64" ]]; then
- DOCKER_FILE="${DOCKER_DIR}/Dockerfile${OS_PLATFORM_SUFFIX}_aarch64"
-fi
-
-if [ ! -e "${DOCKER_FILE}" ] ; then
- echo "'${OS_PLATFORM}' environment not available yet for '${CPU_ARCH}'"
- exit 1
+if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
+ DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
fi
docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR
@@ -92,7 +75,7 @@ RUN rm -f /var/log/faillog /var/log/lastlog
RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME} -d
"${DOCKER_HOME_DIR}"
RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" >
"/etc/sudoers.d/hadoop-build-${USER_ID}"
-ENV HOME="${DOCKER_HOME_DIR}"
+ENV HOME "${DOCKER_HOME_DIR}"
UserSpecificDocker
@@ -110,5 +93,4 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
-v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \
-v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \
-u "${USER_ID}" \
- --name "hadoop-build${OS_PLATFORM_SUFFIX}" \
- "hadoop-build${OS_PLATFORM_SUFFIX}-${USER_ID}" "$@"
+ "hadoop-build-${USER_ID}" "$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
