This is an automated email from the ASF dual-hosted git repository. slfan1989 pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit e50ab07c11be2bad830f531eb2e0646fbca82253 Author: Cheng Pan <[email protected]> AuthorDate: Mon Sep 1 23:31:41 2025 +0800 HADOOP-19659. Upgrade Debian 10 to 11 in build env Dockerfile (#7898) Contributed by Cheng Pan. * HADOOP-19659. Upgrade Debian 10 to 11 in build env Dockerfile Signed-off-by: Shilun Fan <[email protected]> --- BUILDING.txt | 2 +- dev-support/Jenkinsfile | 16 ++--- .../{Dockerfile_debian_10 => Dockerfile_debian_11} | 40 ++++++------ .../docker/pkg-resolver/install-pkg-resolver.sh | 2 +- dev-support/docker/pkg-resolver/packages.json | 71 +++++++++++----------- dev-support/docker/pkg-resolver/platforms.json | 2 +- 6 files changed, 67 insertions(+), 66 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 9a7af891db6..73c50c1f67f 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -31,7 +31,7 @@ On Linux / Mac: $ ./start-build-env.sh [OS platform] - - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10]. + - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10]. Default is 'ubuntu_20'. Note: Currently only default ('ubuntu_20') is supported on arm machine diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 2a2b52b20f9..672061ea09a 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -91,10 +91,10 @@ pipeline { ''' } - dir("${WORKSPACE}/debian-10") { + dir("${WORKSPACE}/debian-11") { sh '''#!/usr/bin/env bash - cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11 ''' } @@ -157,12 +157,12 @@ pipeline { // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is // really needed to ensure that any C++ related/platform change doesn't - // break the Hadoop build on Debian 10. - stage ('precommit-run Debian 10') { + // break the Hadoop build on Debian 11. + stage ('precommit-run Debian 11') { environment { - SOURCEDIR = "${WORKSPACE}/debian-10/src" - PATCHDIR = "${WORKSPACE}/debian-10/out" - DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" + SOURCEDIR = "${WORKSPACE}/debian-11/src" + PATCHDIR = "${WORKSPACE}/debian-11/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11" IS_OPTIONAL = 1 } @@ -182,7 +182,7 @@ pipeline { failure { sh '''#!/usr/bin/env bash - cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" + cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}" ''' archiveArtifacts "out/**" } diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_11 similarity index 71% rename from dev-support/docker/Dockerfile_debian_10 rename to dev-support/docker/Dockerfile_debian_11 index 71446b27f68..2dc47eee6ac 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_11 @@ -17,7 +17,7 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM debian:10 +FROM debian:11 WORKDIR /root @@ -29,26 +29,24 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras -ENV DEBIAN_FRONTEND noninteractive -ENV DEBCONF_TERSE true +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_TERSE=true ###### # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-pkg-resolver.sh -RUN pkg-resolver/install-pkg-resolver.sh debian:10 +RUN pkg-resolver/install-pkg-resolver.sh debian:11 ###### # Install packages from apt ###### # hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \ - && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \ - && apt-get -q update \ - && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \ && apt-get clean \ + && update-java-alternatives -s java-1.11.0-openjdk-amd64 \ && rm -rf /var/lib/apt/lists/* # TODO : Set locale @@ -56,38 +54,38 @@ RUN apt-get -q update \ ###### # Set env vars required to build Hadoop ###### -ENV MAVEN_HOME /usr +ENV MAVEN_HOME=/usr # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) -ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 ####### # Set env vars for SpotBugs 4.2.2 ####### -ENV SPOTBUGS_HOME /opt/spotbugs +ENV SPOTBUGS_HOME=/opt/spotbugs ####### # Set env vars for Google Protobuf 3.21.12 ####### -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" +ENV PROTOBUF_HOME=/opt/protobuf +ENV PATH="${PATH}:/opt/protobuf/bin" ### # Avoid out of memory errors in builds ### -ENV MAVEN_OPTS -Xms256m -Xmx3072m +ENV MAVEN_OPTS="-Xms256m -Xmx3072m" # Skip gpg verification when downloading Yetus via yetus-wrapper -ENV HADOOP_SKIP_YETUS_VERIFICATION true +ENV HADOOP_SKIP_YETUS_VERIFICATION=true #### # Install packages #### -RUN pkg-resolver/install-cmake.sh debian:10 -RUN pkg-resolver/install-spotbugs.sh debian:10 -RUN pkg-resolver/install-boost.sh debian:10 -RUN pkg-resolver/install-protobuf.sh debian:10 -RUN pkg-resolver/install-hadolint.sh debian:10 -RUN pkg-resolver/install-intel-isa-l.sh debian:10 +RUN pkg-resolver/install-cmake.sh debian:11 +RUN pkg-resolver/install-spotbugs.sh debian:11 +RUN pkg-resolver/install-boost.sh debian:11 +RUN pkg-resolver/install-protobuf.sh debian:11 +RUN pkg-resolver/install-hadolint.sh debian:11 +RUN pkg-resolver/install-intel-isa-l.sh debian:11 ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh index 70e94b3792d..81dd0002f40 100644 --- a/dev-support/docker/pkg-resolver/install-pkg-resolver.sh +++ b/dev-support/docker/pkg-resolver/install-pkg-resolver.sh @@ -24,7 +24,7 @@ fi chmod a+x pkg-resolver/*.sh pkg-resolver/*.py chmod a+r pkg-resolver/*.json -if [ "$1" == "debian:10" ]; then +if [ "$1" == "debian:11" ]; then apt-get -q update apt-get -q install -y --no-install-recommends python3 \ python3-pip \ diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index 28b8733e04b..452ed015c32 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -1,38 +1,38 @@ { "ant": { - "debian:10": "ant", + "debian:11": "ant", "ubuntu:focal": "ant", "ubuntu:focal::arch64": "ant", "ubuntu:noble": "ant", "rockylinux:8": "ant" }, "apt-utils": { - "debian:10": "apt-utils", + "debian:11": "apt-utils", "ubuntu:focal": "apt-utils", "ubuntu:noble": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, "automake": { - "debian:10": "automake", + "debian:11": "automake", "ubuntu:focal": "automake", "ubuntu:noble": "automake", "ubuntu:focal::arch64": "automake", "rockylinux:8": "automake" }, "bats": { - "debian:10": "bats", + "debian:11": "bats", "ubuntu:focal": "bats", "ubuntu:noble": "bats", "ubuntu:focal::arch64": "bats" }, "build-essential": { - "debian:10": "build-essential", + "debian:11": "build-essential", "ubuntu:focal": "build-essential", "ubuntu:noble": "build-essential", "ubuntu:focal::arch64": "build-essential" }, "bzip2": { - "debian:10": [ + "debian:11": [ "bzip2", "libbz2-dev" ], @@ -54,7 +54,7 @@ ] }, "clang": { - "debian:10": "clang", + "debian:11": "clang", "ubuntu:focal": "clang", "ubuntu:noble": "clang", "ubuntu:focal::arch64": "clang", @@ -66,7 +66,7 @@ "ubuntu:focal::arch64": "cmake" }, "curl": { - "debian:10": [ + "debian:11": [ "curl", "libcurl4-openssl-dev" ], @@ -88,7 +88,7 @@ ] }, "doxygen": { - "debian:10": "doxygen", + "debian:11": "doxygen", "ubuntu:focal": "doxygen", "ubuntu:noble": "doxygen", "ubuntu:focal::arch64": "doxygen" @@ -97,7 +97,7 @@ "rockylinux:8": "dnf" }, "fuse": { - "debian:10": [ + "debian:11": [ "fuse", "libfuse-dev" ], @@ -120,7 +120,7 @@ ] }, "gcc": { - "debian:10": { + "debian:11": { "bullseye": [ "gcc", "g++" @@ -140,39 +140,39 @@ ] }, "git": { - "debian:10": "git", + "debian:11": "git", "ubuntu:focal": "git", "ubuntu:noble": "git", "ubuntu:focal::arch64": "git", "rockylinux:8": "git" }, "gnupg-agent": { - "debian:10": "gnupg-agent", + "debian:11": "gnupg-agent", "ubuntu:focal": "gnupg-agent", "ubuntu:noble": "gnupg-agent", "ubuntu:focal::arch64": "gnupg-agent" }, "hugo": { - "debian:10": "hugo", + "debian:11": "hugo", "ubuntu:focal": "hugo", "ubuntu:noble": "hugo", "ubuntu:focal::arch64": "hugo" }, "libbcprov-java": { - "debian:10": "libbcprov-java", + "debian:11": "libbcprov-java", "ubuntu:focal": "libbcprov-java", "ubuntu:noble": "libbcprov-java", "ubuntu:focal::arch64": "libbcprov-java" }, "libtool": { - "debian:10": "libtool", + "debian:11": "libtool", "ubuntu:focal": "libtool", "ubuntu:noble": "libtool", "ubuntu:focal::arch64": "libtool", "rockylinux:8": "libtool" }, "openssl": { - "debian:10": "libssl-dev", + "debian:11": "libssl-dev", "ubuntu:focal": "libssl-dev", "ubuntu:noble": "libssl-dev", "ubuntu:focal::arch64": "libssl-dev", @@ -185,7 +185,7 @@ ] }, "protocol-buffers": { - "debian:10": [ + "debian:11": [ "libprotobuf-dev", "libprotoc-dev" ], @@ -203,20 +203,20 @@ ] }, "sasl": { - "debian:10": "libsasl2-dev", + "debian:11": "libsasl2-dev", "ubuntu:focal": "libsasl2-dev", "ubuntu:noble": "libsasl2-dev", "ubuntu:focal::arch64": "libsasl2-dev", "rockylinux:8": "cyrus-sasl-devel" }, "snappy": { - "debian:10": "libsnappy-dev", + "debian:11": "libsnappy-dev", "ubuntu:focal": "libsnappy-dev", "ubuntu:noble": "libsnappy-dev", "ubuntu:focal::arch64": "libsnappy-dev" }, "zlib": { - "debian:10": [ + "debian:11": [ "libzstd-dev", "zlib1g-dev" ], @@ -238,7 +238,7 @@ ] }, "locales": { - "debian:10": "locales", + "debian:11": "locales", "ubuntu:focal": "locales", "ubuntu:noble": "locales", "ubuntu:focal::arch64": "locales" @@ -248,20 +248,23 @@ "ubuntu:noble": "libtirpc-dev" }, "make": { - "debian:10": "make", + "debian:11": "make", "ubuntu:focal": "make", "ubuntu:noble": "make", "ubuntu:focal::arch64": "make", "rockylinux:8": "make" }, "maven": { - "debian:10": "maven", + "debian:11": "maven", "ubuntu:focal": "maven", "ubuntu:noble": "maven", "ubuntu:focal::arch64": "maven" }, "java": { - "debian:10": "openjdk-11-jdk", + "debian:11": [ + "openjdk-11-jdk", + "openjdk-17-jdk" + ], "ubuntu:focal": [ "temurin-24-jdk", "openjdk-8-jdk", @@ -281,21 +284,21 @@ ] }, "pinentry-curses": { - "debian:10": "pinentry-curses", + "debian:11": "pinentry-curses", "ubuntu:focal": "pinentry-curses", "ubuntu:noble": "pinentry-curses", "ubuntu:focal::arch64": "pinentry-curses", "rockylinux:8": "pinentry-curses" }, "pkg-config": { - "debian:10": "pkg-config", + "debian:11": "pkg-config", "ubuntu:focal": "pkg-config", "ubuntu:noble": "pkg-config", "ubuntu:focal::arch64": "pkg-config", "rockylinux:8": "pkg-config" }, "python": { - "debian:10": [ + "debian:11": [ "python3", "python3-pip", "python3-pkg-resources", @@ -332,14 +335,14 @@ ] }, "rsync": { - "debian:10": "rsync", + "debian:11": "rsync", "ubuntu:focal": "rsync", "ubuntu:noble": "rsync", "ubuntu:focal::arch64": "rsync", "rockylinux:8": "rsync" }, "shellcheck": { - "debian:10": "shellcheck", + "debian:11": "shellcheck", "ubuntu:focal": "shellcheck", "ubuntu:noble": "shellcheck", "ubuntu:focal::arch64": "shellcheck" @@ -348,27 +351,27 @@ "rockylinux:8": "perl-Digest-SHA" }, "software-properties-common": { - "debian:10": "software-properties-common", + "debian:11": "software-properties-common", "ubuntu:focal": "software-properties-common", "ubuntu:noble": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" }, "sudo": { - "debian:10": "sudo", + "debian:11": "sudo", "ubuntu:focal": "sudo", "ubuntu:noble": "sudo", "ubuntu:focal::arch64": "sudo", "rockylinux:8": "sudo" }, "valgrind": { - "debian:10": "valgrind", + "debian:11": "valgrind", "ubuntu:focal": "valgrind", "ubuntu:noble": "valgrind", "ubuntu:focal::arch64": "valgrind", "rockylinux:8": "valgrind" }, "yasm": { - "debian:10": "yasm", + "debian:11": "yasm", "ubuntu:focal": "yasm", "ubuntu:noble": "yasm", "ubuntu:focal::arch64": "yasm" diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json index eebf7fc948f..00f6c129b20 100644 --- a/dev-support/docker/pkg-resolver/platforms.json +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -3,5 +3,5 @@ "ubuntu:focal::arch64", "ubuntu:noble", "rockylinux:8", - "debian:10" + "debian:11" ] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
