This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new b2e2523f316 HIVE-29577: Support prefetching Tez snapshot jars into the 
Docker image (#6448)
b2e2523f316 is described below

commit b2e2523f31659bb0ec3f538a30ecfb71be9cf108
Author: Bodor Laszlo <[email protected]>
AuthorDate: Fri Apr 24 11:28:38 2026 +0200

    HIVE-29577: Support prefetching Tez snapshot jars into the Docker image 
(#6448)
---
 packaging/src/docker/Dockerfile    | 43 +++++++++++++++++++++++++++++++++++---
 packaging/src/docker/build.sh      | 14 ++++++++++++-
 packaging/src/docker/entrypoint.sh | 18 +++++++++++++++-
 3 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile
index b400533506e..ba77deefcf1 100644
--- a/packaging/src/docker/Dockerfile
+++ b/packaging/src/docker/Dockerfile
@@ -67,7 +67,8 @@ RUN tar -xzv \
     tar -xzv \
       --exclude="apache-tez-$TEZ_VERSION-bin/share" \
       -f /opt/apache-tez-$TEZ_VERSION-bin.tar.gz \
-      -C /opt
+      -C /opt; \
+    mkdir -p /opt/tez-snapshot;
 
 FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
 
@@ -75,10 +76,13 @@ ARG UID=1000
 ARG HADOOP_VERSION
 ARG HIVE_VERSION
 ARG TEZ_VERSION
+ARG TEZ_SNAPSHOT_VERSION=
+ARG 
TEZ_SNAPSHOT_REPO_URL=https://repository.apache.org/content/repositories/snapshots
+
 # Install dependencies
 RUN set -ex; \
     microdnf update -y; \
-    microdnf -y install procps gettext; \
+    microdnf -y install procps gettext wget xmlstarlet; \
     microdnf clean all; \
     useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive
 
@@ -93,6 +97,38 @@ ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH
 COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME
 COPY --from=env --chown=hive /opt/apache-hive-$HIVE_VERSION-bin $HIVE_HOME
 COPY --from=env --chown=hive /opt/apache-tez-$TEZ_VERSION-bin $TEZ_HOME
+COPY --from=env --chown=hive /opt/tez-snapshot /opt/tez-snapshot
+
+# When TEZ_SNAPSHOT_VERSION is set, fetch Tez snapshot jars from the Maven 
snapshot repository
+# and place them under /opt/tez-snapshot. At runtime, entrypoint.sh symlinks 
these into
+# $HIVE_HOME/lib with a "0-" prefix so they sort first in bin/hive's classpath 
glob, ensuring
+# snapshot classes take precedence over the Tez release jars bundled with Hive.
+# Maven snapshot repositories use timestamped filenames (e.g. 
tez-api-1.0.0-20250101.jar),
+# so we fetch maven-metadata.xml first to resolve the exact filename before 
downloading the jar.
+RUN set -eux; \
+    mkdir -p /opt/tez-snapshot-download; \
+    if [[ -n "${TEZ_SNAPSHOT_VERSION}" ]]; then \
+      base_url="${TEZ_SNAPSHOT_REPO_URL}/org/apache/tez"; \
+      for artifact in tez-common tez-api tez-dag tez-mapreduce 
tez-runtime-internals tez-runtime-library; do \
+        version_url="${base_url}/${artifact}/${TEZ_SNAPSHOT_VERSION}"; \
+        metadata_url="${version_url}/maven-metadata.xml"; \
+        
metadata_file="/opt/tez-snapshot-download/${artifact}-maven-metadata.xml"; \
+        echo "metadata_url=${metadata_url}"; \
+        wget -q "${metadata_url}" -O "${metadata_file}"; \
+        snapshot_value="$(xmlstarlet sel -t -v 
"string(/metadata/versioning/snapshotVersions/snapshotVersion[extension='jar' 
and not(classifier)]/value)" "${metadata_file}")"; \
+        test -n "${snapshot_value}"; \
+        jar_file="${artifact}-${snapshot_value}.jar"; \
+        jar_url="${version_url}/${jar_file}"; \
+        echo "jar_url=${jar_url}"; \
+        wget -q "${jar_url}" -O "/opt/tez-snapshot/${jar_file}"; \
+      done; \
+      echo "Downloaded Tez snapshot jars under /opt/tez-snapshot:"; \
+      ls -1 /opt/tez-snapshot/*.jar; \
+    else \
+      echo "TEZ_SNAPSHOT_VERSION not set. Skipping Tez snapshot download."; \
+    fi; \
+    rm -rf /opt/tez-snapshot-download
+
 
 COPY --chown=hive entrypoint.sh /
 COPY --chown=hive conf $HIVE_HOME/conf
@@ -103,7 +139,8 @@ RUN chmod +x /entrypoint.sh && \
     mkdir -p $HIVE_HOME/scratch && \
     chown hive $HIVE_HOME/scratch && \
     mkdir -p /home/hive/.beeline && \
-    chown hive /home/hive/.beeline
+    chown hive /home/hive/.beeline && \
+    chown -R hive /opt/tez-snapshot
 
 USER hive
 WORKDIR $HIVE_HOME
diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh
index cd6747da53e..356e1bcd6c4 100755
--- a/packaging/src/docker/build.sh
+++ b/packaging/src/docker/build.sh
@@ -20,13 +20,15 @@ set -eux
 HIVE_VERSION=
 HADOOP_VERSION=
 TEZ_VERSION=
+TEZ_SNAPSHOT_VERSION=
 usage() {
     cat <<EOF 1>&2
-Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-hive 
<Hive version>] [-repo <Docker repo>]
+Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> 
[-tez-snapshot [<Maven snapshot version>]] [-hive <Hive version>] [-repo 
<Docker repo>]
 Build the Hive Docker image (reused for LLAP too)
 -help                Display help
 -hadoop              Build image with the specified Hadoop version (default: 
from Maven pom)
 -tez                 Required. Tez release tarball version 
(apache-tez-\$TEZ_VERSION-bin.tar.gz from archive)
+-tez-snapshot <ver>  Optional. When a snapshot version is given, fetch Tez 
Maven snapshot jars into the image. With no version, snapshot prefetch is 
skipped.
 -hive                Build image with the specified Hive version
 -repo                Docker repository
 EOF
@@ -48,6 +50,13 @@ while [ $# -gt 0 ]; do
       TEZ_VERSION=$1
       shift
       ;;
+    -tez-snapshot)
+      shift
+      if [ $# -gt 0 ] && [[ "$1" != -* ]]; then
+        TEZ_SNAPSHOT_VERSION=$1
+        shift
+      fi
+      ;;
     -hive)
       shift
       HIVE_VERSION=$1
@@ -135,6 +144,9 @@ DOCKER_BUILD_ARGS=(
   --build-arg "HADOOP_VERSION=$HADOOP_VERSION"
   --build-arg "TEZ_VERSION=$TEZ_VERSION"
 )
+if [ -n "$TEZ_SNAPSHOT_VERSION" ]; then
+  DOCKER_BUILD_ARGS+=(--build-arg "TEZ_SNAPSHOT_VERSION=$TEZ_SNAPSHOT_VERSION")
+fi
 
 docker build \
         "$WORK_DIR" \
diff --git a/packaging/src/docker/entrypoint.sh 
b/packaging/src/docker/entrypoint.sh
index 377fe594d6e..d06cd06dabb 100644
--- a/packaging/src/docker/entrypoint.sh
+++ b/packaging/src/docker/entrypoint.sh
@@ -147,8 +147,13 @@ function run_tezam {
   export HIVE_HOME="${HIVE_HOME:-/opt/hive}"
   export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}"
   export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}"
+  : "${TEZ_SNAPSHOT_HOME:=/opt/tez-snapshot}"
+  if [[ ! -d "${TEZ_SNAPSHOT_HOME}" ]]; then
+    echo "Tez snapshot home not found at ${TEZ_SNAPSHOT_HOME}. Rebuild image 
to prefetch snapshot artifacts."
+    exit 1
+  fi
   # service_plugins_descriptor.json references 
org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.)
-  
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
+  
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_SNAPSHOT_HOME}/*:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
 
   local java_bin
   local tezam_java_opts
@@ -178,6 +183,17 @@ if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( 
"${SERVICE_NAME}" == "hiveserver2" |
 fi
 
 if [ "${SERVICE_NAME}" == "hiveserver2" ]; then
+  TEZ_SNAPSHOT_HOME="${TEZ_SNAPSHOT_HOME:-/opt/tez-snapshot}"
+  # bin/hive prepends all of $HIVE_HOME/lib/*.jar to $HADOOP_CLASSPATH, so any 
entry
+  # we put first in HADOOP_CLASSPATH ends up after all Hive lib jars.  To get 
snapshot jars
+  # truly first, symlink them into $HIVE_HOME/lib/ with a "0-" prefix: the 
for-loop glob in
+  # bin/hive processes jars alphabetically, and ASCII '0' (48) sorts before 
'a' (97), so these
+  # symlinks become the very first jars on the classpath (right after the conf 
dir entries).
+  if ls "${TEZ_SNAPSHOT_HOME}"/*.jar 1>/dev/null 2>&1; then
+    for snap_jar in "${TEZ_SNAPSHOT_HOME}"/*.jar; do
+      ln -sf "$snap_jar" "${HIVE_HOME}/lib/0-$(basename "$snap_jar")"
+    done
+  fi
   export HADOOP_CLASSPATH="$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_CLASSPATH"
   exec "$HIVE_HOME/bin/hive" --skiphadoopversion --skiphbasecp --service 
"$SERVICE_NAME"
 elif [ "${SERVICE_NAME}" == "metastore" ]; then

Reply via email to