This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b2e2523f316 HIVE-29577: Support prefetching Tez snapshot jars into the
Docker image (#6448)
b2e2523f316 is described below
commit b2e2523f31659bb0ec3f538a30ecfb71be9cf108
Author: Bodor Laszlo <[email protected]>
AuthorDate: Fri Apr 24 11:28:38 2026 +0200
HIVE-29577: Support prefetching Tez snapshot jars into the Docker image
(#6448)
---
packaging/src/docker/Dockerfile | 43 +++++++++++++++++++++++++++++++++++---
packaging/src/docker/build.sh | 14 ++++++++++++-
packaging/src/docker/entrypoint.sh | 18 +++++++++++++++-
3 files changed, 70 insertions(+), 5 deletions(-)
diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile
index b400533506e..ba77deefcf1 100644
--- a/packaging/src/docker/Dockerfile
+++ b/packaging/src/docker/Dockerfile
@@ -67,7 +67,8 @@ RUN tar -xzv \
tar -xzv \
--exclude="apache-tez-$TEZ_VERSION-bin/share" \
-f /opt/apache-tez-$TEZ_VERSION-bin.tar.gz \
- -C /opt
+ -C /opt; \
+ mkdir -p /opt/tez-snapshot;
FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
@@ -75,10 +76,13 @@ ARG UID=1000
ARG HADOOP_VERSION
ARG HIVE_VERSION
ARG TEZ_VERSION
+ARG TEZ_SNAPSHOT_VERSION=
+ARG
TEZ_SNAPSHOT_REPO_URL=https://repository.apache.org/content/repositories/snapshots
+
# Install dependencies
RUN set -ex; \
microdnf update -y; \
- microdnf -y install procps gettext; \
+ microdnf -y install procps gettext wget xmlstarlet; \
microdnf clean all; \
useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive
@@ -93,6 +97,38 @@ ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH
COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME
COPY --from=env --chown=hive /opt/apache-hive-$HIVE_VERSION-bin $HIVE_HOME
COPY --from=env --chown=hive /opt/apache-tez-$TEZ_VERSION-bin $TEZ_HOME
+COPY --from=env --chown=hive /opt/tez-snapshot /opt/tez-snapshot
+
+# When TEZ_SNAPSHOT_VERSION is set, fetch Tez snapshot jars from the Maven
snapshot repository
+# and place them under /opt/tez-snapshot. At runtime, entrypoint.sh symlinks
these into
+# $HIVE_HOME/lib with a "0-" prefix so they sort first in bin/hive's classpath
glob, ensuring
+# snapshot classes take precedence over the Tez release jars bundled with Hive.
+# Maven snapshot repositories use timestamped filenames (e.g.
tez-api-1.0.0-20250101.jar),
+# so we fetch maven-metadata.xml first to resolve the exact filename before
downloading the jar.
+RUN set -eux; \
+ mkdir -p /opt/tez-snapshot-download; \
+ if [[ -n "${TEZ_SNAPSHOT_VERSION}" ]]; then \
+ base_url="${TEZ_SNAPSHOT_REPO_URL}/org/apache/tez"; \
+ for artifact in tez-common tez-api tez-dag tez-mapreduce
tez-runtime-internals tez-runtime-library; do \
+ version_url="${base_url}/${artifact}/${TEZ_SNAPSHOT_VERSION}"; \
+ metadata_url="${version_url}/maven-metadata.xml"; \
+
metadata_file="/opt/tez-snapshot-download/${artifact}-maven-metadata.xml"; \
+ echo "metadata_url=${metadata_url}"; \
+ wget -q "${metadata_url}" -O "${metadata_file}"; \
+ snapshot_value="$(xmlstarlet sel -t -v
"string(/metadata/versioning/snapshotVersions/snapshotVersion[extension='jar'
and not(classifier)]/value)" "${metadata_file}")"; \
+ test -n "${snapshot_value}"; \
+ jar_file="${artifact}-${snapshot_value}.jar"; \
+ jar_url="${version_url}/${jar_file}"; \
+ echo "jar_url=${jar_url}"; \
+ wget -q "${jar_url}" -O "/opt/tez-snapshot/${jar_file}"; \
+ done; \
+ echo "Downloaded Tez snapshot jars under /opt/tez-snapshot:"; \
+ ls -1 /opt/tez-snapshot/*.jar; \
+ else \
+ echo "TEZ_SNAPSHOT_VERSION not set. Skipping Tez snapshot download."; \
+ fi; \
+ rm -rf /opt/tez-snapshot-download
+
COPY --chown=hive entrypoint.sh /
COPY --chown=hive conf $HIVE_HOME/conf
@@ -103,7 +139,8 @@ RUN chmod +x /entrypoint.sh && \
mkdir -p $HIVE_HOME/scratch && \
chown hive $HIVE_HOME/scratch && \
mkdir -p /home/hive/.beeline && \
- chown hive /home/hive/.beeline
+ chown hive /home/hive/.beeline && \
+ chown -R hive /opt/tez-snapshot
USER hive
WORKDIR $HIVE_HOME
diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh
index cd6747da53e..356e1bcd6c4 100755
--- a/packaging/src/docker/build.sh
+++ b/packaging/src/docker/build.sh
@@ -20,13 +20,15 @@ set -eux
HIVE_VERSION=
HADOOP_VERSION=
TEZ_VERSION=
+TEZ_SNAPSHOT_VERSION=
usage() {
cat <<EOF 1>&2
-Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-hive
<Hive version>] [-repo <Docker repo>]
+Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version>
[-tez-snapshot [<Maven snapshot version>]] [-hive <Hive version>] [-repo
<Docker repo>]
Build the Hive Docker image (reused for LLAP too)
-help Display help
-hadoop Build image with the specified Hadoop version (default:
from Maven pom)
-tez Required. Tez release tarball version
(apache-tez-\$TEZ_VERSION-bin.tar.gz from archive)
+-tez-snapshot <ver> Optional. When a snapshot version is given, fetch Tez
Maven snapshot jars into the image. With no version, snapshot prefetch is
skipped.
-hive Build image with the specified Hive version
-repo Docker repository
EOF
@@ -48,6 +50,13 @@ while [ $# -gt 0 ]; do
TEZ_VERSION=$1
shift
;;
+ -tez-snapshot)
+ shift
+ if [ $# -gt 0 ] && [[ "$1" != -* ]]; then
+ TEZ_SNAPSHOT_VERSION=$1
+ shift
+ fi
+ ;;
-hive)
shift
HIVE_VERSION=$1
@@ -135,6 +144,9 @@ DOCKER_BUILD_ARGS=(
--build-arg "HADOOP_VERSION=$HADOOP_VERSION"
--build-arg "TEZ_VERSION=$TEZ_VERSION"
)
+if [ -n "$TEZ_SNAPSHOT_VERSION" ]; then
+ DOCKER_BUILD_ARGS+=(--build-arg "TEZ_SNAPSHOT_VERSION=$TEZ_SNAPSHOT_VERSION")
+fi
docker build \
"$WORK_DIR" \
diff --git a/packaging/src/docker/entrypoint.sh
b/packaging/src/docker/entrypoint.sh
index 377fe594d6e..d06cd06dabb 100644
--- a/packaging/src/docker/entrypoint.sh
+++ b/packaging/src/docker/entrypoint.sh
@@ -147,8 +147,13 @@ function run_tezam {
export HIVE_HOME="${HIVE_HOME:-/opt/hive}"
export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}"
export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}"
+ : "${TEZ_SNAPSHOT_HOME:=/opt/tez-snapshot}"
+ if [[ ! -d "${TEZ_SNAPSHOT_HOME}" ]]; then
+ echo "Tez snapshot home not found at ${TEZ_SNAPSHOT_HOME}. Rebuild image
to prefetch snapshot artifacts."
+ exit 1
+ fi
# service_plugins_descriptor.json references
org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.)
-
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
+
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_SNAPSHOT_HOME}/*:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
local java_bin
local tezam_java_opts
@@ -178,6 +183,17 @@ if [[ "${SKIP_SCHEMA_INIT}" == "false" && (
"${SERVICE_NAME}" == "hiveserver2" |
fi
if [ "${SERVICE_NAME}" == "hiveserver2" ]; then
+ TEZ_SNAPSHOT_HOME="${TEZ_SNAPSHOT_HOME:-/opt/tez-snapshot}"
+ # bin/hive prepends all of $HIVE_HOME/lib/*.jar to $HADOOP_CLASSPATH, so any
entry
+ # we put first in HADOOP_CLASSPATH ends up after all Hive lib jars. To get
snapshot jars
+ # truly first, symlink them into $HIVE_HOME/lib/ with a "0-" prefix: the
for-loop glob in
+ # bin/hive processes jars alphabetically, and ASCII '0' (48) sorts before
'a' (97), so these
+ # symlinks become the very first jars on the classpath (right after the conf
dir entries).
+ if ls "${TEZ_SNAPSHOT_HOME}"/*.jar 1>/dev/null 2>&1; then
+ for snap_jar in "${TEZ_SNAPSHOT_HOME}"/*.jar; do
+ ln -sf "$snap_jar" "${HIVE_HOME}/lib/0-$(basename "$snap_jar")"
+ done
+ fi
export HADOOP_CLASSPATH="$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_CLASSPATH"
exec "$HIVE_HOME/bin/hive" --skiphadoopversion --skiphbasecp --service
"$SERVICE_NAME"
elif [ "${SERVICE_NAME}" == "metastore" ]; then