This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f6e75cdb800 HIVE-29419: Provide a Hive-specific docker image for Tez
AM (#6435)
f6e75cdb800 is described below
commit f6e75cdb800dcce2e4f8b5bd7cb69c1618e164db
Author: Bodor Laszlo <[email protected]>
AuthorDate: Fri Apr 24 00:58:15 2026 +0200
HIVE-29419: Provide a Hive-specific docker image for Tez AM (#6435)
---
.../hive/llap/tezplugins/LlapTaskCommunicator.java | 2 +-
packaging/src/docker/Dockerfile | 1 -
packaging/src/docker/build.sh | 23 ++++--
packaging/src/docker/conf/hive-site.xml.template | 21 ++++++
.../docker/conf/service_plugins_descriptor.json | 22 ++++++
packaging/src/docker/conf/tez-log4j.properties | 5 ++
packaging/src/docker/conf/tez-site.xml.template | 74 ++++++++++++++++++
packaging/src/docker/docker-compose.yml | 37 +++++++--
packaging/src/docker/entrypoint.sh | 88 ++++++++++++++++------
9 files changed, 235 insertions(+), 38 deletions(-)
diff --git
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
index d850caae958..0451848f606 100644
---
a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++
b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -257,7 +257,7 @@ protected void startRpcServer() {
int numHandlers =
HiveConf.getIntVar(conf,
ConfVars.LLAP_TASK_COMMUNICATOR_LISTENER_THREAD_COUNT);
String[] portRange =
- conf.get(HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT.varname)
+ HiveConf.getVar(conf,
HiveConf.ConfVars.LLAP_TASK_UMBILICAL_SERVER_PORT)
.split("-");
boolean isHadoopSecurityAuthorizationEnabled = conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false);
diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile
index 17f7c96eade..b400533506e 100644
--- a/packaging/src/docker/Dockerfile
+++ b/packaging/src/docker/Dockerfile
@@ -75,7 +75,6 @@ ARG UID=1000
ARG HADOOP_VERSION
ARG HIVE_VERSION
ARG TEZ_VERSION
-
# Install dependencies
RUN set -ex; \
microdnf update -y; \
diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh
index 9d8258320f5..cd6747da53e 100755
--- a/packaging/src/docker/build.sh
+++ b/packaging/src/docker/build.sh
@@ -22,11 +22,11 @@ HADOOP_VERSION=
TEZ_VERSION=
usage() {
cat <<EOF 1>&2
-Usage: $0 [-h] [-hadoop <Hadoop version>] [-tez <Tez version>] [-hive <Hive
version>] [-repo <Docker repo>]
+Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-hive
<Hive version>] [-repo <Docker repo>]
Build the Hive Docker image (reused for LLAP too)
-help Display help
--hadoop Build image with the specified Hadoop version
--tez Build image with the specified Tez version
+-hadoop Build image with the specified Hadoop version (default:
from Maven pom)
+-tez Required. Tez release tarball version
(apache-tez-\$TEZ_VERSION-bin.tar.gz from archive)
-hive Build image with the specified Hive version
-repo Docker repository
EOF
@@ -64,6 +64,12 @@ while [ $# -gt 0 ]; do
esac
done
+if [ -z "${TEZ_VERSION}" ]; then
+ echo "Error: -tez <Tez version> is required." >&2
+ usage
+ exit 1
+fi
+
SCRIPT_DIR=$(cd $(dirname $0); pwd)
SOURCE_DIR=${SOURCE_DIR:-"$SCRIPT_DIR/../../.."}
repo=${REPO:-apache}
@@ -123,12 +129,17 @@ cp "$CACHE_DIR/apache-tez-$TEZ_VERSION-bin.tar.gz"
"$WORK_DIR/"
cp -R "$SOURCE_DIR/packaging/src/docker/conf" "$WORK_DIR/"
cp -R "$SOURCE_DIR/packaging/src/docker/entrypoint.sh" "$WORK_DIR/"
cp "$SOURCE_DIR/packaging/src/docker/Dockerfile" "$WORK_DIR/"
+
+DOCKER_BUILD_ARGS=(
+ --build-arg "HIVE_VERSION=$HIVE_VERSION"
+ --build-arg "HADOOP_VERSION=$HADOOP_VERSION"
+ --build-arg "TEZ_VERSION=$TEZ_VERSION"
+)
+
docker build \
"$WORK_DIR" \
-f "$WORK_DIR/Dockerfile" \
-t "$repo/hive:$HIVE_VERSION" \
- --build-arg "HIVE_VERSION=$HIVE_VERSION" \
- --build-arg "HADOOP_VERSION=$HADOOP_VERSION" \
- --build-arg "TEZ_VERSION=$TEZ_VERSION"
+ "${DOCKER_BUILD_ARGS[@]}"
rm -r "${WORK_DIR}"
diff --git a/packaging/src/docker/conf/hive-site.xml.template
b/packaging/src/docker/conf/hive-site.xml.template
index c3d28b29691..13f6b8a0c09 100644
--- a/packaging/src/docker/conf/hive-site.xml.template
+++ b/packaging/src/docker/conf/hive-site.xml.template
@@ -88,4 +88,25 @@
<name>hive.query.results.cache.directory</name>
<value>${HIVE_QUERY_RESULTS_CACHE_DIRECTORY}</value>
</property>
+ <property>
+ <name>hive.server2.tez.initialize.default.sessions</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.server2.tez.use.external.sessions</name>
+ <value>${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS}</value>
+ </property>
+ <!--
+ A registry namespace prefix is a hardcoded prefix for Tez external
sessions.
+ The actual tez.am.registry.namespace value is appended to this prefix.
+ Once hive can use the registry client that Tez provides
(ZkAMRegistryClient), this property will be removed.
+ -->
+ <property>
+ <name>hive.server2.tez.external.sessions.namespace</name>
+ <value>/tez-external-sessions${TEZ_AM_REGISTRY_NAMESPACE}</value>
+ </property>
+ <property>
+ <name>hive.server2.tez.external.sessions.registry.class</name>
+
<value>org.apache.hadoop.hive.ql.exec.tez.ZookeeperExternalSessionsRegistryClient</value>
+ </property>
</configuration>
diff --git a/packaging/src/docker/conf/service_plugins_descriptor.json
b/packaging/src/docker/conf/service_plugins_descriptor.json
new file mode 100644
index 00000000000..133f6eaa649
--- /dev/null
+++ b/packaging/src/docker/conf/service_plugins_descriptor.json
@@ -0,0 +1,22 @@
+{
+ "taskSchedulerDescriptors": [
+ {
+ "className":
"org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService",
+ "entityName": "LLAP"
+ }
+ ],
+ "containerLauncherDescriptors": [
+ {
+ "className":
"org.apache.hadoop.hive.llap.tezplugins.LlapContainerLauncher",
+ "entityName": "LLAP"
+ }
+ ],
+ "taskCommunicatorDescriptors": [
+ {
+ "className":
"org.apache.hadoop.hive.llap.tezplugins.LlapTaskCommunicator",
+ "entityName": "LLAP"
+ }
+ ],
+ "enableContainers": false,
+ "enableUber": false
+}
diff --git a/packaging/src/docker/conf/tez-log4j.properties
b/packaging/src/docker/conf/tez-log4j.properties
new file mode 100644
index 00000000000..6afa36a6356
--- /dev/null
+++ b/packaging/src/docker/conf/tez-log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootLogger=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%n
diff --git a/packaging/src/docker/conf/tez-site.xml.template
b/packaging/src/docker/conf/tez-site.xml.template
new file mode 100644
index 00000000000..a420137a97e
--- /dev/null
+++ b/packaging/src/docker/conf/tez-site.xml.template
@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>tez.am.mode.session</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>tez.am.framework.mode</name>
+ <value>${TEZ_FRAMEWORK_MODE}</value>
+ </property>
+ <property>
+ <name>tez.am.zookeeper.quorum</name>
+ <value>${TEZ_AM_ZOOKEEPER_QUORUM}</value>
+ </property>
+ <property>
+ <name>tez.am.registry.namespace</name>
+ <value>${TEZ_AM_REGISTRY_NAMESPACE}</value>
+ </property>
+ <property>
+ <name>tez.local.mode</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>tez.am.tez-ui.webservice.enable</name>
+ <value>false</value>
+ </property>
+ <!-- Tez AM should not timeout in ZK Mode -->
+ <property>
+ <name>tez.session.am.dag.submit.timeout.secs</name>
+ <value>-1</value>
+ </property>
+ <property>
+ <name>tez.ignore.lib.uris</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>tez.am.disable.client-version-check</name>
+ <value>true</value>
+ </property>
+
+ <!--
+ In standalone AM mode with LLAP, the task scheduler
(LlapTaskSchedulerService) and task
+ communicator (LlapTaskCommunicator) are instantiated at AM startup from
service_plugins_descriptor.json
+ on the classpath — not from the DAG payload as in YARN-submitted mode.
These plugins read their
+ configuration from TezConfiguration, which is built exclusively from
tez-site.xml on the classpath.
+ Hive-specific properties must therefore appear here so the AM can find
LLAP daemons and connect
+ to them, even though they are conceptually Hive settings.
+ -->
+ <property>
+ <name>hive.zookeeper.quorum</name>
+ <value>${HIVE_ZOOKEEPER_QUORUM}</value>
+ </property>
+ <property>
+ <name>hive.llap.daemon.service.hosts</name>
+ <value>${HIVE_LLAP_DAEMON_SERVICE_HOSTS}</value>
+ </property>
+</configuration>
diff --git a/packaging/src/docker/docker-compose.yml
b/packaging/src/docker/docker-compose.yml
index 37b9d70e138..475b9e6669e 100644
--- a/packaging/src/docker/docker-compose.yml
+++ b/packaging/src/docker/docker-compose.yml
@@ -132,14 +132,35 @@ services:
- zookeeper_datalog:/datalog
- zookeeper_logs:/logs
- #TODO Tez AM container (in the meantime, the HS2(with local Tez AM) + LLAP
daemon setup is working properly)
- # 1. Define and use a Tez AM image from HIVE-29419 or TEZ-4682
- # 2. Configure TezAM to use Zookeeper Llap Registry to discover the LLAP
daemon
- # 3. Configure HiveServer2 to use the Tez AM Zookeeper Registry to
discover the Tez AM
- # Prerequisites:
- # - tez-api 1.0.0-SNAPSHOT jar injected into HiveSever2 until Tez
1.0.0 is released
- # - make HIVE-29477 happen to let HiveServer2 use Tez external sessions
- # 4. Define hadoop components here to be used by all the containers
(working example can be found at TEZ-4682), currently a local volume
+ tezam:
+ profiles:
+ - llap
+ image: apache/hive:${HIVE_VERSION}
+ container_name: tezam
+ hostname: tezam
+ depends_on:
+ - zookeeper
+ restart: on-failure:3
+ environment:
+ USER: hive
+ SERVICE_NAME: 'tezam'
+
+ TEZ_FRAMEWORK_MODE: STANDALONE_ZOOKEEPER
+ TEZ_AM_ZOOKEEPER_QUORUM: zookeeper:2181
+
+ # LLAP daemon discovery
+ HIVE_ZOOKEEPER_QUORUM: zookeeper:2181
+ HIVE_LLAP_DAEMON_SERVICE_HOSTS: '@llap0'
+
+ # Directories shared between HiveServer2 and LLAP daemon
+ HIVE_SCRATCH_DIR: /opt/hive/scratch
+ HIVE_QUERY_RESULTS_CACHE_DIRECTORY: /opt/hive/scratch/_resultscache_
+
+ volumes:
+ - warehouse:/opt/hive/data/warehouse
+ - scratch:/opt/hive/scratch
+ networks:
+ - hive
llapdaemon:
profiles:
diff --git a/packaging/src/docker/entrypoint.sh
b/packaging/src/docker/entrypoint.sh
index 5a5979b9ac9..377fe594d6e 100644
--- a/packaging/src/docker/entrypoint.sh
+++ b/packaging/src/docker/entrypoint.sh
@@ -41,8 +41,17 @@ export
HIVE_WAREHOUSE_PATH="${HIVE_WAREHOUSE_PATH:-/opt/hive/data/warehouse}"
export HIVE_SCRATCH_DIR="${HIVE_SCRATCH_DIR:-/opt/hive/scratch}"
export
HIVE_QUERY_RESULTS_CACHE_DIRECTORY="${HIVE_WAREHOUSE_PATH:-/opt/hive/scratch/_resultscache_}"
+export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}"
+export
HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}"
+
+export
HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS="${HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS:-true}"
+export TEZ_FRAMEWORK_MODE="${TEZ_FRAMEWORK_MODE:-STANDALONE_ZOOKEEPER}"
+export TEZ_AM_REGISTRY_NAMESPACE="${TEZ_AM_REGISTRY_NAMESPACE:-/tez_am/server}"
+export
TEZ_AM_ZOOKEEPER_QUORUM="${TEZ_AM_ZOOKEEPER_QUORUM:-${HIVE_ZOOKEEPER_QUORUM}}"
+
envsubst < $HIVE_HOME/conf/core-site.xml.template >
$HIVE_HOME/conf/core-site.xml
envsubst < $HIVE_HOME/conf/hive-site.xml.template >
$HIVE_HOME/conf/hive-site.xml
+envsubst < $HIVE_HOME/conf/tez-site.xml.template > $HIVE_HOME/conf/tez-site.xml
# =========================================================================
: "${DB_DRIVER:=derby}"
@@ -68,9 +77,30 @@ function initialize_hive {
fi
}
+function append_java_opens {
+ local -n _opts=$1
+ local opens=(
+ "--add-opens=java.base/java.lang=ALL-UNNAMED"
+ "--add-opens=java.base/java.util=ALL-UNNAMED"
+ "--add-opens=java.base/java.io=ALL-UNNAMED"
+ "--add-opens=java.base/java.net=ALL-UNNAMED"
+ "--add-opens=java.base/java.nio=ALL-UNNAMED"
+ "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+ "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+ "--add-opens=java.base/java.util.regex=ALL-UNNAMED"
+ "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+ "--add-opens=java.sql/java.sql=ALL-UNNAMED"
+ "--add-opens=java.base/java.text=ALL-UNNAMED"
+ "-Dnet.bytebuddy.experimental=true"
+ )
+ for opt in "${opens[@]}"; do
+ if [[ " ${_opts} " != *" ${opt} "* ]]; then
+ _opts="${_opts} ${opt}"
+ fi
+ done
+}
+
function run_llap {
- export HIVE_ZOOKEEPER_QUORUM="${HIVE_ZOOKEEPER_QUORUM:-zookeeper:2181}"
- export
HIVE_LLAP_DAEMON_SERVICE_HOSTS="${HIVE_LLAP_DAEMON_SERVICE_HOSTS:-@llap0}"
export LLAP_MEMORY_MB="${LLAP_MEMORY_MB:-1024}"
export LLAP_EXECUTORS="${LLAP_EXECUTORS:-1}"
@@ -87,25 +117,8 @@ function run_llap {
export LLAP_DAEMON_CONF_DIR="${LLAP_DAEMON_CONF_DIR:-$HIVE_CONF_DIR}"
export
LLAP_DAEMON_USER_CLASSPATH="${LLAP_DAEMON_USER_CLASSPATH:-$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/tools/lib/*}"
- JAVA_ADD_OPENS=(
- "--add-opens=java.base/java.lang=ALL-UNNAMED"
- "--add-opens=java.base/java.util=ALL-UNNAMED"
- "--add-opens=java.base/java.io=ALL-UNNAMED"
- "--add-opens=java.base/java.net=ALL-UNNAMED"
- "--add-opens=java.base/java.nio=ALL-UNNAMED"
- "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
- "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
- "--add-opens=java.base/java.util.regex=ALL-UNNAMED"
- "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
- "--add-opens=java.sql/java.sql=ALL-UNNAMED"
- "--add-opens=java.base/java.text=ALL-UNNAMED"
- "-Dnet.bytebuddy.experimental=true"
- )
- for opt in "${JAVA_ADD_OPENS[@]}"; do
- if [[ " ${LLAP_DAEMON_OPTS:-} " != *" ${opt} "* ]]; then
- LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${opt}"
- fi
- done
+ LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-}"
+ append_java_opens LLAP_DAEMON_OPTS
if [[ -n "${LLAP_EXTRA_OPTS:-}" ]]; then
export LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS:-} ${LLAP_EXTRA_OPTS}"
@@ -121,6 +134,35 @@ function run_llap {
exec "${LLAP_RUN_SCRIPT}" run "$@"
}
+function run_tezam {
+ : "${USER:=hive}"
+ : "${LOCAL_DIRS:="/tmp"}"
+ : "${LOG_DIRS:="/opt/tez/logs"}"
+ : "${APP_SUBMIT_TIME_ENV:=$(($(date +%s) * 1000))}"
+ : "${TEZ_AM_EXTERNAL_ID:="tez-session-${HOSTNAME:-tezam}"}"
+ export USER LOCAL_DIRS LOG_DIRS APP_SUBMIT_TIME_ENV TEZ_AM_EXTERNAL_ID
+
+ export HADOOP_HOME="${HADOOP_HOME:-/opt/hadoop}"
+ export TEZ_HOME="${TEZ_HOME:-/opt/tez}"
+ export HIVE_HOME="${HIVE_HOME:-/opt/hive}"
+ export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}"
+ export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}"
+ # service_plugins_descriptor.json references
org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.)
+
tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
+
+ local java_bin
+ local tezam_java_opts
+ java_bin="${JAVA_HOME:+$JAVA_HOME/bin/}java"
+ tezam_java_opts="${HADOOP_CLIENT_OPTS:-}
-Dlog4j.configuration=file:${HIVE_CONF_DIR}/tez-log4j.properties"
+ append_java_opens tezam_java_opts
+ "${java_bin}" ${tezam_java_opts} -cp "${tezam_cp}"
org.apache.tez.dag.app.DAGAppMaster --session "$@"
+ local rc=$?
+ if [[ ${rc} -ne 0 ]]; then
+ echo "DAGAppMaster exited with code ${rc}. See logs above for details."
+ fi
+ exit "${rc}"
+}
+
export HIVE_CONF_DIR=$HIVE_HOME/conf
if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then
find "${HIVE_CUSTOM_CONF_DIR}" -type f -exec \
@@ -129,7 +171,7 @@ if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then
export TEZ_CONF_DIR=$HIVE_CONF_DIR
fi
-export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx1G $SERVICE_OPTS"
+export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} -Xmx1G ${SERVICE_OPTS:-}"
if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2"
|| "${SERVICE_NAME}" == "metastore" ) ]]; then
# handles schema initialization
initialize_hive
@@ -147,4 +189,6 @@ elif [ "${SERVICE_NAME}" == "metastore" ]; then
fi
elif [ "${SERVICE_NAME}" == "llap" ]; then
run_llap "$@"
+elif [ "${SERVICE_NAME}" == "tezam" ]; then
+ run_tezam "$@"
fi