This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1b60acc9155 HIVE-29483: Implement an integration testing mechanism
with real Tez on Yarn including resource localization (#6640)
1b60acc9155 is described below
commit 1b60acc9155646721b41a40863850fb013ffab98
Author: Hemanth Umashankar
<[email protected]>
AuthorDate: Thu Oct 22 13:53:00 2026 +0530
HIVE-29483: Implement an integration testing mechanism with real Tez on
Yarn including resource localization (#6640)
---
itests/pom.xml | 6 +
itests/tez-yarn-it/README.md | 169 +++++++
itests/tez-yarn-it/pom.xml | 505 +++++++++++++++++++++
.../src/test/docker/hadoop-yarn/Dockerfile | 35 ++
.../tez-yarn-it/src/test/docker/hadoop-yarn/config | 59 +++
.../src/test/docker/hadoop-yarn/docker-compose.yml | 76 ++++
.../hive/tez/yarn/ComposeImageSubstitutor.java | 47 ++
.../apache/hive/tez/yarn/StartTezYarnCluster.java | 84 ++++
.../hive/tez/yarn/TestTezYarnLocalization.java | 210 +++++++++
.../org/apache/hive/tez/yarn/TestTezYarnUtils.java | 134 ++++++
.../hive/tez/yarn/TezYarnClusterContainer.java | 299 ++++++++++++
.../src/test/resources/custom_hosts_file | 23 +
.../src/test/resources/hive-site-yarn-it.xml | 126 +++++
.../src/test/resources/log4j2-test.properties | 32 ++
.../src/test/resources/testcontainers.properties | 18 +
.../tez-yarn-it/src/test/resources/yarn-site.xml | 33 ++
16 files changed, 1856 insertions(+)
diff --git a/itests/pom.xml b/itests/pom.xml
index 6c64b0d730c..5a2716d1dd2 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -50,6 +50,7 @@
<module>qtest-iceberg</module>
<module>test-docker</module>
<module>hive-iceberg</module>
+ <module>tez-yarn-it</module>
</modules>
<dependencyManagement>
<dependencies>
@@ -497,6 +498,11 @@
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <version>${awaitility.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>
diff --git a/itests/tez-yarn-it/README.md b/itests/tez-yarn-it/README.md
new file mode 100644
index 00000000000..74eb64187dd
--- /dev/null
+++ b/itests/tez-yarn-it/README.md
@@ -0,0 +1,169 @@
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Tez-on-YARN integration test
+
+An opt-in integration test module that runs Hive-on-Tez against a real
Docker-containerized
+HDFS + YARN cluster. It is used to verify YARN container localization behavior
(including
+`hive-exec.jar`) and is also useful as a small "real" Hadoop cluster for
manual Tez-on-YARN
+testing and debugging.
+
+MiniTezCluster/MiniDFSCluster are not sufficient for this class of testing
because they do
+not exercise the real YARN NodeManager localization path (`LocalResource`
download/extract
+to container-local directories before AM/task launch). In mini-clusters, Tez
tasks run without
+that real container-localization lifecycle.
+
+## Prerequisites
+
+- Java 21
+- Maven 3.6.3 or later
+- Docker Desktop (or Docker Engine) with at least **4 GB** of memory assigned
+
+Docker Desktop 25+ exposes a newer Docker API than Testcontainers' default
helper image
+expects. This module ships `ComposeImageSubstitutor` and
`testcontainers.properties` to
+substitute a compatible helper image automatically; no extra configuration is
required on
+supported setups.
+
+## First-time setup
+
+Build the full Hive distribution once to populate `$HIVE_HOME/lib/` and
install all
+artifacts to `~/.m2`:
+
+```bash
+mvn clean install -DskipTests -Pitests,dist
+```
+
+
+
+## Running the automated tests
+
+This module is **opt-in**: tests are skipped by default. Activate the
`tez-yarn` profile
+with `-Ptez-yarn` to run them. Running `mvn test -pl itests/tez-yarn-it`
without the
+profile skips test execution by design.
+
+```bash
+mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it
+```
+
+By default, this module assembles `target/tez-libs.tar.gz` from Maven
dependencies and
+uses that staged archive for `tez.lib.uris`. You can also select an
alternative source:
+
+- **Default staged archive** (same as above):
+ ```bash
+ mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ -Dtez.dist.source=staged
+ ```
+- **Local archive** (for example from a local Tez checkout):
+ ```bash
+ mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ -Dtez.dist.source=local \
+
-Dtez.dist.tarball=$HOME/apache/tez/tez-dist/target/tez-0.10.6-SNAPSHOT-minimal.tar.gz
+ ```
+- **Download archive** (defaults to Apache archive URL for `${tez.version}`):
+ ```bash
+ mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ -Dtez.dist.source=download
+ ```
+- **Download from a custom URL/mirror**:
+ ```bash
+ mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ -Dtez.dist.source=download \
+ -Dtez.dist.download.url=https://<mirror>/apache-tez-<version>-bin.tar.gz
+ ```
+
+The selected archive must be a Tez distribution tarball suitable for
`tez.lib.uris`
+(for example Apache Tez `*-bin.tar.gz` or Tez dist `*-minimal.tar.gz` outputs).
+
+If you have not run a full install recently, add `-am` to build required
upstream modules:
+
+```bash
+mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -am
+```
+
+To run a single test:
+
+```bash
+mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar
+```
+
+
+
+## Starting a keep-alive cluster for manual testing
+
+To start the cluster and keep it running for manual Beeline testing, explicitly
+run `StartTezYarnCluster`.
+
+```bash
+mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=StartTezYarnCluster
+```
+
+Useful web UIs while the cluster is up:
+
+- NameNode UI: `http://localhost:9870`
+- ResourceManager UI: `http://localhost:8088`
+- NodeManager UI: `http://localhost:8042`
+- NodeManager AM RPC range: `nodemanager:41000-41020` (shown in the startup
banner)
+
+Run a Tez-on-YARN query to exercise jar localization (including `INSERT ...
VALUES`):
+
+```sql
+CREATE TABLE test_tez (id INT, name STRING) STORED AS ORC;
+INSERT INTO test_tez VALUES (1, 'hello'), (2, 'world');
+set hive.fetch.task.conversion=none;
+SELECT * FROM test_tez;
+```
+
+
+
+## Troubleshooting
+
+- **Compose hangs or fails during image pull** — confirm Docker is running and
that at
+least 4 GB of memory is assigned to the Docker VM. Retry after `docker system
prune` if
+disk space is low.
+- **Cluster appears to be left running** — tear down the compose stack
manually:
+ ```bash
+ docker compose -f
itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml down -v
--remove-orphans
+ ```
+- **Docker Desktop 25+ API errors** — the bundled `ComposeImageSubstitutor`
rewrites
+Testcontainers' helper image to a version compatible with recent Docker
daemons. Override
+with `-Dtez.yarn.compose.image=docker:<tag>` if needed.
+- **`tez.dist.source` validation failure** — accepted values are `staged`
(default),
+ `local`, and `download`.
+- **`tez.dist.source=local` fails** — pass an existing path with
+ `-Dtez.dist.tarball=/absolute/path/to/tez.tar.gz`.
+- **Zero tests executed** — activate the `tez-yarn` profile with
`-Pitests,tez-yarn`;
+see [Running the automated tests](#running-the-automated-tests).
+- **Missing dependency / compile errors on a partial build** — use `-am` when
running only
+this module (see above).
+
+
+
+## Re-deploying after changing `hive-exec`
+
+The `hive-exec.jar` localized into YARN task containers is resolved from the
Maven test
+classpath (`ql/target/hive-exec-*.jar`). To pick up code changes:
+
+1. Rebuild `hive-exec`:
+ ```bash
+ mvn package -DskipTests -pl ql
+ ```
+2. Re-run the integration test or `StartTezYarnCluster`.
+
diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml
new file mode 100644
index 00000000000..3d6e26a2fb9
--- /dev/null
+++ b/itests/tez-yarn-it/pom.xml
@@ -0,0 +1,505 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it</artifactId>
+ <version>4.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>hive-it-tez-yarn</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Integration - Tez on YARN Integration tests</name>
+ <properties>
+ <hive.path.to.root>../..</hive.path.to.root>
+ <!-- This module needs Docker; skip by default. Opt in with -Ptez-yarn. -->
+ <skipTests>true</skipTests>
+ <!-- Tez dist input mode: staged (default), local, or download. -->
+ <tez.dist.source>staged</tez.dist.source>
+ <tez.dist.path>${project.build.directory}/tez-libs.tar.gz</tez.dist.path>
+ <tez.dist.prepare.skip>${skipTests}</tez.dist.prepare.skip>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- Explicit Tez dependencies: Hive's Tez compilation/execution path
needs these classes
+ on the in-process HS2 JVM classpath. In the main Hive build they are
often profile-gated. -->
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-common</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-dag</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-library</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-runtime-internals</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Provides org.apache.tez.hadoop.shim.HadoopShimsLoader, which
DAGAppMaster.serviceInit()
+ loads at Tez AM startup. Normally transitive via tez-api; declared
explicitly so it is
+ reliably on the classpath of the in-process HiveServer2 JVM. -->
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>hadoop-shim</artifactId>
+ <version>${tez.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it-util</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <!-- Copy Tez framework jars (org.apache.tez group) and the Hadoop MR
client jars
+ required by the Tez AM into target/tez-stage/. The antrun execution
below then
+ packs them into a tez-libs.tar.gz that the integration tests upload
to the
+ containerised HDFS cluster and reference via tez.lib.uris. This
replaces the
+ previous approach of scanning the test JVM's classpath at runtime,
which was
+ fragile and did not resemble real cluster setups. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>stage-tez-framework-jars</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <skip>${tez.dist.prepare.skip}</skip>
+
<outputDirectory>${project.build.directory}/tez-stage</outputDirectory>
+ <includeGroupIds>org.apache.tez</includeGroupIds>
+ <excludeClassifiers>tests</excludeClassifiers>
+ <stripVersion>false</stripVersion>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ <execution>
+ <id>stage-tez-mr-client-jars</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <skip>${tez.dist.prepare.skip}</skip>
+
<outputDirectory>${project.build.directory}/tez-stage</outputDirectory>
+
<includeGroupIds>org.apache.hadoop,commons-collections</includeGroupIds>
+
<includeArtifactIds>hadoop-mapreduce-client-core,hadoop-mapreduce-client-common,commons-collections</includeArtifactIds>
+ <excludeClassifiers>tests</excludeClassifiers>
+ <stripVersion>false</stripVersion>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>assemble-tez-dist</id>
+ <!-- Must run after the dependency-copy executions in
process-test-resources. -->
+ <phase>test-compile</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <skip>${tez.dist.prepare.skip}</skip>
+ <target>
+ <tar destfile="${project.build.directory}/tez-libs.tar.gz"
compression="gzip">
+ <tarfileset dir="${project.build.directory}/tez-stage"
includes="*.jar"/>
+ </tar>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>generate-custom-hosts-file</id>
+ <!-- Must run after the parent "setup-test-dirs" execution
(process-test-resources). -->
+ <phase>test-compile</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <skip>${skipTests}</skip>
+ <target>
+ <mkdir dir="${test.conf.dir}"/>
+ <copy file="${basedir}/src/test/resources/custom_hosts_file"
tofile="${test.conf.dir}/custom_hosts_file" overwrite="true"/>
+ <exec executable="hostname"
output="${test.conf.dir}/local_hostname.txt" failonerror="true"/>
+ <loadfile property="hive.tez.yarn.it.local.hostname"
srcFile="${test.conf.dir}/local_hostname.txt">
+ <filterchain>
+ <striplinebreaks/>
+ </filterchain>
+ </loadfile>
+ <!--
+ When -Djdk.net.hosts.file is set, Java hostname resolution
uses only that file.
+ Ensure the local machine hostname is resolvable so Hive can
start/compile queries.
+ -->
+ <echo file="${test.conf.dir}/custom_hosts_file"
append="true">127.0.0.1 ${hive.tez.yarn.it.local.hostname}</echo>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>validate-tez-dist-source</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireProperty>
+ <property>tez.dist.source</property>
+ <regex>(staged|local|download)</regex>
+ <regexMessage>
+ Invalid tez.dist.source='${tez.dist.source}'. Use one of:
staged, local, download.
+ </regexMessage>
+ </requireProperty>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- Resolve docker-network hostnames (namenode, resourcemanager, …)
to 127.0.0.1
+ so HDFS URIs embedded in Tez LocalResources are reachable from
both the host JVM
+ and inside YARN containers (via the shared docker network
alias). -->
+ <argLine>${maven.test.jvm.args}
-Djdk.net.hosts.file=${test.conf.dir}/custom_hosts_file
-Dtez.dist.path=${tez.dist.path}
-Dtez.yarn.it.log.file=${project.build.directory}/tez-yarn-it.log</argLine>
+ <!-- StartTezYarnCluster prints a connection banner to System.out;
keep it on the live
+ console (the root pom sets this to true) while slf4j logs are
routed to
+ target/tez-yarn-it.log via log4j2-test.properties. -->
+ <redirectTestOutputToFile>false</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <!-- Override the root pom's console-appender
hive-log4j2.properties with a file-only
+ config so StartTezYarnCluster's banner is the only console
output. -->
+
<log4j.configurationFile>log4j2-test.properties</log4j.configurationFile>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>tez-yarn</id>
+ <properties>
+ <skipTests>false</skipTests>
+ </properties>
+ </profile>
+ <profile>
+ <id>tez-dist-local</id>
+ <activation>
+ <property>
+ <name>tez.dist.source</name>
+ <value>local</value>
+ </property>
+ </activation>
+ <properties>
+ <tez.dist.prepare.skip>true</tez.dist.prepare.skip>
+ <tez.dist.path>${tez.dist.tarball}</tez.dist.path>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>validate-local-tez-dist-tarball</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireProperty>
+ <property>tez.dist.tarball</property>
+ <message>
+ When -Dtez.dist.source=local, also set
-Dtez.dist.tarball=/absolute/path/to/tez.tar.gz.
+ </message>
+ </requireProperty>
+ <requireFilesExist>
+ <files>
+ <file>${tez.dist.tarball}</file>
+ </files>
+ <message>Tez tarball file not found at
'${tez.dist.tarball}'.</message>
+ </requireFilesExist>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>tez-dist-download</id>
+ <activation>
+ <property>
+ <name>tez.dist.source</name>
+ <value>download</value>
+ </property>
+ </activation>
+ <properties>
+ <tez.dist.prepare.skip>true</tez.dist.prepare.skip>
+
<tez.dist.download.path>${project.build.directory}/tez-download/apache-tez-${tez.version}-bin.tar.gz</tez.dist.download.path>
+
<tez.dist.download.url>https://archive.apache.org/dist/tez/${tez.version}/apache-tez-${tez.version}-bin.tar.gz</tez.dist.download.url>
+ <tez.dist.path>${tez.dist.download.path}</tez.dist.path>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>download-tez-dist</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <skip>${skipTests}</skip>
+ <target>
+ <mkdir dir="${project.build.directory}/tez-download"/>
+ <get src="${tez.dist.download.url}"
dest="${tez.dist.download.path}" usetimestamp="true" verbose="true"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile
b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile
new file mode 100644
index 00000000000..9cf4bcc30dd
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Custom Hadoop image for the Tez-on-YARN Integration tests.
+#
+# The official apache/hadoop:3.4.2 image ships Java 8 for its HDFS/YARN
daemons.
+# Hive is compiled for Java 21, so Tez AM and task containers -- which load
+# hive-exec classes -- must run on Java 21. This image adds a Java 21 JDK at
+# /opt/jdk21 using a multi-stage build from eclipse-temurin:21-jdk; the daemons
+# continue to use the default Java 8.
+FROM --platform=linux/amd64 eclipse-temurin:21-jdk AS jdk21
+FROM apache/hadoop:3.4.2
+
+USER root
+COPY --from=jdk21 /opt/java/openjdk /opt/jdk21
+# Ensure the JDK binaries are executable and verify the copy is functional.
+# Fails at image-build time if the JDK is incompatible with the base image's
+# glibc, surfacing the problem early rather than silently at YARN container
+# launch time (where YARN shows exit code 1 and empty stderr).
+RUN chmod 755 /opt/jdk21/bin/java && /opt/jdk21/bin/java -version
+
+USER hadoop
diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config
b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config
new file mode 100644
index 00000000000..e08ecb55fc7
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.default.name=hdfs://namenode
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode
+
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
+HDFS-SITE.XML_dfs.replication=1
+# Stable DataNode hostname for host JVM (custom_hosts_file + fixed ports).
+HDFS-SITE.XML_dfs.datanode.hostname=datanode
+HDFS-SITE.XML_dfs.client.use.datanode.hostname=true
+
+MAPRED-SITE.XML_mapreduce.framework.name=yarn
+# $$ keeps literal $HADOOP_HOME through Compose env_file interpolation.
+MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$$HADOOP_HOME
+MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$$HADOOP_HOME
+MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$$HADOOP_HOME
+
+YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
+YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
+YARN-SITE.XML_yarn.nodemanager.webapp.address=nodemanager:8042
+# Docker cgroup limits trip the default memory checks on NodeManager.
+YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
+YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
+YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
+# Generous liveness expiry to tolerate slow container startup (e.g. under
heavy load on CI).
+YARN-SITE.XML_yarn.nm.liveness-monitor.expiry-interval-ms=3600000
+YARN-SITE.XML_yarn.am.liveness-monitor.expiry-interval-ms=3600000
+YARN-SITE.XML_yarn.resourcemanager.nodemanagers.heartbeat-interval-ms=1000
+# Absolute YARN app classpath; child containers cannot expand HADOOP_* env
vars.
+YARN-SITE.XML_yarn.application.classpath=/opt/hadoop/etc/hadoop,/opt/hadoop/share/hadoop/common/*,/opt/hadoop/share/hadoop/common/lib/*,/opt/hadoop/share/hadoop/hdfs/*,/opt/hadoop/share/hadoop/hdfs/lib/*,/opt/hadoop/share/hadoop/yarn/*,/opt/hadoop/share/hadoop/yarn/lib/*,/opt/hadoop/share/hadoop/mapreduce/*,/opt/hadoop/share/hadoop/mapreduce/lib/*
+YARN-SITE.XML_yarn.nodemanager.env-whitelist=JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME,HADOOP_HOME,PATH,LANG,TZ
+
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false
diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml
b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml
new file mode 100644
index 00000000000..f7fcc91e617
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# HDFS + YARN cluster for tez-yarn-it. Image built by TezYarnClusterContainer;
config via env_file.
+
+services:
+ namenode:
+ image: hive-it-hadoop:latest
+ hostname: namenode
+ command: ["hdfs", "namenode"]
+ env_file:
+ - config
+ environment:
+ ENSURE_NAMENODE_DIR: /tmp/hadoop-hadoop/dfs/name
+ ports:
+ - "8020:8020" # NameNode RPC
+ - "9870:9870" # NameNode web UI
+ healthcheck:
+ test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/9870'"]
+ interval: 5s
+ timeout: 3s
+ retries: 30
+ start_period: 10s
+
+ datanode:
+ image: hive-it-hadoop:latest
+ hostname: datanode
+ command: ["hdfs", "datanode"]
+ env_file:
+ - config
+ depends_on:
+ namenode:
+ condition: service_healthy
+ ports:
+ - "9866:9866" # DataNode data transfer
+ - "9864:9864" # DataNode web UI
+
+ resourcemanager:
+ image: hive-it-hadoop:latest
+ hostname: resourcemanager
+ command: ["yarn", "resourcemanager"]
+ env_file:
+ - config
+ depends_on:
+ namenode:
+ condition: service_healthy
+ ports:
+ - "8032:8032" # ResourceManager RPC
+ - "8088:8088" # ResourceManager web UI
+ # No healthcheck: RM webapp binds to hostname "resourcemanager", not
localhost.
+
+ nodemanager:
+ image: hive-it-hadoop:latest
+ hostname: nodemanager
+ command: ["yarn", "nodemanager"]
+ env_file:
+ - config
+ depends_on:
+ resourcemanager:
+ condition: service_started
+ ports:
+ - "8042:8042" # NodeManager web UI
+ - "41000-41020:41000-41020" # Tez AM client RPC range
diff --git
a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/ComposeImageSubstitutor.java
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/ComposeImageSubstitutor.java
new file mode 100644
index 00000000000..045bdd6c7d1
--- /dev/null
+++
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/ComposeImageSubstitutor.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.tez.yarn;
+
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.ImageNameSubstitutor;
+
+/**
+ * Replaces Testcontainers' compose helper with a newer CLI for modern daemons.
+ * Registered via testcontainers.properties.
+ * Override with -Dtez.yarn.compose.image=docker:tag.
+ */
+public class ComposeImageSubstitutor extends ImageNameSubstitutor {
+
+ private static final String REPLACEMENT_IMAGE =
+ System.getProperty("tez.yarn.compose.image", "docker:27");
+
+ @Override
+ public DockerImageName apply(DockerImageName original) {
+ String unversioned = original.getUnversionedPart();
+ if ("docker".equals(unversioned) || unversioned.endsWith("/docker")) {
+ return DockerImageName.parse(REPLACEMENT_IMAGE);
+ }
+ return original;
+ }
+
+ @Override
+ protected String getDescription() {
+ return "tez-yarn-it compose image substitutor (docker -> " +
REPLACEMENT_IMAGE + ")";
+ }
+}
diff --git
a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java
new file mode 100644
index 00000000000..17e068df5cd
--- /dev/null
+++
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.tez.yarn;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hive.service.server.HiveServer2;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Keep-alive HDFS + YARN + HiveServer2 cluster for manual Beeline testing.
+ */
+public class StartTezYarnCluster {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(StartTezYarnCluster.class);
+
+ @Test
+ public void testRunCluster() throws Exception {
+ TezYarnClusterContainer cluster = new TezYarnClusterContainer();
+ cluster.start();
+
+ TestTezYarnUtils.setupHdfs(cluster);
+
+ String tezLibUris = cluster.uploadTezLibsToHdfs();
+ LOG.info("Staged Tez libs to HDFS: {}", tezLibUris);
+
+ Path localScratch = Files.createTempDirectory("hive-tez-loc-");
+
+ int hs2Port =
Integer.parseInt(System.getProperty("tez.yarn.cluster.hs2.port", "10000"));
+ HiveConf conf = TestTezYarnUtils.buildHiveConf(tezLibUris, localScratch,
hs2Port);
+
+ HiveServer2 hs2 = new HiveServer2();
+ hs2.init(conf);
+ hs2.start();
+
+ TestTezYarnUtils.waitForJdbc(hs2Port);
+
+ String jdbcUrl = TestTezYarnUtils.jdbcUrl(hs2Port);
+ String logFile = System.getProperty("tez.yarn.it.log.file");
+ String bar =
"======================================================================";
+ System.out.println(bar);
+ System.out.println(" Tez-on-YARN cluster is up. Ctrl-C this JVM to tear
it down.");
+ System.out.println(bar);
+ System.out.printf(" %-22s %s%n", "HDFS URI", cluster.getHdfsUri());
+ System.out.printf(" %-22s %s%n", "NameNode web UI",
cluster.getNameNodeWebUrl());
+ System.out.printf(" %-22s %s%n", "ResourceManager RPC",
cluster.getResourceManagerAddress());
+ System.out.printf(" %-22s %s%n", "ResourceManager UI",
cluster.getResourceManagerWebUrl());
+ System.out.printf(" %-22s %s%n", "NodeManager UI",
cluster.getNodeManagerWebUrl());
+ System.out.printf(" %-22s %s%n", "NodeManager AM RPC range",
cluster.getNodeManagerAmRpcRange());
+ System.out.printf(" %-22s %s%n", "Tez libs on HDFS", tezLibUris);
+ System.out.printf(" %-22s %s%n", "HDFS warehouse",
TestTezYarnUtils.HDFS_WAREHOUSE);
+ System.out.printf(" %-22s %s%n", "HiveServer2 JDBC", jdbcUrl);
+ System.out.printf(" %-22s %s%n", "Log file (slf4j)",
+ logFile != null ? new File(logFile).getAbsolutePath() : "(configured
by log4j2)");
+ System.out.println(bar);
+ System.out.println(" Beeline:");
+ System.out.println(" beeline -u '" + jdbcUrl + "' -n hive");
+ System.out.println(bar);
+ System.out.flush();
+
+ Thread.currentThread().join();
+ }
+}
diff --git
a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java
new file mode 100644
index 00000000000..3c6fa13e719
--- /dev/null
+++
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.tez.yarn;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hive.service.server.HiveServer2;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.Container;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+
+public class TestTezYarnLocalization {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TestTezYarnLocalization.class);
+
+ private static TezYarnClusterContainer cluster;
+ private static HiveServer2 hs2;
+ private static int hs2Port;
+
+ @BeforeClass
+ public static void startAll() throws Exception {
+ cluster = new TezYarnClusterContainer();
+ cluster.start();
+
+ TestTezYarnUtils.setupHdfs(cluster);
+
+ String tezLibUris = cluster.uploadTezLibsToHdfs();
+ LOG.info("Staged Tez libs to HDFS: {}", tezLibUris);
+
+ Path localScratch = Files.createTempDirectory("hive-tez-loc-");
+ hs2Port = TestTezYarnUtils.findFreePort();
+ HiveConf conf = TestTezYarnUtils.buildHiveConf(tezLibUris, localScratch,
hs2Port);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SESSION_SILENT, true);
+ conf.setVar(HiveConf.ConfVars.PRE_EXEC_HOOKS, "");
+ conf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, "");
+
+ hs2 = new HiveServer2();
+ hs2.init(conf);
+ hs2.start();
+
+ TestTezYarnUtils.waitForJdbc(hs2Port);
+ LOG.info("HiveServer2 is ready on port {}", hs2Port);
+ }
+
+ @AfterClass
+ public static void stopAll() {
+ dumpNodeManagerDiagnostics();
+ if (hs2 != null) {
+ hs2.stop();
+ hs2 = null;
+ }
+ if (cluster != null) {
+ cluster.stop();
+ cluster = null;
+ }
+ }
+
+ @Test
+ public void testQuerySucceedsWithAppJar() throws Exception {
+ String url = TestTezYarnUtils.jdbcUrl(hs2Port);
+ try (Connection conn = DriverManager.getConnection(url, "hive", "")) {
+ try (Statement stmt = conn.createStatement()) {
+
+ stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED
AS ORC");
+ stmt.execute("INSERT INTO tez_loc_test VALUES (42)");
+
+ try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) {
+ Assert.assertTrue("Result set must contain at least one row",
rs.next());
+ int count = rs.getInt(1);
+ Assert.assertEquals(
+ "INSERT VALUES should return the inserted row value
(hive-exec.jar was localized)",
+ 42, count);
+ LOG.info("Tez query succeeded: inserted row value = {}", count);
+ }
+ }
+ }
+
+ verifyTezYarnAppExists();
+ verifyHiveExecJarOnHdfs();
+ verifyHiveExecJarLocalizedInNm();
+ }
+
+ private static void verifyHiveExecJarOnHdfs() throws IOException {
+ List<String> matchingJars = new ArrayList<>();
+ org.apache.hadoop.fs.Path searchRoot =
+ new org.apache.hadoop.fs.Path(TestTezYarnUtils.HDFS_ROOT +
"/user-install");
+ try (FileSystem fs = cluster.newFileSystem()) {
+ RemoteIterator<LocatedFileStatus> files = fs.listFiles(searchRoot, true);
+ while (files.hasNext()) {
+ LocatedFileStatus file = files.next();
+ String fileName = file.getPath().getName();
+ if (fileName.startsWith("hive-exec-") && fileName.endsWith(".jar")) {
+ matchingJars.add(file.getPath().toString());
+ }
+ }
+ }
+ LOG.info("HDFS hive-exec.jar search in {}/user-install: {}",
+ TestTezYarnUtils.HDFS_ROOT,
+ matchingJars.isEmpty() ? "(none found)" : String.join("\n",
matchingJars));
+ Assert.assertFalse(
+ "hive-exec.jar was not staged to HDFS under " +
TestTezYarnUtils.HDFS_ROOT + "/user-install — "
+ + "TezSessionState.buildCommonLocalResources()
localization step 1 appears to have failed.",
+ matchingJars.isEmpty());
+ }
+
+ private static void verifyHiveExecJarLocalizedInNm() throws IOException,
InterruptedException {
+ Container.ExecResult r = cluster.nodeManagerContainer().execInContainer(
+ "bash", "-c", "find /tmp -name 'hive-exec-*.jar' 2>/dev/null |
head -5");
+ LOG.info("NodeManager hive-exec.jar localization check: {}",
+ r.getStdout().trim().isEmpty() ? "(none found)" :
r.getStdout().trim());
+ Assert.assertFalse(
+ "hive-exec.jar was not found in the NodeManager container's local
filesystem after Tez query — "
+ + "YARN localization step 2 appears to have failed.",
+ r.getStdout().trim().isEmpty());
+ }
+
+ /**
+ * Logs Tez AM and NodeManager diagnostics at teardown.
+ */
+ private static void dumpNodeManagerDiagnostics() {
+ if (cluster == null) {
+ return;
+ }
+ LOG.info("########## BEGIN NodeManager diagnostics ##########");
+ try {
+ dumpNmCommand("launch_container.sh (AM launch command + classpath)",
+ "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 "
+ + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}'
2>/dev/null || true");
+
+ dumpNmCommand("container syslog (Tez AM log4j output)",
+ "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null |
head -10 "
+ + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}'
2>/dev/null || true");
+
+ dumpNmCommand("container stdout + stderr + prelaunch.err",
+ "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name
'stderr' -o -name 'prelaunch.err' \\) "
+ + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"---
{} ---\"; cat {}' 2>/dev/null || true");
+ } catch (Exception e) {
+ LOG.warn("Could not dump NodeManager diagnostics", e);
+ }
+ LOG.info("########## END NodeManager diagnostics ##########");
+ }
+
+ private static void dumpNmCommand(String label, String bashCommand) {
+ try {
+ Container.ExecResult r =
+ cluster.nodeManagerContainer().execInContainer("bash", "-c",
bashCommand);
+ String out = r.getStdout();
+ LOG.info("===== NM: {} =====\n{}", label, out.isEmpty() ? "(no output
found)" : out);
+ } catch (Exception e) {
+ LOG.warn("===== NM: {} (dump failed) =====", label, e);
+ }
+ }
+
+ private static void verifyTezYarnAppExists() throws Exception {
+ List<ApplicationReport> applications;
+ try (YarnClient yarnClient = cluster.newYarnClient()) {
+ applications = yarnClient.getApplications(
+ Collections.singleton("TEZ"),
EnumSet.allOf(YarnApplicationState.class));
+ }
+ if (applications.isEmpty()) {
+ LOG.info("YARN application list (TEZ, ALL states): (none found)");
+ } else {
+ LOG.info("YARN application list (TEZ, ALL states):");
+ for (ApplicationReport app : applications) {
+ LOG.info("Found Tez YARN application: {} ({})",
+ app.getApplicationId(), app.getYarnApplicationState());
+ }
+ }
+ Assert.assertFalse(
+ "At least one Tez YARN application must be visible in the
ResourceManager after running a Tez query",
+ applications.isEmpty());
+ }
+}
diff --git
a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnUtils.java
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnUtils.java
new file mode 100644
index 00000000000..8cddf4a5e13
--- /dev/null
+++
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnUtils.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.tez.yarn;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+
+import java.net.ServerSocket;
+import java.net.URL;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.time.Duration;
+
+/**
+ * Shared HDFS paths, fixture setup, Hive configuration and JDBC helpers for
the tez-yarn-it tests.
+ */
+public final class TestTezYarnUtils {
+
+ static final String HDFS_BASE = "hdfs://namenode:8020";
+ static final String HDFS_ROOT = "/tmp/hive-tez-loc";
+ static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse";
+ static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch";
+
+ private TestTezYarnUtils() {
+ }
+
+ /**
+ * Creates the HDFS fixture directories as the HDFS superuser through the
FileSystem API.
+ */
+ static void setupHdfs(TezYarnClusterContainer cluster) throws Exception {
+ String[] dirs = {
+ "/tmp",
+ HDFS_ROOT,
+ HDFS_ROOT + "/warehouse",
+ HDFS_ROOT + "/scratch",
+ HDFS_ROOT + "/user-install"
+ };
+ try (FileSystem fs = cluster.newSuperuserFileSystem()) {
+ for (String dir : dirs) {
+ fs.mkdirs(new org.apache.hadoop.fs.Path(dir));
+ }
+ for (String dir : dirs) {
+ fs.setPermission(new org.apache.hadoop.fs.Path(dir), new
FsPermission((short) 0777));
+ }
+ }
+ }
+
+ static HiveConf buildHiveConf(String tezLibUris, Path localScratch, int
hs2Port) throws Exception {
+ HiveConf conf = new HiveConf();
+
+ URL hiveSite =
TestTezYarnUtils.class.getClassLoader().getResource("hive-site-yarn-it.xml");
+ URL yarnSite =
TestTezYarnUtils.class.getClassLoader().getResource("yarn-site.xml");
+ if (hiveSite != null) {
+ conf.addResource(hiveSite);
+ }
+ if (yarnSite != null) {
+ conf.addResource(yarnSite);
+ }
+
+ // Dynamic properties: values derived at runtime from container ports or
temp directories.
+ conf.set("fs.defaultFS", HDFS_BASE);
+ conf.set("hive.metastore.warehouse.dir", HDFS_WAREHOUSE);
+ conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, HDFS_SCRATCH);
+ conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname,
localScratch.toAbsolutePath().toString());
+ conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, HDFS_ROOT +
"/user-install");
+ conf.set("javax.jdo.option.ConnectionURL",
+ "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath()
+ ";create=true");
+ conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false);
+
+ conf.set("tez.lib.uris", tezLibUris);
+
+ conf.set("tez.am.client.am.port-range",
+ TezYarnClusterContainer.AM_CLIENT_PORT_START + "-" +
TezYarnClusterContainer.AM_CLIENT_PORT_END);
+ String containerEnv = "JAVA_HOME=" +
TezYarnClusterContainer.CONTAINER_JAVA_HOME
+ + ",HADOOP_HOME=/opt/hadoop"
+ + ",HADOOP_MAPRED_HOME=/opt/hadoop";
+ conf.set("tez.am.launch.env", containerEnv);
+ conf.set("tez.task.launch.env", containerEnv);
+
+ conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port);
+ conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort());
+
+ return conf;
+ }
+
+ static void waitForJdbc(int port) {
+ String url = jdbcUrl(port);
+ try {
+ Awaitility.await()
+ .atMost(Duration.ofSeconds(120))
+ .pollInterval(Duration.ofSeconds(2))
+ .ignoreExceptions()
+ .until(() -> {
+ try (Connection c = DriverManager.getConnection(url, "hive",
"")) {
+ return true;
+ }
+ });
+ } catch (ConditionTimeoutException e) {
+ throw new IllegalStateException(
+ "HiveServer2 JDBC endpoint not reachable on port " + port + "
after 120s", e);
+ }
+ }
+
+ static String jdbcUrl(int port) {
+ return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl";
+ }
+
+ static int findFreePort() throws Exception {
+ try (ServerSocket s = new ServerSocket(0)) {
+ s.setReuseAddress(true);
+ return s.getLocalPort();
+ }
+ }
+}
diff --git
a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java
new file mode 100644
index 00000000000..76b557d3c4a
--- /dev/null
+++
b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.tez.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.SafeMode;
+import org.apache.hadoop.fs.SafeModeAction;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionTimeoutException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.ComposeContainer;
+import org.testcontainers.containers.ContainerState;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+
+/**
+ * Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers
ComposeContainer.
+ * Fixed published ports plus custom_hosts_file let the host JVM reach
namenode/resourcemanager.
+ * HDFS and YARN are driven through the Hadoop client APIs rather than
in-container CLIs.
+ */
+public class TezYarnClusterContainer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TezYarnClusterContainer.class);
+
+ public static final String CONTAINER_JAVA_HOME = "/opt/jdk21";
+
+ private final String hadoopImage;
+
+ private static final int NN_RPC_PORT = 8020;
+ private static final int NN_HTTP_PORT = 9870;
+ private static final int RM_RPC_PORT = 8032;
+ private static final int RM_HTTP_PORT = 8088;
+ private static final int NM_HTTP_PORT = 8042;
+ private static final String HDFS_SUPERUSER = "hadoop";
+ // Tez AM client RPC port range, published by the NM container so the host
JVM can reach the AM.
+ public static final int AM_CLIENT_PORT_START = 41000;
+ public static final int AM_CLIENT_PORT_END = 41020;
+
+ // Compose V2 names service containers "<service>-1"; used by
getContainerByServiceName().
+ private static final int SERVICE_INSTANCE = 1;
+
+ private static final Duration CLUSTER_READY_TIMEOUT = Duration.ofMinutes(2);
+ private static final Duration CLUSTER_READY_POLL_INTERVAL =
Duration.ofSeconds(3);
+ private static final long RM_CONNECT_MAX_WAIT_MS = 2000;
+ private static final long RM_CONNECT_RETRY_INTERVAL_MS = 500;
+
+ private final ComposeContainer compose;
+
+ public TezYarnClusterContainer() {
+ hadoopImage = buildHadoopImage();
+ String basedir = System.getProperty("basedir", ".");
+ File composeFile = Paths.get(basedir,
"src/test/docker/hadoop-yarn/docker-compose.yml").toFile();
+ LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile,
hadoopImage);
+ compose = new ComposeContainer(composeFile)
+ .withLocalCompose(false);
+ }
+
+ public void start() throws Exception {
+ try {
+ compose.start();
+ // Avoid flakiness: wait until HDFS has left safemode before running
HDFS operations.
+ waitForSafemodeExit();
+ waitForNodeManagerRegistration();
+ } catch (Exception e) {
+ try {
+ stop();
+ } catch (Exception stopException) {
+ LOG.info("Failed to stop TezYarnClusterContainer after startup
failure", stopException);
+ }
+ throw e;
+ }
+ }
+
+ public void stop() {
+ compose.stop();
+ }
+
+ public String getHdfsUri() {
+ return "hdfs://namenode:" + NN_RPC_PORT;
+ }
+
+ public String getResourceManagerAddress() {
+ return "resourcemanager:" + RM_RPC_PORT;
+ }
+
+ public String getResourceManagerWebAppAddress() {
+ return "resourcemanager:" + RM_HTTP_PORT;
+ }
+
+ // Web UIs are opened from the host, so they use the published localhost
ports;
+ // RPC and AM endpoints use the container hostnames that resolve via
custom_hosts_file.
+ public String getNameNodeWebUrl() {
+ return "http://localhost:" + NN_HTTP_PORT;
+ }
+
+ public String getResourceManagerWebUrl() {
+ return "http://localhost:" + RM_HTTP_PORT;
+ }
+
+ public String getNodeManagerWebUrl() {
+ return "http://localhost:" + NM_HTTP_PORT;
+ }
+
+ public String getNodeManagerAmRpcRange() {
+ return "nodemanager:" + AM_CLIENT_PORT_START + "-" + AM_CLIENT_PORT_END;
+ }
+
+ public String uploadJarToHdfs(Path localJarPath) throws IOException {
+ String fileName = localJarPath.getFileName().toString();
+ String hdfsDir = "/tmp/hive-tez-yarn-jars";
+ String hdfsPath = hdfsDir + "/" + fileName;
+
+ copyToHdfs(localJarPath, hdfsDir, hdfsPath);
+
+ return hdfsPath;
+ }
+
+ public String uploadTezLibsToHdfs() throws IOException {
+ String tezDistPath = System.getProperty("tez.dist.path");
+ if (tezDistPath == null || tezDistPath.isEmpty()) {
+ throw new IllegalStateException(
+ "System property 'tez.dist.path' is not set. "
+ + "It must point to a Tez distribution tarball selected by the Maven
test run "
+ + "(default staged tarball, local tarball, or downloaded tarball). "
+ + "If running tests in isolation, invoke Maven with -Ptez-yarn so
surefire sets it.");
+ }
+
+ Path tarball = Paths.get(tezDistPath);
+ if (!Files.isRegularFile(tarball)) {
+ throw new IllegalStateException(
+ "Tez distribution tarball not found at: " + tarball.toAbsolutePath()
+ + ". Build the staged tarball with "
+ + "'mvn test-compile -Pitests,tez-yarn -pl itests/tez-yarn-it' "
+ + "or pass a valid local/downloaded archive via tez.dist.source
options.");
+ }
+
+ String fileName = tarball.getFileName().toString();
+ String hdfsDir = "/tmp/hive-tez-yarn";
+ String hdfsPath = hdfsDir + "/" + fileName;
+
+ LOG.info("Uploading Tez distribution tarball ({}) to HDFS path {}",
+ tarball.toAbsolutePath(), hdfsPath);
+
+ copyToHdfs(tarball, hdfsDir, hdfsPath);
+
+ // "#tez" is the YARN container link name for the localized archive.
+ return "hdfs://namenode:" + NN_RPC_PORT + hdfsPath + "#tez";
+ }
+
+ private void copyToHdfs(Path localPath, String hdfsDir, String hdfsPath)
throws IOException {
+ // java.nio.file.Path is used for local files here, so the HDFS paths are
qualified explicitly.
+ org.apache.hadoop.fs.Path src = new
org.apache.hadoop.fs.Path(localPath.toUri());
+ org.apache.hadoop.fs.Path dest = new org.apache.hadoop.fs.Path(hdfsPath);
+
+ try (FileSystem fs = newFileSystem()) {
+ fs.mkdirs(new org.apache.hadoop.fs.Path(hdfsDir));
+ // delSrc=false, overwrite=true mirrors the semantics of "hdfs dfs -put
-f".
+ fs.copyFromLocalFile(false, true, src, dest);
+ }
+ }
+
+ ContainerState namenodeContainer() {
+ return serviceContainer("namenode");
+ }
+
+ ContainerState resourceManagerContainer() {
+ return serviceContainer("resourcemanager");
+ }
+
+ ContainerState nodeManagerContainer() {
+ return serviceContainer("nodemanager");
+ }
+
+ private ContainerState serviceContainer(String serviceName) {
+ ContainerState cs = compose.getContainerByServiceName(serviceName + "-" +
SERVICE_INSTANCE)
+ .orElseThrow(() -> new IllegalStateException(
+ "Compose service container not found: " + serviceName));
+
+ if (!cs.isRunning()){
+ throw new IllegalStateException(
+ "Compose service container not running: " + serviceName);
+ }
+ return cs;
+ }
+
+ private void waitForSafemodeExit() {
+ try {
+ Awaitility.await()
+ .pollDelay(Duration.ZERO)
+ .pollInterval(CLUSTER_READY_POLL_INTERVAL)
+ .atMost(CLUSTER_READY_TIMEOUT)
+ // The NameNode healthcheck only covers the web port, so RPC may
still be refused here.
+ .ignoreExceptions()
+ .until(() -> {
+ try (FileSystem fs = newFileSystem()) {
+ return !((SafeMode) fs).setSafeMode(SafeModeAction.GET);
+ }
+ });
+ } catch (ConditionTimeoutException e) {
+ throw new IllegalStateException(
+ "HDFS did not leave safemode within " +
CLUSTER_READY_TIMEOUT.toMinutes() + " minutes", e);
+ }
+ }
+
+ private void waitForNodeManagerRegistration() {
+ try (YarnClient yarnClient = YarnClient.createYarnClient()) {
+ yarnClient.init(buildHadoopConfiguration());
+ yarnClient.start();
+
+ Awaitility.await()
+ .pollDelay(Duration.ZERO)
+ .pollInterval(CLUSTER_READY_POLL_INTERVAL)
+ .atMost(CLUSTER_READY_TIMEOUT)
+ .ignoreExceptions()
+ .until(() ->
!yarnClient.getNodeReports(NodeState.RUNNING).isEmpty());
+ } catch (ConditionTimeoutException e) {
+ throw new IllegalStateException("NodeManager did not register with
ResourceManager within "
+ + CLUSTER_READY_TIMEOUT.toMinutes() + " minutes", e);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to query the ResourceManager for
registered NodeManagers", e);
+ }
+ }
+
+ /**
+ * Configuration for host JVM access to the containerised HDFS and YARN
services.
+ */
+ private Configuration buildHadoopConfiguration() {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, getHdfsUri());
+ // Reach DataNodes by their container hostname (custom_hosts_file), not
their container IP.
+ conf.setBoolean("dfs.client.use.datanode.hostname", true);
+ conf.setInt("dfs.replication", 1);
+ conf.set(YarnConfiguration.RM_ADDRESS, getResourceManagerAddress());
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
RM_CONNECT_MAX_WAIT_MS);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
RM_CONNECT_RETRY_INTERVAL_MS);
+ return conf;
+ }
+
+ /**
+ * Uncached FileSystem so callers can close it without affecting the
HiveServer2 instance.
+ */
+ FileSystem newFileSystem() throws IOException {
+ return FileSystem.newInstance(URI.create(getHdfsUri()),
buildHadoopConfiguration());
+ }
+
+ FileSystem newSuperuserFileSystem() throws IOException {
+ try {
+ return FileSystem.newInstance(URI.create(getHdfsUri()),
buildHadoopConfiguration(), HDFS_SUPERUSER);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while creating HDFS superuser
FileSystem client", e);
+ }
+ }
+
+ YarnClient newYarnClient() {
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(buildHadoopConfiguration());
+ yarnClient.start();
+ return yarnClient;
+ }
+
+ private static String buildHadoopImage() {
+ String basedir = System.getProperty("basedir", ".");
+ Path dockerfile = Paths.get(basedir,
"src/test/docker/hadoop-yarn/Dockerfile");
+ String imageName = "hive-it-hadoop-" + java.util.UUID.randomUUID();
+ return new ImageFromDockerfile(imageName, false)
+ .withDockerfile(dockerfile)
+ .get();
+ }
+}
diff --git a/itests/tez-yarn-it/src/test/resources/custom_hosts_file
b/itests/tez-yarn-it/src/test/resources/custom_hosts_file
new file mode 100644
index 00000000000..867182b1cdd
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/resources/custom_hosts_file
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Maps Docker service hostnames to 127.0.0.1 for the host JVM
(-Djdk.net.hosts.file).
+
+127.0.0.1 localhost
+127.0.0.1 namenode
+127.0.0.1 resourcemanager
+127.0.0.1 datanode
+127.0.0.1 nodemanager
diff --git a/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml
b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml
new file mode 100644
index 00000000000..0b52fe9596b
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml
@@ -0,0 +1,126 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <property>
+ <name>hive.in.test</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.execution.engine</name>
+ <value>tez</value>
+ </property>
+
+ <property>
+ <name>hive.server2.enable.doAs</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.support.concurrency</name>
+ <value>false</value>
+ </property>
+
+ <!-- Disable query history so HMS doesn't need to create additional Iceberg
tables. -->
+ <property>
+ <name>hive.query.history.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>datanucleus.connectionPool.maxPoolSize</name>
+ <value>4</value>
+ </property>
+
+ <!-- HDFS client must use the container hostname so the host JVM resolves
DataNode addresses
+ through the custom hosts file rather than via the raw IP returned by
HDFS. -->
+ <property>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hive.stats.autogather</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.stats.column.autogather</name>
+ <value>false</value>
+ </property>
+
+ <!-- YARN ResourceManager coordinates — "resourcemanager" is the Docker
network alias. -->
+ <property>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>resourcemanager</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>resourcemanager:8032</value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.address</name>
+ <value>resourcemanager:8088</value>
+ </property>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ <property>
+ <name>tez.use.cluster.hadoop-libs</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>tez.lib.uris.classpath</name>
+ <value>$PWD/tez/*</value>
+ </property>
+ <property>
+ <name>hive.server2.tez.initialize.default.sessions</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ <value>0</value>
+ </property>
+
+ <!-- HiveServer2 transport settings. -->
+ <property>
+ <name>hive.server2.thrift.bind.host</name>
+ <value>localhost</value>
+ </property>
+ <property>
+ <name>hive.server2.transport.mode</name>
+ <value>binary</value>
+ </property>
+ <property>
+ <name>hive.server2.authentication</name>
+ <value>NOSASL</value>
+ </property>
+
+</configuration>
diff --git a/itests/tez-yarn-it/src/test/resources/log4j2-test.properties
b/itests/tez-yarn-it/src/test/resources/log4j2-test.properties
new file mode 100644
index 00000000000..d4c1b7dc85f
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/resources/log4j2-test.properties
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Routes all tez-yarn-it test logging to a file so StartTezYarnCluster's
System.out
+# banner is the only thing on the console. The log file path is supplied by
the surefire
+# argLine (-Dtez.yarn.it.log.file); when unset (e.g. running from an IDE) it
falls back
+# to target/tez-yarn-it.log.
+
+status = warn
+name = TezYarnItTestLogging
+
+appender.file.type = File
+appender.file.name = FILE
+appender.file.fileName = ${sys:tez.yarn.it.log.file:-target/tez-yarn-it.log}
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = FILE
diff --git a/itests/tez-yarn-it/src/test/resources/testcontainers.properties
b/itests/tez-yarn-it/src/test/resources/testcontainers.properties
new file mode 100644
index 00000000000..2a45044cc49
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/resources/testcontainers.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Bump Testcontainers' docker CLI helper for modern Docker API versions (see
ComposeImageSubstitutor).
+image.substitutor=org.apache.hive.tez.yarn.ComposeImageSubstitutor
diff --git a/itests/tez-yarn-it/src/test/resources/yarn-site.xml
b/itests/tez-yarn-it/src/test/resources/yarn-site.xml
new file mode 100644
index 00000000000..fda5ad066fa
--- /dev/null
+++ b/itests/tez-yarn-it/src/test/resources/yarn-site.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <!-- Disable memory checks; Docker cgroup limits trip the default
thresholds. -->
+ <property>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+</configuration>