abstractdog commented on code in PR #6640: URL: https://github.com/apache/hive/pull/6640#discussion_r4311997585
########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.SafeMode; +import org.apache.hadoop.fs.SafeModeAction; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** + * Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. + * HDFS and YARN are driven through the Hadoop client APIs rather than in-container CLIs. + */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + public static final String CONTAINER_JAVA_HOME = "/opt/jdk21"; + + private final String hadoop_image; + + private static final int NN_RPC_PORT = 8020; + private static final int NN_HTTP_PORT = 9870; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + private static final int NM_HTTP_PORT = 8042; + // Tez AM client RPC port range, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT_START = 41000; + public static final int AM_CLIENT_PORT_END = 41020; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private static final Duration CLUSTER_READY_TIMEOUT = Duration.ofMinutes(2); + private static final Duration CLUSTER_READY_POLL_INTERVAL = Duration.ofSeconds(3); + private static final long RM_CONNECT_MAX_WAIT_MS = 2000; + private static final long RM_CONNECT_RETRY_INTERVAL_MS = 500; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + hadoop_image = buildHadoopImage(); + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, hadoop_image); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() throws Exception { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + waitForSafemodeExit(); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception stopException) { + LOG.info("Failed to stop TezYarnClusterContainer after startup failure", stopException); + } + throw e; + } + } + + public void stop() { + compose.stop(); + } + + private void verifyJava21InNodeManager() throws Exception { + Container.ExecResult r = nodeManagerContainer().execInContainer( + CONTAINER_JAVA_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_HOME, r.getStderr().trim()); + } else { + throw new IllegalStateException( + "Java check failed in NodeManager (exit " + r.getExitCode() + + "). stderr: " + r.getStderr()); + } + } Review Comment: remove "21" from here too, `verifyJavaInNodeManager` and `Java is functional in NodeManager` also consider if this is needed at all: `waitForNodeManagerRegistration` asserts a working node manager, does adding a java check brings any value here? ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.SafeMode; +import org.apache.hadoop.fs.SafeModeAction; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** + * Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. + * HDFS and YARN are driven through the Hadoop client APIs rather than in-container CLIs. + */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + public static final String CONTAINER_JAVA_HOME = "/opt/jdk21"; + + private final String hadoop_image; Review Comment: `hadoopImage` ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnUtils.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Duration; + +/** + * Shared HDFS paths, fixture setup, Hive configuration and JDBC helpers for the tez-yarn-it tests. + */ +public final class TestTezYarnUtils { + + static final String HDFS_BASE = "hdfs://namenode:8020"; + static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse"; + static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch"; + + private TestTezYarnUtils() { + } + + /** + * Creates the HDFS fixture directories from inside the NameNode container, which runs as the Hadoop superuser. + * A host-side FileSystem client lacks write permission on "/". + */ + static void setupHdfs(ContainerState nn) throws Exception { + String[] dirs = { + "/tmp", + HDFS_ROOT + "/warehouse", + HDFS_ROOT + "/scratch", + HDFS_ROOT + "/user-install" + }; + for (String dir : dirs) { + Container.ExecResult r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", dir); + if (r.getExitCode() != 0) { + throw new IllegalStateException("hdfs dfs -mkdir -p " + dir + " failed:\n" + r.getStderr()); + } + } + for (String dir : new String[]{"/tmp", HDFS_ROOT}) { + Container.ExecResult r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", dir); Review Comment: FileSystem API? ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java: ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + TestTezYarnUtils.setupHdfs(cluster.namenodeContainer()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + hs2Port = TestTezYarnUtils.findFreePort(); + HiveConf conf = TestTezYarnUtils.buildHiveConf(tezLibUris, localScratch, hs2Port); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SESSION_SILENT, true); + conf.setVar(HiveConf.ConfVars.PRE_EXEC_HOOKS, ""); + conf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, ""); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + TestTezYarnUtils.waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = TestTezYarnUtils.jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("INSERT INTO tez_loc_test VALUES (42)"); + + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + int count = rs.getInt(1); + Assert.assertEquals( + "INSERT VALUES should return the inserted row value (hive-exec.jar was localized)", + 42, count); + LOG.info("Tez query succeeded: inserted row value = {}", count); + } + } + } + + verifyTezYarnAppExists(); + verifyHiveExecJarOnHdfs(); + verifyHiveExecJarLocalizedInNm(); + } + + private static void verifyHiveExecJarOnHdfs() throws IOException, InterruptedException { + Container.ExecResult r = cluster.namenodeContainer().execInContainer( + "hdfs", "dfs", "-find", TestTezYarnUtils.HDFS_ROOT + "/user-install", "-name", "hive-exec-*.jar"); + LOG.info("HDFS hive-exec.jar search in {}/user-install: {}", + TestTezYarnUtils.HDFS_ROOT, r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not staged to HDFS under " + TestTezYarnUtils.HDFS_ROOT + "/user-install — " + + "TezSessionState.buildCommonLocalResources() localization step 1 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + private static void verifyHiveExecJarLocalizedInNm() throws IOException, InterruptedException { + Container.ExecResult r = cluster.nodeManagerContainer().execInContainer( + "bash", "-c", "find /tmp -name 'hive-exec-*.jar' 2>/dev/null | head -5"); + LOG.info("NodeManager hive-exec.jar localization check: {}", + r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not found in the NodeManager container's local filesystem after Tez query — " + + "YARN localization step 2 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + /** + * Logs Tez AM and NodeManager diagnostics at teardown. + */ + private static void dumpNodeManagerDiagnostics() { + if (cluster == null) { + return; + } + LOG.info("########## BEGIN NodeManager diagnostics ##########"); + try { + dumpNmCommand("launch_container.sh (AM launch command + classpath)", + "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container syslog (Tez AM log4j output)", + "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container stdout + stderr + prelaunch.err", + "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name 'stderr' -o -name 'prelaunch.err' \\) " + + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + } catch (Exception e) { + LOG.warn("Could not dump NodeManager diagnostics", e); + } + LOG.info("########## END NodeManager diagnostics ##########"); + } + + private static void dumpNmCommand(String label, String bashCommand) { + try { + Container.ExecResult r = + cluster.nodeManagerContainer().execInContainer("bash", "-c", bashCommand); + String out = r.getStdout(); + LOG.info("===== NM: {} =====\n{}", label, out.isEmpty() ? "(no output found)" : out); + } catch (Exception e) { + LOG.warn("===== NM: {} (dump failed) =====", label, e); + } + } + + private static void verifyTezYarnAppExists() throws Exception { + ContainerState rm = cluster.resourceManagerContainer(); + Container.ExecResult result = rm.execInContainer( + "yarn", "application", "-list", "-appTypes", "TEZ", "-appStates", "ALL"); Review Comment: could this be handled via `yarnClient.getApplications`? ########## itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml: ########## @@ -0,0 +1,130 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <property> + <name>hive.in.test</name> + <value>true</value> + </property> + + <property> + <name>hive.execution.engine</name> + <value>tez</value> + </property> + + <property> + <name>hive.server2.enable.doAs</name> + <value>false</value> + </property> + + <property> + <name>hive.support.concurrency</name> + <value>false</value> + </property> + + <!-- Disable query history so HMS doesn't need to create additional Iceberg tables. --> + <property> + <name>hive.query.history.enabled</name> + <value>false</value> + </property> + + <property> + <name>datanucleus.schema.autoCreateAll</name> + <value>true</value> + </property> + + <property> + <name>hive.metastore.schema.verification</name> + <value>false</value> + </property> + + <property> + <name>datanucleus.connectionPool.maxPoolSize</name> + <value>4</value> + </property> + + <!-- HDFS client must use the container hostname so the host JVM resolves DataNode addresses + through the custom hosts file rather than via the raw IP returned by HDFS. --> + <property> + <name>dfs.client.use.datanode.hostname</name> + <value>true</value> + </property> + <property> + <name>hive.stats.autogather</name> + <value>false</value> + </property> + <property> + <name>hive.stats.column.autogather</name> + <value>false</value> + </property> + + <!-- YARN ResourceManager coordinates — "resourcemanager" is the Docker network alias. --> + <property> + <name>yarn.resourcemanager.hostname</name> + <value>resourcemanager</value> + </property> + <property> + <name>yarn.resourcemanager.address</name> + <value>resourcemanager:8032</value> + </property> + <property> + <name>yarn.resourcemanager.webapp.address</name> + <value>resourcemanager:8088</value> + </property> + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + <property> + <name>tez.local.mode</name> + <value>false</value> + </property> Review Comment: this is false by default, is it needed? ########## itests/pom.xml: ########## @@ -31,6 +31,7 @@ <hive.path.to.root>..</hive.path.to.root> <wiremock-jre8.version>2.32.0</wiremock-jre8.version> <assertj.version>3.27.3</assertj.version> + <awaitility.version>4.2.1</awaitility.version> Review Comment: this is already defined in the root pom, should it be redefined here? ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnUtils.java: ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.time.Duration; + +/** + * Shared HDFS paths, fixture setup, Hive configuration and JDBC helpers for the tez-yarn-it tests. + */ +public final class TestTezYarnUtils { + + static final String HDFS_BASE = "hdfs://namenode:8020"; + static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse"; + static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch"; + + private TestTezYarnUtils() { + } + + /** + * Creates the HDFS fixture directories from inside the NameNode container, which runs as the Hadoop superuser. + * A host-side FileSystem client lacks write permission on "/". + */ + static void setupHdfs(ContainerState nn) throws Exception { + String[] dirs = { + "/tmp", + HDFS_ROOT + "/warehouse", + HDFS_ROOT + "/scratch", + HDFS_ROOT + "/user-install" + }; + for (String dir : dirs) { + Container.ExecResult r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", dir); Review Comment: FileSystem API? ########## itests/tez-yarn-it/README.md: ########## @@ -0,0 +1,119 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Tez-on-YARN localization integration test + +An opt-in integration test module that runs Hive-on-Tez against a real Docker-containerized +HDFS + YARN cluster to verify that `hive-exec.jar` is correctly localized for Tez AM and +task containers — a scenario that in-process mini-clusters (MiniTezCluster, MiniDFSCluster) +do not exercise. + +## Prerequisites + +* Java 21 +* Maven 3.6.3 or later +* Docker Desktop (or Docker Engine) with at least **4 GB** of memory assigned + +Docker Desktop 25+ exposes a newer Docker API than Testcontainers' default helper image +expects. This module ships `ComposeImageSubstitutor` and `testcontainers.properties` to +substitute a compatible helper image automatically; no extra configuration is required on +supported setups. + +## First-time setup + +Build the full Hive distribution once to populate `$HIVE_HOME/lib/` and install all +artifacts to `~/.m2`: + +```bash +mvn clean install -DskipTests -Pitests,dist +``` + +## Running the automated tests + +This module is **opt-in**: tests are skipped by default. You must pass `-Pitests,tez-yarn` +(or `-Drun.tez.yarn.tests=true`) for Surefire to execute anything. Running +`mvn test -pl itests/tez-yarn-it` without those flags reports zero tests by design. + +```bash +mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it +``` + +If you have not run a full install recently, add `-am` to build required upstream modules: + +```bash +mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -am +``` + +To run a single test: + +```bash +mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \ + -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar +``` + +## Starting a keep-alive cluster for manual testing + +`StartTezYarnCluster` starts the full HDFS + YARN + HiveServer2 stack and blocks until +`Ctrl+C`. An optional port can be passed via `-Dtez.yarn.cluster.hs2.port` (default: 10000). + +```bash +mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \ + -Dtest=StartTezYarnCluster \ + -Dtez.yarn.cluster.run=true +``` + +Once ready, the log prints the JDBC URL and Beeline command. Connect from a second terminal: + +```bash +beeline -u 'jdbc:hive2://localhost:10000/default;auth=noSasl' -n hive Review Comment: thanks, in this case, it can be removed from the README as well ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java: ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + TestTezYarnUtils.setupHdfs(cluster.namenodeContainer()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + hs2Port = TestTezYarnUtils.findFreePort(); + HiveConf conf = TestTezYarnUtils.buildHiveConf(tezLibUris, localScratch, hs2Port); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SESSION_SILENT, true); + conf.setVar(HiveConf.ConfVars.PRE_EXEC_HOOKS, ""); + conf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, ""); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + TestTezYarnUtils.waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = TestTezYarnUtils.jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("INSERT INTO tez_loc_test VALUES (42)"); + + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + int count = rs.getInt(1); + Assert.assertEquals( + "INSERT VALUES should return the inserted row value (hive-exec.jar was localized)", + 42, count); + LOG.info("Tez query succeeded: inserted row value = {}", count); + } + } + } + + verifyTezYarnAppExists(); + verifyHiveExecJarOnHdfs(); + verifyHiveExecJarLocalizedInNm(); + } + + private static void verifyHiveExecJarOnHdfs() throws IOException, InterruptedException { + Container.ExecResult r = cluster.namenodeContainer().execInContainer( + "hdfs", "dfs", "-find", TestTezYarnUtils.HDFS_ROOT + "/user-install", "-name", "hive-exec-*.jar"); Review Comment: could this be handled with `FileSystem` API? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
