abstractdog commented on code in PR #6640: URL: https://github.com/apache/hive/pull/6640#discussion_r4037079115
########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); + + private static final int NN_RPC_PORT = 8020; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + // Tez AM client RPC port, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT = 41000; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, HADOOP_IMAGE); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + requireSuccess(namenodeContainer().execInContainer("hdfs", "dfsadmin", "-safemode", "wait"), + "hdfs dfsadmin -safemode wait"); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception ignored) { + } + throw new IllegalStateException("Failed to start TezYarnClusterContainer", e); + } + } + + public void stop() { + compose.stop(); + } + + private void verifyJava21InNodeManager() { + try { + Container.ExecResult r = nodeManagerContainer().execInContainer( + CONTAINER_JAVA_21_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_21_HOME, r.getStderr().trim()); + } else { + LOG.warn("Java 21 check FAILED in NodeManager (exit {}). " + + "Tez AM/task containers will fail at launch time. " + + "stderr: {}", r.getExitCode(), r.getStderr()); + } + } catch (Exception e) { + LOG.warn("Could not verify Java 21 in NodeManager container", e); + } + } + + public String getHdfsUri() { + return "hdfs://namenode:" + NN_RPC_PORT; + } + + public String getResourceManagerAddress() { + return "resourcemanager:" + RM_RPC_PORT; + } + + public String getResourceManagerWebAppAddress() { + return "resourcemanager:" + RM_HTTP_PORT; + } + + public String uploadJarToHdfs(Path localJarPath) throws IOException, InterruptedException { + String fileName = localJarPath.getFileName().toString(); + String containerTmp = "/tmp/" + fileName; + String hdfsDir = "/tmp/hive-tez-yarn-jars"; + String hdfsPath = hdfsDir + "/" + fileName; + + ContainerState namenode = namenodeContainer(); + namenode.copyFileToContainer(MountableFile.forHostPath(localJarPath, 0644), containerTmp); Review Comment: same applies here as in case of `execInContainer`, consider using API ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java: ########## Review Comment: consider removing this class, not sure if it brings any value additional to TestTezYarnLocalization ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Keep-alive HDFS + YARN + HiveServer2 cluster for manual Beeline testing. + * Enable with -Dtez.yarn.cluster.run=true. */ +public class StartTezYarnCluster { + + private static final Logger LOG = LoggerFactory.getLogger(StartTezYarnCluster.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch"; + + @Test + public void testRunCluster() throws Exception { + if (!Boolean.parseBoolean(System.getProperty("tez.yarn.cluster.run", "false"))) { + return; + } + + TezYarnClusterContainer cluster = new TezYarnClusterContainer(); + cluster.start(); + + setupHdfs(cluster.namenodeContainer()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + String derbyUrl = "jdbc:derby:" + + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"; + System.setProperty("javax.jdo.option.ConnectionURL", derbyUrl); Review Comment: these system properties look strange to me, I believe metastore should be configured by configuration only ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnClusterContainer.java: ########## Review Comment: I believe this unit test class can also be removed if the hdfs and yarn calls are changed to API calls not sure if these test cases are worth a full cluster startup/teardown ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java: ########## Review Comment: consider refactoring every duplicated thing into a helper class that's shared between `StartTezYarnCluster`, `TestTezYarnLocalization` ``` - HDFS_BASE / HDFS_ROOT / HDFS_WAREHOUSE / HDFS_SCRATCH constants - mkdir -p + chmod -R 777 fixture setup - buildHiveConf / buildConf (fs.defaultFS, warehouse, scratch, ConnectionURL, tez.lib.uris, tez.am.launch.env, tez.am.client.am.port-range) - waitForJdbc, jdbcUrl, findFreePort ``` ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); + + private static final int NN_RPC_PORT = 8020; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + // Tez AM client RPC port, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT = 41000; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, HADOOP_IMAGE); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + requireSuccess(namenodeContainer().execInContainer("hdfs", "dfsadmin", "-safemode", "wait"), + "hdfs dfsadmin -safemode wait"); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception ignored) { + } Review Comment: at least an INFO level message is needed here, should if anything happened during stop or make `start` method throw an exception instead of catch-all and rethrow an `IllegalStateException` I cannot see additional value in using an `IllegalStateException` here ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java: ########## @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + "/tmp/hive-tez-loc/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + "/tmp/hive-tez-loc/scratch"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + ContainerState nn = cluster.namenodeContainer(); + var r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp"); + Assert.assertEquals("hdfs dfs -mkdir -p /tmp failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp"); + Assert.assertEquals("hdfs dfs -chmod -R 777 /tmp failed:\n" + r.getStderr(), 0, r.getExitCode()); + + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/warehouse failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/scratch failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/user-install failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", HDFS_ROOT); + Assert.assertEquals("hdfs dfs -chmod -R 777 " + HDFS_ROOT + " failed:\n" + r.getStderr(), 0, r.getExitCode()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + String derbyUrl = "jdbc:derby:" + + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"; + System.setProperty("javax.jdo.option.ConnectionURL", derbyUrl); + HiveConf conf = buildHiveConf(tezLibUris, localScratch); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + System.clearProperty("javax.jdo.option.ConnectionURL"); + } + + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("INSERT INTO tez_loc_test VALUES (42)"); + + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + int count = rs.getInt(1); + Assert.assertEquals( + "INSERT VALUES should return the inserted row value (hive-exec.jar was localized)", + 42, count); + LOG.info("Tez query succeeded: inserted row value = {}", count); + } + } + } + + verifyTezYarnAppExists(); + verifyHiveExecJarOnHdfs(); + verifyHiveExecJarLocalizedInNm(); + } + + private static void verifyHiveExecJarOnHdfs() throws IOException, InterruptedException { + Container.ExecResult r = cluster.namenodeContainer().execInContainer( + "hdfs", "dfs", "-find", HDFS_ROOT + "/user-install", "-name", "hive-exec-*.jar"); + LOG.info("HDFS hive-exec.jar search in {}/user-install: {}", + HDFS_ROOT, r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not staged to HDFS under " + HDFS_ROOT + "/user-install — " + + "TezSessionState.buildCommonLocalResources() localization step 1 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + private static void verifyHiveExecJarLocalizedInNm() throws IOException, InterruptedException { + Container.ExecResult r = cluster.nodeManagerContainer().execInContainer( + "bash", "-c", "find /tmp -name 'hive-exec-*.jar' 2>/dev/null | head -5"); + LOG.info("NodeManager hive-exec.jar localization check: {}", + r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not found in the NodeManager container's local filesystem after Tez query — " + + "YARN localization step 2 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + /** Logs Tez AM and NodeManager diagnostics at teardown. */ + private static void dumpNodeManagerDiagnostics() { + if (cluster == null) { + return; + } + LOG.info("########## BEGIN NodeManager diagnostics ##########"); + try { + dumpNmCommand("launch_container.sh (AM launch command + classpath)", + "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container syslog (Tez AM log4j output)", + "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container stdout + stderr + prelaunch.err", + "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name 'stderr' -o -name 'prelaunch.err' \\) " + + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + } catch (Exception e) { + LOG.warn("Could not dump NodeManager diagnostics", e); + } + LOG.info("########## END NodeManager diagnostics ##########"); + } + + private static void dumpNmCommand(String label, String bashCommand) { + try { + Container.ExecResult r = + cluster.nodeManagerContainer().execInContainer("bash", "-c", bashCommand); + String out = r.getStdout(); + LOG.info("===== NM: {} =====\n{}", label, out.isEmpty() ? "(no output found)" : out); + } catch (Exception e) { + LOG.warn("===== NM: {} (dump failed) =====", label, e); + } + } + + private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) throws Exception { + HiveConf conf = new HiveConf(); + + URL hiveSite = TestTezYarnLocalization.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestTezYarnLocalization.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) { + conf.addResource(hiveSite); + } + if (yarnSite != null) { + conf.addResource(yarnSite); + } + + // Dynamic properties: values derived at runtime from container ports or temp directories. + conf.set("fs.defaultFS", HDFS_BASE); + conf.set("hive.metastore.warehouse.dir", HDFS_WAREHOUSE); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, HDFS_SCRATCH); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, HDFS_ROOT + "/user-install"); + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false); + + conf.set("tez.lib.uris", tezLibUris); + + conf.set("tez.am.client.am.port-range", + TezYarnClusterContainer.AM_CLIENT_PORT + "-" + TezYarnClusterContainer.AM_CLIENT_PORT); + String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + + ",HADOOP_HOME=/opt/hadoop" + + ",HADOOP_MAPRED_HOME=/opt/hadoop"; + conf.set("tez.am.launch.env", containerEnv); + conf.set("tez.task.launch.env", containerEnv); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + + return conf; + } + + private static void verifyTezYarnAppExists() { + try { + ContainerState rm = cluster.resourceManagerContainer(); + Container.ExecResult result = rm.execInContainer( + "yarn", "application", "-list", "-appTypes", "TEZ", "-appStates", "ALL"); + String out = result.getStdout(); + LOG.info("YARN application list (TEZ, ALL states):\n{}", out); + + Pattern appIdPattern = Pattern.compile("(application_\\d+_\\d+)"); + Matcher matcher = appIdPattern.matcher(out); + boolean found = false; + while (matcher.find()) { + LOG.info("Found Tez YARN application: {}", matcher.group(1)); + found = true; + } + + Assert.assertTrue( + "At least one Tez YARN application must be visible in the ResourceManager after running a Tez query", + found); + + } catch (Exception e) { + LOG.warn("Could not verify Tez YARN application existence via RM exec; " + + "primary query-result assertion already passed. Cause: {}", e.getMessage()); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 120_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException( + "HiveServer2 JDBC endpoint not reachable on port " + port + " after 120s"); + } Review Comment: use awaitility for this too ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Keep-alive HDFS + YARN + HiveServer2 cluster for manual Beeline testing. + * Enable with -Dtez.yarn.cluster.run=true. */ +public class StartTezYarnCluster { + + private static final Logger LOG = LoggerFactory.getLogger(StartTezYarnCluster.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch"; + + @Test + public void testRunCluster() throws Exception { + if (!Boolean.parseBoolean(System.getProperty("tez.yarn.cluster.run", "false"))) { + return; + } + + TezYarnClusterContainer cluster = new TezYarnClusterContainer(); + cluster.start(); + + setupHdfs(cluster.namenodeContainer()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + String derbyUrl = "jdbc:derby:" + + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"; + System.setProperty("javax.jdo.option.ConnectionURL", derbyUrl); + + int hs2Port = Integer.parseInt(System.getProperty("tez.yarn.cluster.hs2.port", "10000")); + HiveConf conf = buildConf(tezLibUris, localScratch, hs2Port); + + HiveServer2 hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + String jdbcUrl = "jdbc:hive2://localhost:" + hs2Port + "/default;auth=noSasl"; + LOG.info("====================================================="); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + LOG.info("JDBC URL : {}", jdbcUrl); + LOG.info("Beeline : beeline -u '{}' -n hive", jdbcUrl); + LOG.info("Press Ctrl+C to stop the cluster."); + LOG.info("====================================================="); + + Thread.currentThread().join(); + } + + private static void setupHdfs(ContainerState nn) throws Exception { + String[] dirs = { + "/tmp", + HDFS_ROOT + "/warehouse", + HDFS_ROOT + "/scratch", + HDFS_ROOT + "/user-install" + }; + for (String dir : dirs) { + Container.ExecResult r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", dir); + if (r.getExitCode() != 0) { + throw new IllegalStateException("hdfs dfs -mkdir -p " + dir + " failed:\n" + r.getStderr()); + } + } + for (String dir : new String[]{"/tmp", HDFS_ROOT}) { + Container.ExecResult r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", dir); + if (r.getExitCode() != 0) { + throw new IllegalStateException("hdfs dfs -chmod -R 777 " + dir + " failed:\n" + r.getStderr()); + } + } + } + + private static HiveConf buildConf(String tezLibUris, Path localScratch, int hs2Port) throws Exception { Review Comment: this has a lot of common code with `TestTezYarnLocalization.buildHiveConf`, consider refactoring ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; Review Comment: should be `CONTAINER_JAVA_HOME`, let's avoid changing it every time we're about to bump java version consider removing every occurrence of "21", unless it's crucial ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/StartTezYarnCluster.java: ########## @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Keep-alive HDFS + YARN + HiveServer2 cluster for manual Beeline testing. + * Enable with -Dtez.yarn.cluster.run=true. */ +public class StartTezYarnCluster { + + private static final Logger LOG = LoggerFactory.getLogger(StartTezYarnCluster.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + HDFS_ROOT + "/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + HDFS_ROOT + "/scratch"; + + @Test + public void testRunCluster() throws Exception { + if (!Boolean.parseBoolean(System.getProperty("tez.yarn.cluster.run", "false"))) { Review Comment: is this needed? if someone wants to run the cluster, there is an explicit opt-in already `-Dtest=StartTezYarnCluster` anyway, change in README.md too if needed ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnClusterContainer.java: ########## @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.Container.ExecResult; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class TestTezYarnClusterContainer { + + private static TezYarnClusterContainer cluster; + + @BeforeClass + public static void startCluster() { + cluster = new TezYarnClusterContainer(); + cluster.start(); + } + + @AfterClass + public static void stopCluster() { + if (cluster != null) { + cluster.stop(); + } + } + + @Test + public void testHdfsUriFormat() { + String uri = cluster.getHdfsUri(); + Assert.assertNotNull("HDFS URI must not be null", uri); + Assert.assertTrue("Expected hdfs:// URI, got: " + uri, uri.startsWith("hdfs://")); + } + + @Test + public void testResourceManagerAddressFormat() { + String addr = cluster.getResourceManagerAddress(); + Assert.assertNotNull("RM address must not be null", addr); + Assert.assertTrue("Expected host:port, got: " + addr, addr.contains(":")); + } + + @Test + public void testHdfsWriteAndRead() throws IOException, InterruptedException { + ExecResult mkdir = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/smoke-test"); + Assert.assertEquals("hdfs mkdir failed:\n" + mkdir.getStderr(), 0, mkdir.getExitCode()); + + ExecResult ls = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-ls", "/tmp"); + Assert.assertTrue("Expected /tmp/smoke-test in HDFS listing:\n" + ls.getStdout(), + ls.getStdout().contains("smoke-test")); + } + + @Test + public void testYarnNodeManagerRegistered() throws IOException, InterruptedException { + ExecResult result = cluster.resourceManagerContainer() + .execInContainer("yarn", "node", "-list"); + String out = result.getStdout(); Review Comment: we tend to run everything in the containers as separate processes, while HDFS and Yarn has their amazing clients (FileSystem, YarnClient, etc.), would you consider using them? that would need the boilerplate of OS process + stdout parsing and would lead to better code here in my opinion ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java: ########## @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + "/tmp/hive-tez-loc/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + "/tmp/hive-tez-loc/scratch"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + ContainerState nn = cluster.namenodeContainer(); + var r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp"); + Assert.assertEquals("hdfs dfs -mkdir -p /tmp failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp"); + Assert.assertEquals("hdfs dfs -chmod -R 777 /tmp failed:\n" + r.getStderr(), 0, r.getExitCode()); + + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/warehouse failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/scratch failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); + Assert.assertEquals("hdfs dfs -mkdir -p " + HDFS_ROOT + "/user-install failed:\n" + r.getStderr(), 0, r.getExitCode()); + r = nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", HDFS_ROOT); + Assert.assertEquals("hdfs dfs -chmod -R 777 " + HDFS_ROOT + " failed:\n" + r.getStderr(), 0, r.getExitCode()); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + String derbyUrl = "jdbc:derby:" + + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"; + System.setProperty("javax.jdo.option.ConnectionURL", derbyUrl); + HiveConf conf = buildHiveConf(tezLibUris, localScratch); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + System.clearProperty("javax.jdo.option.ConnectionURL"); + } + + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("INSERT INTO tez_loc_test VALUES (42)"); + + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + int count = rs.getInt(1); + Assert.assertEquals( + "INSERT VALUES should return the inserted row value (hive-exec.jar was localized)", + 42, count); + LOG.info("Tez query succeeded: inserted row value = {}", count); + } + } + } + + verifyTezYarnAppExists(); + verifyHiveExecJarOnHdfs(); + verifyHiveExecJarLocalizedInNm(); + } + + private static void verifyHiveExecJarOnHdfs() throws IOException, InterruptedException { + Container.ExecResult r = cluster.namenodeContainer().execInContainer( + "hdfs", "dfs", "-find", HDFS_ROOT + "/user-install", "-name", "hive-exec-*.jar"); + LOG.info("HDFS hive-exec.jar search in {}/user-install: {}", + HDFS_ROOT, r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not staged to HDFS under " + HDFS_ROOT + "/user-install — " + + "TezSessionState.buildCommonLocalResources() localization step 1 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + private static void verifyHiveExecJarLocalizedInNm() throws IOException, InterruptedException { + Container.ExecResult r = cluster.nodeManagerContainer().execInContainer( + "bash", "-c", "find /tmp -name 'hive-exec-*.jar' 2>/dev/null | head -5"); + LOG.info("NodeManager hive-exec.jar localization check: {}", + r.getStdout().trim().isEmpty() ? "(none found)" : r.getStdout().trim()); + Assert.assertFalse( + "hive-exec.jar was not found in the NodeManager container's local filesystem after Tez query — " + + "YARN localization step 2 appears to have failed.", + r.getStdout().trim().isEmpty()); + } + + /** Logs Tez AM and NodeManager diagnostics at teardown. */ + private static void dumpNodeManagerDiagnostics() { + if (cluster == null) { + return; + } + LOG.info("########## BEGIN NodeManager diagnostics ##########"); + try { + dumpNmCommand("launch_container.sh (AM launch command + classpath)", + "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container syslog (Tez AM log4j output)", + "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container stdout + stderr + prelaunch.err", + "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name 'stderr' -o -name 'prelaunch.err' \\) " + + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + } catch (Exception e) { + LOG.warn("Could not dump NodeManager diagnostics", e); + } + LOG.info("########## END NodeManager diagnostics ##########"); + } + + private static void dumpNmCommand(String label, String bashCommand) { + try { + Container.ExecResult r = + cluster.nodeManagerContainer().execInContainer("bash", "-c", bashCommand); + String out = r.getStdout(); + LOG.info("===== NM: {} =====\n{}", label, out.isEmpty() ? "(no output found)" : out); + } catch (Exception e) { + LOG.warn("===== NM: {} (dump failed) =====", label, e); + } + } + + private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) throws Exception { + HiveConf conf = new HiveConf(); + + URL hiveSite = TestTezYarnLocalization.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestTezYarnLocalization.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) { + conf.addResource(hiveSite); + } + if (yarnSite != null) { + conf.addResource(yarnSite); + } + + // Dynamic properties: values derived at runtime from container ports or temp directories. + conf.set("fs.defaultFS", HDFS_BASE); + conf.set("hive.metastore.warehouse.dir", HDFS_WAREHOUSE); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, HDFS_SCRATCH); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, HDFS_ROOT + "/user-install"); + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false); + + conf.set("tez.lib.uris", tezLibUris); + + conf.set("tez.am.client.am.port-range", + TezYarnClusterContainer.AM_CLIENT_PORT + "-" + TezYarnClusterContainer.AM_CLIENT_PORT); + String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + + ",HADOOP_HOME=/opt/hadoop" + + ",HADOOP_MAPRED_HOME=/opt/hadoop"; + conf.set("tez.am.launch.env", containerEnv); + conf.set("tez.task.launch.env", containerEnv); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + + return conf; + } + + private static void verifyTezYarnAppExists() { Review Comment: method can throw an Exception instead of wrapping the whole code inside in a try-catch ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); + + private static final int NN_RPC_PORT = 8020; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + // Tez AM client RPC port, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT = 41000; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, HADOOP_IMAGE); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + requireSuccess(namenodeContainer().execInContainer("hdfs", "dfsadmin", "-safemode", "wait"), + "hdfs dfsadmin -safemode wait"); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception ignored) { + } + throw new IllegalStateException("Failed to start TezYarnClusterContainer", e); + } + } + + public void stop() { + compose.stop(); + } + + private void verifyJava21InNodeManager() { Review Comment: method should thrown an exception instead of try-catch, start() will catch and act accordingly ########## itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml: ########## @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# HDFS + YARN cluster for tez-yarn-it. Image built by TezYarnClusterContainer; config via env_file. + +services: + namenode: + image: hive-it-hadoop-jdk21:latest + hostname: namenode + command: ["hdfs", "namenode"] + env_file: + - config + environment: + ENSURE_NAMENODE_DIR: /tmp/hadoop-hadoop/dfs/name + ports: + - "8020:8020" + - "9870:9870" + healthcheck: + test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/9870'"] + interval: 5s + timeout: 3s + retries: 30 + start_period: 10s + + datanode: + image: hive-it-hadoop-jdk21:latest + hostname: datanode + command: ["hdfs", "datanode"] + env_file: + - config + depends_on: + namenode: + condition: service_healthy + ports: + - "9866:9866" + - "9864:9864" + + resourcemanager: + image: hive-it-hadoop-jdk21:latest + hostname: resourcemanager + command: ["yarn", "resourcemanager"] + env_file: + - config + depends_on: + namenode: + condition: service_healthy + ports: + - "8032:8032" + - "8088:8088" + # No healthcheck: RM webapp binds to hostname "resourcemanager", not localhost. + + nodemanager: + image: hive-it-hadoop-jdk21:latest + hostname: nodemanager + command: ["yarn", "nodemanager"] + env_file: + - config + depends_on: + resourcemanager: + condition: service_started + ports: + - "41000:41000" Review Comment: consider using and exposing a port range ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); Review Comment: not necessary to move this to static initialization time, it can lead to much more exotic exception than the original root one HADOOP_IMAGE can be an instance field, and this can be initialized in the contructor ########## itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.tez.yarn; + +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerState; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; + +/** Starts an HDFS + YARN cluster from docker-compose.yml via Testcontainers ComposeContainer. + * Fixed published ports plus custom_hosts_file let the host JVM reach namenode/resourcemanager. */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); + + private static final int NN_RPC_PORT = 8020; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + // Tez AM client RPC port, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT = 41000; + + // Compose V2 names service containers "<service>-1"; used by getContainerByServiceName(). + private static final int SERVICE_INSTANCE = 1; + + private final ComposeContainer compose; + + public TezYarnClusterContainer() { + String basedir = System.getProperty("basedir", "."); + File composeFile = Paths.get(basedir, "src/test/docker/hadoop-yarn/docker-compose.yml").toFile(); + LOG.info("Starting Tez-on-YARN cluster from {} (image {})", composeFile, HADOOP_IMAGE); + // withLocalCompose(false): no local docker-compose binary required on CI. + compose = new ComposeContainer(composeFile) + .withLocalCompose(false); + } + + public void start() { + try { + compose.start(); + // Avoid flakiness: wait until HDFS has left safemode before running HDFS operations. + requireSuccess(namenodeContainer().execInContainer("hdfs", "dfsadmin", "-safemode", "wait"), + "hdfs dfsadmin -safemode wait"); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } catch (Exception e) { + try { + stop(); + } catch (Exception ignored) { + } + throw new IllegalStateException("Failed to start TezYarnClusterContainer", e); + } + } + + public void stop() { + compose.stop(); + } + + private void verifyJava21InNodeManager() { + try { + Container.ExecResult r = nodeManagerContainer().execInContainer( + CONTAINER_JAVA_21_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_21_HOME, r.getStderr().trim()); + } else { + LOG.warn("Java 21 check FAILED in NodeManager (exit {}). " + + "Tez AM/task containers will fail at launch time. " + + "stderr: {}", r.getExitCode(), r.getStderr()); + } + } catch (Exception e) { + LOG.warn("Could not verify Java 21 in NodeManager container", e); + } + } + + public String getHdfsUri() { + return "hdfs://namenode:" + NN_RPC_PORT; + } + + public String getResourceManagerAddress() { + return "resourcemanager:" + RM_RPC_PORT; + } + + public String getResourceManagerWebAppAddress() { + return "resourcemanager:" + RM_HTTP_PORT; + } + + public String uploadJarToHdfs(Path localJarPath) throws IOException, InterruptedException { + String fileName = localJarPath.getFileName().toString(); + String containerTmp = "/tmp/" + fileName; + String hdfsDir = "/tmp/hive-tez-yarn-jars"; + String hdfsPath = hdfsDir + "/" + fileName; + + ContainerState namenode = namenodeContainer(); + namenode.copyFileToContainer(MountableFile.forHostPath(localJarPath, 0644), containerTmp); + + Container.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + Container.ExecResult put = namenode.execInContainer("hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + return hdfsPath; + } + + public String uploadTezLibsToHdfs() throws IOException, InterruptedException { + String tezDistPath = System.getProperty("tez.dist.path"); + if (tezDistPath == null || tezDistPath.isEmpty()) { + throw new IllegalStateException( + "System property 'tez.dist.path' is not set. " + + "It must point to the tez-libs.tar.gz assembled from Tez Maven artifacts. " + + "This is set automatically by the Maven surefire configuration; " + + "if running tests in isolation, ensure the module was built with " + + "'mvn test-compile -Pitests,tez-yarn' first."); + } + + Path tarball = Paths.get(tezDistPath); + if (!Files.isRegularFile(tarball)) { + throw new IllegalStateException( + "Tez distribution tarball not found at: " + tarball.toAbsolutePath() + + ". Run 'mvn test-compile -Pitests,tez-yarn -pl itests/tez-yarn-it' to build it."); + } + + String fileName = tarball.getFileName().toString(); + String containerTmp = "/tmp/" + fileName; + String hdfsDir = "/tmp/hive-tez-yarn"; + String hdfsPath = hdfsDir + "/" + fileName; + + LOG.info("Uploading Tez distribution tarball ({}) to HDFS path {}", + tarball.toAbsolutePath(), hdfsPath); + + ContainerState namenode = namenodeContainer(); + Container.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + namenode.copyFileToContainer(MountableFile.forHostPath(tarball, 0644), containerTmp); + + Container.ExecResult put = namenode.execInContainer( + "hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + // "#tez" is the YARN container link name for the localized archive. + return "hdfs://namenode:" + NN_RPC_PORT + hdfsPath + "#tez"; + } + + ContainerState namenodeContainer() { + return serviceContainer("namenode"); + } + + ContainerState resourceManagerContainer() { + return serviceContainer("resourcemanager"); + } + + ContainerState nodeManagerContainer() { + return serviceContainer("nodemanager"); + } + + private ContainerState serviceContainer(String serviceName) { + return compose.getContainerByServiceName(serviceName + "-" + SERVICE_INSTANCE) + .orElseThrow(() -> new IllegalStateException( + "Compose service container not running: " + serviceName)); Review Comment: make sure the container is running, not just existing `ContainerState cs = compose.getContainerByServiceName( ... cs.isRunning()` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
