This is an automated email from the ASF dual-hosted git repository. abstractdog pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new d2ce078f2d8 HIVE-27295: Improve docker logging in AbstractExternalDB and DatabaseRule (#4268) (Laszlo Bodor reviewed by Stamatis Zampetakis) d2ce078f2d8 is described below commit d2ce078f2d8584f39fbf8329c88c544f019464f8 Author: Bodor Laszlo <bodorlaszlo0...@gmail.com> AuthorDate: Thu Apr 27 09:08:09 2023 +0200 HIVE-27295: Improve docker logging in AbstractExternalDB and DatabaseRule (#4268) (Laszlo Bodor reviewed by Stamatis Zampetakis) --- .../hive/ql/externalDB/AbstractExternalDB.java | 39 ++++++++++++++-------- .../metastore/dbinstall/rules/DatabaseRule.java | 34 +++++++++++++------ 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java index f328bfc4bc6..48da3344277 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java @@ -83,14 +83,12 @@ public abstract class AbstractExternalDB { return new String[] { "docker", "logs", getDockerContainerName() }; } - private ProcessResults runCmd(String[] cmd, long secondsToWait) throws IOException, InterruptedException { LOG.info("Going to run: " + String.join(" ", cmd)); Process proc = Runtime.getRuntime().exec(cmd); - if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) { - throw new RuntimeException( - "Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds"); + if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) { + throw new RuntimeException("Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds"); } BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); final StringBuilder lines = new StringBuilder(); @@ -99,41 +97,54 @@ public abstract class AbstractExternalDB { reader = new BufferedReader(new InputStreamReader(proc.getErrorStream())); final StringBuilder errLines = new StringBuilder(); reader.lines().forEach(s -> errLines.append(s).append('\n')); - LOG.info("Result size: " + lines.length() + ";" + errLines.length()); + LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(), errLines.length()); return new ProcessResults(lines.toString(), errLines.toString(), proc.exitValue()); } - private int runCmdAndPrintStreams(String[] cmd, long secondsToWait) + private ProcessResults runCmdAndPrintStreams(String[] cmd, long secondsToWait) throws InterruptedException, IOException { ProcessResults results = runCmd(cmd, secondsToWait); LOG.info("Stdout from proc: " + results.stdout); LOG.info("Stderr from proc: " + results.stderr); - return results.rc; + return results; } public void launchDockerContainer() throws Exception { runCmdAndPrintStreams(buildRmCmd(), 600); - if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) { - throw new RuntimeException("Unable to start docker container"); + if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) { + printDockerEvents(); + throw new RuntimeException("Unable to start docker container"); } long startTime = System.currentTimeMillis(); ProcessResults pr; do { Thread.sleep(1000); - pr = runCmd(buildLogCmd(), 30); + pr = runCmdAndPrintStreams(buildLogCmd(), 30); if (pr.rc != 0) { - throw new RuntimeException("Failed to get docker logs"); + printDockerEvents(); + throw new RuntimeException("Failed to get docker logs"); } } while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() && !isContainerReady(pr)); if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) { - throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 + - " seconds"); + printDockerEvents(); + throw new RuntimeException( + String.format("Container initialization failed within %d seconds. Please check the hive logs.", + MAX_STARTUP_WAIT / 1000)); } + } + + protected void printDockerEvents() { + try { + runCmdAndPrintStreams(new String[] { "docker", "events", "--since", "24h", "--until", "0s" }, 3); + } catch (Exception e) { + LOG.warn("A problem was encountered while attempting to retrieve Docker events (the system made an analytical" + + " best effort to list the events to reveal the root cause). No further actions are necessary.", e); + } } public void cleanupDockerContainer() throws IOException, InterruptedException { - if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) { + if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) { throw new RuntimeException("Unable to remove docker container"); } } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java index 4fc8d50bb76..23115ab916a 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/DatabaseRule.java @@ -125,21 +125,25 @@ public abstract class DatabaseRule extends ExternalResource { @Override public void before() throws Exception { //runDockerContainer runCmdAndPrintStreams(buildRmCmd(), 600); - if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) { + if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) { + printDockerEvents(); throw new RuntimeException("Unable to start docker container"); } long startTime = System.currentTimeMillis(); ProcessResults pr; do { Thread.sleep(1000); - pr = runCmd(buildLogCmd(), 30); + pr = runCmdAndPrintStreams(buildLogCmd(), 30); if (pr.rc != 0) { + printDockerEvents(); throw new RuntimeException("Failed to get docker logs"); } } while (startTime + MAX_STARTUP_WAIT >= System.currentTimeMillis() && !isContainerReady(pr)); if (startTime + MAX_STARTUP_WAIT < System.currentTimeMillis()) { - throw new RuntimeException("Container failed to be ready in " + MAX_STARTUP_WAIT/1000 + - " seconds"); + printDockerEvents(); + throw new RuntimeException( + String.format("Container initialization failed within %d seconds. Please check the hive logs.", + MAX_STARTUP_WAIT / 1000)); } MetastoreSchemaTool.setHomeDirForTesting(); } @@ -152,7 +156,7 @@ public abstract class DatabaseRule extends ExternalResource { return; } try { - if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) { + if (runCmdAndPrintStreams(buildRmCmd(), 600).rc != 0) { throw new RuntimeException("Unable to remove docker container"); } } catch (InterruptedException | IOException e) { @@ -174,9 +178,8 @@ public abstract class DatabaseRule extends ExternalResource { throws IOException, InterruptedException { LOG.info("Going to run: " + StringUtils.join(cmd, " ")); Process proc = Runtime.getRuntime().exec(cmd); - if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) { - throw new RuntimeException( - "Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds"); + if (!proc.waitFor(Math.abs(secondsToWait), TimeUnit.SECONDS)) { + throw new RuntimeException("Process " + cmd[0] + " failed to run in " + secondsToWait + " seconds"); } BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream())); final StringBuilder lines = new StringBuilder(); @@ -185,16 +188,25 @@ public abstract class DatabaseRule extends ExternalResource { reader = new BufferedReader(new InputStreamReader(proc.getErrorStream())); final StringBuilder errLines = new StringBuilder(); reader.lines().forEach(s -> errLines.append(s).append('\n')); - LOG.info("Result size: " + lines.length() + ";" + errLines.length()); + LOG.info("Result lines#: {}(stdout);{}(stderr)",lines.length(), errLines.length()); return new ProcessResults(lines.toString(), errLines.toString(), proc.exitValue()); } - private int runCmdAndPrintStreams(String[] cmd, long secondsToWait) + private ProcessResults runCmdAndPrintStreams(String[] cmd, long secondsToWait) throws InterruptedException, IOException { ProcessResults results = runCmd(cmd, secondsToWait); LOG.info("Stdout from proc: " + results.stdout); LOG.info("Stderr from proc: " + results.stderr); - return results.rc; + return results; + } + + protected void printDockerEvents() { + try { + runCmdAndPrintStreams(new String[] { "docker", "events", "--since", "24h", "--until", "0s" }, 3); + } catch (Exception e) { + LOG.warn("A problem was encountered while attempting to retrieve Docker events (the system made an analytical" + + " best effort to list the events to reveal the root cause). No further actions are necessary.", e); + } } private String[] buildRunCmd() {