HIVE-16460: In the console output, show vertex list in topological order instead of an alphabetical sort (Prasanth Jayachandran reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/087aafca Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/087aafca Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/087aafca Branch: refs/heads/hive-14535 Commit: 087aafca763ed1f49cab2c389d2e00425c48a6d0 Parents: cea9ea7 Author: Prasanth Jayachandran <[email protected]> Authored: Fri Jun 2 15:30:51 2017 -0700 Committer: Prasanth Jayachandran <[email protected]> Committed: Fri Jun 2 15:30:51 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +- .../ql/exec/tez/monitoring/TezJobMonitor.java | 8 ++-- .../exec/tez/monitoring/TezProgressMonitor.java | 42 ++++++++++---------- .../tez/monitoring/TestTezProgressMonitor.java | 3 +- 4 files changed, 28 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 3356dc9..7ee6186 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -194,7 +194,7 @@ public class TezTask extends Task<TezWork> { additionalLr, inputOutputJars, inputOutputLocalResources); // finally monitor will print progress until the job is done - TezJobMonitor monitor = new TezJobMonitor(work.getWorkMap(),dagClient, conf, dag, ctx); + TezJobMonitor monitor = new TezJobMonitor(work.getAllWork(),dagClient, conf, dag, ctx); rc = monitor.monitorExecution(); if (rc != 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java index 049d7fd..9e2846c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java @@ -68,7 +68,7 @@ public class TezJobMonitor { private final PerfLogger perfLogger = SessionState.getPerfLogger(); private static final List<DAGClient> shutdownList; - private final Map<String, BaseWork> workMap; + private final List<BaseWork> topSortedWorks; transient LogHelper console; @@ -101,9 +101,9 @@ public class TezJobMonitor { private long executionStartTime = 0; private final RenderStrategy.UpdateFunction updateFunction; - public TezJobMonitor(Map<String, BaseWork> workMap, final DAGClient dagClient, HiveConf conf, DAG dag, + public TezJobMonitor(List<BaseWork> topSortedWorks, final DAGClient dagClient, HiveConf conf, DAG dag, Context ctx) { - this.workMap = workMap; + this.topSortedWorks = topSortedWorks; this.dagClient = dagClient; this.hiveConf = conf; this.dag = dag; @@ -323,7 +323,7 @@ public class TezJobMonitor { ProgressMonitor progressMonitor(DAGStatus status, Map<String, Progress> progressMap) { try { - return new TezProgressMonitor(dagClient, status, workMap, progressMap, console, + return new TezProgressMonitor(dagClient, status, topSortedWorks, progressMap, console, executionStartTime); } catch (IOException | TezException e) { console.printInfo("Getting Progress Information: " + e.getMessage() + " stack trace: " + http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java index 9739ad7..7fbd80a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezProgressMonitor.java @@ -42,7 +42,7 @@ import static org.apache.tez.dag.api.client.DAGStatus.State.KILLED; class TezProgressMonitor implements ProgressMonitor { private static final int COLUMN_1_WIDTH = 16; - private final Map<String, BaseWork> workMap; + private final List<BaseWork> topSortedWork; private final SessionState.LogHelper console; private final long executionStartTime; private final DAGStatus status; @@ -53,11 +53,11 @@ class TezProgressMonitor implements ProgressMonitor { * Try to get most the data required from dagClient in the constructor itself so that even after * the tez job has finished this object can be used for later use.s */ - TezProgressMonitor(DAGClient dagClient, DAGStatus status, Map<String, BaseWork> workMap, + TezProgressMonitor(DAGClient dagClient, DAGStatus status, List<BaseWork> topSortedWork, Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime) throws IOException, TezException { this.status = status; - this.workMap = workMap; + this.topSortedWork = topSortedWork; this.console = console; this.executionStartTime = executionStartTime; for (Map.Entry<String, Progress> entry : progressMap.entrySet()) { @@ -88,25 +88,26 @@ class TezProgressMonitor implements ProgressMonitor { public List<List<String>> rows() { try { List<List<String>> results = new ArrayList<>(); - SortedSet<String> keys = new TreeSet<>(progressCountsMap.keySet()); - for (String s : keys) { - VertexProgress progress = progressCountsMap.get(s); + for (BaseWork baseWork : topSortedWork) { + String vertexName = baseWork.getName(); + VertexProgress progress = progressCountsMap.get(vertexName); + if (progress != null) { + // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0 - // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0 - - results.add( + results.add( Arrays.asList( - getNameWithProgress(s, progress.succeededTaskCount, progress.totalTaskCount), - getMode(s, workMap), - progress.vertexStatus(vertexStatusMap.get(s)), - progress.total(), - progress.completed(), - progress.running(), - progress.pending(), - progress.failed(), - progress.killed() + getNameWithProgress(vertexName, progress.succeededTaskCount, progress.totalTaskCount), + getMode(baseWork), + progress.vertexStatus(vertexStatusMap.get(vertexName)), + progress.total(), + progress.completed(), + progress.running(), + progress.pending(), + progress.failed(), + progress.killed() ) - ); + ); + } } return results; } catch (Exception e) { @@ -194,9 +195,8 @@ class TezProgressMonitor implements ProgressMonitor { return result; } - private String getMode(String name, Map<String, BaseWork> workMap) { + private String getMode(BaseWork work) { String mode = "container"; - BaseWork work = workMap.get(name); if (work != null) { // uber > llap > container if (work.getUberMode()) { http://git-wip-us.apache.org/repos/asf/hive/blob/087aafca/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java index 1e0fa14..98067df 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/monitoring/TestTezProgressMonitor.java @@ -32,6 +32,7 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -92,7 +93,7 @@ public class TestTezProgressMonitor { when(dagClient.getVertexStatus(eq(REDUCER), anySet())).thenReturn(running); TezProgressMonitor monitor = - new TezProgressMonitor(dagClient, dagStatus, new HashMap<String, BaseWork>(), progressMap(), console, + new TezProgressMonitor(dagClient, dagStatus, new ArrayList<BaseWork>(), progressMap(), console, Long.MAX_VALUE); verify(dagClient).getVertexStatus(eq(MAPPER), isNull(Set.class));
