This is an automated email from the ASF dual-hosted git repository. gengliang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 999c9ed1 [SPARK-31081][UI][SQL] Make display of stageId/stageAttemptId/taskId of sql metrics toggleable 999c9ed1 is described below commit 999c9ed10c2362d89afd3bbb48e35f3c7ac3cf89 Author: Kousuke Saruta <saru...@oss.nttdata.com> AuthorDate: Tue Mar 24 13:37:13 2020 -0700 [SPARK-31081][UI][SQL] Make display of stageId/stageAttemptId/taskId of sql metrics toggleable ### What changes were proposed in this pull request? This is another solution for `SPARK-31081` and #27849 . I added a checkbox which can toggle display of stageId/taskid in the SQL's DAG page. Mainly, I implemented the toggleable texts in boxes with HTML label feature provided by `dagre-d3`. The additional metrics are enclosed by `<span>` and control the appearance of the text. But the exception is additional metrics in clusters. We can use HTML label for cluster but layout will be broken so I choosed normal text label for clusters. Due to that, this solution contains a little bit tricky code in`spark-sql-viz.js` to manipulate the metric texts and generate DOMs. ### Why are the changes needed? It makes metrics harder to read after #26843 and user may not interest in extra info(stageId/StageAttemptId/taskId ) when they do not need debug. #27849 control the appearance by a new configuration property but providing a checkbox is more flexible. ### Does this PR introduce any user-facing change? Yes. [Additional metrics shown] ![with-checked](https://user-images.githubusercontent.com/4736016/77244214-0f6cd780-6c56-11ea-9275-a30758dd5339.png) [Additional metrics hidden] ![without-chedked](https://user-images.githubusercontent.com/4736016/77244219-14ca2200-6c56-11ea-9874-33a466085fce.png) ### How was this patch tested? Tested manually with a simple DataFrame operation. * The appearance of additional metrics in the boxes are controlled by the newly added checkbox. * No error found with JS-debugger. * Checked/not-checked state is preserved after reloading. Closes #27927 from sarutak/SPARK-31081. Authored-by: Kousuke Saruta <saru...@oss.nttdata.com> Signed-off-by: Gengliang Wang <gengliang.w...@databricks.com> --- .../sql/execution/ui/static/spark-sql-viz.css | 2 +- .../spark/sql/execution/ui/static/spark-sql-viz.js | 74 +++++++++++++++++++++- .../spark/sql/execution/ui/ExecutionPage.scala | 4 ++ .../spark/sql/execution/ui/SparkPlanGraph.scala | 10 +-- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css index 94a6bd8..6ba79dc 100644 --- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css +++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css @@ -35,7 +35,7 @@ } /* Highlight the SparkPlan node name */ -#plan-viz-graph svg text :first-child { +#plan-viz-graph svg text :first-child:not(.stageId-and-taskId-metrics) { font-weight: bold; } diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js index 6244b2c..0fb7dab 100644 --- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js +++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js @@ -47,6 +47,7 @@ function renderPlanViz() { } resizeSvg(svg); + postprocessForAdditionalMetrics(); } /* -------------------- * @@ -70,6 +71,10 @@ function setupTooltipForSparkPlanNode(nodeId) { }) } +// labelSeparator should be a non-graphical character in order not to affect the width of boxes. +var labelSeparator = "\x01"; +var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*attempt.*task[^)]*\\))(.*)$"; + /* * Helper function to pre-process the graph layout. * This step is necessary for certain styles that affect the positioning @@ -79,8 +84,29 @@ function preprocessGraphLayout(g) { g.graph().ranksep = "70"; var nodes = g.nodes(); for (var i = 0; i < nodes.length; i++) { - var node = g.node(nodes[i]); - node.padding = "5"; + var node = g.node(nodes[i]); + node.padding = "5"; + + var firstSearator; + var secondSeparator; + var splitter; + if (node.isCluster) { + firstSearator = secondSeparator = labelSeparator; + splitter = "\\n"; + } else { + firstSearator = "<span class='stageId-and-taskId-metrics'>"; + secondSeparator = "</span>"; + splitter = "<br>"; + } + + node.label.split(splitter).forEach(function(text, i) { + var newTexts = text.match(stageAndTaskMetricsPattern); + if (newTexts) { + node.label = node.label.replace( + newTexts[0], + newTexts[1] + firstSearator + newTexts[2] + secondSeparator + newTexts[3]); + } + }); } // Curve the edges var edges = g.edges(); @@ -158,3 +184,47 @@ function getAbsolutePosition(d3selection) { } return { x: _x, y: _y }; } + +/* + * Helper function for postprocess for additional metrics. + */ +function postprocessForAdditionalMetrics() { + // With dagre-d3, we can choose normal text (default) or HTML as a label type. + // HTML label for node works well but not for cluster so we need to choose the default label type + // and manipulate DOM. + $("g.cluster text tspan") + .each(function() { + var originalText = $(this).text(); + if (originalText.indexOf(labelSeparator) > 0) { + var newTexts = originalText.split(labelSeparator); + var thisD3Node = d3.selectAll($(this)); + thisD3Node.text(newTexts[0]); + thisD3Node.append("tspan").attr("class", "stageId-and-taskId-metrics").text(newTexts[1]); + $(this).append(newTexts[2]); + } else { + return originalText; + } + }); + + var checkboxNode = $("#stageId-and-taskId-checkbox"); + checkboxNode.click(function() { + onClickAdditionalMetricsCheckbox($(this)); + }); + var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") == "true"; + $("#stageId-and-taskId-checkbox").prop("checked", isChecked); + onClickAdditionalMetricsCheckbox(checkboxNode); +} + +/* + * Helper function which defines the action on click the checkbox. + */ +function onClickAdditionalMetricsCheckbox(checkboxNode) { + var additionalMetrics = $(".stageId-and-taskId-metrics"); + var isChecked = checkboxNode.prop("checked"); + if (isChecked) { + additionalMetrics.show(); + } else { + additionalMetrics.hide(); + } + window.localStorage.setItem("stageId-and-taskId-checked", isChecked); +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala index 6084aaf..d304369 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala @@ -71,6 +71,10 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging {jobLinks(JobExecutionStatus.FAILED, "Failed Jobs:")} </ul> </div> + <div> + <input type="checkbox" id="stageId-and-taskId-checkbox"></input> + <span>Show the Stage (Stage Attempt): Task ID that corresponds to the max metric</span> + </div> val metrics = sqlStore.executionMetrics(executionId) val graph = sqlStore.planGraph(executionId) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala index d31d778..6762802 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala @@ -160,7 +160,7 @@ private[ui] class SparkPlanGraphNode( val metrics: Seq[SQLPlanMetric]) { def makeDotNode(metricsValue: Map[Long, String]): String = { - val builder = new mutable.StringBuilder(name) + val builder = new mutable.StringBuilder("<b>" + name + "</b>") val values = for { metric <- metrics @@ -173,9 +173,10 @@ private[ui] class SparkPlanGraphNode( // If there are metrics, display each entry in a separate line. // Note: whitespace between two "\n"s is to create an empty line between the name of // SparkPlan and metrics. If removing it, it won't display the empty line in UI. - builder ++= "\n \n" - builder ++= values.mkString("\n") - s""" $id [label="${StringEscapeUtils.escapeJava(builder.toString())}"];""" + builder ++= "<br><br>" + builder ++= values.mkString("<br>") + val labelStr = StringEscapeUtils.escapeJava(builder.toString().replaceAll("\n", "<br>")) + s""" $id [labelType="html" label="${labelStr}"];""" } else { // SPARK-30684: when there is no metrics, add empty lines to increase the height of the node, // so that there won't be gaps between an edge and a small node. @@ -210,6 +211,7 @@ private[ui] class SparkPlanGraphCluster( } s""" | subgraph cluster${id} { + | isCluster="true"; | label="${StringEscapeUtils.escapeJava(labelStr)}"; | ${nodes.map(_.makeDotNode(metricsValue)).mkString(" \n")} | } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org