HeartSaVioR commented on a change in pull request #30336:
URL: https://github.com/apache/spark/pull/30336#discussion_r525707868
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
##########
@@ -236,12 +239,61 @@ private[ui] class StreamingQueryStatisticsPage(parent:
StreamingQueryTab)
<td
class={"aggregated-num-state-rows-dropped-by-watermark-timeline"}>{graphUIDataForNumRowsDroppedByWatermark.generateTimelineHtml(jsCollector)}</td>
<td
class={"aggregated-num-state-rows-dropped-by-watermark-histogram"}>{graphUIDataForNumRowsDroppedByWatermark.generateHistogramHtml(jsCollector)}</td>
</tr>
- // scalastyle:on
+ // scalastyle:on
+
+ result ++= generateAggregatedCustomMetrics(query, minBatchTime,
maxBatchTime, jsCollector)
+ result
} else {
new NodeBuffer()
}
}
+ def generateAggregatedCustomMetrics(
+ query: StreamingQueryUIData,
+ minBatchTime: Long,
+ maxBatchTime: Long,
+ jsCollector: JsCollector): NodeBuffer = {
+ val result: NodeBuffer = new NodeBuffer
+
+ // This is made sure on caller side but put it here to be defensive
+ require(query.lastProgress.stateOperators.nonEmpty)
+ val enabledCustomMetrics =
parent.parent.conf.get(ENABLED_STREAMING_UI_CUSTOM_METRIC_LIST)
+ logDebug(s"Enabled custom metrics: $enabledCustomMetrics")
+ query.lastProgress.stateOperators.head.customMetrics.keySet().asScala
+ .filter(enabledCustomMetrics.contains(_)).map { metricName =>
+ val data = query.recentProgress.map(p =>
(parseProgressTimestamp(p.timestamp),
+ p.stateOperators.map(_.customMetrics.get(metricName).toDouble).sum))
+ val max = data.maxBy(_._2)._2
+
+ val graphUIData =
+ new GraphUIData(
+ s"aggregated-$metricName-timeline",
+ s"aggregated-$metricName-histogram",
+ data,
+ minBatchTime,
+ maxBatchTime,
+ 0,
+ max,
+ "")
+ graphUIData.generateDataJs(jsCollector)
+
+ result ++=
+ // scalastyle:off
+ <tr>
+ <td style="vertical-align: middle;">
+ <div style="width: 240px;">
+ <div><strong>Aggregated Custom Metric {s"$metricName"}
{SparkUIUtils.tooltip("Custom metric.", "right")}</strong></div>
Review comment:
FYI showing desc looks to be coupled with this comment -
https://github.com/apache/spark/pull/30336#discussion_r525207223
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
