This is an automated email from the ASF dual-hosted git repository. sarutak pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 9ca5934 [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured streaming tab 9ca5934 is described below commit 9ca5934cb6bf8257d91a33fcf6f2738822fae34a Author: iRakson <raksonrak...@gmail.com> AuthorDate: Mon Jun 15 10:39:55 2020 +0900 [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured streaming tab ### What changes were proposed in this pull request? Sorting result for duration column in tables of structured streaming tab is wrong sometimes. <img width="1677" alt="Screenshot 2020-06-13 at 1 58 53 PM" src="https://user-images.githubusercontent.com/15366835/84572178-10755700-adb6-11ea-9131-338e8ba7fb24.png"> We are sorting on string, which results in this behaviour. `sorttable_numeric` and `sorttable_customkey` is used to fix this. Refer [this](https://github.com/apache/spark/pull/28752#issuecomment-643451586) and [this](https://github.com/apache/spark/pull/28752#issuecomment-643569254) After changes : <img width="1677" alt="Screenshot 2020-06-13 at 8 05 32 PM" src="https://user-images.githubusercontent.com/15366835/84572299-a8734080-adb6-11ea-9aa3-b4bc594de4cf.png"> ### Why are the changes needed? Sorting results are wrong for duration column in tables of structured streaming tab. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Screenshots attached. Closes #28823 from iRakson/testsort. Authored-by: iRakson <raksonrak...@gmail.com> Signed-off-by: Kousuke Saruta <saru...@oss.nttdata.com> --- .../sql/streaming/ui/StreamingQueryPage.scala | 23 ++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala index 7336765..43b93a3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala @@ -57,12 +57,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab) val name = UIUtils.getQueryName(query) val status = UIUtils.getQueryStatus(query) val duration = if (queryActive) { - SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp) + System.currentTimeMillis() - query.startTimestamp } else { withNoProgress(query, { val endTimeMs = query.lastProgress.timestamp - SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - query.startTimestamp) - }, "-") + parseProgressTimestamp(endTimeMs) - query.startTimestamp + }, 0) } <tr> @@ -71,7 +71,9 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab) <td> {query.id} </td> <td> <a href={statisticsLink}> {query.runId} </a> </td> <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td> - <td> {duration} </td> + <td sorttable_customkey={duration.toString}> + {SparkUIUtils.formatDurationVerbose(duration)} + </td> <td> {withNoProgress(query, { (query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum / query.recentProgress.length).formatted("%.2f") }, "NaN")} @@ -93,8 +95,13 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab) "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec", "Avg Process /sec", "Lastest Batch") + val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric", + "sorttable_numeric", "") + // header classes size must be equal to header row size + assert(headerRow.size == headerCss.size) + Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true), - activeQueries, true, Some("activeQueries-table"), Seq(null), false)) + activeQueries, true, Some("activeQueries-table"), headerCss, false)) } else { None } @@ -104,8 +111,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab) "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec", "Avg Process /sec", "Lastest Batch", "Error") + val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric", + "sorttable_numeric", "", "") + assert(headerRow.size == headerCss.size) + Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false), - inactiveQueries, true, Some("completedQueries-table"), Seq(null), false)) + inactiveQueries, true, Some("completedQueries-table"), headerCss, false)) } else { None } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org