[spark] branch branch-3.0 updated: [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured streaming tab

[sarutak]

This is an automated email from the ASF dual-hosted git repository.

sarutak pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new 9ca5934  [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in 
structured streaming tab
9ca5934 is described below

commit 9ca5934cb6bf8257d91a33fcf6f2738822fae34a
Author: iRakson <raksonrak...@gmail.com>
AuthorDate: Mon Jun 15 10:39:55 2020 +0900

    [SPARK-31983][WEBUI][3.0] Fix sorting for duration column in structured 
streaming tab
    
    ### What changes were proposed in this pull request?
    Sorting result for duration column in tables of structured streaming tab is 
wrong sometimes.
    <img width="1677" alt="Screenshot 2020-06-13 at 1 58 53 PM" 
src="https://user-images.githubusercontent.com/15366835/84572178-10755700-adb6-11ea-9131-338e8ba7fb24.png";>
    
    We are sorting on string, which results in this behaviour.
    `sorttable_numeric` and `sorttable_customkey` is used to fix this.
    
    Refer 
[this](https://github.com/apache/spark/pull/28752#issuecomment-643451586) and 
[this](https://github.com/apache/spark/pull/28752#issuecomment-643569254)
    
    After changes :
    <img width="1677" alt="Screenshot 2020-06-13 at 8 05 32 PM" 
src="https://user-images.githubusercontent.com/15366835/84572299-a8734080-adb6-11ea-9aa3-b4bc594de4cf.png";>
    
    ### Why are the changes needed?
    Sorting results are wrong for duration column in tables of structured 
streaming tab.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Screenshots attached.
    
    Closes #28823 from iRakson/testsort.
    
    Authored-by: iRakson <raksonrak...@gmail.com>
    Signed-off-by: Kousuke Saruta <saru...@oss.nttdata.com>
---
 .../sql/streaming/ui/StreamingQueryPage.scala      | 23 ++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 7336765..43b93a3 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -57,12 +57,12 @@ private[ui] class StreamingQueryPage(parent: 
StreamingQueryTab)
     val name = UIUtils.getQueryName(query)
     val status = UIUtils.getQueryStatus(query)
     val duration = if (queryActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - 
query.startTimestamp)
+      System.currentTimeMillis() - query.startTimestamp
     } else {
       withNoProgress(query, {
         val endTimeMs = query.lastProgress.timestamp
-        SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - 
query.startTimestamp)
-      }, "-")
+        parseProgressTimestamp(endTimeMs) - query.startTimestamp
+      }, 0)
     }
 
     <tr>
@@ -71,7 +71,9 @@ private[ui] class StreamingQueryPage(parent: 
StreamingQueryTab)
       <td> {query.id} </td>
       <td> <a href={statisticsLink}> {query.runId} </a> </td>
       <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
-      <td> {duration} </td>
+      <td sorttable_customkey={duration.toString}>
+        {SparkUIUtils.formatDurationVerbose(duration)}
+      </td>
       <td> {withNoProgress(query, {
         (query.recentProgress.map(p => 
withNumberInvalid(p.inputRowsPerSecond)).sum /
           query.recentProgress.length).formatted("%.2f") }, "NaN")}
@@ -93,8 +95,13 @@ private[ui] class StreamingQueryPage(parent: 
StreamingQueryTab)
         "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input 
/sec",
         "Avg Process /sec", "Lastest Batch")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", 
"sorttable_numeric",
+        "sorttable_numeric", "")
+      // header classes size must be equal to header row size
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, 
queryActive = true),
-        activeQueries, true, Some("activeQueries-table"), Seq(null), false))
+        activeQueries, true, Some("activeQueries-table"), headerCss, false))
     } else {
       None
     }
@@ -104,8 +111,12 @@ private[ui] class StreamingQueryPage(parent: 
StreamingQueryTab)
         "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input 
/sec",
         "Avg Process /sec", "Lastest Batch", "Error")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", 
"sorttable_numeric",
+        "sorttable_numeric", "", "")
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, 
queryActive = false),
-        inactiveQueries, true, Some("completedQueries-table"), Seq(null), 
false))
+        inactiveQueries, true, Some("completedQueries-table"), headerCss, 
false))
     } else {
       None
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org