Repository: spark Updated Branches: refs/heads/master dd72b10ab -> e3967dc5c
[SPARK-21254][WEBUI] History UI performance fixes ## What changes were proposed in this pull request? As described in JIRA ticket, History page is taking ~1min to load for cases when amount of jobs is 10k+. Most of the time is currently being spent on DOM manipulations and all additional costs implied by this (browser repaints and reflows). PR's goal is not to change any behavior but to optimize time of History UI rendering: 1. The most costly operation is setting `innerHTML` for `duration` column within a loop, which is [extremely unperformant](https://jsperf.com/jquery-append-vs-html-list-performance/24). [Refactoring ](https://github.com/criteo-forks/spark/commit/114943b21a730092aa3249b7a905b240bd46e531) this helped to get page load time **down to 10-15s** 2. Second big gain bringing page load time **down to 4s** was [was achieved](https://github.com/criteo-forks/spark/commit/f35fdcd5f129339fce75996e9242c88085a9b8ab) by detaching table's DOM before parsing it with DataTables jQuery plugin. 3. Another chunk of improvements ([1](https://github.com/criteo-forks/spark/commit/332b398db7eb3052484d436919185cb0b62b2385), [2](https://github.com/criteo-forks/spark/commit/0af596a547e3a1f2b594a83cbda1f6ef559de86b), [3](https://github.com/criteo-forks/spark/commit/235f164178a09e22306f05090ee1ff5f314a6710)) was focused on removing unnecessary DOM manipulations that in total contributed ~250ms to page load time. ## How was this patch tested? Tested by existing Selenium tests in `org.apache.spark.deploy.history.HistoryServerSuite`. Changes were also tested on Criteo's spark-2.1 fork with 20k+ number of rows in the table, reducing load time to 4s. Author: Dmitry Parfenchik <d.parfenc...@criteo.com> Author: Anna Savarin <a.sava...@criteo.com> Closes #18783 from 2ooom/history-ui-perf-fix-upstream-master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3967dc5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3967dc5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3967dc5 Branch: refs/heads/master Commit: e3967dc5cc5912155c9d67d422438162a5a9ff44 Parents: dd72b10 Author: Dmitry Parfenchik <d.parfenc...@criteo.com> Authored: Fri Aug 4 08:19:54 2017 +0100 Committer: Sean Owen <so...@cloudera.com> Committed: Fri Aug 4 08:19:54 2017 +0100 ---------------------------------------------------------------------- .../spark/ui/static/historypage-template.html | 24 ++-- .../org/apache/spark/ui/static/historypage.js | 118 +++++++++---------- 2 files changed, 73 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/e3967dc5/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index 6cff006..18d921a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -29,26 +29,30 @@ App Name </span> </th> - <th class="attemptIDSpan"> + {{#hasMultipleAttempts}} + <th> <span data-toggle="tooltip" data-placement="top" title="The attempt ID of this application since one application might be launched several times"> Attempt ID </span> </th> + {{/hasMultipleAttempts}} <th> <span data-toggle="tooltip" data-placement="top" title="Started time of this application."> Started </span> </th> - <th class="completedColumn"> + {{#showCompletedColumns}} + <th> <span data-toggle="tooltip" data-placement="top" title="The completed time of this application."> Completed </span> </th> - <th class="durationColumn"> + <th> <span data-toggle="tooltip" data-placement="top" title="The duration time of this application."> Duration </span> </th> + {{/showCompletedColumns}} <th> <span data-toggle="tooltip" data-placement="top" title="The Spark user of this application"> Spark User @@ -68,13 +72,17 @@ <tbody> {{#applications}} <tr> - <td class="rowGroupColumn"><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> - <td class="rowGroupColumn">{{name}}</td> + <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> + <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{name}}</td> {{#attempts}} - <td class="attemptIDSpan"><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> + {{#hasMultipleAttempts}} + <td><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> + {{/hasMultipleAttempts}} <td>{{startTime}}</td> - <td class="completedColumn">{{endTime}}</td> - <td class="durationColumn"><span title="{{duration}}" class="durationClass">{{duration}}</span></td> + {{#showCompletedColumns}} + <td>{{endTime}}</td> + <td><span title="{{durationMillisec}}">{{duration}}</span></td> + {{/showCompletedColumns}} <td>{{sparkUser}}</td> <td>{{lastUpdated}}</td> <td><a href="{{log}}" class="btn btn-info btn-mini">Download</a></td> http://git-wip-us.apache.org/repos/asf/spark/blob/e3967dc5/core/src/main/resources/org/apache/spark/ui/static/historypage.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 9edd3ba..aa7e860 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -48,6 +48,18 @@ function getParameterByName(name, searchString) { return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); } +function removeColumnByName(columns, columnName) { + return columns.filter(function(col) {return col.name != columnName}) +} + +function getColumnIndex(columns, columnName) { + for(var i = 0; i < columns.length; i++) { + if (columns[i].name == columnName) + return i; + } + return -1; +} + jQuery.extend( jQuery.fn.dataTableExt.oSort, { "title-numeric-pre": function ( a ) { var x = a.match(/title="*(-?[0-9\.]+)/)[1]; @@ -122,84 +134,68 @@ $(document).ready(function() { attempt["lastUpdated"] = formatDate(attempt["lastUpdated"]); attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" + (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs"; - + attempt["durationMillisec"] = attempt["duration"]; + attempt["duration"] = formatDuration(attempt["duration"]); var app_clone = {"id" : id, "name" : name, "num" : num, "attempts" : [attempt]}; array.push(app_clone); } } + if(array.length < 20) { + $.fn.dataTable.defaults.paging = false; + } var data = { "uiroot": uiRoot, - "applications": array - } + "applications": array, + "hasMultipleAttempts": hasMultipleAttempts, + "showCompletedColumns": !requestedIncomplete, + } $.get("static/historypage-template.html", function(template) { - historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); - var selector = "#history-summary-table"; + var sibling = historySummary.prev(); + historySummary.detach(); + var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); + var attemptIdColumnName = 'attemptId'; + var startedColumnName = 'started'; + var defaultSortColumn = completedColumnName = 'completed'; + var durationColumnName = 'duration'; var conf = { - "columns": [ - {name: 'first', type: "appid-numeric"}, - {name: 'second'}, - {name: 'third'}, - {name: 'fourth'}, - {name: 'fifth'}, - {name: 'sixth', type: "title-numeric"}, - {name: 'seventh'}, - {name: 'eighth'}, - {name: 'ninth'}, - ], - "columnDefs": [ - {"searchable": false, "targets": [5]} - ], - "autoWidth": false, - "order": [[ 4, "desc" ]] - }; - - var rowGroupConf = { - "rowsGroup": [ - 'first:name', - 'second:name' - ], + "columns": [ + {name: 'appId', type: "appid-numeric"}, + {name: 'appName'}, + {name: attemptIdColumnName}, + {name: startedColumnName}, + {name: completedColumnName}, + {name: durationColumnName, type: "title-numeric"}, + {name: 'user'}, + {name: 'lastUpdated'}, + {name: 'eventLog'}, + ], + "autoWidth": false, }; if (hasMultipleAttempts) { - jQuery.extend(conf, rowGroupConf); - var rowGroupCells = document.getElementsByClassName("rowGroupColumn"); - for (i = 0; i < rowGroupCells.length; i++) { - rowGroupCells[i].style='background-color: #ffffff'; - } - } - - if (!hasMultipleAttempts) { - var attemptIDCells = document.getElementsByClassName("attemptIDSpan"); - for (i = 0; i < attemptIDCells.length; i++) { - attemptIDCells[i].style.display='none'; - } - } - - if (requestedIncomplete) { - var completedCells = document.getElementsByClassName("completedColumn"); - for (i = 0; i < completedCells.length; i++) { - completedCells[i].style.display='none'; - } - - var durationCells = document.getElementsByClassName("durationColumn"); - for (i = 0; i < durationCells.length; i++) { - durationCells[i].style.display='none'; - } + conf.rowsGroup = [ + 'appId:name', + 'appName:name' + ]; } else { - var durationCells = document.getElementsByClassName("durationClass"); - for (i = 0; i < durationCells.length; i++) { - var timeInMilliseconds = parseInt(durationCells[i].title); - durationCells[i].innerHTML = formatDuration(timeInMilliseconds); - } + conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); } - if ($(selector.concat(" tr")).length < 20) { - $.extend(conf, {paging: false}); + var defaultSortColumn = completedColumnName; + if (requestedIncomplete) { + defaultSortColumn = startedColumnName; + conf.columns = removeColumnByName(conf.columns, completedColumnName); + conf.columns = removeColumnByName(conf.columns, durationColumnName); } - - $(selector).DataTable(conf); + conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; + conf.columnDefs = [ + {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} + ]; + historySummary.append(apps); + apps.DataTable(conf); + sibling.after(historySummary); $('#history-summary [data-toggle="tooltip"]').tooltip(); }); }); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
