[GitHub] spark issue #23088: [SPARK-26119][CORE][WEBUI]Task summary table should cont...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/23088 PR : https://github.com/apache/spark/pull/23205 for displaying the message on empty data in task summary metrics table. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark issue #23205: [SPARK-26253][WebUI] : Task Summary Metrics Table on Sta...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/23205 cc @tgravescs @abellina --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #23205: [SPARK-26253][WebUI] : Task Summary Metrics Table...
GitHub user pgandhi999 opened a pull request: https://github.com/apache/spark/pull/23205 [SPARK-26253][WebUI] : Task Summary Metrics Table on Stage Page shows empty table when no data is present Task Summary Metrics Table on Stage Page shows empty table when no data is present instead of showing a message. ## What changes were proposed in this pull request? Added a custom message to show on the task summary metrics table as well as executor summary table when no data is present. ## How was this patch tested? **Before:**  **After:** https://user-images.githubusercontent.com/8190/49362019-8699f780-f703-11e8-93e1-d02df6572923.png;> You can merge this pull request into a Git repository by running: $ git pull https://github.com/pgandhi999/spark SPARK-26253 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/spark/pull/23205.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #23205 commit e5f5774a12e2e1b67c6089269900fd4f3cbe8ca1 Author: pgandhi Date: 2018-12-03T08:11:59Z [SPARK-26253] : Task Summary Metrics Table on Stage Page shows empty table when no data is present Added a custom message to show on the task summary metrics table as well as executor summary table when no data is present. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark issue #23160: [SPARK-26196][WebUI] Total tasks title in the stage page...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/23160 LGTM --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark issue #23088: [SPARK-26119][CORE][WEBUI]Task summary table should cont...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/23088 Hi @shahidki31 ,thank you for pointing that out. Will put up a fix for it shortly. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #23160: [SPARK-26196][WebUI] Total tasks title in the sta...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/23160#discussion_r238055413
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -661,8 +662,8 @@ $(document).ready(function () {
{data : "launchTime", name: "Launch Time", render:
formatDate},
{
data : function (row, type) {
-if (row.duration) {
-return type === 'display' ?
formatDuration(row.duration) : row.duration;
+if (row.taskMetrics &&
row.taskMetrics.executorRunTime) {
+return type === 'display' ?
formatDuration(row.taskMetrics.executorRunTime) :
row.taskMetrics.executorRunTime;
--- End diff --
Ok, that makes sense. However, you also need to update the search filter
method to not search on duration in StagesResource.scala here:
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala#L213.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #23160: [SPARK-26196][WebUI] Total tasks title in the sta...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/23160#discussion_r238055402
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -661,8 +662,8 @@ $(document).ready(function () {
{data : "launchTime", name: "Launch Time", render:
formatDate},
{
data : function (row, type) {
-if (row.duration) {
-return type === 'display' ?
formatDuration(row.duration) : row.duration;
+if (row.taskMetrics &&
row.taskMetrics.executorRunTime) {
+return type === 'display' ?
formatDuration(row.taskMetrics.executorRunTime) :
row.taskMetrics.executorRunTime;
--- End diff --
Ok, that makes sense. However, you also need to update the search filter
method to not search on duration in StagesResource.scala here:
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala#L213.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #23160: [SPARK-26196][WebUI] Total tasks title in the sta...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/23160#discussion_r237843584
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -610,7 +610,8 @@ $(document).ready(function () {
$("#accumulator-table").DataTable(accumulatorConf);
// building tasks table that uses server side functionality
-var totalTasksToShow = responseBody.numCompleteTasks +
responseBody.numActiveTasks;
+var totalTasksToShow = responseBody.numCompleteTasks +
responseBody.numActiveTasks +
--- End diff --
Ah that makes sense. Looks good to me, however could you also please verify
whether the data in tasks table reflects information about the failed / killed
tasks as well, as I am passing the value of that variable in the ajax request
while fetching data for task tables. Thank you for catching that.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark issue #23160: [SPARK-26196]Total tasks title in the stage page is inco...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/23160 What I meant was like in the screenshot posted above, whether the task table has 7 entries and they all show accurate information. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r235184492
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -92,6 +92,14 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
val parameterTaskSortDesc =
UIUtils.stripXSS(request.getParameter("task.desc"))
val parameterTaskPageSize =
UIUtils.stripXSS(request.getParameter("task.pageSize"))
--- End diff --
Done, https://issues.apache.org/jira/browse/SPARK-26130.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r235184400
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -259,7 +278,8 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
}
- def makeTimeline(tasks: Seq[TaskData], currentTime: Long): Seq[Node] = {
+ def makeTimeline(tasks: Seq[TaskData], currentTime: Long, page: Int,
pageSize: Int,
+totalPages: Int, stageId: Int, stageAttemptId: Int, totalTasks: Int):
Seq[Node] = {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r235184320
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: Option[String] = None
+ var filteredRecords = totalRecords
+ // The datatables client API sends a list of query parameters to the
server which contain
+ // information like the columns to be sorted, search value typed by
the user in the search
+ // box, pagination index etc. For more information on these query
parameters,
+ // refer https://datatables.net/manual/server-side.
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+isSearch = true
+searchValue = Some(uriQueryParameters.getFirst("search[value]"))
+ }
+ val _tasksToShow: Seq[TaskData] = doPagination(uriQueryParameters,
stageId, stageAttemptId,
+isSearch, totalRecords.toInt)
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow,
searchValue.get)
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r235184363
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -315,187 +241,22 @@ private[ui] class StagePage(parent: StagesTab,
store: AppStatusStore) extends We
}
-val metricsSummary = store.taskSummary(stageData.stageId,
stageData.attemptId,
- Array(0, 0.25, 0.5, 0.75, 1.0))
-
-val summaryTable = metricsSummary.map { metrics =>
- def timeQuantiles(data: IndexedSeq[Double]): Seq[Node] = {
-data.map { millis =>
- {UIUtils.formatDuration(millis.toLong)}
-}
- }
-
- def sizeQuantiles(data: IndexedSeq[Double]): Seq[Node] = {
-data.map { size =>
- {Utils.bytesToString(size.toLong)}
-}
- }
-
- def sizeQuantilesWithRecords(
- data: IndexedSeq[Double],
- records: IndexedSeq[Double]) : Seq[Node] = {
-data.zip(records).map { case (d, r) =>
- {s"${Utils.bytesToString(d.toLong)} / ${r.toLong}"}
-}
- }
-
- def titleCell(title: String, tooltip: String): Seq[Node] = {
-
-
-{title}
-
-
- }
-
- def simpleTitleCell(title: String): Seq[Node] = {title}
-
- val deserializationQuantiles = titleCell("Task Deserialization Time",
-ToolTips.TASK_DESERIALIZATION_TIME) ++
timeQuantiles(metrics.executorDeserializeTime)
-
- val serviceQuantiles = simpleTitleCell("Duration") ++
timeQuantiles(metrics.executorRunTime)
-
- val gcQuantiles = titleCell("GC Time", ToolTips.GC_TIME) ++
timeQuantiles(metrics.jvmGcTime)
-
- val serializationQuantiles = titleCell("Result Serialization Time",
-ToolTips.RESULT_SERIALIZATION_TIME) ++
timeQuantiles(metrics.resultSerializationTime)
-
- val gettingResultQuantiles = titleCell("Getting Result Time",
ToolTips.GETTING_RESULT_TIME) ++
-timeQuantiles(metrics.gettingResultTime)
-
- val peakExecutionMemoryQuantiles = titleCell("Peak Execution Memory",
-ToolTips.PEAK_EXECUTION_MEMORY) ++
sizeQuantiles(metrics.peakExecutionMemory)
-
- // The scheduler delay includes the network delay to send the task
to the worker
- // machine and to send back the result (but not the time to fetch
the task result,
- // if it needed to be fetched from the block manager on the worker).
- val schedulerDelayQuantiles = titleCell("Scheduler Delay",
ToolTips.SCHEDULER_DELAY) ++
-timeQuantiles(metrics.schedulerDelay)
-
- def inputQuantiles: Seq[Node] = {
-simpleTitleCell("Input Size / Records") ++
- sizeQuantilesWithRecords(metrics.inputMetrics.bytesRead,
metrics.inputMetrics.recordsRead)
- }
-
- def outputQuantiles: Seq[Node] = {
-simpleTitleCell("Output Size / Records") ++
- sizeQuantilesWithRecords(metrics.outputMetrics.bytesWritten,
-metrics.outputMetrics.recordsWritten)
- }
-
- def shuffleReadBlockedQuantiles: Seq[Node] = {
-titleCell("Shuffle Read Blocked Time",
ToolTips.SHUFFLE_READ_BLOCKED_TIME) ++
- timeQuantiles(metrics.shuffleReadMetrics.fetchWaitTime)
- }
-
- def shuffleReadTotalQuantiles: Seq[Node] = {
-titleCell("Shuffle Read Size / Records", ToolTips.SHUFFLE_READ) ++
- sizeQuantilesWithRecords(metrics.shuffleReadMetrics.readBytes,
-metrics.shuffleReadMetrics.readRecords)
- }
-
- def shuffleReadRemoteQuantiles: Seq[Node] = {
-titleCell("Shuffle Remote Reads",
ToolTips.SHUFFLE_READ_REMOTE_SIZE) ++
- sizeQuantiles(metrics.shuffleReadMetrics.remoteBytesRead)
- }
-
- def shuffleWriteQuantiles: Seq[Node] = {
-simpleTitleCell("Shuffle Write Size / Records") ++
- sizeQuantilesWithRecords(metrics.shuffleWriteMetrics.writeBytes,
-metrics.shuffleWriteMetrics.writeRecords)
- }
-
- def memoryBytesSpilledQuantiles: Seq[Node] = {
-simpleTitleCell("Shuffle spill (memory)") ++
sizeQuantiles(metrics.memoryBytesSpilled)
- }
-
- def diskBytesSpilledQuantiles: Seq[Node] = {
-simpleTitleCell("Shuffle spill (disk)") ++
sizeQuantiles(metrics.diskBytesSpilled)
- }
-
- val listings: Seq[Seq[Node]] = Seq(
-{serviceQuantiles},
-{sche
[GitHub] spark issue #21688: [SPARK-21809] : Change Stage Page to use datatables to s...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/21688 Hi @gengliangwang , your comment does make sense. However, the arrows are positioned according to the width of the individual column which gets automatically set by datatables during runtime. To circumvent the issue, I added cell borders to the entire datatable, so the arrow will now appear within the borders of the corresponding clicked column. Thank you for the observation. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232838411
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -162,26 +165,29 @@ private[v1] class StagesResource extends
BaseAppResource {
// Performs pagination on the server side
def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
-val queryParams = queryParameters.keySet()
var columnNameToSort = queryParameters.getFirst("columnNameToSort")
+// Sorting on Logs column will default to Index column sort
if (columnNameToSort.equalsIgnoreCase("Logs")) {
columnNameToSort = "Index"
}
val isAscendingStr = queryParameters.getFirst("order[0][dir]")
var pageStartIndex = 0
var pageLength = totalRecords
+// We fetch only the desired rows upto the specified page length for
all cases except when a
+// search query is present, in that case, we need to fetch all the
rows to perform the search
+// on the entire table
if (!isSearch) {
pageStartIndex = queryParameters.getFirst("start").toInt
pageLength = queryParameters.getFirst("length").toInt
}
-return withUI(_.store.taskList(stageId, stageAttemptId,
pageStartIndex, pageLength,
+withUI(_.store.taskList(stageId, stageAttemptId, pageStartIndex,
pageLength,
indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
}
// Filters task list based on search parameter
def filterTaskList(
taskDataList: Seq[TaskData],
-searchValue: String): Seq[TaskData] = {
+searchValue: String): Option[Seq[TaskData]] = {
--- End diff --
Have reverted the particular code change. Thank you.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232838308
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -117,35 +117,38 @@ private[v1] class StagesResource extends
BaseAppResource {
val uriQueryParameters = uriInfo.getQueryParameters(true)
val totalRecords = uriQueryParameters.getFirst("numTasks")
var isSearch = false
- var searchValue: String = null
+ var searchValue: Option[String] = None
var filteredRecords = totalRecords
- var _tasksToShow: Seq[TaskData] = null
// The datatables client API sends a list of query parameters to the
server which contain
// information like the columns to be sorted, search value typed by
the user in the search
// box, pagination index etc. For more information on these query
parameters,
// refer https://datatables.net/manual/server-side.
if (uriQueryParameters.getFirst("search[value]") != null &&
uriQueryParameters.getFirst("search[value]").length > 0) {
-_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
- totalRecords.toInt)
isSearch = true
-searchValue = uriQueryParameters.getFirst("search[value]")
- } else {
-_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
- totalRecords.toInt)
+searchValue = try {
--- End diff --
Yep, makes sense, have changed the code.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809301
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809245
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ // The datatables client API sends a list of query parameters to the
server which contain
+ // information like the columns to be sorted, search value typed by
the user in the search
+ // box, pagination index etc. For more information on these query
parameters,
+ // refer https://datatables.net/manual/server-side.
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
+var columnNameToSort = queryParameters.getFirst("columnNameToSort")
+if (columnNameToSort.equalsIgnoreCase("Logs")) {
+ columnNameToSort = "Index"
+}
+val isAscendingStr = queryParameters.getFirst("order[0][dir]")
+var pageStartIndex = 0
+var pageLength = totalRecords
+if (!isSearch) {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809385
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ // The datatables client API sends a list of query parameters to the
server which contain
+ // information like the columns to be sorted, search value typed by
the user in the search
+ // box, pagination index etc. For more information on these query
parameters,
+ // refer https://datatables.net/manual/server-side.
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
+var columnNameToSort = queryParameters.getFirst("columnNameToSort")
+if (columnNameToSort.equalsIgnoreCase("Logs")) {
+ columnNameToSort = "Index"
+}
+val isAscendingStr = queryParameters.getFirst("order[0][dir]")
+var pageStartIndex = 0
+var pageLength = totalRecords
+if (!isSearch) {
+ pageStartIndex = queryParameters.getFirst("start").toInt
+ pageLength = queryParameters.getFirst("length").toInt
+}
+return withUI(_.store.taskList(stageId, stageAttemptId,
pageStartIndex, pageLength,
+ indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
+ }
+
+ // Filters task list based on search parameter
+ def filterTaskList(
+taskDataList: Seq[TaskData],
+searchValue: String): Seq[TaskData] = {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809338
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ // The datatables client API sends a list of query parameters to the
server which contain
+ // information like the columns to be sorted, search value typed by
the user in the search
+ // box, pagination index etc. For more information on these query
parameters,
+ // refer https://datatables.net/manual/server-side.
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809274
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r232809213
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,120 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ // The datatables client API sends a list of query parameters to the
server which contain
+ // information like the columns to be sorted, search value typed by
the user in the search
+ // box, pagination index etc. For more information on these query
parameters,
+ // refer https://datatables.net/manual/server-side.
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r231932184 --- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js --- @@ -0,0 +1,960 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var blockUICount = true; --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231932218
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,960 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = true;
+
+$(document).ajaxStop(function () {
+if (blockUICount) {
+$.unblockUI();
+blockUICount = false;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231932079
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231931747
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
--- End diff --
Yep makes sense, have removed it. Thank you.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231605935
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,965 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231605818
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,965 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r231605894
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,965 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102715
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,959 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102675
--- Diff: core/src/main/resources/org/apache/spark/ui/static/webui.css ---
@@ -260,4 +260,45 @@ a.expandbutton {
.paginate_button.active > a {
color: #99;
text-decoration: underline;
+}
+
+.title-table {
+ clear: left;
+ display: inline-block;
+}
+
+.table-dataTable {
+ width="100%";
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102603
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,959 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102635
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,959 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102497
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,959 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r230102562
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,959 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark issue #21688: [SPARK-21809] : Change Stage Page to use datatables to s...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/21688 @tgravescs Have fixed the bug where the page used to hang on clicking on the "Summary Metrics for 2 Completed Tasks" link and then refreshing the page. Thank you for pointing that out. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r229115481
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,944 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r229115449
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,944 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r229115504
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,944 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var indexOfProxy = words.indexOf("proxy");
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (indexOfProxy > 0) {
+var appId = words[indexOfProxy + 1];
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+var indexOfHistory = words.indexOf("history");
+if (indexOfHistory > 0) {
+var appId = words[indexOfHistory + 1];
+var appAttemptId = words[indexOfHistory + 2];
+var newBaseURI = words.slice(0,
words.indexOf("history")).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSp
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r227515255
--- Diff: core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
---
@@ -547,6 +519,24 @@ private[spark] class AppStatusStore(
store.close()
}
+ def interceptAndModifyTaskData(taskDataWrapper: TaskDataWrapper) :
v1.TaskData = {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r227515191
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -52,22 +52,20 @@ $.extend( $.fn.dataTable.ext.type.order, {
function stageEndPoint(appId) {
var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
var urlArray = urlRegex.exec(document.baseURI);
-var ind = urlArray.indexOf("proxy");
+var indexOfProxy = urlArray.indexOf("proxy");
var queryString = document.baseURI.split('?');
var words = document.baseURI.split('/');
var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
-if (ind > 0) {
+if (indexOfProxy > 0) {
var appId = urlArray[2];
-var indexOfProxy = words.indexOf("proxy");
-var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+var newBaseURI = words.slice(0, words.indexOf("proxy") +
2).join('/');
return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
}
-ind = urlArray.indexOf("history");
-if (ind > 0) {
+var indexOfHistory = urlArray.indexOf("history");
+if (indexOfHistory > 0) {
var appId = urlArray[2];
-var appAttemptId = urlArray[ind + 2];
-var indexOfHistory = words.indexOf("history");
-var newBaseURI = words.slice(0, indexOfHistory).join('/');
+var appAttemptId = urlArray[indexOfHistory + 2];
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r227515143
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -52,22 +52,20 @@ $.extend( $.fn.dataTable.ext.type.order, {
function stageEndPoint(appId) {
var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
var urlArray = urlRegex.exec(document.baseURI);
-var ind = urlArray.indexOf("proxy");
+var indexOfProxy = urlArray.indexOf("proxy");
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #22806: [SPARK-25250] : On successful completion of a tas...
GitHub user pgandhi999 opened a pull request: https://github.com/apache/spark/pull/22806 [SPARK-25250] : On successful completion of a task attempt on a parti… …tion id, kill other running task attempts on that same partition We recently had a scenario where a race condition occurred when a task from previous stage attempt just finished before new attempt for the same stage was created due to fetch failure, so the new task created in the second attempt on the same partition id was retrying multiple times due to TaskCommitDenied Exception without realizing that the task in earlier attempt was already successful. For example, consider a task with partition id 9000 and index 9000 running in stage 4.0. We see a fetch failure so thus, we spawn a new stage attempt 4.1. Just within this timespan, the above task completes successfully, thus, marking the partition id 9000 as complete for 4.0. However, as stage 4.1 has not yet been created, the taskset info for that stage is not available to the TaskScheduler so, naturally, the partition id 9000 has not been marked completed for 4.1. Stage 4.1 now spawns task with index 2000 on the same partition id 9000. This task fails due to CommitDeniedException and since, it does not see the corresponding partition id as been marked successful, it keeps retrying multiple times until the job finally succeeds. It doesn't cause any job failures because the DAG scheduler is tracking the partitions separate from the task set managers. ## What changes were proposed in this pull request? The fix that this PR addresses is as follows: Whenever any Result Task gets successfully completed, we simply mark the corresponding partition id as completed in all attempts for that particular stage. As a result, we do not see any Killed tasks due to TaskCommitDenied Exceptions showing up in the UI. Also, since, the method defined uses hash maps and arrays for efficient searching and processing, so as a result, it's time complexity is not related to the number of tasks, hence, it is also efficient. You can merge this pull request into a Git repository by running: $ git pull https://github.com/pgandhi999/spark SPARK-25250 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/spark/pull/22806.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #22806 commit 5ad6efde552fc541330386f5124c6b4055f82256 Author: pgandhi Date: 2018-10-23T14:31:27Z [SPARK-25250] : On successful completion of a task attempt on a partition id, kill other running task attempts on that same partition The fix that this PR addresses is as follows: Whenever any Result Task gets successfully completed, we simply mark the corresponding partition id as completed in all attempts for that particular stage. As a result, we do not see any Killed tasks due to TaskCommitDenied Exceptions showing up in the UI. Also, since, the method defined uses hash maps and arrays for efficient searching and processing, so as a result, it's time complexity is not related to the number of tasks, hence, it is also efficient. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225964304
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r225963998 --- Diff: core/src/main/scala/org/apache/spark/status/LiveEntity.scala --- @@ -341,7 +341,9 @@ private class LiveExecutorStageSummary( metrics.shuffleWriteMetrics.recordsWritten, metrics.memoryBytesSpilled, metrics.diskBytesSpilled, - isBlacklisted) + isBlacklisted, --- End diff -- Have made the required changes and tested them. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225559330
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,159 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
+var columnToSort = 0
+if (queryParams.contains("order[0][column]")) {
+ columnToSort = queryParameters.getFirst("order[0][column]").toInt
+}
+var columnNameToSort = queryParameters.getFirst("columns[" +
columnToSort + "][name]")
+if (columnNameToSort.equalsIgnoreCase("Logs")) {
+ columnNameToSort = "Index"
+ columnToSort = 0
+}
+val isAscendingStr = queryParameters.getFirst("order[0][dir]")
+var pageStartIndex = 0
+var pageLength = totalRecords
+if (!isSearch) {
+ pageStartIndex = queryParameters.getFirst("start").toInt
+ pageLength = queryParameters.getFirst("length").toInt
+}
+return withUI(_.store.taskList(stageId, stageAttemptId,
pageStartIndex, pageLength,
+ indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
+ }
+
+ // Filters task list based on search parameter
+ def filterTaskList(
+taskDataList: Seq[TaskData],
+searchValue: String): Seq[TaskData] = {
+val defaultOptionString: String = "d"
+// The task metrics dummy object below has been added to avoid
throwing exception in cases
+// when task metrics for a particular task do not exist as of yet
+val dummyTaskMetrics: TaskMetrics = new TaskMetrics(0, 0, 0, 0, 0, 0,
0, 0, 0, 0,
--- End diff --
ok, thanks for clarifying @abellina , fixed it.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558995
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,124 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558870
--- Diff: core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
---
@@ -349,7 +349,23 @@ private[spark] class AppStatusStore(
def taskList(stageId: Int, stageAttemptId: Int, maxTasks: Int):
Seq[v1.TaskData] = {
val stageKey = Array(stageId, stageAttemptId)
store.view(classOf[TaskDataWrapper]).index("stage").first(stageKey).last(stageKey).reverse()
- .max(maxTasks).asScala.map(_.toApi).toSeq.reverse
+ .max(maxTasks).asScala.map { taskDataWrapper =>
+ val taskDataOld: v1.TaskData = taskDataWrapper.toApi
+ val executorLogs: Option[Map[String, String]] = try {
+Some(executorSummary(taskDataOld.executorId).executorLogs)
+ } catch {
+case e: NoSuchElementException => e.getMessage
+ None
+ }
+ new v1.TaskData(taskDataOld.taskId, taskDataOld.index,
+taskDataOld.attempt, taskDataOld.launchTime,
taskDataOld.resultFetchStart,
+taskDataOld.duration, taskDataOld.executorId, taskDataOld.host,
taskDataOld.status,
+taskDataOld.taskLocality, taskDataOld.speculative,
taskDataOld.accumulatorUpdates,
+taskDataOld.errorMessage, taskDataOld.taskMetrics,
+executorLogs.getOrElse(Map[String, String]()),
+AppStatusUtils.schedulerDelay(taskDataOld),
+AppStatusUtils.gettingResultTime(taskDataOld))
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558750
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,872 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (ind > 0) {
+var appId = urlArray[2];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serializ
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558583
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,872 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (ind > 0) {
+var appId = urlArray[2];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serializ
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558809
--- Diff: core/src/main/resources/org/apache/spark/ui/static/utils.js ---
@@ -74,3 +74,99 @@ function getTimeZone() {
return new Date().toString().match(/\((.*)\)/)[1];
}
}
+
+function formatLogsCells(execLogs, type) {
+ if (type !== 'display') return Object.keys(execLogs);
+ if (!execLogs) return;
+ var result = '';
+ $.each(execLogs, function (logName, logUrl) {
+result += '' + logName + ''
+ });
+ return result;
+}
+
+function getStandAloneAppId(cb) {
+ var words = document.baseURI.split('/');
+ var ind = words.indexOf("proxy");
+ if (ind > 0) {
+var appId = words[ind + 1];
+cb(appId);
+return;
+ }
+ ind = words.indexOf("history");
+ if (ind > 0) {
+var appId = words[ind + 1];
+cb(appId);
+return;
+ }
+ //Looks like Web UI is running in standalone mode
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r225558670 --- Diff: core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html --- @@ -0,0 +1,124 @@ + + +
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558381
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,872 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (ind > 0) {
+var appId = urlArray[2];
+var indexOfProxy = words.indexOf("proxy");
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558473
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,872 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+if (ind > 0) {
+var appId = urlArray[2];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var indexOfHistory = words.indexOf("history");
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r225558309
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,872 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r225558239 --- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala --- @@ -290,17 +210,11 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We desc = taskSortDesc, store = parent.store ) - (_taskTable, _taskTable.table(page)) + _taskTable.table(page) --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r223746447
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
--- End diff --
Here is the link to the one that I used: https://regex101.com/r/BVAlE0/2/
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark issue #21809: [SPARK-24851][UI] Map a Stage ID to it's Associated Job ...
Github user pgandhi999 commented on the issue: https://github.com/apache/spark/pull/21809 test this please --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r223127069 --- Diff: core/src/main/scala/org/apache/spark/status/LiveEntity.scala --- @@ -341,7 +341,9 @@ private class LiveExecutorStageSummary( metrics.shuffleWriteMetrics.recordsWritten, metrics.memoryBytesSpilled, metrics.diskBytesSpilled, - isBlacklisted) + isBlacklisted, + null, --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r223127036
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
--- End diff --
I created this regex through regex testers available online and this is the
pattern that I got. I agree with you that it might not improve the function.
Let me see what I can do and get back on this one.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r223126675
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,869 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r223126640 --- Diff: core/src/main/scala/org/apache/spark/status/storeTypes.scala --- @@ -283,7 +283,10 @@ private[spark] class TaskDataWrapper( speculative, accumulatorUpdates, errorMessage, - metrics) + metrics, + null, + 0L, --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r223126615
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,159 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
+var columnToSort = 0
+if (queryParams.contains("order[0][column]")) {
+ columnToSort = queryParameters.getFirst("order[0][column]").toInt
+}
+var columnNameToSort = queryParameters.getFirst("columns[" +
columnToSort + "][name]")
+if (columnNameToSort.equalsIgnoreCase("Logs")) {
+ columnNameToSort = "Index"
+ columnToSort = 0
+}
+val isAscendingStr = queryParameters.getFirst("order[0][dir]")
+var pageStartIndex = 0
+var pageLength = totalRecords
+if (!isSearch) {
+ pageStartIndex = queryParameters.getFirst("start").toInt
+ pageLength = queryParameters.getFirst("length").toInt
+}
+return withUI(_.store.taskList(stageId, stageAttemptId,
pageStartIndex, pageLength,
+ indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
+ }
+
+ // Filters task list based on search parameter
+ def filterTaskList(
+taskDataList: Seq[TaskData],
+searchValue: String): Seq[TaskData] = {
+val defaultOptionString: String = "d"
+// The task metrics dummy object below has been added to avoid
throwing exception in cases
+// when task metrics for a particular task do not exist as of yet
+val dummyTaskMetrics: TaskMetrics = new TaskMetrics(0, 0, 0, 0, 0, 0,
0, 0, 0, 0,
--- End diff --
I did not get your comment clearly here, do you mean that I should have all
task metric filters in a separate function or should I wrap dummytaskmetrics in
an Option? Can you please elaborate a little? Thank you.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r223124830
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +103,159 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ // This api needs to stay formatted exactly as it is below, since, it is
being used by the
+ // datatables for the stages page.
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
+@Context uriInfo: UriInfo):
+ HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var filteredRecords = totalRecords
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, true,
+ totalRecords.toInt)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId, false,
+ totalRecords.toInt)
+ }
+ val ret = new HashMap[String, Object]()
+ if (_tasksToShow.nonEmpty) {
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = filterTaskList(_tasksToShow, searchValue)
+ filteredRecords = filteredTaskList.length.toString
+ if (filteredTaskList.length > 0) {
+val pageStartIndex = uriQueryParameters.getFirst("start").toInt
+val pageLength = uriQueryParameters.getFirst("length").toInt
+ret.put("aaData", filteredTaskList.slice(pageStartIndex,
pageStartIndex + pageLength))
+ } else {
+ret.put("aaData", filteredTaskList)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ } else {
+ret.put("aaData", _tasksToShow)
+ }
+ ret.put("recordsTotal", totalRecords)
+ ret.put("recordsFiltered", filteredRecords)
+ ret
+}
+ }
+
+ // Performs pagination on the server side
+ def doPagination(queryParameters: MultivaluedMap[String, String],
stageId: Int,
+stageAttemptId: Int, isSearch: Boolean, totalRecords: Int):
Seq[TaskData] = {
+val queryParams = queryParameters.keySet()
+var columnToSort = 0
+if (queryParams.contains("order[0][column]")) {
+ columnToSort = queryParameters.getFirst("order[0][column]").toInt
+}
+var columnNameToSort = queryParameters.getFirst("columns[" +
columnToSort + "][name]")
+if (columnNameToSort.equalsIgnoreCase("Logs")) {
+ columnNameToSort = "Index"
+ columnToSort = 0
+}
+val isAscendingStr = queryParameters.getFirst("order[0][dir]")
+var pageStartIndex = 0
+var pageLength = totalRecords
+if (!isSearch) {
+ pageStartIndex = queryParameters.getFirst("start").toInt
+ pageLength = queryParameters.getFirst("length").toInt
+}
+return withUI(_.store.taskList(stageId, stageAttemptId,
pageStartIndex, pageLength,
+ indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
+ }
+
+ // Filters task list based on search parameter
+ def filterTaskList(
+taskDataList: Seq[TaskData],
+searchValue: String): Seq[TaskData] = {
+val defaultOptionString: String = "d"
+// The task metrics dummy object below has been added to avoid
throwing exception in cases
+// when task metrics for a particular task do not exist as of yet
+val dummyTaskMetrics: TaskMetrics = new TaskMetrics(0, 0, 0, 0, 0, 0,
0, 0, 0, 0,
+ new InputMetrics(0, 0), new OutputMetrics(0, 0),
+ new ShuffleReadMetrics(0, 0, 0, 0, 0, 0, 0), new
ShuffleWriteMetrics(0, 0, 0))
+val searchValueLowerCase = searchValue.toLowerCase
[GitHub] spark pull request #21809: [SPARK-24851][UI] Map a Stage ID to it's Associat...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21809#discussion_r223124683
--- Diff: core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
---
@@ -112,10 +112,13 @@ private[spark] class AppStatusStore(
}
}
- def stageAttempt(stageId: Int, stageAttemptId: Int, details: Boolean =
false): v1.StageData = {
+ def stageAttempt(stageId: Int, stageAttemptId: Int,
+ details: Boolean = false): (v1.StageData, Seq[Int]) = {
val stageKey = Array(stageId, stageAttemptId)
-val stage = store.read(classOf[StageDataWrapper], stageKey).info
-if (details) stageWithDetails(stage) else stage
+val stageDataWrapper = store.read(classOf[StageDataWrapper], stageKey)
+val stage = if (details) stageWithDetails(stageDataWrapper.info) else
stageDataWrapper.info
+val jobIds = stageDataWrapper.jobIds
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733733
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222734361
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,906 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222734020
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,906 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733950
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,906 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733891
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733377
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733306
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222733191
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222732441
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r222732070
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,905 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var blockUICount = 0;
+
+$(document).ajaxStop(function () {
+if (blockUICount == 0) {
+$.unblockUI();
+blockUICount++;
+}
+});
+
+$(document).ajaxStart(function () {
+if (blockUICount == 0) {
+$.blockUI({message: 'Loading Stage Page...'});
+}
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"duration-pre": ConvertDurationString,
+
+"duration-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"duration-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g. (history)
https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4=1
+// e.g. (proxy)
https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4=1
+function stageEndPoint(appId) {
+var urlRegex =
/https\:\/\/[^\/]+\/([^\/]+)\/([^\/]+)\/([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?\/?([^\/]+)?/gm;
+var urlArray = urlRegex.exec(document.baseURI);
+var ind = urlArray.indexOf("proxy");
+var queryString = document.baseURI.split('?');
+var words = document.baseURI.split('/');
+if (ind > 0) {
+var appId = urlArray[2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfProxy = words.indexOf("proxy");
+var newBaseURI = words.slice(0, indexOfProxy + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = urlArray.indexOf("history");
+if (ind > 0) {
+var appId = urlArray[2];
+var appAttemptId = urlArray[ind + 2];
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+var indexOfHistory = words.indexOf("history");
+var newBaseURI = words.slice(0, indexOfHistory).join('/');
+if (isNaN(appAttemptId) || appAttemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
appAttemptId + "/stages/" + stageId;
+}
+}
+var stageId = queryString[1].split("&").filter(word =>
word.includes("id="))[0].split("=")[1];
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size /
[GitHub] spark pull request #21809: [SPARK-24851][UI] Map a Stage ID to it's Associat...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21809#discussion_r221295745
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -182,6 +197,15 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
{Utils.bytesToString(stageData.diskBytesSpilled)}
}}
+ {if (!stageJobIds.isEmpty) {
--- End diff --
Since, we are returning from the code if it is None, it should never reach
there.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21809: [SPARK-24851][UI] Map a Stage ID to it's Associat...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21809#discussion_r221295304
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -105,15 +105,30 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
val stageAttemptId = parameterAttempt.toInt
val stageHeader = s"Details for Stage $stageId (Attempt
$stageAttemptId)"
-val stageData = parent.store
- .asOption(parent.store.stageAttempt(stageId, stageAttemptId, details
= false))
- .getOrElse {
-val content =
-
-No information to display for Stage {stageId} (Attempt
{stageAttemptId})
-
-return UIUtils.headerSparkPage(request, stageHeader, content,
parent)
- }
+var stageDataTuple: Option[Tuple2[StageData, Seq[Int]]] = try {
--- End diff --
Makes sense, simplified the code and tested it. Looks good. Thank you.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r221263645
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -40,8 +42,29 @@ private[v1] class StagesResource extends BaseAppResource
{
@PathParam("stageId") stageId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean):
Seq[StageData] = {
withUI { ui =>
- val ret = ui.store.stageData(stageId, details = details)
+ var ret = ui.store.stageData(stageId, details = details)
if (ret.nonEmpty) {
+// Some of the data that we want to display for the tasks like
executorLogs,
+// schedulerDelay etc. comes from other sources, thus, we
basically add them to the
+// executor and task data object before passing them to the client.
+for (r <- ret) {
+ for (execId <- r.executorSummary.get.keys.toArray) {
+val executorLogs =
ui.store.executorSummary(execId).executorLogs
+val hostPort = ui.store.executorSummary(execId).hostPort
+val taskDataArray = r.tasks.get.keys.toArray
+var execStageSummary = r.executorSummary.get.get(execId).get
+execStageSummary.executorLogs = executorLogs
+execStageSummary.hostPort = hostPort
+for (taskData <- taskDataArray) {
+ var taskDataObject = r.tasks.get.get(taskData).get
+ taskDataObject.executorLogs = executorLogs
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220264300
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +124,87 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
@Context uriInfo: UriInfo):
+ util.HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = ui.store.taskList(stageId, stageAttemptId, 0,
totalRecords.toInt,
+ indexName("Index"), true)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId)
+ }
+ if (_tasksToShow.nonEmpty) {
+val iterator = _tasksToShow.iterator
+while(iterator.hasNext) {
+ val t1: TaskData = iterator.next()
+ val execId = t1.executorId
+ val executorLogs = ui.store.executorSummary(execId).executorLogs
+ t1.executorLogs = executorLogs
+ t1.schedulerDelay = AppStatusUtils.schedulerDelay(t1)
+ t1.gettingResultTime = AppStatusUtils.gettingResultTime(t1)
+}
+val ret = new util.HashMap[String, Object]()
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = ui.store.filterTaskList(_tasksToShow,
searchValue)
+ if (filteredTaskList.length > 0) {
+ret.put("aaData", filteredTaskList)
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId)
+val iterator = _tasksToShow.iterator
+while(iterator.hasNext) {
+ val t1: TaskData = iterator.next()
+ val execId = t1.executorId
+ val executorLogs =
ui.store.executorSummary(execId).executorLogs
+ t1.executorLogs = executorLogs
+ t1.schedulerDelay = AppStatusUtils.schedulerDelay(t1)
+ t1.gettingResultTime = AppStatusUtils.gettingResultTime(t1)
+}
+ret.put("aaData", _tasksToShow)
+ }
+} else {
+ ret.put("aaData", _tasksToShow)
+}
+ret.put("recordsTotal", totalRecords)
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220264258
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +124,87 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
@Context uriInfo: UriInfo):
+ util.HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = ui.store.taskList(stageId, stageAttemptId, 0,
totalRecords.toInt,
+ indexName("Index"), true)
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r220264190 --- Diff: core/src/main/scala/org/apache/spark/status/AppStatusStore.scala --- @@ -391,6 +391,41 @@ private[spark] class AppStatusStore( ordered.skip(offset).max(length).asScala.map(_.toApi).toSeq } + // Filters task list based on search parameter + def filterTaskList( --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220264138
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +124,87 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
@Context uriInfo: UriInfo):
+ util.HashMap[String, Object] = {
+withUI { ui =>
+ val uriQueryParameters = uriInfo.getQueryParameters(true)
+ val totalRecords = uriQueryParameters.getFirst("numTasks")
+ var isSearch = false
+ var searchValue: String = null
+ var _tasksToShow: Seq[TaskData] = null
+ if (uriQueryParameters.getFirst("search[value]") != null &&
+uriQueryParameters.getFirst("search[value]").length > 0) {
+_tasksToShow = ui.store.taskList(stageId, stageAttemptId, 0,
totalRecords.toInt,
+ indexName("Index"), true)
+isSearch = true
+searchValue = uriQueryParameters.getFirst("search[value]")
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId)
+ }
+ if (_tasksToShow.nonEmpty) {
+val iterator = _tasksToShow.iterator
+while(iterator.hasNext) {
+ val t1: TaskData = iterator.next()
+ val execId = t1.executorId
+ val executorLogs = ui.store.executorSummary(execId).executorLogs
+ t1.executorLogs = executorLogs
+ t1.schedulerDelay = AppStatusUtils.schedulerDelay(t1)
+ t1.gettingResultTime = AppStatusUtils.gettingResultTime(t1)
+}
+val ret = new util.HashMap[String, Object]()
+// Performs server-side search based on input from user
+if (isSearch) {
+ val filteredTaskList = ui.store.filterTaskList(_tasksToShow,
searchValue)
+ if (filteredTaskList.length > 0) {
+ret.put("aaData", filteredTaskList)
+ } else {
+_tasksToShow = doPagination(uriQueryParameters, stageId,
stageAttemptId)
--- End diff --
Added sorting and pagination capability on search results as well.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263918
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +124,87 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263860
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -40,8 +43,27 @@ private[v1] class StagesResource extends BaseAppResource
{
@PathParam("stageId") stageId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean):
Seq[StageData] = {
withUI { ui =>
- val ret = ui.store.stageData(stageId, details = details)
+ var ret = ui.store.stageData(stageId, details = details)
if (ret.nonEmpty) {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r220263804 --- Diff: core/src/main/scala/org/apache/spark/status/api/v1/api.scala --- @@ -215,7 +217,10 @@ class TaskData private[spark]( val speculative: Boolean, val accumulatorUpdates: Seq[AccumulableInfo], val errorMessage: Option[String] = None, -val taskMetrics: Option[TaskMetrics] = None) +val taskMetrics: Option[TaskMetrics] = None, +var executorLogs: Map[String, String], --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r220263752 --- Diff: core/src/main/scala/org/apache/spark/status/api/v1/api.scala --- @@ -196,8 +198,8 @@ class StageData private[spark]( val rddIds: Seq[Int], val accumulatorUpdates: Seq[AccumulableInfo], -val tasks: Option[Map[Long, TaskData]], -val executorSummary: Option[Map[String, ExecutorStageSummary]], +var tasks: Option[Map[Long, TaskData]], --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request: https://github.com/apache/spark/pull/21688#discussion_r220263706 --- Diff: core/src/main/scala/org/apache/spark/status/api/v1/api.scala --- @@ -71,7 +71,9 @@ class ExecutorStageSummary private[spark]( val shuffleWriteRecords : Long, val memoryBytesSpilled : Long, val diskBytesSpilled : Long, -val isBlacklistedForStage: Boolean) +val isBlacklistedForStage: Boolean, +var executorLogs: Map[String, String], --- End diff -- Done --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263640
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -102,4 +124,87 @@ private[v1] class StagesResource extends
BaseAppResource {
withUI(_.store.taskList(stageId, stageAttemptId, offset, length,
sortBy))
}
+ @GET
+ @Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
+ def taskTable(
+@PathParam("stageId") stageId: Int,
+@PathParam("stageAttemptId") stageAttemptId: Int,
+@QueryParam("details") @DefaultValue("true") details: Boolean,
@Context uriInfo: UriInfo):
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263590
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -40,8 +43,27 @@ private[v1] class StagesResource extends BaseAppResource
{
@PathParam("stageId") stageId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean):
Seq[StageData] = {
withUI { ui =>
- val ret = ui.store.stageData(stageId, details = details)
+ var ret = ui.store.stageData(stageId, details = details)
if (ret.nonEmpty) {
+for (r <- ret) {
+ val executorIdArray = r.executorSummary.get.keys.toArray
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263551
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -40,8 +43,27 @@ private[v1] class StagesResource extends BaseAppResource
{
@PathParam("stageId") stageId: Int,
@QueryParam("details") @DefaultValue("true") details: Boolean):
Seq[StageData] = {
withUI { ui =>
- val ret = ui.store.stageData(stageId, details = details)
+ var ret = ui.store.stageData(stageId, details = details)
if (ret.nonEmpty) {
+for (r <- ret) {
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263505
--- Diff:
core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala ---
@@ -16,15 +16,18 @@
*/
package org.apache.spark.status.api.v1
-import java.util.{List => JList}
+import java.util
--- End diff --
Done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220263423
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala ---
@@ -278,7 +198,7 @@ private[ui] class StagePage(parent: StagesTab, store:
AppStatusStore) extends We
}
}
val currentTime = System.currentTimeMillis()
-val (taskTable, taskTableHTML) = try {
+val taskTable = try {
val _taskTable = new TaskPagedTable(
--- End diff --
taskTable is needed to display the Event Timeline graph in Stage Page.
Whatever was needed has already been removed. If I am still missing something,
please let me know. Thank you.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262782
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262739
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262700
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262591
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262544
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262644
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262453
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
[GitHub] spark pull request #21688: [SPARK-21809] : Change Stage Page to use datatabl...
Github user pgandhi999 commented on a diff in the pull request:
https://github.com/apache/spark/pull/21688#discussion_r220262322
--- Diff: core/src/main/resources/org/apache/spark/ui/static/stagepage.js
---
@@ -0,0 +1,926 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$(document).ajaxStop($.unblockUI);
+$(document).ajaxStart(function () {
+$.blockUI({message: 'Loading Stage Page...'});
+});
+
+$.extend( $.fn.dataTable.ext.type.order, {
+"file-size-pre": ConvertDurationString,
+
+"file-size-asc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+},
+
+"file-size-desc": function ( a, b ) {
+a = ConvertDurationString( a );
+b = ConvertDurationString( b );
+return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+}
+} );
+
+// This function will only parse the URL under certain format
+// e.g.
https://domain:50509/history/application_1502220952225_59143/stages/stage/?id=0=0
+function stageEndPoint(appId) {
+var words = document.baseURI.split('/');
+var words2 = document.baseURI.split('?');
+var ind = words.indexOf("proxy");
+if (ind > 0) {
+var appId = words[ind + 1];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind + 2).join('/');
+return newBaseURI + "/api/v1/applications/" + appId + "/stages/" +
stageId;
+}
+ind = words.indexOf("history");
+if (ind > 0) {
+var appId = words[ind + 1];
+var attemptId = words[ind + 2];
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+var newBaseURI = words.slice(0, ind).join('/');
+if (isNaN(attemptId) || attemptId == "0") {
+return newBaseURI + "/api/v1/applications/" + appId +
"/stages/" + stageId;
+} else {
+return newBaseURI + "/api/v1/applications/" + appId + "/" +
attemptId + "/stages/" + stageId;
+}
+}
+var stageIdLen = words2[1].indexOf('&');
+var stageId = words2[1].substr(3, stageIdLen - 3);
+return location.origin + "/api/v1/applications/" + appId + "/stages/"
+ stageId;
+}
+
+function getColumnNameForTaskMetricSummary(columnKey) {
+switch(columnKey) {
+case "executorRunTime":
+return "Duration";
+break;
+
+case "jvmGcTime":
+return "GC Time";
+break;
+
+case "gettingResultTime":
+return "Getting Result Time";
+break;
+
+case "inputMetrics":
+return "Input Size / Records";
+break;
+
+case "outputMetrics":
+return "Output Size / Records";
+break;
+
+case "peakExecutionMemory":
+return "Peak Execution Memory";
+break;
+
+case "resultSerializationTime":
+return "Result Serialization Time";
+break;
+
+case "schedulerDelay":
+return "Scheduler Delay";
+break;
+
+case "diskBytesSpilled":
+return "Shuffle spill (disk)";
+break;
+
+case "memoryBytesSpilled":
+return "Shuffle spill (memory)";
+break;
+
