[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/13620 --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user zsxwing commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r72121948
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -210,64 +214,69 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
- private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
+ private def jobsTable(
+ request: HttpServletRequest,
+ jobTag: String,
+ jobs: Seq[JobUIData]): Seq[Node] = {
+val allParameters = request.getParameterMap.asScala.toMap
+val parameterOtherTable =
allParameters.filterNot(_._1.startsWith(jobTag))
+ .map(para => para._1 + "=" + para._2(0))
+
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
+val jobIdTitle = if (someJobHasJobGroup) "Job Id (Job Group)" else
"Job Id"
-val columns: Seq[Node] = {
- {if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"}
- Description
- Submitted
- Duration
- Stages: Succeeded/Total
- Tasks (for all stages):
Succeeded/Total
-}
+val parameterJobPage = request.getParameter(jobTag + ".page")
+val parameterJobSortColumn = request.getParameter(jobTag + ".sort")
+val parameterJobSortDesc = request.getParameter(jobTag + ".desc")
+val parameterJobPageSize = request.getParameter(jobTag + ".pageSize")
+val parameterJobPrevPageSize = request.getParameter(jobTag +
".prevPageSize")
-def makeRow(job: JobUIData): Seq[Node] = {
- val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(job)
- val duration: Option[Long] = {
-job.submissionTime.map { start =>
- val end =
job.completionTime.getOrElse(System.currentTimeMillis())
- end - start
-}
+val jobPage = Option(parameterJobPage).map(_.toInt).getOrElse(1)
+val jobSortColumn = Option(parameterJobSortColumn).map { sortColumn =>
+ UIUtils.decodeURLParameter(sortColumn)
+}.getOrElse(jobIdTitle)
+val jobSortDesc =
Option(parameterJobSortDesc).map(_.toBoolean).getOrElse(
+ // New jobs should be shown above old jobs by default.
+ if (jobSortColumn == jobIdTitle) true else false
+)
+val jobPageSize =
Option(parameterJobPageSize).map(_.toInt).getOrElse(100)
+val jobPrevPageSize =
Option(parameterJobPrevPageSize).map(_.toInt).getOrElse(jobPageSize)
+
+val page: Int = {
+ // If the user has changed to a larger page size, then go to page 1
in order to avoid
+ // IndexOutOfBoundsException.
+ if (jobPageSize <= jobPrevPageSize) {
+jobPage
+ } else {
+1
}
- val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
- val formattedSubmissionTime =
job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
- val basePathUri = UIUtils.prependBaseUri(parent.basePath)
- val jobDescription =
-UIUtils.makeDescription(lastStageDescription, basePathUri,
plainText = false)
-
- val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId)
-
-
- {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")}
-
-
- {jobDescription}
- {lastStageName}
-
-
- {formattedSubmissionTime}
-
-{formattedDuration}
-
- {job.completedStageIndices.size}/{job.stageIds.size -
job.numSkippedStages}
- {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
- {if (job.numSkippedStages > 0) s"(${job.numSkippedStages}
skipped)"}
-
-
- {UIUtils.makeProgressBar(started = job.numActiveTasks, completed
= job.numCompletedTasks,
- failed = job.numFailedTasks, skipped = job.numSkippedTasks,
killed = job.numKilledTasks,
- total = job.numTasks - job.numSkippedTasks)}
-
-
}
+val currentTime = System.currentTimeMillis()
-
- {columns}
-
-{jobs.map(makeRow)}
-
-
+try {
+ new JobPagedTable(
+jobs,
+jobTag,
+UIUtils.prependBaseUri(parent.basePath),
+"jobs", // subPath
+parameterOtherTable,
+parent.jobProgresslistener.stageIdToInfo,
+parent.jobProgresslistener.stageIdToData,
+currentTime,
+jobIdTitle,
+pageSize = jobPageSize,
+sortColumn = jobSortColumn,
+desc = jobSortDesc
+ ).table(page)
+} catch {
+ case e @ (_ : IllegalArgument
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r72038829
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -210,64 +214,69 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
- private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
+ private def jobsTable(
+ request: HttpServletRequest,
+ jobTag: String,
+ jobs: Seq[JobUIData]): Seq[Node] = {
+val allParameters = request.getParameterMap.asScala.toMap
+val parameterOtherTable =
allParameters.filterNot(_._1.startsWith(jobTag))
+ .map(para => para._1 + "=" + para._2(0))
+
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
+val jobIdTitle = if (someJobHasJobGroup) "Job Id (Job Group)" else
"Job Id"
-val columns: Seq[Node] = {
- {if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"}
- Description
- Submitted
- Duration
- Stages: Succeeded/Total
- Tasks (for all stages):
Succeeded/Total
-}
+val parameterJobPage = request.getParameter(jobTag + ".page")
+val parameterJobSortColumn = request.getParameter(jobTag + ".sort")
+val parameterJobSortDesc = request.getParameter(jobTag + ".desc")
+val parameterJobPageSize = request.getParameter(jobTag + ".pageSize")
+val parameterJobPrevPageSize = request.getParameter(jobTag +
".prevPageSize")
-def makeRow(job: JobUIData): Seq[Node] = {
- val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(job)
- val duration: Option[Long] = {
-job.submissionTime.map { start =>
- val end =
job.completionTime.getOrElse(System.currentTimeMillis())
- end - start
-}
+val jobPage = Option(parameterJobPage).map(_.toInt).getOrElse(1)
+val jobSortColumn = Option(parameterJobSortColumn).map { sortColumn =>
+ UIUtils.decodeURLParameter(sortColumn)
+}.getOrElse(jobIdTitle)
+val jobSortDesc =
Option(parameterJobSortDesc).map(_.toBoolean).getOrElse(
+ // New jobs should be shown above old jobs by default.
+ if (jobSortColumn == jobIdTitle) true else false
+)
+val jobPageSize =
Option(parameterJobPageSize).map(_.toInt).getOrElse(100)
+val jobPrevPageSize =
Option(parameterJobPrevPageSize).map(_.toInt).getOrElse(jobPageSize)
+
+val page: Int = {
+ // If the user has changed to a larger page size, then go to page 1
in order to avoid
+ // IndexOutOfBoundsException.
+ if (jobPageSize <= jobPrevPageSize) {
+jobPage
+ } else {
+1
}
- val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
- val formattedSubmissionTime =
job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
- val basePathUri = UIUtils.prependBaseUri(parent.basePath)
- val jobDescription =
-UIUtils.makeDescription(lastStageDescription, basePathUri,
plainText = false)
-
- val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId)
-
-
- {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")}
-
-
- {jobDescription}
- {lastStageName}
-
-
- {formattedSubmissionTime}
-
-{formattedDuration}
-
- {job.completedStageIndices.size}/{job.stageIds.size -
job.numSkippedStages}
- {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
- {if (job.numSkippedStages > 0) s"(${job.numSkippedStages}
skipped)"}
-
-
- {UIUtils.makeProgressBar(started = job.numActiveTasks, completed
= job.numCompletedTasks,
- failed = job.numFailedTasks, skipped = job.numSkippedTasks,
killed = job.numKilledTasks,
- total = job.numTasks - job.numSkippedTasks)}
-
-
}
+val currentTime = System.currentTimeMillis()
-
- {columns}
-
-{jobs.map(makeRow)}
-
-
+try {
+ new JobPagedTable(
+jobs,
+jobTag,
+UIUtils.prependBaseUri(parent.basePath),
+"jobs", // subPath
+parameterOtherTable,
+parent.jobProgresslistener.stageIdToInfo,
+parent.jobProgresslistener.stageIdToData,
+currentTime,
+jobIdTitle,
+pageSize = jobPageSize,
+sortColumn = jobSortColumn,
+desc = jobSortDesc
+ ).table(page)
+} catch {
+ case e @ (_ : IllegalArgumen
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r71999178
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +375,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+}
+private[ui] class JobPagedTable(
+data: S
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r71999045
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -210,64 +214,69 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
- private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
+ private def jobsTable(
+ request: HttpServletRequest,
+ jobTag: String,
+ jobs: Seq[JobUIData]): Seq[Node] = {
+val allParameters = request.getParameterMap.asScala.toMap
+val parameterOtherTable =
allParameters.filterNot(_._1.startsWith(jobTag))
+ .map(para => para._1 + "=" + para._2(0))
--- End diff --
It keeps the status of other tables on the page. See
https://github.com/apache/spark/pull/13708#issuecomment-226642404
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r71997586
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +375,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+}
+private[ui] class JobPagedTable(
+data: Seq[
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r71997489
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -210,64 +214,69 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
- private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
+ private def jobsTable(
+ request: HttpServletRequest,
+ jobTag: String,
+ jobs: Seq[JobUIData]): Seq[Node] = {
+val allParameters = request.getParameterMap.asScala.toMap
+val parameterOtherTable =
allParameters.filterNot(_._1.startsWith(jobTag))
+ .map(para => para._1 + "=" + para._2(0))
+
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
+val jobIdTitle = if (someJobHasJobGroup) "Job Id (Job Group)" else
"Job Id"
-val columns: Seq[Node] = {
- {if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"}
- Description
- Submitted
- Duration
- Stages: Succeeded/Total
- Tasks (for all stages):
Succeeded/Total
-}
+val parameterJobPage = request.getParameter(jobTag + ".page")
+val parameterJobSortColumn = request.getParameter(jobTag + ".sort")
+val parameterJobSortDesc = request.getParameter(jobTag + ".desc")
+val parameterJobPageSize = request.getParameter(jobTag + ".pageSize")
+val parameterJobPrevPageSize = request.getParameter(jobTag +
".prevPageSize")
-def makeRow(job: JobUIData): Seq[Node] = {
- val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(job)
- val duration: Option[Long] = {
-job.submissionTime.map { start =>
- val end =
job.completionTime.getOrElse(System.currentTimeMillis())
- end - start
-}
+val jobPage = Option(parameterJobPage).map(_.toInt).getOrElse(1)
+val jobSortColumn = Option(parameterJobSortColumn).map { sortColumn =>
+ UIUtils.decodeURLParameter(sortColumn)
+}.getOrElse(jobIdTitle)
+val jobSortDesc =
Option(parameterJobSortDesc).map(_.toBoolean).getOrElse(
+ // New jobs should be shown above old jobs by default.
+ if (jobSortColumn == jobIdTitle) true else false
+)
+val jobPageSize =
Option(parameterJobPageSize).map(_.toInt).getOrElse(100)
+val jobPrevPageSize =
Option(parameterJobPrevPageSize).map(_.toInt).getOrElse(jobPageSize)
+
+val page: Int = {
+ // If the user has changed to a larger page size, then go to page 1
in order to avoid
+ // IndexOutOfBoundsException.
+ if (jobPageSize <= jobPrevPageSize) {
+jobPage
+ } else {
+1
}
- val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
- val formattedSubmissionTime =
job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
- val basePathUri = UIUtils.prependBaseUri(parent.basePath)
- val jobDescription =
-UIUtils.makeDescription(lastStageDescription, basePathUri,
plainText = false)
-
- val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId)
-
-
- {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")}
-
-
- {jobDescription}
- {lastStageName}
-
-
- {formattedSubmissionTime}
-
-{formattedDuration}
-
- {job.completedStageIndices.size}/{job.stageIds.size -
job.numSkippedStages}
- {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
- {if (job.numSkippedStages > 0) s"(${job.numSkippedStages}
skipped)"}
-
-
- {UIUtils.makeProgressBar(started = job.numActiveTasks, completed
= job.numCompletedTasks,
- failed = job.numFailedTasks, skipped = job.numSkippedTasks,
killed = job.numKilledTasks,
- total = job.numTasks - job.numSkippedTasks)}
-
-
}
+val currentTime = System.currentTimeMillis()
-
- {columns}
-
-{jobs.map(makeRow)}
-
-
+try {
+ new JobPagedTable(
+jobs,
+jobTag,
+UIUtils.prependBaseUri(parent.basePath),
+"jobs", // subPath
+parameterOtherTable,
+parent.jobProgresslistener.stageIdToInfo,
+parent.jobProgresslistener.stageIdToData,
+currentTime,
+jobIdTitle,
+pageSize = jobPageSize,
+sortColumn = jobSortColumn,
+desc = jobSortDesc
+ ).table(page)
+} catch {
+ case e @ (_ : IllegalArgumentEx
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user yhuai commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r71997400
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -210,64 +214,69 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
- private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = {
+ private def jobsTable(
+ request: HttpServletRequest,
+ jobTag: String,
+ jobs: Seq[JobUIData]): Seq[Node] = {
+val allParameters = request.getParameterMap.asScala.toMap
+val parameterOtherTable =
allParameters.filterNot(_._1.startsWith(jobTag))
+ .map(para => para._1 + "=" + para._2(0))
--- End diff --
What does `parameterOtherTable` do?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r67087787
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r67087737
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r67028036
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66896475
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66896310
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66895790
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user nblintao commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66894846
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
--- End diff --
Sorry, it's deprecated. I will remove it. Thanks.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66885556
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66885396
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
--- End diff --
used anywhere?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66885448
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/13620#discussion_r66885512
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala ---
@@ -369,3 +361,246 @@ private[ui] class AllJobsPage(parent: JobsTab)
extends WebUIPage("") {
}
}
}
+
+private[ui] class JobTableRowData(
+val jobData: JobUIData,
+val lastStageName: String,
+val lastStageDescription: String,
+val duration: Long,
+val formattedDuration: String,
+val submissionTime: Long,
+val formattedSubmissionTime: String,
+val jobDescription: NodeSeq,
+val detailUrl: String)
+
+private[ui] class JobDataSource(
+jobs: Seq[JobUIData],
+stageIdToInfo: HashMap[Int, StageInfo],
+stageIdToData: HashMap[(Int, Int), StageUIData],
+basePath: String,
+currentTime: Long,
+pageSize: Int,
+sortColumn: String,
+desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
+
+ // Convert JobUIData to JobTableRowData which contains the final
contents to show in the table
+ // so that we can avoid creating duplicate contents during sorting the
data
+ private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))
+
+ private var _slicedJobIds: Set[Int] = null
+
+ override def dataSize: Int = data.size
+
+ override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
+val r = data.slice(from, to)
+_slicedJobIds = r.map(_.jobData.jobId).toSet
+r
+ }
+
+ def slicedJobIds: Set[Int] = _slicedJobIds
+
+ private def getLastStageNameAndDescription(job: JobUIData): (String,
String) = {
+val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => stageIdToInfo.get(ids.max)}
+val lastStageData = lastStageInfo.flatMap { s =>
+ stageIdToData.get((s.stageId, s.attemptId))
+}
+val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
+val description = lastStageData.flatMap(_.description).getOrElse("")
+(name, description)
+ }
+
+ private def jobRow(jobData: JobUIData): JobTableRowData = {
+val (lastStageName, lastStageDescription) =
getLastStageNameAndDescription(jobData)
+val duration: Option[Long] = {
+ jobData.submissionTime.map { start =>
+val end =
jobData.completionTime.getOrElse(System.currentTimeMillis())
+end - start
+ }
+}
+val formattedDuration = duration.map(d =>
UIUtils.formatDuration(d)).getOrElse("Unknown")
+val submissionTime = jobData.submissionTime
+val formattedSubmissionTime =
submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
+val jobDescription = UIUtils.makeDescription(lastStageDescription,
basePath, plainText = false)
+
+val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
+
+new JobTableRowData (
+ jobData,
+ lastStageName,
+ lastStageDescription,
+ duration.getOrElse(-1),
+ formattedDuration,
+ submissionTime.getOrElse(-1),
+ formattedSubmissionTime,
+ jobDescription,
+ detailUrl
+)
+ }
+
+ /**
+ * Return Ordering according to sortColumn and desc
+ */
+ private def ordering(sortColumn: String, desc: Boolean):
Ordering[JobTableRowData] = {
+val ordering = sortColumn match {
+ case "Job Id" | "Job Id (Job Group)" => new
Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Int.compare(x.jobData.jobId, y.jobData.jobId)
+ }
+ case "Description" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.String.compare(x.lastStageDescription,
y.lastStageDescription)
+ }
+ case "Submitted" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.submissionTime, y.submissionTime)
+ }
+ case "Duration" => new Ordering[JobTableRowData] {
+override def compare(x: JobTableRowData, y: JobTableRowData): Int =
+ Ordering.Long.compare(x.duration, y.duration)
+ }
+ case "Stages: Succeeded/Total" | "Tasks (for all stages):
Succeeded/Total" =>
+throw new IllegalArgumentException(s"Unsortable column:
$sortColumn")
+ case unknownColumn => throw new IllegalArgumentException(s"Unknown
column: $unknownColumn")
+}
+if (desc) {
+ ordering.reverse
+} else {
+ ordering
+}
+ }
+
[GitHub] spark pull request #13620: [SPARK-15590] [WEBUI] Paginate Job Table in Jobs ...
GitHub user nblintao opened a pull request:
https://github.com/apache/spark/pull/13620
[SPARK-15590] [WEBUI] Paginate Job Table in Jobs tab
## What changes were proposed in this pull request?
This patch adds pagination support for the Job Tables in the Jobs tab.
Pagination is provided for all of the three Job Tables (active, completed, and
failed). Interactions (jumping, sorting, and setting page size) for paged
tables are also included.
## How was this patch tested?
Tested manually by using checking the Web UI after completing and failing
hundreds of jobs.
Generate completed jobs by:
```scala
val d = sc.parallelize(Array(1,2,3,4,5))
for(i <- 1 to 255){ var b = d.collect() }
```
Generate failed jobs by calling the following code multiple times:
```scala
var b = d.map(_/0).collect()
```
Interactions like jumping, sorting, and setting page size are all tested.
This shows the pagination for completed jobs:
This shows the sorting works in job tables:
This shows the pagination for failed jobs and the effect of jumping and
setting page size:
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/nblintao/spark dev
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/13620.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #13620
commit 4db0e0978cd2cc57814733e6a6360407f10cb37b
Author: Tao Lin
Date: 2016-06-11T16:04:50Z
paginate job tables
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
