[SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting Builds on top of work in SPARK-8425 to update Application Level Blacklisting in the scheduler.
## What changes were proposed in this pull request? Adds a UI to these patches by: - defining new listener events for blacklisting and unblacklisting, nodes and executors; - sending said events at the relevant points in BlacklistTracker; - adding JSON (de)serialization code for these events; - augmenting the Executors UI page to show which, and how many, executors are blacklisted; - adding a unit test to make sure events are being fired; - adding HistoryServerSuite coverage to verify that the SHS reads these events correctly. - updates the Executor UI to show Blacklisted/Active/Dead as a tri-state in Executors Status Updates .rat-excludes to pass tests. username squito ## How was this patch tested? ./dev/run-tests testOnly org.apache.spark.util.JsonProtocolSuite testOnly org.apache.spark.scheduler.BlacklistTrackerSuite testOnly org.apache.spark.deploy.history.HistoryServerSuite https://github.com/jsoltren/jose-utils/blob/master/blacklist/test-blacklist.sh ![blacklist-20161219](https://cloud.githubusercontent.com/assets/1208477/21335321/9eda320a-c623-11e6-8b8c-9c912a73c276.jpg) Author: José Hiram Soltren <j...@cloudera.com> Closes #16346 from jsoltren/SPARK-16654-submit. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/640f9423 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/640f9423 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/640f9423 Branch: refs/heads/master Commit: 640f942337e1ce87075195998bd051e19c4b50b9 Parents: 064fadd Author: José Hiram Soltren <j...@cloudera.com> Authored: Thu Jan 19 09:08:18 2017 -0600 Committer: Imran Rashid <iras...@cloudera.com> Committed: Thu Jan 19 09:08:18 2017 -0600 ---------------------------------------------------------------------- .../org/apache/spark/SparkFirehoseListener.java | 20 ++++ .../spark/ui/static/executorspage-template.html | 5 + .../org/apache/spark/ui/static/executorspage.js | 39 ++++-- .../spark/scheduler/BlacklistTracker.scala | 16 ++- .../spark/scheduler/EventLoggingListener.scala | 16 +++ .../apache/spark/scheduler/SparkListener.scala | 54 +++++++++ .../spark/scheduler/SparkListenerBus.scala | 8 ++ .../spark/scheduler/TaskSchedulerImpl.scala | 10 +- .../org/apache/spark/status/api/v1/api.scala | 1 + .../scala/org/apache/spark/ui/ToolTips.scala | 3 + .../apache/spark/ui/exec/ExecutorsPage.scala | 2 + .../org/apache/spark/ui/exec/ExecutorsTab.scala | 57 ++++++++- .../apache/spark/ui/jobs/ExecutorTable.scala | 6 + .../scala/org/apache/spark/ui/jobs/UIData.scala | 2 + .../application_list_json_expectation.json | 44 +++++-- .../completed_app_list_json_expectation.json | 44 +++++-- .../executor_list_json_expectation.json | 1 + .../executor_node_blacklisting_expectation.json | 118 +++++++++++++++++++ ...blacklisting_unblacklisting_expectation.json | 118 +++++++++++++++++++ .../limit_app_list_json_expectation.json | 64 ++++------ .../maxDate2_app_list_json_expectation.json | 2 +- .../maxDate_app_list_json_expectation.json | 4 +- .../minDate_app_list_json_expectation.json | 40 ++++++- .../one_app_json_expectation.json | 2 +- .../one_app_multi_attempt_json_expectation.json | 4 +- .../one_stage_attempt_json_expectation.json | 80 ++++++------- .../one_stage_json_expectation.json | 80 ++++++------- ...summary_w__custom_quantiles_expectation.json | 2 +- ...stage_with_accumulable_json_expectation.json | 100 ++++++++-------- .../spark-events/app-20161115172038-0000 | 75 ++++++++++++ .../spark-events/app-20161116163331-0000 | 68 +++++++++++ .../deploy/history/HistoryServerSuite.scala | 4 +- .../spark/scheduler/BlacklistTrackerSuite.scala | 42 ++++++- .../spark/scheduler/TaskSetManagerSuite.scala | 3 +- .../apache/spark/util/JsonProtocolSuite.scala | 49 ++++++++ dev/.rat-excludes | 2 + 36 files changed, 950 insertions(+), 235 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/java/org/apache/spark/SparkFirehoseListener.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java index 97eed61..9fe97b4 100644 --- a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java +++ b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java @@ -114,6 +114,26 @@ public class SparkFirehoseListener implements SparkListenerInterface { } @Override + public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executorBlacklisted) { + onEvent(executorBlacklisted); + } + + @Override + public final void onExecutorUnblacklisted(SparkListenerExecutorUnblacklisted executorUnblacklisted) { + onEvent(executorUnblacklisted); + } + + @Override + public final void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) { + onEvent(nodeBlacklisted); + } + + @Override + public final void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) { + onEvent(nodeUnblacklisted); + } + + @Override public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) { onEvent(blockUpdated); } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html index 64ea719..4e83d6d 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html @@ -45,6 +45,11 @@ limitations under the License. title="Bytes and records written to disk in order to be read by a shuffle in a future stage."> Shuffle Write</span> </th> + <th> + <span data-toggle="tooltip" data-placement="left" + title="Number of executors blacklisted by the scheduler due to task failures."> + Blacklisted</span> + </th> </thead> <tbody> </tbody> http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/resources/org/apache/spark/ui/static/executorspage.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index fe5db6a..7dbfe32 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -182,7 +182,7 @@ $(document).ready(function () { executorsSummary = $("#active-executors"); getStandAloneppId(function (appId) { - + var endPoint = createRESTEndPoint(appId); $.getJSON(endPoint, function (response, status, jqXHR) { var summary = []; @@ -202,7 +202,8 @@ $(document).ready(function () { var allTotalInputBytes = 0; var allTotalShuffleRead = 0; var allTotalShuffleWrite = 0; - + var allTotalBlacklisted = 0; + var activeExecCnt = 0; var activeRDDBlocks = 0; var activeMemoryUsed = 0; @@ -219,7 +220,8 @@ $(document).ready(function () { var activeTotalInputBytes = 0; var activeTotalShuffleRead = 0; var activeTotalShuffleWrite = 0; - + var activeTotalBlacklisted = 0; + var deadExecCnt = 0; var deadRDDBlocks = 0; var deadMemoryUsed = 0; @@ -236,7 +238,8 @@ $(document).ready(function () { var deadTotalInputBytes = 0; var deadTotalShuffleRead = 0; var deadTotalShuffleWrite = 0; - + var deadTotalBlacklisted = 0; + response.forEach(function (exec) { allExecCnt += 1; allRDDBlocks += exec.rddBlocks; @@ -254,6 +257,7 @@ $(document).ready(function () { allTotalInputBytes += exec.totalInputBytes; allTotalShuffleRead += exec.totalShuffleRead; allTotalShuffleWrite += exec.totalShuffleWrite; + allTotalBlacklisted += exec.isBlacklisted ? 1 : 0; if (exec.isActive) { activeExecCnt += 1; activeRDDBlocks += exec.rddBlocks; @@ -271,6 +275,7 @@ $(document).ready(function () { activeTotalInputBytes += exec.totalInputBytes; activeTotalShuffleRead += exec.totalShuffleRead; activeTotalShuffleWrite += exec.totalShuffleWrite; + activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0; } else { deadExecCnt += 1; deadRDDBlocks += exec.rddBlocks; @@ -288,9 +293,10 @@ $(document).ready(function () { deadTotalInputBytes += exec.totalInputBytes; deadTotalShuffleRead += exec.totalShuffleRead; deadTotalShuffleWrite += exec.totalShuffleWrite; + deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0; } }); - + var totalSummary = { "execCnt": ( "Total(" + allExecCnt + ")"), "allRDDBlocks": allRDDBlocks, @@ -307,7 +313,8 @@ $(document).ready(function () { "allTotalGCTime": allTotalGCTime, "allTotalInputBytes": allTotalInputBytes, "allTotalShuffleRead": allTotalShuffleRead, - "allTotalShuffleWrite": allTotalShuffleWrite + "allTotalShuffleWrite": allTotalShuffleWrite, + "allTotalBlacklisted": allTotalBlacklisted }; var activeSummary = { "execCnt": ( "Active(" + activeExecCnt + ")"), @@ -325,7 +332,8 @@ $(document).ready(function () { "allTotalGCTime": activeTotalGCTime, "allTotalInputBytes": activeTotalInputBytes, "allTotalShuffleRead": activeTotalShuffleRead, - "allTotalShuffleWrite": activeTotalShuffleWrite + "allTotalShuffleWrite": activeTotalShuffleWrite, + "allTotalBlacklisted": activeTotalBlacklisted }; var deadSummary = { "execCnt": ( "Dead(" + deadExecCnt + ")" ), @@ -343,12 +351,13 @@ $(document).ready(function () { "allTotalGCTime": deadTotalGCTime, "allTotalInputBytes": deadTotalInputBytes, "allTotalShuffleRead": deadTotalShuffleRead, - "allTotalShuffleWrite": deadTotalShuffleWrite + "allTotalShuffleWrite": deadTotalShuffleWrite, + "allTotalBlacklisted": deadTotalBlacklisted }; - + var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; $.get(createTemplateURI(appId), function (template) { - + executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); var selector = "#active-executors-table"; var conf = { @@ -360,7 +369,12 @@ $(document).ready(function () { } }, {data: 'hostPort'}, - {data: 'isActive', render: formatStatus}, + {data: 'isActive', render: function (data, type, row) { + if (type !== 'display') return data; + if (row.isBlacklisted) return "Blacklisted"; + else return formatStatus (data, type); + } + }, {data: 'rddBlocks'}, { data: function (row, type) { @@ -474,7 +488,8 @@ $(document).ready(function () { }, {data: 'allTotalInputBytes', render: formatBytes}, {data: 'allTotalShuffleRead', render: formatBytes}, - {data: 'allTotalShuffleWrite', render: formatBytes} + {data: 'allTotalShuffleWrite', render: formatBytes}, + {data: 'allTotalBlacklisted'} ], "paging": false, "searching": false, http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala index bf7a62e..77d5c97 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala @@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicReference import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.internal.Logging import org.apache.spark.internal.config import org.apache.spark.util.{Clock, SystemClock, Utils} @@ -48,9 +48,14 @@ import org.apache.spark.util.{Clock, SystemClock, Utils} * one exception is [[nodeBlacklist()]], which can be called without holding a lock. */ private[scheduler] class BlacklistTracker ( + private val listenerBus: LiveListenerBus, conf: SparkConf, clock: Clock = new SystemClock()) extends Logging { + def this(sc: SparkContext) = { + this(sc.listenerBus, sc.conf) + } + BlacklistTracker.validateBlacklistConfs(conf) private val MAX_FAILURES_PER_EXEC = conf.get(config.MAX_FAILURES_PER_EXEC) private val MAX_FAILED_EXEC_PER_NODE = conf.get(config.MAX_FAILED_EXEC_PER_NODE) @@ -103,6 +108,7 @@ private[scheduler] class BlacklistTracker ( execsToUnblacklist.foreach { exec => val status = executorIdToBlacklistStatus.remove(exec).get val failedExecsOnNode = nodeToBlacklistedExecs(status.node) + listenerBus.post(SparkListenerExecutorUnblacklisted(now, exec)) failedExecsOnNode.remove(exec) if (failedExecsOnNode.isEmpty) { nodeToBlacklistedExecs.remove(status.node) @@ -114,7 +120,10 @@ private[scheduler] class BlacklistTracker ( // Un-blacklist any nodes that have been blacklisted longer than the blacklist timeout. logInfo(s"Removing nodes $nodesToUnblacklist from blacklist because the blacklist " + s"has timed out") - nodeIdToBlacklistExpiryTime --= nodesToUnblacklist + nodesToUnblacklist.foreach { node => + nodeIdToBlacklistExpiryTime.remove(node) + listenerBus.post(SparkListenerNodeUnblacklisted(now, node)) + } _nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet) } updateNextExpiryTime() @@ -161,6 +170,8 @@ private[scheduler] class BlacklistTracker ( s" task failures in successful task sets") val node = failuresInTaskSet.node executorIdToBlacklistStatus.put(exec, BlacklistedExecutor(node, expiryTimeForNewBlacklists)) + listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, newTotal)) + executorIdToFailureList.remove(exec) updateNextExpiryTime() // In addition to blacklisting the executor, we also update the data for failures on the @@ -174,6 +185,7 @@ private[scheduler] class BlacklistTracker ( logInfo(s"Blacklisting node $node because it has ${blacklistedExecsOnNode.size} " + s"executors blacklisted: ${blacklistedExecsOnNode}") nodeIdToBlacklistExpiryTime.put(node, expiryTimeForNewBlacklists) + listenerBus.post(SparkListenerNodeBlacklisted(now, node, blacklistedExecsOnNode.size)) _nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet) } } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala index f39565e..af9bdef 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala @@ -193,6 +193,22 @@ private[spark] class EventLoggingListener( logEvent(event, flushLogger = true) } + override def onExecutorBlacklisted(event: SparkListenerExecutorBlacklisted): Unit = { + logEvent(event, flushLogger = true) + } + + override def onExecutorUnblacklisted(event: SparkListenerExecutorUnblacklisted): Unit = { + logEvent(event, flushLogger = true) + } + + override def onNodeBlacklisted(event: SparkListenerNodeBlacklisted): Unit = { + logEvent(event, flushLogger = true) + } + + override def onNodeUnblacklisted(event: SparkListenerNodeUnblacklisted): Unit = { + logEvent(event, flushLogger = true) + } + // No-op because logging every update would be overkill override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {} http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala index 7618dfe..1b12af7 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala @@ -106,6 +106,28 @@ case class SparkListenerExecutorRemoved(time: Long, executorId: String, reason: extends SparkListenerEvent @DeveloperApi +case class SparkListenerExecutorBlacklisted( + time: Long, + executorId: String, + taskFailures: Int) + extends SparkListenerEvent + +@DeveloperApi +case class SparkListenerExecutorUnblacklisted(time: Long, executorId: String) + extends SparkListenerEvent + +@DeveloperApi +case class SparkListenerNodeBlacklisted( + time: Long, + hostId: String, + executorFailures: Int) + extends SparkListenerEvent + +@DeveloperApi +case class SparkListenerNodeUnblacklisted(time: Long, hostId: String) + extends SparkListenerEvent + +@DeveloperApi case class SparkListenerBlockUpdated(blockUpdatedInfo: BlockUpdatedInfo) extends SparkListenerEvent /** @@ -239,6 +261,26 @@ private[spark] trait SparkListenerInterface { def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit /** + * Called when the driver blacklists an executor for a Spark application. + */ + def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit + + /** + * Called when the driver re-enables a previously blacklisted executor. + */ + def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit + + /** + * Called when the driver blacklists a node for a Spark application. + */ + def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit + + /** + * Called when the driver re-enables a previously blacklisted node. + */ + def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit + + /** * Called when the driver receives a block update info. */ def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit @@ -293,6 +335,18 @@ abstract class SparkListener extends SparkListenerInterface { override def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit = { } + override def onExecutorBlacklisted( + executorBlacklisted: SparkListenerExecutorBlacklisted): Unit = { } + + override def onExecutorUnblacklisted( + executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit = { } + + override def onNodeBlacklisted( + nodeBlacklisted: SparkListenerNodeBlacklisted): Unit = { } + + override def onNodeUnblacklisted( + nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit = { } + override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = { } override def onOtherEvent(event: SparkListenerEvent): Unit = { } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala index 471586a..3ff3633 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala @@ -61,6 +61,14 @@ private[spark] trait SparkListenerBus listener.onExecutorAdded(executorAdded) case executorRemoved: SparkListenerExecutorRemoved => listener.onExecutorRemoved(executorRemoved) + case executorBlacklisted: SparkListenerExecutorBlacklisted => + listener.onExecutorBlacklisted(executorBlacklisted) + case executorUnblacklisted: SparkListenerExecutorUnblacklisted => + listener.onExecutorUnblacklisted(executorUnblacklisted) + case nodeBlacklisted: SparkListenerNodeBlacklisted => + listener.onNodeBlacklisted(nodeBlacklisted) + case nodeUnblacklisted: SparkListenerNodeUnblacklisted => + listener.onNodeUnblacklisted(nodeUnblacklisted) case blockUpdated: SparkListenerBlockUpdated => listener.onBlockUpdated(blockUpdated) case logStart: SparkListenerLogStart => // ignore event log metadata http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 9a8e313..72ed55a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -63,14 +63,14 @@ private[spark] class TaskSchedulerImpl private[scheduler]( this( sc, sc.conf.get(config.MAX_TASK_FAILURES), - TaskSchedulerImpl.maybeCreateBlacklistTracker(sc.conf)) + TaskSchedulerImpl.maybeCreateBlacklistTracker(sc)) } def this(sc: SparkContext, maxTaskFailures: Int, isLocal: Boolean) = { this( sc, maxTaskFailures, - TaskSchedulerImpl.maybeCreateBlacklistTracker(sc.conf), + TaskSchedulerImpl.maybeCreateBlacklistTracker(sc), isLocal = isLocal) } @@ -717,9 +717,9 @@ private[spark] object TaskSchedulerImpl { retval.toList } - private def maybeCreateBlacklistTracker(conf: SparkConf): Option[BlacklistTracker] = { - if (BlacklistTracker.isBlacklistEnabled(conf)) { - Some(new BlacklistTracker(conf)) + private def maybeCreateBlacklistTracker(sc: SparkContext): Option[BlacklistTracker] = { + if (BlacklistTracker.isBlacklistEnabled(sc.conf)) { + Some(new BlacklistTracker(sc)) } else { None } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/status/api/v1/api.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 44a929b..7d035b1 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -73,6 +73,7 @@ class ExecutorSummary private[spark]( val totalInputBytes: Long, val totalShuffleRead: Long, val totalShuffleWrite: Long, + val isBlacklisted: Boolean, val maxMemory: Long, val executorLogs: Map[String, String]) http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/ToolTips.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala index 3cc5353..766cc65 100644 --- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala +++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala @@ -91,6 +91,9 @@ private[spark] object ToolTips { val TASK_TIME = "Shaded red when garbage collection (GC) time is over 10% of task time" + val BLACKLISTED = + "Shows if this executor has been blacklisted by the scheduler due to task failures." + val APPLICATION_EXECUTOR_LIMIT = """Maximum number of executors that this application will use. This limit is finite only when dynamic allocation is enabled. The number of granted executors may exceed the limit http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 7953d77..2d1691e 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -39,6 +39,7 @@ private[ui] case class ExecutorSummaryInfo( totalInputBytes: Long, totalShuffleRead: Long, totalShuffleWrite: Long, + isBlacklisted: Int, maxMemory: Long, executorLogs: Map[String, String]) @@ -101,6 +102,7 @@ private[spark] object ExecutorsPage { taskSummary.inputBytes, taskSummary.shuffleRead, taskSummary.shuffleWrite, + taskSummary.isBlacklisted, maxMem, taskSummary.executorLogs ) http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index 678571f..8ae712f 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -53,7 +53,8 @@ private[ui] case class ExecutorTaskSummary( var shuffleRead: Long = 0L, var shuffleWrite: Long = 0L, var executorLogs: Map[String, String] = Map.empty, - var isAlive: Boolean = true + var isAlive: Boolean = true, + var isBlacklisted: Boolean = false ) /** @@ -73,7 +74,8 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar def deadStorageStatusList: Seq[StorageStatus] = storageStatusListener.deadStorageStatusList - override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized { + override def onExecutorAdded( + executorAdded: SparkListenerExecutorAdded): Unit = synchronized { val eid = executorAdded.executorId val taskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid)) taskSummary.executorLogs = executorAdded.executorInfo.logUrlMap @@ -100,7 +102,8 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar executorToTaskSummary.get(executorRemoved.executorId).foreach(e => e.isAlive = false) } - override def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit = { + override def onApplicationStart( + applicationStart: SparkListenerApplicationStart): Unit = { applicationStart.driverLogs.foreach { logs => val storageStatus = activeStorageStatusList.find { s => s.blockManagerId.executorId == SparkContext.LEGACY_DRIVER_IDENTIFIER || @@ -114,13 +117,15 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar } } - override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = synchronized { + override def onTaskStart( + taskStart: SparkListenerTaskStart): Unit = synchronized { val eid = taskStart.taskInfo.executorId val taskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid)) taskSummary.tasksActive += 1 } - override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized { + override def onTaskEnd( + taskEnd: SparkListenerTaskEnd): Unit = synchronized { val info = taskEnd.taskInfo if (info != null) { val eid = info.executorId @@ -157,4 +162,46 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar } } + private def updateExecutorBlacklist( + eid: String, + isBlacklisted: Boolean): Unit = { + val execTaskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid)) + execTaskSummary.isBlacklisted = isBlacklisted + } + + override def onExecutorBlacklisted( + executorBlacklisted: SparkListenerExecutorBlacklisted) + : Unit = synchronized { + updateExecutorBlacklist(executorBlacklisted.executorId, true) + } + + override def onExecutorUnblacklisted( + executorUnblacklisted: SparkListenerExecutorUnblacklisted) + : Unit = synchronized { + updateExecutorBlacklist(executorUnblacklisted.executorId, false) + } + + override def onNodeBlacklisted( + nodeBlacklisted: SparkListenerNodeBlacklisted) + : Unit = synchronized { + // Implicitly blacklist every executor associated with this node, and show this in the UI. + activeStorageStatusList.foreach { status => + if (status.blockManagerId.host == nodeBlacklisted.hostId) { + updateExecutorBlacklist(status.blockManagerId.executorId, true) + } + } + } + + override def onNodeUnblacklisted( + nodeUnblacklisted: SparkListenerNodeUnblacklisted) + : Unit = synchronized { + // Implicitly unblacklist every executor associated with this node, regardless of how + // they may have been blacklisted initially (either explicitly through executor blacklisting + // or implicitly through node blacklisting). Show this in the UI. + activeStorageStatusList.foreach { status => + if (status.blockManagerId.host == nodeUnblacklisted.hostId) { + updateExecutorBlacklist(status.blockManagerId.executorId, false) + } + } + } } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala index 9fb3f35..cd1b02a 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala @@ -85,6 +85,11 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage <th>Shuffle Spill (Memory)</th> <th>Shuffle Spill (Disk)</th> }} + <th> + <span data-toggle="tooltip" title={ToolTips.BLACKLISTED}> + Blacklisted + </span> + </th> </thead> <tbody> {createExecutorTable()} @@ -160,6 +165,7 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage {Utils.bytesToString(v.diskBytesSpilled)} </td> }} + <td>{v.isBlacklisted}</td> </tr> } case None => http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index 9ce8542..371dad9 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -43,6 +43,7 @@ private[spark] object UIData { var shuffleWriteRecords : Long = 0 var memoryBytesSpilled : Long = 0 var diskBytesSpilled : Long = 0 + var isBlacklisted : Int = 0 } class JobUIData( @@ -92,6 +93,7 @@ private[spark] object UIData { var shuffleWriteRecords: Long = _ var memoryBytesSpilled: Long = _ var diskBytesSpilled: Long = _ + var isBlacklisted: Int = _ var schedulingPool: String = "" var description: Option[String] = None http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json index cba44c8..10902ab 100644 --- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json @@ -1,4 +1,32 @@ [ { + "id" : "app-20161116163331-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-16T22:33:29.916GMT", + "endTime" : "2016-11-16T22:33:40.587GMT", + "lastUpdated" : "", + "duration" : 10671, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479335620587, + "startTimeEpoch" : 1479335609916, + "lastUpdatedEpoch" : 0 + } ] +}, { + "id" : "app-20161115172038-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-15T23:20:37.079GMT", + "endTime" : "2016-11-15T23:22:18.874GMT", + "lastUpdated" : "", + "duration" : 101795, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479252138874, + "startTimeEpoch" : 1479252037079, + "lastUpdatedEpoch" : 0 + } ] +}, { "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { @@ -8,8 +36,8 @@ "duration" : 10505, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917391398, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 } ] }, { @@ -23,8 +51,8 @@ "duration" : 57, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917380950, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -34,8 +62,8 @@ "duration" : 10, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380880, "endTimeEpoch" : 1430917380890, + "startTimeEpoch" : 1430917380880, "lastUpdatedEpoch" : 0 } ] }, { @@ -49,8 +77,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426633910242, "endTimeEpoch" : 1426633945177, + "startTimeEpoch" : 1426633910242, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -60,8 +88,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426533910242, "endTimeEpoch" : 1426533945177, + "startTimeEpoch" : 1426533910242, "lastUpdatedEpoch" : 0 } ] }, { @@ -74,8 +102,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1425081758277, "endTimeEpoch" : 1425081766912, + "startTimeEpoch" : 1425081758277, "lastUpdatedEpoch" : 0 } ] }, { @@ -88,8 +116,8 @@ "duration" : 9011, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981779720, "endTimeEpoch" : 1422981788731, + "startTimeEpoch" : 1422981779720, "lastUpdatedEpoch" : 0 } ] }, { @@ -102,8 +130,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981758277, "endTimeEpoch" : 1422981766912, + "startTimeEpoch" : 1422981758277, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json index cba44c8..10902ab 100644 --- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json @@ -1,4 +1,32 @@ [ { + "id" : "app-20161116163331-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-16T22:33:29.916GMT", + "endTime" : "2016-11-16T22:33:40.587GMT", + "lastUpdated" : "", + "duration" : 10671, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479335620587, + "startTimeEpoch" : 1479335609916, + "lastUpdatedEpoch" : 0 + } ] +}, { + "id" : "app-20161115172038-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-15T23:20:37.079GMT", + "endTime" : "2016-11-15T23:22:18.874GMT", + "lastUpdated" : "", + "duration" : 101795, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479252138874, + "startTimeEpoch" : 1479252037079, + "lastUpdatedEpoch" : 0 + } ] +}, { "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { @@ -8,8 +36,8 @@ "duration" : 10505, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917391398, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 } ] }, { @@ -23,8 +51,8 @@ "duration" : 57, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917380950, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -34,8 +62,8 @@ "duration" : 10, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380880, "endTimeEpoch" : 1430917380890, + "startTimeEpoch" : 1430917380880, "lastUpdatedEpoch" : 0 } ] }, { @@ -49,8 +77,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426633910242, "endTimeEpoch" : 1426633945177, + "startTimeEpoch" : 1426633910242, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -60,8 +88,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426533910242, "endTimeEpoch" : 1426533945177, + "startTimeEpoch" : 1426533910242, "lastUpdatedEpoch" : 0 } ] }, { @@ -74,8 +102,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1425081758277, "endTimeEpoch" : 1425081766912, + "startTimeEpoch" : 1425081758277, "lastUpdatedEpoch" : 0 } ] }, { @@ -88,8 +116,8 @@ "duration" : 9011, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981779720, "endTimeEpoch" : 1422981788731, + "startTimeEpoch" : 1422981779720, "lastUpdatedEpoch" : 0 } ] }, { @@ -102,8 +130,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981758277, "endTimeEpoch" : 1422981766912, + "startTimeEpoch" : 1422981758277, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index e7db674..6b9f29e 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -16,6 +16,7 @@ "totalInputBytes" : 28000288, "totalShuffleRead" : 0, "totalShuffleWrite" : 13180, + "isBlacklisted" : false, "maxMemory" : 278302556, "executorLogs" : { } } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json new file mode 100644 index 0000000..5914a1c --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json @@ -0,0 +1,118 @@ +[ { + "id" : "2", + "hostPort" : "172.22.0.167:51487", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 4, + "completedTasks" : 0, + "totalTasks" : 4, + "totalDuration" : 2537, + "totalGCTime" : 88, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : true, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout", + "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr" + } +}, { + "id" : "driver", + "hostPort" : "172.22.0.167:51475", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 0, + "maxTasks" : 0, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 0, + "totalTasks" : 0, + "totalDuration" : 0, + "totalGCTime" : 0, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : true, + "maxMemory" : 384093388, + "executorLogs" : { } +}, { + "id" : "1", + "hostPort" : "172.22.0.167:51490", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 4, + "totalTasks" : 4, + "totalDuration" : 3152, + "totalGCTime" : 68, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : true, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout", + "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr" + } +}, { + "id" : "0", + "hostPort" : "172.22.0.167:51491", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 4, + "completedTasks" : 0, + "totalTasks" : 4, + "totalDuration" : 2551, + "totalGCTime" : 116, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : true, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout", + "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr" + } +}, { + "id" : "3", + "hostPort" : "172.22.0.167:51485", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 12, + "totalTasks" : 12, + "totalDuration" : 2453, + "totalGCTime" : 72, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : true, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout", + "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr" + } +} ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json new file mode 100644 index 0000000..92e249c --- /dev/null +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json @@ -0,0 +1,118 @@ +[ { + "id" : "2", + "hostPort" : "172.22.0.111:64539", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 6, + "completedTasks" : 0, + "totalTasks" : 6, + "totalDuration" : 2792, + "totalGCTime" : 128, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : false, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout", + "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr" + } +}, { + "id" : "driver", + "hostPort" : "172.22.0.111:64527", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 0, + "maxTasks" : 0, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 0, + "totalTasks" : 0, + "totalDuration" : 0, + "totalGCTime" : 0, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : false, + "maxMemory" : 384093388, + "executorLogs" : { } +}, { + "id" : "1", + "hostPort" : "172.22.0.111:64541", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 12, + "totalTasks" : 12, + "totalDuration" : 2613, + "totalGCTime" : 84, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : false, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stdout", + "stderr" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stderr" + } +}, { + "id" : "0", + "hostPort" : "172.22.0.111:64540", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 4, + "completedTasks" : 0, + "totalTasks" : 4, + "totalDuration" : 2741, + "totalGCTime" : 120, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : false, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout", + "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr" + } +}, { + "id" : "3", + "hostPort" : "172.22.0.111:64543", + "isActive" : true, + "rddBlocks" : 0, + "memoryUsed" : 0, + "diskUsed" : 0, + "totalCores" : 4, + "maxTasks" : 4, + "activeTasks" : 0, + "failedTasks" : 0, + "completedTasks" : 4, + "totalTasks" : 4, + "totalDuration" : 3457, + "totalGCTime" : 72, + "totalInputBytes" : 0, + "totalShuffleRead" : 0, + "totalShuffleWrite" : 0, + "isBlacklisted" : false, + "maxMemory" : 384093388, + "executorLogs" : { + "stdout" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stdout", + "stderr" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stderr" + } +} ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json index 9165f54..8820c71 100644 --- a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json @@ -1,67 +1,43 @@ [ { - "id" : "local-1430917381534", + "id" : "app-20161116163331-0000", "name" : "Spark shell", "attempts" : [ { - "startTime" : "2015-05-06T13:03:00.893GMT", - "endTime" : "2015-05-06T13:03:11.398GMT", + "startTime" : "2016-11-16T22:33:29.916GMT", + "endTime" : "2016-11-16T22:33:40.587GMT", "lastUpdated" : "", - "duration" : 10505, - "sparkUser" : "irashid", + "duration" : 10671, + "sparkUser" : "jose", "completed" : true, - "startTimeEpoch" : 1430917380893, - "endTimeEpoch" : 1430917391398, + "endTimeEpoch" : 1479335620587, + "startTimeEpoch" : 1479335609916, "lastUpdatedEpoch" : 0 } ] }, { - "id" : "local-1430917381535", + "id" : "app-20161115172038-0000", "name" : "Spark shell", "attempts" : [ { - "attemptId" : "2", - "startTime" : "2015-05-06T13:03:00.893GMT", - "endTime" : "2015-05-06T13:03:00.950GMT", - "lastUpdated" : "", - "duration" : 57, - "sparkUser" : "irashid", - "completed" : true, - "startTimeEpoch" : 1430917380893, - "endTimeEpoch" : 1430917380950, - "lastUpdatedEpoch" : 0 - }, { - "attemptId" : "1", - "startTime" : "2015-05-06T13:03:00.880GMT", - "endTime" : "2015-05-06T13:03:00.890GMT", + "startTime" : "2016-11-15T23:20:37.079GMT", + "endTime" : "2016-11-15T23:22:18.874GMT", "lastUpdated" : "", - "duration" : 10, - "sparkUser" : "irashid", + "duration" : 101795, + "sparkUser" : "jose", "completed" : true, - "startTimeEpoch" : 1430917380880, - "endTimeEpoch" : 1430917380890, + "endTimeEpoch" : 1479252138874, + "startTimeEpoch" : 1479252037079, "lastUpdatedEpoch" : 0 } ] }, { - "id" : "local-1426533911241", + "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { - "attemptId" : "2", - "startTime" : "2015-03-17T23:11:50.242GMT", - "endTime" : "2015-03-17T23:12:25.177GMT", - "lastUpdated" : "", - "duration" : 34935, - "sparkUser" : "irashid", - "completed" : true, - "startTimeEpoch" : 1426633910242, - "endTimeEpoch" : 1426633945177, - "lastUpdatedEpoch" : 0 - }, { - "attemptId" : "1", - "startTime" : "2015-03-16T19:25:10.242GMT", - "endTime" : "2015-03-16T19:25:45.177GMT", + "startTime" : "2015-05-06T13:03:00.893GMT", + "endTime" : "2015-05-06T13:03:11.398GMT", "lastUpdated" : "", - "duration" : 34935, + "duration" : 10505, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426533910242, - "endTimeEpoch" : 1426533945177, + "endTimeEpoch" : 1430917391398, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json index a525d61..c3fe4db 100644 --- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json @@ -8,8 +8,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981758277, "endTimeEpoch" : 1422981766912, + "startTimeEpoch" : 1422981758277, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json index cc567f6..8281fa7 100644 --- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json @@ -8,8 +8,8 @@ "duration" : 9011, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981779720, "endTimeEpoch" : 1422981788731, + "startTimeEpoch" : 1422981779720, "lastUpdatedEpoch" : 0 } ] }, { @@ -22,8 +22,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981758277, "endTimeEpoch" : 1422981766912, + "startTimeEpoch" : 1422981758277, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json index c934a87..1930281 100644 --- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json @@ -1,4 +1,32 @@ [ { + "id" : "app-20161116163331-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-16T22:33:29.916GMT", + "endTime" : "2016-11-16T22:33:40.587GMT", + "lastUpdated" : "", + "duration" : 10671, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479335620587, + "startTimeEpoch" : 1479335609916, + "lastUpdatedEpoch" : 0 + } ] +}, { + "id" : "app-20161115172038-0000", + "name" : "Spark shell", + "attempts" : [ { + "startTime" : "2016-11-15T23:20:37.079GMT", + "endTime" : "2016-11-15T23:22:18.874GMT", + "lastUpdated" : "", + "duration" : 101795, + "sparkUser" : "jose", + "completed" : true, + "endTimeEpoch" : 1479252138874, + "startTimeEpoch" : 1479252037079, + "lastUpdatedEpoch" : 0 + } ] +}, { "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { @@ -8,8 +36,8 @@ "duration" : 10505, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917391398, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 } ] }, { @@ -23,8 +51,8 @@ "duration" : 57, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380893, "endTimeEpoch" : 1430917380950, + "startTimeEpoch" : 1430917380893, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -34,8 +62,8 @@ "duration" : 10, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1430917380880, "endTimeEpoch" : 1430917380890, + "startTimeEpoch" : 1430917380880, "lastUpdatedEpoch" : 0 } ] }, { @@ -49,8 +77,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426633910242, "endTimeEpoch" : 1426633945177, + "startTimeEpoch" : 1426633910242, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -60,8 +88,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426533910242, "endTimeEpoch" : 1426533945177, + "startTimeEpoch" : 1426533910242, "lastUpdatedEpoch" : 0 } ] }, { @@ -74,8 +102,8 @@ "duration" : 8635, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1425081758277, "endTimeEpoch" : 1425081766912, + "startTimeEpoch" : 1425081758277, "lastUpdatedEpoch" : 0 } ] } ] http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json index f486d46..e8ed96d 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json @@ -8,8 +8,8 @@ "duration" : 9011, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1422981779720, "endTimeEpoch" : 1422981788731, + "startTimeEpoch" : 1422981779720, "lastUpdatedEpoch" : 0 } ] } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json index e63039f..88c6015 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json @@ -9,8 +9,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426633910242, "endTimeEpoch" : 1426633945177, + "startTimeEpoch" : 1426633910242, "lastUpdatedEpoch" : 0 }, { "attemptId" : "1", @@ -20,8 +20,8 @@ "duration" : 34935, "sparkUser" : "irashid", "completed" : true, - "startTimeEpoch" : 1426533910242, "endTimeEpoch" : 1426533945177, + "startTimeEpoch" : 1426533910242, "lastUpdatedEpoch" : 0 } ] } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 0084339..cdebb5f 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -68,9 +68,9 @@ } } }, - "11" : { - "taskId" : 11, - "index" : 3, + "9" : { + "taskId" : 9, + "index" : 1, "attempt" : 0, "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", @@ -79,13 +79,13 @@ "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 2, + "executorDeserializeTime" : 1, "executorDeserializeCpuTime" : 0, - "executorRunTime" : 434, + "executorRunTime" : 436, "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 1, + "resultSerializationTime" : 0, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -105,17 +105,17 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1647, - "writeTime" : 83000, + "bytesWritten" : 1648, + "writeTime" : 98000, "recordsWritten" : 0 } } }, - "14" : { - "taskId" : 14, - "index" : 6, + "10" : { + "taskId" : 10, + "index" : 2, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.832GMT", + "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -149,16 +149,16 @@ }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, - "writeTime" : 88000, + "writeTime" : 76000, "recordsWritten" : 0 } } }, - "13" : { - "taskId" : 13, - "index" : 5, + "11" : { + "taskId" : 11, + "index" : 3, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", + "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -171,7 +171,7 @@ "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 2, + "resultSerializationTime" : 1, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -191,17 +191,17 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 73000, + "bytesWritten" : 1647, + "writeTime" : 83000, "recordsWritten" : 0 } } }, - "10" : { - "taskId" : 10, - "index" : 2, + "12" : { + "taskId" : 12, + "index" : 4, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", + "launchTime" : "2015-02-03T16:43:05.831GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -234,30 +234,30 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 76000, + "bytesWritten" : 1645, + "writeTime" : 101000, "recordsWritten" : 0 } } }, - "9" : { - "taskId" : 9, - "index" : 1, + "13" : { + "taskId" : 13, + "index" : 5, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", + "launchTime" : "2015-02-03T16:43:05.831GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 1, + "executorDeserializeTime" : 2, "executorDeserializeCpuTime" : 0, - "executorRunTime" : 436, + "executorRunTime" : 434, "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 0, + "resultSerializationTime" : 2, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -278,16 +278,16 @@ }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, - "writeTime" : 98000, + "writeTime" : 73000, "recordsWritten" : 0 } } }, - "12" : { - "taskId" : 12, - "index" : 4, + "14" : { + "taskId" : 14, + "index" : 6, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", + "launchTime" : "2015-02-03T16:43:05.832GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -320,8 +320,8 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1645, - "writeTime" : 101000, + "bytesWritten" : 1648, + "writeTime" : 88000, "recordsWritten" : 0 } } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 63fe3b2..85dafcf 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -68,9 +68,9 @@ } } }, - "11" : { - "taskId" : 11, - "index" : 3, + "9" : { + "taskId" : 9, + "index" : 1, "attempt" : 0, "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", @@ -79,13 +79,13 @@ "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 2, + "executorDeserializeTime" : 1, "executorDeserializeCpuTime" : 0, - "executorRunTime" : 434, + "executorRunTime" : 436, "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 1, + "resultSerializationTime" : 0, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -105,17 +105,17 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1647, - "writeTime" : 83000, + "bytesWritten" : 1648, + "writeTime" : 98000, "recordsWritten" : 0 } } }, - "14" : { - "taskId" : 14, - "index" : 6, + "10" : { + "taskId" : 10, + "index" : 2, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.832GMT", + "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -149,16 +149,16 @@ }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, - "writeTime" : 88000, + "writeTime" : 76000, "recordsWritten" : 0 } } }, - "13" : { - "taskId" : 13, - "index" : 5, + "11" : { + "taskId" : 11, + "index" : 3, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", + "launchTime" : "2015-02-03T16:43:05.830GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -171,7 +171,7 @@ "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 2, + "resultSerializationTime" : 1, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -191,17 +191,17 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 73000, + "bytesWritten" : 1647, + "writeTime" : 83000, "recordsWritten" : 0 } } }, - "10" : { - "taskId" : 10, - "index" : 2, + "12" : { + "taskId" : 12, + "index" : 4, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", + "launchTime" : "2015-02-03T16:43:05.831GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -234,30 +234,30 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1648, - "writeTime" : 76000, + "bytesWritten" : 1645, + "writeTime" : 101000, "recordsWritten" : 0 } } }, - "9" : { - "taskId" : 9, - "index" : 1, + "13" : { + "taskId" : 13, + "index" : 5, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.830GMT", + "launchTime" : "2015-02-03T16:43:05.831GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 1, + "executorDeserializeTime" : 2, "executorDeserializeCpuTime" : 0, - "executorRunTime" : 436, + "executorRunTime" : 434, "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, - "resultSerializationTime" : 0, + "resultSerializationTime" : 2, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -278,16 +278,16 @@ }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, - "writeTime" : 98000, + "writeTime" : 73000, "recordsWritten" : 0 } } }, - "12" : { - "taskId" : 12, - "index" : 4, + "14" : { + "taskId" : 14, + "index" : 6, "attempt" : 0, - "launchTime" : "2015-02-03T16:43:05.831GMT", + "launchTime" : "2015-02-03T16:43:05.832GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -320,8 +320,8 @@ "recordsRead" : 0 }, "shuffleWriteMetrics" : { - "bytesWritten" : 1645, - "writeTime" : 101000, + "bytesWritten" : 1648, + "writeTime" : 88000, "recordsWritten" : 0 } } http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json index 5dcbc89..0ed609d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json @@ -3,7 +3,7 @@ "executorDeserializeTime" : [ 1.0, 3.0, 36.0 ], "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0 ], "executorRunTime" : [ 16.0, 28.0, 351.0 ], - "executorCpuTime" : [ 0.0, 0.0, 0.0], + "executorCpuTime" : [ 0.0, 0.0, 0.0 ], "resultSize" : [ 2010.0, 2065.0, 2065.0 ], "jvmGcTime" : [ 0.0, 0.0, 7.0 ], "resultSerializationTime" : [ 0.0, 0.0, 2.0 ], http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index aaeef1f..c2ece6f 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -29,11 +29,11 @@ "value" : "5050" } ], "tasks" : { - "2" : { - "taskId" : 2, - "index" : 2, + "0" : { + "taskId" : 0, + "index" : 0, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.522GMT", + "launchTime" : "2015-03-16T19:25:36.515GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -41,11 +41,11 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "378", - "value" : "378" + "update" : "78", + "value" : "5050" } ], "taskMetrics" : { - "executorDeserializeTime" : 13, + "executorDeserializeTime" : 14, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, @@ -77,11 +77,11 @@ } } }, - "5" : { - "taskId" : 5, - "index" : 5, + "1" : { + "taskId" : 1, + "index" : 1, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.523GMT", + "launchTime" : "2015-03-16T19:25:36.521GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -89,11 +89,11 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "897", - "value" : "3750" + "update" : "247", + "value" : "2175" } ], "taskMetrics" : { - "executorDeserializeTime" : 12, + "executorDeserializeTime" : 14, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, @@ -125,9 +125,9 @@ } } }, - "4" : { - "taskId" : 4, - "index" : 4, + "2" : { + "taskId" : 2, + "index" : 2, "attempt" : 0, "launchTime" : "2015-03-16T19:25:36.522GMT", "executorId" : "<driver>", @@ -137,17 +137,17 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "678", - "value" : "2853" + "update" : "378", + "value" : "378" } ], "taskMetrics" : { - "executorDeserializeTime" : 12, + "executorDeserializeTime" : 13, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, - "resultSerializationTime" : 1, + "resultSerializationTime" : 2, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -173,11 +173,11 @@ } } }, - "7" : { - "taskId" : 7, - "index" : 7, + "3" : { + "taskId" : 3, + "index" : 3, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.524GMT", + "launchTime" : "2015-03-16T19:25:36.522GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -185,11 +185,11 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "1222", - "value" : "4972" + "update" : "572", + "value" : "950" } ], "taskMetrics" : { - "executorDeserializeTime" : 12, + "executorDeserializeTime" : 13, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, @@ -221,11 +221,11 @@ } } }, - "1" : { - "taskId" : 1, - "index" : 1, + "4" : { + "taskId" : 4, + "index" : 4, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.521GMT", + "launchTime" : "2015-03-16T19:25:36.522GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -233,17 +233,17 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "247", - "value" : "2175" + "update" : "678", + "value" : "2853" } ], "taskMetrics" : { - "executorDeserializeTime" : 14, + "executorDeserializeTime" : 12, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, - "resultSerializationTime" : 2, + "resultSerializationTime" : 1, "memoryBytesSpilled" : 0, "diskBytesSpilled" : 0, "inputMetrics" : { @@ -269,11 +269,11 @@ } } }, - "3" : { - "taskId" : 3, - "index" : 3, + "5" : { + "taskId" : 5, + "index" : 5, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.522GMT", + "launchTime" : "2015-03-16T19:25:36.523GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -281,11 +281,11 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "572", - "value" : "950" + "update" : "897", + "value" : "3750" } ], "taskMetrics" : { - "executorDeserializeTime" : 13, + "executorDeserializeTime" : 12, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, @@ -365,11 +365,11 @@ } } }, - "0" : { - "taskId" : 0, - "index" : 0, + "7" : { + "taskId" : 7, + "index" : 7, "attempt" : 0, - "launchTime" : "2015-03-16T19:25:36.515GMT", + "launchTime" : "2015-03-16T19:25:36.524GMT", "executorId" : "<driver>", "host" : "localhost", "taskLocality" : "PROCESS_LOCAL", @@ -377,11 +377,11 @@ "accumulatorUpdates" : [ { "id" : 1, "name" : "my counter", - "update" : "78", - "value" : "5050" + "update" : "1222", + "value" : "4972" } ], "taskMetrics" : { - "executorDeserializeTime" : 14, + "executorDeserializeTime" : 12, "executorDeserializeCpuTime" : 0, "executorRunTime" : 15, "executorCpuTime" : 0, --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org