Repository: spark Updated Branches: refs/heads/master 19f32f2d9 -> abc6c42c2
[SPARK-13241][WEB UI] Added long values for dates in ApplicationAttemptInfo API ## What changes were proposed in this pull request? Adding long values for each Date in the ApplicationAttemptInfo API for easier use in code ## How was the this patch tested? Tested with dev/run-tests Author: Alex Bozarth <ajboz...@us.ibm.com> Closes #11326 from ajbozarth/spark13241. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/abc6c42c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/abc6c42c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/abc6c42c Branch: refs/heads/master Commit: abc6c42c2d76d6aa4a8cac605cb4214e8f8f3328 Parents: 19f32f2 Author: Alex Bozarth <ajboz...@us.ibm.com> Authored: Fri Apr 1 16:18:09 2016 -0700 Committer: Sean Owen <so...@cloudera.com> Committed: Fri Apr 1 16:18:09 2016 -0700 ---------------------------------------------------------------------- .../org/apache/spark/status/api/v1/api.scala | 6 ++++- .../application_list_json_expectation.json | 24 ++++++++++++++++++++ .../completed_app_list_json_expectation.json | 24 ++++++++++++++++++++ .../maxDate2_app_list_json_expectation.json | 3 +++ .../maxDate_app_list_json_expectation.json | 6 +++++ .../minDate_app_list_json_expectation.json | 18 +++++++++++++++ .../one_app_json_expectation.json | 3 +++ .../one_app_multi_attempt_json_expectation.json | 6 +++++ .../deploy/history/HistoryServerSuite.scala | 4 +++- 9 files changed, 92 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/main/scala/org/apache/spark/status/api/v1/api.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index d43868b..ebbbf48 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -38,7 +38,11 @@ class ApplicationAttemptInfo private[spark]( val lastUpdated: Date, val duration: Long, val sparkUser: String, - val completed: Boolean = false) + val completed: Boolean = false) { + def getStartTimeEpoch: Long = startTime.getTime + def getEndTimeEpoch: Long = endTime.getTime + def getLastUpdatedEpoch: Long = lastUpdated.getTime +} class ExecutorStageSummary private[spark]( val taskTime : Long, http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json index 5bbb4ce..1a13233 100644 --- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917391398, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:11.398GMT", "lastUpdated" : "", @@ -14,6 +17,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917380950, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:00.950GMT", "lastUpdated" : "", @@ -22,6 +28,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1430917380880, + "endTimeEpoch" : 1430917380890, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.880GMT", "endTime" : "2015-05-06T13:03:00.890GMT", "lastUpdated" : "", @@ -34,6 +43,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1426633910242, + "endTimeEpoch" : 1426633945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-17T23:11:50.242GMT", "endTime" : "2015-03-17T23:12:25.177GMT", "lastUpdated" : "", @@ -42,6 +54,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1426533910242, + "endTimeEpoch" : 1426533945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-16T19:25:10.242GMT", "endTime" : "2015-03-16T19:25:45.177GMT", "lastUpdated" : "", @@ -53,6 +68,9 @@ "id" : "local-1425081759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1425081758277, + "endTimeEpoch" : 1425081766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-28T00:02:38.277GMT", "endTime" : "2015-02-28T00:02:46.912GMT", "lastUpdated" : "", @@ -64,6 +82,9 @@ "id" : "local-1422981780767", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981779720, + "endTimeEpoch" : 1422981788731, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:59.720GMT", "endTime" : "2015-02-03T16:43:08.731GMT", "lastUpdated" : "", @@ -75,6 +96,9 @@ "id" : "local-1422981759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981758277, + "endTimeEpoch" : 1422981766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:38.277GMT", "endTime" : "2015-02-03T16:42:46.912GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json index 5bbb4ce..1a13233 100644 --- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917391398, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:11.398GMT", "lastUpdated" : "", @@ -14,6 +17,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917380950, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:00.950GMT", "lastUpdated" : "", @@ -22,6 +28,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1430917380880, + "endTimeEpoch" : 1430917380890, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.880GMT", "endTime" : "2015-05-06T13:03:00.890GMT", "lastUpdated" : "", @@ -34,6 +43,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1426633910242, + "endTimeEpoch" : 1426633945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-17T23:11:50.242GMT", "endTime" : "2015-03-17T23:12:25.177GMT", "lastUpdated" : "", @@ -42,6 +54,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1426533910242, + "endTimeEpoch" : 1426533945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-16T19:25:10.242GMT", "endTime" : "2015-03-16T19:25:45.177GMT", "lastUpdated" : "", @@ -53,6 +68,9 @@ "id" : "local-1425081759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1425081758277, + "endTimeEpoch" : 1425081766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-28T00:02:38.277GMT", "endTime" : "2015-02-28T00:02:46.912GMT", "lastUpdated" : "", @@ -64,6 +82,9 @@ "id" : "local-1422981780767", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981779720, + "endTimeEpoch" : 1422981788731, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:59.720GMT", "endTime" : "2015-02-03T16:43:08.731GMT", "lastUpdated" : "", @@ -75,6 +96,9 @@ "id" : "local-1422981759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981758277, + "endTimeEpoch" : 1422981766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:38.277GMT", "endTime" : "2015-02-03T16:42:46.912GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json index 3f80a52..eacf04b 100644 --- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1422981759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981758277, + "endTimeEpoch" : 1422981766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:38.277GMT", "endTime" : "2015-02-03T16:42:46.912GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json index 508bdc1..adad25b 100644 --- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1422981780767", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981779720, + "endTimeEpoch" : 1422981788731, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:59.720GMT", "endTime" : "2015-02-03T16:43:08.731GMT", "lastUpdated" : "", @@ -13,6 +16,9 @@ "id" : "local-1422981759269", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981758277, + "endTimeEpoch" : 1422981766912, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:38.277GMT", "endTime" : "2015-02-03T16:42:46.912GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json index 5dca7d7..a658909 100644 --- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1430917381534", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917391398, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:11.398GMT", "lastUpdated" : "", @@ -14,6 +17,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1430917380893, + "endTimeEpoch" : 1430917380950, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.893GMT", "endTime" : "2015-05-06T13:03:00.950GMT", "lastUpdated" : "", @@ -22,6 +28,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1430917380880, + "endTimeEpoch" : 1430917380890, + "lastUpdatedEpoch" : 0, "startTime" : "2015-05-06T13:03:00.880GMT", "endTime" : "2015-05-06T13:03:00.890GMT", "lastUpdated" : "", @@ -34,6 +43,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1426633910242, + "endTimeEpoch" : 1426633945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-17T23:11:50.242GMT", "endTime" : "2015-03-17T23:12:25.177GMT", "lastUpdated" : "", @@ -42,6 +54,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1426533910242, + "endTimeEpoch" : 1426533945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-16T19:25:10.242GMT", "endTime" : "2015-03-16T19:25:45.177GMT", "lastUpdated" : "", @@ -54,6 +69,9 @@ "name": "Spark shell", "attempts": [ { + "startTimeEpoch" : 1425081758277, + "endTimeEpoch" : 1425081766912, + "lastUpdatedEpoch" : 0, "startTime": "2015-02-28T00:02:38.277GMT", "endTime": "2015-02-28T00:02:46.912GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json index cca32c7..0217fac 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json @@ -2,6 +2,9 @@ "id" : "local-1422981780767", "name" : "Spark shell", "attempts" : [ { + "startTimeEpoch" : 1422981779720, + "endTimeEpoch" : 1422981788731, + "lastUpdatedEpoch" : 0, "startTime" : "2015-02-03T16:42:59.720GMT", "endTime" : "2015-02-03T16:43:08.731GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json ---------------------------------------------------------------------- diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json index 1ea1779..b20a266 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json @@ -3,6 +3,9 @@ "name" : "Spark shell", "attempts" : [ { "attemptId" : "2", + "startTimeEpoch" : 1426633910242, + "endTimeEpoch" : 1426633945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-17T23:11:50.242GMT", "endTime" : "2015-03-17T23:12:25.177GMT", "lastUpdated" : "", @@ -11,6 +14,9 @@ "completed" : true }, { "attemptId" : "1", + "startTimeEpoch" : 1426533910242, + "endTimeEpoch" : 1426533945177, + "lastUpdatedEpoch" : 0, "startTime" : "2015-03-16T19:25:10.242GMT", "endTime" : "2015-03-16T19:25:45.177GMT", "lastUpdated" : "", http://git-wip-us.apache.org/repos/asf/spark/blob/abc6c42c/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 79e4efb..2a013ac 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -162,7 +162,9 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers val json = if (jsonOrg.indexOf("lastUpdated") >= 0) { val subStrings = jsonOrg.split(",") for (i <- subStrings.indices) { - if (subStrings(i).indexOf("lastUpdated") >= 0) { + if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) { + subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0") + } else if (subStrings(i).indexOf("lastUpdated") >= 0) { subStrings(i) = "\"lastUpdated\":\"\"" } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org