Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20138#discussion_r161882635
--- Diff:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
@@ -663,6 +665,95 @@ class FsHistoryProviderSuite extends SparkFunSuite
with BeforeAndAfter with Matc
freshUI.get.ui.store.job(0)
}
+ test("clean up stale app information") {
+ val storeDir = Utils.createTempDir()
+ val conf = createTestConf().set(LOCAL_STORE_DIR,
storeDir.getAbsolutePath())
+ val provider = spy(new FsHistoryProvider(conf))
+ val appId = "new1"
+
+ // Write logs for two app attempts.
+ doReturn(1L).when(provider).getNewLastScanTime()
+ val attempt1 = newLogFile(appId, Some("1"), inProgress = false)
+ writeFile(attempt1, true, None,
+ SparkListenerApplicationStart(appId, Some(appId), 1L, "test",
Some("1")),
+ SparkListenerJobStart(0, 1L, Nil, null),
+ SparkListenerApplicationEnd(5L)
+ )
+ val attempt2 = newLogFile(appId, Some("2"), inProgress = false)
+ writeFile(attempt2, true, None,
+ SparkListenerApplicationStart(appId, Some(appId), 1L, "test",
Some("2")),
+ SparkListenerJobStart(0, 1L, Nil, null),
+ SparkListenerApplicationEnd(5L)
+ )
+ updateAndCheck(provider) { list =>
+ assert(list.size === 1)
+ assert(list(0).id === appId)
+ assert(list(0).attempts.size === 2)
+ }
+
+ // Load the app's UI.
+ val ui = provider.getAppUI(appId, Some("1"))
+ assert(ui.isDefined)
+
+ // Delete the underlying log file for attempt 1 and rescan. The UI
should go away, but since
+ // attempt 2 still exists, listing data should be there.
+ doReturn(2L).when(provider).getNewLastScanTime()
+ attempt1.delete()
+ updateAndCheck(provider) { list =>
+ assert(list.size === 1)
+ assert(list(0).id === appId)
+ assert(list(0).attempts.size === 1)
+ }
+ assert(!ui.get.valid)
+ assert(provider.getAppUI(appId, None) === None)
+
+ // Delete the second attempt's log file. Now everything should go away.
+ doReturn(3L).when(provider).getNewLastScanTime()
+ attempt2.delete()
+ updateAndCheck(provider) { list =>
+ assert(list.isEmpty)
+ }
+ }
+
+ test("SPARK-21571: clean up removes invalid history files") {
+ val clock = new ManualClock(TimeUnit.DAYS.toMillis(120))
--- End diff --
This line:
val maxTime = clock.getTimeMillis() - conf.get(MAX_LOG_AGE_S) * 1000
Without that `maxTime` would be negative and that seems to be triggering a
bug somewhere else. I need to take a look at exactly what's happening there,
but it seems unrelated to this change.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
