Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/23158#discussion_r237280881
--- Diff:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
@@ -334,6 +334,42 @@ class FsHistoryProviderSuite extends SparkFunSuite
with BeforeAndAfter with Matc
assert(!log2.exists())
}
+ test("should not clean inprogress application with lastUpdated time less
the maxTime") {
+ val firstFileModifiedTime = TimeUnit.DAYS.toMillis(1)
+ val secondFileModifiedTime = TimeUnit.DAYS.toMillis(6)
+ val maxAge = TimeUnit.DAYS.toMillis(7)
+ val clock = new ManualClock(0)
+ val provider = new FsHistoryProvider(
+ createTestConf().set("spark.history.fs.cleaner.maxAge",
s"${maxAge}ms"), clock)
+ val log = newLogFile("inProgressApp1", None, inProgress = true)
+ writeFile(log, true, None,
+ SparkListenerApplicationStart(
+ "inProgressApp1", Some("inProgressApp1"), 3L, "test",
Some("attempt1"))
+ )
+ clock.setTime(firstFileModifiedTime)
+ provider.checkForLogs()
--- End diff --
You need to set the log file's modified time before calling this, otherwise
the cleaner won't be checking what you expect.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
