Github user vanzin commented on a diff in the pull request: https://github.com/apache/spark/pull/2471#discussion_r18737263 --- Diff: core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala --- @@ -54,35 +66,57 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis // is already known. private var lastModifiedTime = -1L + // A timestamp of when the disk was last accessed to check for event log to delete + private var lastLogCleanTimeMs = -1L + // Mapping of application IDs to their metadata, in descending end time order. Apps are inserted // into the map in order, so the LinkedHashMap maintains the correct ordering. @volatile private var applications: mutable.LinkedHashMap[String, FsApplicationHistoryInfo] = new mutable.LinkedHashMap() /** - * A background thread that periodically checks for event log updates on disk. - * - * If a log check is invoked manually in the middle of a period, this thread re-adjusts the - * time at which it performs the next log check to maintain the same period as before. + * A background thread that periodically do something about event log. * - * TODO: Add a mechanism to update manually. + * If operateFun is invoked manually in the middle of a period, this thread re-adjusts the + * time at which it does operateFun to maintain the same period as before. */ - private val logCheckingThread = new Thread("LogCheckingThread") { - override def run() = Utils.logUncaughtExceptions { - while (true) { - val now = getMonotonicTimeMs() - if (now - lastLogCheckTimeMs > UPDATE_INTERVAL_MS) { - Thread.sleep(UPDATE_INTERVAL_MS) - } else { - // If the user has manually checked for logs recently, wait until - // UPDATE_INTERVAL_MS after the last check time - Thread.sleep(lastLogCheckTimeMs + UPDATE_INTERVAL_MS - now) + private def getThread( + name: String, + lastTimeMsFun: () => Long, --- End diff -- You could manage this value inside the thread itself and avoid having the fields for each thread and the extra code to pass it to this method.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org