xuanyuanking commented on a change in pull request #32933: URL: https://github.com/apache/spark/pull/32933#discussion_r661308771
########## File path: sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBFileManager.scala ########## @@ -153,6 +156,139 @@ class RocksDBFileManager( logInfo(s"Saved checkpoint file for version $version") } + /** + * Load all necessary files for specific checkpoint version from DFS to given local directory. + * If version is 0, then it will deleted all files in the directory. For other versions, it + * ensures that only the exact files generated during checkpointing will be present in the + * local directory. + */ + def loadCheckpointFromDfs(version: Long, localDir: File): RocksDBCheckpointMetadata = { + logInfo(s"Loading checkpoint files for version $version") + val metadata = if (version == 0) { + if (localDir.exists) Utils.deleteRecursively(localDir) + localDir.mkdirs() + RocksDBCheckpointMetadata(Seq.empty, 0) + } else { + // Delete all non-immutable files in local dir, and unzip new ones from DFS commit file + listRocksDBFiles(localDir)._2.foreach(_.delete()) + Utils.unzipFromFile(fs, dfsBatchZipFile(version), localDir) + + // Copy the necessary immutable files + val metadataFile = localMetadataFile(localDir) + val metadata = RocksDBCheckpointMetadata.readFromFile(metadataFile) + logInfo(s"Read metadata for version $version:\n${metadata.prettyJson}") + loadImmutableFilesFromDfs(metadata.immutableFiles, localDir) + versionToRocksDBFiles.put(version, metadata.immutableFiles) + metadataFile.delete() + metadata + } + logFilesInDir(localDir, s"Loaded checkpoint files for version $version") + metadata + } + + /** Get the latest version available in the DFS directory. If no data present, it returns 0. */ + def getLatestVersion(): Long = { + val path = new Path(dfsRootDir) + if (fm.exists(path)) { + fm.list(path, onlyZipFiles) + .map(_.getPath.getName.stripSuffix(".zip")) + .map(_.toLong) + .foldLeft(0L)(math.max) + } else { + 0 + } + } + + /** + * Delete old versions by deleting the associated version and SST files. + * At a high-level, this method finds which versions to delete, and which SST files that were + * last used in those versions. Its safe to delete these SST files because a SST file can + * be reused only in successive versions. Therefore, if a SST file F was last used in version + * V, then it wont be used in version V+1 or later, and if version V can be deleted, then + * F can safely be deleted as well. + * + * To find old files, it does the following. + * - List all the existing [version].zip files + * - Find the min version that needs to be retained based on the given `numVersionsToRetain`. + * - Accordingly decide which versions should be deleted. + * - Resolve all SSTs files of all the existing versions, if not already resolved. + * - Find what was the latest version in which each SST file was used. + * - Delete the files that were last used in the to-be-deleted versions as we will not + * need those files any more. + * + * Note that it only deletes files that it knows are safe to delete. + * It may not delete the following files. + * - Partially written SST files + * - SST files that were used in a version, but that version got overwritten with a different + * set of SST files. + */ + def deleteOldVersions(numVersionsToRetain: Int): Unit = { + val path = new Path(dfsRootDir) + + // All versions present in DFS, sorted + val sortedVersions = fm.list(path, onlyZipFiles) + .map(_.getPath.getName.stripSuffix(".zip")) + .map(_.toLong) + .sorted + + // Return if no versions generated yet + if (sortedVersions.isEmpty) return + + // Find the versions to delete + val maxVersionPresent = sortedVersions.last + val minVersionPresent = sortedVersions.head + val minVersionToRetain = + math.max(minVersionPresent, maxVersionPresent - numVersionsToRetain + 1) + val versionsToDelete = sortedVersions.takeWhile(_ < minVersionToRetain).toSet[Long] + + // Return if no version to delete + if (versionsToDelete.isEmpty) return + + logInfo( + s"Versions present: (min $minVersionPresent, max $maxVersionPresent), " + + s"cleaning up all versions older than $minVersionToRetain to retain last " + + s"$numVersionsToRetain versions") + + // Resolve RocksDB files for all the versions and find the max version each file is used + val fileToMaxUsedVersion = new mutable.HashMap[RocksDBImmutableFile, Long] + sortedVersions.foreach { version => + val files = Option(versionToRocksDBFiles.get(version)).getOrElse { + val newResolvedFiles = getImmutableFilesFromVersionZip(version) + versionToRocksDBFiles.put(version, newResolvedFiles) + newResolvedFiles + } + files.foreach(f => fileToMaxUsedVersion(f) = version) + } + + // Best effort attempt to delete SST files that were last used in to-be-deleted versions + val filesToDelete = fileToMaxUsedVersion.filter { case (_, v) => versionsToDelete.contains(v)} + logInfo(s"Deleting ${filesToDelete.size} files not used in versions >= $minVersionToRetain") + filesToDelete.foreach { case (file, maxUsedVersion) => + try { + val dfsFile = dfsFilePath(file.dfsFileName) + fm.delete(dfsFile) + logDebug(s"Deleted file $file that was last used in version $maxUsedVersion") + } catch { + case e: Exception => + logWarning(s"Error deleting file $file, last used in version $maxVersionPresent", e) + } + } + + // Delete the version files and forget about them + versionsToDelete.foreach { version => + try { + val versionFile = dfsBatchZipFile(version) + fm.delete(versionFile) + versionToRocksDBFiles.remove(version) + logDebug(s"Deleted version $version") + } catch { + case e: Exception => + logWarning(s"Error deleting version file $version", e) Review comment: Make sense, done in the next commit. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org