This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1f23edfa84aa [SPARK-46827][CORE] Make `RocksDBPersistenceEngine` to support a symbolic link 1f23edfa84aa is described below commit 1f23edfa84aa3318791d5fbbbae22d479a49134a Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Wed Jan 24 07:35:14 2024 -0800 [SPARK-46827][CORE] Make `RocksDBPersistenceEngine` to support a symbolic link ### What changes were proposed in this pull request? This PR aims to make `RocksDBPersistenceEngine` to support a symbolic link location. ### Why are the changes needed? To be consistent with `FileSystemPersistenceEngine` which supports symbolic link locations. https://github.com/apache/spark/blob/7004dd9edcad32d34d0448df9498d32c444ab082/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala#L45-L50 ### Does this PR introduce _any_ user-facing change? No. This is a new feature at 4.0.0. ### How was this patch tested? Pass the CIs with a newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44867 from dongjoon-hyun/SPARK-46827. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../spark/deploy/master/RocksDBPersistenceEngine.scala | 9 +++++++-- .../spark/deploy/master/PersistenceEngineSuite.scala | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala index 5c43dab4d066..8364dbd693b1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala @@ -19,7 +19,7 @@ package org.apache.spark.deploy.master import java.nio.ByteBuffer import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Paths} +import java.nio.file.{FileAlreadyExistsException, Files, Paths} import scala.collection.mutable.ArrayBuffer import scala.reflect.ClassTag @@ -43,7 +43,12 @@ private[master] class RocksDBPersistenceEngine( RocksDB.loadLibrary() - private val path = Files.createDirectories(Paths.get(dir)) + private val path = try { + Files.createDirectories(Paths.get(dir)) + } catch { + case _: FileAlreadyExistsException if Files.isSymbolicLink(Paths.get(dir)) => + Files.createDirectories(Paths.get(dir).toRealPath()) + } /** * Use full filter. diff --git a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala index b977a1142444..01b7e46eb2a8 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala @@ -88,6 +88,21 @@ class PersistenceEngineSuite extends SparkFunSuite { } } + test("SPARK-46827: RocksDBPersistenceEngine with a symbolic link") { + withTempDir { dir => + val target = Paths.get(dir.getAbsolutePath(), "target") + val link = Paths.get(dir.getAbsolutePath(), "symbolic_link"); + + Files.createDirectories(target) + Files.createSymbolicLink(link, target); + + val conf = new SparkConf() + testPersistenceEngine(conf, serializer => + new RocksDBPersistenceEngine(link.toAbsolutePath.toString, serializer) + ) + } + } + test("SPARK-46205: Support KryoSerializer in FileSystemPersistenceEngine") { withTempDir { dir => val conf = new SparkConf() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org