Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/20584#discussion_r167469495
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala ---
@@ -102,17 +104,27 @@ class FileBasedDataSourceSuite extends QueryTest with
SharedSQLContext {
def testIgnoreMissingFiles(): Unit = {
withTempDir { dir =>
val basePath = dir.getCanonicalPath
+
Seq("0").toDF("a").write.format(format).save(new Path(basePath,
"first").toString)
Seq("1").toDF("a").write.format(format).save(new Path(basePath,
"second").toString)
+
val thirdPath = new Path(basePath, "third")
+ val fs =
thirdPath.getFileSystem(spark.sparkContext.hadoopConfiguration)
Seq("2").toDF("a").write.format(format).save(thirdPath.toString)
+ val files =
fs.listStatus(thirdPath).filter(_.isFile).map(_.getPath)
+
val df = spark.read.format(format).load(
new Path(basePath, "first").toString,
new Path(basePath, "second").toString,
new Path(basePath, "third").toString)
- val fs =
thirdPath.getFileSystem(spark.sparkContext.hadoopConfiguration)
+ // Make sure all data files are deleted and can't be opened.
+ files.foreach(f => fs.delete(f, false))
assert(fs.delete(thirdPath, true))
--- End diff --
Hmmmm .. but it asserts true on the delete completion .. I would be
surprised if it's something not guaranteed ..
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
