This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-2.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push: new 26ad3fe [SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle non-existed path 26ad3fe is described below commit 26ad3fe2604506a2cbc5de0ecd1b5771f3965cb4 Author: yi.wu <yi...@databricks.com> AuthorDate: Mon Mar 16 11:45:25 2020 -0700 [SPARK-31163][SQL] TruncateTableCommand with acl/permission should handle non-existed path ### What changes were proposed in this pull request? This fix #26956 Wrap try-catch on `fs.getFileStatus(path)` within acl/permission in case of the path doesn't exist. ### Why are the changes needed? `truncate table` may fail to re-create path in case of interruption or something else. As a result, next time we `truncate table` on the same table with acl/permission, it will fail due to `FileNotFoundException`. And it also brings behavior change compares to previous Spark version, which could still `truncate table` successfully even if the path doesn't exist. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Added UT. Closes #27923 from Ngone51/fix_truncate. Authored-by: yi.wu <yi...@databricks.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> (cherry picked from commit cb26f636b08aea4c5c6bf5035a359cd3cbf335c0) Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../apache/spark/sql/execution/command/tables.scala | 2 +- .../spark/sql/execution/command/DDLSuite.scala | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 28dc4a4..8f504d3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -470,8 +470,8 @@ case class TruncateTableCommand( var optPermission: Option[FsPermission] = None var optAcls: Option[java.util.List[AclEntry]] = None if (!ignorePermissionAcl) { - val fileStatus = fs.getFileStatus(path) try { + val fileStatus = fs.getFileStatus(path) optPermission = Some(fileStatus.getPermission()) } catch { case NonFatal(_) => // do nothing diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 0aabe86..73565f2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -2009,6 +2009,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { } } + test("SPARK-31163: acl/permission should handle non-existed path when truncating table") { + withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") { + withTable("tab1") { + sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet PARTITIONED BY (col2)") + sql("INSERT INTO tab1 SELECT 'one', 1") + checkAnswer(spark.table("tab1"), Row("one", 1)) + val part = spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head + val path = new File(part.location.getPath) + sql("TRUNCATE TABLE tab1") + // simulate incomplete/unsuccessful truncate + assert(path.exists()) + path.delete() + assert(!path.exists()) + // execute without java.io.FileNotFoundException + sql("TRUNCATE TABLE tab1") + // partition path should be re-created + assert(path.exists()) + } + } + } + test("create temporary view with mismatched schema") { withTable("tab1") { spark.range(10).write.saveAsTable("tab1") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org