This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 303df64 [SPARK-33889][SQL] Fix NPE from `SHOW PARTITIONS` on V2 tables 303df64 is described below commit 303df64b466b7734b3c497955d1cca3e34fb663e Author: Max Gekk <max.g...@gmail.com> AuthorDate: Wed Dec 23 14:34:01 2020 +0000 [SPARK-33889][SQL] Fix NPE from `SHOW PARTITIONS` on V2 tables ### What changes were proposed in this pull request? At `ShowPartitionsExec.run()`, check that a row returned by `listPartitionIdentifiers()` contains a `null` field, and convert it to `"null"`. ### Why are the changes needed? Because `SHOW PARTITIONS` throws NPE on V2 table with `null` partition values. ### Does this PR introduce _any_ user-facing change? Yes ### How was this patch tested? Added new UT to `v2.ShowPartitionsSuite`. Closes #30904 from MaxGekk/fix-npe-show-partitions. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../execution/datasources/v2/ShowPartitionsExec.scala | 6 +++--- .../sql/execution/command/v2/ShowPartitionsSuite.scala | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala index 416dce6..ac24094 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala @@ -53,9 +53,9 @@ case class ShowPartitionsExec( var i = 0 while (i < len) { val dataType = schema(i).dataType - val partValue = row.get(i, dataType) - val partValueStr = Cast(Literal(partValue, dataType), StringType, Some(timeZoneId)) - .eval().toString + val partValueUTF8String = + Cast(Literal(row.get(i, dataType), dataType), StringType, Some(timeZoneId)).eval() + val partValueStr = if (partValueUTF8String == null) "null" else partValueUTF8String.toString partitions(i) = escapePathName(schema(i).name) + "=" + escapePathName(partValueStr) i += 1 } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala index e52c60d..ed0a7df 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.command.v2 -import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.{AnalysisException, Row, SaveMode} import org.apache.spark.sql.execution.command class ShowPartitionsSuite extends command.ShowPartitionsSuiteBase with CommandSuiteBase { @@ -34,4 +34,18 @@ class ShowPartitionsSuite extends command.ShowPartitionsSuiteBase with CommandSu "SHOW PARTITIONS cannot run for a table which does not support partitioning")) } } + + test("SPARK-33889: null and empty string as partition values") { + import testImplicits._ + withNamespaceAndTable("ns", "tbl") { t => + val df = Seq((0, ""), (1, null)).toDF("a", "part") + df.write + .partitionBy("part") + .format("parquet") + .mode(SaveMode.Overwrite) + .saveAsTable(t) + + runShowPartitionsSql(s"SHOW PARTITIONS $t", Row("part=") :: Row("part=null") :: Nil) + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org