This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new ac4d04e [SPARK-33777][SQL] Sort output of V2 SHOW PARTITIONS ac4d04e is described below commit ac4d04e480004e8b5de55e1323d27bd3c8bf28be Author: Max Gekk <max.g...@gmail.com> AuthorDate: Mon Dec 14 14:28:47 2020 -0800 [SPARK-33777][SQL] Sort output of V2 SHOW PARTITIONS ### What changes were proposed in this pull request? List partitions returned by the V2 `SHOW PARTITIONS` command in alphabetical order. ### Why are the changes needed? To have the same behavior as: 1. V1 in-memory catalog, see https://github.com/apache/spark/blob/a28ed86a387b286745b30cd4d90b3d558205a5a7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala#L546 2. V1 Hive catalogs, see https://github.com/apache/spark/blob/fab2995972761503563fa2aa547c67047c51bd33/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala#L715 ### Does this PR introduce _any_ user-facing change? Yes, after the changes, V2 SHOW PARTITIONS sorts its output. ### How was this patch tested? Added new UT to the base trait `ShowPartitionsSuiteBase` which contains tests for V1 and V2. Closes #30764 from MaxGekk/sort-show-partitions. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> (cherry picked from commit f156718587fc33b9bf8e5abc4ae1f6fa0a5da887) Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../execution/datasources/v2/ShowPartitionsExec.scala | 5 +++-- .../sql/execution/command/ShowPartitionsSuiteBase.scala | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala index c4b6aa8..416dce6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala @@ -49,7 +49,7 @@ case class ShowPartitionsExec( val len = schema.length val partitions = new Array[String](len) val timeZoneId = SQLConf.get.sessionLocalTimeZone - partitionIdentifiers.map { row => + val output = partitionIdentifiers.map { row => var i = 0 while (i < len) { val dataType = schema(i).dataType @@ -59,7 +59,8 @@ case class ShowPartitionsExec( partitions(i) = escapePathName(schema(i).name) + "=" + escapePathName(partValueStr) i += 1 } - InternalRow(UTF8String.fromString(partitions.mkString("/"))) + partitions.mkString("/") } + output.sorted.map(p => InternalRow(UTF8String.fromString(p))) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala index b695dec..56c6e5a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala @@ -173,4 +173,21 @@ trait ShowPartitionsSuiteBase extends QueryTest with SQLTestUtils { } } } + + test("SPARK-33777: sorted output") { + withNamespace(s"$catalog.ns") { + sql(s"CREATE NAMESPACE $catalog.ns") + val table = s"$catalog.ns.dateTable" + withTable(table) { + sql(s""" + |CREATE TABLE $table (id int, part string) + |$defaultUsing + |PARTITIONED BY (part)""".stripMargin) + sql(s"ALTER TABLE $table ADD PARTITION(part = 'b')") + sql(s"ALTER TABLE $table ADD PARTITION(part = 'a')") + val partitions = sql(s"show partitions $table") + assert(partitions.first().getString(0) === "part=a") + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org