This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3094e495095 [SPARK-39149][SQL] SHOW DATABASES command should not quote
database names under legacy mode
3094e495095 is described below
commit 3094e495095635f6c9e83f4646d3321c2a9311f4
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Thu May 12 11:18:18 2022 +0800
[SPARK-39149][SQL] SHOW DATABASES command should not quote database names
under legacy mode
### What changes were proposed in this pull request?
This is a bug of the command legacy mode as it does not fully restore to
the legacy behavior. The legacy v1 SHOW DATABASES command does not quote the
database names. This PR fixes it.
### Why are the changes needed?
bug fix
### Does this PR introduce _any_ user-facing change?
no change by default, unless people turn on legacy mode, in which case SHOW
DATABASES common won't quote the database names.
### How was this patch tested?
new tests
Closes #36508 from cloud-fan/regression.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../execution/datasources/v2/ShowNamespacesExec.scala | 11 ++++++++++-
.../sql/execution/command/ShowNamespacesSuiteBase.scala | 17 +++++++++++++++++
.../sql/execution/command/v1/ShowNamespacesSuite.scala | 13 +++++++++++++
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
index 9dafbd79a52..c55c7b9f985 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
@@ -42,8 +42,17 @@ case class ShowNamespacesExec(
catalog.listNamespaces()
}
+ // Please refer to the rule `KeepLegacyOutputs` for details about legacy
command.
+ // The legacy SHOW DATABASES command does not quote the database names.
+ val isLegacy = output.head.name == "databaseName"
+ val namespaceNames = if (isLegacy && namespaces.forall(_.length == 1)) {
+ namespaces.map(_.head)
+ } else {
+ namespaces.map(_.quoted)
+ }
+
val rows = new ArrayBuffer[InternalRow]()
- namespaces.map(_.quoted).map { ns =>
+ namespaceNames.map { ns =>
if (pattern.map(StringUtils.filterPattern(Seq(ns),
_).nonEmpty).getOrElse(true)) {
rows += toCatalystRow(ns)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
index b3693845c3b..80e545f6e3c 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowNamespacesSuiteBase.scala
@@ -42,6 +42,9 @@ trait ShowNamespacesSuiteBase extends QueryTest with
DDLCommandTestUtils {
protected def builtinTopNamespaces: Seq[String] = Seq.empty
protected def isCasePreserving: Boolean = true
+ protected def createNamespaceWithSpecialName(ns: String): Unit = {
+ sql(s"CREATE NAMESPACE $catalog.`$ns`")
+ }
test("default namespace") {
withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
@@ -124,6 +127,20 @@ trait ShowNamespacesSuiteBase extends QueryTest with
DDLCommandTestUtils {
}
}
+ test("SPARK-39149: keep the legacy no-quote behavior") {
+ Seq(true, false).foreach { legacy =>
+ withSQLConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA.key ->
legacy.toString) {
+ withNamespace(s"$catalog.`123`") {
+ createNamespaceWithSpecialName("123")
+ val res = if (legacy) "123" else "`123`"
+ checkAnswer(
+ sql(s"SHOW NAMESPACES IN $catalog"),
+ (res +: builtinTopNamespaces).map(Row(_)))
+ }
+ }
+ }
+ }
+
test("case sensitivity of the pattern string") {
Seq(true, false).foreach { caseSensitive =>
withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
index a1b32e42ae2..b65a9acb656 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowNamespacesSuite.scala
@@ -18,7 +18,9 @@
package org.apache.spark.sql.execution.command.v1
import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.catalog.CatalogDatabase
import org.apache.spark.sql.execution.command
+import org.apache.spark.util.Utils
/**
* This base suite contains unified tests for the `SHOW NAMESPACES` and `SHOW
DATABASES` commands
@@ -31,6 +33,17 @@ import org.apache.spark.sql.execution.command
trait ShowNamespacesSuiteBase extends command.ShowNamespacesSuiteBase {
override protected def builtinTopNamespaces: Seq[String] = Seq("default")
+ override protected def createNamespaceWithSpecialName(ns: String): Unit = {
+ // Call `ExternalCatalog` directly to bypass the database name validation
in `SessionCatalog`.
+ spark.sharedState.externalCatalog.createDatabase(
+ CatalogDatabase(
+ name = ns,
+ description = "",
+ locationUri = Utils.createTempDir().toURI,
+ properties = Map.empty),
+ ignoreIfExists = false)
+ }
+
test("IN namespace doesn't exist") {
val errMsg = intercept[AnalysisException] {
sql("SHOW NAMESPACES in dummy")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
