This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 54aabb012e8 [SPARK-39350][SQL] Add flag to control breaking change process for: DESC NAMESPACE EXTENDED should redact properties 54aabb012e8 is described below commit 54aabb012e85b5c46773b57960d57de580fa8bba Author: Daniel Tenedorio <daniel.tenedo...@databricks.com> AuthorDate: Wed Jun 8 15:56:13 2022 +0900 [SPARK-39350][SQL] Add flag to control breaking change process for: DESC NAMESPACE EXTENDED should redact properties ### What changes were proposed in this pull request? Add a flag to control breaking change process for: DESC NAMESPACE EXTENDED should redact properties. ### Why are the changes needed? This lets Spark users control how the new behavior rolls out to users. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? This PR extends unit test coverage. Closes #36799 from dtenedor/desc-namespace-breaking-change. Authored-by: Daniel Tenedorio <daniel.tenedo...@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../org/apache/spark/sql/internal/SQLConf.scala | 10 ++++ .../apache/spark/sql/execution/command/ddl.scala | 2 + .../datasources/v2/DescribeNamespaceExec.scala | 3 + .../command/v2/DescribeNamespaceSuite.scala | 64 +++++++++++++++------- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8c7702efd47..4b0d110b077 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3818,6 +3818,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES = + buildConf("spark.sql.legacy.descNamespaceRedactProperties") + .internal() + .doc("When set to false, redact sensitive information in the result of DESC NAMESPACE " + + "EXTENDED. If set to true, it restores the legacy behavior that this sensitive " + + "information was included in the output.") + .version("3.4.0") + .booleanConf + .createWithDefault(false) + /** * Holds information about keys that have been deprecated. * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 5cdcf33d6cd..19b737d7d80 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -190,6 +190,8 @@ case class DescribeDatabaseCommand( val propertiesStr = if (properties.isEmpty) { "" + } else if (SQLConf.get.getConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES)) { + properties.toSeq.sortBy(_._1).mkString("(", ", ", ")") } else { conf.redactOptions(properties).toSeq.sortBy(_._1).mkString("(", ", ", ")") } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala index 75c12ea4201..950511e16c8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala @@ -23,6 +23,7 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsNamespaces} +import org.apache.spark.sql.internal.SQLConf /** * Physical plan node for describing a namespace. @@ -48,6 +49,8 @@ case class DescribeNamespaceExec( val propertiesStr = if (properties.isEmpty) { "" + } else if (SQLConf.get.getConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES)) { + properties.toSeq.sortBy(_._1).mkString("(", ", ", ")") } else { conf.redactOptions(properties.toMap).toSeq.sortBy(_._1).mkString("(", ", ", ")") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala index 645399b9026..3f1108f379e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.command.v2 import org.apache.spark.sql.Row import org.apache.spark.sql.connector.catalog.SupportsNamespaces import org.apache.spark.sql.execution.command +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType} import org.apache.spark.util.Utils @@ -30,28 +31,49 @@ class DescribeNamespaceSuite extends command.DescribeNamespaceSuiteBase with Com override def notFoundMsgPrefix: String = "Namespace" test("DescribeNamespace using v2 catalog") { - withNamespace(s"$catalog.ns1.ns2") { - sql( - s""" - | CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 - | COMMENT 'test namespace' - | LOCATION '/tmp/ns_test' - | WITH DBPROPERTIES (password = 'password') + withSQLConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES.key -> "false") { + withNamespace(s"$catalog.ns1.ns2") { + sql( + s""" + | CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 + | COMMENT 'test namespace' + | LOCATION '/tmp/ns_test' + | WITH DBPROPERTIES (password = 'password') """.stripMargin) - val descriptionDf = sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.ns1.ns2") - assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === - Seq( - ("info_name", StringType), - ("info_value", StringType) - )) - val description = descriptionDf.collect() - assert(description === Seq( - Row("Namespace Name", "ns2"), - Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), - Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), - Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName()), - Row("Properties", "((password,*********(redacted)))")) - ) + val descriptionDf = sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.ns1.ns2") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("info_name", StringType), + ("info_value", StringType) + )) + val description = descriptionDf.collect() + assert(description === Seq( + Row("Namespace Name", "ns2"), + Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), + Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), + Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName()), + Row("Properties", "((password,*********(redacted)))")) + ) + } + } + withSQLConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES.key -> "true") { + withNamespace(s"$catalog.ns1.ns2") { + sql(s"CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 COMMENT " + + "'test namespace' LOCATION '/tmp/ns_test'") + val descriptionDf = sql(s"DESCRIBE NAMESPACE $catalog.ns1.ns2") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("info_name", StringType), + ("info_value", StringType) + )) + val description = descriptionDf.collect() + assert(description === Seq( + Row("Namespace Name", "ns2"), + Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), + Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), + Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName())) + ) + } } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org