This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 47af892 [SPARK-28675][SQL] Remove maskCredentials and use
redactOptions
47af892 is described below
commit 47af8925b60509d2a2c932e2bcf25394721c6f12
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Sat Aug 10 16:45:59 2019 -0700
[SPARK-28675][SQL] Remove maskCredentials and use redactOptions
## What changes were proposed in this pull request?
This PR replaces `CatalogUtils.maskCredentials` with
`SQLConf.get.redactOptions` to match other redacts.
## How was this patch tested?
unit test and manual tests:
Before this PR:
```sql
spark-sql> DESC EXTENDED test_spark_28675;
id int NULL
# Detailed Table Information
Database default
Table test_spark_28675
Owner root
Created Time Fri Aug 09 08:23:17 GMT-07:00 2019
Last Access Wed Dec 31 17:00:00 GMT-07:00 1969
Created By Spark 3.0.0-SNAPSHOT
Type MANAGED
Provider org.apache.spark.sql.jdbc
Location file:/user/hive/warehouse/test_spark_28675
Serde Library org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties [url=###, driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
spark-sql> SHOW TABLE EXTENDED LIKE 'test_spark_28675';
default test_spark_28675 false Database: default
Table: test_spark_28675
Owner: root
Created Time: Fri Aug 09 08:23:17 GMT-07:00 2019
Last Access: Wed Dec 31 17:00:00 GMT-07:00 1969
Created By: Spark 3.0.0-SNAPSHOT
Type: MANAGED
Provider: org.apache.spark.sql.jdbc
Location: file:/user/hive/warehouse/test_spark_28675
Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties: [url=###, driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
Schema: root
|-- id: integer (nullable = true)
```
After this PR:
```sql
spark-sql> DESC EXTENDED test_spark_28675;
id int NULL
# Detailed Table Information
Database default
Table test_spark_28675
Owner root
Created Time Fri Aug 09 08:19:49 GMT-07:00 2019
Last Access Wed Dec 31 17:00:00 GMT-07:00 1969
Created By Spark 3.0.0-SNAPSHOT
Type MANAGED
Provider org.apache.spark.sql.jdbc
Location file:/user/hive/warehouse/test_spark_28675
Serde Library org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties [url=*********(redacted), driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
spark-sql> SHOW TABLE EXTENDED LIKE 'test_spark_28675';
default test_spark_28675 false Database: default
Table: test_spark_28675
Owner: root
Created Time: Fri Aug 09 08:19:49 GMT-07:00 2019
Last Access: Wed Dec 31 17:00:00 GMT-07:00 1969
Created By: Spark 3.0.0-SNAPSHOT
Type: MANAGED
Provider: org.apache.spark.sql.jdbc
Location: file:/user/hive/warehouse/test_spark_28675
Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Storage Properties: [url=*********(redacted), driver=com.mysql.jdbc.Driver,
dbtable=test_spark_28675]
Schema: root
|-- id: integer (nullable = true)
```
Closes #25395 from wangyum/SPARK-28675.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../catalyst/catalog/ExternalCatalogUtils.scala | 15 --------
.../spark/sql/catalyst/catalog/interface.scala | 2 +-
.../spark/sql/execution/datasources/ddl.scala | 5 +--
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 42 ++++++++++++++++++++++
4 files changed, 46 insertions(+), 18 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index 50f32e8..4cff162 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -174,21 +174,6 @@ object ExternalCatalogUtils {
}
object CatalogUtils {
- /**
- * Masking credentials in the option lists. For example, in the sql plan
explain output
- * for JDBC data sources.
- */
- def maskCredentials(options: Map[String, String]): Map[String, String] = {
- options.map {
- case (key, _) if key.toLowerCase(Locale.ROOT) == "password" => (key,
"###")
- case (key, value)
- if key.toLowerCase(Locale.ROOT) == "url" &&
- value.toLowerCase(Locale.ROOT).contains("password") =>
- (key, "###")
- case o => o
- }
- }
-
def normalizePartCols(
tableName: String,
tableCols: Seq[String],
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index d32d492..8d89baf 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -74,7 +74,7 @@ case class CatalogStorageFormat(
inputFormat.foreach(map.put("InputFormat", _))
outputFormat.foreach(map.put("OutputFormat", _))
if (compressed) map.put("Compressed", "")
- CatalogUtils.maskCredentials(properties) match {
+ SQLConf.get.redactOptions(properties) match {
case props if props.isEmpty => // No-op
case props =>
map.put("Storage Properties", props.map(p => p._1 + "=" +
p._2).mkString("[", ", ", "]"))
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
index 042320e..4022640 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
@@ -21,10 +21,11 @@ import java.util.Locale
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogUtils}
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.command.{DDLUtils, RunnableCommand}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
/**
@@ -73,7 +74,7 @@ case class CreateTempViewUsing(
userSpecifiedSchema.map(_ + " ").getOrElse("") +
s"replace:$replace " +
s"provider:$provider " +
- CatalogUtils.maskCredentials(options)
+ SQLConf.get.redactOptions(options)
}
override def run(sparkSession: SparkSession): Seq[Row] = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 9f7faa2..158085c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1056,6 +1056,48 @@ class JDBCSuite extends QueryTest
}
}
+ test("Replace CatalogUtils.maskCredentials with SQLConf.get.redactOptions") {
+ val password = "testPass"
+ val tableName = "tab1"
+ withTable(tableName) {
+ sql(
+ s"""
+ |CREATE TABLE $tableName
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (
+ | url '$urlWithUserAndPass',
+ | dbtable 'TEST.PEOPLE',
+ | user 'testUser',
+ | password '$password')
+ """.stripMargin)
+
+ val storageProps = sql(s"DESC FORMATTED $tableName")
+ .filter("col_name = 'Storage Properties'")
+ .select("data_type").collect()
+ assert(storageProps.length === 1)
+ storageProps.foreach { r =>
+
assert(r.getString(0).contains(s"url=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+
assert(r.getString(0).contains(s"password=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+ }
+
+ val information = sql(s"SHOW TABLE EXTENDED LIKE '$tableName'")
+ .select("information").collect()
+ assert(information.length === 1)
+ information.foreach { r =>
+
assert(r.getString(0).contains(s"url=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+
assert(r.getString(0).contains(s"password=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+ }
+
+ val createTabStmt = sql(s"SHOW CREATE TABLE $tableName")
+ .select("createtab_stmt").collect()
+ assert(createTabStmt.length === 1)
+ createTabStmt.foreach { r =>
+ assert(r.getString(0).contains(s"`url`
'${Utils.REDACTION_REPLACEMENT_TEXT}'"))
+ assert(r.getString(0).contains(s"`password`
'${Utils.REDACTION_REPLACEMENT_TEXT}'"))
+ }
+ }
+ }
+
test("SPARK 12941: The data type mapping for StringType to Oracle") {
val oracleDialect = JdbcDialects.get("jdbc:oracle://127.0.0.1/db")
assert(oracleDialect.getJDBCType(StringType).
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
