This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 49f94eb6e88a [SPARK-46579][SQL] Redact JDBC url in errors and logs 49f94eb6e88a is described below commit 49f94eb6e88a9e5aaff675fb53125ce6091529fa Author: Max Gekk <max.g...@gmail.com> AuthorDate: Wed Jan 3 12:02:02 2024 -0800 [SPARK-46579][SQL] Redact JDBC url in errors and logs ### What changes were proposed in this pull request? In the PR, I propose to redact the JDBC url in error message parameters and logs. ### Why are the changes needed? To avoid leaking of user's secrets. ### Does this PR introduce _any_ user-facing change? Yes, it can. ### How was this patch tested? By running the modified test suites: ``` $ build/sbt "test:testOnly *JDBCTableCatalogSuite" ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44574 from MaxGekk/redact-jdbc-url. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../execution/datasources/jdbc/JDBCOptions.scala | 3 +++ .../jdbc/connection/BasicConnectionProvider.scala | 3 ++- .../execution/datasources/v2/jdbc/JDBCTable.scala | 4 ++-- .../datasources/v2/jdbc/JDBCTableCatalog.scala | 22 +++++++++++----------- .../org/apache/spark/sql/jdbc/JdbcDialects.scala | 2 +- .../v2/jdbc/JDBCTableCatalogSuite.scala | 15 +++++++++++---- 6 files changed, 30 insertions(+), 19 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala index 28fa7b8bf561..43db0c6eef11 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.TimestampNTZType +import org.apache.spark.util.Utils /** * Options for the JDBC data source. @@ -248,6 +249,8 @@ class JDBCOptions( otherOption.parameters.equals(this.parameters) case _ => false } + + def getRedactUrl(): String = Utils.redact(SQLConf.get.stringRedactionPattern, url) } class JdbcOptionsInWrite( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala index 369cf59e0599..57902336ebf2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/connection/BasicConnectionProvider.scala @@ -45,7 +45,8 @@ private[jdbc] class BasicConnectionProvider extends JdbcConnectionProvider with jdbcOptions.asConnectionProperties.asScala.foreach { case(k, v) => properties.put(k, v) } - logDebug(s"JDBC connection initiated with URL: ${jdbcOptions.url} and properties: $properties") + logDebug(s"JDBC connection initiated with URL: ${jdbcOptions.getRedactUrl()} " + + s"and properties: $properties") driver.connect(jdbcOptions.url, properties) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala index c251010881f3..120a68075a8f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala @@ -66,7 +66,7 @@ case class JDBCTable(ident: Identifier, schema: StructType, jdbcOptions: JDBCOpt JdbcUtils.classifyException( errorClass = "FAILED_JDBC.CREATE_INDEX", messageParameters = Map( - "url" -> jdbcOptions.url, + "url" -> jdbcOptions.getRedactUrl(), "indexName" -> toSQLId(indexName), "tableName" -> toSQLId(name)), dialect = JdbcDialects.get(jdbcOptions.url)) { @@ -87,7 +87,7 @@ case class JDBCTable(ident: Identifier, schema: StructType, jdbcOptions: JDBCOpt JdbcUtils.classifyException( errorClass = "FAILED_JDBC.DROP_INDEX", messageParameters = Map( - "url" -> jdbcOptions.url, + "url" -> jdbcOptions.getRedactUrl(), "indexName" -> toSQLId(indexName), "tableName" -> toSQLId(name)), dialect = JdbcDialects.get(jdbcOptions.url)) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala index 976cd3f6e9aa..2eb61b3d01fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala @@ -71,7 +71,7 @@ class JDBCTableCatalog extends TableCatalog val rs = JdbcUtils.classifyException( errorClass = "FAILED_JDBC.GET_TABLES", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(namespace.toSeq)), dialect) { conn.getMetaData.getTables(null, schemaPattern, "%", Array("TABLE")) @@ -90,7 +90,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.TABLE_EXISTS", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "tableName" -> toSQLId(ident)), dialect) { JdbcUtils.withConnection(options)(JdbcUtils.tableExists(_, writeOptions)) @@ -115,7 +115,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.RENAME_TABLE", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "oldName" -> toSQLId(oldIdent), "newName" -> toSQLId(newIdent)), dialect) { @@ -181,7 +181,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.CREATE_TABLE", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "tableName" -> toSQLId(ident)), dialect) { JdbcUtils.createTable(conn, getTableName(ident), schema, caseSensitive, writeOptions) @@ -197,7 +197,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.ALTER_TABLE", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "tableName" -> toSQLId(ident)), dialect) { JdbcUtils.alterTable(conn, getTableName(ident), changes, options) @@ -212,7 +212,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.NAMESPACE_EXISTS", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(namespace.toSeq)), dialect) { JdbcUtils.schemaExists(conn, options, db) @@ -225,7 +225,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.withConnection(options) { conn => JdbcUtils.classifyException( errorClass = "FAILED_JDBC.LIST_NAMESPACES", - messageParameters = Map("url" -> options.url), + messageParameters = Map("url" -> options.getRedactUrl()), dialect) { JdbcUtils.listSchemas(conn, options) } @@ -277,7 +277,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.CREATE_NAMESPACE", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(db)), dialect) { JdbcUtils.createSchema(conn, options, db, comment) @@ -301,7 +301,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.CREATE_NAMESPACE_COMMENT", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(db)), dialect) { JdbcUtils.alterSchemaComment(conn, options, db, set.value) @@ -317,7 +317,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.REMOVE_NAMESPACE_COMMENT", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(db)), dialect) { JdbcUtils.removeSchemaComment(conn, options, db) @@ -344,7 +344,7 @@ class JDBCTableCatalog extends TableCatalog JdbcUtils.classifyException( errorClass = "FAILED_JDBC.DROP_NAMESPACE", messageParameters = Map( - "url" -> options.url, + "url" -> options.getRedactUrl(), "namespace" -> toSQLId(db)), dialect) { JdbcUtils.dropSchema(conn, options, db, cascade) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala index 4825568d88eb..87d3e2d97eba 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala @@ -159,7 +159,7 @@ abstract class JdbcDialect extends Serializable with Logging { val connection = ConnectionProvider.create(driver, options.parameters, options.connectionProviderName) require(connection != null, - s"The driver could not open a JDBC connection. Check the URL: ${options.url}") + s"The driver could not open a JDBC connection. Check the URL: ${options.getRedactUrl()}") connection } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala index 1cd4077b4ec1..fa35c0f33ced 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalogSuite.scala @@ -554,15 +554,22 @@ class JDBCTableCatalogSuite extends QueryTest with SharedSparkSession { test("CREATE TABLE with table property") { withTable("h2.test.new_table") { + val sqlText = "CREATE TABLE h2.test.new_table(i INT, j STRING)" + + " TBLPROPERTIES('ENGINE'='tableEngineName')" checkError( - exception = intercept[AnalysisException] { - sql("CREATE TABLE h2.test.new_table(i INT, j STRING)" + - " TBLPROPERTIES('ENGINE'='tableEngineName')") - }, + exception = intercept[AnalysisException] { sql(sqlText) }, errorClass = "FAILED_JDBC.CREATE_TABLE", parameters = Map( "url" -> url, "tableName" -> "`test`.`new_table`")) + withSQLConf(SQLConf.SQL_STRING_REDACTION_PATTERN.key -> ".*password=.*") { + checkError( + exception = intercept[AnalysisException] { sql(sqlText) }, + errorClass = "FAILED_JDBC.CREATE_TABLE", + parameters = Map( + "url" -> "*********(redacted)", + "tableName" -> "`test`.`new_table`")) + } } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org