This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 0975fe9 [SPARK-27244][CORE] Redact Passwords While Using Option
logConf=true
0975fe9 is described below
commit 0975fe9f7e6ed4154988b8df1487ee90976eef49
Author: Ninad Ingole <robert.wal...@example.com>
AuthorDate: Fri Mar 29 14:16:53 2019 -0500
[SPARK-27244][CORE] Redact Passwords While Using Option logConf=true
When logConf is set to true, config keys that contain password were printed
in cleartext in driver log. This change uses the already present redact method
in Utils, to redact all the passwords based on redact pattern in SparkConf and
then print the conf to driver log thus ensuring that sensitive information like
passwords is not printed in clear text.
This patch was tested through `SparkConfSuite` & then entire unit test
through sbt
Please review http://spark.apache.org/contributing.html before opening a
pull request.
Closes #24196 from ninadingole/SPARK-27244.
Authored-by: Ninad Ingole <robert.wal...@example.com>
Signed-off-by: Sean Owen <sean.o...@databricks.com>
(cherry picked from commit dbc7ce18b934fbfd0743b1348fc1265778f07027)
Signed-off-by: Sean Owen <sean.o...@databricks.com>
---
core/src/main/scala/org/apache/spark/SparkConf.scala | 2 +-
core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 10 +++++++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala
b/core/src/main/scala/org/apache/spark/SparkConf.scala
index f53b2be..cfceedf 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -579,7 +579,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable
with Logging with Seria
* configuration out for debugging.
*/
def toDebugString: String = {
- getAll.sorted.map{case (k, v) => k + "=" + v}.mkString("\n")
+ Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v
}.mkString("\n")
}
}
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index bff808e..42d66ec 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -30,7 +30,7 @@ import org.apache.spark.deploy.history.config._
import org.apache.spark.internal.config._
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.serializer.{JavaSerializer, KryoRegistrator,
KryoSerializer}
-import org.apache.spark.util.{ResetSystemProperties, RpcUtils}
+import org.apache.spark.util.{ResetSystemProperties, RpcUtils, Utils}
class SparkConfSuite extends SparkFunSuite with LocalSparkContext with
ResetSystemProperties {
test("Test byteString conversion") {
@@ -339,6 +339,14 @@ class SparkConfSuite extends SparkFunSuite with
LocalSparkContext with ResetSyst
}
}
+ test("SPARK-27244 toDebugString should redact passwords") {
+ val conf = new SparkConf().set("dummy.password", "dummy-password")
+ conf.validateSettings()
+
+ assert(conf.get("dummy.password") === "dummy-password")
+
assert(conf.toDebugString.contains(s"dummy.password=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+ }
+
}
class Class1 {}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
