This is an automated email from the ASF dual-hosted git repository. mridulm80 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 9c9020ebafd [SPARK-46058][CORE] Add separate flag for privateKeyPassword 9c9020ebafd is described below commit 9c9020ebafd88684d7f10a2b871f9bc14ebba8b4 Author: Hasnain Lakhani <hasnain.lakh...@databricks.com> AuthorDate: Wed Dec 6 19:58:51 2023 -0600 [SPARK-46058][CORE] Add separate flag for privateKeyPassword ### What changes were proposed in this pull request? This PR adds a separate way of configuring the private key for RPC SSL support when using openssl. ### Why are the changes needed? Right now with config inheritance we support: * JKS with password A, PEM with password B * JKS with no password, PEM with password A * JKS and PEM with no password But we do not support the case where JKS has a password and PEM does not. If we set `keyPassword` we will attempt to use it, and cannot set `spark.ssl.rpc.keyPassword` to null to override the password. So let's make it a separate flag as the easiest workaround. This was noticed while migrating some existing deployments to the RPC SSL support where we use openssl support for RPC and use a key with no password ### Does this PR introduce _any_ user-facing change? Yes, this affects how the (currently unreleased) RPC SSL feature is configured going forward ### How was this patch tested? Updated test configs to match the issue I saw, which would fail `SSLFactory.init()` saying key was invalid. Tests now pass. ``` build/sbt > project network-common > testOnly > project network-shuffle > testOnly > project core > test *Ssl* ``` ### Was this patch authored or co-authored using generative AI tooling? No Closes #43998 from hasnain-db/new-flag. Authored-by: Hasnain Lakhani <hasnain.lakh...@databricks.com> Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com> --- .../org/apache/spark/network/TransportContext.java | 1 + .../org/apache/spark/network/ssl/SSLFactory.java | 18 +++++++++++--- .../apache/spark/network/util/TransportConf.java | 11 +++++++-- .../apache/spark/network/ssl/SslSampleConfigs.java | 22 +++++++++++------ .../src/test/resources/unencrypted-certchain.pem | 21 ++++++++++++++++ .../src/test/resources/unencrypted-key.pem | 28 ++++++++++++++++++++++ .../src/test/resources/unencrypted-certchain.pem | 21 ++++++++++++++++ .../src/test/resources/unencrypted-key.pem | 28 ++++++++++++++++++++++ .../main/scala/org/apache/spark/SSLOptions.scala | 23 ++++++++++++++---- .../scala/org/apache/spark/SecurityManager.scala | 2 ++ core/src/test/resources/unencrypted-certchain.pem | 21 ++++++++++++++++ core/src/test/resources/unencrypted-key.pem | 28 ++++++++++++++++++++++ .../scala/org/apache/spark/SSLOptionsSuite.scala | 2 ++ docs/security.md | 8 +++++++ .../src/test/resources/unencrypted-certchain.pem | 21 ++++++++++++++++ .../yarn/src/test/resources/unencrypted-key.pem | 28 ++++++++++++++++++++++ 16 files changed, 266 insertions(+), 17 deletions(-) diff --git a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java index 90ca4f4c46a..9f3b9c59256 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java +++ b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java @@ -262,6 +262,7 @@ public class TransportContext implements Closeable { .requestedCiphers(conf.sslRpcRequestedCiphers()) .keyStore(conf.sslRpcKeyStore(), conf.sslRpcKeyStorePassword()) .privateKey(conf.sslRpcPrivateKey()) + .privateKeyPassword(conf.sslRpcPrivateKeyPassword()) .keyPassword(conf.sslRpcKeyPassword()) .certChain(conf.sslRpcCertChain()) .trustStore( diff --git a/common/network-common/src/main/java/org/apache/spark/network/ssl/SSLFactory.java b/common/network-common/src/main/java/org/apache/spark/network/ssl/SSLFactory.java index 19c19ec2820..0ae83eb5fd6 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/ssl/SSLFactory.java +++ b/common/network-common/src/main/java/org/apache/spark/network/ssl/SSLFactory.java @@ -106,7 +106,7 @@ public class SSLFactory { .build(); nettyServerSslContext = SslContextBuilder - .forServer(b.certChain, b.privateKey, b.keyPassword) + .forServer(b.certChain, b.privateKey, b.privateKeyPassword) .sslProvider(getSslProvider(b)) .build(); } @@ -160,6 +160,7 @@ public class SSLFactory { private File keyStore; private String keyStorePassword; private File privateKey; + private String privateKeyPassword; private String keyPassword; private File certChain; private File trustStore; @@ -215,9 +216,9 @@ public class SSLFactory { } /** - * Sets the Key password + * Sets the key password * - * @param keyPassword The password for the private key + * @param keyPassword The password for the private key in the key store * @return The builder object */ public Builder keyPassword(String keyPassword) { @@ -225,6 +226,17 @@ public class SSLFactory { return this; } + /** + * Sets the private key password + * + * @param privateKeyPassword The password for the private key + * @return The builder object + */ + public Builder privateKeyPassword(String privateKeyPassword) { + this.privateKeyPassword = privateKeyPassword; + return this; + } + /** * Sets a X.509 certificate chain file in PEM format * diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java index eb85d2bb561..53d2c7ab3ef 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java @@ -299,6 +299,13 @@ public class TransportConf { return conf.get("spark.ssl.rpc.keyStorePassword", null); } + /** + * The password to the private key in the key store + */ + public String sslRpcKeyPassword() { + return conf.get("spark.ssl.rpc.keyPassword", null); + } + /** * A PKCS#8 private key file in PEM format; can be relative to the current directory */ @@ -314,8 +321,8 @@ public class TransportConf { /** * The password to the private key */ - public String sslRpcKeyPassword() { - return conf.get("spark.ssl.rpc.keyPassword", null); + public String sslRpcPrivateKeyPassword() { + return conf.get("spark.ssl.rpc.privateKeyPassword", null); } /** diff --git a/common/network-common/src/test/java/org/apache/spark/network/ssl/SslSampleConfigs.java b/common/network-common/src/test/java/org/apache/spark/network/ssl/SslSampleConfigs.java index 2a04d740e8a..04aac240159 100644 --- a/common/network-common/src/test/java/org/apache/spark/network/ssl/SslSampleConfigs.java +++ b/common/network-common/src/test/java/org/apache/spark/network/ssl/SslSampleConfigs.java @@ -50,28 +50,35 @@ public class SslSampleConfigs { public static final String certChainPath = getAbsolutePath("/certchain.pem"); public static final String untrustedKeyStorePath = getAbsolutePath("/untrusted-keystore"); public static final String trustStorePath = getAbsolutePath("/truststore"); - + public static final String unencryptedPrivateKeyPath = getAbsolutePath("/unencrypted-key.pem"); + public static final String unencryptedCertChainPath = + getAbsolutePath("/unencrypted-certchain.pem"); /** * Creates a config map containing the settings needed to enable the RPC SSL feature * All the settings (except the enabled one) are intentionally set on the parent namespace - * so that we can verify settings inheritance works + * so that we can verify settings inheritance works. We intentionally set conflicting + * options for the key password to verify that is handled correctly. */ public static Map<String, String> createDefaultConfigMap() { Map<String, String> confMap = new HashMap<String, String>(); confMap.put("spark.ssl.rpc.enabled", "true"); - // Need this so the other settings get parsed + confMap.put("spark.ssl.rpc.openSslEnabled", "true"); + confMap.put("spark.ssl.rpc.privateKey", SslSampleConfigs.unencryptedPrivateKeyPath); + // intentionally not set + // confMap.put("spark.ssl.rpc.privateKeyPassword", "password"); + confMap.put("spark.ssl.rpc.certChain", SslSampleConfigs.unencryptedCertChainPath); confMap.put("spark.ssl.enabled", "true"); + confMap.put("spark.ssl.keyPassword", "password"); confMap.put("spark.ssl.trustStoreReloadingEnabled", "false"); - confMap.put("spark.ssl.openSslEnabled", "false"); confMap.put("spark.ssl.trustStoreReloadIntervalMs", "10000"); confMap.put("spark.ssl.keyStore", SslSampleConfigs.keyStorePath); confMap.put("spark.ssl.keyStorePassword", "password"); - confMap.put("spark.ssl.privateKey", SslSampleConfigs.privateKeyPath); - confMap.put("spark.ssl.keyPassword", "password"); - confMap.put("spark.ssl.certChain", SslSampleConfigs.certChainPath); confMap.put("spark.ssl.trustStore", SslSampleConfigs.trustStorePath); confMap.put("spark.ssl.trustStorePassword", "password"); + confMap.put("spark.ssl.protocol", "TLSv1.3"); + confMap.put("spark.ssl.standalone.enabled", "true"); + confMap.put("spark.ssl.ui.enabled", "true"); return confMap; } @@ -90,6 +97,7 @@ public class SslSampleConfigs { confMap.put("spark.ssl.rpc.keyStorePassword", "password"); confMap.put("spark.ssl.rpc.privateKey", SslSampleConfigs.privateKeyPath); confMap.put("spark.ssl.rpc.keyPassword", "password"); + confMap.put("spark.ssl.rpc.privateKeyPassword", "password"); confMap.put("spark.ssl.rpc.certChain", SslSampleConfigs.certChainPath); confMap.put("spark.ssl.rpc.trustStore", SslSampleConfigs.trustStorePath); confMap.put("spark.ssl.rpc.trustStorePassword", "password"); diff --git a/common/network-common/src/test/resources/unencrypted-certchain.pem b/common/network-common/src/test/resources/unencrypted-certchain.pem new file mode 100644 index 00000000000..0fbdfaaa3c3 --- /dev/null +++ b/common/network-common/src/test/resources/unencrypted-certchain.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDizCCAnMCFF9A5eNs0Twi7AJJwWunO+KQRT2mMA0GCSqGSIb3DQEBCwUAMIGB +MRgwFgYDVQQDDA9EYXRhYnJpY2tzIHRlc3QxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xETAPBgNVBAsMCFNlcnZpY2VzMRgwFgYD +VQQKDA9EYXRhYnJpY2tzIEluYy4xCzAJBgNVBAYTAlVTMB4XDTIzMTEyMjA2MDgw +M1oXDTMzMTExOTA2MDgwM1owgYExGDAWBgNVBAMMD0RhdGFicmlja3MgdGVzdDET +MBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzERMA8G +A1UECwwIU2VydmljZXMxGDAWBgNVBAoMD0RhdGFicmlja3MgSW5jLjELMAkGA1UE +BhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAL4EfS4hPPKHwTOkQ+sNldSP +YvgLEFI8LORogSex5IHZTAzBktcaTc0W3/xS8Rd0pGOUzlDw6lLeR0M7g2pBdxsU +VZc1YDUt6R5QmBXuRco1jtPsp/Yll3LaQAk56WkiSbgPscm2QqAs1kKWd4/o9Iyt +JcwyUp7DwOVJX9Fohkayf7ktPgNZnTU0/nFaTXYwKSd2vbBs8Rq2oXlEQ88kRM3a +gUmc0y5UeFN6jt6gLNhxzJj6bZpMfojDRlW6nMMQ15Q1dps8dJWsGcOILMqQ6Deh +faS4JfAZZE6uA3b6uyNN8PalnIkJ6G+haXxsitlCprB1TF0y+gUmSPdPrZhA9nc= +-----END CERTIFICATE----- diff --git a/common/network-common/src/test/resources/unencrypted-key.pem b/common/network-common/src/test/resources/unencrypted-key.pem new file mode 100644 index 00000000000..371cbbfb0a0 --- /dev/null +++ b/common/network-common/src/test/resources/unencrypted-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAECggEADSDhb+oe/yCdluntNBpRVjbTSKr6yqsRavX8cSrnt5Rk +eb/I/5elKnM+a1cfPwx0GJbrMqABmm5wL4qOr80FM6rBQX52tgL41sSfcmnKFjGN +RaoCQgKBPVHZVOnL3xfrDQG3WSE+5hNydYBZKjE2gOqJ8KROKC2rpbjv5AOruvX3 +VS9800jq6HLmiQkjZ0eCIDlJe7f9oaDjGid2Mk1Sy/991F3pFRIuywa5OuoC7yBT +r1A2VRRQWuhXktkz8u23dMbEhU1oh2PDXDzb9gCX72c8BBZkol8YFh/64GsLmHmb +wwg1CKgPAs9R68TqXsE3RrjZ9t7iqtEIW9JDmMEGSQKBgQD4Z7ha6ukX91f8ioCm +8O9X0PjLv90VE/JFIR7Ym6YX6qrngf2iOns0utYImrYDbRhPjVBurmB+8E7LY5pS +qYjdGtT3dC6dAXrElED0siEZzORseg89I8dlWEjVa0VzUKPos3GcoLagH+mRXhhu +aXkdSaUy9jWuxwISA3MaaZvGnQKBgQDVGVUb/FI9ES7pd5RzJu+vH43EZKOMG+nF +dk2tZPU0BA4Cfjfk/GOfb7U9mWuKHRXykqPjAP6ZSK4bSAZDSGh75cApDh7WAprX +0JH5iD68Tm8KL/Lo6QAPS2/ON9hG0SX9jUdIQhsQEJgQcHTHEV0RdAKo2nsrT7tr +2F0gbgZInwKBgQChdZlozyPvRgBU0BnLaPPJWrU8iltDZhGlSV/pX1JYXVn03JNl +rSmEHqUcNqN0GqcgnjPXnVRvbfdpUDZw4G1rehNPPJ9HwjxwJgUKh/Xn9TvMHpJl +JSpn/zhoMC+WQqYnjOud6QCLl/KTYFv0+G2W0dWlCE/gaM45szBPzLFKKQKBgQCl +GV5WM1Qn2eNFoI7T9Guoe0Lj0LDhQVMJ2JFv8JME/Ms55T4628v3X53EntOxir1R +VYlBu6iFa8jwfAnWIQhKTYNmi3kah6Qd5oriEEvCquXet610A+k28FQsKhoXK71K +RyXd9tFuzdxyiB4BiRNZDU9uMO9SbBCiClyEXpnhswKBgFRwNyETdflcT3QdLSbL +FM7WWRYxum64ijMqqSPyTuvsIO8c9qwlLgqiFgiawz+MSRVX/dmhrwzBIXKDxYU1 +S/pGdZO63ynOD19xSSoDh7qyPglxkGm5d8vQvmY9myUyEqqwpJHD28dBOrOyLv1K +GdxQh/QJQRFxn4SbkHG3AuiB +-----END PRIVATE KEY----- diff --git a/common/network-shuffle/src/test/resources/unencrypted-certchain.pem b/common/network-shuffle/src/test/resources/unencrypted-certchain.pem new file mode 100644 index 00000000000..0fbdfaaa3c3 --- /dev/null +++ b/common/network-shuffle/src/test/resources/unencrypted-certchain.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDizCCAnMCFF9A5eNs0Twi7AJJwWunO+KQRT2mMA0GCSqGSIb3DQEBCwUAMIGB +MRgwFgYDVQQDDA9EYXRhYnJpY2tzIHRlc3QxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xETAPBgNVBAsMCFNlcnZpY2VzMRgwFgYD +VQQKDA9EYXRhYnJpY2tzIEluYy4xCzAJBgNVBAYTAlVTMB4XDTIzMTEyMjA2MDgw +M1oXDTMzMTExOTA2MDgwM1owgYExGDAWBgNVBAMMD0RhdGFicmlja3MgdGVzdDET +MBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzERMA8G +A1UECwwIU2VydmljZXMxGDAWBgNVBAoMD0RhdGFicmlja3MgSW5jLjELMAkGA1UE +BhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAL4EfS4hPPKHwTOkQ+sNldSP +YvgLEFI8LORogSex5IHZTAzBktcaTc0W3/xS8Rd0pGOUzlDw6lLeR0M7g2pBdxsU +VZc1YDUt6R5QmBXuRco1jtPsp/Yll3LaQAk56WkiSbgPscm2QqAs1kKWd4/o9Iyt +JcwyUp7DwOVJX9Fohkayf7ktPgNZnTU0/nFaTXYwKSd2vbBs8Rq2oXlEQ88kRM3a +gUmc0y5UeFN6jt6gLNhxzJj6bZpMfojDRlW6nMMQ15Q1dps8dJWsGcOILMqQ6Deh +faS4JfAZZE6uA3b6uyNN8PalnIkJ6G+haXxsitlCprB1TF0y+gUmSPdPrZhA9nc= +-----END CERTIFICATE----- diff --git a/common/network-shuffle/src/test/resources/unencrypted-key.pem b/common/network-shuffle/src/test/resources/unencrypted-key.pem new file mode 100644 index 00000000000..371cbbfb0a0 --- /dev/null +++ b/common/network-shuffle/src/test/resources/unencrypted-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAECggEADSDhb+oe/yCdluntNBpRVjbTSKr6yqsRavX8cSrnt5Rk +eb/I/5elKnM+a1cfPwx0GJbrMqABmm5wL4qOr80FM6rBQX52tgL41sSfcmnKFjGN +RaoCQgKBPVHZVOnL3xfrDQG3WSE+5hNydYBZKjE2gOqJ8KROKC2rpbjv5AOruvX3 +VS9800jq6HLmiQkjZ0eCIDlJe7f9oaDjGid2Mk1Sy/991F3pFRIuywa5OuoC7yBT +r1A2VRRQWuhXktkz8u23dMbEhU1oh2PDXDzb9gCX72c8BBZkol8YFh/64GsLmHmb +wwg1CKgPAs9R68TqXsE3RrjZ9t7iqtEIW9JDmMEGSQKBgQD4Z7ha6ukX91f8ioCm +8O9X0PjLv90VE/JFIR7Ym6YX6qrngf2iOns0utYImrYDbRhPjVBurmB+8E7LY5pS +qYjdGtT3dC6dAXrElED0siEZzORseg89I8dlWEjVa0VzUKPos3GcoLagH+mRXhhu +aXkdSaUy9jWuxwISA3MaaZvGnQKBgQDVGVUb/FI9ES7pd5RzJu+vH43EZKOMG+nF +dk2tZPU0BA4Cfjfk/GOfb7U9mWuKHRXykqPjAP6ZSK4bSAZDSGh75cApDh7WAprX +0JH5iD68Tm8KL/Lo6QAPS2/ON9hG0SX9jUdIQhsQEJgQcHTHEV0RdAKo2nsrT7tr +2F0gbgZInwKBgQChdZlozyPvRgBU0BnLaPPJWrU8iltDZhGlSV/pX1JYXVn03JNl +rSmEHqUcNqN0GqcgnjPXnVRvbfdpUDZw4G1rehNPPJ9HwjxwJgUKh/Xn9TvMHpJl +JSpn/zhoMC+WQqYnjOud6QCLl/KTYFv0+G2W0dWlCE/gaM45szBPzLFKKQKBgQCl +GV5WM1Qn2eNFoI7T9Guoe0Lj0LDhQVMJ2JFv8JME/Ms55T4628v3X53EntOxir1R +VYlBu6iFa8jwfAnWIQhKTYNmi3kah6Qd5oriEEvCquXet610A+k28FQsKhoXK71K +RyXd9tFuzdxyiB4BiRNZDU9uMO9SbBCiClyEXpnhswKBgFRwNyETdflcT3QdLSbL +FM7WWRYxum64ijMqqSPyTuvsIO8c9qwlLgqiFgiawz+MSRVX/dmhrwzBIXKDxYU1 +S/pGdZO63ynOD19xSSoDh7qyPglxkGm5d8vQvmY9myUyEqqwpJHD28dBOrOyLv1K +GdxQh/QJQRFxn4SbkHG3AuiB +-----END PRIVATE KEY----- diff --git a/core/src/main/scala/org/apache/spark/SSLOptions.scala b/core/src/main/scala/org/apache/spark/SSLOptions.scala index 51b6b4445ea..26108d885e4 100644 --- a/core/src/main/scala/org/apache/spark/SSLOptions.scala +++ b/core/src/main/scala/org/apache/spark/SSLOptions.scala @@ -45,6 +45,7 @@ import org.apache.spark.network.util.MapConfigProvider * @param keyStore a path to the key-store file * @param keyStorePassword a password to access the key-store file * @param privateKey a PKCS#8 private key file in PEM format + * @param privateKeyPassword a password to access the privateKey file * @param keyPassword a password to access the private key in the key-store * @param keyStoreType the type of the key-store * @param needClientAuth set true if SSL needs client authentication @@ -79,7 +80,8 @@ private[spark] case class SSLOptions( trustStoreReloadIntervalMs: Int = 10000, openSslEnabled: Boolean = false, protocol: Option[String] = None, - enabledAlgorithms: Set[String] = Set.empty) + enabledAlgorithms: Set[String] = Set.empty, + privateKeyPassword: Option[String] = None) extends Logging { /** @@ -170,6 +172,7 @@ private[spark] case class SSLOptions( trustStorePassword.foreach(confMap.put(s"$nsp.trustStorePassword", _)) protocol.foreach(confMap.put(s"$nsp.protocol", _)) confMap.put(s"$nsp.enabledAlgorithms", enabledAlgorithms.mkString(",")) + privateKeyPassword.foreach(confMap.put(s"$nsp.privateKeyPassword", _)) new MapConfigProvider(confMap) } @@ -178,8 +181,8 @@ private[spark] case class SSLOptions( override def toString: String = s"SSLOptions{enabled=$enabled, port=$port, " + s"keyStore=$keyStore, keyStorePassword=${keyStorePassword.map(_ => "xxx")}, " + s"privateKey=$privateKey, keyPassword=${keyPassword.map(_ => "xxx")}, " + - s"keyStoreType=$keyStoreType, needClientAuth=$needClientAuth, " + - s"certChain=$certChain, trustStore=$trustStore, " + + s"privateKeyPassword=${privateKeyPassword.map(_ => "xxx")}, keyStoreType=$keyStoreType, " + + s"needClientAuth=$needClientAuth, certChain=$certChain, trustStore=$trustStore, " + s"trustStorePassword=${trustStorePassword.map(_ => "xxx")}, " + s"trustStoreReloadIntervalMs=$trustStoreReloadIntervalMs, " + s"trustStoreReloadingEnabled=$trustStoreReloadingEnabled, openSSLEnabled=$openSslEnabled, " + @@ -197,7 +200,8 @@ private[spark] object SSLOptions extends Logging { * $ - `[ns].keyStore` - a path to the key-store file; can be relative to the current directory * $ - `[ns].keyStorePassword` - a password to the key-store file * $ - `[ns].privateKey` - a PKCS#8 private key file in PEM format - * $ - `[ns].keyPassword` - a password to the private key + * $ - `[ns].privateKeyPassword` - a password for the above key + * $ - `[ns].keyPassword` - a password to the private key in the key store * $ - `[ns].keyStoreType` - the type of the key-store * $ - `[ns].needClientAuth` - whether SSL needs client authentication * $ - `[ns].certChain` - an X.509 certificate chain file in PEM format @@ -260,6 +264,10 @@ private[spark] object SSLOptions extends Logging { val privateKey = conf.getOption(s"$ns.privateKey").map(new File(_)) .orElse(defaults.flatMap(_.privateKey)) + val privateKeyPassword = conf.getWithSubstitution(s"$ns.privateKeyPassword") + .orElse(Option(conf.getenv(ENV_RPC_SSL_PRIVATE_KEY_PASSWORD)).filter(_.trim.nonEmpty)) + .orElse(defaults.flatMap(_.privateKeyPassword)) + val keyPassword = conf.getWithSubstitution(s"$ns.keyPassword") .orElse(Option(hadoopConf.getPassword(s"$ns.keyPassword")).map(new String(_))) .orElse(Option(conf.getenv(ENV_RPC_SSL_KEY_PASSWORD)).filter(_.trim.nonEmpty)) @@ -320,24 +328,29 @@ private[spark] object SSLOptions extends Logging { trustStoreReloadIntervalMs, openSslEnabled, protocol, - enabledAlgorithms) + enabledAlgorithms, + privateKeyPassword) } // Config names and environment variables for propagating SSL passwords val SPARK_RPC_SSL_KEY_PASSWORD_CONF = "spark.ssl.rpc.keyPassword" + val SPARK_RPC_SSL_PRIVATE_KEY_PASSWORD_CONF = "spark.ssl.rpc.privateKeyPassword" val SPARK_RPC_SSL_KEY_STORE_PASSWORD_CONF = "spark.ssl.rpc.keyStorePassword" val SPARK_RPC_SSL_TRUST_STORE_PASSWORD_CONF = "spark.ssl.rpc.trustStorePassword" val SPARK_RPC_SSL_PASSWORD_FIELDS: Seq[String] = Seq( SPARK_RPC_SSL_KEY_PASSWORD_CONF, + SPARK_RPC_SSL_PRIVATE_KEY_PASSWORD_CONF, SPARK_RPC_SSL_KEY_STORE_PASSWORD_CONF, SPARK_RPC_SSL_TRUST_STORE_PASSWORD_CONF ) val ENV_RPC_SSL_KEY_PASSWORD = "_SPARK_SSL_RPC_KEY_PASSWORD" + val ENV_RPC_SSL_PRIVATE_KEY_PASSWORD = "_SPARK_SSL_RPC_PRIVATE_KEY_PASSWORD" val ENV_RPC_SSL_KEY_STORE_PASSWORD = "_SPARK_SSL_RPC_KEY_STORE_PASSWORD" val ENV_RPC_SSL_TRUST_STORE_PASSWORD = "_SPARK_SSL_RPC_TRUST_STORE_PASSWORD" val SPARK_RPC_SSL_PASSWORD_ENVS: Seq[String] = Seq( ENV_RPC_SSL_KEY_PASSWORD, + ENV_RPC_SSL_PRIVATE_KEY_PASSWORD, ENV_RPC_SSL_KEY_STORE_PASSWORD, ENV_RPC_SSL_TRUST_STORE_PASSWORD ) diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala index ee9051d024c..9771609f01b 100644 --- a/core/src/main/scala/org/apache/spark/SecurityManager.scala +++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala @@ -429,6 +429,8 @@ private[spark] class SecurityManager( val map = scala.collection.mutable.Map[String, String]() rpcSSLOptions.keyPassword.foreach(password => map += (SSLOptions.ENV_RPC_SSL_KEY_PASSWORD -> password)) + rpcSSLOptions.privateKeyPassword.foreach(password => + map += (SSLOptions.ENV_RPC_SSL_PRIVATE_KEY_PASSWORD -> password)) rpcSSLOptions.keyStorePassword.foreach(password => map += (SSLOptions.ENV_RPC_SSL_KEY_STORE_PASSWORD -> password)) rpcSSLOptions.trustStorePassword.foreach(password => diff --git a/core/src/test/resources/unencrypted-certchain.pem b/core/src/test/resources/unencrypted-certchain.pem new file mode 100644 index 00000000000..0fbdfaaa3c3 --- /dev/null +++ b/core/src/test/resources/unencrypted-certchain.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDizCCAnMCFF9A5eNs0Twi7AJJwWunO+KQRT2mMA0GCSqGSIb3DQEBCwUAMIGB +MRgwFgYDVQQDDA9EYXRhYnJpY2tzIHRlc3QxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xETAPBgNVBAsMCFNlcnZpY2VzMRgwFgYD +VQQKDA9EYXRhYnJpY2tzIEluYy4xCzAJBgNVBAYTAlVTMB4XDTIzMTEyMjA2MDgw +M1oXDTMzMTExOTA2MDgwM1owgYExGDAWBgNVBAMMD0RhdGFicmlja3MgdGVzdDET +MBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzERMA8G +A1UECwwIU2VydmljZXMxGDAWBgNVBAoMD0RhdGFicmlja3MgSW5jLjELMAkGA1UE +BhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAL4EfS4hPPKHwTOkQ+sNldSP +YvgLEFI8LORogSex5IHZTAzBktcaTc0W3/xS8Rd0pGOUzlDw6lLeR0M7g2pBdxsU +VZc1YDUt6R5QmBXuRco1jtPsp/Yll3LaQAk56WkiSbgPscm2QqAs1kKWd4/o9Iyt +JcwyUp7DwOVJX9Fohkayf7ktPgNZnTU0/nFaTXYwKSd2vbBs8Rq2oXlEQ88kRM3a +gUmc0y5UeFN6jt6gLNhxzJj6bZpMfojDRlW6nMMQ15Q1dps8dJWsGcOILMqQ6Deh +faS4JfAZZE6uA3b6uyNN8PalnIkJ6G+haXxsitlCprB1TF0y+gUmSPdPrZhA9nc= +-----END CERTIFICATE----- diff --git a/core/src/test/resources/unencrypted-key.pem b/core/src/test/resources/unencrypted-key.pem new file mode 100644 index 00000000000..371cbbfb0a0 --- /dev/null +++ b/core/src/test/resources/unencrypted-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAECggEADSDhb+oe/yCdluntNBpRVjbTSKr6yqsRavX8cSrnt5Rk +eb/I/5elKnM+a1cfPwx0GJbrMqABmm5wL4qOr80FM6rBQX52tgL41sSfcmnKFjGN +RaoCQgKBPVHZVOnL3xfrDQG3WSE+5hNydYBZKjE2gOqJ8KROKC2rpbjv5AOruvX3 +VS9800jq6HLmiQkjZ0eCIDlJe7f9oaDjGid2Mk1Sy/991F3pFRIuywa5OuoC7yBT +r1A2VRRQWuhXktkz8u23dMbEhU1oh2PDXDzb9gCX72c8BBZkol8YFh/64GsLmHmb +wwg1CKgPAs9R68TqXsE3RrjZ9t7iqtEIW9JDmMEGSQKBgQD4Z7ha6ukX91f8ioCm +8O9X0PjLv90VE/JFIR7Ym6YX6qrngf2iOns0utYImrYDbRhPjVBurmB+8E7LY5pS +qYjdGtT3dC6dAXrElED0siEZzORseg89I8dlWEjVa0VzUKPos3GcoLagH+mRXhhu +aXkdSaUy9jWuxwISA3MaaZvGnQKBgQDVGVUb/FI9ES7pd5RzJu+vH43EZKOMG+nF +dk2tZPU0BA4Cfjfk/GOfb7U9mWuKHRXykqPjAP6ZSK4bSAZDSGh75cApDh7WAprX +0JH5iD68Tm8KL/Lo6QAPS2/ON9hG0SX9jUdIQhsQEJgQcHTHEV0RdAKo2nsrT7tr +2F0gbgZInwKBgQChdZlozyPvRgBU0BnLaPPJWrU8iltDZhGlSV/pX1JYXVn03JNl +rSmEHqUcNqN0GqcgnjPXnVRvbfdpUDZw4G1rehNPPJ9HwjxwJgUKh/Xn9TvMHpJl +JSpn/zhoMC+WQqYnjOud6QCLl/KTYFv0+G2W0dWlCE/gaM45szBPzLFKKQKBgQCl +GV5WM1Qn2eNFoI7T9Guoe0Lj0LDhQVMJ2JFv8JME/Ms55T4628v3X53EntOxir1R +VYlBu6iFa8jwfAnWIQhKTYNmi3kah6Qd5oriEEvCquXet610A+k28FQsKhoXK71K +RyXd9tFuzdxyiB4BiRNZDU9uMO9SbBCiClyEXpnhswKBgFRwNyETdflcT3QdLSbL +FM7WWRYxum64ijMqqSPyTuvsIO8c9qwlLgqiFgiawz+MSRVX/dmhrwzBIXKDxYU1 +S/pGdZO63ynOD19xSSoDh7qyPglxkGm5d8vQvmY9myUyEqqwpJHD28dBOrOyLv1K +GdxQh/QJQRFxn4SbkHG3AuiB +-----END PRIVATE KEY----- diff --git a/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala b/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala index ee6bf071ef6..de1aa1ad7c4 100644 --- a/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala +++ b/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala @@ -284,6 +284,7 @@ class SSLOptionsSuite extends SparkFunSuite { test("get passwords from environment") { val conf = new SparkConfWithEnv(Map( SSLOptions.ENV_RPC_SSL_KEY_PASSWORD -> "val1", + SSLOptions.ENV_RPC_SSL_PRIVATE_KEY_PASSWORD -> "val4", SSLOptions.ENV_RPC_SSL_KEY_STORE_PASSWORD -> "val2", SSLOptions.ENV_RPC_SSL_TRUST_STORE_PASSWORD -> "val3")) val hadoopConf = new Configuration() @@ -292,6 +293,7 @@ class SSLOptionsSuite extends SparkFunSuite { val opts = SSLOptions.parse(conf, hadoopConf, "spark.ssl", defaults = None) assert(opts.keyPassword === Some("val1")) + assert(opts.privateKeyPassword === Some("val4")) assert(opts.keyStorePassword === Some("val2")) assert(opts.trustStorePassword === Some("val3")) } diff --git a/docs/security.md b/docs/security.md index 755c7ce8b43..00e35ce2f49 100644 --- a/docs/security.md +++ b/docs/security.md @@ -636,6 +636,14 @@ replaced with one of the above namespaces. </td> <td>rpc</td> </tr> + <tr> + <td><code>${ns}.privateKeyPassword</code></td> + <td>None</td> + <td> + The password to the above private key file in PEM format. + </td> + <td>rpc</td> + </tr> <tr> <td><code>${ns}.certChain</code></td> <td>None</td> diff --git a/resource-managers/yarn/src/test/resources/unencrypted-certchain.pem b/resource-managers/yarn/src/test/resources/unencrypted-certchain.pem new file mode 100644 index 00000000000..0fbdfaaa3c3 --- /dev/null +++ b/resource-managers/yarn/src/test/resources/unencrypted-certchain.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDizCCAnMCFF9A5eNs0Twi7AJJwWunO+KQRT2mMA0GCSqGSIb3DQEBCwUAMIGB +MRgwFgYDVQQDDA9EYXRhYnJpY2tzIHRlc3QxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xETAPBgNVBAsMCFNlcnZpY2VzMRgwFgYD +VQQKDA9EYXRhYnJpY2tzIEluYy4xCzAJBgNVBAYTAlVTMB4XDTIzMTEyMjA2MDgw +M1oXDTMzMTExOTA2MDgwM1owgYExGDAWBgNVBAMMD0RhdGFicmlja3MgdGVzdDET +MBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzERMA8G +A1UECwwIU2VydmljZXMxGDAWBgNVBAoMD0RhdGFicmlja3MgSW5jLjELMAkGA1UE +BhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAL4EfS4hPPKHwTOkQ+sNldSP +YvgLEFI8LORogSex5IHZTAzBktcaTc0W3/xS8Rd0pGOUzlDw6lLeR0M7g2pBdxsU +VZc1YDUt6R5QmBXuRco1jtPsp/Yll3LaQAk56WkiSbgPscm2QqAs1kKWd4/o9Iyt +JcwyUp7DwOVJX9Fohkayf7ktPgNZnTU0/nFaTXYwKSd2vbBs8Rq2oXlEQ88kRM3a +gUmc0y5UeFN6jt6gLNhxzJj6bZpMfojDRlW6nMMQ15Q1dps8dJWsGcOILMqQ6Deh +faS4JfAZZE6uA3b6uyNN8PalnIkJ6G+haXxsitlCprB1TF0y+gUmSPdPrZhA9nc= +-----END CERTIFICATE----- diff --git a/resource-managers/yarn/src/test/resources/unencrypted-key.pem b/resource-managers/yarn/src/test/resources/unencrypted-key.pem new file mode 100644 index 00000000000..371cbbfb0a0 --- /dev/null +++ b/resource-managers/yarn/src/test/resources/unencrypted-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOxuEaPlFpPj6a +JgdieE5KdCA1DLPAITpjQzVpeRJNZ004jmlpzH1kH8y5pnUmzY860Upaow1BJ/eC +KxSh6YtFNiLWdErXjzfJNfgwT4TznIt8yYv3rgEYvOrxvADLA+KGibY5QOjDiyNP +uQrsRi4HE+zBE41ZgZ6h19zd8093SGNyVl7lH8gLKSKqoyAl9GaXpjrPQoMj1TIX +nMeScyCzfiX6SW2OzdcCVt8w0trSbPB19Uga9GC2DAEDp2cCt9jBeRcpZX6hqh0Z +9pCkWiwd2+MmbzaGFkutoXHjlo93cTJlonEmzvXzAMjw/qrJAf1FoYCeuW/RsTKb +MLmFSoODAgMBAAECggEADSDhb+oe/yCdluntNBpRVjbTSKr6yqsRavX8cSrnt5Rk +eb/I/5elKnM+a1cfPwx0GJbrMqABmm5wL4qOr80FM6rBQX52tgL41sSfcmnKFjGN +RaoCQgKBPVHZVOnL3xfrDQG3WSE+5hNydYBZKjE2gOqJ8KROKC2rpbjv5AOruvX3 +VS9800jq6HLmiQkjZ0eCIDlJe7f9oaDjGid2Mk1Sy/991F3pFRIuywa5OuoC7yBT +r1A2VRRQWuhXktkz8u23dMbEhU1oh2PDXDzb9gCX72c8BBZkol8YFh/64GsLmHmb +wwg1CKgPAs9R68TqXsE3RrjZ9t7iqtEIW9JDmMEGSQKBgQD4Z7ha6ukX91f8ioCm +8O9X0PjLv90VE/JFIR7Ym6YX6qrngf2iOns0utYImrYDbRhPjVBurmB+8E7LY5pS +qYjdGtT3dC6dAXrElED0siEZzORseg89I8dlWEjVa0VzUKPos3GcoLagH+mRXhhu +aXkdSaUy9jWuxwISA3MaaZvGnQKBgQDVGVUb/FI9ES7pd5RzJu+vH43EZKOMG+nF +dk2tZPU0BA4Cfjfk/GOfb7U9mWuKHRXykqPjAP6ZSK4bSAZDSGh75cApDh7WAprX +0JH5iD68Tm8KL/Lo6QAPS2/ON9hG0SX9jUdIQhsQEJgQcHTHEV0RdAKo2nsrT7tr +2F0gbgZInwKBgQChdZlozyPvRgBU0BnLaPPJWrU8iltDZhGlSV/pX1JYXVn03JNl +rSmEHqUcNqN0GqcgnjPXnVRvbfdpUDZw4G1rehNPPJ9HwjxwJgUKh/Xn9TvMHpJl +JSpn/zhoMC+WQqYnjOud6QCLl/KTYFv0+G2W0dWlCE/gaM45szBPzLFKKQKBgQCl +GV5WM1Qn2eNFoI7T9Guoe0Lj0LDhQVMJ2JFv8JME/Ms55T4628v3X53EntOxir1R +VYlBu6iFa8jwfAnWIQhKTYNmi3kah6Qd5oriEEvCquXet610A+k28FQsKhoXK71K +RyXd9tFuzdxyiB4BiRNZDU9uMO9SbBCiClyEXpnhswKBgFRwNyETdflcT3QdLSbL +FM7WWRYxum64ijMqqSPyTuvsIO8c9qwlLgqiFgiawz+MSRVX/dmhrwzBIXKDxYU1 +S/pGdZO63ynOD19xSSoDh7qyPglxkGm5d8vQvmY9myUyEqqwpJHD28dBOrOyLv1K +GdxQh/QJQRFxn4SbkHG3AuiB +-----END PRIVATE KEY----- --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org