This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 59570d6202e HIVE-27675: Support keystore/truststore types for hive to zookeeper integration (Naveen Gangam) (#4691) 59570d6202e is described below commit 59570d6202e6c29dad0824af80e241339cf89f83 Author: Naveen Gangam <ngan...@cloudera.com> AuthorDate: Fri Sep 15 10:38:20 2023 -0400 HIVE-27675: Support keystore/truststore types for hive to zookeeper integration (Naveen Gangam) (#4691) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 16 +++++++++++++-- .../hcatalog/templeton/tool/ZooKeeperStorage.java | 4 ++++ .../security/ZooKeeperTokenStoreTestBase.java | 5 +++++ .../org/apache/hive/jdbc/TestRestrictedList.java | 2 ++ .../InformationSchemaWithPrivilegeTestBase.java | 5 +++++ .../java/org/apache/hive/jdbc/HiveConnection.java | 4 +++- jdbc/src/java/org/apache/hive/jdbc/Utils.java | 24 +++++++++++++++++++++- .../hive/jdbc/ZooKeeperHiveClientHelper.java | 9 ++++++-- .../hadoop/hive/registry/impl/ZkRegistryBase.java | 2 ++ .../hadoop/hive/common/SSLZookeeperFactory.java | 14 ++++++++++--- .../hadoop/hive/common/ZooKeeperHiveHelper.java | 24 +++++++++++++++++++++- .../hadoop/hive/metastore/conf/MetastoreConf.java | 14 ++++++++++++- .../security/MetastoreDelegationTokenManager.java | 4 ++++ .../metastore/security/ZooKeeperTokenStore.java | 10 +++++++++ 14 files changed, 126 insertions(+), 11 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 10b8a34b76e..0d7a8d072e5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3012,6 +3012,10 @@ public class HiveConf extends Configuration { "Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." + "Overrides any explicit value set via the zookeeper.ssl.keyStore.password " + "system property (note the camelCase)."), + HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE("hive.zookeeper.ssl.keystore.type", "", + "Keystore type when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.type " + + "system property (note the camelCase)."), HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("hive.zookeeper.ssl.truststore.location", "", "Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + "Overrides any explicit value set via the zookeeper.ssl.trustStore.location" + @@ -3020,6 +3024,10 @@ public class HiveConf extends Configuration { "Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." + "Overrides any explicit value set via the zookeeper.ssl.trustStore.password " + "system property (note the camelCase)."), + HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE("hive.zookeeper.ssl.truststore.type", "", + "Truststore type when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.type " + + "system property (note the camelCase)."), HIVE_ZOOKEEPER_KILLQUERY_ENABLE("hive.zookeeper.killquery.enable", true, "Whether enabled kill query coordination with zookeeper, " + "when hive.server2.support.dynamic.service.discovery is enabled."), @@ -5550,8 +5558,10 @@ public class HiveConf extends Configuration { "hive.driver.parallel.compilation.global.limit," + "hive.zookeeper.ssl.keystore.location," + "hive.zookeeper.ssl.keystore.password," + + "hive.zookeeper.ssl.keystore.type," + "hive.zookeeper.ssl.truststore.location," + - "hive.zookeeper.ssl.truststore.password", + "hive.zookeeper.ssl.truststore.password," + + "hive.zookeeper.ssl.truststore.type", "Comma separated list of configuration options which are immutable at runtime"), HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list", METASTOREPWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname @@ -6377,8 +6387,10 @@ public class HiveConf extends Configuration { .sslEnabled(getBoolVar(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE)) .keyStoreLocation(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION)) .keyStorePassword(keyStorePassword) + .keyStoreType(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE)) .trustStoreLocation(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION)) - .trustStorePassword(trustStorePassword).build(); + .trustStorePassword(trustStorePassword) + .trustStoreType(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE)).build(); } public HiveConf() { diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java index 2919038c78d..f54c866118d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java @@ -55,8 +55,10 @@ public class ZooKeeperStorage implements TempletonStorage { public static final String ZK_SSL_ENABLE = "templeton.zookeeper.ssl.client.enable"; public static final String ZK_KEYSTORE_LOCATION = "templeton.zookeeper.keystore.location"; public static final String ZK_KEYSTORE_PASSWORD = "templeton.zookeeper.keystore.password"; + public static final String ZK_KEYSTORE_TYPE = "templeton.zookeeper.keystore.type"; public static final String ZK_TRUSTSTORE_LOCATION = "templeton.zookeeper.truststore.location"; public static final String ZK_TRUSTSTORE_PASSWORD = "templeton.zookeeper.truststore.password"; + public static final String ZK_TRUSTSTORE_TYPE = "templeton.zookeeper.truststore.type"; public static final String ENCODING = "UTF-8"; @@ -77,8 +79,10 @@ public class ZooKeeperStorage implements TempletonStorage { .sslEnabled(conf.getBoolean(ZK_SSL_ENABLE, false)) .keyStoreLocation(conf.get(ZK_KEYSTORE_LOCATION, "")) .keyStorePassword(conf.get(ZK_KEYSTORE_PASSWORD, "")) + .keyStoreType(conf.get(ZK_KEYSTORE_TYPE, "")) .trustStoreLocation(conf.get(ZK_TRUSTSTORE_LOCATION, "")) .trustStorePassword(conf.get(ZK_TRUSTSTORE_PASSWORD, "")) + .trustStoreType(conf.get(ZK_TRUSTSTORE_TYPE, "")) .build(); CuratorFramework zk = xkHelper.getNewZookeeperClient(); zk.start(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java index 35053e70b0a..9841a185dc0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java @@ -54,6 +54,7 @@ public abstract class ZooKeeperTokenStoreTestBase { private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; private static final String TRUST_STORE_NAME = "truststore.jks"; private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + private static final String KEY_STORE_TRUST_STORE_TYPE = "JKS"; private static MiniZooKeeperCluster zkCluster = null; private static int zkPort = -1; @@ -96,10 +97,14 @@ public abstract class ZooKeeperTokenStoreTestBase { dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD, KEY_STORE_TRUST_STORE_PASSWORD); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE, + KEY_STORE_TRUST_STORE_TYPE); conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION, dataFileDir + File.separator + TRUST_STORE_NAME); conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD, KEY_STORE_TRUST_STORE_PASSWORD); + conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE, + KEY_STORE_TRUST_STORE_TYPE); conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_SSL_ENABLE, "true"); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java index 04780ad0546..aeec57757c2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java @@ -105,8 +105,10 @@ public class TestRestrictedList { addToExpectedRestrictedMap("hive.driver.parallel.compilation.global.limit"); addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.location"); addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.password"); + addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.type"); addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.location"); addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.password"); + addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.type"); checkRestrictedListMatch(); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java index 9573e5050c6..6f58d265e12 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java @@ -180,6 +180,7 @@ public abstract class InformationSchemaWithPrivilegeTestBase { private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; private static final String TRUST_STORE_NAME = "truststore.jks"; private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + private static final String KEY_STORE_TRUST_STORE_TYPE = "JKS"; private static MiniHS2 miniHS2 = null; private static MiniZooKeeperCluster zkCluster = null; @@ -223,10 +224,14 @@ public abstract class InformationSchemaWithPrivilegeTestBase { dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE.varname, + KEY_STORE_TRUST_STORE_TYPE); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION.varname, dataFileDir + File.separator + TRUST_STORE_NAME); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname, KEY_STORE_TRUST_STORE_PASSWORD); + confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE.varname, + KEY_STORE_TRUST_STORE_TYPE); confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE.varname, "true"); } miniHS2.start(confOverlay); diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 5850105b5fa..197640cd388 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -1009,7 +1009,9 @@ public class HiveConnection implements java.sql.Connection { JdbcConnectionParams.SUNJSSE_ALGORITHM_STRING); String keyStorePath = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE); String keyStorePassword = Utils.getPassword(sessConfMap, JdbcConnectionParams.SSL_KEY_STORE_PASSWORD); - KeyStore sslKeyStore = KeyStore.getInstance(JdbcConnectionParams.SSL_KEY_STORE_TYPE); + String keyStoreType = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE_TYPE); + keyStoreType = (!StringUtils.isBlank(keyStoreType)) ? keyStoreType : KeyStore.getDefaultType(); + KeyStore sslKeyStore = KeyStore.getInstance(keyStoreType); if (keyStorePath == null || keyStorePath.isEmpty()) { throw new IllegalArgumentException(JdbcConnectionParams.SSL_KEY_STORE diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java index e6f07032e92..e3fb3edf810 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -146,8 +146,10 @@ public class Utils { public static final String ZOOKEEPER_SSL_ENABLE = "zooKeeperSSLEnable"; public static final String ZOOKEEPER_KEYSTORE_LOCATION = "zooKeeperKeystoreLocation"; public static final String ZOOKEEPER_KEYSTORE_PASSWORD= "zooKeeperKeystorePassword"; + public static final String ZOOKEEPER_KEYSTORE_TYPE= "zooKeeperKeystoreType"; public static final String ZOOKEEPER_TRUSTSTORE_LOCATION = "zooKeeperTruststoreLocation"; public static final String ZOOKEEPER_TRUSTSTORE_PASSWORD = "zooKeeperTruststorePassword"; + public static final String ZOOKEEPER_TRUSTSTORE_TYPE = "zooKeeperTruststoreType"; // Default namespace value on ZooKeeper. // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri. static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2"; @@ -185,7 +187,7 @@ public class Utils { static final String TRUE = "true"; static final String SSL_KEY_STORE = "sslKeyStore"; static final String SSL_KEY_STORE_PASSWORD = "keyStorePassword"; - static final String SSL_KEY_STORE_TYPE = "JKS"; + static final String SSL_KEY_STORE_TYPE = "keyStoreType"; static final String SUNX509_ALGORITHM_STRING = "SunX509"; static final String SUNJSSE_ALGORITHM_STRING = "SunJSSE"; // --------------- End 2 way ssl options ---------------------------- @@ -207,8 +209,10 @@ public class Utils { private boolean zooKeeperSslEnabled = false; private String zookeeperKeyStoreLocation = ""; private String zookeeperKeyStorePassword = ""; + private String zookeeperKeyStoreType; private String zookeeperTrustStoreLocation = ""; private String zookeeperTrustStorePassword = ""; + private String zookeeperTrustStoreType; private String currentHostZnodePath; private final List<String> rejectedHostZnodePaths = new ArrayList<String>(); @@ -233,8 +237,10 @@ public class Utils { this.zooKeeperSslEnabled = params.zooKeeperSslEnabled; this.zookeeperKeyStoreLocation = params.zookeeperKeyStoreLocation; this.zookeeperKeyStorePassword = params.zookeeperKeyStorePassword; + this.zookeeperKeyStoreType = params.zookeeperKeyStoreType; this.zookeeperTrustStoreLocation = params.zookeeperTrustStoreLocation; this.zookeeperTrustStorePassword = params.zookeeperTrustStorePassword; + this.zookeeperTrustStoreType = params.zookeeperTrustStoreType; this.currentHostZnodePath = params.currentHostZnodePath; this.rejectedHostZnodePaths.addAll(rejectedHostZnodePaths); @@ -291,6 +297,10 @@ public class Utils { return zookeeperKeyStorePassword; } + public String getZookeeperKeyStoreType() { + return zookeeperKeyStoreType; + } + public String getZookeeperTrustStoreLocation() { return zookeeperTrustStoreLocation; } @@ -299,6 +309,10 @@ public class Utils { return zookeeperTrustStorePassword; } + public String getZookeeperTrustStoreType() { + return zookeeperTrustStoreType; + } + public List<String> getRejectedHostZnodePaths() { return rejectedHostZnodePaths; } @@ -359,6 +373,10 @@ public class Utils { this.zookeeperKeyStorePassword = zookeeperKeyStorePassword; } + public void setZookeeperKeyStoreType(String zookeeperKeyStoreType) { + this.zookeeperKeyStoreType = zookeeperKeyStoreType; + } + public void setZookeeperTrustStoreLocation(String zookeeperTrustStoreLocation) { this.zookeeperTrustStoreLocation = zookeeperTrustStoreLocation; } @@ -367,6 +385,10 @@ public class Utils { this.zookeeperTrustStorePassword = zookeeperTrustStorePassword; } + public void setZookeeperTrustStoreType(String zookeeperTrustStoreType) { + this.zookeeperTrustStoreType = zookeeperTrustStoreType; + } + public void setCurrentHostZnodePath(String currentHostZnodePath) { this.currentHostZnodePath = currentHostZnodePath; } diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java index 70091343430..dfe21e71331 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java +++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java @@ -103,11 +103,15 @@ class ZooKeeperHiveClientHelper { connParams.setZookeeperKeyStorePassword( StringUtils.defaultString(Utils.getPassword(sessionConf, JdbcConnectionParams.ZOOKEEPER_KEYSTORE_PASSWORD), "")); + connParams.setZookeeperKeyStoreType( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_KEYSTORE_TYPE),"")); connParams.setZookeeperTrustStoreLocation( StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_LOCATION), "")); connParams.setZookeeperTrustStorePassword( StringUtils.defaultString(Utils.getPassword(sessionConf, JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_PASSWORD), "")); + connParams.setZookeeperTrustStoreType( + StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_TYPE),"")); } } @@ -119,8 +123,9 @@ class ZooKeeperHiveClientHelper { .retryPolicy(new ExponentialBackoffRetry(1000, 3)) .zookeeperFactory( new SSLZookeeperFactory(connParams.isZooKeeperSslEnabled(), connParams.getZookeeperKeyStoreLocation(), - connParams.getZookeeperKeyStorePassword(), connParams.getZookeeperTrustStoreLocation(), - connParams.getZookeeperTrustStorePassword())) + connParams.getZookeeperKeyStorePassword(), connParams.getZookeeperKeyStoreType(), + connParams.getZookeeperTrustStoreLocation(), + connParams.getZookeeperTrustStorePassword(), connParams.getZookeeperTrustStoreType())) .build(); zooKeeperClient.start(); return zooKeeperClient; diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index 73290586b43..9da200f2e92 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -238,8 +238,10 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> { .sslEnabled(HiveConf.getBoolVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE)) .keyStoreLocation(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION)) .keyStorePassword(keyStorePassword) + .keyStoreType(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE)) .trustStoreLocation(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION)) .trustStorePassword(trustStorePassword) + .trustStoreType(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE)) .build().getNewZookeeperClient(zooKeeperAclProvider, namespace); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java index ee01731fa95..514199f2155 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java @@ -27,6 +27,8 @@ import org.apache.zookeeper.common.ClientX509Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.security.KeyStore; + /** * Factory to create Zookeeper clients with the zookeeper.client.secure enabled, * allowing SSL communication with the Zookeeper server. @@ -38,22 +40,26 @@ public class SSLZookeeperFactory implements ZookeeperFactory { private boolean sslEnabled; private String keyStoreLocation; private String keyStorePassword; + private String keyStoreType; private String trustStoreLocation; private String trustStorePassword; + private String trustStoreType; public SSLZookeeperFactory(boolean sslEnabled, String keyStoreLocation, String keyStorePassword, - String trustStoreLocation, String trustStorePassword) { + String keyStoreType, String trustStoreLocation, String trustStorePassword, String trustStoreType) { this.sslEnabled = sslEnabled; this.keyStoreLocation = keyStoreLocation; this.keyStorePassword = keyStorePassword; + this.keyStoreType = (!StringUtils.isBlank(keyStoreType)) ? keyStoreType : KeyStore.getDefaultType(); this.trustStoreLocation = trustStoreLocation; this.trustStorePassword = trustStorePassword; + this.trustStoreType = (!StringUtils.isBlank(trustStoreType)) ? trustStoreType : KeyStore.getDefaultType(); if (sslEnabled) { - if (StringUtils.isEmpty(keyStoreLocation)) { + if (StringUtils.isBlank(keyStoreLocation)) { LOG.warn("Missing keystoreLocation parameter"); } - if (StringUtils.isEmpty(trustStoreLocation)) { + if (StringUtils.isBlank(trustStoreLocation)) { LOG.warn("Missing trustStoreLocation parameter"); } } @@ -71,8 +77,10 @@ public class SSLZookeeperFactory implements ZookeeperFactory { ClientX509Util x509Util = new ClientX509Util(); clientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(), this.keyStoreLocation); clientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(), this.keyStorePassword); + clientConfig.setProperty(x509Util.getSslKeystoreTypeProperty(), this.keyStoreType); clientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(), this.trustStoreLocation); clientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(), this.trustStorePassword); + clientConfig.setProperty(x509Util.getSslTruststoreTypeProperty(), this.trustStoreType); return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly, clientConfig); } } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java index 1e35795d63c..c7da6259f53 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java @@ -71,8 +71,10 @@ public class ZooKeeperHiveHelper { private boolean sslEnabled = false; private String keyStoreLocation = null; private String keyStorePassword = null; + private String keyStoreType = null; private String trustStoreLocation = null; private String trustStorePassword = null; + private String trustStoreType = null; public ZooKeeperHiveHelper build() { return new ZooKeeperHiveHelper(this); @@ -128,6 +130,11 @@ public class ZooKeeperHiveHelper { return this; } + public ZooKeeperHiveHelperBuilder keyStoreType(String keyStoreType) { + this.keyStoreType = keyStoreType; + return this; + } + public ZooKeeperHiveHelperBuilder trustStoreLocation(String trustStoreLocation) { this.trustStoreLocation = trustStoreLocation; return this; @@ -138,6 +145,11 @@ public class ZooKeeperHiveHelper { return this; } + public ZooKeeperHiveHelperBuilder trustStoreType(String trustStoreType) { + this.trustStoreType = trustStoreType; + return this; + } + public String getQuorum() { return quorum; } @@ -178,6 +190,10 @@ public class ZooKeeperHiveHelper { return keyStorePassword; } + public String getKeyStoreType() { + return keyStoreType; + } + public String getTrustStoreLocation() { return trustStoreLocation; } @@ -185,6 +201,10 @@ public class ZooKeeperHiveHelper { public String getTrustStorePassword() { return trustStorePassword; } + + public String getTrustStoreType() { + return trustStoreType; + } } public static ZooKeeperHiveHelper.ZooKeeperHiveHelperBuilder builder() { @@ -233,8 +253,10 @@ public class ZooKeeperHiveHelper { new SSLZookeeperFactory(sslEnabled, builder.getKeyStoreLocation(), builder.getKeyStorePassword(), + builder.getKeyStoreType(), builder.getTrustStoreLocation(), - builder.getTrustStorePassword()); + builder.getTrustStorePassword(), + builder.getTrustStoreType()); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 71ba8d520ff..27646119d6e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1536,6 +1536,11 @@ public class MetastoreConf { "Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." + "Overrides any explicit value set via the zookeeper.ssl.keyStore.password" + "system property (note the camelCase)."), + THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE("metastore.zookeeper.ssl.keystore.type", + "hive.zookeeper.ssl.keystore.type", "", + "Keystore type when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.keyStore.type" + + "system property (note the camelCase)."), THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("metastore.zookeeper.ssl.truststore.location", "hive.zookeeper.ssl.truststore.location", "", "Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " + @@ -1546,6 +1551,11 @@ public class MetastoreConf { "Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." + "Overrides any explicit value set via the zookeeper.ssl.trustStore.password " + "system property (note the camelCase)."), + THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE("metastore.zookeeper.ssl.truststore.type", + "hive.zookeeper.ssl.truststore.type", "", + "Truststore type when using a client-side certificate with TLS connectivity to ZooKeeper." + + "Overrides any explicit value set via the zookeeper.ssl.trustStore.type" + + "system property (note the camelCase)."), THRIFT_URI_SELECTION("metastore.thrift.uri.selection", "hive.metastore.uri.selection", "RANDOM", new StringSetValidator("RANDOM", "SEQUENTIAL"), "Determines the selection mechanism used by metastore client to connect to remote " + @@ -2597,8 +2607,10 @@ public class MetastoreConf { .sslEnabled(MetastoreConf.getBoolVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE)) .keyStoreLocation(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION)) .keyStorePassword(keyStorePassword) + .keyStoreType(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE)) .trustStoreLocation(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION)) - .trustStorePassword(trustStorePassword).build(); + .trustStorePassword(trustStorePassword) + .trustStoreType(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE)).build(); } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index 239bff6dc9d..e43d4b15a94 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -52,10 +52,14 @@ public class MetastoreDelegationTokenManager { "hive.cluster.delegation.token.store.zookeeper.keystore.location"; public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD = "hive.cluster.delegation.token.store.zookeeper.keystore.password"; + public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE = + "hive.cluster.delegation.token.store.zookeeper.keystore.type"; public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION = "hive.cluster.delegation.token.store.zookeeper.truststore.location"; public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD = "hive.cluster.delegation.token.store.zookeeper.truststore.password"; + public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE = + "hive.cluster.delegation.token.store.zookeeper.truststore.type"; public MetastoreDelegationTokenManager() { } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java index 94066857398..aa2f0cccffa 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java @@ -69,8 +69,10 @@ public class ZooKeeperTokenStore implements DelegationTokenStore { private boolean sslEnabled; private String keyStoreLocation; private String keyStorePassword; + private String keyStoreType; private String trustStoreLocation; private String trustStorePassword; + private String trustStoreType; private List<ACL> newNodeAcl; private Configuration conf; @@ -144,8 +146,10 @@ public class ZooKeeperTokenStore implements DelegationTokenStore { .sslEnabled(sslEnabled) .keyStoreLocation(keyStoreLocation) .keyStorePassword(keyStorePassword) + .keyStoreType(keyStoreType) .trustStoreLocation(trustStoreLocation) .trustStorePassword(trustStorePassword) + .trustStoreType(trustStoreType) .build(); zkSession = zkHelper.getNewZookeeperClient(aclDefaultProvider); zkSession.start(); @@ -499,10 +503,14 @@ public class ZooKeeperTokenStore implements DelegationTokenStore { keyStoreLocation = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION); keyStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD); + keyStoreType = + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE); trustStoreLocation = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION); trustStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD); + trustStoreType = + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE); } catch (IOException ex) { throw new RuntimeException("Failed to read zookeeper configuration passwords", ex); } @@ -517,10 +525,12 @@ public class ZooKeeperTokenStore implements DelegationTokenStore { keyStoreLocation = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_LOCATION, ""); char[] pwd = conf.getPassword(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD); keyStorePassword = pwd == null ? null : new String(pwd); + keyStoreType = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE, ""); trustStoreLocation = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION, ""); pwd = conf.getPassword(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD); trustStorePassword = pwd == null ? null : new String(pwd); + trustStoreType = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE, ""); } catch (IOException ex) { throw new RuntimeException("Failed to read zookeeper configuration passwords", ex); }