This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new be508b8 HIVE-22195: Configure authentication type for Zookeeper when different from the default cluster wide (Denys Kuzmenko via Peter Vary) be508b8 is described below commit be508b8d16c465ac17f3b8c09d7cbb27c4eee47a Author: denys kuzmenko <dkuzme...@cloudera.com> AuthorDate: Mon Sep 23 11:04:08 2019 +0200 HIVE-22195: Configure authentication type for Zookeeper when different from the default cluster wide (Denys Kuzmenko via Peter Vary) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 8 ++- .../hadoop/hive/registry/impl/ZkRegistryBase.java | 13 +++-- .../hadoop/hive/registry/impl/ZookeeperUtils.java | 18 +++++- .../hive/registry/impl/TestZookeeperUtils.java | 66 ++++++++++++++++++++++ .../server/HS2ActivePassiveHARegistryClient.java | 2 +- .../apache/hive/service/server/HiveServer2.java | 50 ++++++++-------- .../metastore/security/ZooKeeperTokenStore.java | 4 +- 7 files changed, 129 insertions(+), 32 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 97cffff..de28e92 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2582,7 +2582,13 @@ public class HiveConf extends Configuration { "this is set to false, however unless MAPREDUCE-7086 fix is present, queries that\n" + "read MM tables with original files will fail. The default in Hive 3.0 is false."), - // Zookeeper related configs + // Zookeeper related configs + HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION("hive.security.zookeeper.authentication", + "DEFAULT", new StringSet("DEFAULT", "SIMPLE"), + "Whether the authentication type for Zookeeper is different from the cluster wide\n" + + "`hadoop.security.authentication` configuration. This could be useful when cluster\n" + + "is kerberized, but Zookeeper is not."), + HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "", "List of ZooKeeper servers to talk to. This is needed for: \n" + "1. Read/write locks - when hive.lock.manager is set to \n" + diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java index e56ae11..5751b8e 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java @@ -160,7 +160,7 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> { this.stateChangeListeners = new HashSet<>(); this.pathToInstanceCache = new ConcurrentHashMap<>(); this.nodeToInstanceCache = new ConcurrentHashMap<>(); - final String namespace = getRootNamespace(rootNs, nsPrefix); + final String namespace = getRootNamespace(conf, rootNs, nsPrefix); ACLProvider aclProvider; // get acl provider for most outer path that is non-null if (userPathPrefix == null) { @@ -180,8 +180,9 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> { this.zooKeeperClient.getConnectionStateListenable().addListener(new ZkConnectionStateListener()); } - public static String getRootNamespace(String userProvidedNamespace, String defaultNamespacePrefix) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + public static String getRootNamespace(Configuration conf, String userProvidedNamespace, + String defaultNamespacePrefix) { + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf); String rootNs = userProvidedNamespace; if (rootNs == null) { rootNs = defaultNamespacePrefix + (isSecure ? SASL_NAMESPACE : UNSECURE_NAMESPACE); @@ -190,7 +191,7 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> { } private ACLProvider getACLProviderForZKPath(String zkPath) { - final boolean isSecure = UserGroupInformation.isSecurityEnabled(); + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(conf); return new ACLProvider() { @Override public List<ACL> getDefaultAcl() { @@ -366,7 +367,9 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> { } private void checkAndSetAcls() throws Exception { - if (!UserGroupInformation.isSecurityEnabled()) return; + if (!ZookeeperUtils.isKerberosEnabled(conf)) { + return; + } // We are trying to check ACLs on the "workers" directory, which noone except us should be // able to write to. Higher-level directories shouldn't matter - we don't read them. String pathToCheck = workersPath; diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java index 454d503..7a4274e 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java +++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZookeeperUtils.java @@ -13,6 +13,8 @@ */ package org.apache.hadoop.hive.registry.impl; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +34,7 @@ public class ZookeeperUtils { public static String setupZookeeperAuth(Configuration conf, String saslLoginContextName, String zkPrincipal, String zkKeytab) throws IOException { // If the login context name is not set, we are in the client and don't need auth. - if (UserGroupInformation.isSecurityEnabled() && saslLoginContextName != null) { + if (isKerberosEnabled(conf) && saslLoginContextName != null) { LOG.info("UGI security is enabled. Setting up ZK auth."); if (zkPrincipal == null || zkPrincipal.isEmpty()) { @@ -53,7 +55,19 @@ public class ZookeeperUtils { } /** - * Dynamically sets up the JAAS configuration that uses kerberos + * Check if Kerberos is enabled. + */ + public static boolean isKerberosEnabled(Configuration conf) { + try { + return UserGroupInformation.getLoginUser().isFromKeytab() && !AuthenticationMethod.SIMPLE.name().equalsIgnoreCase( + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION)); + } catch (IOException e) { + return false; + } + } + + /** + * Dynamically sets up the JAAS configuration that uses kerberos. * * @param principal * @param keyTabFile diff --git a/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java new file mode 100644 index 0000000..c486274 --- /dev/null +++ b/llap-client/src/test/org/apache/hadoop/hive/registry/impl/TestZookeeperUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.registry.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * ZookeeperUtils test suite. + */ +public class TestZookeeperUtils { + + private Configuration conf; + private UserGroupInformation ugi; + + @Before + public void setup() { + conf = new Configuration(); + ugi = Mockito.mock(UserGroupInformation.class); + UserGroupInformation.setLoginUser(ugi); + } + + @Test + public void testHadoopKerberosZookeeperDefault() { + Mockito.when(ugi.isFromKeytab()).thenReturn(true); + Assert.assertTrue(ZookeeperUtils.isKerberosEnabled(conf)); + } + + @Test + public void testHadoopKerberosZookeeperSimple(){ + Mockito.when(ugi.isFromKeytab()).thenReturn(true); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, + AuthenticationMethod.SIMPLE.name()); + Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf)); + } + + @Test + public void testHadoopSimpleZookeeperDefault(){ + Mockito.when(ugi.isFromKeytab()).thenReturn(false); + conf.set(HiveConf.ConfVars.HIVE_SECURITY_ZOOKEEPER_AUTHENTICATION.varname, + AuthenticationMethod.SIMPLE.name()); + Assert.assertFalse(ZookeeperUtils.isKerberosEnabled(conf)); + } +} diff --git a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java index f87b610..122742e 100644 --- a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java +++ b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistryClient.java @@ -40,7 +40,7 @@ public class HS2ActivePassiveHARegistryClient { String namespace = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE); Preconditions.checkArgument(!StringUtils.isBlank(namespace), HiveConf.ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE.varname + " cannot be null or empty"); - String nsKey = ZkRegistryBase.getRootNamespace(null, namespace + "-"); + String nsKey = ZkRegistryBase.getRootNamespace(conf, null, namespace + "-"); HS2ActivePassiveHARegistry registry = hs2Registries.get(nsKey); if (registry == null) { registry = HS2ActivePassiveHARegistry.create(conf, true); diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 5d81668..17570f7 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -86,9 +86,9 @@ import org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.compactor.CompactorThread; import org.apache.hadoop.hive.ql.txn.compactor.Worker; +import org.apache.hadoop.hive.registry.impl.ZookeeperUtils; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.HiveVersionInfo; import org.apache.hive.common.util.ShutdownHookManager; @@ -278,6 +278,7 @@ public class HiveServer2 extends CompositeService { } wmQueue = hiveConf.get(ConfVars.HIVE_SERVER2_TEZ_INTERACTIVE_QUEUE.varname, "").trim(); + this.zooKeeperAclProvider = getACLProvider(hiveConf); this.serviceDiscovery = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY); this.activePassiveHA = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE); @@ -453,29 +454,34 @@ public class HiveServer2 extends CompositeService { /** * ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory */ - private final ACLProvider zooKeeperAclProvider = new ACLProvider() { + private ACLProvider zooKeeperAclProvider; - @Override - public List<ACL> getDefaultAcl() { - List<ACL> nodeAcls = new ArrayList<ACL>(); - if (UserGroupInformation.isSecurityEnabled()) { - // Read all to the world - nodeAcls.addAll(Ids.READ_ACL_UNSAFE); - // Create/Delete/Write/Admin to the authenticated user - nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); - } else { - // ACLs for znodes on a non-kerberized cluster - // Create/Read/Delete/Write/Admin to the world - nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + private ACLProvider getACLProvider(HiveConf hiveConf) { + final boolean isSecure = ZookeeperUtils.isKerberosEnabled(hiveConf); + + return new ACLProvider() { + @Override + public List<ACL> getDefaultAcl() { + List<ACL> nodeAcls = new ArrayList<ACL>(); + if (isSecure) { + // Read all to the world + nodeAcls.addAll(Ids.READ_ACL_UNSAFE); + // Create/Delete/Write/Admin to the authenticated user + nodeAcls.add(new ACL(Perms.ALL, Ids.AUTH_IDS)); + } else { + // ACLs for znodes on a non-kerberized cluster + // Create/Read/Delete/Write/Admin to the world + nodeAcls.addAll(Ids.OPEN_ACL_UNSAFE); + } + return nodeAcls; } - return nodeAcls; - } - @Override - public List<ACL> getAclForPath(String path) { - return getDefaultAcl(); - } - }; + @Override + public List<ACL> getAclForPath(String path) { + return getDefaultAcl(); + } + }; + } /** * Add conf keys, values that HiveServer2 will publish to ZooKeeper. @@ -522,7 +528,7 @@ public class HiveServer2 extends CompositeService { * @throws Exception */ private void setUpZooKeeperAuth(HiveConf hiveConf) throws Exception { - if (UserGroupInformation.isSecurityEnabled()) { + if (ZookeeperUtils.isKerberosEnabled(hiveConf)) { String principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL); if (principal.isEmpty()) { throw new IOException("HiveServer2 Kerberos principal is empty"); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java index 953c5fd..ba1f177 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java @@ -34,6 +34,7 @@ import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.MetastoreDelegationTokenSupport; import org.apache.zookeeper.CreateMode; @@ -111,7 +112,8 @@ public class ZooKeeperTokenStore implements DelegationTokenStore { } private void setupJAASConfig(Configuration conf) throws IOException { - if (!UserGroupInformation.getLoginUser().isFromKeytab()) { + if (!UserGroupInformation.getLoginUser().isFromKeytab() || AuthenticationMethod.SIMPLE.name().equalsIgnoreCase( + getNonEmptyConfVar(conf, "hive.security.zookeeper.authentication"))) { // The process has not logged in using keytab // this should be a test mode, can't use keytab to authenticate // with zookeeper.