Repository: ambari Updated Branches: refs/heads/branch-2.5 947b7191b -> 82e22171d
AMBARI-19462. Setup the correct authentication and authorization between Hive/Hcatalog and ZooKeeper. (Attila Magyar via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/82e22171 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/82e22171 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/82e22171 Branch: refs/heads/branch-2.5 Commit: 82e22171daaa8e8ded880f7bb721131006920ec0 Parents: 947b719 Author: Attila Magyar <amag...@hortonworks.com> Authored: Wed Jan 11 20:17:54 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Wed Jan 11 20:18:19 2017 +0100 ---------------------------------------------------------------------- .../java/org/apache/ambari/tools/zk/ZkAcl.java | 23 +++- .../org/apache/ambari/tools/zk/ZkMigrator.java | 2 +- .../apache/ambari/tools/zk/ZkPathPattern.java | 114 +++++++++++++++++++ .../apache/ambari/tools/zk/ZkMigratorTest.java | 45 +++++++- .../core/resources/zkmigrator.py | 2 +- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 7 ++ .../0.12.0.2.0/package/scripts/hive_server.py | 20 ++++ .../0.12.0.2.0/package/scripts/params_linux.py | 6 + .../package/templates/zkmigrator_jaas.conf.j2 | 26 +++++ .../stacks/2.0.6/HIVE/test_hive_client.py | 5 + .../stacks/2.0.6/HIVE/test_hive_metastore.py | 7 +- .../stacks/2.0.6/HIVE/test_hive_server.py | 5 + .../stacks/2.1/HIVE/test_hive_metastore.py | 5 + 13 files changed, 257 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkAcl.java ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkAcl.java b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkAcl.java index 420ab0a..5c66612 100644 --- a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkAcl.java +++ b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkAcl.java @@ -54,16 +54,27 @@ public class ZkAcl { } /** - * Sets the ACL on the given znode according to my state + * Sets the ACL on the znode indicated by the given pattern */ - public void setRecursivelyOn(ZooKeeper zkClient, String node) throws KeeperException, InterruptedException { - zkClient.setACL(node, singletonList(new ACL(permission.code, new Id(scheme.value, id))), ANY_NODE_VER); - for (String child : zkClient.getChildren(node, null)) { - setRecursivelyOn(zkClient, path(node, child)); + public void setRecursivelyOn(ZooKeeper zkClient, ZkPathPattern pattern) throws KeeperException, InterruptedException { + for (String each : pattern.findMatchingPaths(zkClient, "/")) { + System.out.println("Set ACL " + asZkAcl() + " recursively on " + each); + setRecursivelyOnSingle(zkClient, each); } } - private String path(String node, String child) { + public void setRecursivelyOnSingle(ZooKeeper zkClient, String baseNode) throws KeeperException, InterruptedException { + zkClient.setACL(baseNode, singletonList(asZkAcl()), ANY_NODE_VER); + for (String child : zkClient.getChildren(baseNode, null)) { + setRecursivelyOnSingle(zkClient, append(baseNode, child)); + } + } + + private ACL asZkAcl() { + return new ACL(permission.code, new Id(scheme.value, id)); + } + + public static String append(String node, String child) { return node.endsWith("/") ? node + child : node + "/" + child; } http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java index b4da1ed..c100b85 100644 --- a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java +++ b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java @@ -73,7 +73,7 @@ public class ZkMigrator { private static void setAcls(String connectionString, String znode, ZkAcl acl) throws IOException, InterruptedException, KeeperException { ZooKeeper client = ZkConnection.open(connectionString, SESSION_TIMEOUT_MILLIS, CONNECTION_TIMEOUT_MILLIS); try { - acl.setRecursivelyOn(client, znode); + acl.setRecursivelyOn(client, ZkPathPattern.fromString(znode)); } catch (KeeperException.NoNodeException e) { System.out.println("Could not set ACL on " + znode + ". Reason: " + e.getMessage()); } finally { http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkPathPattern.java ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkPathPattern.java b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkPathPattern.java new file mode 100644 index 0000000..a376975 --- /dev/null +++ b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkPathPattern.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.tools.zk; + +import static java.util.Collections.singletonList; +import static org.apache.ambari.tools.zk.ZkAcl.append; + +import java.nio.file.FileSystems; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; + +/** + * Represents a Zookeeper path where each '/' separated segment might contain wildcard mask (*) + * E.g.: /ab/c/*//def + */ +public class ZkPathPattern { + private final List<Segment> segments; + + /** + * Creates a instance from a pattern like /*//abc* + */ + public static ZkPathPattern fromString(String pattern) { + pattern = pattern.trim(); + if (!pattern.startsWith("/")) { + throw new IllegalArgumentException("ZkPath must start with: '/'. Invalid path: " + pattern); + } + if ("/".equals(pattern)) { + return new ZkPathPattern(singletonList(new Segment("*"))); + } + List<Segment> segments = new ArrayList<>(); + for (String segment : pattern.substring(1).split("/")) { + if (segment.isEmpty()) { + throw new IllegalArgumentException("Invalid ZkPath: " + pattern); + } + segments.add(new Segment(segment)); + } + if (segments.isEmpty()) { + throw new IllegalArgumentException("Empty ZkPath: " + pattern); + } + return new ZkPathPattern(segments); + } + + private ZkPathPattern(List<Segment> segments) { + this.segments = segments; + } + + /** + * Finds matching paths from the given base node + * E.g.: + * If the pattern is /*//abc* it will find nodes like: + * /a/abcdef + * /xy/abc/efg + * /def/abc + */ + public List<String> findMatchingPaths(ZooKeeper zkClient, String basePath) throws KeeperException, InterruptedException { + List<String> result = new ArrayList<>(); + collectMatching(zkClient, basePath, result); + return result; + } + + private void collectMatching(ZooKeeper zkClient, String basePath, List<String> result) throws KeeperException, InterruptedException { + for (String child : zkClient.getChildren(basePath, null)) { + if (first().matches(child)) { + if (rest() == null) { + result.add(append(basePath, child)); + } else { + rest().collectMatching(zkClient, append(basePath, child), result); + } + } + } + } + + private Segment first() { + return segments.get(0); + } + + private ZkPathPattern rest() { + List<Segment> tail = segments.subList(1, segments.size()); + return tail.isEmpty() ? null : new ZkPathPattern(tail); + } + + public static class Segment { + private final PathMatcher glob; + + public Segment(String segment) { + this.glob = FileSystems.getDefault().getPathMatcher("glob:" + segment); + } + + public boolean matches(String node) { + return glob.matches(Paths.get(node)); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java index b2c9899..b7e9c0c 100644 --- a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java +++ b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java @@ -21,6 +21,7 @@ package org.apache.ambari.tools.zk; import static org.apache.zookeeper.ZooDefs.Perms.DELETE; import static org.apache.zookeeper.ZooDefs.Perms.READ; import static org.apache.zookeeper.ZooDefs.Perms.WRITE; +import static org.apache.zookeeper.ZooDefs.Perms.ALL; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -84,6 +85,48 @@ public class ZkMigratorTest { } @Test + public void testSupportsWildcard() throws Exception { + // Given + path("/abc123"); + path("/abcdef/efg"); + path("/abc/123"); + path("/x"); + path("/y/a"); + path("/ab"); + // When + setAcls("/abc*", "ip:127.0.0.1:r"); + // Then + assertHasAcl("/abc123", "ip", "127.0.0.1", READ); + assertHasAcl("/abcdef/efg", "ip", "127.0.0.1", READ); + assertHasAcl("/abc/123", "ip", "127.0.0.1", READ); + assertHasAcl("/x", "world", "anyone", ALL); + assertHasAcl("/y/a", "world", "anyone", ALL); + assertHasAcl("/ab", "world", "anyone", ALL); + } + + @Test + public void testSupportsMultupleWildcards() throws Exception { + // Given + path("/abc123"); + path("/a/abcdef"); + path("/def/abc"); + path("/xy/abc/efg"); + path("/a/xyabc"); + path("/a/b/abc"); + path("/b"); + // When + setAcls("/*/abc*", "ip:127.0.0.1:r"); + // Then + assertHasAcl("/a/abcdef", "ip", "127.0.0.1", READ); + assertHasAcl("/xy/abc/efg", "ip", "127.0.0.1", READ); + assertHasAcl("/def/abc", "ip", "127.0.0.1", READ); + assertHasAcl("/a/xyabc", "world", "anyone", ALL); + assertHasAcl("/abc123", "world", "anyone", ALL); + assertHasAcl("/a/b/abc", "world", "anyone", ALL); + assertHasAcl("/b", "world", "anyone", ALL); + } + + @Test public void testSupportsWorldScheme() throws Exception { // Given path("/unprotected"); @@ -125,7 +168,7 @@ public class ZkMigratorTest { } private String path(String s) throws Exception { - return cli.create().forPath(s, "any".getBytes()); + return cli.create().creatingParentsIfNeeded().forPath(s, "any".getBytes()); } private void setAcls(String path, String acl) throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py index 5e86e05..344b9a4 100644 --- a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py +++ b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py @@ -33,7 +33,7 @@ class ZkMigrator: self.user = user self.zkmigrator_jar = "/var/lib/ambari-agent/tools/zkmigrator.jar" - def set_acls(self, znode, acl, tries=1): + def set_acls(self, znode, acl, tries=3): Logger.info(format("Setting ACL on znode {znode} to {acl}")) Execute( self._command(znode, acl), \ http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py index 67138a4..58f01e6 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py @@ -273,6 +273,13 @@ def hive(name=None): mode=0644, content=Template("hive.conf.j2") ) + if params.security_enabled: + File(os.path.join(params.hive_config_dir, 'zkmigrator_jaas.conf'), + owner=params.hive_user, + group=params.user_group, + content=Template("zkmigrator_jaas.conf.j2") + ) + if name == 'metastore' or name == 'hiveserver2': if params.hive_jdbc_target is not None and not os.path.exists(params.hive_jdbc_target): http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index d212f2a..dfd1df1 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -39,6 +39,7 @@ from resource_management.core.logger import Logger import hive_server_upgrade from hive import hive from hive_service import hive_service +from resource_management.core.resources.zkmigrator import ZkMigrator class HiveServer(Script): @@ -196,6 +197,25 @@ class HiveServerDefault(HiveServer): else: self.put_structured_out({"securityState": "UNSECURED"}) + def _base_node(self, path): + if not path.startswith('/'): + path = '/' + path + try: + return '/' + path.split('/')[1] + except IndexError: + return path + + def disable_security(self, env): + import params + zkmigrator = ZkMigrator(params.hive_zookeeper_quorum, params.java_exec, params.java64_home, params.jaas_file, params.hive_user) + if params.hive_cluster_token_zkstore: + zkmigrator.set_acls(self._base_node(params.hive_cluster_token_zkstore), 'world:anyone:crdwa') + if params.hive_zk_namespace: + zkmigrator.set_acls( + params.hive_zk_namespace if params.hive_zk_namespace.startswith('/') else '/' + params.hive_zk_namespace, + 'world:anyone:crdwa') + zkmigrator.set_acls(params.zkdtsm_pattern, 'world:anyone:crdwa') + def get_log_folder(self): import params return params.hive_log_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 61034cf..82c6804 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -415,6 +415,7 @@ else: hive_metastore_heapsize = config['configurations']['hive-env']['hive.metastore.heapsize'] java64_home = config['hostLevelParams']['java_home'] +java_exec = format("{java64_home}/bin/java") java_version = expect("/hostLevelParams/java_version", int) ##### MYSQL @@ -653,6 +654,11 @@ ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-hive-plugin-properties'] policy_user = config['configurations']['ranger-hive-plugin-properties']['policy_user'] +hive_cluster_token_zkstore = default("/configurations/hive-site/hive.cluster.delegation.token.store.zookeeper.znode", None) +jaas_file = os.path.join(hive_config_dir, 'zkmigrator_jaas.conf') +zkdtsm_pattern = '/zkdtsm_*' +hive_zk_namespace = default("/configurations/hive-site/hive.zookeeper.namespace", None) + if security_enabled: hive_principal = hive_server_principal.replace('_HOST',hostname.lower()) hive_keytab = config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab'] http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/zkmigrator_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/zkmigrator_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/zkmigrator_jaas.conf.j2 new file mode 100644 index 0000000..e7adfd3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/templates/zkmigrator_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{hive_keytab}}" + principal="{{hive_principal}}"; +}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py index 265f040..2653254 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py @@ -177,6 +177,11 @@ class TestHiveClient(RMFTestCase): group = 'root', mode = 0644, ) + self.assertResourceCalled('File', '/usr/hdp/current/hive-client/conf/zkmigrator_jaas.conf', + content = Template('zkmigrator_jaas.conf.j2'), + owner = 'hive', + group = 'hadoop', + ) self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar', content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'), mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 25ce513..bf9c208 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -95,7 +95,7 @@ class TestHiveMetastore(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.assert_configure_default() + self.assert_configure_secured() self.assertNoMoreResources() def test_start_secured(self): @@ -334,6 +334,11 @@ class TestHiveMetastore(RMFTestCase): group = 'root', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hive/conf.server/zkmigrator_jaas.conf', + content = Template('zkmigrator_jaas.conf.j2'), + owner = 'hive', + group = 'hadoop', + ) self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/usr/share/java/mysql-connector-java.jar', http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 4ea7bf4..6e97d15 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -693,6 +693,11 @@ class TestHiveServer(RMFTestCase): group='root', mode=0644, ) + self.assertResourceCalled('File', '/etc/hive/conf.server/zkmigrator_jaas.conf', + content = Template('zkmigrator_jaas.conf.j2'), + owner = 'hive', + group = 'hadoop', + ) self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/usr/share/java/mysql-connector-java.jar', http://git-wip-us.apache.org/repos/asf/ambari/blob/82e22171/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 635c297..a904bc2 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -348,6 +348,11 @@ class TestHiveMetastore(RMFTestCase): group = 'root', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hive/conf.server/zkmigrator_jaas.conf', + content = Template('zkmigrator_jaas.conf.j2'), + owner = 'hive', + group = 'hadoop', + ) self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/usr/share/java/mysql-connector-java.jar',
