AMBARI-19568. Setup the correct authentication and authorization between ZooKeeper and oozie. (Attila Magyar via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aae7013f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aae7013f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aae7013f Branch: refs/heads/branch-dev-patch-upgrade Commit: aae7013f48983b093b31897997c9e725d8110f16 Parents: 285666f Author: Attila Magyar <amag...@hortonworks.com> Authored: Tue Jan 17 09:18:15 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Tue Jan 17 09:18:15 2017 +0100 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 +++++ .../4.0.0.2.0/package/scripts/oozie_server.py | 11 +++++++-- .../4.0.0.2.0/package/scripts/params_linux.py | 12 ++++++--- .../package/templates/zkmigrator_jaas.conf.j2 | 26 ++++++++++++++++++++ .../OOZIE/4.2.0.2.3/kerberos.json | 3 ++- .../HDP/2.0.6/properties/stack_features.json | 3 +-- .../HDP/3.0/properties/stack_features.json | 3 +-- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 13 +++++++--- .../HDF/2.0/properties/stack_features.json | 3 +-- 9 files changed, 63 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 252f60e..3cdafe9 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -353,6 +353,12 @@ def oozie_server_specific(): group = params.user_group, recursive_ownership = True, ) + if params.security_enabled: + File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), + owner=params.oozie_user, + group=params.user_group, + content=Template("zkmigrator_jaas.conf.j2") + ) def __parse_sharelib_from_output(output): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index e0778da..1a34b87 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -42,7 +42,7 @@ from oozie_service import oozie_service from oozie_server_upgrade import OozieUpgrade from check_oozie_server_status import check_oozie_server_status - +from resource_management.core.resources.zkmigrator import ZkMigrator class OozieServer(Script): @@ -193,7 +193,14 @@ class OozieServerDefault(OozieServer): stack_select.select("oozie-server", params.version) OozieUpgrade.prepare_libext_directory() - + + def disable_security(self, env): + import params + if not params.zk_connection_string: + return + zkmigrator = ZkMigrator(params.zk_connection_string, params.java_exec, params.java64_home, params.jaas_file, params.oozie_user) + zkmigrator.set_acls(params.zk_namespace if params.zk_namespace.startswith('/') else '/' + params.zk_namespace, 'world:anyone:crdwa') + def get_log_folder(self): import params return params.oozie_log_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index f9c608e..48c8ef0 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -138,6 +138,8 @@ oozie_pid_dir = status_params.oozie_pid_dir pid_file = status_params.pid_file hadoop_jar_location = "/usr/lib/hadoop/" java_share_dir = "/usr/share/java" +java64_home = config['hostLevelParams']['java_home'] +java_exec = format("{java64_home}/bin/java") ext_js_file = "ext-2.2.zip" ext_js_path = format("/usr/share/{stack_name_uppercase}-oozie/{ext_js_file}") security_enabled = config['configurations']['cluster-env']['security_enabled'] @@ -157,9 +159,13 @@ oozie_site = config['configurations']['oozie-site'] # Need this for yarn.nodemanager.recovery.dir in yarn-site yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix'] yarn_resourcemanager_address = config['configurations']['yarn-site']['yarn.resourcemanager.address'] +zk_namespace = default('/configurations/oozie-site/oozie.zookeeper.namespace', 'oozie') +zk_connection_string = default('/configurations/oozie-site/oozie.zookeeper.connection.string', None) +jaas_file = os.path.join(conf_dir, 'zkmigrator_jaas.conf') if security_enabled: oozie_site = dict(config['configurations']['oozie-site']) + oozie_principal_with_host = oozie_principal.replace('_HOST', hostname) # If a user-supplied oozie.ha.authentication.kerberos.principal property exists in oozie-site, # use it to replace the existing oozie.authentication.kerberos.principal value. This is to ensure @@ -176,10 +182,8 @@ if security_enabled: if stack_version_formatted and check_stack_feature(StackFeature.OOZIE_HOST_KERBEROS, stack_version_formatted): #older versions of oozie have problems when using _HOST in principal - oozie_site['oozie.service.HadoopAccessorService.kerberos.principal'] = \ - oozie_principal.replace('_HOST', hostname) - oozie_site['oozie.authentication.kerberos.principal'] = \ - http_principal.replace('_HOST', hostname) + oozie_site['oozie.service.HadoopAccessorService.kerberos.principal'] = oozie_principal_with_host + oozie_site['oozie.authentication.kerberos.principal'] = http_principal.replace('_HOST', hostname) smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] oozie_keytab = default("/configurations/oozie-env/oozie_keytab", oozie_service_keytab) http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/zkmigrator_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/zkmigrator_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/zkmigrator_jaas.conf.j2 new file mode 100644 index 0000000..fbc0ce5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/templates/zkmigrator_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{oozie_keytab}}" + principal="{{oozie_principal_with_host}}"; +}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json index d2e2ab8..f1092f5 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json @@ -20,7 +20,8 @@ "oozie.service.AuthorizationService.authorization.enabled": "true", "oozie.service.HadoopAccessorService.kerberos.enabled": "true", "local.realm": "${realm}", - "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" + "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials", + "oozie.zookeeper.secure" : "true" } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index fd7fac9..a64af73 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -162,8 +162,7 @@ { "name": "oozie_host_kerberos", "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)", - "min_version": "2.0.0.0", - "max_version": "2.2.0.0" + "min_version": "2.0.0.0" }, { "name": "falcon_extensions", http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json index dd87b72..ddf8348 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/properties/stack_features.json @@ -162,8 +162,7 @@ { "name": "oozie_host_kerberos", "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)", - "min_version": "2.0.0.0", - "max_version": "2.2.0.0" + "min_version": "2.0.0.0" }, { "name": "falcon_extensions", http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index d24d0b9..f5bd4aa 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -601,7 +601,7 @@ class TestOozieServer(RMFTestCase): self.assertNoMoreResources() @patch.object(shell, "call") - @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) def test_configure_secured(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py", @@ -616,7 +616,7 @@ class TestOozieServer(RMFTestCase): self.assertNoMoreResources() @patch.object(shell, "call") - @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) def test_configure_secured_ha(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) @@ -648,7 +648,7 @@ class TestOozieServer(RMFTestCase): @patch.object(shell, "call") @patch("os.path.isfile") - @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True])) def test_start_secured(self, isfile_mock, call_mocks): isfile_mock.return_value = True call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) @@ -1125,9 +1125,14 @@ class TestOozieServer(RMFTestCase): group = 'hadoop', recursive_ownership = True, ) + self.assertResourceCalled('File', '/etc/oozie/conf/zkmigrator_jaas.conf', + owner = 'oozie', + group = 'hadoop', + content = Template('zkmigrator_jaas.conf.j2') + ) @patch.object(shell, "call") - @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True])) + @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True])) def test_configure_default_hdp22(self, call_mocks): call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added")) config_file = "stacks/2.0.6/configs/default.json" http://git-wip-us.apache.org/repos/asf/ambari/blob/aae7013f/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json index 645e357..0b6b3ab 100644 --- a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/stacks/HDF/2.0/properties/stack_features.json @@ -162,8 +162,7 @@ { "name": "oozie_host_kerberos", "description": "Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)", - "min_version": "0.0.0.0", - "max_version": "0.2.0.0" + "min_version": "0.0.0.0" }, { "name": "falcon_extensions",