Repository: ambari Updated Branches: refs/heads/branch-2.5 2741265f9 -> 7338d5280
AMBARI-19668. Supporting zookeeper security only from HDP 2.6. (Attila Magyar via stoader) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7338d528 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7338d528 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7338d528 Branch: refs/heads/branch-2.5 Commit: 7338d5280333b574bd001465cc8422dfa1b5edc8 Parents: 2741265 Author: Attila Magyar <amag...@hortonworks.com> Authored: Tue Jan 24 16:07:22 2017 +0100 Committer: Toader, Sebastian <stoa...@hortonworks.com> Committed: Tue Jan 24 16:07:56 2017 +0100 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 5 - .../HDFS/2.1.0.2.0/kerberos.json | 3 +- .../2.1.0.2.0/package/scripts/params_linux.py | 5 + .../2.1.0.2.0/package/scripts/zkfc_slave.py | 18 +- .../4.0.0.2.0/package/scripts/oozie_server.py | 4 + .../4.0.0.2.0/package/scripts/params_linux.py | 3 + .../OOZIE/4.2.0.2.3/kerberos.json | 3 +- .../YARN/2.1.0.2.0/kerberos.json | 3 +- .../2.1.0.2.0/package/scripts/params_linux.py | 1 + .../package/scripts/resourcemanager.py | 5 +- .../2.0.6/hooks/before-ANY/scripts/params.py | 9 +- .../HDP/2.0.6/properties/stack_features.json | 5 + .../services/HDFS/configuration/hadoop-env.xml | 5 - .../stacks/HDP/2.2/services/YARN/kerberos.json | 3 +- .../HDP/2.3.ECS/services/YARN/kerberos.json | 3 +- .../services/HDFS/configuration/hadoop-env.xml | 5 - .../stacks/HDP/2.3/services/YARN/kerberos.json | 3 +- .../services/HDFS/configuration/hadoop-env.xml | 5 - .../stacks/HDP/2.5/services/HDFS/kerberos.json | 3 +- .../stacks/HDP/2.5/services/YARN/kerberos.json | 5 +- .../services/HDFS/configuration/hadoop-env.xml | 181 ++++++++++++ .../stacks/HDP/2.6/services/HDFS/kerberos.json | 247 ++++++++++++++++ .../stacks/HDP/2.6/services/OOZIE/kerberos.json | 70 +++++ .../stacks/HDP/2.6/services/YARN/kerberos.json | 278 +++++++++++++++++++ .../PERF/1.0/properties/stack_features.json | 5 + .../test/python/stacks/2.0.6/HDFS/test_zkfc.py | 9 +- 27 files changed, 837 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 02ce194..8fd5c8d 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -49,6 +49,7 @@ class StackFeature: CONFIG_VERSIONING = "config_versioning" FALCON_EXTENSIONS = "falcon_extensions" DATANODE_NON_ROOT = "datanode_non_root" + SECURE_ZOOKEEPER = "secure_zookeeper" REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env" RANGER = "ranger" RANGER_TAGSYNC_COMPONENT = "ranger_tagsync_component" http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml index bc64d1f..89d5001 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hadoop-env.xml @@ -377,11 +377,6 @@ if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$H ulimit -l {{datanode_max_locked_memory}} fi {% endif %} - -# Enable ACLs on zookeper znodes if required -{% if hadoop_zkfc_opts is defined %} - export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" -{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/kerberos.json index 630c200..3cb83ae 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/kerberos.json @@ -24,8 +24,7 @@ "core-site": { "hadoop.security.authentication": "kerberos", "hadoop.security.authorization": "true", - "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}", - "ha.zookeeper.acl":"sasl:nn:rwcda" + "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}" } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 22e2ee6..07cb409 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -75,6 +75,7 @@ version_for_stack_feature_checks = get_stack_feature_version(config) stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) +stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user = status_params.hdfs_user @@ -281,6 +282,9 @@ dfs_ha_automatic_failover_enabled = default("/configurations/hdfs-site/dfs.ha.au dfs_ha_namenode_active = default("/configurations/hadoop-env/dfs_ha_initial_namenode_active", None) # hostname of the standby HDFS HA Namenode (only used when HA is enabled) dfs_ha_namenode_standby = default("/configurations/hadoop-env/dfs_ha_initial_namenode_standby", None) +ha_zookeeper_quorum = config['configurations']['core-site']['ha.zookeeper.quorum'] +jaas_file = os.path.join(hadoop_conf_secure_dir, 'hdfs_jaas.conf') +zk_namespace = default('/configurations/hdfs-site/ha.zookeeper.parent-znode', '/hadoop-ha') # Values for the current Host namenode_id = None @@ -376,6 +380,7 @@ name_node_params = default("/commandParams/namenode", None) java_home = config['hostLevelParams']['java_home'] java_version = expect("/hostLevelParams/java_version", int) +java_exec = format("{java_home}/bin/java") hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index c4091ff..94bc64c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -37,6 +37,7 @@ from resource_management.libraries.functions.security_commons import validate_se from resource_management.libraries.functions.security_commons import FILE_TYPE_XML from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.script import Script +from resource_management.core.resources.zkmigrator import ZkMigrator @@ -62,8 +63,10 @@ class ZkfcSlave(Script): env.set_params(params) hdfs("zkfc_slave") - # set up failover / zookeper ACLs - utils.set_up_zkfc_security(params) + # set up failover / zookeper ACLs, this feature is supported from HDP 2.6 ownwards + if params.stack_supports_zk_security: + utils.set_up_zkfc_security(params) + pass @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) @@ -164,7 +167,16 @@ class ZkfcSlaveDefault(ZkfcSlave): self.put_structured_out({"securityState": "UNSECURED"}) else: self.put_structured_out({"securityState": "UNSECURED"}) - + + def disable_security(self, env): + import params + + if not params.stack_supports_zk_security: + return + + zkmigrator = ZkMigrator(params.ha_zookeeper_quorum, params.java_exec, params.java_home, params.jaas_file, params.hdfs_user) + zkmigrator.set_acls(params.zk_namespace if params.zk_namespace.startswith('/') else '/' + params.zk_namespace, 'world:anyone:crdwa') + def get_log_folder(self): import params return params.hdfs_log_dir http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index 1e4eeff..4397fe2 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -196,7 +196,11 @@ class OozieServerDefault(OozieServer): def disable_security(self, env): import params + if not params.stack_supports_zk_security: + Logger.info("Stack doesn't support zookeeper security") + return if not params.zk_connection_string: + Logger.info("No zookeeper connection string. Skipping reverting ACL") return zkmigrator = ZkMigrator(params.zk_connection_string, params.java_exec, params.java64_home, params.jaas_file, params.oozie_user) zkmigrator.set_acls(params.zk_namespace if params.zk_namespace.startswith('/') else '/' + params.zk_namespace, 'world:anyone:crdwa') http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py index 005bcc7..3dc55f1 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py @@ -34,6 +34,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.expect import expect from resource_management.libraries.functions.get_architecture import get_architecture +from resource_management.libraries.functions.stack_features import get_stack_feature_version from urlparse import urlparse @@ -64,6 +65,7 @@ agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int stack_root = status_params.stack_root stack_version_unformatted = status_params.stack_version_unformatted stack_version_formatted = status_params.stack_version_formatted +version_for_stack_feature_checks = get_stack_feature_version(config) hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") @@ -160,6 +162,7 @@ yarn_resourcemanager_address = config['configurations']['yarn-site']['yarn.resou zk_namespace = default('/configurations/oozie-site/oozie.zookeeper.namespace', 'oozie') zk_connection_string = default('/configurations/oozie-site/oozie.zookeeper.connection.string', None) jaas_file = os.path.join(conf_dir, 'zkmigrator_jaas.conf') +stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) if security_enabled: oozie_site = dict(config['configurations']['oozie-site']) http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json index f1092f5..d2e2ab8 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/kerberos.json @@ -20,8 +20,7 @@ "oozie.service.AuthorizationService.authorization.enabled": "true", "oozie.service.HadoopAccessorService.kerberos.enabled": "true", "local.realm": "${realm}", - "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials", - "oozie.zookeeper.secure" : "true" + "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json index bd2b285..35552c9 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/kerberos.json @@ -30,8 +30,7 @@ "yarn.resourcemanager.proxyuser.*.groups": "", "yarn.resourcemanager.proxyuser.*.hosts": "", "yarn.resourcemanager.proxyuser.*.users": "", - "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + "yarn.resourcemanager.proxy-user-privileges.enabled": "true" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index b2c7b01..c56e72f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -249,6 +249,7 @@ nodemanager_kinit_cmd = "" rm_zk_address = config['configurations']['yarn-site']['yarn.resourcemanager.zk-address'] rm_zk_znode = config['configurations']['yarn-site']['yarn.resourcemanager.zk-state-store.parent-path'] rm_zk_store_class = config['configurations']['yarn-site']['yarn.resourcemanager.store.class'] +stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) if security_enabled: rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal'] http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py index 15cb3be..79b5810 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py @@ -228,8 +228,11 @@ class ResourcemanagerDefault(Resourcemanager): def disable_security(self, env): import params + if not params.stack_supports_zk_security: + Logger.info("Stack doesn't support zookeeper security") + return if not params.rm_zk_address: - Logger.info("Skipping reverting ACL") + Logger.info("No zookeeper connection string. Skipping reverting ACL") return zkmigrator = ZkMigrator( params.rm_zk_address, \ http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py index d4e505a..8e0e783 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py @@ -32,6 +32,9 @@ from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version from ambari_commons.constants import AMBARI_SUDO_BINARY @@ -181,6 +184,9 @@ ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) zeppelin_master_hosts = default("/clusterHostInfo/zeppelin_master_hosts", []) zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", []) +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + has_namenode = not len(namenode_host) == 0 has_ganglia_server = not len(ganglia_server_hosts) == 0 @@ -191,6 +197,7 @@ has_falcon_server_hosts = not len(falcon_server_hosts) == 0 has_ranger_admin = not len(ranger_admin_hosts) == 0 has_zeppelin_master = not len(zeppelin_master_hosts) == 0 has_zkfc_hosts = not len(zkfc_hosts)== 0 +stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) if has_namenode or dfs_type == 'HCFS': hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True) @@ -239,5 +246,5 @@ tez_am_view_acls = config['configurations']['tez-site']["tez.am.view-acls"] override_uid = str(default("/configurations/cluster-env/override_uid", "true")).lower() # if NN HA on secure clutser, access Zookeper securely -if has_zkfc_hosts and security_enabled: +if stack_supports_zk_security and has_zkfc_hosts and security_enabled: hadoop_zkfc_opts=format("-Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config={hadoop_conf_secure_dir}/hdfs_jaas.conf -Dzookeeper.sasl.clientconfig=Client") http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index e4a499b..0fd1766 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -22,6 +22,11 @@ "min_version": "2.2.0.0" }, { + "name": "secure_zookeeper", + "description": "Protect ZNodes with SASL acl in secure clusters", + "min_version": "2.6.0.0" + }, + { "name": "config_versioning", "description": "Configurable versions support", "min_version": "2.3.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml index ef111e0..5be2b74 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/hadoop-env.xml @@ -180,11 +180,6 @@ if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$H ulimit -l {{datanode_max_locked_memory}} fi {% endif %} - -# Enable ACLs on zookeper znodes if required -{% if hadoop_zkfc_opts is defined %} - export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}}" -{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json index a8ef83c..8618804 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/kerberos.json @@ -31,8 +31,7 @@ "yarn.resourcemanager.proxyuser.*.hosts": "", "yarn.resourcemanager.proxyuser.*.users": "", "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure", - "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + "yarn.resourcemanager.zk-state-store.parent-path": "/rmstore-secure" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json index 3059f14..4c5bcdb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3.ECS/services/YARN/kerberos.json @@ -33,8 +33,7 @@ "yarn.resourcemanager.proxyuser.*.groups": "", "yarn.resourcemanager.proxyuser.*.hosts": "", "yarn.resourcemanager.proxyuser.*.users": "", - "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + "yarn.resourcemanager.proxy-user-privileges.enabled": "true" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml index cce3bdb..b6c3738 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml @@ -156,11 +156,6 @@ if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$H {% endif %} ulimit -n {{hdfs_user_nofile_limit}} fi - -# Enable ACLs on zookeper znodes if required -{% if hadoop_zkfc_opts is defined %} - export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}}" -{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json index 5fff05c..0e7a5de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/kerberos.json @@ -31,8 +31,7 @@ "yarn.resourcemanager.proxyuser.*.groups": "", "yarn.resourcemanager.proxyuser.*.hosts": "", "yarn.resourcemanager.proxyuser.*.users": "", - "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + "yarn.resourcemanager.proxy-user-privileges.enabled": "true" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.4/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/HDFS/configuration/hadoop-env.xml index 0212ba0..24e0193 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/HDFS/configuration/hadoop-env.xml @@ -156,11 +156,6 @@ if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$H {% endif %} ulimit -n {{hdfs_user_nofile_limit}} fi - -# Enable ACLs on zookeper znodes if required -{% if hadoop_zkfc_opts is defined %} - export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}}" -{% endif %} </value> <value-attributes> <type>content</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/kerberos.json index 0623db4..826d019 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HDFS/kerberos.json @@ -24,8 +24,7 @@ "core-site": { "hadoop.security.authentication": "kerberos", "hadoop.security.authorization": "true", - "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}", - "ha.zookeeper.acl":"sasl:nn:rwcda" + "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}" } }, { http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json index eaffec6..395840e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -31,8 +31,7 @@ "yarn.resourcemanager.proxyuser.*.groups": "", "yarn.resourcemanager.proxyuser.*.hosts": "", "yarn.resourcemanager.proxyuser.*.users": "", - "yarn.resourcemanager.proxy-user-privileges.enabled": "true", - "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + "yarn.resourcemanager.proxy-user-privileges.enabled": "true" } }, { @@ -275,4 +274,4 @@ ] } ] -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-env.xml new file mode 100644 index 0000000..768ca82 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/configuration/hadoop-env.xml @@ -0,0 +1,181 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- hadoop-env.sh --> + <property> + <name>content</name> + <display-name>hadoop-env template</display-name> + <description>This is the jinja template for hadoop-env.sh file</description> + <value> + # Set Hadoop-specific environment variables here. + + # The only required environment variable is JAVA_HOME. All others are + # optional. When running a distributed configuration it is best to + # set JAVA_HOME in this file, so that it is correctly defined on + # remote nodes. + + # The java implementation to use. Required. + export JAVA_HOME={{java_home}} + export HADOOP_HOME_WARN_SUPPRESS=1 + + # Hadoop home directory + export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + + # Hadoop Configuration Directory + + {# this is different for HDP1 #} + # Path to jsvc required by secure HDP 2.0 datanode + export JSVC_HOME={{jsvc_path}} + + + # The maximum amount of heap to use, in MB. Default is 1000. + export HADOOP_HEAPSIZE="{{hadoop_heapsize}}" + + export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}" + + # Extra Java runtime options. Empty by default. + export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" + + # Command specific options appended to HADOOP_OPTS when specified + HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" + + HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" + + {% if java_version < 8 %} + SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" + export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" + export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly" + + export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + + # The following applies to multiple commands (fs, dfs, fsck, distcp etc) + export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS" + + {% else %} + SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" + export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" + export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly" + + export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + + # The following applies to multiple commands (fs, dfs, fsck, distcp etc) + export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" + {% endif %} + + HADOOP_NFS3_OPTS="-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}" + HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}" + + + # On secure datanodes, user to run the datanode as after dropping privileges + export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}} + + # Extra ssh options. Empty by default. + export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" + + # Where log files are stored. $HADOOP_HOME/logs by default. + export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER + + # History server logs + export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER + + # Where log files are stored in the secure data environment. + export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER + + # File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. + # export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + + # host:path where hadoop code should be rsync'd from. Unset by default. + # export HADOOP_MASTER=master:/home/$USER/src/hadoop + + # Seconds to sleep between slave commands. Unset by default. This + # can be useful in large clusters, where, e.g., slave rsyncs can + # otherwise arrive faster than the master can service them. + # export HADOOP_SLAVE_SLEEP=0.1 + + # The directory where pid files are stored. /tmp by default. + export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER + export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER + + # History server pid + export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER + + YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" + + # A string representing this instance of hadoop. $USER by default. + export HADOOP_IDENT_STRING=$USER + + # The scheduling priority for daemon processes. See 'man nice'. + + # export HADOOP_NICENESS=10 + + # Add database libraries + JAVA_JDBC_LIBS="" + if [ -d "/usr/share/java" ]; then + for jarFile in `ls /usr/share/java | grep -E "(mysql|ojdbc|postgresql|sqljdbc)" 2>/dev/null` + do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile + done + fi + + # Add libraries to the hadoop classpath - some may not need a colon as they already include it + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS} + + # Setting path to hdfs command line + export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} + + # Mostly required for hadoop 2.0 + export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} + + export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" + + + # Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. + # Makes sense to fix only when runing DN as root + if [ "$command" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then + {% if is_datanode_max_locked_memory_set %} + ulimit -l {{datanode_max_locked_memory}} + {% endif %} + ulimit -n {{hdfs_user_nofile_limit}} + fi + + # Enable ACLs on zookeper znodes if required + {% if hadoop_zkfc_opts is defined %} + export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" + {% endif %} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nfsgateway_heapsize</name> + <display-name>NFSGateway maximum Java heap size</display-name> + <value>1024</value> + <description>Maximum Java heap size for NFSGateway (Java option -Xmx)</description> + <value-attributes> + <type>int</type> + <unit>MB</unit> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/kerberos.json new file mode 100644 index 0000000..b5acf92 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HDFS/kerberos.json @@ -0,0 +1,247 @@ +{ + "services": [ + { + "name": "HDFS", + "identities": [ + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.keytab" + } + }, + { + "name": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "core-site/hadoop.security.auth_to_local" + ], + "configurations": [ + { + "core-site": { + "hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true", + "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}", + "ha.zookeeper.acl":"sasl:nn:rwcda" + } + }, + { + "ranger-hdfs-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ], + "components": [ + { + "name": "HDFS_CLIENT", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + }, + { + "name": "NAMENODE", + "identities": [ + { + "name": "hdfs", + "principal": { + "value": "${hadoop-env/hdfs_user}${principal_suffix}@${realm}", + "type" : "user" , + "configuration": "hadoop-env/hdfs_principal_name", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/hdfs.headless.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hadoop-env/hdfs_user_keytab" + } + }, + { + "name": "namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.namenode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.namenode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.namenode.kerberos.internal.spnego.principal" + } + }, + { + "name": "/HDFS/NAMENODE/namenode_nn", + "principal": { + "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-hdfs-audit/xasecure.audit.jaas.Client.option.keyTab" + } + } + ], + "configurations": [ + { + "hdfs-site": { + "dfs.block.access.token.enable": "true" + } + } + ] + }, + { + "name": "DATANODE", + "identities": [ + { + "name": "datanode_dn", + "principal": { + "value": "dn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.datanode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/dn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.datanode.keytab.file" + } + } + ], + "configurations" : [ + { + "hdfs-site" : { + "dfs.datanode.address" : "0.0.0.0:1019", + "dfs.datanode.http.address": "0.0.0.0:1022" + } + } + ] + }, + { + "name": "SECONDARY_NAMENODE", + "identities": [ + { + "name": "secondary_namenode_nn", + "principal": { + "value": "nn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.secondary.namenode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.secondary.namenode.kerberos.internal.spnego.principal" + } + } + ] + }, + { + "name": "NFS_GATEWAY", + "identities": [ + { + "name": "nfsgateway", + "principal": { + "value": "nfs/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/nfs.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/nfs.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/nfs.keytab.file" + } + } + ] + }, + { + "name": "JOURNALNODE", + "identities": [ + { + "name": "journalnode_jn", + "principal": { + "value": "jn/_HOST@${realm}", + "type" : "service", + "configuration": "hdfs-site/dfs.journalnode.kerberos.principal", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/jn.service.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hdfs-site/dfs.journalnode.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.journalnode.kerberos.internal.spnego.principal" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/kerberos.json new file mode 100644 index 0000000..f1092f5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/OOZIE/kerberos.json @@ -0,0 +1,70 @@ +{ + "services": [ + { + "name": "OOZIE", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "oozie-site/oozie.authentication.kerberos.name.rules" + ], + "configurations": [ + { + "oozie-site": { + "oozie.authentication.type": "kerberos", + "oozie.service.AuthorizationService.authorization.enabled": "true", + "oozie.service.HadoopAccessorService.kerberos.enabled": "true", + "local.realm": "${realm}", + "oozie.credentials.credentialclasses": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials", + "oozie.zookeeper.secure" : "true" + } + } + ], + "components": [ + { + "name": "OOZIE_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "oozie_server", + "principal": { + "value": "oozie/_HOST@${realm}", + "type" : "service", + "configuration": "oozie-site/oozie.service.HadoopAccessorService.kerberos.principal", + "local_username" : "${oozie-env/oozie_user}" + }, + "keytab": { + "file": "${keytab_dir}/oozie.service.keytab", + "owner": { + "name": "${oozie-env/oozie_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "oozie-site/oozie.service.HadoopAccessorService.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "oozie-site/oozie.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "oozie-site/oozie.authentication.kerberos.keytab" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json new file mode 100644 index 0000000..eaffec6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json @@ -0,0 +1,278 @@ +{ + "services": [ + { + "name": "YARN", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "configurations": [ + { + "yarn-site": { + "yarn.timeline-service.enabled": "true", + "yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true", + "yarn.admin.acl": "${yarn-env/yarn_user},dr.who", + "yarn.timeline-service.http-authentication.signature.secret": "", + "yarn.timeline-service.http-authentication.signature.secret.file": "", + "yarn.timeline-service.http-authentication.signer.secret.provider": "", + "yarn.timeline-service.http-authentication.signer.secret.provider.object": "", + "yarn.timeline-service.http-authentication.token.validity": "", + "yarn.timeline-service.http-authentication.cookie.domain": "", + "yarn.timeline-service.http-authentication.cookie.path": "", + "yarn.timeline-service.http-authentication.proxyuser.*.hosts": "", + "yarn.timeline-service.http-authentication.proxyuser.*.users": "", + "yarn.timeline-service.http-authentication.proxyuser.*.groups": "", + "yarn.timeline-service.http-authentication.kerberos.name.rules": "", + "yarn.resourcemanager.proxyuser.*.groups": "", + "yarn.resourcemanager.proxyuser.*.hosts": "", + "yarn.resourcemanager.proxyuser.*.users": "", + "yarn.resourcemanager.proxy-user-privileges.enabled": "true", + "yarn.resourcemanager.zk-acl" : "sasl:rm:rwcda" + } + }, + { + "core-site": { + "hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*", + "hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/rm_host}" + } + }, + { + "capacity-scheduler": { + "yarn.scheduler.capacity.root.acl_administer_queue": "${yarn-env/yarn_user}", + "yarn.scheduler.capacity.root.default.acl_administer_queue": "${yarn-env/yarn_user}", + "yarn.scheduler.capacity.root.acl_administer_jobs": "${yarn-env/yarn_user}", + "yarn.scheduler.capacity.root.default.acl_administer_jobs": "${yarn-env/yarn_user}", + "yarn.scheduler.capacity.root.default.acl_submit_applications": "${yarn-env/yarn_user}" + } + }, + { + "ranger-yarn-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ], + "components": [ + { + "name": "NODEMANAGER", + "identities": [ + { + "name": "nodemanager_nm", + "principal": { + "value": "nm/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.nodemanager.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/nm.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.nodemanager.keytab" + } + }, + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "hive-interactive-site/hive.llap.daemon.service.principal" + }, + "keytab": { + "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file" + }, + "when" : { + "contains" : ["services", "HIVE"] + } + }, + { + "name": "llap_zk_hive", + "principal": { + "value": "hive/_HOST@${realm}", + "type" : "service", + "configuration": "hive-interactive-site/hive.llap.zk.sm.principal" + }, + "keytab": { + "file": "${keytab_dir}/hive.llap.zk.sm.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file" + }, + "when" : { + "contains" : ["services", "HIVE"] + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-keytab-file" + } + } + ], + "configurations": [ + { + "yarn-site": { + "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor" + } + } + ] + }, + { + "name": "RESOURCEMANAGER", + "identities": [ + { + "name": "resource_manager_rm", + "principal": { + "value": "rm/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.resourcemanager.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/rm.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.resourcemanager.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file" + } + }, + { + "name": "/YARN/RESOURCEMANAGER/resource_manager_rm", + "principal": { + "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.principal" + }, + "keytab": { + "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.keyTab" + } + } + ] + }, + { + "name": "APP_TIMELINE_SERVER", + "identities": [ + { + "name": "app_timeline_server_yarn", + "principal": { + "value": "yarn/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.timeline-service.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/yarn.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.timeline-service.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab" + } + }, + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + } + ] + }, + { + "name": "MAPREDUCE2", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "HISTORYSERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "history_server_jhs", + "principal": { + "value": "jhs/_HOST@${realm}", + "type" : "service", + "configuration": "mapred-site/mapreduce.jobhistory.principal", + "local_username": "${mapred-env/mapred_user}" + }, + "keytab": { + "file": "${keytab_dir}/jhs.service.keytab", + "owner": { + "name": "${mapred-env/mapred_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "mapred-site/mapreduce.jobhistory.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-principal" + }, + "keytab": { + "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file" + } + } + ] + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/main/resources/stacks/PERF/1.0/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/properties/stack_features.json b/ambari-server/src/main/resources/stacks/PERF/1.0/properties/stack_features.json index 81640b6..e9e0ed2 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/properties/stack_features.json @@ -6,6 +6,11 @@ "min_version": "1.0.0.0" }, { + "name": "secure_zookeeper", + "description": "Protect ZNodes with SASL acl in secure clusters", + "min_version": "2.6.0.0" + }, + { "name": "config_versioning", "description": "Configurable versions support", "min_version": "1.0.0.0" http://git-wip-us.apache.org/repos/asf/ambari/blob/7338d528/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py index aa9e9bc..0f64291 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py @@ -174,13 +174,6 @@ class TestZkfc(RMFTestCase): owner = 'root', ) - self.assertResourceCalled('File', '/etc/hadoop/conf/secure/hdfs_jaas.conf', - owner='root', - group='root', - mode=0644, - content=Template("hdfs_jaas.conf.j2") - ) - self.assertResourceCalled('Directory', '/var/run/hadoop', owner = 'hdfs', group = 'hadoop', @@ -488,4 +481,4 @@ class TestZkfc(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) \ No newline at end of file + put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
