AMBARI-22634. Kerberos support for OneFS (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/37a2ca70 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/37a2ca70 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/37a2ca70 Branch: refs/heads/branch-feature-AMBARI-22008-isilon Commit: 37a2ca707567c5b87be150f4a6a5bd13bcb71bc1 Parents: 9fee038 Author: Attila Magyar <amag...@hortonworks.com> Authored: Thu Dec 14 12:59:07 2017 +0100 Committer: Attila Magyar <amag...@hortonworks.com> Committed: Mon Dec 18 09:12:47 2017 +0100 ---------------------------------------------------------------------- .../main/admin/kerberos/step1_controller.js | 105 ++++++++++--------- ambari-web/app/messages.js | 1 + .../app/templates/main/admin/kerberos/step1.hbs | 14 +-- .../addon-services/ONEFS/1.0.0/kerberos.json | 73 +++++++++++++ .../ONEFS/1.0.0/package/scripts/params_linux.py | 12 +-- .../1.0.0/package/scripts/service_check.py | 80 +------------- 6 files changed, 138 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js b/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js index 9c864a8..690843b 100644 --- a/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js +++ b/ambari-web/app/controllers/main/admin/kerberos/step1_controller.js @@ -18,29 +18,44 @@ var App = require('app'); +var PreCondition = Ember.Object.extend({ + displayText: null, + checked: false, + visibilityCriteria: function() { return true; }, + + hidden: function() { + return !this.get('visibilityCriteria')(); + }.property('visibilityCriteria'), + + satisfied: function() { + return this.get('checked') || this.get('hidden'); + }.property('checked', 'hidden') +}); + App.KerberosWizardStep1Controller = Em.Controller.extend({ name: "kerberosWizardStep1Controller", selectedItem: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'), - isSubmitDisabled: Em.computed.someBy('selectedOption.preConditions', 'checked', false), + isSubmitDisabled: Em.computed.someBy('selectedOption.preConditions', 'satisfied', false), options: Em.A([ Em.Object.create({ displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'), value: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc'), preConditions: [ - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.1'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.1') + }), + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.2') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.2'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.3') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.3'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.kdc.condition.4'), + visibilityCriteria: function() { return App.Service.find().someProperty('serviceName', 'ONEFS') } }) ] }), @@ -48,25 +63,20 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({ displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.ad'), value: Em.I18n.t('admin.kerberos.wizard.step1.option.ad'), preConditions: [ - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.1'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.1') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.2'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.2') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.3'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.3') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.4'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.4') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.5'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ad.condition.5') }) ] }), @@ -74,21 +84,17 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({ displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa'), value: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa'), preConditions: [ - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.1'), - checked: false + PreCondition.create({ + dsplayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.1'), }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.2'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.2') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.3'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.3') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.4'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.ipa.condition.4') }) ] }), @@ -96,25 +102,20 @@ App.KerberosWizardStep1Controller = Em.Controller.extend({ displayName: Em.I18n.t('admin.kerberos.wizard.step1.option.manual'), value: Em.I18n.t('admin.kerberos.wizard.step1.option.manual'), preConditions: [ - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.1'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.1') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.2'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.2') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.3'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.3') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.4'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.4') }), - Em.Object.create({ - displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.5'), - checked: false + PreCondition.create({ + displayText: Em.I18n.t('admin.kerberos.wizard.step1.option.manual.condition.5') }) ] }) http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index b305196..1345d65 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1266,6 +1266,7 @@ Em.I18n.translations = { 'admin.kerberos.wizard.step1.option.kdc.condition.1': 'Ambari Server and cluster hosts have network access to both the KDC and KDC admin hosts.', 'admin.kerberos.wizard.step1.option.kdc.condition.2': 'KDC administrative credentials are on-hand.', 'admin.kerberos.wizard.step1.option.kdc.condition.3': 'The Java Cryptography Extensions (JCE) have been setup on the Ambari Server host and all hosts in the cluster.', + 'admin.kerberos.wizard.step1.option.kdc.condition.4': 'The Isilon administrator has setup all appropriate principals in OneFS', 'admin.kerberos.wizard.step1.option.manual': 'Manage Kerberos principals and keytabs manually', 'admin.kerberos.wizard.step1.option.manual.condition.1': 'Cluster hosts have network access to the KDC', 'admin.kerberos.wizard.step1.option.manual.condition.2': 'Kerberos client utilities (such as kinit) have been installed on every cluster host', http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/ambari-web/app/templates/main/admin/kerberos/step1.hbs ---------------------------------------------------------------------- diff --git a/ambari-web/app/templates/main/admin/kerberos/step1.hbs b/ambari-web/app/templates/main/admin/kerberos/step1.hbs index 57d0637..df15daf 100644 --- a/ambari-web/app/templates/main/admin/kerberos/step1.hbs +++ b/ambari-web/app/templates/main/admin/kerberos/step1.hbs @@ -44,12 +44,14 @@ <h5>{{selectedOption.displayName}}:</h5> <b>{{t admin.kerberos.wizard.step1.prerequisites.label}}</b> <br/> <br/> {{#each condition in selectedOption.preConditions}} - {{view App.CheckboxView - classNames="checkbox" - checkedBinding="condition.checked" - labelClassNames="kerberos-step1-prerequiste-checkboxes" - labelBinding="condition.displayText" - }} + {{#unless condition.hidden}} + {{view App.CheckboxView + classNames="checkbox" + checkedBinding="condition.checked" + labelClassNames="kerberos-step1-prerequiste-checkboxes" + labelBinding="condition.displayText" + }} + {{/unless}} {{/each}} </div> </div> http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json new file mode 100644 index 0000000..6078985 --- /dev/null +++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/kerberos.json @@ -0,0 +1,73 @@ +{ + "services": [ + { + "name": "ONEFS", + "identities": [ + { + "name": "hdfs_spnego", + "reference": "/spnego", + "principal": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.principal" + }, + "keytab": { + "configuration": "hdfs-site/dfs.web.authentication.kerberos.keytab" + } + }, + { + "name": "hdfs_smokeuser", + "reference": "/smokeuser" + } + ], + "auth_to_local_properties" : [ + "core-site/hadoop.security.auth_to_local" + ], + "configurations": [ + { + "core-site": { + "hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true", + "hadoop.proxyuser.HTTP.groups": "${hadoop-env/proxyuser_group}" + } + }, + { + "ranger-hdfs-audit": { + "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", + "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", + "xasecure.audit.jaas.Client.option.useKeyTab": "true", + "xasecure.audit.jaas.Client.option.storeKey": "false", + "xasecure.audit.jaas.Client.option.serviceName": "solr", + "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true" + } + } + ], + "components": [ + { + "name": "ONEFS_CLIENT", + "identities": [ + { + "name": "hdfs_hdfs_client_hdfs", + "principal": { + "value": "${hadoop-env/hdfs_user}${principal_suffix}@${realm}", + "type" : "user" , + "configuration": "hadoop-env/hdfs_principal_name", + "local_username" : "${hadoop-env/hdfs_user}" + }, + "keytab": { + "file": "${keytab_dir}/hdfs.headless.keytab", + "owner": { + "name": "${hadoop-env/hdfs_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "hadoop-env/hdfs_user_keytab" + } + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py index b823fc5..953efdc 100644 --- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py +++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/params_linux.py @@ -32,24 +32,16 @@ config = Script.get_config() hostname = config["hostname"] hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) user_group = config['configurations']['cluster-env']['user_group'] hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir'] security_enabled = config['configurations']['cluster-env']['security_enabled'] -namenode_host = default("/clusterHostInfo/namenode_host", []) -journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", []) -zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", []) - -has_journalnode_hosts = not len(journalnode_hosts) == 0 -has_zkfc_hosts = not len(zkfc_hosts) == 0 -is_namenode_master = hostname in namenode_host - dfs_type = default("/commandParams/dfs_type", "") -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_conf_dir = conf_select.get_hadoop_conf_dir() -hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] http://git-wip-us.apache.org/repos/asf/ambari/blob/37a2ca70/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py index 3d798a3..4c92866 100644 --- a/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py +++ b/contrib/management-packs/isilon-onefs-mpack/src/main/resources/addon-services/ONEFS/1.0.0/package/scripts/service_check.py @@ -18,11 +18,7 @@ limitations under the License. """ from resource_management import * -from resource_management.core.shell import as_user from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons import OSConst -from resource_management.libraries.functions.curl_krb_request import curl_krb_request -from resource_management.core.logger import Logger class HdfsServiceCheck(Script): pass @@ -38,7 +34,7 @@ class HdfsServiceCheckDefault(HdfsServiceCheck): tmp_file = format("{dir}/{unique}") if params.security_enabled: - Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + Execute(format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}"), user=params.hdfs_user ) params.HdfsResource(dir, @@ -58,80 +54,6 @@ class HdfsServiceCheckDefault(HdfsServiceCheck): ) params.HdfsResource(None, action="execute") - if params.has_journalnode_hosts: - if params.security_enabled: - for host in params.journalnode_hosts: - if params.https_only: - uri = format("https://{host}:{journalnode_port}") - else: - uri = format("http://{host}:{journalnode_port}") - response, errmsg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab, - params.smokeuser_principal, uri, "jn_service_check", - params.kinit_path_local, False, None, params.smoke_user) - if not response: - Logger.error("Cannot access WEB UI on: {0}. Error : {1}", uri, errmsg) - return 1 - else: - journalnode_port = params.journalnode_port - checkWebUIFileName = "checkWebUI.py" - checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}") - comma_sep_jn_hosts = ",".join(params.journalnode_hosts) - - checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only} -o {script_https_protocol}") - File(checkWebUIFilePath, - content=StaticFile(checkWebUIFileName), - mode=0775) - - Execute(checkWebUICmd, - logoutput=True, - try_sleep=3, - tries=5, - user=params.smoke_user - ) - - if params.is_namenode_master: - if params.has_zkfc_hosts: - pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") - pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid") - check_zkfc_process_cmd = as_user(format( - "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.hdfs_user) - Execute(check_zkfc_process_cmd, - logoutput=True, - try_sleep=3, - tries=5 - ) - -@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) -class HdfsServiceCheckWindows(HdfsServiceCheck): - def service_check(self, env): - import params - env.set_params(params) - - unique = functions.get_unique_id_and_date() - - #Hadoop uses POSIX-style paths, separator is always / - dir = params.hdfs_tmp_dir - tmp_file = dir + '/' + unique - - #commands for execution - hadoop_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hadoop.cmd")) - create_dir_cmd = "%s fs -mkdir %s" % (hadoop_cmd, dir) - own_dir = "%s fs -chmod 777 %s" % (hadoop_cmd, dir) - test_dir_exists = "%s fs -test -e %s" % (hadoop_cmd, dir) - cleanup_cmd = "%s fs -rm %s" % (hadoop_cmd, tmp_file) - create_file_cmd = "%s fs -put %s %s" % (hadoop_cmd, os.path.join(params.hadoop_conf_dir, "core-site.xml"), tmp_file) - test_cmd = "%s fs -test -e %s" % (hadoop_cmd, tmp_file) - - hdfs_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hdfs.cmd")) - safemode_command = "%s dfsadmin -safemode get | %s OFF" % (hdfs_cmd, params.grep_exe) - - Execute(safemode_command, logoutput=True, try_sleep=3, tries=20) - Execute(create_dir_cmd, user=params.hdfs_user,logoutput=True, ignore_failures=True) - Execute(own_dir, user=params.hdfs_user,logoutput=True) - Execute(test_dir_exists, user=params.hdfs_user,logoutput=True) - Execute(create_file_cmd, user=params.hdfs_user,logoutput=True) - Execute(test_cmd, user=params.hdfs_user,logoutput=True) - Execute(cleanup_cmd, user=params.hdfs_user,logoutput=True) if __name__ == "__main__": HdfsServiceCheck().execute()