Repository: ambari Updated Branches: refs/heads/trunk 2ef578f3c -> 414dbe008
AMBARI-11614 : Stack Advisor changes needed for Ranger Kafka Plugin (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/414dbe00 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/414dbe00 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/414dbe00 Branch: refs/heads/trunk Commit: 414dbe00893d7daa59fdfb5e28e3852f80f5a44b Parents: 2ef578f Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Wed Jun 3 14:40:43 2015 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Wed Jun 3 14:40:43 2015 -0700 ---------------------------------------------------------------------- .../functions/setup_ranger_plugin_xml.py | 3 +- .../KAFKA/0.8.1.2.2/package/scripts/params.py | 8 ++- .../package/scripts/setup_ranger_kafka.py | 10 +++- .../0.5.0.2.2/package/scripts/params_linux.py | 2 +- .../0.5.0.2.3/configuration/ranger-kms-site.xml | 2 +- .../0.9.1.2.1/package/scripts/params_linux.py | 2 +- .../stacks/HDP/2.3/services/RANGER/alerts.json | 62 ++++++++++++++++++++ .../stacks/HDP/2.3/services/stack_advisor.py | 34 ++++++++++- ambari-web/app/data/HDP2.3/site_properties.js | 27 +++++++++ 9 files changed, 142 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py index 1a24723..228c367 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_ranger_plugin_xml.py @@ -120,7 +120,8 @@ def setup_ranger_plugin(component_select_name, service_name, group = component_group, mode=0744) - setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list) + #This should be done by rpm + #setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py index d8c4507..708e7cd 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py @@ -22,6 +22,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions from resource_management.libraries.functions.default import default from utils import get_bare_principal +from resource_management.libraries.functions.get_hdp_version import get_hdp_version import status_params @@ -198,10 +199,15 @@ if has_ranger_admin and is_supported_kafka_ranger: downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") + driver_curl_target = format("{kafka_home}libs/{jdbc_jar_name}") ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None + + hdp_version = get_hdp_version('kafka-broker') + setup_ranger_env_sh_source = format('/usr/hdp/{hdp_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh') + setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh") + http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py index 2ad9bbf..94bbafa 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/setup_ranger_kafka.py @@ -15,13 +15,15 @@ See the License for the specific language governing permissions and limitations under the License. """ from resource_management.core.logger import Logger -from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin +from resource_management.core.resources import Execute +from resource_management.libraries.functions.format import format def setup_ranger_kafka(): import params if params.has_ranger_admin: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin setup_ranger_plugin('kafka-broker', 'kafka', params.downloaded_custom_connector, params.driver_curl_source, params.driver_curl_target, params.java64_home, @@ -37,5 +39,11 @@ def setup_ranger_kafka(): credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, api_version = 'v2') + + if params.enable_ranger_kafka: + Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target), + not_if=format("test -f {setup_ranger_env_sh_target}"), + sudo=True + ) else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index 9ff7902..8cebd64 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -210,7 +210,7 @@ if has_ranger_admin: downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") + driver_curl_target = format("/usr/hdp/current/knox-server/ext/{jdbc_jar_name}") knox_ranger_plugin_config = { 'username': repo_config_username, http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-site.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-site.xml index 0cdc653..647e2a4 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-site.xml @@ -22,7 +22,7 @@ <configuration> <property> <name>ranger.service.host</name> - <value>{{ranger_admin_hosts}}</value> + <value>{{kms_host}}</value> </property> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py index b8819e7..d0ed8f0 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py @@ -216,7 +216,7 @@ if has_ranger_admin: downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") + driver_curl_target = format("{storm_component_home_dir}/lib/{jdbc_jar_name}") storm_ranger_plugin_config = { 'username': repo_config_username, http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/alerts.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/alerts.json new file mode 100644 index 0000000..2de5f87 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/alerts.json @@ -0,0 +1,62 @@ +{ + "RANGER": { + "service": [], + "RANGER_ADMIN": [ + { + "name": "ranger_admin_process", + "label": "Ranger Admin Process", + "description": "This host-level alert is triggered if the Ranger Admin Web UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{admin-properties/policymgr_external_url}}", + "https": "{{admin-properties/policymgr_external_url}}", + "https_property": "{{ranger-admin-site/ranger.service.https.attrib.ssl.enabled}}", + "https_property_value": "true", + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning": { + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + } + ], + "RANGER_USERSYNC": [ + { + "name": "ranger_usersync_process", + "label": "Ranger Usersync Process", + "description": "This host-level alert is triggered if the Ranger Usersync cannot be determined to be up.", + "interval": 1, + "scope": "HOST", + "source": { + "type": "PORT", + "uri": "{{ranger-ugsync-site/ranger.usersync.port}}", + "default_port": 5151, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index 58a8f7d..86a150d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -25,7 +25,8 @@ class HDP23StackAdvisor(HDP22StackAdvisor): "TEZ": self.recommendTezConfigurations, "HDFS": self.recommendHDFSConfigurations, "HIVE": self.recommendHIVEConfigurations, - "HBASE": self.recommendHBASEConfigurations + "HBASE": self.recommendHBASEConfigurations, + "KAFKA": self.recommendKAFKAConfigurations, } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -86,12 +87,22 @@ class HDP23StackAdvisor(HDP22StackAdvisor): if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == 'Yes'.lower()): putHdfsSiteProperty("dfs.namenode.inode.attributes.provider.class",'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer') + def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts): + putKafkaBrokerProperty = self.putProperty(configurations, "kafka-broker", services) + + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if 'ranger-kafka-plugin-properties' in services['configurations'] and ('ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']): + rangerPluginEnabled = services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'] + if ("RANGER" in servicesList) and (rangerPluginEnabled.lower() == "Yes".lower()): + putKafkaBrokerProperty("authorizer.class.name", 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer') + def getServiceConfigurationValidators(self): parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators() childValidators = { "HDFS": {"hdfs-site": self.validateHDFSConfigurations}, "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations}, - "HBASE": {"hbase-site": self.validateHBASEConfigurations} + "HBASE": {"hbase-site": self.validateHBASEConfigurations}, + "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations} } self.mergeValidators(parentValidators, childValidators) return parentValidators @@ -197,6 +208,25 @@ class HDP23StackAdvisor(HDP22StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "hbase-site") + def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + kafka_broker = properties + validationItems = [] + + #Adding Ranger Plugin logic here + ranger_plugin_properties = getSiteProperties(configurations, "ranger-kafka-plugin-properties") + ranger_plugin_enabled = ranger_plugin_properties['ranger-kafka-plugin-enabled'] + prop_name = 'authorizer.class.name' + prop_val = "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" + servicesList = [service["StackServices"]["service_name"] for service in services["services"]] + if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()): + if kafka_broker[prop_name] != prop_val: + validationItems.append({"config-name": prop_name, + "item": self.getWarnItem( + "If Ranger Kafka Plugin is enabled."\ + "{0} needs to be set to {1}".format(prop_name,prop_val))}) + + return self.toConfigurationValidationProblems(validationItems, "kafka-broker") + def isComponentUsingCardinalityForLayout(self, componentName): return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER'] http://git-wip-us.apache.org/repos/asf/ambari/blob/414dbe00/ambari-web/app/data/HDP2.3/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2.3/site_properties.js b/ambari-web/app/data/HDP2.3/site_properties.js index a2963da..53d663c 100644 --- a/ambari-web/app/data/HDP2.3/site_properties.js +++ b/ambari-web/app/data/HDP2.3/site_properties.js @@ -395,6 +395,33 @@ hdp23properties.push({ }, { "id": "site property", + "name": "xasecure.audit.destination.db", + "displayName": "Audit to DB", + "displayType": "checkbox", + "filename": "ranger-kafka-audit.xml", + "category": "Advanced ranger-kafka-audit", + "serviceName": "KAFKA" + }, + { + "id": "site property", + "name": "xasecure.audit.destination.hdfs", + "displayName": "Audit to HDFS", + "displayType": "checkbox", + "filename": "ranger-kafka-audit.xml", + "category": "Advanced ranger-kafka-audit", + "serviceName": "KAFKA" + }, + { + "id": "site property", + "name": "xasecure.audit.destination.solr", + "displayName": "Audit to SOLR", + "displayType": "checkbox", + "filename": "ranger-kafka-audit.xml", + "category": "Advanced ranger-kafka-audit", + "serviceName": "KAFKA" + }, + { + "id": "site property", "name": "xasecure.audit.provider.summary.enabled", "displayName": "Audit provider summary enabled", "displayType": "checkbox",
