Repository: metron Updated Branches: refs/heads/master 7f9f86c00 -> 8560d2dfc
METRON-1122: Add support for the profiler in the management pack this closes apache/incubator-metron#709 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8560d2df Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8560d2df Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8560d2df Branch: refs/heads/master Commit: 8560d2dfc96dd0215ca50acc3148300e7ef2ad75 Parents: 7f9f86c Author: cstella <ceste...@gmail.com> Authored: Tue Aug 29 14:41:10 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Tue Aug 29 14:41:10 2017 -0400 ---------------------------------------------------------------------- .../METRON/CURRENT/role_command_order.json | 2 + .../configuration/metron-indexing-env.xml | 2 +- .../configuration/metron-profiler-env.xml | 155 +++++++++++++++ .../common-services/METRON/CURRENT/metainfo.xml | 43 ++++ .../package/scripts/params/params_linux.py | 25 +++ .../package/scripts/params/status_params.py | 11 ++ .../package/scripts/profiler_commands.py | 194 ++++++++++++++++++ .../CURRENT/package/scripts/profiler_master.py | 94 +++++++++ .../package/templates/profiler.properties.j2 | 47 +++++ .../METRON/CURRENT/service_advisor.py | 5 + .../METRON/CURRENT/themes/metron_theme.json | 198 ++++++++++++++++++- .../roles/ambari_config/vars/single_node_vm.yml | 7 +- .../roles/ambari_config/vars/small_cluster.yml | 5 +- 13 files changed, 776 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json index 820179a..015f026 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json @@ -5,11 +5,13 @@ "_comment" : "dependencies for all cases", "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"], "METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"], + "METRON_PROFILER-INSTALL": ["METRON_ENRICHMENT-INSTALL"], "METRON_REST-INSTALL": ["METRON_PARSERS-INSTALL"], "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"], "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"], "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"], "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"], + "METRON_PROFILER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_ENRICHMENT-INSTALL"], "METRON_REST-START": ["KAFKA_BROKER-START","STORM_REST_API-START","ZOOKEEPER_SERVER-START","NAMENODE-START","METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"], "METRON_MANAGEMENT_UI-START": ["METRON_REST-START"], "STORM_REST_API-STOP" : ["METRON_ENRICHMENT_MASTER-STOP","METRON_PARSERS-STOP","METRON_INDEXING-STOP","METRON_REST-STOP","METRON_MANAGEMENT_UI-STOP"], http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index e36730a..6abbe77 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -59,7 +59,7 @@ <description>Indexing Writer Class Name</description> <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value> <display-name>Indexing Writer Class Name</display-name> - </property> + </property> <property> <name>update_table</name> <description>The HBase table which will hold edits to indexed data</description> http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml new file mode 100644 index 0000000..c7f6ce2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -0,0 +1,155 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>profiler_kafka_start</name> + <value>UNCOMMITTED_EARLIEST</value> + <description>One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST</description> + <display-name>Input Topic Start</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>EARLIEST</value> + </entry> + <entry> + <value>LATEST</value> + </entry> + <entry> + <value>UNCOMMITTED_EARLIEST</value> + </entry> + <entry> + <value>UNCOMMITTED_LATEST</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + <property> + <name>profiler_period_duration</name> + <value>15</value> + <description>The duration of each profile period. This value should be defined along with profiler.period.duration.units</description> + <display-name>Period Duration</display-name> + </property> + <property> + <name>profiler_period_units</name> + <value>MINUTES</value> + <description>The units used to specify the profiler.period.duration. This value should be defined along with profiler.period.duration.</description> + <display-name>Period Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>SECONDS</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + + </property> + <property> + <name>profiler_ttl</name> + <value>30</value> + <description>If a message has not been applied to a Profile in this period of time, the Profile will be terminated and its resources will be cleaned up. This value should be defined along with profiler.ttl.units. + This time-to-live does not affect the persisted Profile data in HBase. It only affects the state stored in memory during the execution of the latest profile period. This state will be deleted if the time-to-live is exceeded. + </description> + <display-name>Time to Live</display-name> + </property> + <property> + <name>profiler_ttl_units</name> + <value>MINUTES</value> + <description>The units used to specify the profiler.ttl.</description> + <display-name>Time To Live Units</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>DAYS</value> + </entry> + <entry> + <value>HOURS</value> + </entry> + <entry> + <value>MINUTES</value> + </entry> + <entry> + <value>SECONDS</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + + + </property> + <property> + <name>profiler_hbase_table</name> + <value>profiler</value> + <description>The name of the HBase table that profile data is written to. The Profiler expects that the table exists and is writable.</description> + <display-name>HBase Table</display-name> + </property> + <property> + <name>profiler_hbase_cf</name> + <value>P</value> + <description>The column family used to store profile data in HBase.</description> + <display-name>HBase Table Column Family</display-name> + </property> + <property> + <name>profiler_hbase_batch</name> + <value>10</value> + <description>The number of puts that are written to HBase in a single batch.</description> + <display-name>HBase Batch Size</display-name> + </property> + <property> + <name>profiler_hbase_flush_interval</name> + <value>30</value> + <description>The maximum number of seconds between batch writes to HBase.</description> + <display-name>HBase Flush Interval</display-name> + </property> + <property> + <name>profiler_topology_worker_childopts</name> + <value/> + <description>Extra topology child opts for the storm topology.</description> + <display-name>topology.worker.childopts</display-name> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + </property> + <property> + <name>profiler_topology_workers</name> + <value>1</value> + <description>The profiler storm topology workers</description> + <display-name>Number of Workers</display-name> + </property> + <property> + <name>profiler_acker_executors</name> + <value>1</value> + <description>The profiler storm topology acker executors</description> + <display-name>Number of Acker Executors</display-name> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index b2dda25..4213214 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -119,6 +119,46 @@ </component> <component> + <name>METRON_PROFILER</name> + <displayName>Metron Profiler</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <versionAdvertised>false</versionAdvertised> + <dependencies> + <dependency> + <name>HBASE/HBASE_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>ZOOKEEPER/ZOOKEEPER_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>KAFKA/KAFKA_BROKER</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/profiler_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <configuration-dependencies> + <config-type>metron-enrichment-env</config-type> + <config-type>metron-profiler-env</config-type> + </configuration-dependencies> + </component> + + <component> <name>METRON_INDEXING</name> <displayName>Metron Indexing</displayName> <category>MASTER</category> @@ -321,6 +361,9 @@ <name>metron-enrichment</name> </package> <package> + <name>metron-profiler</name> + </package> + <package> <name>metron-indexing</name> </package> <package> http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index b524940..7855d6c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -172,6 +172,7 @@ enrichment_cf = status_params.enrichment_cf update_table = status_params.update_table update_cf = status_params.update_cf + threatintel_table = status_params.threatintel_table threatintel_cf = status_params.threatintel_cf @@ -202,6 +203,7 @@ security_enabled = status_params.security_enabled client_jaas_path = metron_home + '/client_jaas.conf' client_jaas_arg = '-Djava.security.auth.login.config=' + metron_home + '/client_jaas.conf' enrichment_topology_worker_childopts = client_jaas_arg if security_enabled else '' +profiler_topology_worker_childopts = client_jaas_arg if security_enabled else '' indexing_topology_worker_childopts = client_jaas_arg if security_enabled else '' metron_jvm_flags += (' ' + client_jaas_arg) if security_enabled else '' topology_auto_credentials = config['configurations']['storm-site'].get('nimbus.credential.renewers.classes', []) @@ -252,6 +254,29 @@ threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-e threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism'] kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism'] +# Profiler + +metron_profiler_topology = 'profiler' +profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] +profiler_kafka_start = config['configurations']['metron-profiler-env']['profiler_kafka_start'] +profiler_period_duration = config['configurations']['metron-profiler-env']['profiler_period_duration'] +profiler_period_units = config['configurations']['metron-profiler-env']['profiler_period_units'] +profiler_ttl = config['configurations']['metron-profiler-env']['profiler_ttl'] +profiler_ttl_units = config['configurations']['metron-profiler-env']['profiler_ttl_units'] +profiler_hbase_batch = config['configurations']['metron-profiler-env']['profiler_hbase_batch'] +profiler_hbase_flush_interval = config['configurations']['metron-profiler-env']['profiler_hbase_flush_interval'] +profiler_topology_workers = config['configurations']['metron-profiler-env']['profiler_topology_workers'] +profiler_acker_executors = config['configurations']['metron-profiler-env']['profiler_acker_executors'] +profiler_hbase_table = config['configurations']['metron-profiler-env']['profiler_hbase_table'] +profiler_hbase_cf = config['configurations']['metron-profiler-env']['profiler_hbase_cf'] +profiler_configured_flag_file = status_params.profiler_configured_flag_file +profiler_acl_configured_flag_file = status_params.indexing_acl_configured_flag_file +profiler_hbase_configured_flag_file = status_params.profiler_hbase_configured_flag_file +profiler_hbase_acl_configured_flag_file = status_params.profiler_hbase_acl_configured_flag_file +if not len(profiler_topology_worker_childopts) == 0: + profiler_topology_worker_childopts += ' ' +profiler_topology_worker_childopts += config['configurations']['metron-profiler-env']['profiler_topology_worker_childopts'] + # Indexing indexing_kafka_start = config['configurations']['metron-indexing-env']['indexing_kafka_start'] indexing_input_topic = status_params.indexing_input_topic http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index dd2ba9b..1935f2f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -52,6 +52,17 @@ threatintel_cf = 't' update_table = 'metron_update' update_cf = 't' +# Profiler +metron_profiler_topology = 'profiler' +profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] +profiler_hbase_table = config['configurations']['metron-profiler-env']['profiler_hbase_table'] +profiler_hbase_cf = config['configurations']['metron-profiler-env']['profiler_hbase_cf'] +profiler_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_configured' +profiler_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_acl_configured' +profiler_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_hbase_configured' +profiler_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_profiler_hbase_acl_configured' + + # Indexing metron_indexing_topology = 'indexing' indexing_input_topic = config['configurations']['metron-indexing-env']['indexing_input_topic'] http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py new file mode 100644 index 0000000..ddd66cb --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute, File + +import metron_service +import metron_security + + +# Wrap major operations and functionality in this class +class ProfilerCommands: + __params = None + __profiler_topic = None + __profiler_topology = None + __configured = False + __acl_configured = False + __hbase_configured = False + __hbase_acl_configured = False + + def __init__(self, params): + if params is None: + raise ValueError("params argument is required for initialization") + self.__params = params + self.__profiler_topology = params.metron_profiler_topology + self.__profiler_topic = params.profiler_input_topic + self.__configured = os.path.isfile(self.__params.profiler_configured_flag_file) + self.__acl_configured = os.path.isfile(self.__params.profiler_acl_configured_flag_file) + self.__hbase_configured = os.path.isfile(self.__params.profiler_hbase_configured_flag_file) + self.__hbase_acl_configured = os.path.isfile(self.__params.profiler_hbase_acl_configured_flag_file) + + def is_configured(self): + return self.__configured + + def is_acl_configured(self): + return self.__acl_configured + + def set_configured(self): + File(self.__params.profiler_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def is_hbase_configured(self): + return self.__hbase_configured + + def is_hbase_acl_configured(self): + return self.__hbase_acl_configured + + def set_hbase_configured(self): + Logger.info("Setting HBase Configured to True") + File(self.__params.profiler_hbase_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def set_hbase_acl_configured(self): + Logger.info("Setting HBase ACL Configured to True") + File(self.__params.profiler_hbase_acl_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def create_hbase_tables(self): + Logger.info("Creating HBase Tables") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"create '{0}','{1}'\" | hbase shell -n" + add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf) + Execute(add_table_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done creating HBase Tables") + self.set_hbase_configured() + + def set_hbase_acls(self): + Logger.info("Setting HBase ACLs") + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.hbase_keytab_path, + self.__params.hbase_principal_name, + execute_user=self.__params.hbase_user) + cmd = "echo \"grant '{0}', 'RW', '{1}'\" | hbase shell -n" + add_table_acl_cmd = cmd.format(self.__params.metron_user, self.__params.profiler_hbase_table) + Execute(add_table_acl_cmd, + tries=3, + try_sleep=5, + logoutput=False, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user=self.__params.hbase_user + ) + + Logger.info("Done setting HBase ACLs") + self.set_hbase_acl_configured() + + def set_acl_configured(self): + File(self.__params.profiler_acl_configured_flag_file, + content="", + owner=self.__params.metron_user, + mode=0755) + + def start_profiler_topology(self, env): + Logger.info('Starting ' + self.__profiler_topology) + + if not self.is_topology_active(env): + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.metron_keytab_path, + self.__params.metron_principal_name, + execute_user=self.__params.metron_user) + start_cmd_template = """{0}/bin/start_profiler_topology.sh \ + -s {1} \ + -z {2}""" + Execute(start_cmd_template.format(self.__params.metron_home, + self.__profiler_topology, + self.__params.zookeeper_quorum), + user=self.__params.metron_user) + + else: + Logger.info('Profiler topology already running') + + Logger.info('Finished starting profiler topology') + + def stop_profiler_topology(self, env): + Logger.info('Stopping ' + self.__profiler_topology) + + if self.is_topology_active(env): + if self.__params.security_enabled: + metron_security.kinit(self.__params.kinit_path_local, + self.__params.metron_keytab_path, + self.__params.metron_principal_name, + execute_user=self.__params.metron_user) + stop_cmd = 'storm kill ' + self.__profiler_topology + Execute(stop_cmd, user=self.__params.metron_user) + + else: + Logger.info("Profiler topology already stopped") + + Logger.info('Done stopping profiler topologies') + + def restart_profiler_topology(self, env): + Logger.info('Restarting the profiler topologies') + self.stop_profiler_topology(env) + + # Wait for old topology to be cleaned up by Storm, before starting again. + retries = 0 + topology_active = self.is_topology_active(env) + while self.is_topology_active(env) and retries < 3: + Logger.info('Existing topology still active. Will wait and retry') + time.sleep(10) + retries += 1 + + if not topology_active: + Logger.info('Waiting for storm kill to complete') + time.sleep(30) + self.start_profiler_topology(env) + Logger.info('Done restarting the profiler topologies') + else: + Logger.warning('Retries exhausted. Existing topology not cleaned up. Aborting topology start.') + + def is_topology_active(self, env): + env.set_params(self.__params) + active = True + topologies = metron_service.get_running_topologies(self.__params) + is_running = False + if self.__profiler_topology in topologies: + is_running = topologies[self.__profiler_topology] in ['ACTIVE', 'REBALANCING'] + active &= is_running + return active http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py new file mode 100644 index 0000000..4946ab0 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_master.py @@ -0,0 +1,94 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.core.source import Template +from resource_management.libraries.functions.format import format +from resource_management.core.source import StaticFile +from resource_management.libraries.functions import format as ambari_format +from resource_management.libraries.script import Script + +from metron_security import storm_security_setup +import metron_service +from profiler_commands import ProfilerCommands + + +class Profiler(Script): + __configured = False + + def install(self, env): + from params import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + from params import params + env.set_params(params) + + Logger.info("Running profiler configure") + File(format("{metron_config_path}/profiler.properties"), + content=Template("profiler.properties.j2"), + owner=params.metron_user, + group=params.metron_group + ) + + commands = ProfilerCommands(params) + metron_service.load_global_config(params) + + if not commands.is_configured(): + commands.set_configured() + + if not commands.is_hbase_configured(): + commands.create_hbase_tables() + if params.security_enabled and not commands.is_hbase_acl_configured(): + commands.set_hbase_acls() + + Logger.info("Calling security setup") + storm_security_setup(params) + + def start(self, env, upgrade_type=None): + from params import params + env.set_params(params) + self.configure(env) + commands = ProfilerCommands(params) + commands.start_profiler_topology(env) + + def stop(self, env, upgrade_type=None): + from params import params + env.set_params(params) + commands = ProfilerCommands(params) + commands.stop_profiler_topology(env) + + def status(self, env): + from params import status_params + env.set_params(status_params) + commands = ProfilerCommands(status_params) + if not commands.is_topology_active(env): + raise ComponentIsNotRunning() + + def restart(self, env): + from params import params + env.set_params(params) + self.configure(env) + commands = ProfilerCommands(params) + commands.restart_profiler_topology(env) + +if __name__ == "__main__": + Profiler().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 new file mode 100644 index 0000000..cf2ad58 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/profiler.properties.j2 @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +##### Storm ##### + +topology.worker.childopts={{profiler_topology_worker_childopts}} + +##### Profiler ##### + +profiler.input.topic={{enrichment_output_topic}} +profiler.output.topic={{enrichment_input_topic}} +profiler.period.duration={{profiler_period_duration}} +profiler.period.duration.units={{profiler_period_units}} +profiler.workers={{profiler_topology_workers}} +profiler.executors={{profiler_acker_executors}} +profiler.ttl={{profiler_ttl}} +profiler.ttl.units={{profiler_ttl_units}} +profiler.hbase.salt.divisor=1000 +profiler.hbase.table={{profiler_hbase_table}} +profiler.hbase.column.family={{profiler_hbase_cf}} +profiler.hbase.batch={{profiler_hbase_batch}} +profiler.hbase.flush.interval.seconds={{profiler_hbase_flush_interval}} + +##### Kafka ##### + +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{profiler_kafka_start}} +kafka.security.protocol={{kafka_security_protocol}} http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py index 4a95e63..2fb1ab0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py @@ -43,6 +43,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0] metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0] + metronProfilerMaster = self.getHosts(componentsList, "METRON_PROFILER")[0] metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0] metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0] @@ -76,6 +77,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost) items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost }) + if metronParsersHost != metronProfilerHost: + message = "Metron Profiler must be co-located with Metron Parsers on {0}".format(metronParsersHost) + items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_PROFILER', "host": metronProfilerHost }) + # Enrichment Master also needs ZK Client, but this is already guaranteed by being colocated with Parsers Master if metronParsersHost not in zookeeperClientHosts: message = "Metron must be co-located with an instance of Zookeeper Client" http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index ce352a4..748feb8 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -65,7 +65,7 @@ "display-name": "Enrichment", "layout": { "tab-columns": "1", - "tab-rows": "4", + "tab-rows": "3", "sections": [ { "name": "section-enrichment-adapters", @@ -132,7 +132,7 @@ "display-name": "Indexing", "layout": { "tab-columns": "1", - "tab-rows": "3", + "tab-rows": "4", "sections": [ { "name": "section-indexing-kafka", @@ -155,7 +155,7 @@ }, { "name": "section-indexing-update", - "row-index": "0", + "row-index": "1", "column-index": "0", "row-span": "1", "column-span": "1", @@ -174,7 +174,7 @@ }, { "name": "section-indexing-storm", - "row-index": "1", + "row-index": "2", "column-index": "0", "row-span": "1", "column-span": "1", @@ -193,7 +193,7 @@ }, { "name": "section-indexing-hdfs", - "row-index": "2", + "row-index": "3", "column-index": "0", "row-span": "1", "column-span": "1", @@ -214,6 +214,73 @@ } }, { + "name": "profiler", + "display-name": "Profiler", + "layout": { + "tab-columns": "1", + "tab-rows": "3", + "sections": [ + { + "name": "section-profiler-kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-kafka", + "display-name": "Kafka", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-profiler-setup", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-setup", + "display-name": "Profiler Setup", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-profiler-storm", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-profiler-storm", + "display-name": "Storm", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + }, + { "name": "rest", "display-name": "REST", "layout": { @@ -454,6 +521,54 @@ }, { + "config": "metron-profiler-env/profiler_kafka_start", + "subsection-name": "subsection-profiler-kafka" + }, + { + "config": "metron-profiler-env/profiler_period_duration", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_period_units", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_ttl", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_ttl_units", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_table", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_cf", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_batch", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_hbase_flush_interval", + "subsection-name": "subsection-profiler-setup" + }, + { + "config": "metron-profiler-env/profiler_topology_worker_childopts", + "subsection-name": "subsection-profiler-storm" + }, + { + "config": "metron-profiler-env/profiler_topology_workers", + "subsection-name": "subsection-profiler-storm" + }, + { + "config": "metron-profiler-env/profiler_acker_executors", + "subsection-name": "subsection-profiler-storm" + }, + { "config": "metron-rest-env/metron_rest_port", "subsection-name": "subsection-rest" }, @@ -780,7 +895,78 @@ "type": "text-field" } }, - + { + "config": "metron-profiler-env/profiler_kafka_start", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_period_duration", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_period_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_ttl", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_ttl_units", + "widget": { + "type": "combo" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_table", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_cf", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_batch", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_hbase_flush_interval", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_topology_worker_childopts", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_topology_workers", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-profiler-env/profiler_acker_executors", + "widget": { + "type": "text-field" + } + }, { "config": "metron-rest-env/metron_rest_port", http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index 482fafd..4c3cbce 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -31,6 +31,7 @@ hbase_slave: [HBASE_REGIONSERVER] es_master: [ES_MASTER] kibana_master: [KIBANA_MASTER] metron_indexing: [METRON_INDEXING] +metron_profiler: [METRON_PROFILER] metron_enrichment_master : [METRON_ENRICHMENT_MASTER] metron_parsers : [METRON_PARSERS] metron_rest: [METRON_REST] @@ -38,8 +39,8 @@ metron_management_ui: [METRON_MANAGEMENT_UI] metron_components: > {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | - union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | - union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(es_master) }} + union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) | + union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(es_master) }} cluster_name: "metron_cluster" blueprint_name: "metron_blueprint" @@ -82,7 +83,7 @@ configurations: yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}' yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' - storm-site: - supervisor.slots.ports: "[6700, 6701, 6702, 6703]" + supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]" storm.local.dir: '{{ storm_local_dir }}' topology.classpath: '{{ topology_classpath }}' - kafka-env: http://git-wip-us.apache.org/repos/asf/metron/blob/8560d2df/metron-deployment/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index af22cb6..45b15f3 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -31,6 +31,7 @@ es_master: [ES_MASTER] es_slave: [ES_SLAVE] kibana_master: [KIBANA_MASTER] metron_indexing: [METRON_INDEXING] +metron_profiler: [METRON_PROFILER] metron_enrichment_master : [METRON_ENRICHMENT_MASTER] metron_parsers : [METRON_PARSERS] metron_rest: [METRON_REST] @@ -43,7 +44,7 @@ master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_ma master_2_host: - "{{groups.ambari_slave[1]}}" metron_components: > - {{ metron_indexing | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) | + {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(hadoop_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }} metron_host: - "{{ groups.metron[0] }}" @@ -80,7 +81,7 @@ configurations: yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}' yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}' - storm-site: - supervisor.slots.ports: "[6700, 6701, 6702, 6703]" + supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704]" storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}' topology.classpath: '{{ topology_classpath }}' - kafka-broker:
