METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44 Parents: 3fcbf8b Author: nickwallen <n...@nickallen.org> Authored: Tue Apr 17 09:43:16 2018 -0400 Committer: nickallen <nickal...@apache.org> Committed: Tue Apr 17 09:43:16 2018 -0400 ---------------------------------------------------------------------- .../packaging/ambari/metron-mpack/pom.xml | 7 +- .../configuration/metron-enrichment-env.xml | 130 +++- .../package/scripts/enrichment_commands.py | 20 +- .../package/scripts/enrichment_master.py | 12 +- .../package/scripts/params/params_linux.py | 19 +- .../enrichment-splitjoin.properties.j2 | 63 ++ .../templates/enrichment-unified.properties.j2 | 60 ++ .../METRON/CURRENT/themes/metron_theme.json | 151 ++++- .../docker/rpm-docker/SPECS/metron.spec | 5 +- .../main/config/enrichment-splitjoin.properties | 63 ++ .../config/enrichment-splitjoin.properties.j2 | 63 ++ .../main/config/enrichment-unified.properties | 69 +++ .../config/enrichment-unified.properties.j2 | 60 ++ .../src/main/config/enrichment.properties | 64 -- .../src/main/config/enrichment.properties.j2 | 63 -- .../main/flux/enrichment/remote-splitjoin.yaml | 590 +++++++++++++++++++ .../main/flux/enrichment/remote-unified.yaml | 71 ++- .../src/main/flux/enrichment/remote.yaml | 590 ------------------- .../main/scripts/start_enrichment_topology.sh | 16 +- .../integration/EnrichmentIntegrationTest.java | 61 +- .../UnifiedEnrichmentIntegrationTest.java | 71 +++ 21 files changed, 1433 insertions(+), 815 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml index 491e8dd..1843eb7 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml @@ -110,7 +110,8 @@ <resource> <directory>${basedir}/../../../../metron-platform/metron-enrichment/src/main/config</directory> <includes> - <include>enrichment.properties.j2</include> + <include>enrichment-splitjoin.properties.j2</include> + <include>enrichment-unified.properties.j2</include> </includes> <filtering>false</filtering> </resource> @@ -171,8 +172,10 @@ <fileset> <directory>${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates</directory> <includes> - <include>enrichment.properties.j2</include> + <include>enrichment-unified.properties.j2</include> + <include>enrichment-splitjoin.properties.j2</include> <include>elasticsearch.properties.j2</include> + <include>hdfs.properties.j2</include> </includes> </fileset> </filesets> http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml index 9737660..81b135c 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml @@ -18,6 +18,10 @@ limitations under the License. --> <configuration supports_final="true"> + + <!-- + enrichment adapter parameters + --> <property> <name>geoip_url</name> <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value> @@ -33,6 +37,10 @@ <type>content</type> </value-attributes> </property> + + <!-- + kafka parameters + --> <property> <name>enrichment_kafka_start</name> <description>Enrichment Topology Spout Offset</description> @@ -81,6 +89,10 @@ <value>indexing</value> <display-name>Threat Intel Error Topic</display-name> </property> + + <!-- + hbase parameters + --> <property> <name>enrichment_hbase_table</name> <value>enrichment</value> @@ -105,6 +117,10 @@ <description>The HBase column family which will hold threatintel data in HBase.</description> <display-name>HBase Table Column Family</display-name> </property> + + <!-- + storm common parameters + --> <property> <name>enrichment_workers</name> <description>Number of Workers for the Enrichment Topology</description> @@ -129,70 +145,156 @@ <property> <name>enrichment_topology_max_spout_pending</name> <description>Spout Max Pending Tuples for the Enrichment Topology</description> - <value/> - <display-name>Enrichment Max Pending</display-name> + <value>500</value> + <display-name>Enrichment Max Spout Pending</display-name> <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> </property> <property> + <name>enrichment_topology</name> + <description>Which Enrichment topology to execute</description> + <value>Split-Join</value> + <display-name>Enrichment Topology</display-name> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>Split-Join</value> + </entry> + <entry> + <value>Unified</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> + + <!-- + split-join topology parameters + --> + <property> <name>enrichment_join_cache_size</name> - <description>Enrichment Join Bolt Cache Max Size</description> + <description>Enrichment join bolt max cache size for the Split Join Enrichment Topology</description> <value>100000</value> - <display-name>Enrichment Join Size</display-name> + <display-name>Enrichment Join Max Cache Size</display-name> </property> <property> <name>threatintel_join_cache_size</name> - <description>Threat Intel Join Bolt Cache Max Size</description> + <description>Threat Intel join bolt max cache size for the Split Join Enrichment Topology</description> <value>100000</value> - <display-name>Threat Intel Join Size</display-name> + <display-name>Threat Intel Join Max Cache Size</display-name> </property> <property> <name>enrichment_kafka_spout_parallelism</name> - <description>Kafka Spout Parallelism for the Enrichment Topology</description> + <description>Kafka spout parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Enrichment Spout Parallelism</display-name> </property> <property> <name>enrichment_split_parallelism</name> - <description>Enrichment Split Bolt Parallelism for the Enrichment Topology</description> + <description>Enrichment split bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Enrichment Split Parallelism</display-name> </property> <property> <name>enrichment_stellar_parallelism</name> - <description>Stellar Enrichment Bolt Parallelism for the Enrichment Topology</description> + <description>Stellar enrichment bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Stellar Enrichment Parallelism</display-name> </property> <property> <name>enrichment_join_parallelism</name> - <description>Enrichment Join Bolt Parallelism for the Enrichment Topology</description> + <description>Enrichment join bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Enrichment Join Parallelism</display-name> </property> <property> <name>threat_intel_split_parallelism</name> - <description>Threat Intel Split Bolt Parallelism for the Enrichment Topology</description> + <description>Threat Intel split bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Threat Intel Split Parallelism</display-name> </property> <property> <name>threat_intel_stellar_parallelism</name> - <description>Threat Intel Stellar Bolt Parallelism for the Enrichment Topology</description> + <description>Threat Intel stellar bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Threat Intel Stellar Parallelism</display-name> </property> <property> <name>threat_intel_join_parallelism</name> - <description>Threat Intel Join Bolt Parallelism for the Enrichment Topology</description> + <description>Threat Intel join bolt parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Threat Intel Join Parallelism</display-name> </property> <property> <name>kafka_writer_parallelism</name> - <description>Kafka Writer Parallelism for the Enrichment Topology</description> + <description>Kafka writer parallelism for the Split Join Enrichment Topology</description> <value>1</value> <display-name>Enrichment Kafka Writer Parallelism</display-name> </property> + + <!-- + unified topology parameters + --> + <property> + <name>unified_kafka_spout_parallelism</name> + <description>Kafka spout parallelism for the Unified Enrichment Topology</description> + <value>1</value> + <display-name>Unified Enrichment Spout Parallelism</display-name> + </property> + <property> + <name>unified_enrichment_parallelism</name> + <description>Enrichment parallelism for the Unified Enrichment Topology</description> + <value>1</value> + <display-name>Unified Enrichment Parallelism</display-name> + </property> + <property> + <name>unified_threat_intel_parallelism</name> + <description>Threat Intel parallelism for the Unified Enrichment Topology</description> + <value>1</value> + <display-name>Unified Threat Intel Parallelism</display-name> + </property> + <property> + <name>unified_kafka_writer_parallelism</name> + <description>Kafka writer parallelism for the Unified Enrichment Topology</description> + <value>1</value> + <display-name>Unified Kafka Writer Parallelism</display-name> + </property> + <property> + <name>unified_enrichment_cache_size</name> + <description>Enrichment max cache size for the Unified Enrichment Topology</description> + <value>100000</value> + <display-name>Unified Enrichment Cache Size</display-name> + </property> + <property> + <name>unified_threat_intel_cache_size</name> + <description>Threat Intel Max Cache Size for the Unified Enrichment Topology</description> + <value>100000</value> + <display-name>Unified Threat Intel Cache Size</display-name> + </property> + <property> + <name>unified_enrichment_threadpool_size</name> + <description>Enrichment thread pool size for the Unified Enrichment Topology</description> + <value>1</value> + <display-name>Unified Enrichment Thread Pool Size</display-name> + </property> + <property> + <name>unified_enrichment_threadpool_type</name> + <description>Enrichment thread pool type for the Unified Enrichment Topology</description> + <display-name>Unified Enrichment Thread Pool Type</display-name> + <value>FIXED</value> + <value-attributes> + <type>value-list</type> + <entries> + <entry> + <value>FIXED</value> + </entry> + <entry> + <value>WORK_STEALING</value> + </entry> + </entries> + <selection-cardinality>1</selection-cardinality> + </value-attributes> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index f9ec547..a1bdbed 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -129,13 +129,21 @@ class EnrichmentCommands: Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology)) if not self.is_topology_active(env): - start_cmd_template = """{0}/bin/start_enrichment_topology.sh \ - -s {1} \ - -z {2}""" + + # which enrichment topology needs started? + if self.__params.enrichment_topology == "Unified": + topology_flux = "{0}/flux/enrichment/remote-unified.yaml".format(self.__params.metron_home) + topology_props = "{0}/config/enrichment-unified.properties".format(self.__params.metron_home) + elif self.__params.enrichment_topology == "Split-Join": + topology_flux = "{0}/flux/enrichment/remote-splitjoin.yaml".format(self.__params.metron_home) + topology_props = "{0}/config/enrichment-splitjoin.properties".format(self.__params.metron_home) + else: + raise Fail("Unexpected enrichment topology; name=" + self.__params.enrichment_topology) + + # start the topology + start_cmd_template = """{0}/bin/start_enrichment_topology.sh --remote {1} --filter {2}""" Logger.info('Starting ' + self.__enrichment_topology) - start_cmd = start_cmd_template.format(self.__params.metron_home, - self.__enrichment_topology, - self.__params.zookeeper_quorum) + start_cmd = start_cmd_template.format(self.__params.metron_home, topology_flux, topology_props) Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True) else: Logger.info('Enrichment topology already running') http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index 24feb81..cada1d2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -38,11 +38,15 @@ class Enrichment(Script): env.set_params(params) Logger.info("Running enrichment configure") - File(format("{metron_config_path}/enrichment.properties"), - content=Template("enrichment.properties.j2"), + File(format("{metron_config_path}/enrichment-splitjoin.properties"), + content=Template("enrichment-splitjoin.properties.j2"), owner=params.metron_user, - group=params.metron_group - ) + group=params.metron_group) + + File(format("{metron_config_path}/enrichment-unified.properties"), + content=Template("enrichment-unified.properties.j2"), + owner=params.metron_user, + group=params.metron_group) if not metron_service.is_zk_configured(params): metron_service.init_zk_config(params) http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index ccce022..f44d05f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -256,20 +256,27 @@ user_settings_hbase_table = status_params.user_settings_hbase_table user_settings_hbase_cf = status_params.user_settings_hbase_cf # Enrichment +metron_enrichment_topology = status_params.metron_enrichment_topology geoip_url = config['configurations']['metron-enrichment-env']['geoip_url'] enrichment_host_known_hosts = config['configurations']['metron-enrichment-env']['enrichment_host_known_hosts'] + +# Enrichment - Kafka enrichment_kafka_start = config['configurations']['metron-enrichment-env']['enrichment_kafka_start'] enrichment_input_topic = status_params.enrichment_input_topic enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic'] threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic'] -metron_enrichment_topology = status_params.metron_enrichment_topology + +# Enrichment - Storm common parameters enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers'] enrichment_acker_executors = config['configurations']['metron-enrichment-env']['enrichment_acker_executors'] if not len(enrichment_topology_worker_childopts) == 0: enrichment_topology_worker_childopts += ' ' enrichment_topology_worker_childopts += config['configurations']['metron-enrichment-env']['enrichment_topology_worker_childopts'] enrichment_topology_max_spout_pending = config['configurations']['metron-enrichment-env']['enrichment_topology_max_spout_pending'] +enrichment_topology = config['configurations']['metron-enrichment-env']['enrichment_topology'] + +# Enrichment - Split Join topology enrichment_join_cache_size = config['configurations']['metron-enrichment-env']['enrichment_join_cache_size'] threatintel_join_cache_size = config['configurations']['metron-enrichment-env']['threatintel_join_cache_size'] enrichment_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['enrichment_kafka_spout_parallelism'] @@ -281,6 +288,16 @@ threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-e threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism'] kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism'] +# Enrichment - Unified topology +unified_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_spout_parallelism'] +unified_enrichment_parallelism = config['configurations']['metron-enrichment-env']['unified_enrichment_parallelism'] +unified_threat_intel_parallelism = config['configurations']['metron-enrichment-env']['unified_threat_intel_parallelism'] +unified_kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_writer_parallelism'] +unified_enrichment_cache_size = config['configurations']['metron-enrichment-env']['unified_enrichment_cache_size'] +unified_threat_intel_cache_size = config['configurations']['metron-enrichment-env']['unified_threat_intel_cache_size'] +unified_enrichment_threadpool_size = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_size'] +unified_enrichment_threadpool_type = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_type'] + # Profiler metron_profiler_topology = 'profiler' profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic'] http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 new file mode 100644 index 0000000..a0b21c9 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{enrichment_kafka_start}} + +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} + +##### JoinBolt ##### +enrichment.join.cache.size={{enrichment_join_cache_size}} +threat.intel.join.cache.size={{threatintel_join_cache_size}} + +##### Enrichment ##### +hbase.provider.impl={{enrichment_hbase_provider_impl}} +enrichment.simple.hbase.table={{enrichment_hbase_table}} +enrichment.simple.hbase.cf={{enrichment_hbase_cf}} +enrichment.host.known_hosts={{enrichment_host_known_hosts}} + +##### Threat Intel ##### +threat.intel.tracker.table={{threatintel_hbase_table}} +threat.intel.tracker.cf={{threatintel_hbase_cf}} +threat.intel.simple.hbase.table={{threatintel_hbase_table}} +threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} + +##### Parallelism ##### +kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} +enrichment.split.parallelism={{enrichment_split_parallelism}} +enrichment.stellar.parallelism={{enrichment_stellar_parallelism}} +enrichment.join.parallelism={{enrichment_join_parallelism}} +threat.intel.split.parallelism={{threat_intel_split_parallelism}} +threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}} +threat.intel.join.parallelism={{threat_intel_join_parallelism}} +kafka.writer.parallelism={{kafka_writer_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 new file mode 100644 index 0000000..8c28c49 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} +kafka.start={{enrichment_kafka_start}} +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} + +##### Enrichment ##### +hbase.provider.impl={{enrichment_hbase_provider_impl}} +enrichment.simple.hbase.table={{enrichment_hbase_table}} +enrichment.simple.hbase.cf={{enrichment_hbase_cf}} +enrichment.host.known_hosts={{enrichment_host_known_hosts}} + +##### Threat Intel ##### +threat.intel.tracker.table={{threatintel_hbase_table}} +threat.intel.tracker.cf={{threatintel_hbase_cf}} +threat.intel.simple.hbase.table={{threatintel_hbase_table}} +threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} + +##### Parallelism ##### +kafka.spout.parallelism={{unified_kafka_spout_parallelism}} +enrichment.parallelism={{unified_enrichment_parallelism}} +threat.intel.parallelism={{unified_threat_intel_parallelism}} +kafka.writer.parallelism={{unified_kafka_writer_parallelism}} + +##### Caches ##### +enrichment.cache.size={{unified_enrichment_cache_size}} +threat.intel.cache.size={{unified_threat_intel_cache_size}} + +##### Threads ##### +enrichment.threadpool.size={{unified_enrichment_threadpool_size}} +enrichment.threadpool.type={{unified_enrichment_threadpool_type}} http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 364b3ef..06bc155 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -65,7 +65,7 @@ "display-name": "Enrichment", "layout": { "tab-columns": "1", - "tab-rows": "3", + "tab-rows": "5", "sections": [ { "name": "section-enrichment-adapters", @@ -123,6 +123,44 @@ "column-span": "1" } ] + }, + { + "name": "section-enrichment-splitjoin", + "row-index": "3", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-splitjoin", + "display-name": "Split Join Topology", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-enrichment-unified", + "row-index": "4", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-enrichment-unified", + "display-name": "Unified Topology", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] } ] } @@ -462,44 +500,80 @@ "subsection-name": "subsection-enrichment-storm" }, { - "config": "metron-enrichment-env/enrichment_join_cache_size", + "config": "metron-enrichment-env/enrichment_topology", "subsection-name": "subsection-enrichment-storm" }, { + "config": "metron-enrichment-env/enrichment_join_cache_size", + "subsection-name": "subsection-enrichment-splitjoin" + }, + { "config": "metron-enrichment-env/threatintel_join_cache_size", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/enrichment_kafka_spout_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/enrichment_split_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/enrichment_stellar_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/enrichment_join_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/threat_intel_split_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/threat_intel_stellar_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/threat_intel_join_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" }, { "config": "metron-enrichment-env/kafka_writer_parallelism", - "subsection-name": "subsection-enrichment-storm" + "subsection-name": "subsection-enrichment-splitjoin" + }, + { + "config": "metron-enrichment-env/unified_kafka_spout_parallelism", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_enrichment_parallelism", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_threat_intel_parallelism", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_kafka_writer_parallelism", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_enrichment_cache_size", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_threat_intel_cache_size", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_enrichment_threadpool_size", + "subsection-name": "subsection-enrichment-unified" + }, + { + "config": "metron-enrichment-env/unified_enrichment_threadpool_type", + "subsection-name": "subsection-enrichment-unified" }, { "config": "metron-indexing-env/ra_indexing_kafka_start", @@ -838,6 +912,12 @@ } }, { + "config": "metron-enrichment-env/enrichment_topology", + "widget": { + "type": "combo" + } + }, + { "config": "metron-enrichment-env/enrichment_join_cache_size", "widget": { "type": "text-field" @@ -897,7 +977,54 @@ "type": "text-field" } }, - + { + "config": "metron-enrichment-env/unified_kafka_spout_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_enrichment_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_threat_intel_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_kafka_writer_parallelism", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_enrichment_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_threat_intel_cache_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_enrichment_threadpool_size", + "widget": { + "type": "text-field" + } + }, + { + "config": "metron-enrichment-env/unified_enrichment_threadpool_type", + "widget": { + "type": "combo" + } + }, { "config": "metron-indexing-env/batch_indexing_kafka_start", "widget": { http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec index 6b35dae..1f40105 100644 --- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec +++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec @@ -273,13 +273,14 @@ This package installs the Metron Enrichment files %dir %{metron_home}/flux/enrichment %{metron_home}/bin/latency_summarizer.sh %{metron_home}/bin/start_enrichment_topology.sh -%{metron_home}/config/enrichment.properties +%{metron_home}/config/enrichment-splitjoin.properties +%{metron_home}/config/enrichment-unified.properties %{metron_home}/config/zookeeper/enrichments/bro.json %{metron_home}/config/zookeeper/enrichments/snort.json %{metron_home}/config/zookeeper/enrichments/websphere.json %{metron_home}/config/zookeeper/enrichments/yaf.json %{metron_home}/config/zookeeper/enrichments/asa.json -%{metron_home}/flux/enrichment/remote.yaml +%{metron_home}/flux/enrichment/remote-splitjoin.yaml %{metron_home}/flux/enrichment/remote-unified.yaml %attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties new file mode 100644 index 0000000..109c2ee --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### Storm ##### +enrichment.workers=1 +enrichment.acker.executors=0 +topology.worker.childopts= +topology.auto-credentials= +topology.max.spout.pending=500 + +##### Kafka ##### +kafka.zk=node1:2181 +kafka.broker=node1:6667 +kafka.security.protocol=PLAINTEXT + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST + +enrichment.input.topic=enrichments +enrichment.output.topic=indexing +enrichment.error.topic=indexing +threat.intel.error.topic=indexing + +##### JoinBolt ##### +enrichment.join.cache.size=100000 +threat.intel.join.cache.size=100000 + +##### Enrichment ##### +hbase.provider.impl=org.apache.metron.hbase.HTableProvider +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t +enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ +{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + +##### Threat Intel ##### +threat.intel.tracker.table=access_tracker +threat.intel.tracker.cf=t +threat.intel.simple.hbase.table=threatintel +threat.intel.simple.hbase.cf=t + +##### Parallelism ##### +kafka.spout.parallelism=1 +enrichment.split.parallelism=1 +enrichment.stellar.parallelism=1 +enrichment.join.parallelism=1 +threat.intel.split.parallelism=1 +threat.intel.stellar.parallelism=1 +threat.intel.join.parallelism=1 +kafka.writer.parallelism=1 http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 new file mode 100755 index 0000000..a0b21c9 --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start={{enrichment_kafka_start}} + +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} + +##### JoinBolt ##### +enrichment.join.cache.size={{enrichment_join_cache_size}} +threat.intel.join.cache.size={{threatintel_join_cache_size}} + +##### Enrichment ##### +hbase.provider.impl={{enrichment_hbase_provider_impl}} +enrichment.simple.hbase.table={{enrichment_hbase_table}} +enrichment.simple.hbase.cf={{enrichment_hbase_cf}} +enrichment.host.known_hosts={{enrichment_host_known_hosts}} + +##### Threat Intel ##### +threat.intel.tracker.table={{threatintel_hbase_table}} +threat.intel.tracker.cf={{threatintel_hbase_cf}} +threat.intel.simple.hbase.table={{threatintel_hbase_table}} +threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} + +##### Parallelism ##### +kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} +enrichment.split.parallelism={{enrichment_split_parallelism}} +enrichment.stellar.parallelism={{enrichment_stellar_parallelism}} +enrichment.join.parallelism={{enrichment_join_parallelism}} +threat.intel.split.parallelism={{threat_intel_split_parallelism}} +threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}} +threat.intel.join.parallelism={{threat_intel_join_parallelism}} +kafka.writer.parallelism={{kafka_writer_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties new file mode 100644 index 0000000..5338ead --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +enrichment.workers=1 +enrichment.acker.executors=0 +topology.worker.childopts= +topology.auto-credentials= +topology.max.spout.pending=500 + +##### Kafka ##### +kafka.zk=node1:2181 +kafka.broker=node1:6667 +kafka.security.protocol=PLAINTEXT + +# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST +kafka.start=UNCOMMITTED_EARLIEST + +enrichment.input.topic=enrichments +enrichment.output.topic=indexing +enrichment.error.topic=indexing +threat.intel.error.topic=indexing + +##### JoinBolt ##### +enrichment.join.cache.size=100000 +threat.intel.join.cache.size=100000 + +##### Enrichment ##### +hbase.provider.impl=org.apache.metron.hbase.HTableProvider +enrichment.simple.hbase.table=enrichment +enrichment.simple.hbase.cf=t +enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ +{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ +{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] + +##### Threat Intel ##### +threat.intel.tracker.table=access_tracker +threat.intel.tracker.cf=t +threat.intel.simple.hbase.table=threatintel +threat.intel.simple.hbase.cf=t + +##### Parallelism ##### +kafka.spout.parallelism=1 +enrichment.parallelism=1 +threat.intel.parallelism=1 +kafka.writer.parallelism=1 + +##### Caches ##### +enrichment.cache.size=100000 +threat.intel.cache.size=100000 + +##### Threads ##### +enrichment.threadpool.size=1 +enrichment.threadpool.type=FIXED http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 new file mode 100644 index 0000000..8c28c49 --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +##### Storm ##### +enrichment.workers={{enrichment_workers}} +enrichment.acker.executors={{enrichment_acker_executors}} +topology.worker.childopts={{enrichment_topology_worker_childopts}} +topology.auto-credentials={{topology_auto_credentials}} +topology.max.spout.pending={{enrichment_topology_max_spout_pending}} + +##### Kafka ##### +kafka.zk={{zookeeper_quorum}} +kafka.broker={{kafka_brokers}} +kafka.security.protocol={{kafka_security_protocol}} +kafka.start={{enrichment_kafka_start}} +enrichment.input.topic={{enrichment_input_topic}} +enrichment.output.topic={{enrichment_output_topic}} +enrichment.error.topic={{enrichment_error_topic}} +threat.intel.error.topic={{threatintel_error_topic}} + +##### Enrichment ##### +hbase.provider.impl={{enrichment_hbase_provider_impl}} +enrichment.simple.hbase.table={{enrichment_hbase_table}} +enrichment.simple.hbase.cf={{enrichment_hbase_cf}} +enrichment.host.known_hosts={{enrichment_host_known_hosts}} + +##### Threat Intel ##### +threat.intel.tracker.table={{threatintel_hbase_table}} +threat.intel.tracker.cf={{threatintel_hbase_cf}} +threat.intel.simple.hbase.table={{threatintel_hbase_table}} +threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} + +##### Parallelism ##### +kafka.spout.parallelism={{unified_kafka_spout_parallelism}} +enrichment.parallelism={{unified_enrichment_parallelism}} +threat.intel.parallelism={{unified_threat_intel_parallelism}} +kafka.writer.parallelism={{unified_kafka_writer_parallelism}} + +##### Caches ##### +enrichment.cache.size={{unified_enrichment_cache_size}} +threat.intel.cache.size={{unified_threat_intel_cache_size}} + +##### Threads ##### +enrichment.threadpool.size={{unified_enrichment_threadpool_size}} +enrichment.threadpool.type={{unified_enrichment_threadpool_type}} http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties deleted file mode 100644 index 9592968..0000000 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##### Storm ##### -enrichment.workers=1 -enrichment.acker.executors=0 -topology.worker.childopts= -topology.auto-credentials= -topology.max.spout.pending= - -##### Kafka ##### -kafka.zk=node1:2181 -kafka.broker=node1:6667 -kafka.security.protocol=PLAINTEXT - -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start=UNCOMMITTED_EARLIEST - -enrichment.input.topic=enrichments -enrichment.output.topic=indexing -enrichment.error.topic=indexing -threat.intel.error.topic=indexing - -##### JoinBolt ##### -enrichment.join.cache.size=100000 -threat.intel.join.cache.size=100000 - -##### Enrichment ##### -hbase.provider.impl=org.apache.metron.hbase.HTableProvider -enrichment.simple.hbase.table=enrichment -enrichment.simple.hbase.cf=t -enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\ -{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\ -{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}] - - -##### Threat Intel ##### -threat.intel.tracker.table=access_tracker -threat.intel.tracker.cf=t -threat.intel.simple.hbase.table=threatintel -threat.intel.simple.hbase.cf=t - -##### Parallelism ##### -kafka.spout.parallelism=1 -enrichment.split.parallelism=1 -enrichment.stellar.parallelism=1 -enrichment.join.parallelism=1 -threat.intel.split.parallelism=1 -threat.intel.stellar.parallelism=1 -threat.intel.join.parallelism=1 -kafka.writer.parallelism=1 http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 deleted file mode 100755 index 133f9c5..0000000 --- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 +++ /dev/null @@ -1,63 +0,0 @@ -{# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#} - -##### Storm ##### -enrichment.workers={{enrichment_workers}} -enrichment.acker.executors={{enrichment_acker_executors}} -topology.worker.childopts={{enrichment_topology_worker_childopts}} -topology.auto-credentials={{topology_auto_credentials}} -topology.max.spout.pending={{enrichment_topology_max_spout_pending}} - -##### Kafka ##### -kafka.zk={{zookeeper_quorum}} -kafka.broker={{kafka_brokers}} -kafka.security.protocol={{kafka_security_protocol}} - -# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST -kafka.start={{enrichment_kafka_start}} - -enrichment.input.topic={{enrichment_input_topic}} -enrichment.output.topic={{enrichment_output_topic}} -enrichment.error.topic={{enrichment_error_topic}} -threat.intel.error.topic={{threatintel_error_topic}} - -##### JoinBolt ##### -enrichment.join.cache.size={{enrichment_join_cache_size}} -threat.intel.join.cache.size={{threatintel_join_cache_size}} - -##### Enrichment ##### -hbase.provider.impl={{enrichment_hbase_provider_impl}} -enrichment.simple.hbase.table={{enrichment_hbase_table}} -enrichment.simple.hbase.cf={{enrichment_hbase_cf}} -enrichment.host.known_hosts={{enrichment_host_known_hosts}} - -##### Threat Intel ##### -threat.intel.tracker.table={{threatintel_hbase_table}} -threat.intel.tracker.cf={{threatintel_hbase_cf}} -threat.intel.simple.hbase.table={{threatintel_hbase_table}} -threat.intel.simple.hbase.cf={{threatintel_hbase_cf}} - -##### Parallelism ##### -kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}} -enrichment.split.parallelism={{enrichment_split_parallelism}} -enrichment.stellar.parallelism={{enrichment_stellar_parallelism}} -enrichment.join.parallelism={{enrichment_join_parallelism}} -threat.intel.split.parallelism={{threat_intel_split_parallelism}} -threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}} -threat.intel.join.parallelism={{threat_intel_join_parallelism}} -kafka.writer.parallelism={{kafka_writer_parallelism}} http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml new file mode 100644 index 0000000..fd7ceff --- /dev/null +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml @@ -0,0 +1,590 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "enrichment" +config: + topology.workers: ${enrichment.workers} + topology.acker.executors: ${enrichment.acker.executors} + topology.worker.childopts: ${topology.worker.childopts} + topology.auto-credentials: ${topology.auto-credentials} + topology.max.spout.pending: ${topology.max.spout.pending} + +components: + +# Enrichment + - id: "stellarEnrichmentAdapter" + className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" + configMethods: + - name: "ofType" + args: + - "ENRICHMENT" + + # Any kafka props for the producer go here. + - id: "kafkaWriterProps" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: + - "security.protocol" + - "${kafka.security.protocol}" + + - id: "stellarEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "stellar" + - ref: "stellarEnrichmentAdapter" + + - id: "geoEnrichmentAdapter" + className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter" + - id: "geoEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "geo" + - ref: "geoEnrichmentAdapter" + - id: "hostEnrichmentAdapter" + className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" + constructorArgs: + - '${enrichment.host.known_hosts}' + - id: "hostEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "host" + - ref: "hostEnrichmentAdapter" + + - id: "simpleHBaseEnrichmentConfig" + className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig" + configMethods: + - name: "withProviderImpl" + args: + - "${hbase.provider.impl}" + - name: "withHBaseTable" + args: + - "${enrichment.simple.hbase.table}" + - name: "withHBaseCF" + args: + - "${enrichment.simple.hbase.cf}" + - id: "simpleHBaseEnrichmentAdapter" + className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter" + configMethods: + - name: "withConfig" + args: + - ref: "simpleHBaseEnrichmentConfig" + - id: "simpleHBaseEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "hbaseEnrichment" + - ref: "simpleHBaseEnrichmentAdapter" + - id: "enrichments" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: + - ref: "geoEnrichment" + - name: "add" + args: + - ref: "hostEnrichment" + - name: "add" + args: + - ref: "simpleHBaseEnrichment" + - name: "add" + args: + - ref: "stellarEnrichment" + + #enrichment error + - id: "enrichmentErrorKafkaWriter" + className: "org.apache.metron.writer.kafka.KafkaWriter" + configMethods: + - name: "withTopic" + args: + - "${enrichment.error.topic}" + - name: "withZkQuorum" + args: + - "${kafka.zk}" + - name: "withProducerConfigs" + args: + - ref: "kafkaWriterProps" + +# Threat Intel + - id: "stellarThreatIntelAdapter" + className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" + configMethods: + - name: "ofType" + args: + - "THREAT_INTEL" + - id: "stellarThreatIntelEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "stellar" + - ref: "stellarThreatIntelAdapter" + - id: "simpleHBaseThreatIntelConfig" + className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig" + configMethods: + - name: "withProviderImpl" + args: + - "${hbase.provider.impl}" + - name: "withTrackerHBaseTable" + args: + - "${threat.intel.tracker.table}" + - name: "withTrackerHBaseCF" + args: + - "${threat.intel.tracker.cf}" + - name: "withHBaseTable" + args: + - "${threat.intel.simple.hbase.table}" + - name: "withHBaseCF" + args: + - "${threat.intel.simple.hbase.cf}" + - id: "simpleHBaseThreatIntelAdapter" + className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter" + configMethods: + - name: "withConfig" + args: + - ref: "simpleHBaseThreatIntelConfig" + - id: "simpleHBaseThreatIntelEnrichment" + className: "org.apache.metron.enrichment.configuration.Enrichment" + constructorArgs: + - "hbaseThreatIntel" + - ref: "simpleHBaseThreatIntelAdapter" + + - id: "threatIntels" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: + - ref: "simpleHBaseThreatIntelEnrichment" + - name: "add" + args: + - ref: "stellarThreatIntelEnrichment" + + #threatintel error + - id: "threatIntelErrorKafkaWriter" + className: "org.apache.metron.writer.kafka.KafkaWriter" + configMethods: + - name: "withTopic" + args: + - "${threat.intel.error.topic}" + - name: "withZkQuorum" + args: + - "${kafka.zk}" + - name: "withProducerConfigs" + args: + - ref: "kafkaWriterProps" +#indexing + - id: "kafkaWriter" + className: "org.apache.metron.writer.kafka.KafkaWriter" + configMethods: + - name: "withTopic" + args: + - "${enrichment.output.topic}" + - name: "withZkQuorum" + args: + - "${kafka.zk}" + - name: "withProducerConfigs" + args: + - ref: "kafkaWriterProps" + +#kafka/zookeeper + # Any kafka props for the consumer go here. + - id: "kafkaProps" + className: "java.util.HashMap" + configMethods: + - name: "put" + args: + - "value.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "key.deserializer" + - "org.apache.kafka.common.serialization.ByteArrayDeserializer" + - name: "put" + args: + - "group.id" + - "enrichments" + - name: "put" + args: + - "security.protocol" + - "${kafka.security.protocol}" + + + # The fields to pull out of the kafka messages + - id: "fields" + className: "java.util.ArrayList" + configMethods: + - name: "add" + args: + - "value" + + - id: "kafkaConfig" + className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder" + constructorArgs: + - ref: "kafkaProps" + # topic name + - "${enrichment.input.topic}" + - "${kafka.zk}" + - ref: "fields" + configMethods: + - name: "setFirstPollOffsetStrategy" + args: + - "${kafka.start}" + + +spouts: + - id: "kafkaSpout" + className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" + constructorArgs: + - ref: "kafkaConfig" + parallelism: ${kafka.spout.parallelism} + +bolts: +# Enrichment Bolts + - id: "enrichmentSplitBolt" + className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichments" + args: + - ref: "enrichments" + parallelism: ${enrichment.split.parallelism} + + - id: "geoEnrichmentBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "geoEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + + - id: "stellarEnrichmentBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "stellarEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + parallelism: ${enrichment.stellar.parallelism} + + - id: "hostEnrichmentBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "hostEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + + - id: "simpleHBaseEnrichmentBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "simpleHBaseEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + + - id: "enrichmentJoinBolt" + className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withMaxCacheSize" + args: [${enrichment.join.cache.size}] + - name: "withMaxTimeRetain" + args: [10] + parallelism: ${enrichment.join.parallelism} + + - id: "enrichmentErrorOutputBolt" + className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withMessageWriter" + args: + - ref: "enrichmentErrorKafkaWriter" + + +# Threat Intel Bolts + - id: "threatIntelSplitBolt" + className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichments" + args: + - ref: "threatIntels" + - name: "withMessageFieldName" + args: ["message"] + parallelism: ${threat.intel.split.parallelism} + + - id: "simpleHBaseThreatIntelBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "simpleHBaseThreatIntelEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + - id: "stellarThreatIntelBolt" + className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withEnrichment" + args: + - ref: "stellarThreatIntelEnrichment" + - name: "withMaxCacheSize" + args: [10000] + - name: "withMaxTimeRetain" + args: [10] + parallelism: ${threat.intel.stellar.parallelism} + + - id: "threatIntelJoinBolt" + className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withMaxCacheSize" + args: [${threat.intel.join.cache.size}] + - name: "withMaxTimeRetain" + args: [10] + parallelism: ${threat.intel.join.parallelism} + + - id: "threatIntelErrorOutputBolt" + className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withMessageWriter" + args: + - ref: "threatIntelErrorKafkaWriter" + +# Indexing Bolts + - id: "outputBolt" + className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" + constructorArgs: + - "${kafka.zk}" + configMethods: + - name: "withMessageWriter" + args: + - ref: "kafkaWriter" + parallelism: ${kafka.writer.parallelism} + + +streams: +#parser + - name: "spout -> enrichmentSplit" + from: "kafkaSpout" + to: "enrichmentSplitBolt" + grouping: + type: LOCAL_OR_SHUFFLE + +#enrichment + - name: "enrichmentSplit -> host" + from: "enrichmentSplitBolt" + to: "hostEnrichmentBolt" + grouping: + streamId: "host" + type: FIELDS + args: ["message"] + + - name: "enrichmentSplit -> geo" + from: "enrichmentSplitBolt" + to: "geoEnrichmentBolt" + grouping: + streamId: "geo" + type: FIELDS + args: ["message"] + + - name: "enrichmentSplit -> stellar" + from: "enrichmentSplitBolt" + to: "stellarEnrichmentBolt" + grouping: + streamId: "stellar" + type: FIELDS + args: ["message"] + + + - name: "enrichmentSplit -> simpleHBaseEnrichmentBolt" + from: "enrichmentSplitBolt" + to: "simpleHBaseEnrichmentBolt" + grouping: + streamId: "hbaseEnrichment" + type: FIELDS + args: ["message"] + + - name: "splitter -> join" + from: "enrichmentSplitBolt" + to: "enrichmentJoinBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + + - name: "geo -> join" + from: "geoEnrichmentBolt" + to: "enrichmentJoinBolt" + grouping: + streamId: "geo" + type: FIELDS + args: ["key"] + + - name: "stellar -> join" + from: "stellarEnrichmentBolt" + to: "enrichmentJoinBolt" + grouping: + streamId: "stellar" + type: FIELDS + args: ["key"] + + - name: "simpleHBaseEnrichmentBolt -> join" + from: "simpleHBaseEnrichmentBolt" + to: "enrichmentJoinBolt" + grouping: + streamId: "hbaseEnrichment" + type: FIELDS + args: ["key"] + + - name: "host -> join" + from: "hostEnrichmentBolt" + to: "enrichmentJoinBolt" + grouping: + streamId: "host" + type: FIELDS + args: ["key"] + + # Error output + - name: "geoEnrichmentBolt -> enrichmentErrorOutputBolt" + from: "geoEnrichmentBolt" + to: "enrichmentErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + + - name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt" + from: "stellarEnrichmentBolt" + to: "enrichmentErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + + - name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt" + from: "hostEnrichmentBolt" + to: "enrichmentErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + + - name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt" + from: "simpleHBaseEnrichmentBolt" + to: "enrichmentErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + +#threat intel + - name: "enrichmentJoin -> threatSplit" + from: "enrichmentJoinBolt" + to: "threatIntelSplitBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] + + - name: "threatSplit -> simpleHBaseThreatIntel" + from: "threatIntelSplitBolt" + to: "simpleHBaseThreatIntelBolt" + grouping: + streamId: "hbaseThreatIntel" + type: FIELDS + args: ["message"] + + - name: "threatSplit -> stellarThreatIntel" + from: "threatIntelSplitBolt" + to: "stellarThreatIntelBolt" + grouping: + streamId: "stellar" + type: FIELDS + args: ["message"] + + + - name: "simpleHBaseThreatIntel -> join" + from: "simpleHBaseThreatIntelBolt" + to: "threatIntelJoinBolt" + grouping: + streamId: "hbaseThreatIntel" + type: FIELDS + args: ["key"] + + - name: "stellarThreatIntel -> join" + from: "stellarThreatIntelBolt" + to: "threatIntelJoinBolt" + grouping: + streamId: "stellar" + type: FIELDS + args: ["key"] + + - name: "threatIntelSplit -> threatIntelJoin" + from: "threatIntelSplitBolt" + to: "threatIntelJoinBolt" + grouping: + streamId: "message" + type: FIELDS + args: ["key"] +#output + - name: "threatIntelJoin -> output" + from: "threatIntelJoinBolt" + to: "outputBolt" + grouping: + streamId: "message" + type: LOCAL_OR_SHUFFLE + + # Error output + - name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt" + from: "simpleHBaseThreatIntelBolt" + to: "threatIntelErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + + - name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt" + from: "stellarThreatIntelBolt" + to: "threatIntelErrorOutputBolt" + grouping: + streamId: "error" + type: LOCAL_OR_SHUFFLE + http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml index ddc5ffc..d7107d9 100644 --- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml +++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml @@ -27,13 +27,12 @@ config: topology.worker.childopts: ${topology.worker.childopts} topology.auto-credentials: ${topology.auto-credentials} topology.max.spout.pending: ${topology.max.spout.pending} - # Change this if you want to adjust the threadpool size - metron.threadpool.size: "2C" # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores) - # Change this if you want to adjust the threadpool type - metron.threadpool.type: "FIXED" # FIXED or WORK_STEALING + metron.threadpool.size: ${enrichment.threadpool.size} # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores) + metron.threadpool.type: ${enrichment.threadpool.type} # FIXED or WORK_STEALING + components: -# Enrichment + # enrichment - id: "stellarEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" configMethods: @@ -41,7 +40,7 @@ components: args: - "ENRICHMENT" - # Any kafka props for the producer go here. + # any kafka props for the producer go here. - id: "kafkaWriterProps" className: "java.util.HashMap" configMethods: @@ -58,15 +57,18 @@ components: - id: "geoEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter" + - id: "geoEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: - "geo" - ref: "geoEnrichmentAdapter" + - id: "hostEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter" constructorArgs: - '${enrichment.host.known_hosts}' + - id: "hostEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -85,17 +87,20 @@ components: - name: "withHBaseCF" args: - "${enrichment.simple.hbase.cf}" + - id: "simpleHBaseEnrichmentAdapter" className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter" configMethods: - name: "withConfig" args: - ref: "simpleHBaseEnrichmentConfig" + - id: "simpleHBaseEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: - "hbaseEnrichment" - ref: "simpleHBaseEnrichmentAdapter" + - id: "enrichments" className: "java.util.ArrayList" configMethods: @@ -112,7 +117,7 @@ components: args: - ref: "stellarEnrichment" - #enrichment error + # enrichment error - id: "enrichmentErrorKafkaWriter" className: "org.apache.metron.writer.kafka.KafkaWriter" configMethods: @@ -123,21 +128,23 @@ components: args: - "${kafka.zk}" - name: "withProducerConfigs" - args: + args: - ref: "kafkaWriterProps" -# Threat Intel + # threat Intel - id: "stellarThreatIntelAdapter" className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter" configMethods: - name: "ofType" args: - "THREAT_INTEL" + - id: "stellarThreatIntelEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: - "stellar" - ref: "stellarThreatIntelAdapter" + - id: "simpleHBaseThreatIntelConfig" className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig" configMethods: @@ -156,12 +163,14 @@ components: - name: "withHBaseCF" args: - "${threat.intel.simple.hbase.cf}" + - id: "simpleHBaseThreatIntelAdapter" className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter" configMethods: - name: "withConfig" args: - ref: "simpleHBaseThreatIntelConfig" + - id: "simpleHBaseThreatIntelEnrichment" className: "org.apache.metron.enrichment.configuration.Enrichment" constructorArgs: @@ -178,7 +187,7 @@ components: args: - ref: "stellarThreatIntelEnrichment" - #threatintel error + # threatintel error - id: "threatIntelErrorKafkaWriter" className: "org.apache.metron.writer.kafka.KafkaWriter" configMethods: @@ -189,9 +198,9 @@ components: args: - "${kafka.zk}" - name: "withProducerConfigs" - args: + args: - ref: "kafkaWriterProps" -#indexing + # indexing - id: "kafkaWriter" className: "org.apache.metron.writer.kafka.KafkaWriter" configMethods: @@ -202,11 +211,11 @@ components: args: - "${kafka.zk}" - name: "withProducerConfigs" - args: + args: - ref: "kafkaWriterProps" -#kafka/zookeeper - # Any kafka props for the consumer go here. + # kafka/zookeeper + # any kafka props for the consumer go here. - id: "kafkaProps" className: "java.util.HashMap" configMethods: @@ -228,7 +237,7 @@ components: - "${kafka.security.protocol}" - # The fields to pull out of the kafka messages + # the fields to pull out of the kafka messages - id: "fields" className: "java.util.ArrayList" configMethods: @@ -251,6 +260,7 @@ components: spouts: + - id: "kafkaSpout" className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout" constructorArgs: @@ -258,7 +268,8 @@ spouts: parallelism: ${kafka.spout.parallelism} bolts: -# Enrichment Bolts + + # enrichment bolt - id: "enrichmentBolt" className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt" constructorArgs: @@ -268,7 +279,7 @@ bolts: args: - ref: "enrichments" - name: "withMaxCacheSize" - args: [${enrichment.join.cache.size}] + args: [${enrichment.cache.size}] - name: "withMaxTimeRetain" args: [10] - name: "withCaptureCacheStats" @@ -278,7 +289,7 @@ bolts: - "ENRICHMENT" - name: "withMessageGetter" args: ["JSON_FROM_POSITION"] - parallelism: ${enrichment.join.parallelism} + parallelism: ${enrichment.parallelism} - id: "enrichmentErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" @@ -290,7 +301,7 @@ bolts: - ref: "enrichmentErrorKafkaWriter" -# Threat Intel Bolts + # threat intel bolts - id: "threatIntelBolt" className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt" constructorArgs: @@ -300,7 +311,7 @@ bolts: args: - ref: "threatIntels" - name: "withMaxCacheSize" - args: [${enrichment.join.cache.size}] + args: [${threat.intel.cache.size}] - name: "withMaxTimeRetain" args: [10] - name: "withCaptureCacheStats" @@ -312,7 +323,7 @@ bolts: args: ["message"] - name: "withMessageGetter" args: ["JSON_FROM_FIELD_BY_REFERENCE"] - parallelism: ${threat.intel.join.parallelism} + parallelism: ${threat.intel.parallelism} - id: "threatIntelErrorOutputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" @@ -323,7 +334,7 @@ bolts: args: - ref: "threatIntelErrorKafkaWriter" -# Indexing Bolts + # output bolt - id: "outputBolt" className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt" constructorArgs: @@ -334,16 +345,16 @@ bolts: - ref: "kafkaWriter" parallelism: ${kafka.writer.parallelism} - streams: -#parser + + # parser - name: "spout -> enrichmentBolt" from: "kafkaSpout" to: "enrichmentBolt" grouping: type: LOCAL_OR_SHUFFLE - # Error output + # error output - name: "enrichmentBolt -> enrichmentErrorOutputBolt" from: "enrichmentBolt" to: "enrichmentErrorOutputBolt" @@ -351,7 +362,7 @@ streams: streamId: "error" type: LOCAL_OR_SHUFFLE -#threat intel + # threat intel - name: "enrichmentBolt -> threatIntelBolt" from: "enrichmentBolt" to: "threatIntelBolt" @@ -359,7 +370,7 @@ streams: streamId: "message" type: LOCAL_OR_SHUFFLE -#output + # output - name: "threatIntelBolt -> output" from: "threatIntelBolt" to: "outputBolt" @@ -367,12 +378,10 @@ streams: streamId: "message" type: LOCAL_OR_SHUFFLE - # Error output + # error output - name: "threatIntelBolt -> threatIntelErrorOutputBolt" from: "threatIntelBolt" to: "threatIntelErrorOutputBolt" grouping: streamId: "error" type: LOCAL_OR_SHUFFLE - -