http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py new file mode 100755 index 0000000..f0903ac --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +import cPickle as pickle +import argparse, sys, os.path +import errno +import os + + +class DashboardIndex(object): + + def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): + """ + :arg host: hostname of the node (default: localhost) + :arg port: port to use (integer, default: 9200) + :arg url_prefix: optional url prefix for elasticsearch + :arg timeout: default timeout in seconds (float, default: 10) + """ + self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) + + def get(self): + """ + Get .kibana index from Elasticsearch + """ + dotkibana = self.es.search(index='.kibana', size = 100) + return dotkibana['hits']['hits'] + + def load(self,filespec): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + data=[] + with open(filespec,'rb') as fp: + data = pickle.load(fp) + return data + + def save(self,filename,data): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + with open(filename,'wb') as fp: + pickle.dump(data,fp) + + def put(self,data): + """ + Bulk write data to Elasticsearch + :args data: data to be written (note: index name is specified in data) + """ + bulk(self.es,data) + + def main(self,args): + + if args.save: + print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) + self.save(filename=args.filespec,data=di.get()) + else: + """ + Loads Kibana Dashboard definition from disk and replaces .kibana on index + :args filespec: path/filename for saved file + """ + if not os.path.isfile(args.filespec): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) + self.es.indices.delete(index='.kibana', ignore=[400, 404]) + self.put(data=di.load(filespec=args.filespec)) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument("hostname", help="ES Hostname or IP", type=str) + parser.add_argument("port", help="ES Port", type=int) + parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) + parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") + args = parser.parse_args() + di = DashboardIndex(host=args.hostname,port=args.port) + di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template new file mode 100644 index 0000000..6f38ed5 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template @@ -0,0 +1,233 @@ +{ + "template" : ".kibana", + "mappings" : { + "search" : { + "dynamic" : "strict", + "properties" : { + "columns" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "sort" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "url" : { + "dynamic" : "strict", + "properties" : { + "accessCount" : { + "type" : "long" + }, + "accessDate" : { + "type" : "date" + }, + "createDate" : { + "type" : "date" + }, + "url" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 2048 + } + } + } + } + }, + "dashboard" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "optionsJSON" : { + "type" : "text" + }, + "panelsJSON" : { + "type" : "text" + }, + "refreshInterval" : { + "properties" : { + "display" : { + "type" : "keyword" + }, + "pause" : { + "type" : "boolean" + }, + "section" : { + "type" : "integer" + }, + "value" : { + "type" : "integer" + } + } + }, + "timeFrom" : { + "type" : "keyword" + }, + "timeRestore" : { + "type" : "boolean" + }, + "timeTo" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "index-pattern" : { + "dynamic" : "strict", + "properties" : { + "fieldFormatMap" : { + "type" : "text" + }, + "fields" : { + "type" : "text" + }, + "intervalName" : { + "type" : "keyword" + }, + "notExpandable" : { + "type" : "boolean" + }, + "sourceFilters" : { + "type" : "text" + }, + "timeFieldName" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + } + } + }, + "timelion-sheet" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "timelion_chart_height" : { + "type" : "integer" + }, + "timelion_columns" : { + "type" : "integer" + }, + "timelion_interval" : { + "type" : "keyword" + }, + "timelion_other_interval" : { + "type" : "keyword" + }, + "timelion_rows" : { + "type" : "integer" + }, + "timelion_sheet" : { + "type" : "text" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "visualization" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "savedSearchId" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + }, + "visState" : { + "type" : "text" + } + } + }, + "server" : { + "dynamic" : "strict", + "properties" : { + "uuid" : { + "type" : "keyword" + } + } + }, + "_default_" : { + "dynamic" : "strict" + }, + "config" : { + "dynamic" : "true", + "properties" : { + "buildNum" : { + "type" : "keyword" + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py new file mode 100755 index 0000000..4c001ef --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py @@ -0,0 +1,131 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +kibana_master + +""" + +import errno +import os + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions.format import format as ambari_format +from resource_management.libraries.script import Script + + +class Kibana(Script): + def install(self, env): + import params + env.set_params(params) + Logger.info("Install Kibana Master") + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + + Logger.info("Configure Kibana for Metron") + + directories = [params.log_dir, params.pid_dir, params.conf_dir] + Directory(directories, + create_parents=True, + mode=0755, + owner=params.kibana_user, + group=params.kibana_user + ) + + File("{0}/kibana.yml".format(params.conf_dir), + owner=params.kibana_user, + content=InlineTemplate(params.kibana_yml_template) + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Logger.info("Stop Kibana Master") + + Execute("service kibana stop") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + self.configure(env) + + Logger.info("Start the Master") + + + Execute("service kibana start") + + def restart(self, env): + import params + env.set_params(params) + + self.configure(env) + + Logger.info("Restarting the Master") + + Execute("service kibana restart") + + def status(self, env): + import params + env.set_params(params) + + Logger.info("Status of the Master") + + Execute("service kibana status") + + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) + def load_template(self, env): + import params + env.set_params(params) + + hostname = ambari_format("{es_host}") + port = int(ambari_format("{es_port}")) + + Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port)) + + kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') + if not os.path.isfile(kibanaTemplate): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) + + Logger.info("Loading .kibana index template from %s" % kibanaTemplate) + template_cmd = ambari_format( + 'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate) + Execute(template_cmd, logoutput=True) + + kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') + if not os.path.isfile(kibanaDashboardLoad): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) + + Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) + + kibana_cmd = ambari_format( + 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) + Execute(kibana_cmd, logoutput=True) + + +if __name__ == "__main__": + Kibana().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py new file mode 100755 index 0000000..ef4cb62 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Kibana Params configurations + +""" + +from urlparse import urlparse + +from resource_management.libraries.functions import format +from resource_management.libraries.script import Script + +# server configurations +config = Script.get_config() + +kibana_home = '/usr/share/kibana/' +kibana_bin = '/usr/share/kibana/bin/' + +conf_dir = "/etc/kibana" +kibana_user = config['configurations']['kibana-env']['kibana_user'] +kibana_group = config['configurations']['kibana-env']['kibana_group'] +log_dir = config['configurations']['kibana-env']['kibana_log_dir'] +pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] +pid_file = format("{pid_dir}/kibanasearch.pid") +es_url = config['configurations']['kibana-env']['kibana_es_url'] +parsed = urlparse(es_url) +es_host = parsed.netloc.split(':')[0] +es_port = parsed.netloc.split(':')[1] +kibana_port = config['configurations']['kibana-env']['kibana_server_port'] +kibana_server_host = config['configurations']['kibana-env']['kibana_server_host'] +kibana_default_application = config['configurations']['kibana-env']['kibana_default_application'] +hostname = config['hostname'] +java64_home = config['hostLevelParams']['java_home'] +kibana_yml_template = config['configurations']['kibana-site']['content'] + http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json new file mode 100755 index 0000000..448e102 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json @@ -0,0 +1,28 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "metron_ui", + "label": "Metron UI", + "requires_user_name": "false", + "component_name": "KIBANA_MASTER", + "url":"%@://%@:%@/", + "port":{ + "http_property": "kibana_server_port", + "http_default_port": "5601", + "https_property": "kibana_server_port", + "https_default_port": "5601", + "regex": "^(\\d+)$", + "site": "kibana-env" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template index b0103f2..01d4f7b 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template @@ -2,9 +2,6 @@ "template": "bro_index*", "mappings": { "bro_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -30,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -40,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -50,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -60,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -105,7 +97,8 @@ { "threat_triage_reason": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat:triage:rules:*:reason", "match_mapping_type": "*" @@ -114,7 +107,8 @@ { "threat_triage_name": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat:triage:rules:*:name", "match_mapping_type": "*" @@ -131,14 +125,13 @@ * Be careful when modifying this file to not unintentionally affect other logs. * For instance, the "version" field exists in the HTTP, SSL, and SSH logs. If you * were to only consider the SSH log, you would set the type to integer, but because - * in the SSL and HTTP logs version is a string, we must set the type to string. + * in the SSL and HTTP logs version is a string, we must set the type to keyword. */ /* * Metron-specific fields */ "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * Widely-used Bro fields (potentially renamed during Metron ingest) @@ -148,8 +141,7 @@ "format": "epoch_millis" }, "uid": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "alert": { "type": "nested" @@ -214,28 +206,24 @@ "type": "integer" }, "method": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "host": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "uri": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", "ignore_above": 8191 }, "referrer": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "user_agent": { - "type": "string" + "type": "text", + "fielddata": "true" }, "request_body_len": { "type": "long" @@ -247,8 +235,7 @@ "type": "integer" }, "status_msg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "info_code": { "type": "integer" @@ -262,12 +249,10 @@ "index": "not_analyzed" }, "username": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "password": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "proxied": { "type": "string", @@ -307,8 +292,7 @@ * https://www.bro.org/sphinx-git/script-reference/types.html#type-interval */ "proto": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "trans_id": { "type": "long" @@ -318,29 +302,25 @@ "index": "not_analyzed" }, "query": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "qclass": { "type": "integer" }, "qclass_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "qtype": { "type": "integer" }, "qtype_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "rcode": { "type": "integer" }, "rcode_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "AA": { "type": "boolean" @@ -358,7 +338,8 @@ "type": "integer" }, "answers": { - "type": "string" + "type": "text", + "fielddata": "true" }, "TTLs": { "type": "string" @@ -381,58 +362,46 @@ * Notes: Field exists in the Conn and Files logs */ "service": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "duration": { "type": "float" }, "orig_bytes": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "resp_bytes": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "conn_state": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "local_orig": { "type": "boolean" }, "local_resp": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "missed_bytes": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "history": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "orig_pkts": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "orig_ip_bytes": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "resp_pkts": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "resp_ip_bytes": { - "type": "long", - "index": "not_analyzed" + "type": "long" }, "tunnel_parents": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * DPD log support @@ -443,12 +412,10 @@ * Notes: Field exists in the DNS, Conn, DPD, and Notice logs */ "analyzer": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "failure_reason": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * FTP log support @@ -468,19 +435,19 @@ * Notes: Field exists in the FTP, Files, and Notice logs */ "user": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "command": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "arg": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "mime_type": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "file_size": { @@ -490,8 +457,7 @@ "type": "integer" }, "reply_msg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "data_channel:passive": { "type": "boolean" @@ -506,15 +472,15 @@ "type": "integer" }, "cwd": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "passive": { "type": "boolean" }, "fuid": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * Files log support @@ -540,23 +506,23 @@ * Notes: Field exists in the FTP, Files, and Notice logs */ "conn_uids": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "source": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "depth": { "type": "integer" }, "analyzers": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "filename": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "is_orig": { "type": "boolean" @@ -577,20 +543,16 @@ "type": "boolean" }, "parent_fuid": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "md5": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sha1": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sha256": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "extracted": { "type": "string", @@ -617,16 +579,17 @@ "type": "integer" }, "subject": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "issuer_subject": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "serial": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * SMTP log support @@ -649,27 +612,31 @@ * Notes: Field exists in the HTTP, SMTP, and SIP logs */ "helo": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "mailfrom": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "rcptto": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "date": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "from": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "to": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "cc": { @@ -677,42 +644,42 @@ "analyzer": "simple" }, "reply_to": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "msg_id": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "in_reply_to": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "x_originating_ip": { "type": "ip" }, "first_received": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "second_received": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "last_reply": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, "path": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "tls": { "type": "boolean" }, "fuids": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "is_webmail": { "type": "boolean" @@ -729,16 +696,13 @@ * Notes: Field exists in the KnownCerts, SMTP, SIP, and SSL logs */ "cipher": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "curve": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "server_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "resumed": { "type": "boolean" @@ -751,12 +715,10 @@ "type": "boolean" }, "last_alert": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "next_protocol": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "established": { "type": "boolean" @@ -795,19 +757,16 @@ * Notes: Field exists in the Weird and LoadedScripts logs */ "name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "addl": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "notice": { "type": "boolean" }, "peer": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * Notice log support @@ -836,24 +795,19 @@ * Notes: Field exists in the Notice and SSH logs */ "file_mime_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "file_desc": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "note": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "msg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sub": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "src": { "type": "ip" @@ -862,27 +816,22 @@ "type": "ip" }, "p": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "n": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "src_peer": { "type": "ip" }, "peer_descr": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "actions": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "suppress_for": { - "type": "double", - "index": "not_analyzed" + "type": "double" }, "dropped": { "type": "boolean" @@ -914,15 +863,13 @@ * Notes: Field exists in the DNS and DHCP logs */ "mac": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "assigned_ip": { "type": "ip" }, "lease_time": { - "type": "float", - "index": "not_analyzed" + "type": "float" }, /* * SSH log support @@ -951,44 +898,34 @@ "type": "boolean" }, "auth_attempts": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "direction": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "client": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "server": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "cipher_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "mac_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "compression_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "kex_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "host_key_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "host_key": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * Software log support @@ -999,35 +936,29 @@ * Notes: Field exists in the HTTP, KnownCerts, and Software logs */ "host_p": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "software_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version:major": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version:minor": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version:minor2": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version:minor3": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "version:addl": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "unparsed_version": { - "type": "string", + "type": "text", + "fielddata": "true", "analyzer": "simple" }, /* @@ -1052,16 +983,14 @@ "type": "ip" }, "connect_info": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "reply_msg": { "type": "string", "index": "not_analyzed" }, "result": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "ttl": { "type": "string", @@ -1076,82 +1005,64 @@ * Notes: In other bro records, the id field is of type conn_id, so it is * expanded before being logged into 4 fields, all of which are addressed * under the "Widely-used Bro fields" section of this template. In X509 - * logs, however, id is a string to identify the certificate file id. + * logs, however, id is a keyword to identify the certificate file id. */ "id": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:version": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "certificate:serial": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:subject": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:issuer": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:not_valid_before": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:not_valid_after": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:key_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:sig_alg": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:key_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:key_length": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, "certificate:exponent": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "certificate:curve": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "san:dns": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "san:uri": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "san:email": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "san:ip": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "basic_constraints:ca": { "type": "boolean" }, "basic_constraints:path_len": { - "type": "integer", - "index": "not_analyzed" + "type": "integer" }, /* * Known::DevicesInfo log support @@ -1162,32 +1073,26 @@ * Notes: Field exists in the DHCP, RADIUS, and KnownDevices logs */ "dhcp_host_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * RFB::Info log support * https://www.bro.org/sphinx-git/scripts/base/protocols/rfb/main.bro.html#type-RFB::Info */ "client_major_version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "client_minor_version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "server_major_version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "server_minor_version": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "authentication_method": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "auth": { "type": "boolean" @@ -1196,8 +1101,7 @@ "type": "boolean" }, "desktop_name": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "width": { "type": "integer" @@ -1233,8 +1137,7 @@ "type": "integer" }, "pkt_lag": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "events_proc": { "type": "integer" @@ -1303,8 +1206,7 @@ * Notes: Field exists in the Weird, CaptureLoss, and Stats logs */ "ts_delta": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "gaps": { "type": "integer", @@ -1323,15 +1225,15 @@ * https://www.bro.org/sphinx/scripts/base/frameworks/reporter/main.bro.html#type-Reporter::Info */ "level": { - "type": "string" + "type": "text", + "fielddata": "true", + "analyzer": "simple" }, "message": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "location": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, /* * SIP::Info log support @@ -1372,42 +1274,41 @@ * Notes: Field exists in the HTTP and SIP logs */ "request_from": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "request_to": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "response_from": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "response_to": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "call_id": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "seq": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "request_path": { - "type": "string" + "type": "text", + "fielddata": "true", + "analyzer": "simple" }, "response_path": { - "type": "string" + "type": "text", + "fielddata": "true", + "analyzer": "simple" }, "warning": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "content_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" + }, + "guid": { + "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template index e79d482..00aaf87 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/error_index.template @@ -2,54 +2,41 @@ "template": "error_index*", "mappings": { "error_doc": { - "_timestamp": { - "enabled": true - }, "properties": { "exception": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "hostname": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "stack": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "timestamp": { "type": "date", "format": "epoch_millis" }, "message": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "raw_message": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", "ignore_above": 8191 }, "raw_message_bytes": { - "type": "binary", - "index": "no" + "type": "binary" }, "error_fields": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "error_hash": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "failed_sensor_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "error_type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "alert": { "type": "nested" http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template index 5af9251..24e8357 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template @@ -2,17 +2,13 @@ "template": "metaalert_index*", "mappings": { "metaalert_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "alert_template": { "path_match": "alert.*", "match_mapping_type": "string", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -28,16 +24,13 @@ ], "properties": { "guid": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "score": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "status": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "timestamp": { "type": "date", @@ -47,8 +40,7 @@ "type": "nested" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template index 7c6b401..43b3ca4 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template @@ -2,9 +2,6 @@ "template": "snort_index*", "mappings": { "snort_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -30,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -40,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -50,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -60,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -102,24 +94,26 @@ "match_mapping_type": "*" } }, - { - "threat_triage_reason": { - "mapping": { - "type": "string" - }, - "match": "threat:triage:rules:*:reason", - "match_mapping_type": "*" - } - }, - { - "threat_triage_name": { - "mapping": { - "type": "string" - }, - "match": "threat:triage:rules:*:name", - "match_mapping_type": "*" + { + "threat_triage_reason": { + "mapping": { + "type": "text", + "fielddata": "true" + }, + "match": "threat.triage.rules:*:reason", + "match_mapping_type": "*" + } + }, + { + "threat_triage_name": { + "mapping": { + "type": "text", + "fielddata": "true" + }, + "match": "threat.triage.rules:*:name", + "match_mapping_type": "*" + } } - } ], "properties": { "timestamp": { @@ -127,8 +121,7 @@ "format": "epoch_millis" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "ip_dst_addr": { "type": "ip" @@ -146,16 +139,13 @@ "type": "integer" }, "ethdst": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "ethlen": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "ethsrc": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "id": { "type": "integer" @@ -167,33 +157,37 @@ "type": "boolean" }, "msg": { - "type": "string" + "type": "text", + "fielddata": "true" }, "protocol": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sig_generator": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sig_id": { "type": "integer" }, "sig_rev": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpack": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpflags": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpseq": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tcpwindow": { - "type": "string" + "type": "text", + "fielddata": "true" }, "tos": { "type": "integer" @@ -201,6 +195,9 @@ "ttl": { "type": "integer" }, + "guid": { + "type": "keyword" + }, "alert": { "type": "nested" } http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template index d100eb0..b6965f9 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template @@ -2,9 +2,6 @@ "template": "yaf_index*", "mappings": { "yaf_doc": { - "_timestamp": { - "enabled": true - }, "dynamic_templates": [ { "geo_location_point": { @@ -20,8 +17,7 @@ "match": "enrichments:geo:*:country", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -30,8 +26,7 @@ "match": "enrichments:geo:*:city", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -40,8 +35,7 @@ "match": "enrichments:geo:*:locID", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -50,8 +44,7 @@ "match": "enrichments:geo:*:dmaCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -60,8 +53,7 @@ "match": "enrichments:geo:*:postalCode", "match_mapping_type": "*", "mapping": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" } } }, @@ -105,7 +97,8 @@ { "threat_triage_reason": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat:triage:rules:*:reason", "match_mapping_type": "*" @@ -114,7 +107,8 @@ { "threat_triage_name": { "mapping": { - "type": "string" + "type": "text", + "fielddata": "true" }, "match": "threat:triage:rules:*:name", "match_mapping_type": "*" @@ -127,8 +121,7 @@ "format": "epoch_millis" }, "source:type": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "ip_dst_addr": { "type": "ip" @@ -157,55 +150,44 @@ "type": "double" }, "proto": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sip": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "sp": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "dip": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "dp": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "iflags": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "uflags": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "riflags": { - "type": "string" + "type": "text", + "fielddata": "true" }, "ruflags": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "isn": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "risn": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "tag": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "rtag": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "pkt": { "type": "integer" @@ -220,14 +202,17 @@ "type": "integer" }, "app": { - "type": "string", - "index": "not_analyzed" + "type": "keyword" }, "end-reason": { - "type": "string" + "type": "text", + "fielddata": "true" }, "alert": { "type": "nested" + }, + "guid": { + "type": "keyword" } } } http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index c79dfd5..e92785a 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -138,7 +138,6 @@ class Indexing(Script): commands = IndexingCommands(params) for template_name in commands.get_templates(): - # delete the index template cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""; Execute( http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json index bf665f2..7a9d892 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json @@ -25,7 +25,7 @@ "service_versions_map": [ { "service_name" : "KIBANA", - "service_version" : "4.5.1", + "service_version" : "5.6.2", "applicable_stacks" : [ { "stack_name" : "HDP", @@ -43,7 +43,7 @@ }, { "service_name" : "ELASTICSEARCH", - "service_version" : "2.3.3", + "service_version" : "5.6.2", "applicable_stacks" : [ { "stack_name" : "HDP", http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index 024582c..839e04d 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -120,7 +120,8 @@ required_configurations: kibana_es_url: http://{{ groups.search[0] }}:9200 kibana_log_dir: /var/log/kibana kibana_server_port: 5000 - kibana_default_application: "dashboard/Metron-Dashboard" + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" + kibana_server_host: 0.0.0.0 blueprint: stack_name: HDP http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 1e652da..568b41b 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -114,7 +114,8 @@ required_configurations: kibana_es_url: http://{{ groups.web[0] }}:9200 kibana_log_dir: /var/log/kibana kibana_server_port: 5000 - kibana_default_application: "dashboard/Metron-Dashboard" + kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9" + kibana_server_host: 0.0.0.0 blueprint: stack_name: HDP
