http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py new file mode 100755 index 0000000..f0903ac --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk +import cPickle as pickle +import argparse, sys, os.path +import errno +import os + + +class DashboardIndex(object): + + def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): + """ + :arg host: hostname of the node (default: localhost) + :arg port: port to use (integer, default: 9200) + :arg url_prefix: optional url prefix for elasticsearch + :arg timeout: default timeout in seconds (float, default: 10) + """ + self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) + + def get(self): + """ + Get .kibana index from Elasticsearch + """ + dotkibana = self.es.search(index='.kibana', size = 100) + return dotkibana['hits']['hits'] + + def load(self,filespec): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + data=[] + with open(filespec,'rb') as fp: + data = pickle.load(fp) + return data + + def save(self,filename,data): + """ + Save Index data on local filesystem + :args filespec: path/filename for saved file + """ + with open(filename,'wb') as fp: + pickle.dump(data,fp) + + def put(self,data): + """ + Bulk write data to Elasticsearch + :args data: data to be written (note: index name is specified in data) + """ + bulk(self.es,data) + + def main(self,args): + + if args.save: + print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) + self.save(filename=args.filespec,data=di.get()) + else: + """ + Loads Kibana Dashboard definition from disk and replaces .kibana on index + :args filespec: path/filename for saved file + """ + if not os.path.isfile(args.filespec): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) + self.es.indices.delete(index='.kibana', ignore=[400, 404]) + self.put(data=di.load(filespec=args.filespec)) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument("hostname", help="ES Hostname or IP", type=str) + parser.add_argument("port", help="ES Port", type=int) + parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) + parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") + args = parser.parse_args() + di = DashboardIndex(host=args.hostname,port=args.port) + di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template new file mode 100644 index 0000000..6f38ed5 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template @@ -0,0 +1,233 @@ +{ + "template" : ".kibana", + "mappings" : { + "search" : { + "dynamic" : "strict", + "properties" : { + "columns" : { + "type" : "keyword" + }, + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "sort" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "url" : { + "dynamic" : "strict", + "properties" : { + "accessCount" : { + "type" : "long" + }, + "accessDate" : { + "type" : "date" + }, + "createDate" : { + "type" : "date" + }, + "url" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 2048 + } + } + } + } + }, + "dashboard" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "optionsJSON" : { + "type" : "text" + }, + "panelsJSON" : { + "type" : "text" + }, + "refreshInterval" : { + "properties" : { + "display" : { + "type" : "keyword" + }, + "pause" : { + "type" : "boolean" + }, + "section" : { + "type" : "integer" + }, + "value" : { + "type" : "integer" + } + } + }, + "timeFrom" : { + "type" : "keyword" + }, + "timeRestore" : { + "type" : "boolean" + }, + "timeTo" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "index-pattern" : { + "dynamic" : "strict", + "properties" : { + "fieldFormatMap" : { + "type" : "text" + }, + "fields" : { + "type" : "text" + }, + "intervalName" : { + "type" : "keyword" + }, + "notExpandable" : { + "type" : "boolean" + }, + "sourceFilters" : { + "type" : "text" + }, + "timeFieldName" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + } + } + }, + "timelion-sheet" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "hits" : { + "type" : "integer" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "timelion_chart_height" : { + "type" : "integer" + }, + "timelion_columns" : { + "type" : "integer" + }, + "timelion_interval" : { + "type" : "keyword" + }, + "timelion_other_interval" : { + "type" : "keyword" + }, + "timelion_rows" : { + "type" : "integer" + }, + "timelion_sheet" : { + "type" : "text" + }, + "title" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + } + } + }, + "visualization" : { + "dynamic" : "strict", + "properties" : { + "description" : { + "type" : "text" + }, + "kibanaSavedObjectMeta" : { + "properties" : { + "searchSourceJSON" : { + "type" : "text" + } + } + }, + "savedSearchId" : { + "type" : "keyword" + }, + "title" : { + "type" : "text" + }, + "uiStateJSON" : { + "type" : "text" + }, + "version" : { + "type" : "integer" + }, + "visState" : { + "type" : "text" + } + } + }, + "server" : { + "dynamic" : "strict", + "properties" : { + "uuid" : { + "type" : "keyword" + } + } + }, + "_default_" : { + "dynamic" : "strict" + }, + "config" : { + "dynamic" : "true", + "properties" : { + "buildNum" : { + "type" : "keyword" + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index ce8c074..1cd6f4c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -14,8 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. """ +import errno import os import requests + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute @@ -149,6 +153,34 @@ class Indexing(Script): cmd.format(params.es_http_url, template_name), logoutput=True) + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) + def kibana_dashboard_install(self, env): + from params import params + env.set_params(params) + + Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url)) + + kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') + if not os.path.isfile(kibanaTemplate): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) + + Logger.info("Loading .kibana index template from %s" % kibanaTemplate) + template_cmd = ambari_format( + 'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate) + Execute(template_cmd, logoutput=True) + + kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') + if not os.path.isfile(kibanaDashboardLoad): + raise IOError( + errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) + + Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) + + kibana_cmd = ambari_format( + 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) + Execute(kibana_cmd, logoutput=True) + def zeppelin_notebook_import(self, env): from params import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml index fe76ade..f59324c 100644 --- a/metron-deployment/pom.xml +++ b/metron-deployment/pom.xml @@ -49,6 +49,7 @@ <activeByDefault>true</activeByDefault> </activation> <modules> + <module>packaging/ambari/elasticsearch-mpack</module> <module>packaging/ambari/metron-mpack</module> </modules> </profile>
