http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py deleted file mode 100755 index f0903ac..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/python -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk -import cPickle as pickle -import argparse, sys, os.path -import errno -import os - - -class DashboardIndex(object): - - def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs): - """ - :arg host: hostname of the node (default: localhost) - :arg port: port to use (integer, default: 9200) - :arg url_prefix: optional url prefix for elasticsearch - :arg timeout: default timeout in seconds (float, default: 10) - """ - self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}]) - - def get(self): - """ - Get .kibana index from Elasticsearch - """ - dotkibana = self.es.search(index='.kibana', size = 100) - return dotkibana['hits']['hits'] - - def load(self,filespec): - """ - Save Index data on local filesystem - :args filespec: path/filename for saved file - """ - data=[] - with open(filespec,'rb') as fp: - data = pickle.load(fp) - return data - - def save(self,filename,data): - """ - Save Index data on local filesystem - :args filespec: path/filename for saved file - """ - with open(filename,'wb') as fp: - pickle.dump(data,fp) - - def put(self,data): - """ - Bulk write data to Elasticsearch - :args data: data to be written (note: index name is specified in data) - """ - bulk(self.es,data) - - def main(self,args): - - if args.save: - print("running save with host:%s on port %d, filespec: %s" % (args.hostname, args.port, args.filespec)) - self.save(filename=args.filespec,data=di.get()) - else: - """ - Loads Kibana Dashboard definition from disk and replaces .kibana on index - :args filespec: path/filename for saved file - """ - if not os.path.isfile(args.filespec): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), args.filespec) - self.es.indices.delete(index='.kibana', ignore=[400, 404]) - self.put(data=di.load(filespec=args.filespec)) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument("hostname", help="ES Hostname or IP", type=str) - parser.add_argument("port", help="ES Port", type=int) - parser.add_argument("filespec", help="file to be pushed from or saved to", type=str) - parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and saved to filespec",action="store_true") - args = parser.parse_args() - di = DashboardIndex(host=args.hostname,port=args.port) - di.main(args)
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template deleted file mode 100644 index 6f38ed5..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template +++ /dev/null @@ -1,233 +0,0 @@ -{ - "template" : ".kibana", - "mappings" : { - "search" : { - "dynamic" : "strict", - "properties" : { - "columns" : { - "type" : "keyword" - }, - "description" : { - "type" : "text" - }, - "hits" : { - "type" : "integer" - }, - "kibanaSavedObjectMeta" : { - "properties" : { - "searchSourceJSON" : { - "type" : "text" - } - } - }, - "sort" : { - "type" : "keyword" - }, - "title" : { - "type" : "text" - }, - "version" : { - "type" : "integer" - } - } - }, - "url" : { - "dynamic" : "strict", - "properties" : { - "accessCount" : { - "type" : "long" - }, - "accessDate" : { - "type" : "date" - }, - "createDate" : { - "type" : "date" - }, - "url" : { - "type" : "text", - "fields" : { - "keyword" : { - "type" : "keyword", - "ignore_above" : 2048 - } - } - } - } - }, - "dashboard" : { - "dynamic" : "strict", - "properties" : { - "description" : { - "type" : "text" - }, - "hits" : { - "type" : "integer" - }, - "kibanaSavedObjectMeta" : { - "properties" : { - "searchSourceJSON" : { - "type" : "text" - } - } - }, - "optionsJSON" : { - "type" : "text" - }, - "panelsJSON" : { - "type" : "text" - }, - "refreshInterval" : { - "properties" : { - "display" : { - "type" : "keyword" - }, - "pause" : { - "type" : "boolean" - }, - "section" : { - "type" : "integer" - }, - "value" : { - "type" : "integer" - } - } - }, - "timeFrom" : { - "type" : "keyword" - }, - "timeRestore" : { - "type" : "boolean" - }, - "timeTo" : { - "type" : "keyword" - }, - "title" : { - "type" : "text" - }, - "uiStateJSON" : { - "type" : "text" - }, - "version" : { - "type" : "integer" - } - } - }, - "index-pattern" : { - "dynamic" : "strict", - "properties" : { - "fieldFormatMap" : { - "type" : "text" - }, - "fields" : { - "type" : "text" - }, - "intervalName" : { - "type" : "keyword" - }, - "notExpandable" : { - "type" : "boolean" - }, - "sourceFilters" : { - "type" : "text" - }, - "timeFieldName" : { - "type" : "keyword" - }, - "title" : { - "type" : "text" - } - } - }, - "timelion-sheet" : { - "dynamic" : "strict", - "properties" : { - "description" : { - "type" : "text" - }, - "hits" : { - "type" : "integer" - }, - "kibanaSavedObjectMeta" : { - "properties" : { - "searchSourceJSON" : { - "type" : "text" - } - } - }, - "timelion_chart_height" : { - "type" : "integer" - }, - "timelion_columns" : { - "type" : "integer" - }, - "timelion_interval" : { - "type" : "keyword" - }, - "timelion_other_interval" : { - "type" : "keyword" - }, - "timelion_rows" : { - "type" : "integer" - }, - "timelion_sheet" : { - "type" : "text" - }, - "title" : { - "type" : "text" - }, - "version" : { - "type" : "integer" - } - } - }, - "visualization" : { - "dynamic" : "strict", - "properties" : { - "description" : { - "type" : "text" - }, - "kibanaSavedObjectMeta" : { - "properties" : { - "searchSourceJSON" : { - "type" : "text" - } - } - }, - "savedSearchId" : { - "type" : "keyword" - }, - "title" : { - "type" : "text" - }, - "uiStateJSON" : { - "type" : "text" - }, - "version" : { - "type" : "integer" - }, - "visState" : { - "type" : "text" - } - } - }, - "server" : { - "dynamic" : "strict", - "properties" : { - "uuid" : { - "type" : "keyword" - } - } - }, - "_default_" : { - "dynamic" : "strict" - }, - "config" : { - "dynamic" : "true", - "properties" : { - "buildNum" : { - "type" : "keyword" - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py deleted file mode 100755 index a15f709..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import errno -import os - -from ambari_commons.os_check import OSCheck -from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl - -from resource_management.core.logger import Logger -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import Execute -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from resource_management.libraries.functions.format import format as ambari_format -from resource_management.libraries.script import Script - -from common import service_check - -class Kibana(Script): - - def install(self, env): - import params - env.set_params(params) - Logger.info("Installing Kibana") - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - Logger.info("Configuring Kibana") - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - Directory(directories, - mode=0755, - owner=params.kibana_user, - group=params.kibana_user - ) - - File("{0}/kibana.yml".format(params.conf_dir), - owner=params.kibana_user, - content=InlineTemplate(params.kibana_yml_template) - ) - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - Logger.info("Stopping Kibana") - Execute("service kibana stop") - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - Logger.info("Starting Kibana") - Execute("service kibana start") - - def restart(self, env): - import params - env.set_params(params) - self.configure(env) - Logger.info("Restarting Kibana") - Execute("service kibana restart") - - def status(self, env): - import params - env.set_params(params) - Logger.info('Status check Kibana') - service_check("service kibana status", user=params.kibana_user, label="Kibana") - - @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) - def load_template(self, env): - import params - env.set_params(params) - - hostname = ambari_format("{es_host}") - port = int(ambari_format("{es_port}")) - - Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port)) - - kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'kibana.template') - if not os.path.isfile(kibanaTemplate): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate) - - Logger.info("Loading .kibana index template from %s" % kibanaTemplate) - template_cmd = ambari_format( - 'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate) - Execute(template_cmd, logoutput=True) - - kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard', 'dashboard-bulkload.json') - if not os.path.isfile(kibanaDashboardLoad): - raise IOError( - errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad) - - Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad) - - kibana_cmd = ambari_format( - 'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk --data-binary @%s' % kibanaDashboardLoad) - Execute(kibana_cmd, logoutput=True) - - -if __name__ == "__main__": - Kibana().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py deleted file mode 100755 index ef4cb62..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Kibana Params configurations - -""" - -from urlparse import urlparse - -from resource_management.libraries.functions import format -from resource_management.libraries.script import Script - -# server configurations -config = Script.get_config() - -kibana_home = '/usr/share/kibana/' -kibana_bin = '/usr/share/kibana/bin/' - -conf_dir = "/etc/kibana" -kibana_user = config['configurations']['kibana-env']['kibana_user'] -kibana_group = config['configurations']['kibana-env']['kibana_group'] -log_dir = config['configurations']['kibana-env']['kibana_log_dir'] -pid_dir = config['configurations']['kibana-env']['kibana_pid_dir'] -pid_file = format("{pid_dir}/kibanasearch.pid") -es_url = config['configurations']['kibana-env']['kibana_es_url'] -parsed = urlparse(es_url) -es_host = parsed.netloc.split(':')[0] -es_port = parsed.netloc.split(':')[1] -kibana_port = config['configurations']['kibana-env']['kibana_server_port'] -kibana_server_host = config['configurations']['kibana-env']['kibana_server_host'] -kibana_default_application = config['configurations']['kibana-env']['kibana_default_application'] -hostname = config['hostname'] -java64_home = config['hostLevelParams']['java_home'] -kibana_yml_template = config['configurations']['kibana-site']['content'] - http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json deleted file mode 100755 index 448e102..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "metron_ui", - "label": "Metron UI", - "requires_user_name": "false", - "component_name": "KIBANA_MASTER", - "url":"%@://%@:%@/", - "port":{ - "http_property": "kibana_server_port", - "http_default_port": "5601", - "https_property": "kibana_server_port", - "https_default_port": "5601", - "regex": "^(\\d+)$", - "site": "kibana-env" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml index 6a20af8..38bd94a 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml @@ -211,6 +211,15 @@ </commandScript> </customCommand> <customCommand> + <name>KIBANA_DASHBOARD_INSTALL</name> + <background>false</background> + <commandScript> + <script>scripts/indexing_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + <customCommand> <name>ZEPPELIN_NOTEBOOK_IMPORT</name> <commandScript> <script>scripts/indexing_master.py</script> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py new file mode 100755 index 0000000..8d2bad8 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file
