METRON-939: Upgrade ElasticSearch and Kibana (mmiklavc via mmiklavc) closes apache/metron#840
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e8213918 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e8213918 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e8213918 Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual Commit: e821391895f0f3523b4e05ae5d639c13b0113855 Parents: 0996b73 Author: mmiklavc <michael.miklav...@gmail.com> Authored: Mon Jan 8 12:08:40 2018 -0700 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Mon Jan 8 12:08:40 2018 -0700 ---------------------------------------------------------------------- dependencies_with_url.csv | 23 + metron-analytics/metron-profiler-client/pom.xml | 8 + metron-analytics/metron-profiler-common/pom.xml | 8 + metron-analytics/metron-profiler/pom.xml | 20 + .../manual-install/Manual_Install_CentOS6.md | 2 +- metron-deployment/packaging/ambari/README.md | 209 ++ .../packaging/ambari/metron-mpack/README.md | 95 +- .../ELASTICSEARCH/2.3.3/metainfo.xml | 29 - .../ELASTICSEARCH/2.3.3/repos/repoinfo.xml | 32 - .../ELASTICSEARCH/5.6.2/metainfo.xml | 29 + .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 32 + .../addon-services/KIBANA/4.5.1/metainfo.xml | 29 - .../KIBANA/4.5.1/quicklinks/quicklinks.json | 27 - .../KIBANA/4.5.1/repos/repoinfo.xml | 41 - .../addon-services/KIBANA/5.6.2/metainfo.xml | 30 + .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 + .../KIBANA/5.6.2/repos/repoinfo.xml | 41 + .../2.3.3/configuration/elastic-env.xml | 67 - .../2.3.3/configuration/elastic-site.xml | 203 -- .../2.3.3/configuration/elastic-sysconfig.xml | 104 - .../ELASTICSEARCH/2.3.3/metainfo.xml | 79 - .../2.3.3/package/scripts/elastic.py | 68 - .../2.3.3/package/scripts/elastic_master.py | 73 - .../2.3.3/package/scripts/elastic_slave.py | 71 - .../2.3.3/package/scripts/params.py | 80 - .../2.3.3/package/scripts/properties_config.py | 34 - .../2.3.3/package/scripts/service_check.py | 80 - .../2.3.3/package/scripts/slave.py | 66 - .../2.3.3/package/scripts/status_params.py | 26 - .../templates/elasticsearch.master.yaml.j2 | 85 - .../templates/elasticsearch.slave.yaml.j2 | 85 - .../2.3.3/quicklinks/quicklinks.json | 43 - .../ELASTICSEARCH/2.3.3/role_command_order.json | 8 - .../5.6.2/configuration/elastic-env.xml | 86 + .../5.6.2/configuration/elastic-site.xml | 198 ++ .../5.6.2/configuration/elastic-sysconfig.xml | 102 + .../ELASTICSEARCH/5.6.2/metainfo.xml | 79 + .../5.6.2/package/scripts/elastic.py | 86 + .../5.6.2/package/scripts/elastic_master.py | 72 + .../5.6.2/package/scripts/elastic_slave.py | 72 + .../5.6.2/package/scripts/params.py | 92 + .../5.6.2/package/scripts/properties_config.py | 34 + .../5.6.2/package/scripts/service_check.py | 77 + .../5.6.2/package/scripts/slave.py | 80 + .../5.6.2/package/scripts/status_params.py | 26 + .../templates/elasticsearch.master.yaml.j2 | 77 + .../templates/elasticsearch.slave.yaml.j2 | 78 + .../templates/elasticsearch_limits.conf.j2 | 20 + .../5.6.2/quicklinks/quicklinks.json | 43 + .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 + .../KIBANA/4.5.1/configuration/kibana-env.xml | 67 - .../KIBANA/4.5.1/configuration/kibana-site.xml | 112 - .../common-services/KIBANA/4.5.1/metainfo.xml | 75 - .../4.5.1/package/scripts/dashboard/__init__.py | 16 - .../4.5.1/package/scripts/dashboard/dashboard.p | 2341 ------------------ .../package/scripts/dashboard/dashboardindex.py | 95 - .../4.5.1/package/scripts/kibana_master.py | 128 - .../KIBANA/4.5.1/package/scripts/params.py | 49 - .../KIBANA/4.5.1/quicklinks/quicklinks.json | 28 - .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 + .../KIBANA/5.6.2/configuration/kibana-site.xml | 113 + .../common-services/KIBANA/5.6.2/metainfo.xml | 75 + .../5.6.2/package/scripts/dashboard/__init__.py | 16 + .../scripts/dashboard/dashboard-bulkload.json | 88 + .../package/scripts/dashboard/dashboardindex.py | 95 + .../package/scripts/dashboard/kibana.template | 233 ++ .../5.6.2/package/scripts/kibana_master.py | 131 + .../KIBANA/5.6.2/package/scripts/params.py | 50 + .../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 + .../CURRENT/package/files/bro_index.template | 463 ++-- .../CURRENT/package/files/error_index.template | 33 +- .../package/files/metaalert_index.template | 18 +- .../CURRENT/package/files/snort_index.template | 93 +- .../CURRENT/package/files/yaf_index.template | 75 +- .../CURRENT/package/scripts/indexing_master.py | 1 - .../metron-mpack/src/main/resources/mpack.json | 4 +- .../roles/ambari_config/vars/single_node_vm.yml | 3 +- .../roles/ambari_config/vars/small_cluster.yml | 3 +- .../e2e/mock-data/alerts_ui_e2e_index.template | 827 +++++-- .../metron-alerts/e2e/utils/e2e_util.ts | 32 +- .../rest/controller/RestExceptionHandler.java | 5 + .../rest/service/impl/UpdateServiceImpl.java | 6 +- .../SearchControllerIntegrationTest.java | 40 +- metron-platform/elasticsearch-shaded/pom.xml | 60 +- .../metron/netty/utils/NettyRuntimeWrapper.java | 34 + .../META-INF/log4j-provider.properties | 18 + metron-platform/metron-common/pom.xml | 8 + .../metron-data-management/README.md | 47 + metron-platform/metron-data-management/pom.xml | 31 +- .../dataloads/bulk/ElasticsearchDataPruner.java | 130 - .../bulk/ElasticsearchDataPrunerRunner.java | 200 -- .../main/scripts/prune_elasticsearch_indices.sh | 3 +- .../bulk/ElasticsearchDataPrunerRunnerTest.java | 72 - .../bulk/ElasticsearchDataPrunerTest.java | 855 ------- metron-platform/metron-elasticsearch/README.md | 235 +- metron-platform/metron-elasticsearch/pom.xml | 18 +- .../bulk/ElasticsearchImportExport.java | 93 + .../dao/ElasticsearchColumnMetadataDao.java | 3 +- .../elasticsearch/dao/ElasticsearchDao.java | 105 +- .../dao/ElasticsearchMetaAlertDao.java | 94 +- .../elasticsearch/utils/ElasticsearchUtils.java | 41 +- .../writer/ElasticsearchWriter.java | 13 +- .../bulk/ElasticsearchImportExportTest.java | 69 + .../elasticsearch/dao/ElasticsearchDaoTest.java | 27 +- .../dao/ElasticsearchRequestSubmitterTest.java | 10 +- .../ElasticsearchIndexingIntegrationTest.java | 23 +- .../ElasticsearchMetaAlertIntegrationTest.java | 63 +- .../ElasticsearchSearchIntegrationTest.java | 52 +- .../components/ElasticSearchComponent.java | 390 +-- metron-platform/metron-enrichment/pom.xml | 21 +- metron-platform/metron-hbase/pom.xml | 8 + metron-platform/metron-indexing/pom.xml | 10 + .../metron/indexing/dao/search/FieldType.java | 6 +- .../indexing/dao/SearchIntegrationTest.java | 670 +++-- .../integration/IndexingIntegrationTest.java | 10 +- metron-platform/metron-integration-test/pom.xml | 8 + .../metron/integration/utils/TestUtils.java | 27 + metron-platform/metron-parsers/pom.xml | 20 + metron-platform/metron-pcap-backend/pom.xml | 20 + metron-platform/metron-pcap/pom.xml | 8 + metron-platform/metron-solr/pom.xml | 20 + .../metron-storm-kafka-override/pom.xml | 8 + metron-platform/metron-storm-kafka/pom.xml | 8 + metron-platform/metron-test-utilities/pom.xml | 8 + metron-platform/metron-writer/pom.xml | 8 + pom.xml | 8 +- 126 files changed, 5268 insertions(+), 6793 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 38a9f5e..b94a984 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -310,6 +310,29 @@ org.springframework.security.kerberos:spring-security-kerberos-core:jar:1.0.1.RE org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j +com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE +io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-codec-http:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-codec:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-common:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-handler:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty:jar:3.10.6.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-resolver:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +io.netty:netty-transport:jar:4.1.13.Final:compile,ASLv2,http://netty.io/ +joda-time:joda-time:jar:2.9.5:compile,ASLv2,https://github.com/JodaOrg/joda-time +net.sf.jopt-simple:jopt-simple:jar:5.0.2:compile,The MIT License,http://jopt-simple.sourceforge.net +org.elasticsearch.client:elasticsearch-rest-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.client:transport:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:elasticsearch:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:jna:jar:4.4.0-1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:lang-mustache-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:parent-join-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:percolator-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:reindex-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:transport-netty3-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch.plugin:transport-netty4-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.elasticsearch:securesm:jar:1.1:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt +org.hdrhistogram:HdrHistogram:jar:2.1.9:compile,BSD,https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt com.trendmicro:tlsh:jar:3.7.1:compile,ASLv2,https://github.com/trendmicro/tlsh org.glassfish:javax.json:jar:1.0.4:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/jsonp org.eclipse.persistence:javax.persistence:jar:2.1.1:compile,EPL 1.0,http://www.eclipse.org/eclipselink http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-analytics/metron-profiler-client/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index 342160d..932284d 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -140,6 +140,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-analytics/metron-profiler-common/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/pom.xml b/metron-analytics/metron-profiler-common/pom.xml index b686f6b..f24b1d2 100644 --- a/metron-analytics/metron-profiler-common/pom.xml +++ b/metron-analytics/metron-profiler-common/pom.xml @@ -115,6 +115,14 @@ <scope>provided</scope> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>log4j-slf4j-impl</artifactId> <groupId>org.apache.logging.log4j</groupId> </exclusion> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-analytics/metron-profiler/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml index 6dcb595..d634cef 100644 --- a/metron-analytics/metron-profiler/pom.xml +++ b/metron-analytics/metron-profiler/pom.xml @@ -203,6 +203,14 @@ <version>${global_storm_version}</version> <exclusions> <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> <artifactId>servlet-api</artifactId> <groupId>javax.servlet</groupId> </exclusion> @@ -222,6 +230,18 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${global_log4j_core_version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>${global_kafka_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md ---------------------------------------------------------------------- diff --git a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md index 0c9df5e..e1305b6 100644 --- a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md +++ b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md @@ -458,7 +458,7 @@ Client - Kibana: * Set "kibana_es_url" to `http://<replace_with_elasticsearch_master_hostname>:9200`. "replace_with_elasticsearch_master_hostname" is the IP of the node where you assigned ElasticSearch Master on the Assign Master tab. - * Change kibana_default_application to "dashboard/Metron-Dashboard" + * Change kibana_default_application to "dashboard/AV-YpDmwdXwc6Ua9Muh9" - Metron: Set "Elasticsearch Hosts" to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab. http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md index bdb8b49..82e1537 100644 --- a/metron-deployment/packaging/ambari/README.md +++ b/metron-deployment/packaging/ambari/README.md @@ -27,6 +27,7 @@ limitations under the License. * [Configuration involving dependency services](#configuration-involving-dependency-services) * [Kerberos](#kerberos) * [Best practices](#best-practices) +* [Upgrading MPack Services](#upgrading-mpack-services) ## Overview Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files) @@ -464,3 +465,211 @@ This is checked in the indexing master * Make sure to `kinit` as the correct user for setting up ACLs in a secured cluster. This is usually kafka for Kafka and hbase for HBase. * See `set_hbase_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` for an HBase example * See `init_kafka_acls` in `METRON.CURRENT/package/scripts/enrichment_commands.py` and `METRON.CURRENT/package/scripts/metron_service.py` for an Kafka example + +## Upgrading MPack Services + +Apache Metron currently provides three services as part of its MPack +* Elasticsearch +* Kibana +* Metron + +There is currently no mechanism provided for multi-version or backwards compatibility. If you upgrade a service, e.g. Elasticsearch 2.x to 5.x, that is the only version that will be +supported by Ambari via MPack. + +The main steps for upgrading a service are split into add-on and common services for each service within the MPack as follows: +* Update the common services + * Change the service directory to use the new product version number + * Update metainfo.xml +* Update the add-on services + * Change the service directory to use the new product version number + * Update repoinfo.xml + * Update metainfo.xml +* Update mpack.json + +### Update Elasticsearch + +#### Update Common Services + +1. Change service directory names for Elasticsearch to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + <version>5.6.2</version> + ... + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>elasticsearch-5.6.2</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + ``` + +#### Update Add-on Services + +1. Change service directory names for Elasticsearch to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2 + ``` + +1. Update repoinfo.xml + + See [https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html) for the latest info. + + Modify the baseurl and repoid in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g. + + ``` + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + ``` + +1. Update metainfo.xml + + Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`. + Also make sure to update the "extends" version to point to the updated common-services version, e.g. + + ``` + <name>ELASTICSEARCH</name> + <version>5.6.2</version> + <extends>common-services/ELASTICSEARCH/5.6.2</extends> + ``` + +#### Update mpack.json + +1. Update the corresponding service_version in the service_versions_map, e.g. + + ``` + ... + "service_versions_map": [ + { + "service_name" : "ELASTICSEARCH", + "service_version" : "5.6.2", + "applicable_stacks" : [ + ... + ] + }, + ... + ] + ... + ``` + +### Kibana + +**Note:** Curator is included with the Kibana service + +#### Update Common Services + +1. Change service directory names for Kibana to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2 + ``` + +1. Update metainfo.xml + + Change the version number and package name in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`, e.g. + + ``` + <version>5.6.2</version> + ... + <packages> + ... + <package> + <name>kibana-5.6.2</name> + </package> + </packages> + ``` + +#### Update Add-on Services + +1. Change service directory names for Kibana to the new desired version + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE} + ``` + + e.g. + + ``` + metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2 + ``` + +1. Update repoinfo.xml + + **Note:** for Curator, there is a different repo for rhel 6 vs rhel 7 + + See the following links for current repo information for Kibana and Curator. + * [https://www.elastic.co/guide/en/kibana/current/rpm.html](https://www.elastic.co/guide/en/kibana/current/rpm.html) + * [https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/yum-repository.html) + + Modify the baseurl's and repoid's in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/repos/repoinfo.xml`, e.g. + + ``` + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + ... + <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + ``` + +1. Update metainfo.xml + + Change the version number in `metron/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/${YOUR_VERSION_NUMBER_HERE}/metainfo.xml`. + Also make sure to update the "extends" version to point to the updated common-services version, e.g. + ``` + <name>KIBANA</name> + <version>5.6.2</version> + <extends>common-services/KIBANA/5.6.2</extends> + ``` + +#### Update mpack.json + +1. Update the corresponding service_version in the service_versions_map, e.g. + + ``` + ... + "service_versions_map": [ + { + "service_name" : "KIBANA", + "service_version" : "5.6.2", + "applicable_stacks" : [ + ... + ] + }, + ... + ] + ... + ``` + http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md index 17fc4f6..5179b5f 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/README.md +++ b/metron-deployment/packaging/ambari/metron-mpack/README.md @@ -92,13 +92,94 @@ A custom action is available in Ambari to import Zeppelin dashboards. See the [m #### Kibana Dashboards -The dashboards installed by the Kibana custom action are managed by the `dashboard.p` file. This file is created by exporting existing dashboards from a running Kibana instance. - -To create a new version of the file, make any necessary changes to Kibana and run the following commands to export your changes. - ``` - cd packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/dashboard - python dashboardindex.py $ES_HOST 9200 dashboard.p -s - ``` +The dashboards installed by the Kibana custom action are managed by two JSON files: +* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template +* metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboard-bulkload.json + +The first file, `kibana.template`, is an Elasticsearch template that specifies the proper mapping types for the Kibana index. This configuration is necessary due to a bug +in the default dynamic mappings provided by Elasticsearch for long types versus integer that are incompatible with Kibana \[1\]. The second file, `dashboard-bulkload.json`, +contains all of the dashboard metadata necessary to create the Metron dashboard. It is an Elasticsearch bulk-insert formatted file \[2\] that contains a series +of documents necessary for setting up the dashboard in Elasticsearch. The main features installed are index patterns, searches, and a variety of visualizations +that are used in the Metron dashboard. + +Deploying the existing dashboard is easy. Once the MPack is installed, run the Kibana service's action "Load Template" to install dashboards. This will no longer overwrite +the .kibana in Elasticsearch. The bulk load is configured to fail inserts for existing documents. If you want to _completely_ reload the dashboard, you would need to delete +the .kibana index and reload again from Ambari. + +1. [https://github.com/elastic/kibana/issues/9888#issuecomment-298096954](https://github.com/elastic/kibana/issues/9888#issuecomment-298096954) +2. [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-bulk.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docs-bulk.html) + +##### Modifying Pre-Installed Dashboards + +You can modify dashboards in Kibana and bring those changes into the core MPack distribution by performing the following steps: + +1. Export the .kibana index from ES +2. Convert the data into the ES bulk load format +3. Replace the dashboard-bulkload.json file in the Kibana MPack. + +You can export the .kibana index using a tool like [https://github.com/taskrabbit/elasticsearch-dump](https://github.com/taskrabbit/elasticsearch-dump). The important +feature is to have one document per line. Here's an exmaple export using elasticsearch-dump + +``` +elasticdump \ + --input=http://node1:9200/.kibana \ + --output=~/dashboard-data.json \ + --type=data +``` + +Once you've exported the data, you can now format it as a bulk load ES file by running the import/export tool located in +metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java. This tool can be run from full-dev +as follows + +``` +java -cp $METRON_HOME/lib/metron-elasticsearch-0.4.2-uber.jar org.apache.metron.elasticsearch.bulk.ElasticsearchImportExport \ + ~/dashboard-data.json \ + ~/dashboard-bulkload.json +``` + +Locate the "create" command for setting the default index by searching for "5.6.2". Change "create" to "index" so that it modifies the existing value. It should look similar to line 1 below. + +``` +{ "index" : { "_id": "5.6.2", "_type": "config" } } +{"defaultIndex":"AV-S2e81hKs1cXXnFMqN"} +``` + +Now copy this file to the Kibana MPack, overwriting the existing bulk load file. That should be everything needed to backup the dashboard. + +**Note**: the dashboard Python Pickle binary file is deprecated and no longer used for backing up and restoring Kibana dashboards. The tooling is still provided as of this +version but is expected to be removed in the future. A section describing the deprecated backup process remains below. + +##### Deprecated Dashboard Install/Backup Instructions + +The dashboards installed by the Kibana custom action are managed by the dashboard.p file. This file is created by exporting existing dashboards from a running Kibana instance. + +To create a new version of the file, make any necessary changes to Kibana (e.g. on full-dev), and export with the appropriate script. + +**Script Options** +``` +[elasticsearch_host] ES host +[elasticsearch_port] ES port number +[input_output_filename] Filename used for reading or writing out pickle file +[-s] Flag to indicate that the .kibana index should be saved locally. Not including this flag will overwrite the .kibana + index completely with the contents of 'input_output_filename'. Careful with this. +``` + +**Saving a Backup** +``` +python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \ +$ES_HOST 9200 \ +~/dashboard.p -s +``` + +**Restoring From a Backup** +``` +python packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py \ +$ES_HOST 9200 \ +~/dashboard.p +``` + +**Note**: This method of writing the Kibana dashboard to Elasticsearch will overwrite the entire .kibana index. Be sure to first backup the index first using either the new JSON +method described above, or writing out the dashboard.p pickle file using the old method (passing -s option to dashboardindex.py) described here. #### Offline Installation http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml deleted file mode 100755 index e2e6cdd..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ /dev/null @@ -1,29 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>ELASTICSEARCH</name> - <version>2.3.3</version> - <extends>common-services/ELASTICSEARCH/2.3.3</extends> - </service> - </services> -</metainfo> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml deleted file mode 100644 index 0a9caac..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<reposinfo> - <os family="redhat6"> - <repo> - <baseurl>https://packages.elastic.co/elasticsearch/2.x/centos</baseurl> - <repoid>elastic-2.x</repoid> - <reponame>ELASTICSEARCH</reponame> - </repo> - </os> - <os family="redhat7"> - <repo> - <baseurl>https://packages.elastic.co/elasticsearch/2.x/centos</baseurl> - <repoid>elasticsearch-2.x</repoid> - <reponame>ELASTICSEARCH</reponame> - </repo> - </os> -</reposinfo> - http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml new file mode 100755 index 0000000..accf7da --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -0,0 +1,29 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ELASTICSEARCH</name> + <version>5.6.2</version> + <extends>common-services/ELASTICSEARCH/5.6.2</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml new file mode 100644 index 0000000..2c64518 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + </repo> + </os> +</reposinfo> + http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml deleted file mode 100755 index 494f71b..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml +++ /dev/null @@ -1,29 +0,0 @@ -<?xml version="1.0"?> -<!-- - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>KIBANA</name> - <extends>common-services/KIBANA/4.5.1</extends> - </service> - </services> -</metainfo> - http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json deleted file mode 100755 index 622a512..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/quicklinks/quicklinks.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "default", - "description": "default quick links configuration", - "configuration": { - "protocol": - { - "type":"HTTP_ONLY" - }, - - "links": [ - { - "name": "kibana_master_ui", - "label": "Metron Dashboard", - "requires_user_name": "false", - "url":"%@://%@:%@/", - "port":{ - "http_property": "kibana_server_port", - "http_default_port": "5601", - "https_property": "kibana_server_port", - "https_default_port": "5601", - "regex": "^(\\d+)$", - "site": "kibana-env" - } - } - ] - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml deleted file mode 100644 index b05c2cf..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<reposinfo> - <os family="redhat6"> - <repo> - <baseurl>http://packages.elastic.co/kibana/4.5/centos</baseurl> - <repoid>kibana-4.x</repoid> - <reponame>KIBANA</reponame> - </repo> - <repo> - <baseurl>http://packages.elastic.co/curator/4/centos/6</baseurl> - <repoid>ES-Curator-4.x</repoid> - <reponame>CURATOR</reponame> - </repo> - </os> - <os family="redhat7"> - <repo> - <baseurl>http://packages.elastic.co/kibana/4.5/centos</baseurl> - <repoid>kibana-4.x</repoid> - <reponame>KIBANA</reponame> - </repo> - <repo> - <baseurl>http://packages.elastic.co/curator/4/centos/7</baseurl> - <repoid>ES-Curator-4.x</repoid> - <reponame>CURATOR</reponame> - </repo> - </os> -</reposinfo> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml new file mode 100755 index 0000000..8a4fba2 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml @@ -0,0 +1,30 @@ +<?xml version="1.0"?> +<!-- + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>KIBANA</name> + <version>5.6.2</version> + <extends>common-services/KIBANA/5.6.2</extends> + </service> + </services> +</metainfo> + http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json new file mode 100755 index 0000000..622a512 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json @@ -0,0 +1,27 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "kibana_master_ui", + "label": "Metron Dashboard", + "requires_user_name": "false", + "url":"%@://%@:%@/", + "port":{ + "http_property": "kibana_server_port", + "http_default_port": "5601", + "https_property": "kibana_server_port", + "https_default_port": "5601", + "regex": "^(\\d+)$", + "site": "kibana-env" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml new file mode 100644 index 0000000..d0bd284 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + </repo> + <repo> + <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + </repo> + <repo> + <baseurl>http://packages.elastic.co/curator/5/centos/7</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml deleted file mode 100755 index a4de039..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml +++ /dev/null @@ -1,67 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration> - <property> - <name>elastic_user</name> - <value>elasticsearch</value> - <property-type>USER</property-type> - <description>Service user for Elasticsearch</description> - <value-attributes> - <type>user</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>elastic_group</name> - <value>elasticsearch</value> - <property-type>GROUP</property-type> - <description>Service group for Elasticsearch</description> - <value-attributes> - <type>user</type> - <overridable>false</overridable> - </value-attributes> - </property> - <property> - <name>elastic_log_dir</name> - <value>/var/log/elasticsearch</value> - <description>Log directory for elastic</description> - </property> - <property> - <name>elastic_pid_dir</name> - <value>/var/run/elasticsearch</value> - <description>The directory for pid files</description> - </property> - - <!-- elasticsearch-env.sh --> - <property> - <name>content</name> - <description>This is the jinja template for elastic-env.sh file</description> - <value> -#!/bin/bash - -# Set ELASTICSEARCH specific environment variables here. - -# The java implementation to use. -export JAVA_HOME={{java64_home}} -export PATH=$PATH:$JAVA_HOME/bin - </value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml deleted file mode 100755 index 7369f92..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ /dev/null @@ -1,203 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<!-- Elastic search Configurations --> - -<configuration supports_final="true"> - <!-- Configurations --> - <property> - <name>cluster_name</name> - <value>metron</value> - <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description> - </property> - <property> - <name>masters_also_are_datanodes</name> - <value>"false"</value> - <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description> - <value-attributes> - <type>string</type> - </value-attributes> - </property> - <property> - <name>zen_discovery_ping_unicast_hosts</name> - <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> - <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.--> - <value></value> - <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description> - </property> - <property> - <name>index_number_of_shards</name> - <value>4</value> - <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description> - </property> - <property> - <name>index_number_of_replicas</name> - <value>2</value> - <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description> - </property> - <property> - <name>path_data</name> - <value>"/opt/lmm/es_data"</value> - <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description> - </property> - <property> - <name>http_cors_enabled</name> - <value>"false"</value> - <description>Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false.</description> - <value-attributes> - <type>string</type> - </value-attributes> - </property> - <property> - <name>http_port</name> - <value>9200-9300</value> - <description>Set a custom port to listen for HTTP traffic</description> - </property> - <property> - <name>transport_tcp_port</name> - <value>9300-9400</value> - <description>Set a custom port for the node to node communication</description> - </property> - <!-- Multi-node Discovery --> - <property> - <name>discovery_zen_ping_multicast_enabled</name> - <value>false</value> - <description>Whether to use multicast</description> - </property> - <property> - <name>discovery_zen_ping_timeout</name> - <value>3s</value> - <description>Wait for ping responses for master discovery</description> - </property> - <property> - <name>discovery_zen_fd_ping_interval</name> - <value>15s</value> - <description>Wait for ping for cluster discovery</description> - </property> - <property> - <name>discovery_zen_fd_ping_timeout</name> - <value>60s</value> - <description>Wait for ping for cluster discovery</description> - </property> - <property> - <name>discovery_zen_fd_ping_retries</name> - <value>5</value> - <description>Number of ping retries before blacklisting</description> - </property> - <!-- Gateway --> - <property> - <name>gateway_recover_after_data_nodes</name> - <value>3</value> - <description>Recover as long as this many data or master nodes have joined the cluster.</description> - </property> - <property> - <name>recover_after_time</name> - <value>15m</value> - <description>recover_after_time</description> - </property> - <property> - <name>expected_data_nodes</name> - <value>0</value> - <description>expected_data_nodes</description> - </property> - <!-- Index --> - <property> - <name>index_merge_scheduler_max_thread_count</name> - <value>5</value> - <description>index.merge.scheduler.max_thread_count</description> - </property> - <property> - <name>indices_memory_index_store_throttle_type</name> - <value>none</value> - <description>index_store_throttle_type</description> - </property> - <property> - <name>index_refresh_interval</name> - <value>1s</value> - <description>index refresh interval</description> - </property> - <property> - <name>index_translog_flush_threshold_size</name> - <value>5g</value> - <description>index_translog_flush_threshold_size</description> - </property> - <property> - <name>indices_memory_index_buffer_size</name> - <value>10%</value> - <description>Percentage of heap used for write buffers</description> - </property> - <property> - <name>bootstrap_mlockall</name> - <value>true</value> - <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description> - </property> - <property> - <name>threadpool_bulk_queue_size</name> - <value>3000</value> - <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description> - </property> - <property> - <name>threadpool_index_queue_size</name> - <value>1000</value> - <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description> - </property> - <property> - <name>indices_cluster_send_refresh_mapping</name> - <value>false</value> - <description>In order to make the index request more efficient, we have set this property on our data nodes</description> - </property> - <property> - <name>indices_fielddata_cache_size</name> - <value>25%</value> - <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description> - </property> - <property> - <name>cluster_routing_allocation_disk_watermark_high</name> - <value>0.99</value> - <description>Property used when multiple drives are used to understand max thresholds</description> - </property> - <property> - <name>cluster_routing_allocation_disk_threshold_enabled</name> - <value>true</value> - <description>Property used when multiple drives are used to understand if thresholding is active</description> - </property> - <property> - <name>cluster_routing_allocation_disk_watermark_low</name> - <value>.97</value> - <description>Property used when multiple drives are used to understand min thresholds</description> - </property> - <property> - <name>cluster_routing_allocation_node_concurrent_recoveries</name> - <value>4</value> - <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description> - </property> - <property> - <name>network_host</name> - <value>[ _local_, _site_ ]</value> - <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> - </property> - <property> - <name>network_publish_host</name> - <value></value> - <value-attributes> - <empty-value-valid>true</empty-value-valid> - </value-attributes> - <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml deleted file mode 100755 index 44ecf05..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml +++ /dev/null @@ -1,104 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration> - <property> - <name>elastic_home</name> - <value>/usr/share/elasticsearch/</value> - <description>Elasticsearch Home Directory</description> - </property> - <property> - <name>data_dir</name> - <value>/var/lib/elasticsearch/</value> - <description>Elasticsearch Data Directory</description> - </property> - <property> - <name>work_dir</name> - <value>/tmp/elasticsearch/</value> - <description>Elasticsearch Work Directory</description> - </property> - <property> - <name>conf_dir</name> - <value>/etc/elasticsearch/</value> - <description>Elasticsearch Configuration Directory</description> - </property> - <property> - <name>heap_size</name> - <value>128m</value> - <description>Heap size</description> - </property> - <property> - <name>max_open_files</name> - <value>65535</value> - <description>Maximum number of open files</description> - </property> - <property> - <name>max_map_count</name> - <value>262144</value> - <description>Maximum number of memory map areas for process</description> - </property> - - <!--/etc/sysconfig/elasticsearch--> - <property> - <name>content</name> - <description>This is the jinja template for elastic sysconfig file (/etc/sysconfig/elasticsearch)</description> - <value> -# Directory where the Elasticsearch binary distribution resides -ES_HOME={{elastic_home}} - -# Heap Size (defaults to 256m min, 1g max) -ES_HEAP_SIZE={{heap_size}} - -# Maximum number of open files -MAX_OPEN_FILES={{max_open_files}} - -# Maximum number of VMA (Virtual Memory Areas) a process can own -MAX_MAP_COUNT={{max_map_count}} - -# Elasticsearch log directory -LOG_DIR={{log_dir}} - -# Elasticsearch data directory -DATA_DIR={{data_dir}} - -# Elasticsearch work directory -WORK_DIR={{work_dir}} - -# Elasticsearch conf directory -CONF_DIR={{conf_dir}} - -# User to run as, change this to a specific elasticsearch user if possible -# Also make sure, this user can write into the log directories in case you change them -# This setting only works for the init script, but has to be configured separately for systemd startup -ES_USER={{elastic_user}} - -# Elasticsearch pid directory -PID_DIR={{pid_dir}} - -# JAVA_HOME must be provided here for OS that use systemd service launch -JAVA_HOME={{java64_home}} - -# Additional Java OPTS -ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \ --XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \ --XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8" - </value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml deleted file mode 100755 index 4373e38..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ /dev/null @@ -1,79 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<metainfo> - <schemaVersion>2.0</schemaVersion> - <services> - <service> - <name>ELASTICSEARCH</name> - <displayName>Elasticsearch</displayName> - <comment>Indexing and Search</comment> - <version>2.3.3</version> - <components> - <component> - <name>ES_MASTER</name> - <displayName>Elasticsearch Master</displayName> - <category>MASTER</category> - <cardinality>1+</cardinality> - <commandScript> - <script>scripts/elastic_master.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - <component> - <name>ES_SLAVE</name> - <displayName>Elasticsearch Data Node</displayName> - <category>SLAVE</category> - <cardinality>0+</cardinality> - <commandScript> - <script>scripts/elastic_slave.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </component> - </components> - <osSpecifics> - <osSpecific> - <osFamily>any</osFamily> - <packages> - <package> - <name>elasticsearch-2.3.3</name> - </package> - </packages> - </osSpecific> - </osSpecifics> - <commandScript> - <script>scripts/service_check.py</script> - <scriptType>PYTHON</scriptType> - <timeout>300</timeout> - </commandScript> - <configuration-dependencies> - <config-type>elastic-env</config-type> - <config-type>elastic-site</config-type> - <config-type>elastic-sysconfig</config-type> - </configuration-dependencies> - <restartRequiredAfterChange>true</restartRequiredAfterChange> - <quickLinksConfigurations> - <quickLinksConfiguration> - <fileName>quicklinks.json</fileName> - <default>true</default> - </quickLinksConfiguration> - </quickLinksConfigurations> - </service> - </services> -</metainfo> http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py deleted file mode 100755 index c44d5ef..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Directory -from resource_management.core.resources.system import File -from resource_management.core.source import InlineTemplate -from resource_management.core.source import Template -from resource_management.core.resources import User - -def elastic(): - import params - - User(params.elastic_user, action = "create", groups = params.elastic_group) - params.path_data = params.path_data.replace('"', '') - data_path = params.path_data.replace(' ', '').split(',') - data_path[:] = [x.replace('"', '') for x in data_path] - - directories = [params.log_dir, params.pid_dir, params.conf_dir] - directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)] - - Directory(directories, - create_parents=True, - mode=0755, - owner=params.elastic_user, - group=params.elastic_group - ) - - print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir) - File("{0}/elastic-env.sh".format(params.conf_dir), - owner=params.elastic_user, - group=params.elastic_group, - content=InlineTemplate(params.elastic_env_sh_template) - ) - - configurations = params.config['configurations']['elastic-site'] - - print "Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir) - File("{0}/elasticsearch.yml".format(params.conf_dir), - content=Template( - "elasticsearch.master.yaml.j2", - configurations=configurations), - owner=params.elastic_user, - group=params.elastic_group - ) - - print "Master sysconfig: /etc/sysconfig/elasticsearch" - File("/etc/sysconfig/elasticsearch", - owner="root", - group="root", - content=InlineTemplate(params.sysconfig_template) - ) http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py deleted file mode 100755 index 3cd63c3..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script -from resource_management.core.logger import Logger - -from elastic import elastic - - -class Elasticsearch(Script): - def install(self, env): - import params - env.set_params(params) - Logger.info('Install ES Master Node') - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - - elastic() - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - stop_cmd = "service elasticsearch stop" - print 'Stop the Master' - Execute(stop_cmd) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - - self.configure(env) - start_cmd = "service elasticsearch start" - print 'Start the Master' - Execute(start_cmd) - - def status(self, env): - import params - env.set_params(params) - status_cmd = "service elasticsearch status" - print 'Status of the Master' - Execute(status_cmd) - - def restart(self, env): - import params - env.set_params(params) - self.configure(env) - restart_cmd = "service elasticsearch restart" - print 'Restarting the Master' - Execute(restart_cmd) - - -if __name__ == "__main__": - Elasticsearch().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py deleted file mode 100755 index 633ddd9..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.core.resources.system import Execute -from resource_management.libraries.script import Script -from resource_management.core.logger import Logger - -from slave import slave - - -class Elasticsearch(Script): - def install(self, env): - import params - env.set_params(params) - Logger.info('Install ES Data Node') - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - slave() - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - stop_cmd = "service elasticsearch stop" - print 'Stop the Slave' - Execute(stop_cmd) - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - self.configure(env) - start_cmd = "service elasticsearch start" - print 'Start the Slave' - Execute(start_cmd) - - def status(self, env): - import params - env.set_params(params) - status_cmd = "service elasticsearch status" - print 'Status of the Slave' - Execute(status_cmd) - - def restart(self, env): - import params - env.set_params(params) - self.configure(env) - restart_cmd = "service elasticsearch restart" - print 'Restarting the Slave' - Execute(restart_cmd) - - -if __name__ == "__main__": - Elasticsearch().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/e8213918/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py deleted file mode 100755 index e7b8d85..0000000 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.script import Script - -# server configurations -config = Script.get_config() - -masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes'] -elastic_home = config['configurations']['elastic-sysconfig']['elastic_home'] -data_dir = config['configurations']['elastic-sysconfig']['data_dir'] -work_dir = config['configurations']['elastic-sysconfig']['work_dir'] -conf_dir = config['configurations']['elastic-sysconfig']['conf_dir'] -heap_size = config['configurations']['elastic-sysconfig']['heap_size'] -max_open_files = config['configurations']['elastic-sysconfig']['max_open_files'] -max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] - -elastic_user = config['configurations']['elastic-env']['elastic_user'] -elastic_group = config['configurations']['elastic-env']['elastic_group'] -log_dir = config['configurations']['elastic-env']['elastic_log_dir'] -pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] - -hostname = config['hostname'] -java64_home = config['hostLevelParams']['java_home'] -elastic_env_sh_template = config['configurations']['elastic-env']['content'] -sysconfig_template = config['configurations']['elastic-sysconfig']['content'] - -cluster_name = config['configurations']['elastic-site']['cluster_name'] -zen_discovery_ping_unicast_hosts = config['configurations']['elastic-site']['zen_discovery_ping_unicast_hosts'] - -path_data = config['configurations']['elastic-site']['path_data'] -http_cors_enabled = config['configurations']['elastic-site']['http_cors_enabled'] -http_port = config['configurations']['elastic-site']['http_port'] -transport_tcp_port = config['configurations']['elastic-site']['transport_tcp_port'] - -recover_after_time = config['configurations']['elastic-site']['recover_after_time'] -gateway_recover_after_data_nodes = config['configurations']['elastic-site']['gateway_recover_after_data_nodes'] -expected_data_nodes = config['configurations']['elastic-site']['expected_data_nodes'] -discovery_zen_ping_multicast_enabled = config['configurations']['elastic-site']['discovery_zen_ping_multicast_enabled'] -index_merge_scheduler_max_thread_count = config['configurations']['elastic-site']['index_merge_scheduler_max_thread_count'] -index_translog_flush_threshold_size = config['configurations']['elastic-site']['index_translog_flush_threshold_size'] -index_refresh_interval = config['configurations']['elastic-site']['index_refresh_interval'] -indices_memory_index_store_throttle_type = config['configurations']['elastic-site']['indices_memory_index_store_throttle_type'] -index_number_of_shards = config['configurations']['elastic-site']['index_number_of_shards'] -index_number_of_replicas = config['configurations']['elastic-site']['index_number_of_replicas'] -indices_memory_index_buffer_size = config['configurations']['elastic-site']['indices_memory_index_buffer_size'] -bootstrap_mlockall = config['configurations']['elastic-site']['bootstrap_mlockall'] -threadpool_bulk_queue_size = config['configurations']['elastic-site']['threadpool_bulk_queue_size'] -cluster_routing_allocation_node_concurrent_recoveries = config['configurations']['elastic-site']['cluster_routing_allocation_node_concurrent_recoveries'] -cluster_routing_allocation_disk_watermark_low = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_low'] -cluster_routing_allocation_disk_threshold_enabled = config['configurations']['elastic-site']['cluster_routing_allocation_disk_threshold_enabled'] -cluster_routing_allocation_disk_watermark_high = config['configurations']['elastic-site']['cluster_routing_allocation_disk_watermark_high'] -indices_fielddata_cache_size = config['configurations']['elastic-site']['indices_fielddata_cache_size'] -indices_cluster_send_refresh_mapping = config['configurations']['elastic-site']['indices_cluster_send_refresh_mapping'] -threadpool_index_queue_size = config['configurations']['elastic-site']['threadpool_index_queue_size'] - -discovery_zen_ping_timeout = config['configurations']['elastic-site']['discovery_zen_ping_timeout'] -discovery_zen_fd_ping_interval = config['configurations']['elastic-site']['discovery_zen_fd_ping_interval'] -discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discovery_zen_fd_ping_timeout'] -discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] - -network_host = config['configurations']['elastic-site']['network_host'] -network_publish_host = config['configurations']['elastic-site']['network_publish_host']
