METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0ab39a32 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0ab39a32 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0ab39a32 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 0ab39a32b61104812a6a9e69e999e7053e3e7e90 Parents: 0d847cf Author: mmiklavc <michael.miklav...@gmail.com> Authored: Thu Apr 5 09:07:48 2018 -0600 Committer: Michael Miklavcic <michael.miklav...@gmail.com> Committed: Thu Apr 5 09:07:48 2018 -0600 ---------------------------------------------------------------------- .../roles/ambari_master/defaults/main.yml | 2 + .../ambari_master/tasks/elasticsearch_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/main.yml | 3 +- .../roles/ambari_master/tasks/metron_mpack.yml | 26 ++ .../ansible/roles/ambari_master/tasks/mpack.yml | 26 -- .../roles/load_web_templates/tasks/main.yml | 2 +- .../manual-install/Manual_Install_CentOS6.md | 4 +- metron-deployment/packaging/ambari/README.md | 25 +- .../ambari/elasticsearch-mpack/README.md | 62 +++++ .../ambari/elasticsearch-mpack/pom.xml | 95 +++++++ .../src/main/assemblies/elasticsearch-mpack.xml | 43 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml | 29 ++ .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 ++++ .../addon-services/KIBANA/5.6.2/metainfo.xml | 30 +++ .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 ++ .../KIBANA/5.6.2/repos/repoinfo.xml | 60 +++++ .../5.6.2/configuration/elastic-env.xml | 86 ++++++ .../5.6.2/configuration/elastic-jvm-options.xml | 144 ++++++++++ .../5.6.2/configuration/elastic-site.xml | 198 ++++++++++++++ .../5.6.2/configuration/elastic-sysconfig.xml | 97 +++++++ .../5.6.2/configuration/elastic-systemd.xml | 30 +++ .../ELASTICSEARCH/5.6.2/metainfo.xml | 97 +++++++ .../5.6.2/package/scripts/elastic_commands.py | 266 +++++++++++++++++++ .../5.6.2/package/scripts/elastic_master.py | 72 +++++ .../5.6.2/package/scripts/elastic_slave.py | 71 +++++ .../5.6.2/package/scripts/params.py | 108 ++++++++ .../5.6.2/package/scripts/properties_config.py | 34 +++ .../5.6.2/package/scripts/service_check.py | 114 ++++++++ .../5.6.2/package/scripts/status_params.py | 27 ++ .../templates/elasticsearch.master.yaml.j2 | 77 ++++++ .../templates/elasticsearch.slave.yaml.j2 | 78 ++++++ .../templates/elasticsearch_limits.conf.j2 | 20 ++ .../5.6.2/quicklinks/quicklinks.json | 43 +++ .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 + .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 +++++ .../KIBANA/5.6.2/configuration/kibana-site.xml | 113 ++++++++ .../common-services/KIBANA/5.6.2/metainfo.xml | 84 ++++++ .../KIBANA/5.6.2/package/scripts/common.py | 56 ++++ .../5.6.2/package/scripts/kibana_master.py | 81 ++++++ .../KIBANA/5.6.2/package/scripts/params.py | 50 ++++ .../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 ++ .../src/main/resources/mpack.json | 76 ++++++ .../packaging/ambari/metron-mpack/README.md | 20 +- .../src/main/assemblies/metron-mpack.xml | 14 - .../ELASTICSEARCH/5.6.2/metainfo.xml | 29 -- .../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 45 ---- .../addon-services/KIBANA/5.6.2/metainfo.xml | 30 --- .../KIBANA/5.6.2/quicklinks/quicklinks.json | 27 -- .../KIBANA/5.6.2/repos/repoinfo.xml | 60 ----- .../5.6.2/configuration/elastic-env.xml | 86 ------ .../5.6.2/configuration/elastic-jvm-options.xml | 144 ---------- .../5.6.2/configuration/elastic-site.xml | 198 -------------- .../5.6.2/configuration/elastic-sysconfig.xml | 97 ------- .../5.6.2/configuration/elastic-systemd.xml | 30 --- .../ELASTICSEARCH/5.6.2/metainfo.xml | 97 ------- .../5.6.2/package/scripts/elastic_commands.py | 266 ------------------- .../5.6.2/package/scripts/elastic_master.py | 72 ----- .../5.6.2/package/scripts/elastic_slave.py | 71 ----- .../5.6.2/package/scripts/params.py | 108 -------- .../5.6.2/package/scripts/properties_config.py | 34 --- .../5.6.2/package/scripts/service_check.py | 114 -------- .../5.6.2/package/scripts/status_params.py | 27 -- .../templates/elasticsearch.master.yaml.j2 | 77 ------ .../templates/elasticsearch.slave.yaml.j2 | 78 ------ .../templates/elasticsearch_limits.conf.j2 | 20 -- .../5.6.2/quicklinks/quicklinks.json | 43 --- .../ELASTICSEARCH/5.6.2/role_command_order.json | 8 - .../KIBANA/5.6.2/configuration/kibana-env.xml | 72 ----- .../KIBANA/5.6.2/configuration/kibana-site.xml | 113 -------- .../common-services/KIBANA/5.6.2/metainfo.xml | 94 ------- .../KIBANA/5.6.2/package/scripts/common.py | 56 ---- .../5.6.2/package/scripts/dashboard/__init__.py | 16 -- .../scripts/dashboard/dashboard-bulkload.json | 88 ------ .../package/scripts/dashboard/dashboardindex.py | 95 ------- .../package/scripts/dashboard/kibana.template | 233 ---------------- .../5.6.2/package/scripts/kibana_master.py | 119 --------- .../KIBANA/5.6.2/package/scripts/params.py | 50 ---- .../KIBANA/5.6.2/quicklinks/quicklinks.json | 28 -- .../common-services/METRON/CURRENT/metainfo.xml | 9 + .../package/scripts/dashboard/__init__.py | 16 ++ .../scripts/dashboard/dashboard-bulkload.json | 88 ++++++ .../package/scripts/dashboard/dashboardindex.py | 95 +++++++ .../package/scripts/dashboard/kibana.template | 233 ++++++++++++++++ .../CURRENT/package/scripts/indexing_master.py | 32 +++ metron-deployment/pom.xml | 1 + 85 files changed, 3082 insertions(+), 2786 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/defaults/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml index 0719b7f..52d220a 100644 --- a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml +++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml @@ -19,3 +19,5 @@ ambari_server_mem: 2048 ambari_mpack_version: 0.4.3.0 metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}" +elasticsearch_mpack_name: elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz +elasticsearch_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/elasticsearch-mpack/target/{{ elasticsearch_mpack_name }}" http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml new file mode 100644 index 0000000..7ce1a13 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/tasks/elasticsearch_mpack.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Copy Elasticsearch MPack to Ambari Host + copy: + src: "{{ elasticsearch_mpack_path }}" + dest: /tmp + +- name: Install Elasticsearch MPack on Ambari Host + shell: ambari-server install-mpack --mpack=/tmp/elasticsearch_mpack-{{ ambari_mpack_version }}.tar.gz + args: + creates: /var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-{{ ambari_mpack_version }}/addon-services http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml index 77a2e74..b6c351d 100644 --- a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml +++ b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml @@ -16,7 +16,8 @@ # --- - include: ambari.yml -- include: mpack.yml +- include: metron_mpack.yml +- include: elasticsearch_mpack.yml - name: start ambari server service: http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml new file mode 100644 index 0000000..a44ea17 --- /dev/null +++ b/metron-deployment/ansible/roles/ambari_master/tasks/metron_mpack.yml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +- name: Copy Metron MPack to Ambari Host + copy: + src: "{{ metron_mpack_path }}" + dest: /tmp + +- name: Install Metron MPack on Ambari Host + shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz + args: + creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml deleted file mode 100644 index 16289bb..0000000 --- a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -- name: Copy MPack to Ambari Host - copy: - src: "{{ metron_mpack_path }}" - dest: /tmp - -- name: Install MPack on Ambari Host - shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz - args: - creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml index 3a91960..55927c3 100644 --- a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml +++ b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml @@ -17,7 +17,7 @@ --- - name: Load Kibana Dashboard command: > - curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests + curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "{{ metron_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests args: warn: off register: result http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md ---------------------------------------------------------------------- diff --git a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md index e1305b6..1631fac 100644 --- a/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md +++ b/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.md @@ -878,11 +878,11 @@ curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d - Load Kibana Dashboard with: ``` -curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests +curl -s -w "%{http_code}" -u <USERNAME>:<PASSWORD> -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "<HOSTNAME>"}]}' http://<AMBARI HOST>:8080/api/v1/clusters/<CLUSTERNAME>/requests ``` For example: ``` -curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests +curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "KIBANA_DASHBOARD_INSTALL"},"Requests/resource_filters": [{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "metron"}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests ``` - If you installed Metron on a single node, you might have to increase the number of Storm supervisor slots from the default 2 to 5 or more. This can be done by editing the "supervisor.slots.ports" under Storm in the Ambari UI. http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/README.md b/metron-deployment/packaging/ambari/README.md index 82e1537..52c7570 100644 --- a/metron-deployment/packaging/ambari/README.md +++ b/metron-deployment/packaging/ambari/README.md @@ -30,13 +30,19 @@ limitations under the License. * [Upgrading MPack Services](#upgrading-mpack-services) ## Overview + Typically, Ambari Management Pack development will be done in the Vagrant environments. These instructions are specific to Vagrant, but can be adapted for other environemnts (e.g. make sure to be on the correct nodes for server vs agent files) -There is an `mpack.json` file which describes what services the mpack will contains, versions, etc. +There are two MPacks: + +* Metron - contains artifacts for deploying the Metron service +* Elasticsearch - contains artifacts for installing Elasticsearch and Kibana services -Alongside this are two directories, `addon-services` and `common-services`. +There is an `mpack.json` file for each which describes what services the mpack will contain, versions, etc. -The layout of `/common-services/METRON.CURRENT` is +Alongside this are two directories, `addon-services` and `common-services`. Below the Metron MPack is described, but this also applies similarly to the Elasticsearch MPack. + +The layout of `/common-services/METRON/CURRENT` is * `/configuration` * This contains a set of `*-env.xml` files, relevent to particular components or the service as a whole. These are where properties are defined. * `/package` @@ -59,11 +65,11 @@ The layout of `/common-services/METRON.CURRENT` is * `service_advisor.py` * Handles component layout and validation, along with handling some configurations for other services or that needs configs from other services. -The layout of `/addon-services/METRON.CURRENT` is +The layout of `/addon-services/METRON/CURRENT` is * `/repos` * Contains `repoinfo.xml` that defines repositories to install packages from * `metainfo.xml` - * Limited info version of `/common-services/METRON.CURRENT/metainfo.xml` + * Limited info version of `/common-services/METRON/CURRENT/metainfo.xml` * `role_command_order.json` * Defines the order of service startup and other actions relative to each other. @@ -361,6 +367,7 @@ Ambari stores the Python files from the service in a couple places. We'll want t Specifically, the server files live in ``` /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-0.4.0.0/common-services +/var/lib/ambari-server/resources/mpacks/elasticsearch-ambari.mpack-0.4.0.0/common-services /var/lib/ambari-agent/cache/common-services ``` @@ -409,6 +416,8 @@ After we've modified files in Ambari and the mpack is working, it is a good idea ``` 1. Install the mpack through Ambari as you normally would +1. The same steps can be followed for Elasticsearch and Kibana by similary deploying the ES MPack located in elasticsearch-mpack/target. + ## Configuration involving dependency services Metron can define expectations on other services, e.g. Storm's `topology.classpath` should be `/etc/hbase/conf:/etc/hadoop/conf`. This happens in `METRON.CURRENT/service_advisor.py`. @@ -468,10 +477,12 @@ This is checked in the indexing master ## Upgrading MPack Services -Apache Metron currently provides three services as part of its MPack +Apache Metron currently provides one service as part of its Metron MPack +* Metron + +Apache Metron currently provides two services as part of its Elasticsearch MPack * Elasticsearch * Kibana -* Metron There is currently no mechanism provided for multi-version or backwards compatibility. If you upgrade a service, e.g. Elasticsearch 2.x to 5.x, that is the only version that will be supported by Ambari via MPack. http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md b/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md new file mode 100644 index 0000000..e9a20cc --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/README.md @@ -0,0 +1,62 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +This provides a Management Pack (MPack) extension for [Apache Ambari](https://ambari.apache.org/) that simplifies the provisioning, management and monitoring of Elasticsearch and Kibana on clusters of any size. + +This allows you to easily install Elasticsearch and Kibana using a simple, guided process. This also allows you to monitor cluster health. + +### Prerequisites + +* Ambari 2.4.2+ + +### Quick Start + +1. Build the Elasticsearch MPack. Execute the following command from the project's root directory. + ``` + mvn clean package -Pmpack -DskipTests + ``` + +1. This results in the Mpack being produced at the following location. + ``` + metron-deployment/packaging/ambari/elasticsearch-mpack/target/elasticsearch_mpack-x.y.z.0.tar.gz + ``` + +1. Copy the tarball to the host where Ambari Server is installed. + +1. Ensure that Ambari Server is stopped. + +1. Install the MPack. + ``` + ambari-server install-mpack --mpack=elasticsearch_mpack-x.y.z.0.tar.gz --verbose + ``` + +1. Elasticsearch and Kibana will now be available as an installable service within Ambari. + +### Installation Notes + +The MPack will make all Elasticsearch services available in Ambari in the same manner as any other services in a stack. These can be installed using Ambari's user interface using "Add Services" or during an initial cluster install. + +#### Kerberization + +Elasticsearch does not provide free native Kerberos support. + +#### Limitations + +There are a few limitations that should be addressed to improve the Elasticsearch MPack installation. + +* The MPack does not support upgrades. http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml new file mode 100644 index 0000000..931ce63 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml @@ -0,0 +1,95 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.metron.packaging.mpacks</groupId> + <artifactId>elasticsearch_mpack</artifactId> + <version>0.4.3.0</version> + <name>Elasticsearch Ambari Management Pack</name> + + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-deployment</artifactId> + <version>0.4.3</version> + <relativePath>../../..</relativePath> + </parent> + + <dependencies> + </dependencies> + + <build> + <plugins> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>compile</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/target/</outputDirectory> + <resources> + <resource> + <directory>${basedir}/src/main/resources</directory> + <includes> + <include>mpack.json</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-jar-plugin</artifactId> + <version>${global_jar_version}</version> + <executions> + <execution> + <id>default-jar</id> + <!-- put the default-jar in the none phase to skip it from being created --> + <phase>none</phase> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>src/main/assemblies/elasticsearch-mpack.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml new file mode 100644 index 0000000..238b028 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/assemblies/elasticsearch-mpack.xml @@ -0,0 +1,43 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd";> + <id>archive</id> + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>src/main/resources/common-services</directory> + <outputDirectory>common-services</outputDirectory> + </fileSet> + <fileSet> + <directory>src/main/resources/addon-services</directory> + <outputDirectory>addon-services</outputDirectory> + </fileSet> + </fileSets> + <files> + <file> + <source>target/mpack.json</source> + </file> + </files> + <dependencySets> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml new file mode 100755 index 0000000..accf7da --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -0,0 +1,29 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ELASTICSEARCH</name> + <version>5.6.2</version> + <extends>common-services/ELASTICSEARCH/5.6.2</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml new file mode 100644 index 0000000..ba21fb1 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml @@ -0,0 +1,45 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + </repo> + </os> + <os family="ubuntu14"> + <!-- + see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+ + https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773 + https://issues.apache.org/jira/browse/AMBARI-21856 + --> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl> + <repoid>elasticsearch-5.x</repoid> + <reponame>ELASTICSEARCH</reponame> + <distribution>stable</distribution> + </repo> + </os> +</reposinfo> + http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml new file mode 100755 index 0000000..8a4fba2 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/metainfo.xml @@ -0,0 +1,30 @@ +<?xml version="1.0"?> +<!-- + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>KIBANA</name> + <version>5.6.2</version> + <extends>common-services/KIBANA/5.6.2</extends> + </service> + </services> +</metainfo> + http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json new file mode 100755 index 0000000..622a512 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/quicklinks/quicklinks.json @@ -0,0 +1,27 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "kibana_master_ui", + "label": "Metron Dashboard", + "requires_user_name": "false", + "url":"%@://%@:%@/", + "port":{ + "http_property": "kibana_server_port", + "http_default_port": "5601", + "https_property": "kibana_server_port", + "https_default_port": "5601", + "regex": "^(\\d+)$", + "site": "kibana-env" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml new file mode 100644 index 0000000..2755818 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml @@ -0,0 +1,60 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + </repo> + <repo> + <baseurl>http://packages.elastic.co/curator/5/centos/6</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/yum</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + </repo> + <repo> + <baseurl>http://packages.elastic.co/curator/5/centos/7</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + </repo> + </os> + <os family="ubuntu14"> + <!-- + see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+ + https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773 + https://issues.apache.org/jira/browse/AMBARI-21856 + --> + <repo> + <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl> + <repoid>kibana-5.x</repoid> + <reponame>KIBANA</reponame> + <distribution>stable</distribution> + </repo> + <repo> + <baseurl>https://packages.elastic.co/curator/5/debian</baseurl> + <repoid>ES-Curator-5.x</repoid> + <reponame>CURATOR</reponame> + <distribution>stable</distribution> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml new file mode 100755 index 0000000..9e4f8ad --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-env.xml @@ -0,0 +1,86 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>elastic_user</name> + <value>elasticsearch</value> + <property-type>USER</property-type> + <description>Service user for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>elastic_group</name> + <value>elasticsearch</value> + <property-type>GROUP</property-type> + <description>Service group for Elasticsearch</description> + <value-attributes> + <type>user</type> + <overridable>false</overridable> + </value-attributes> + </property> + <property> + <name>elastic_log_dir</name> + <value>/var/log/elasticsearch</value> + <description>Log directory for elastic</description> + </property> + <property> + <name>elastic_pid_dir</name> + <value>/var/run/elasticsearch</value> + <description>The directory for pid files</description> + </property> + <!-- elasticsearch-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for elastic-env.sh file</description> + <value> +#!/bin/bash + +# Set ELASTICSEARCH specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java64_home}} +export PATH=$PATH:$JAVA_HOME/bin + </value> + </property> + <property> + <name>elastic_user_nofile_limit</name> + <value>65536</value> + <description>Max open file limit for Elasticsearch user.</description> + </property> + <property> + <name>elastic_user_nproc_limit</name> + <value>2048</value> + <description>Max number of processes for Elasticsearch user.</description> + </property> + <property> + <name>elastic_user_memlock_soft_limit</name> + <value>unlimited</value> + <description>Max locked-in memory address space (soft memlock limit).</description> + </property> + <property> + <name>elastic_user_memlock_hard_limit</name> + <value>unlimited</value> + <description>Max locked-in memory address space (hard memlock limit).</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml new file mode 100644 index 0000000..5c6aaca --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml @@ -0,0 +1,144 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>heap_size</name> + <value>512m</value> + <description>JVM heap size</description> + </property> + <property> + <name>content</name> + <description>The jinja template for the Elasticsearch JVM options file.</description> + <value> +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{heap_size}} +-Xmx{{heap_size}} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM (remove on 32-bit client JVMs) +-server + +# explicitly set the stack size (reduce to 320k on 32-bit client JVMs) +-Xss1m + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# use old-style file permissions on JDK9 +-Djdk.io.permissionsUseCanonicalPath=true + +# flags to configure Netty +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# By default, the GC log file will not rotate. +# By uncommenting the lines below, the GC log file +# will be rotated every 128MB at most 32 times. +#-XX:+UseGCLogFileRotation +#-XX:NumberOfGCLogFiles=32 +#-XX:GCLogFileSize=128M + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml new file mode 100755 index 0000000..34df1e4 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml @@ -0,0 +1,198 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- Elastic search Configurations --> + +<configuration supports_final="true"> + <!-- Configurations --> + <property> + <name>cluster_name</name> + <value>metron</value> + <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description> + </property> + <property> + <name>masters_also_are_datanodes</name> + <value>"false"</value> + <description>ES Masters and Slaves cannot be installed on the same nodes. Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description> + <value-attributes> + <type>string</type> + </value-attributes> + </property> + <property> + <name>zen_discovery_ping_unicast_hosts</name> + <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> + <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.--> + <value></value> + <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description> + </property> + <property> + <name>index_number_of_shards</name> + <value>4</value> + <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description> + </property> + <property> + <name>index_number_of_replicas</name> + <value>2</value> + <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description> + </property> + <property> + <name>path_data</name> + <value>"/opt/lmm/es_data"</value> + <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description> + </property> + <property> + <name>http_cors_enabled</name> + <value>"false"</value> + <description>Enable or disable cross-origin resource sharing, i.e. whether a browser on another origin can do requests to Elasticsearch. Defaults to false.</description> + <value-attributes> + <type>string</type> + </value-attributes> + </property> + <property> + <name>http_port</name> + <value>9200-9300</value> + <description>Set a custom port to listen for HTTP traffic</description> + </property> + <property> + <name>transport_tcp_port</name> + <value>9300-9400</value> + <description>Set a custom port for the node to node communication</description> + </property> + <!-- Multi-node Discovery --> + <property> + <name>discovery_zen_ping_timeout</name> + <value>3s</value> + <description>Wait for ping responses for master discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_interval</name> + <value>15s</value> + <description>Wait for ping for cluster discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_timeout</name> + <value>60s</value> + <description>Wait for ping for cluster discovery</description> + </property> + <property> + <name>discovery_zen_fd_ping_retries</name> + <value>5</value> + <description>Number of ping retries before blacklisting</description> + </property> + <!-- Gateway --> + <property> + <name>gateway_recover_after_data_nodes</name> + <value>3</value> + <description>Recover as long as this many data or master nodes have joined the cluster.</description> + </property> + <property> + <name>recover_after_time</name> + <value>15m</value> + <description>recover_after_time</description> + </property> + <property> + <name>expected_data_nodes</name> + <value>0</value> + <description>expected_data_nodes</description> + </property> + <!-- Index --> + <property> + <name>index_merge_scheduler_max_thread_count</name> + <value>5</value> + <description>index.merge.scheduler.max_thread_count</description> + </property> + <property> + <name>indices_memory_index_store_throttle_type</name> + <value>none</value> + <description>index_store_throttle_type</description> + </property> + <property> + <name>index_refresh_interval</name> + <value>1s</value> + <description>index refresh interval</description> + </property> + <property> + <name>index_translog_flush_threshold_size</name> + <value>5g</value> + <description>index_translog_flush_threshold_size</description> + </property> + <property> + <name>indices_memory_index_buffer_size</name> + <value>10%</value> + <description>Percentage of heap used for write buffers</description> + </property> + <property> + <name>bootstrap_memory_lock</name> + <value>true</value> + <description>The third option on Linux/Unix systems only, is to use mlockall to try to lock the process address space into RAM, preventing any Elasticsearch memory from being swapped out</description> + </property> + <property> + <name>threadpool_bulk_queue_size</name> + <value>3000</value> + <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute a bulk request</description> + </property> + <property> + <name>threadpool_index_queue_size</name> + <value>1000</value> + <description>It tells ES the number of requests that can be queued for execution in the node when there is no thread available to execute index request</description> + </property> + <property> + <name>indices_cluster_send_refresh_mapping</name> + <value>false</value> + <description>In order to make the index request more efficient, we have set this property on our data nodes</description> + </property> + <property> + <name>indices_fielddata_cache_size</name> + <value>25%</value> + <description>You need to keep in mind that not setting this value properly can cause:Facet searches and sorting to have very poor performance:The ES node to run out of memory if you run the facet query against a large index</description> + </property> + <property> + <name>cluster_routing_allocation_disk_watermark_high</name> + <value>0.99</value> + <description>Property used when multiple drives are used to understand max thresholds</description> + </property> + <property> + <name>cluster_routing_allocation_disk_threshold_enabled</name> + <value>true</value> + <description>Property used when multiple drives are used to understand if thresholding is active</description> + </property> + <property> + <name>cluster_routing_allocation_disk_watermark_low</name> + <value>.97</value> + <description>Property used when multiple drives are used to understand min thresholds</description> + </property> + <property> + <name>cluster_routing_allocation_node_concurrent_recoveries</name> + <value>4</value> + <description>Max concurrent recoveries, useful for fast recovery of the cluster nodes on restart</description> + </property> + <property> + <name>network_host</name> + <value>[ _local_, _site_ ]</value> + <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + </property> + <property> + <name>network_publish_host</name> + <value>[]</value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml new file mode 100755 index 0000000..ea6ca38 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml @@ -0,0 +1,97 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>elastic_home</name> + <value>/usr/share/elasticsearch/</value> + <description>Elasticsearch Home Directory</description> + </property> + <property> + <name>data_dir</name> + <value>/var/lib/elasticsearch/</value> + <description>Elasticsearch Data Directory</description> + </property> + <property> + <name>work_dir</name> + <value>/tmp/elasticsearch/</value> + <description>Elasticsearch Work Directory</description> + </property> + <property> + <name>conf_dir</name> + <value>/etc/elasticsearch/</value> + <description>Elasticsearch Configuration Directory</description> + </property> + <property> + <name>max_open_files</name> + <value>65536</value> + <description>Maximum number of open files</description> + </property> + <property> + <name>max_map_count</name> + <value>262144</value> + <description>Maximum number of memory map areas for process</description> + </property> + + <!-- Elasticsearch sysconfig --> + <property> + <name>content</name> + <description>This is the jinja template for elastic sysconfig file</description> + <value> +# Directory where the Elasticsearch binary distribution resides +ES_HOME={{elastic_home}} + +# Maximum number of open files +MAX_OPEN_FILES={{max_open_files}} + +# Maximum number of VMA (Virtual Memory Areas) a process can own +MAX_MAP_COUNT={{max_map_count}} + +# Elasticsearch log directory +LOG_DIR={{log_dir}} + +# Elasticsearch data directory +DATA_DIR={{data_dir}} + +# Elasticsearch work directory +WORK_DIR={{work_dir}} + +# Elasticsearch conf directory +CONF_DIR={{conf_dir}} + +# User to run as, change this to a specific elasticsearch user if possible +# Also make sure, this user can write into the log directories in case you change them +# This setting only works for the init script, but has to be configured separately for systemd startup +ES_USER={{elastic_user}} + +# Elasticsearch pid directory +PID_DIR={{pid_dir}} + +# JAVA_HOME must be provided here for OS that use systemd service launch +JAVA_HOME={{java64_home}} + +# Additional Java options - now preferential to use 'jvm.options' file instead +ES_JAVA_OPTS="" + +# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html +MAX_LOCKED_MEMORY=unlimited + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml new file mode 100644 index 0000000..311e3c0 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml @@ -0,0 +1,30 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>content</name> + <description>The jinja template for the Elasticsearch systemd override file. Applies only to platforms that use systemd.</description> + <value> +[Service] +LimitMEMLOCK=infinity + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml new file mode 100755 index 0000000..47abb45 --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml @@ -0,0 +1,97 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>ELASTICSEARCH</name> + <displayName>Elasticsearch</displayName> + <comment>Indexing and Search</comment> + <version>5.6.2</version> + <components> + <component> + <name>ES_MASTER</name> + <displayName>Elasticsearch Master</displayName> + <category>MASTER</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/elastic_master.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + <component> + <name>ES_SLAVE</name> + <displayName>Elasticsearch Data Node</displayName> + <category>SLAVE</category> + <cardinality>0+</cardinality> + <commandScript> + <script>scripts/elastic_slave.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>redhat6</osFamily> + <packages> + <package> + <name>elasticsearch-5.6.2</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>redhat7</osFamily> + <packages> + <package> + <name>elasticsearch-5.6.2</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>ubuntu14</osFamily> + <packages> + <package> + <name>elasticsearch=5.6.2</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + <configuration-dependencies> + <config-type>elastic-env</config-type> + <config-type>elastic-site</config-type> + <config-type>elastic-sysconfig</config-type> + <config-type>elastic-systemd</config-type> + <config-type>elastic-jvm-options</config-type> + </configuration-dependencies> + <restartRequiredAfterChange>true</restartRequiredAfterChange> + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py new file mode 100644 index 0000000..618d10a --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +from ambari_commons.os_check import OSCheck +from resource_management.core.exceptions import ExecutionFailed +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.core.resources import User +from resource_management.core.logger import Logger +from resource_management.libraries.functions import format as ambari_format +from resource_management.libraries.functions.get_user_call_output import get_user_call_output + + +def service_check(cmd, user, label): + """ + Executes a SysV service check command that adheres to LSB-compliant + return codes. The return codes are interpreted as defined + by the LSB. + + See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html + for more information. + + :param cmd: The service check command to execute. + :param label: The name of the service. + """ + Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label)) + rc, out, err = get_user_call_output(cmd, user, is_checked_call=False) + + if rc in [1, 2, 3]: + # if return code in [1, 2, 3], then 'program is not running' or 'program is dead' + Logger.info("{0} is not running".format(label)) + raise ComponentIsNotRunning() + + elif rc == 0: + # if return code = 0, then 'program is running or service is OK' + Logger.info("{0} is running".format(label)) + + else: + # else service state is unknown + err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc) + Logger.error(err_msg) + raise ExecutionFailed(err_msg, rc, out, err) + +def is_systemd_running(): + """ + Determines if the platform is running Systemd. + :return True, if the platform is running Systemd. False, otherwise. + """ + Logger.info("Is the platform running Systemd?") + rc, out, err = get_user_call_output("pidof systemd", "root", is_checked_call=False) + if rc == 0: + Logger.info("Systemd was found") + return True + else: + Logger.info("Systemd was NOT found") + return False + +def configure_systemd(params): + """ + Configure Systemd for Elasticsearch. + """ + Logger.info("Configuring Systemd for Elasticsearch"); + + # ensure the systemd directory for elasticsearch overrides exists + Logger.info("Create Systemd directory for Elasticsearch: {0}".format(params.systemd_elasticsearch_dir)) + Directory(params.systemd_elasticsearch_dir, + create_parents=True, + owner='root', + group='root') + + # when using Elasticsearch packages on systems that use systemd, system + # limits must also be specified via systemd. + # see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd + Logger.info("Elasticsearch systemd limits: {0}".format(params.systemd_override_file)) + File(params.systemd_override_file, + content=InlineTemplate(params.systemd_override_template), + owner="root", + group="root") + + # reload the configuration + Execute("systemctl daemon-reload") + +def create_user(params): + """ + Creates the user required for Elasticsearch. + """ + Logger.info("Creating user={0} in group={1}".format(params.elastic_user, params.elastic_group)) + User(params.elastic_user, action = "create", groups = params.elastic_group) + +def create_directories(params, directories): + """ + Creates one or more directories. + """ + Logger.info("Creating directories: {0}".format(directories)) + Directory(directories, + create_parents=True, + mode=0755, + owner=params.elastic_user, + group=params.elastic_group + ) + +def create_elastic_env(params): + """ + Creates the Elasticsearch environment file. + """ + Logger.info("Create Elasticsearch environment file.") + File("{0}/elastic-env.sh".format(params.conf_dir), + owner=params.elastic_user, + group=params.elastic_group, + content=InlineTemplate(params.elastic_env_sh_template)) + +def create_elastic_site(params, template_name): + """ + Creates the Elasticsearch site file. + """ + Logger.info("Creating Elasticsearch site file; template={0}".format(template_name)) + + elastic_site = params.config['configurations']['elastic-site'] + path = "{0}/elasticsearch.yml".format(params.conf_dir) + template = Template(template_name, configurations=elastic_site) + File(path, + content=template, + owner=params.elastic_user, + group=params.elastic_group) + +def get_elastic_config_path(default="/etc/default/elasticsearch"): + """ + Defines the path to the Elasticsearch environment file. This path will + differ based on the OS family. + :param default: The path used if the OS family is not recognized. + """ + path = default + if OSCheck.is_redhat_family(): + path = "/etc/sysconfig/elasticsearch" + elif OSCheck.is_ubuntu_family(): + path = "/etc/default/elasticsearch" + else: + Logger.error("Unexpected OS family; using default path={0}".format(path)) + + return path + +def create_elastic_config(params): + """ + Creates the Elasticsearch system config file. Usually lands at either + /etc/sysconfig/elasticsearch or /etc/default/elasticsearch. + """ + path = get_elastic_config_path() + Logger.info("Creating the Elasticsearch system config; path={0}".format(path)) + File(path, owner="root", group="root", content=InlineTemplate(params.sysconfig_template)) + +def create_elastic_pam_limits(params): + """ + Creates the PAM limits for Elasticsearch. + """ + Logger.info("Creating Elasticsearch PAM limits.") + + # in some OS this folder may not exist, so create it + Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir)) + Directory(params.limits_conf_dir, + create_parents=True, + owner='root', + group='root') + + Logger.info("Creating Elasticsearch PAM limits; file={0}".format(params.limits_conf_file)) + File(params.limits_conf_file, + content=Template('elasticsearch_limits.conf.j2'), + owner="root", + group="root") + +def create_elastic_jvm_options(params): + """ + Defines the jvm.options file used to specify JVM options. + """ + path = "{0}/jvm.options".format(params.conf_dir) + Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path)) + File(path, + content=InlineTemplate(params.jvm_options_template), + owner=params.elastic_user, + group=params.elastic_group) + +def get_data_directories(params): + """ + Returns the directories to use for storing Elasticsearch data. + """ + path = params.path_data + path = path.replace('"', '') + path = path.replace(' ', '') + path = path.split(',') + dirs = [p.replace('"', '') for p in path] + + Logger.info("Elasticsearch data directories: dirs={0}".format(dirs)) + return dirs + +def configure_master(): + """ + Configures the Elasticsearch master node. + """ + import params + + # define the directories required + dirs = [ + params.log_dir, + params.pid_dir, + params.conf_dir, + "{0}/scripts".format(params.conf_dir) + ] + dirs += get_data_directories(params) + + # configure the elasticsearch master + create_user(params) + create_directories(params, dirs) + create_elastic_env(params) + create_elastic_site(params, "elasticsearch.master.yaml.j2") + create_elastic_config(params) + create_elastic_pam_limits(params) + create_elastic_jvm_options(params) + if is_systemd_running(): + configure_systemd(params) + +def configure_slave(): + """ + Configures the Elasticsearch slave node. + """ + import params + + # define the directories required + dirs = [ + params.log_dir, + params.pid_dir, + params.conf_dir, + ] + dirs += get_data_directories(params) + + # configure the elasticsearch slave + create_user(params) + create_directories(params, dirs) + create_elastic_env(params) + create_elastic_site(params, "elasticsearch.slave.yaml.j2") + create_elastic_config(params) + create_elastic_pam_limits(params) + create_elastic_jvm_options(params) + if is_systemd_running(): + configure_systemd(params) http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py new file mode 100755 index 0000000..142ce4e --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py @@ -0,0 +1,72 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core import shell +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger +from elastic_commands import service_check +from elastic_commands import configure_master + +class Elasticsearch(Script): + + def install(self, env): + import params + env.set_params(params) + Logger.info('Install Elasticsearch master node') + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + Logger.info('Configure Elasticsearch master node') + configure_master() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Stop Elasticsearch master node') + Execute("service elasticsearch stop") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Start Elasticsearch master node') + self.configure(env) + Execute("service elasticsearch start") + + def status(self, env): + import params + env.set_params(params) + Logger.info('Status check Elasticsearch master node') + service_check( + cmd="service elasticsearch status", + user=params.elastic_status_check_user, + label="Elasticsearch Master") + + def restart(self, env): + import params + env.set_params(params) + Logger.info('Restart Elasticsearch master node') + self.configure(env) + Execute("service elasticsearch restart") + + +if __name__ == "__main__": + Elasticsearch().execute() http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py new file mode 100755 index 0000000..2d559ff --- /dev/null +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py @@ -0,0 +1,71 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.script import Script + +from elastic_commands import service_check +from elastic_commands import configure_slave + +class Elasticsearch(Script): + + def install(self, env): + import params + env.set_params(params) + Logger.info('Install Elasticsearch slave node') + self.install_packages(env) + + def configure(self, env, upgrade_type=None, config_dir=None): + import params + env.set_params(params) + Logger.info('Configure Elasticsearch slave node') + configure_slave() + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + Logger.info('Stop Elasticsearch slave node') + Execute("service elasticsearch stop") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + Execute("service elasticsearch start") + + def status(self, env): + import params + env.set_params(params) + Logger.info('Status check Elasticsearch slave node') + service_check( + cmd="service elasticsearch status", + user=params.elastic_status_check_user, + label="Elasticsearch Slave") + + def restart(self, env): + import params + env.set_params(params) + Logger.info('Restart Elasticsearch slave node') + self.configure(env) + Execute("service elasticsearch restart") + + +if __name__ == "__main__": + Elasticsearch().execute()
