METRON-634 Mpack bug fixes and improvements (not related to singlenode install) (mattf-horton) closes apache/incubator-metron#532
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/df44f077 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/df44f077 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/df44f077 Branch: refs/heads/Metron_0.4.0 Commit: df44f077a97d0495fea511f5d9f900e4d3a25147 Parents: 29d7cb3 Author: mattf-horton <mfo...@hortonworks.com> Authored: Wed Apr 19 21:43:05 2017 -0700 Committer: mattf <ma...@apache.org> Committed: Wed Apr 19 21:43:05 2017 -0700 ---------------------------------------------------------------------- .../2.3.3/configuration/elastic-site.xml | 33 ++++++++------- .../2.3.3/configuration/elastic-sysconfig.xml | 8 +++- .../ELASTICSEARCH/2.3.3/metainfo.xml | 8 +++- .../2.3.3/package/scripts/elastic.py | 6 +-- .../2.3.3/package/scripts/elastic_master.py | 8 ++-- .../2.3.3/package/scripts/elastic_slave.py | 8 ++-- .../2.3.3/package/scripts/params.py | 4 +- .../2.3.3/quicklinks/quicklinks.json | 43 ++++++++++++++++++++ .../METRON/CURRENT/configuration/metron-env.xml | 2 +- 9 files changed, 87 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml index cbcebdd..fd29e63 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -24,7 +24,7 @@ <property> <name>cluster_name</name> <value>metron</value> - <description>Cluster name identifies your cluster</description> + <description>Elasticsearch Cluster Name identifies your Elasticsearch subsystem</description> </property> <property> <name>masters_also_are_datanodes</name> @@ -37,24 +37,24 @@ <property> <name>zen_discovery_ping_unicast_hosts</name> <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> + <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.--> <value></value> - <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format.</description> + <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format: eshost1,eshost2</description> </property> <property> <name>index_number_of_shards</name> <value>4</value> - <description>Set the number of shards (splits) of an index</description> + <description>Set the number of shards (splits) of an index. Changes are not effective after index creation. Usually set to 1 for single-node install.</description> </property> <property> <name>index_number_of_replicas</name> <value>2</value> - <description>Set the number of replicas (additional copies) of an index</description> + <description>Set the number of replicas (copies in addition to the first) of an index. Usually set to 0 for single-node install.</description> </property> - <!-- Logging Configurations --> <property> <name>path_data</name> <value>"/opt/lmm/es_data"</value> - <description>Path to directory where to store index data allocated for this node. e.g. "/mnt/first", "/mnt/second"</description> + <description>Comma-separated list of directories where to store index data allocated for each node: "/mnt/first","/mnt/second". Number of paths should relate to number of shards, and preferably should be on separate physical volumes.</description> </property> <property> <name>http_cors_enabled</name> @@ -64,22 +64,21 @@ <type>string</type> </value-attributes> </property> - <!-- Discovery --> - <property> - <name>transport_tcp_port</name> - <value>9300-9400</value> - <description>Set a custom port for the node to node communication</description> - </property> <property> <name>http_port</name> <value>9200-9300</value> <description>Set a custom port to listen for HTTP traffic</description> </property> - <!-- Discovery --> + <property> + <name>transport_tcp_port</name> + <value>9300-9400</value> + <description>Set a custom port for the node to node communication</description> + </property> + <!-- Multi-node Discovery --> <property> <name>discovery_zen_ping_multicast_enabled</name> <value>false</value> - <description>master eligible nodes</description> + <description>Whether to use multicast</description> </property> <property> <name>discovery_zen_ping_timeout</name> @@ -190,7 +189,7 @@ </property> <property> <name>network_host</name> - <value>_lo_,_eth0_</value> - <description>Network interface(s) will bind to. </description> + <value>"_lo:ipv4_","_eth0:ipv4_"</value> + <description>Network interface(s) ES will bind to within each node. Confirm names via ifconfig. Should have entries for primary external and loopback interfaces, with :ipv4 annotation and quote marks around each entry: "_lo:ipv4_","_eth0:ipv4_"</description> </property> -</configuration> \ No newline at end of file +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml index ae26498..44ecf05 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml @@ -58,7 +58,7 @@ <!--/etc/sysconfig/elasticsearch--> <property> <name>content</name> - <description>This is the jinja template for elastic-env.sh file</description> + <description>This is the jinja template for elastic sysconfig file (/etc/sysconfig/elasticsearch)</description> <value> # Directory where the Elasticsearch binary distribution resides ES_HOME={{elastic_home}} @@ -89,6 +89,12 @@ CONF_DIR={{conf_dir}} # This setting only works for the init script, but has to be configured separately for systemd startup ES_USER={{elastic_user}} +# Elasticsearch pid directory +PID_DIR={{pid_dir}} + +# JAVA_HOME must be provided here for OS that use systemd service launch +JAVA_HOME={{java64_home}} + # Additional Java OPTS ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \ -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \ http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml index 53909a2..4373e38 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml @@ -68,6 +68,12 @@ <config-type>elastic-sysconfig</config-type> </configuration-dependencies> <restartRequiredAfterChange>true</restartRequiredAfterChange> + <quickLinksConfigurations> + <quickLinksConfiguration> + <fileName>quicklinks.json</fileName> + <default>true</default> + </quickLinksConfiguration> + </quickLinksConfigurations> </service> </services> -</metainfo> \ No newline at end of file +</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py index 2e638de..c44d5ef 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py @@ -33,7 +33,7 @@ def elastic(): data_path[:] = [x.replace('"', '') for x in data_path] directories = [params.log_dir, params.pid_dir, params.conf_dir] - directories = directories + data_path + directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)] Directory(directories, create_parents=True, @@ -57,11 +57,11 @@ def elastic(): "elasticsearch.master.yaml.j2", configurations=configurations), owner=params.elastic_user, - group=params.elastic_user + group=params.elastic_group ) print "Master sysconfig: /etc/sysconfig/elasticsearch" - File(format("/etc/sysconfig/elasticsearch"), + File("/etc/sysconfig/elasticsearch", owner="root", group="root", content=InlineTemplate(params.sysconfig_template) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py index 706610e..3cd63c3 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py @@ -40,7 +40,7 @@ class Elasticsearch(Script): def stop(self, env, upgrade_type=None): import params env.set_params(params) - stop_cmd = format("service elasticsearch stop") + stop_cmd = "service elasticsearch stop" print 'Stop the Master' Execute(stop_cmd) @@ -49,14 +49,14 @@ class Elasticsearch(Script): env.set_params(params) self.configure(env) - start_cmd = format("service elasticsearch start") + start_cmd = "service elasticsearch start" print 'Start the Master' Execute(start_cmd) def status(self, env): import params env.set_params(params) - status_cmd = format("service elasticsearch status") + status_cmd = "service elasticsearch status" print 'Status of the Master' Execute(status_cmd) @@ -64,7 +64,7 @@ class Elasticsearch(Script): import params env.set_params(params) self.configure(env) - restart_cmd = format("service elasticsearch restart") + restart_cmd = "service elasticsearch restart" print 'Restarting the Master' Execute(restart_cmd) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py index 7f7f21a..633ddd9 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py @@ -39,7 +39,7 @@ class Elasticsearch(Script): def stop(self, env, upgrade_type=None): import params env.set_params(params) - stop_cmd = format("service elasticsearch stop") + stop_cmd = "service elasticsearch stop" print 'Stop the Slave' Execute(stop_cmd) @@ -47,14 +47,14 @@ class Elasticsearch(Script): import params env.set_params(params) self.configure(env) - start_cmd = format("service elasticsearch start") + start_cmd = "service elasticsearch start" print 'Start the Slave' Execute(start_cmd) def status(self, env): import params env.set_params(params) - status_cmd = format("service elasticsearch status") + status_cmd = "service elasticsearch status" print 'Status of the Slave' Execute(status_cmd) @@ -62,7 +62,7 @@ class Elasticsearch(Script): import params env.set_params(params) self.configure(env) - restart_cmd = format("service elasticsearch restart") + restart_cmd = "service elasticsearch restart" print 'Restarting the Slave' Execute(restart_cmd) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py index aea81fb..00929a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -35,8 +35,8 @@ max_map_count = config['configurations']['elastic-sysconfig']['max_map_count'] elastic_user = config['configurations']['elastic-env']['elastic_user'] elastic_group = config['configurations']['elastic-env']['elastic_group'] log_dir = config['configurations']['elastic-env']['elastic_log_dir'] -pid_dir = '/var/run/elasticsearch' -pid_file = '/var/run/elasticsearch/elasticsearch.pid' +pid_dir = config['configurations']['elastic-env']['elastic_pid_dir'] + hostname = config['hostname'] java64_home = config['hostLevelParams']['java_home'] elastic_env_sh_template = config['configurations']['elastic-env']['content'] http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json new file mode 100644 index 0000000..909828b --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/quicklinks/quicklinks.json @@ -0,0 +1,43 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"HTTP_ONLY" + }, + + "links": [ + { + "name": "es_health_link", + "label": "Elasticsearch Health", + "requires_user_name": "false", + "component_name": "ES_MASTER", + "url":"%@://%@:%@/_cat/health?v", + "port":{ + "http_property": "http_port", + "http_default_port": "9200", + "https_property": "http_port", + "https_default_port": "9200", + "regex": "^(\\d+)", + "site": "elastic-site" + } + }, + { + "name": "es_indices_link", + "label": "Elasticsearch Indexes", + "requires_user_name": "false", + "component_name": "ES_MASTER", + "url":"%@://%@:%@/_cat/indices?v", + "port":{ + "http_property": "http_port", + "http_default_port": "9200", + "https_property": "http_port", + "https_default_port": "9200", + "regex": "^(\\d+)", + "site": "elastic-site" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df44f077/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 90e0757..218e8e5 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -97,7 +97,7 @@ <property require-input="true"> <name>es_hosts</name> <value></value> - <description>Comma delimited list of Elasticsearch Hosts. (eshost1,eshost2)</description> + <description>Comma delimited list of Elasticsearch Master Hosts: eshost1,eshost2</description> <display-name>Elasticsearch Hosts</display-name> </property> <property>
