METRON-905 Fix square-bracket behavior and default network interface bindings for ES (mattf-horton) closes apache/incubator-metron#564
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/21b79dc8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/21b79dc8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/21b79dc8 Branch: refs/heads/Metron_0.4.0 Commit: 21b79dc8eb0ffd7f0a3ea87744cef3d792456af7 Parents: 876a5fc Author: mattf-horton <mfo...@hortonworks.com> Authored: Tue May 9 13:12:16 2017 -0700 Committer: mattf <ma...@apache.org> Committed: Tue May 9 13:12:16 2017 -0700 ---------------------------------------------------------------------- .../2.3.3/configuration/elastic-site.xml | 14 +++++++++++--- .../ELASTICSEARCH/2.3.3/package/scripts/params.py | 1 + .../package/templates/elasticsearch.master.yaml.j2 | 5 +++-- .../package/templates/elasticsearch.slave.yaml.j2 | 5 +++-- .../roles/ambari_config/vars/single_node_vm.yml | 4 ++-- .../roles/ambari_config/vars/small_cluster.yml | 4 ++-- 6 files changed, 22 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml index fd29e63..33e01bf 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml @@ -39,7 +39,7 @@ <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default). Unsure how to do this.--> <!--Also need to document whether should list masters only, or all ES nodes. I think this one is all nodes, but previous inline comment said Masters.--> <value></value> - <description>Unicast discovery list of hosts to act as gossip routers, in comma separated format: eshost1,eshost2</description> + <description>Unicast discovery list of hosts to act as gossip routers, comma-separated list with square brackets: [ eshost1, eshost2 ]</description> </property> <property> <name>index_number_of_shards</name> @@ -189,7 +189,15 @@ </property> <property> <name>network_host</name> - <value>"_lo:ipv4_","_eth0:ipv4_"</value> - <description>Network interface(s) ES will bind to within each node. Confirm names via ifconfig. Should have entries for primary external and loopback interfaces, with :ipv4 annotation and quote marks around each entry: "_lo:ipv4_","_eth0:ipv4_"</description> + <value>[ _local_ ]</value> + <description>Network interface(s) ES will bind to within each node. Use [ _local_ ] for single-node install; change to [ _local_, _site_ ] (including square brackets) for typical cluster install. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + </property> + <property> + <name>network_publish_host</name> + <value></value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> </property> </configuration> http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py index 00929a7..e7b8d85 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py @@ -77,3 +77,4 @@ discovery_zen_fd_ping_timeout = config['configurations']['elastic-site']['discov discovery_zen_fd_ping_retries = config['configurations']['elastic-site']['discovery_zen_fd_ping_retries'] network_host = config['configurations']['elastic-site']['network_host'] +network_publish_host = config['configurations']['elastic-site']['network_publish_host'] http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 index ca3630a..f0d2a8f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 @@ -30,7 +30,7 @@ discovery: multicast: enabled: {{ discovery_zen_ping_multicast_enabled }} unicast: - hosts: "{{zen_discovery_ping_unicast_hosts}}" + hosts: {{zen_discovery_ping_unicast_hosts}} node: data: {{ masters_also_are_datanodes }} @@ -81,4 +81,5 @@ discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} -network.host: {{network_host}} \ No newline at end of file +network.host: {{network_host}} +network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 index 58ce3d3..7d2d0cf 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.slave.yaml.j2 @@ -30,7 +30,7 @@ discovery: multicast: enabled: {{discovery_zen_ping_multicast_enabled}} unicast: - hosts: "{{zen_discovery_ping_unicast_hosts}}" + hosts: {{zen_discovery_ping_unicast_hosts}} node: data: true @@ -81,4 +81,5 @@ discovery.zen.fd.ping_interval: {{discovery_zen_fd_ping_interval}} discovery.zen.fd.ping_timeout: {{discovery_zen_fd_ping_timeout}} discovery.zen.fd.ping_retries: {{discovery_zen_fd_ping_retries}} -network.host: {{network_host}} \ No newline at end of file +network.host: {{network_host}} +network.publish_host: {{network_publish_host}} http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml index fe25eed..b24c153 100644 --- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml @@ -95,9 +95,9 @@ configurations: - elastic-site: index_number_of_shards: 1 index_number_of_replicas: 0 - zen_discovery_ping_unicast_hosts: "{{ groups.search | join(',') }}" + zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]" gateway_recover_after_data_nodes: 1 - network_host: "_lo_,_eth0_,_eth1_" + network_host: "[ _local_ ]" masters_also_are_datanodes: "1" required_configurations: http://git-wip-us.apache.org/repos/asf/metron/blob/21b79dc8/metron-deployment/roles/ambari_config/vars/small_cluster.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml index 5f261a0..70c31ea 100644 --- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml +++ b/metron-deployment/roles/ambari_config/vars/small_cluster.yml @@ -89,9 +89,9 @@ configurations: - elastic-site: index_number_of_shards: 2 index_number_of_replicas: 1 - zen_discovery_ping_unicast_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}" + zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]" gateway_recover_after_data_nodes: 1 - network_host: _lo_,_{{ elasticsearch_network_interface }}_ + network_host: "[ _local_, _{{ elasticsearch_network_interface }}_ ]" required_configurations: - metron-env: