METRON-1508 In Ubuntu14 Dev Indexing Fails to Write to Elasticsearch (nickwallen) closes apache/metron#1185
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08fe1cff Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08fe1cff Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08fe1cff Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 08fe1cff1430c71205d62e375cbacd6906e4fc4e Parents: 0d34e77 Author: nickwallen <n...@nickallen.org> Authored: Fri Sep 21 15:31:24 2018 -0400 Committer: nickallen <nickal...@apache.org> Committed: Fri Sep 21 15:31:24 2018 -0400 ---------------------------------------------------------------------- .../ansible/roles/ambari_config/vars/single_node_vm.yml | 4 ++-- .../development/centos6/ansible/inventory/group_vars/all | 1 + .../development/ubuntu14/ansible/inventory/group_vars/all | 1 + .../ELASTICSEARCH/5.6.2/configuration/elastic-site.xml | 8 ++++---- 4 files changed, 8 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml index 962726c..2ec956a 100644 --- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml +++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml @@ -92,7 +92,7 @@ configurations: # Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]' topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]' - # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties + # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER' supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER' worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER" @@ -117,7 +117,7 @@ configurations: required_configurations: - metron-env: storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744" - es_hosts: "{{ groups.search | join(',') }}" + es_hosts: "{{ elasticsearch_hosts }}" zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995" solr_zookeeper_url: "{{ groups.search[0] }}:9983" - metron-rest-env: http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/centos6/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all index dcb7b42..550eb8f 100644 --- a/metron-deployment/development/centos6/ansible/inventory/group_vars/all +++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all @@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa" snort_version: "2.9.8.0-1" snort_alert_csv_path: "/var/log/snort/alert.csv" threat_intel_bulk_load: False +elasticsearch_hosts: "{{ groups.search | join(',') }}" # data directories - only required to override defaults zookeeper_data_dir: "/data1/hadoop/zookeeper" http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all ---------------------------------------------------------------------- diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all index ef9f705..1338d7c 100644 --- a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all +++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all @@ -46,6 +46,7 @@ pycapa_home: "/opt/pycapa" snort_version: "2.9.8.0-1" snort_alert_csv_path: "/var/log/snort/alert.csv" threat_intel_bulk_load: False +elasticsearch_hosts: "localhost" # data directories - only required to override defaults zookeeper_data_dir: "/data1/hadoop/zookeeper" http://git-wip-us.apache.org/repos/asf/metron/blob/08fe1cff/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml index 34df1e4..440c66d 100755 --- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml +++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-site.xml @@ -111,7 +111,7 @@ <value>0</value> <description>expected_data_nodes</description> </property> - <!-- Index --> + <!-- Index --> <property> <name>index_merge_scheduler_max_thread_count</name> <value>5</value> @@ -171,7 +171,7 @@ <name>cluster_routing_allocation_disk_threshold_enabled</name> <value>true</value> <description>Property used when multiple drives are used to understand if thresholding is active</description> - </property> + </property> <property> <name>cluster_routing_allocation_disk_watermark_low</name> <value>.97</value> @@ -185,7 +185,7 @@ <property> <name>network_host</name> <value>[ _local_, _site_ ]</value> - <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + <description>Network interface(s) ES will bind to within each node. "_site_" or a more specific external address is required for all multi-node clusters, and also recommended for single-node installs to allow access to ES reports from non-local hosts. Always include the square brackets. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description> </property> <property> <name>network_publish_host</name> @@ -193,6 +193,6 @@ <value-attributes> <empty-value-valid>true</empty-value-valid> </value-attributes> - <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/2.3/modules-network.html for ES documentation.</description> + <description>Network address ES will publish for client and peer use. Empty value causes it to pick from the values in network_host, which works in most simple environments. MUST set explicitly for MULTI-HOMED SYSTEMS. See https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-network.html for ES documentation.</description> </property> </configuration>