Gehel has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/344965 )
Change subject: Update elk stack to 5.x ...................................................................... Update elk stack to 5.x * Upgrades elasticsearch to 5.1.2 * Upgrades kibana to 5.1.2 * Uses experimental wikimedia apt repository for all logstash nodes to access new versions of kibana and elasticsearch packages * Updates templates used for logstash for compatability with es 5.x ** index is now a boolean, instead of analyzed/not_analyzed/no. New settings are basically: analyzed => type: text index: true, not_analyzed => type:keyword index:true, no => index:false * updates logstash_checker.py to not use filtered queries which were removed in elasticsearch 5.x. Also reformats as "standard" json format and sets size to 0 since we only need the aggregation results. Bug: T154473 Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929 --- M hieradata/role/common/logstash/collector.yaml M modules/kibana/manifests/init.pp M modules/role/files/logstash/apifeatureusage-template.json M modules/role/files/logstash/elasticsearch-template.json M modules/service/files/logstash_checker.py 5 files changed, 130 insertions(+), 163 deletions(-) Approvals: jenkins-bot: Verified Gehel: Looks good to me, approved diff --git a/hieradata/role/common/logstash/collector.yaml b/hieradata/role/common/logstash/collector.yaml index 4bbcc51..20ef97f 100644 --- a/hieradata/role/common/logstash/collector.yaml +++ b/hieradata/role/common/logstash/collector.yaml @@ -56,4 +56,3 @@ debdeploy::grains: debdeploy-logstash: value: standard - diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp index 9eebac6..695a883 100644 --- a/modules/kibana/manifests/init.pp +++ b/modules/kibana/manifests/init.pp @@ -18,8 +18,7 @@ ) { require_package('kibana') - # kibana 4 - file { '/opt/kibana/config/kibana.yml': + file { '/etc/kibana/kibana.yml': ensure => file, owner => 'root', group => 'root', @@ -36,7 +35,13 @@ enable => true, require => [ Package['kibana'], - File['/opt/kibana/config/kibana.yml'], + File['/etc/kibana/kibana.yml'], ], } + + # kibana 4.x deployment. With 5.x things are + # installed normally and not in /opt + file { '/opt/kibana': + ensure => absent + } } diff --git a/modules/role/files/logstash/apifeatureusage-template.json b/modules/role/files/logstash/apifeatureusage-template.json index 2228152..37fd1dd 100644 --- a/modules/role/files/logstash/apifeatureusage-template.json +++ b/modules/role/files/logstash/apifeatureusage-template.json @@ -21,15 +21,15 @@ "match" : "*", "match_mapping_type" : "string", "mapping" : { - "index" : "no" + "index" : false } } } ], "properties" : { - "@timestamp" : { "type" : "date", "index" : "not_analyzed" }, - "@version" : { "type" : "string", "index" : "not_analyzed" }, - "feature" : { "type" : "string", "index" : "not_analyzed" }, - "agent" : { "type" : "string", "index" : "not_analyzed" } + "@timestamp" : { "type" : "date", "index" : true }, + "@version" : { "type" : "keyword", "index" : true }, + "feature" : { "type" : "keyword", "index" : true }, + "agent" : { "type" : "keyword", "index" : true } } } } diff --git a/modules/role/files/logstash/elasticsearch-template.json b/modules/role/files/logstash/elasticsearch-template.json index 21e909f..d9a7145 100644 --- a/modules/role/files/logstash/elasticsearch-template.json +++ b/modules/role/files/logstash/elasticsearch-template.json @@ -3,7 +3,6 @@ "settings" : { "number_of_shards" : 1, "number_of_replicas" : 2, - "index.cache.field.type" : "soft", "index.refresh_interval" : "5s", "analysis" : { "analyzer" : { @@ -26,19 +25,15 @@ { "string_fields" : { "match" : "*", - "match_mapping_type" : "string", + "match_mapping_type" : "text", "mapping" : { - "type" : "multi_field", + "type" : "text", + "index" : true, + "omit_norms" : true, "fields" : { - "{name}" : { - "type" : "string", - "index" : "analyzed", - "omit_norms" : true - }, "raw" : { - "type" : "string", - "index" : "not_analyzed", - "ignore_above" : 256, + "type" : "keyword", + "index" : true, "doc_values": true } } @@ -49,20 +44,9 @@ "boolean_to_string" : { "match_mapping_type" : "boolean", "mapping" : { - "type" : "multi_field", - "fields" : { - "{name}" : { - "type" : "string", - "index" : "analyzed", - "omit_norms" : true - }, - "raw" : { - "type" : "string", - "index" : "not_analyzed", - "ignore_above" : 256, - "doc_values": true - } - } + "type" : "keyword", + "index" : true, + "doc_values": true } } }, @@ -70,20 +54,9 @@ "double_to_string" : { "match_mapping_type" : "double", "mapping" : { - "type" : "multi_field", - "fields" : { - "{name}" : { - "type" : "string", - "index" : "analyzed", - "omit_norms" : true - }, - "raw" : { - "type" : "string", - "index" : "not_analyzed", - "ignore_above" : 256, - "doc_values": true - } - } + "type" : "keyword", + "index" : true, + "doc_values": true } } }, @@ -91,20 +64,9 @@ "long_to_string" : { "match_mapping_type" : "long", "mapping" : { - "type" : "multi_field", - "fields" : { - "{name}" : { - "type" : "string", - "index" : "analyzed", - "omit_norms" : true - }, - "raw" : { - "type" : "string", - "index" : "not_analyzed", - "ignore_above" : 256, - "doc_values": true - } - } + "type" : "keyword", + "index" : true, + "doc_values": true } } } @@ -112,12 +74,12 @@ "properties" : { "@timestamp" : { "type" : "date", - "index" : "not_analyzed", + "index" : true, "doc_values": true }, "@version" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true }, "geoip" : { @@ -131,32 +93,29 @@ } }, "host" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true }, "level" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true }, "message" : { - "type" : "string", - "index" : "analyzed", + "type" : "text", + "index" : true, "omit_norms" : true }, "normalized_message" : { - "type": "multi_field", + "type": "text", + "index" : true, + "omit_norms" : true, "fields": { "raw": { - "type": "string", - "index": "not_analyzed", + "type": "keyword", + "index": true, "doc_values": true - }, - "normalized_message": { - "index" : "analyzed", - "omit_norms" : true, - "type" : "string" } } }, @@ -165,18 +124,18 @@ "doc_values": true }, "tags" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true }, "type" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true }, "udp_sender" : { - "type" : "string", - "index" : "not_analyzed", + "type" : "keyword", + "index" : true, "doc_values": true } } diff --git a/modules/service/files/logstash_checker.py b/modules/service/files/logstash_checker.py index e3eea54..6239544 100755 --- a/modules/service/files/logstash_checker.py +++ b/modules/service/files/logstash_checker.py @@ -120,94 +120,98 @@ 'OR channel:error)) ' 'OR type:hhvm)') % vars(self) - return {"aggs": { - "2": { - "date_histogram": { - "interval": "10s", - "field": "@timestamp" - } - } - }, "query": { - "filtered": { - "filter": { - "bool": { - "must": [ - { - "range": { - "@timestamp": { - "lte": "now", - "gte": "now-60m" - } - } - } - ], - "must_not": [ - {"terms": { - "level": [ - "INFO", - "DEBUG" - ] - }}, { - "query": { - "match": { - "message": { - "query": "SlowTimer", - "type": "phrase" - } - } - }}, { - "query": { - "match": { - "message": { - "query": "Invalid host name", - "type": "phrase" - } - } - }} - ] - } - }, - "query": { - "query_string": { - "query": query + return { + "size": 0, + "aggs": { + "2": { + "date_histogram": { + "interval": "10s", + "field": "@timestamp" } } - } - }} - - def _make_logstash_query(self): - query = ('host:("%(host)s") ' - 'AND (level:("ERROR") OR level:("FATAL")) ' - 'AND type:("%(service_name)s")') % vars(self) - - return {"aggs": { - "2": { - "date_histogram": { - "interval": "10s", - "field": "@timestamp" - } - } - }, "query": { - "filtered": { - "filter": { - "bool": { - "must": [{ + }, + "query": { + "bool": { + "filter": [ + { "range": { "@timestamp": { "lte": "now", "gte": "now-60m" } } - }] - } - }, - "query": { - "query_string": { - "query": query - } + }, + { + "query_string": { + "query": query + } + } + ], + "must_not": [ + { + "terms": { + "level": [ + "INFO", + "DEBUG" + ] + } + }, + { + "match": { + "message": { + "query": "SlowTimer", + "type": "phrase" + } + } + }, + { + "match": { + "message": { + "query": "Invalid host name", + "type": "phrase" + } + } + } + ] } } - }} + } + + def _make_logstash_query(self): + query = ('host:("%(host)s") ' + 'AND (level:("ERROR") OR level:("FATAL")) ' + 'AND type:("%(service_name)s")') % vars(self) + + return { + "size": 0, + "aggs": { + "2": { + "date_histogram": { + "interval": "10s", + "field": "@timestamp" + } + } + }, + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "lte": "now", + "gte": "now-60m" + } + } + }, + { + "query_string": { + "query": query + } + } + ] + } + } + } def run(self): """ -- To view, visit https://gerrit.wikimedia.org/r/344965 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929 Gerrit-PatchSet: 18 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits