Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/344965 )
Change subject: Update elk stack to 5.x
......................................................................
Update elk stack to 5.x
* Upgrades elasticsearch to 5.1.2
* Upgrades kibana to 5.1.2
* Uses experimental wikimedia apt repository for all logstash nodes
to access new versions of kibana and elasticsearch packages
* Updates templates used for logstash for compatability with es 5.x
** index is now a boolean, instead of analyzed/not_analyzed/no. New
settings are basically: analyzed => type: text index: true,
not_analyzed => type:keyword index:true, no => index:false
* updates logstash_checker.py to not use filtered queries which were
removed in elasticsearch 5.x. Also reformats as "standard" json
format and sets size to 0 since we only need the aggregation results.
Bug: T154473
Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929
---
M hieradata/role/common/logstash/collector.yaml
M modules/kibana/manifests/init.pp
M modules/role/files/logstash/apifeatureusage-template.json
M modules/role/files/logstash/elasticsearch-template.json
M modules/service/files/logstash_checker.py
5 files changed, 130 insertions(+), 163 deletions(-)
Approvals:
jenkins-bot: Verified
Gehel: Looks good to me, approved
diff --git a/hieradata/role/common/logstash/collector.yaml
b/hieradata/role/common/logstash/collector.yaml
index 4bbcc51..20ef97f 100644
--- a/hieradata/role/common/logstash/collector.yaml
+++ b/hieradata/role/common/logstash/collector.yaml
@@ -56,4 +56,3 @@
debdeploy::grains:
debdeploy-logstash:
value: standard
-
diff --git a/modules/kibana/manifests/init.pp b/modules/kibana/manifests/init.pp
index 9eebac6..695a883 100644
--- a/modules/kibana/manifests/init.pp
+++ b/modules/kibana/manifests/init.pp
@@ -18,8 +18,7 @@
) {
require_package('kibana')
- # kibana 4
- file { '/opt/kibana/config/kibana.yml':
+ file { '/etc/kibana/kibana.yml':
ensure => file,
owner => 'root',
group => 'root',
@@ -36,7 +35,13 @@
enable => true,
require => [
Package['kibana'],
- File['/opt/kibana/config/kibana.yml'],
+ File['/etc/kibana/kibana.yml'],
],
}
+
+ # kibana 4.x deployment. With 5.x things are
+ # installed normally and not in /opt
+ file { '/opt/kibana':
+ ensure => absent
+ }
}
diff --git a/modules/role/files/logstash/apifeatureusage-template.json
b/modules/role/files/logstash/apifeatureusage-template.json
index 2228152..37fd1dd 100644
--- a/modules/role/files/logstash/apifeatureusage-template.json
+++ b/modules/role/files/logstash/apifeatureusage-template.json
@@ -21,15 +21,15 @@
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
- "index" : "no"
+ "index" : false
}
}
} ],
"properties" : {
- "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
- "@version" : { "type" : "string", "index" : "not_analyzed" },
- "feature" : { "type" : "string", "index" : "not_analyzed" },
- "agent" : { "type" : "string", "index" : "not_analyzed" }
+ "@timestamp" : { "type" : "date", "index" : true },
+ "@version" : { "type" : "keyword", "index" : true },
+ "feature" : { "type" : "keyword", "index" : true },
+ "agent" : { "type" : "keyword", "index" : true }
}
}
}
diff --git a/modules/role/files/logstash/elasticsearch-template.json
b/modules/role/files/logstash/elasticsearch-template.json
index 21e909f..d9a7145 100644
--- a/modules/role/files/logstash/elasticsearch-template.json
+++ b/modules/role/files/logstash/elasticsearch-template.json
@@ -3,7 +3,6 @@
"settings" : {
"number_of_shards" : 1,
"number_of_replicas" : 2,
- "index.cache.field.type" : "soft",
"index.refresh_interval" : "5s",
"analysis" : {
"analyzer" : {
@@ -26,19 +25,15 @@
{
"string_fields" : {
"match" : "*",
- "match_mapping_type" : "string",
+ "match_mapping_type" : "text",
"mapping" : {
- "type" : "multi_field",
+ "type" : "text",
+ "index" : true,
+ "omit_norms" : true,
"fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
"raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
}
}
@@ -49,20 +44,9 @@
"boolean_to_string" : {
"match_mapping_type" : "boolean",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
},
@@ -70,20 +54,9 @@
"double_to_string" : {
"match_mapping_type" : "double",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
},
@@ -91,20 +64,9 @@
"long_to_string" : {
"match_mapping_type" : "long",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
}
@@ -112,12 +74,12 @@
"properties" : {
"@timestamp" : {
"type" : "date",
- "index" : "not_analyzed",
+ "index" : true,
"doc_values": true
},
"@version" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"geoip" : {
@@ -131,32 +93,29 @@
}
},
"host" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"level" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"message" : {
- "type" : "string",
- "index" : "analyzed",
+ "type" : "text",
+ "index" : true,
"omit_norms" : true
},
"normalized_message" : {
- "type": "multi_field",
+ "type": "text",
+ "index" : true,
+ "omit_norms" : true,
"fields": {
"raw": {
- "type": "string",
- "index": "not_analyzed",
+ "type": "keyword",
+ "index": true,
"doc_values": true
- },
- "normalized_message": {
- "index" : "analyzed",
- "omit_norms" : true,
- "type" : "string"
}
}
},
@@ -165,18 +124,18 @@
"doc_values": true
},
"tags" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"type" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"udp_sender" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
}
}
diff --git a/modules/service/files/logstash_checker.py
b/modules/service/files/logstash_checker.py
index e3eea54..6239544 100755
--- a/modules/service/files/logstash_checker.py
+++ b/modules/service/files/logstash_checker.py
@@ -120,94 +120,98 @@
'OR channel:error)) '
'OR type:hhvm)') % vars(self)
- return {"aggs": {
- "2": {
- "date_histogram": {
- "interval": "10s",
- "field": "@timestamp"
- }
- }
- }, "query": {
- "filtered": {
- "filter": {
- "bool": {
- "must": [
- {
- "range": {
- "@timestamp": {
- "lte": "now",
- "gte": "now-60m"
- }
- }
- }
- ],
- "must_not": [
- {"terms": {
- "level": [
- "INFO",
- "DEBUG"
- ]
- }}, {
- "query": {
- "match": {
- "message": {
- "query": "SlowTimer",
- "type": "phrase"
- }
- }
- }}, {
- "query": {
- "match": {
- "message": {
- "query": "Invalid host name",
- "type": "phrase"
- }
- }
- }}
- ]
- }
- },
- "query": {
- "query_string": {
- "query": query
+ return {
+ "size": 0,
+ "aggs": {
+ "2": {
+ "date_histogram": {
+ "interval": "10s",
+ "field": "@timestamp"
}
}
- }
- }}
-
- def _make_logstash_query(self):
- query = ('host:("%(host)s") '
- 'AND (level:("ERROR") OR level:("FATAL")) '
- 'AND type:("%(service_name)s")') % vars(self)
-
- return {"aggs": {
- "2": {
- "date_histogram": {
- "interval": "10s",
- "field": "@timestamp"
- }
- }
- }, "query": {
- "filtered": {
- "filter": {
- "bool": {
- "must": [{
+ },
+ "query": {
+ "bool": {
+ "filter": [
+ {
"range": {
"@timestamp": {
"lte": "now",
"gte": "now-60m"
}
}
- }]
- }
- },
- "query": {
- "query_string": {
- "query": query
- }
+ },
+ {
+ "query_string": {
+ "query": query
+ }
+ }
+ ],
+ "must_not": [
+ {
+ "terms": {
+ "level": [
+ "INFO",
+ "DEBUG"
+ ]
+ }
+ },
+ {
+ "match": {
+ "message": {
+ "query": "SlowTimer",
+ "type": "phrase"
+ }
+ }
+ },
+ {
+ "match": {
+ "message": {
+ "query": "Invalid host name",
+ "type": "phrase"
+ }
+ }
+ }
+ ]
}
}
- }}
+ }
+
+ def _make_logstash_query(self):
+ query = ('host:("%(host)s") '
+ 'AND (level:("ERROR") OR level:("FATAL")) '
+ 'AND type:("%(service_name)s")') % vars(self)
+
+ return {
+ "size": 0,
+ "aggs": {
+ "2": {
+ "date_histogram": {
+ "interval": "10s",
+ "field": "@timestamp"
+ }
+ }
+ },
+ "query": {
+ "bool": {
+ "filter": [
+ {
+ "range": {
+ "@timestamp": {
+ "lte": "now",
+ "gte": "now-60m"
+ }
+ }
+ },
+ {
+ "query_string": {
+ "query": query
+ }
+ }
+ ]
+ }
+ }
+ }
def run(self):
"""
--
To view, visit https://gerrit.wikimedia.org/r/344965
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929
Gerrit-PatchSet: 18
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org>
Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
