[MediaWiki-commits] [Gerrit] logstash: Add normalized_message field to all events - change (operations/puppet)
Ori.livneh has submitted this change and it was merged. Change subject: logstash: Add normalized_message field to all events .. logstash: Add normalized_message field to all events Copy the message of all events destined for storage in Elasticsearch into a "normalized_message" field that is truncated to 255 characters. This can be used in dashboards as a term search to correlate common messages. Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31 --- A files/logstash/filter-add-normalized-message.conf M manifests/role/logstash.pp 2 files changed, 40 insertions(+), 1 deletion(-) Approvals: Ori.livneh: Looks good to me, approved jenkins-bot: Verified diff --git a/files/logstash/filter-add-normalized-message.conf b/files/logstash/filter-add-normalized-message.conf new file mode 100644 index 000..b77662a --- /dev/null +++ b/files/logstash/filter-add-normalized-message.conf @@ -0,0 +1,34 @@ +# vim:set sw=2 ts=2 sts=2 et +# Add normalized_message field to events bound for logstash +filter { + + if "es" in [tags] and ![normalized_message] { +mutate { + # Create a copy of message field that can be normalized + add_field => [ "normalized_message", "%{message}" ] +} +# Remove documentation anchor tags +mutate { + gsub => [ +"normalized_message", +" \[[^<]*\]", +"" + ] +} +# Trim the normalized_message to a maximum of 255 characters +# This is done because our Elasticsearch schema doesn't store raw fields +# for strings longer than 255 characters and we want something to show +# in terms queries even if it's shortened. +grok { + match => [ +"normalized_message", +"^(?.{255}).*$" + ] + overwrite => [ "normalized_message" ] + named_captures_only => true + add_tag => [ "normalized_message_trimmed" ] + tag_on_failure => [ "normalized_message_untrimmed" ] +} + } + +} diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp index d17f25a..2ad37d2 100644 --- a/manifests/role/logstash.pp +++ b/manifests/role/logstash.pp @@ -57,7 +57,7 @@ logstash::conf { 'filter_strip_ansi_color': source => 'puppet:///files/logstash/filter-strip-ansi-color.conf', -priority => 50, +priority => 40, } logstash::conf { 'filter_syslog': @@ -70,6 +70,11 @@ priority => 50, } +logstash::conf { 'filter_add_normalized_message': +source => 'puppet:///files/logstash/filter-add-normalized-message.conf', +priority => 60, +} + class { '::logstash::output::elasticsearch': host=> '127.0.0.1', replication => 'async', -- To view, visit https://gerrit.wikimedia.org/r/112149 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BryanDavis Gerrit-Reviewer: Faidon Liambotis Gerrit-Reviewer: Ori.livneh Gerrit-Reviewer: jenkins-bot <> ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] logstash: Add normalized_message field to all events - change (operations/puppet)
BryanDavis has uploaded a new change for review. https://gerrit.wikimedia.org/r/112149 Change subject: logstash: Add normalized_message field to all events .. logstash: Add normalized_message field to all events Copy the message of all events destined for storage in Elasticsearch into a "normalized_message" field that is truncated to 255 characters. This can be used in dashboards as a term search to correlate common messages. Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31 --- A files/logstash/filter-add-normalized-message.conf M manifests/role/logstash.pp 2 files changed, 40 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/49/112149/1 diff --git a/files/logstash/filter-add-normalized-message.conf b/files/logstash/filter-add-normalized-message.conf new file mode 100644 index 000..b77662a --- /dev/null +++ b/files/logstash/filter-add-normalized-message.conf @@ -0,0 +1,34 @@ +# vim:set sw=2 ts=2 sts=2 et +# Add normalized_message field to events bound for logstash +filter { + + if "es" in [tags] and ![normalized_message] { +mutate { + # Create a copy of message field that can be normalized + add_field => [ "normalized_message", "%{message}" ] +} +# Remove documentation anchor tags +mutate { + gsub => [ +"normalized_message", +" \[[^<]*\]", +"" + ] +} +# Trim the normalized_message to a maximum of 255 characters +# This is done because our Elasticsearch schema doesn't store raw fields +# for strings longer than 255 characters and we want something to show +# in terms queries even if it's shortened. +grok { + match => [ +"normalized_message", +"^(?.{255}).*$" + ] + overwrite => [ "normalized_message" ] + named_captures_only => true + add_tag => [ "normalized_message_trimmed" ] + tag_on_failure => [ "normalized_message_untrimmed" ] +} + } + +} diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp index d17f25a..2ad37d2 100644 --- a/manifests/role/logstash.pp +++ b/manifests/role/logstash.pp @@ -57,7 +57,7 @@ logstash::conf { 'filter_strip_ansi_color': source => 'puppet:///files/logstash/filter-strip-ansi-color.conf', -priority => 50, +priority => 40, } logstash::conf { 'filter_syslog': @@ -70,6 +70,11 @@ priority => 50, } +logstash::conf { 'filter_add_normalized_message': +source => 'puppet:///files/logstash/filter-add-normalized-message.conf', +priority => 60, +} + class { '::logstash::output::elasticsearch': host=> '127.0.0.1', replication => 'async', -- To view, visit https://gerrit.wikimedia.org/r/112149 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I01c50456cf0e334075acacbe2aebe5d8fc941d31 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BryanDavis ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits