BryanDavis has uploaded a new change for review.
https://gerrit.wikimedia.org/r/179758
Change subject: logstash: port udp2log rules to monolog input
......................................................................
logstash: port udp2log rules to monolog input
Change-Id: I3b6e37a2b6b9389c1a03bd572f422f898970c5b4
---
M files/logstash/filter-mw-via-redis.conf
1 file changed, 99 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/58/179758/1
diff --git a/files/logstash/filter-mw-via-redis.conf
b/files/logstash/filter-mw-via-redis.conf
index c9acec5..97f3a27 100644
--- a/files/logstash/filter-mw-via-redis.conf
+++ b/files/logstash/filter-mw-via-redis.conf
@@ -8,6 +8,105 @@
add_tag => [ "es" ]
}
+ if [channel] == "api" {
+ # Excluded because the message volume is too high
+ drop {}
+ }
+
+ if [channel] == "exception" {
+ # Excluded because events are duplicated by exception-json
+ drop {}
+ }
+
+ if [channel] == "exception-json" {
+ # Parse message as json and put elements in event
+ json {
+ source => "message"
+ add_tag => [ "json" ]
+ }
+ mutate {
+ # Rename the `id` field to `exeception_id`
+ rename => [ "id", "exception_id" ]
+ # Add a field to compute a checksum value based on message + file +
line
+ add_field => [ "message_checksum", "%{message}|%{file}|%{line}" ]
+ # Rename the channel to "exception"
+ replace => [ "channel", "exception" ]
+ }
+ # Convert message_checksum field to md5 hash
+ anonymize {
+ fields => [ "message_checksum" ]
+ algorithm => "MD5"
+ key => "boringsalt"
+ }
+ } # end [channel] == "exception-json"
+
+ if [channel] == "api-feature-usage" {
+ grok {
+ match => [
+ "message",
+ "^(?m)%{QS:feature} %{QS:username} %{QS:ip} %{QS:referer}
%{QS:agent}$"
+ ]
+ named_captures_only => true
+ }
+
+ if !("_grokparsefailure" in [tags]) {
+ # Unquote ('"foo \"bar\""' to 'foo "bar"')
+ mutate {
+ # Strip outer quotes
+ gsub => [
+ "feature", '^"|"$', "",
+ "username", '^"|"$', "",
+ "ip", '^"|"$', "",
+ "referer", '^"|"$', "",
+ "agent", '^"|"$', ""
+ ]
+ }
+ mutate {
+ # Strip backslash escape characters
+ gsub => [
+ "feature", '\\(.)', '\1',
+ "username", '\\(.)', '\1',
+ "ip", '\\(.)', '\1',
+ "referer", '\\(.)', '\1',
+ "agent", '\\(.)', '\1'
+ ]
+ }
+
+ mutate {
+ replace => [ "message", "%{feature}" ]
+ }
+
+ urldecode {
+ field => "username"
+ }
+
+ useragent {
+ source => "agent"
+ prefix => "ua_"
+ }
+
+ # Ignore this one for now, too many hits
+ if [feature] == "action=query&!rawcontinue&!continue" {
+ drop {}
+ }
+ }
+ } # end [channel] == "api-feature-usage"
+
+ if [type] == "xff" {
+ # Copy XFF addresses from message
+ grok {
+ match => [
+ "message",
+ "^%{URI:url}\t(?:, )?(?<xff>(?:%{IP}(?:, )?)+)\t"
+ ]
+ named_captures_only => true
+ }
+ # Turn comma separated list of XFF addresses into a real list
+ mutate {
+ split => [ "xff", ", " ]
+ }
+ } # end [type] == "xff"
+
} # end [type] == "mediawiki"
}
--
To view, visit https://gerrit.wikimedia.org/r/179758
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3b6e37a2b6b9389c1a03bd572f422f898970c5b4
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
