Gage has uploaded a new change for review. https://gerrit.wikimedia.org/r/168935
Change subject: logstash: hadoop: extract and infer job, task, attempt IDs ...................................................................... logstash: hadoop: extract and infer job, task, attempt IDs Change-Id: I5fd74143384e7346bde36d6584d07a5db511bdab --- M files/logstash/filter-gelf.conf 1 file changed, 19 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/35/168935/1 diff --git a/files/logstash/filter-gelf.conf b/files/logstash/filter-gelf.conf index 40499eb..7e15f87 100644 --- a/files/logstash/filter-gelf.conf +++ b/files/logstash/filter-gelf.conf @@ -5,6 +5,8 @@ replace => [ "channel", "%{SourceSimpleClassName}" ] } grok { + # Oniguruma syntax for ‘named capture’: (?<field_name>the pattern here) + # this overwrites the Thread field with a shorter more generic version, saving details to separate fields match => [ "Thread", "(?<Thread>IPC Server handler) %{NUMBER:IPC_Server_handler_id} on %{NUMBER:IPC_Server_handler_port}" ] match => [ "Thread", "(?<Thread>DeletionService) #%{NUMBER:DeletionService_id}" ] match => [ "Thread", "(?<Thread>LocalizerRunner) for %{NOTSPACE:ContainerId}" ] @@ -13,6 +15,23 @@ match => [ "Thread", "(?<Thread>CacheReplicationMonitor)\(%{NUMBER:CacheReplicationMonitor_id}\)" ] overwrite => [ "Thread" ] } + grok { + # so that we can search by job id and find tasks and attempts: + # extract attempt ID to field + # attempt_1409078537822_52431_m_000009_1 + # attempt_1409078537822_55176_r_000000_0 + match => [ "message", ".*attempt_(?<Attempt_id>[:digit:]+_[:digit:]+_[mr]_[:digit:]+_[:digit:]+).*" ] + # extract task ID to field + # task_1409078537822_52431_m_000044 + match => [ "message", ".*task_(?<Task_id>[:digit:]+_[:digit:]+_[mr]_[:digit:]+).*" ] + # extract job ID to field + # job_1409078537822_52431 + match => [ "message", ".*job_(?<Job_id>[:digit:]+_[:digit:]+.*" ] + # infer task ID from attempt ID + match => [ "Attempt_id", "(?<Task_id>[:digit:]+_[:digit:]+_[mr]_[:digit:]+)_[:digit:]+" ] + # infer job ID from task ID + match => [ "Task_id", "(?<Job_id>[:digit:]+_[:digit:]+)_[mr]_[:digit:]+" ] + } dns { reverse => [ "host" ] action => "replace" -- To view, visit https://gerrit.wikimedia.org/r/168935 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5fd74143384e7346bde36d6584d07a5db511bdab Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Gage <jger...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits