Ottomata has submitted this change and it was merged. Change subject: Run webstatscollector (modified) with kafkatee on analytics1003 ......................................................................
Run webstatscollector (modified) with kafkatee on analytics1003 This commit also starts kafkatee consuming from the full webrequest stream (all 4 topics). Here we go! Change-Id: I7be5f44075cae9899ece80de8e16c71b0f2ddc56 --- M manifests/role/analytics/kafkatee.pp M manifests/site.pp 2 files changed, 108 insertions(+), 0 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/manifests/role/analytics/kafkatee.pp b/manifests/role/analytics/kafkatee.pp index 56b73bb..5b10312 100644 --- a/manifests/role/analytics/kafkatee.pp +++ b/manifests/role/analytics/kafkatee.pp @@ -88,6 +88,69 @@ } } +# == role::analytics::kafkatee::webstatscollector +# We want to run webstatscollector via kafkatee for testing. +# Some of the production (role::logging::webstatscollector) +# configs are not relevant here, so we copy the class +# and edit it. +# +# webstatscollector needs all of the webrequest logs, +# so this class makes sure all webrequest topic input +# classes are included. +class role::analytics::kafkatee::webrequest::webstatscollector { + include role::analytics::kafkatee::input::webrequest + + # webstats-collector process writes dump files here. + $webstats_dumps_directory = '/srv/webstats/dumps' + + package { 'webstatscollector': ensure => installed } + service { 'webstats-collector': + ensure => 'running', + hasstatus => 'false', + hasrestart => 'true', + require => Package['webstatscollector'], + } + + # Gzip pagecounts files hourly. + cron { 'webstats-dumps-gzip': + command => "/bin/gzip ${webstats_dumps_directory}/pagecounts-????????-?????? 2> /dev/null", + minute => 2, + user => 'nobody', + require => Service['webstats-collector'], + } + + # Delete webstats dumps that are older than 10 days daily. + cron { 'webstats-dumps-delete': + command => "/usr/bin/find ${webstats_dumps_directory} -maxdepth 1 -type f -mtime +10 -delete", + minute => 28, + hour => 1, + user => 'nobody', + require => Service['webstats-collector'], + } + + # kafkatee outputs into webstats filter and forwards to webstats collector via log2udp + ::kafkatee::output { 'webstatscollector': + destination => "/usr/local/bin/filter | /usr/bin/log2udp -h localhost -p 3815", + type => 'pipe', + require => Service['webstats-collector'], + } +} + + + +# == Class role::analytics::kafkatee::input::webrequest +# Includes each of the 4 webrequest topics as input +# You can use this class, or if you want to consume +# only an individual topic, include one of the +# topic specific classes manually. +class role::analytics::kafkatee::input::webrequest { + include role::analytics::kafkatee::input::webrequest::mobile + include role::analytics::kafkatee::input::webrequest::text + include role::analytics::kafkatee::input::webrequest::bits + include role::analytics::kafkatee::input::webrequest::upload +} + + # == Class role::analytics::kafkatee::input::webrequest::mobile # Sets up a kafkatee input to consume from the webrequest_mobile topic @@ -103,3 +166,47 @@ offset => 'stored', } } +# == Class role::analytics::kafkatee::input::webrequest::text +# Sets up a kafkatee input to consume from the webrequest_text topic +# This is its own class so that if a kafkatee instance wants +# to consume from multiple topics, it may include each +# topic as a class. +# +class role::analytics::kafkatee::input::webrequest::text { + ::kafkatee::input { 'kafka-webrequest_text': + topic => 'webrequest_text', + partitions => '0-11', + options => { 'encoding' => 'json' }, + offset => 'stored', + } +} +# == Class role::analytics::kafkatee::input::webrequest::bits +# Sets up a kafkatee input to consume from the webrequest_bits topic +# This is its own class so that if a kafkatee instance wants +# to consume from multiple topics, it may include each +# topic as a class. +# +class role::analytics::kafkatee::input::webrequest::bits { + ::kafkatee::input { 'kafka-webrequest_bits': + topic => 'webrequest_bits', + partitions => '0-11', + options => { 'encoding' => 'json' }, + offset => 'stored', + } +} +# == Class role::analytics::kafkatee::input::webrequest::upload +# Sets up a kafkatee input to consume from the webrequest_upload topic +# This is its own class so that if a kafkatee instance wants +# to consume from multiple topics, it may include each +# topic as a class. +# +class role::analytics::kafkatee::input::webrequest::upload { + ::kafkatee::input { 'kafka-webrequest_upload': + topic => 'webrequest_upload', + partitions => '0-11', + options => { 'encoding' => 'json' }, + offset => 'stored', + } +} + + diff --git a/manifests/site.pp b/manifests/site.pp index a2b2cf9..d87505a 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -109,6 +109,7 @@ include role::analytics include role::analytics::kafkatee::webrequest::mobile + include role::analytics::kafkatee::webrequest::webstatscollector # run misc udp2log here for sqstat include role::logging::udp2log::misc -- To view, visit https://gerrit.wikimedia.org/r/155740 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7be5f44075cae9899ece80de8e16c71b0f2ddc56 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <o...@wikimedia.org> Gerrit-Reviewer: Ottomata <o...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits