Ottomata has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/403067 )
Change subject: [WIP] Refactor cache::kafka::eventlogging into profile and enable TLS ...................................................................... [WIP] Refactor cache::kafka::eventlogging into profile and enable TLS Bug: T183297 Change-Id: I4096fe7efda237bac162dfb5dc8af1262c445503 --- A modules/profile/manifests/cache/kafka/eventlogging.pp M modules/profile/manifests/cache/text.pp D modules/role/manifests/cache/kafka/eventlogging.pp M modules/role/manifests/cache/text.pp 4 files changed, 86 insertions(+), 74 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/67/403067/1 diff --git a/modules/profile/manifests/cache/kafka/eventlogging.pp b/modules/profile/manifests/cache/kafka/eventlogging.pp new file mode 100644 index 0000000..2f4aa2f --- /dev/null +++ b/modules/profile/manifests/cache/kafka/eventlogging.pp @@ -0,0 +1,79 @@ +# === Class profile::cache::kafka::eventlogging +# +# Sets up a varnishkafka logging endpoint for collecting +# analytics events coming from external clients. +# +# TODO: This class is still in test mode +# +# More info: https://wikitech.wikimedia.org/wiki/Analytics/EventLogging +# +# === Parameters +# +# [*kafka_cluster_name*] +# Name of the Kafka cluster in the kafka_clusters hash to be passed to the +# kafka_config() function. Default: jumbo. +# +# [*cache_cluster*] +# The name of the cache cluster. +# +# [*statsd*] +# The host to send statsd data to. +# +class profile::cache::kafka::eventlogging( + $kafka_cluster_name = hiera('profile::cache::kafka::eventlogging::kafka_cluster_name', 'jumbo') + $cache_cluster = hiera('cache::cluster'), + $statsd = hiera('statsd'), +) { + # Include this class to get key and certificate for varnishkafka + # to produce to Kafka over SSL/TLS. + require ::profile::cache::kafka::certificate + + # Set varnish.arg.q or varnish.arg.m according to Varnish version + $varnish_opts = { 'q' => 'ReqURL ~ "^/(beacon/)?event(\.gif)?\?"' } + + $config = kafka_config($kafka_cluster_name) + # Array of kafka brokers in jumbo-eqiad with SSL port 9093 + $kafka_brokers = $config['brokers']['ssl_array'] + + $topic = "webrequest_${cache_cluster}_test" + $varnish_name = 'frontend' + $varnish_svc_name = 'varnish-frontend' + + varnishkafka::instance { 'eventlogging': + brokers => $kafka_brokers, + # Note that this format uses literal tab characters. + # The '-' in this string used to be %{X-Client-IP@ip}o. + # EventLogging clientIp logging has been removed as part of T128407. + format => '%q %l %n %{%FT%T}t - "%{User-agent}i"', + format_type => 'string', + topic => 'eventlogging-client-side', + varnish_name => $varnish_name, + varnish_svc_name => $varnish_svc_name, + varnish_opts => $varnish_opts, + topic_request_required_acks => '1', + } + + include ::standard + + # Generate icinga alert if varnishkafka is not running. + nrpe::monitor_service { 'varnishkafka-eventlogging': + description => 'eventlogging Varnishkafka log producer', + nrpe_command => "/usr/lib/nagios/plugins/check_procs -c 1 -a '/usr/bin/varnishkafka -S /etc/varnishkafka/eventlogging.conf'", + contact_group => 'admins,analytics', + require => Varnishkafka::Instance['eventlogging'], + } + + $cache_type = hiera('cache::cluster') + $graphite_metric_prefix = "varnishkafka.${::hostname}.eventlogging.${cache_cluster}" + + # Sets up Logster to read from the Varnishkafka instance stats JSON file + # and report metrics to statsd. + varnishkafka::monitor::statsd { 'eventlogging': + graphite_metric_prefix => $graphite_metric_prefix, + statsd_host_port => $statsd, + } + + # Make sure varnishes are configured and started for the first time + # before the instances as well, or they fail to start initially... + Service <| tag == 'varnish_instance' |> -> Varnishkafka::Instance['eventlogging'] +} diff --git a/modules/profile/manifests/cache/text.pp b/modules/profile/manifests/cache/text.pp index d4225b8..743d017 100644 --- a/modules/profile/manifests/cache/text.pp +++ b/modules/profile/manifests/cache/text.pp @@ -87,14 +87,6 @@ backend_warming => $backend_warming, } - # varnishkafka eventlogging listens for eventlogging - # requests and logs them to the eventlogging-client-side - # topic. EventLogging servers consume and process this - # topic into many JSON based kafka topics for further - # consumption. - # TODO: Move this to profile, include from role::cache::text. - class { '::role::cache::kafka::eventlogging': } - # ResourceLoader browser cache hit rate and request volume stats. ::varnish::logging::rls { 'rls': statsd_server => $statsd_host, diff --git a/modules/role/manifests/cache/kafka/eventlogging.pp b/modules/role/manifests/cache/kafka/eventlogging.pp deleted file mode 100644 index 9bb5870..0000000 --- a/modules/role/manifests/cache/kafka/eventlogging.pp +++ /dev/null @@ -1,66 +0,0 @@ -# === Define role::cache::kafka::eventlogging -# -# Sets up a varnishkafka logging endpoint for collecting -# analytics events coming from external clients. -# -# More info: https://wikitech.wikimedia.org/wiki/Analytics/EventLogging -# -# === Parameters -# -# [*varnish_name*] -# The name of the varnish instance to read shared logs from. -# Default 'frontend' -# [*varnish_svc_name*] -# The name of the init unit for the above. -# Default 'varnish-frontend' -# [*kafka_protocol_version*] -# Kafka API version to use, needed for brokers < 0.10 -# https://issues.apache.org/jira/browse/KAFKA-3547 -# -class role::cache::kafka::eventlogging( - $varnish_name = 'frontend', - $varnish_svc_name = 'varnish-frontend', - $kafka_protocol_version = '0.9.0.1', -) inherits role::cache::kafka -{ - # Set varnish.arg.q or varnish.arg.m according to Varnish version - $varnish_opts = { 'q' => 'ReqURL ~ "^/(beacon/)?event(\.gif)?\?"' } - - varnishkafka::instance { 'eventlogging': - # FIXME - top-scope var without namespace, will break in puppet 2.8 - # lint:ignore:variable_scope - brokers => $kafka_brokers, - # lint:endignore - # Note that this format uses literal tab characters. - # The '-' in this string used to be %{X-Client-IP@ip}o. - # EventLogging clientIp logging has been removed as part of T128407. - format => '%q %l %n %{%FT%T}t - "%{User-agent}i"', - format_type => 'string', - topic => 'eventlogging-client-side', - varnish_name => $varnish_name, - varnish_svc_name => $varnish_svc_name, - varnish_opts => $varnish_opts, - topic_request_required_acks => '1', - force_protocol_version => $kafka_protocol_version, - } - - include ::standard - - # Generate icinga alert if varnishkafka is not running. - nrpe::monitor_service { 'varnishkafka-eventlogging': - description => 'eventlogging Varnishkafka log producer', - nrpe_command => "/usr/lib/nagios/plugins/check_procs -c 1 -a '/usr/bin/varnishkafka -S /etc/varnishkafka/eventlogging.conf'", - contact_group => 'admins,analytics', - require => Class['::varnishkafka'], - } - - $cache_type = hiera('cache::cluster') - $graphite_metric_prefix = "varnishkafka.${::hostname}.eventlogging.${cache_type}" - - # Sets up Logster to read from the Varnishkafka instance stats JSON file - # and report metrics to statsd. - varnishkafka::monitor::statsd { 'eventlogging': - graphite_metric_prefix => $graphite_metric_prefix, - statsd_host_port => hiera('statsd'), - } -} diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index d7d9c33..2e08cba 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -14,6 +14,13 @@ # (called 'statsv') then consumes these and emits metrics. include ::profile::cache::kafka::statsv + # varnishkafka eventlogging listens for eventlogging beacon + # requests and logs them to the eventlogging-client-side + # topic. EventLogging servers consume and process this + # topic into many JSON based kafka topics for further + # consumption. + include ::profile::cache::kafka::eventlogging + # TODO: refactor all this so that we have separate roles for production and labs if $::realm == 'production' and $::hostname != 'cp1008' { include ::role::ipsec -- To view, visit https://gerrit.wikimedia.org/r/403067 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4096fe7efda237bac162dfb5dc8af1262c445503 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <ao...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
