Ottomata has submitted this change and it was merged.
Change subject: Setting up varnishkafka on 3 mobile varnish hosts.
......................................................................
Setting up varnishkafka on 3 mobile varnish hosts.
cp1046|cp3011|cp4011
Change-Id: I23d545e48b9046dd6d54df2281b3d76c82758771
---
M .gitmodules
M manifests/role/cache.pp
A modules/varnishkafka
3 files changed, 46 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/.gitmodules b/.gitmodules
index a5b3306..a78db6c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,3 +10,6 @@
[submodule "modules/kafka"]
path = modules/kafka
url = https://gerrit.wikimedia.org/r/p/operations/puppet/kafka
+[submodule "modules/varnishkafka"]
+ path = modules/varnishkafka
+ url = https://gerrit.wikimedia.org/r/operations/puppet/varnishkafka
diff --git a/manifests/role/cache.pp b/manifests/role/cache.pp
index 98db4be..a993aea 100644
--- a/manifests/role/cache.pp
+++ b/manifests/role/cache.pp
@@ -463,6 +463,36 @@
}
}
+ # == Class varnish::kafka
+ # Sets up a varnishkafka instance producing varnish
+ # logs to the analytics Kafka brokers in eqiad.
+ class varnish::kafka($topic)
+ {
+ # ToDo: Remove production conditional once this works
+ # is verified to work in labs.
+ if $::realm == 'production' {
+ require role::analytics::kafka::config
+ $kafka_brokers =
keys($role::analytics::kafka::config::brokers)
+
+ class { '::varnishkafka':
+ brokers => $kafka_brokers,
+ topic => $topic,
+ format_type => 'json',
+ compression_codec => 'snappy',
+ # Note: fake_tag tricks varnishkafka into
allowing hardcoded string into a JSON field.
+ # Hardcoding the $fqdn into hostname rather
than using %l to account for
+ # possible slip ups where varnish only writes
the short hostname for %l.
+ format =>
"%{fake_tag0@hostname?${::fqdn}}x %{@sequence!num?0}n %{%FT%T@dt}t
%{Varnish:time_firstbyte@time_firstbyte!num?0.0}x %{@ip}h
%{Varnish:handling@cache_status}x %{@http_status}s %{@response_size!num?0}b
%{@http_method}m %{Host@uri_host}i %{@uri_path}U %{@uri_query}q
%{Content-Type@content_type}o %{Referer@referer}i
%{X-Forwarded-For@x_forwarded_for}i %{User-Agent@user_agent}i
%{Accept-Language@accept_language}i %{X-Analytics@x_analytics}i",
+ message_send_max_retries => 3,
+ queue_buffering_max_messages => 1000000,
+ # large timeout to account for potential cross
DC latencies
+ topic_request_timeout_ms => 30000, #
request ack timeout
+ }
+
+ class { '::varnishkafka::monitoring': }
+ }
+ }
+
class varnish::logging::eventlistener {
if $::realm == 'production' {
$event_listener = $::site ? {
@@ -1086,7 +1116,20 @@
cluster_options => $cluster_options,
}
+ # varnish::logging to be removed once
+ # udp2log kafka consumer is implemented and deployed.
include role::cache::varnish::logging
+
+ # 2013-12-03: Rolling deployment of varnishkafka to mobiles.
+ # This conditional will be removed once all production mobiles
are running
+ # varnishkafka.
+ if ($::realm == 'labs' or $hostname =~ /cp1046|cp3011|cp4011/) {
+ # Install a varnishkafka producer to send
+ # varnish webrequest logs to Kafka.
+ class { 'role::cache::varnish::kafka':
+ topic => 'webrequest-mobile'
+ }
+ }
}
class parsoid inherits role::cache::varnish::2layer {
diff --git a/modules/varnishkafka b/modules/varnishkafka
new file mode 160000
index 0000000..4a9d826
--- /dev/null
+++ b/modules/varnishkafka
+Subproject commit 4a9d826ffd2aa0a07cc6d6ac0be6fd2c293febbc
--
To view, visit https://gerrit.wikimedia.org/r/94169
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I23d545e48b9046dd6d54df2281b3d76c82758771
Gerrit-PatchSet: 12
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Edenhill <mag...@edenhill.se>
Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org>
Gerrit-Reviewer: Mark Bergsma <m...@wikimedia.org>
Gerrit-Reviewer: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
