BBlack has submitted this change and it was merged.
Change subject: caches: refactor around cache::cluster
......................................................................
caches: refactor around cache::cluster
This adds a new per-cache-cluster-role hiera key 'cache::cluster',
which is the short cache cluster name (e.g. 'text' for
'cache_text'), and then moves the common code for system::role,
webrequest, and reqstats out of the per-cluster roles and into
role::cache::base using the new hieradata (since all clusters
share these, and they only differ on the cluster name's use in
strings).
Change-Id: Ib9539891069c8f667d94922d6fb81da19bbddd7b
---
M hieradata/role/common/cache/maps.yaml
M hieradata/role/common/cache/misc.yaml
M hieradata/role/common/cache/text.yaml
M hieradata/role/common/cache/upload.yaml
M modules/role/manifests/cache/base.pp
M modules/role/manifests/cache/maps.pp
M modules/role/manifests/cache/misc.pp
M modules/role/manifests/cache/text.pp
M modules/role/manifests/cache/upload.pp
9 files changed, 22 insertions(+), 64 deletions(-)
Approvals:
BBlack: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/cache/maps.yaml
b/hieradata/role/common/cache/maps.yaml
index adf8c77..1554a96 100644
--- a/hieradata/role/common/cache/maps.yaml
+++ b/hieradata/role/common/cache/maps.yaml
@@ -1,4 +1,5 @@
cluster: cache_maps
+cache::cluster: maps
# The contents of this hash control our DC->DC routing for varnish backend
# daemons. There should be a key for every cache datacenter, and the
# values can be another datacenter or 'direct', which means contact the
diff --git a/hieradata/role/common/cache/misc.yaml
b/hieradata/role/common/cache/misc.yaml
index 48ca1af..6ef96d4 100644
--- a/hieradata/role/common/cache/misc.yaml
+++ b/hieradata/role/common/cache/misc.yaml
@@ -1,4 +1,5 @@
cluster: cache_misc
+cache::cluster: misc
# The contents of this hash control our DC->DC routing for varnish backend
# daemons. There should be a key for every cache datacenter, and the
# values can be another datacenter or 'direct', which means contact the
diff --git a/hieradata/role/common/cache/text.yaml
b/hieradata/role/common/cache/text.yaml
index 0616360..ae2fe39 100644
--- a/hieradata/role/common/cache/text.yaml
+++ b/hieradata/role/common/cache/text.yaml
@@ -1,4 +1,5 @@
cluster: cache_text
+cache::cluster: text
admin::groups:
- perf-roots
# The contents of this hash control our DC->DC routing for varnish backend
diff --git a/hieradata/role/common/cache/upload.yaml
b/hieradata/role/common/cache/upload.yaml
index bfde655..3b09e28 100644
--- a/hieradata/role/common/cache/upload.yaml
+++ b/hieradata/role/common/cache/upload.yaml
@@ -1,4 +1,5 @@
cluster: cache_upload
+cache::cluster: upload
admin::groups:
- perf-roots
# The contents of this hash control our DC->DC routing for varnish backend
diff --git a/modules/role/manifests/cache/base.pp
b/modules/role/manifests/cache/base.pp
index c103fa2..026e9ad 100644
--- a/modules/role/manifests/cache/base.pp
+++ b/modules/role/manifests/cache/base.pp
@@ -14,6 +14,12 @@
include network::constants
include conftool::scripts
+ $cache_cluster = hiera('cache::cluster')
+
+ system::role { "role::cache::${cache_cluster}":
+ description => "${cache_cluster} Varnish cache server",
+ }
+
# Client connection stats from the 'X-Connection-Properties'
# header set by the SSL terminators.
::varnish::logging::xcps { 'xcps':
@@ -25,6 +31,18 @@
key_prefix => "varnish.${::site}.backends",
}
+ # Install a varnishkafka producer to send
+ # varnish webrequest logs to Kafka.
+ class { 'role::cache::kafka::webrequest':
+ topic => "webrequest_${cache_cluster}",
+ }
+
+ # Parse varnishlogs for request statistics and send to statsd.
+ varnish::logging::reqstats { 'frontend':
+ metric_prefix => "varnish.${::site}.${cache_cluster}.frontend.request",
+ statsd => hiera('statsd'),
+ }
+
# Only production needs system perf tweaks and NFS client disable
if $::realm == 'production' {
include role::cache::perf
diff --git a/modules/role/manifests/cache/maps.pp
b/modules/role/manifests/cache/maps.pp
index 80cc7e6..adb6889 100644
--- a/modules/role/manifests/cache/maps.pp
+++ b/modules/role/manifests/cache/maps.pp
@@ -1,8 +1,4 @@
class role::cache::maps {
- system::role { 'role::cache::maps':
- description => 'maps Varnish cache server',
- }
-
include role::cache::2layer
include role::cache::ssl::unified
if $::standard::has_ganglia {
@@ -68,17 +64,5 @@
fe_cache_be_opts => $fe_cache_be_opts,
be_cache_be_opts => $be_cache_be_opts,
cluster_nodes => hiera('cache::maps::nodes'),
- }
-
- # Install a varnishkafka producer to send
- # varnish webrequest logs to Kafka.
- class { 'role::cache::kafka::webrequest':
- topic => 'webrequest_maps',
- }
-
- # Parse varnishlogs for request statistics and send to statsd.
- varnish::logging::reqstats { 'frontend':
- metric_prefix => "varnish.${::site}.maps.frontend.request",
- statsd => hiera('statsd'),
}
}
diff --git a/modules/role/manifests/cache/misc.pp
b/modules/role/manifests/cache/misc.pp
index 24ba614..695dc90 100644
--- a/modules/role/manifests/cache/misc.pp
+++ b/modules/role/manifests/cache/misc.pp
@@ -1,8 +1,4 @@
class role::cache::misc {
- system::role { 'role::cache::misc':
- description => 'misc Varnish cache server'
- }
-
include role::cache::2layer
include role::cache::ssl::misc
@@ -276,17 +272,5 @@
fe_cache_be_opts => $fe_cache_be_opts,
be_cache_be_opts => $be_cache_be_opts,
cluster_nodes => hiera('cache::misc::nodes'),
- }
-
- # Install a varnishkafka producer to send
- # varnish webrequest logs to Kafka.
- class { 'role::cache::kafka::webrequest':
- topic => 'webrequest_misc',
- }
-
- # Parse varnishlogs for request statistics and send to statsd.
- varnish::logging::reqstats { 'frontend':
- metric_prefix => "varnish.${::site}.misc.frontend.request",
- statsd => hiera('statsd'),
}
}
diff --git a/modules/role/manifests/cache/text.pp
b/modules/role/manifests/cache/text.pp
index 0a9ac56..7a77446 100644
--- a/modules/role/manifests/cache/text.pp
+++ b/modules/role/manifests/cache/text.pp
@@ -1,8 +1,4 @@
class role::cache::text {
- system::role { 'role::cache::text':
- description => 'text Varnish cache server',
- }
-
require geoip
require geoip::dev # for VCL compilation using libGeoIP
include role::cache::2layer
@@ -143,18 +139,6 @@
# consumption.
class { '::role::cache::kafka::eventlogging':
varnish_name => 'frontend',
- }
-
- # Install a varnishkafka producer to send
- # varnish webrequest logs to Kafka.
- class { 'role::cache::kafka::webrequest':
- topic => 'webrequest_text',
- }
-
- # Parse varnishlogs for request statistics and send to statsd.
- varnish::logging::reqstats { 'frontend':
- metric_prefix => "varnish.${::site}.text.frontend.request",
- statsd => hiera('statsd'),
}
# ResourceLoader browser cache hit rate and request volume stats.
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index a3cc388..39e0a4d 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -1,8 +1,4 @@
class role::cache::upload {
- system::role { 'role::cache::upload':
- description => 'upload Varnish cache server',
- }
-
include role::cache::2layer
include role::cache::ssl::unified
if $::standard::has_ganglia {
@@ -107,20 +103,8 @@
cluster_nodes => hiera('cache::upload::nodes'),
}
- # Install a varnishkafka producer to send
- # varnish webrequest logs to Kafka.
- class { 'role::cache::kafka::webrequest':
- topic => 'webrequest_upload',
- }
-
# Media browser cache hit rate and request volume stats.
::varnish::logging::media { 'media':
statsd_server => 'statsd.eqiad.wmnet',
- }
-
- # Parse varnishlogs for request statistics and send to statsd.
- varnish::logging::reqstats { 'frontend':
- metric_prefix => "varnish.${::site}.upload.frontend.request",
- statsd => hiera('statsd'),
}
}
--
To view, visit https://gerrit.wikimedia.org/r/289222
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib9539891069c8f667d94922d6fb81da19bbddd7b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>
Gerrit-Reviewer: BBlack <bbl...@wikimedia.org>
Gerrit-Reviewer: Ema <e...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
