BBlack has submitted this change and it was merged.
Change subject: VCL: explicit applayer backend selection
......................................................................
VCL: explicit applayer backend selection
1. Renames the director named "backend" from tier-one varnish-be
instances; all applayer backends have explicit descriptive
names, even if they're the only one.
2. Separates t1be backend selection into a new subroutine named
cluster_be_recv_applayer_backend.
3. Gets rid of "has_def_backend", which was only used by the misc
cluster t1be instances for similar purposes.
Bug: T127481
Change-Id: I21035e0d83c4d45d39399cc2b215f535e3b8d9b3
---
M modules/role/manifests/cache/maps.pp
M modules/role/manifests/cache/misc.pp
M modules/role/manifests/cache/text.pp
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
M templates/varnish/maps-backend.inc.vcl.erb
M templates/varnish/misc-backend.inc.vcl.erb
M templates/varnish/text-backend.inc.vcl.erb
M templates/varnish/upload-backend.inc.vcl.erb
11 files changed, 48 insertions(+), 33 deletions(-)
Approvals:
BBlack: Verified; Looks good to me, approved
diff --git a/modules/role/manifests/cache/maps.pp
b/modules/role/manifests/cache/maps.pp
index 84bbdad..e89fadc 100644
--- a/modules/role/manifests/cache/maps.pp
+++ b/modules/role/manifests/cache/maps.pp
@@ -22,7 +22,7 @@
$varnish_be_directors = {
'one' => {
- 'backend' => {
+ 'kartotherian' => {
'dynamic' => 'no',
'type' => 'random',
# XXX note explicit abnormal hack: service only exists in
codfw, but eqiad is Tier-1 in general
diff --git a/modules/role/manifests/cache/misc.pp
b/modules/role/manifests/cache/misc.pp
index c10e791..74d33c3 100644
--- a/modules/role/manifests/cache/misc.pp
+++ b/modules/role/manifests/cache/misc.pp
@@ -259,22 +259,12 @@
}
$fe_vcl_config = merge($common_vcl_config, {
- 'layer' => 'frontend',
+ 'layer' => 'frontend',
})
- $be_one_vcl_config = merge($common_vcl_config, {
- 'layer' => 'backend',
- 'has_def_backend' => 'no',
+ $be_vcl_config = merge($common_vcl_config, {
+ 'layer' => 'backend',
})
-
- $be_two_vcl_config = merge($common_vcl_config, {
- 'layer' => 'backend',
- })
-
- $be_vcl_config = $::site_tier ? {
- 'one' => $be_one_vcl_config,
- default => $be_two_vcl_config,
- }
varnish::instance { 'misc-backend':
name => '',
diff --git a/modules/role/manifests/cache/text.pp
b/modules/role/manifests/cache/text.pp
index 88175ce..ff21e85 100644
--- a/modules/role/manifests/cache/text.pp
+++ b/modules/role/manifests/cache/text.pp
@@ -26,7 +26,7 @@
$varnish_be_directors = {
'one' => {
- 'backend' => {
+ 'appservers' => {
'dynamic' => 'no',
'type' => 'random',
'backends' =>
$role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary],
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index 8fdf456..0e7eed7 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -23,7 +23,7 @@
$varnish_be_directors = {
'one' => {
- 'backend' => {
+ 'swift' => {
'dynamic' => 'no',
'type' => 'random',
'backends' =>
$role::cache::configuration::backends[$::realm]['swift'][$::mw_primary],
diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
index 6329f5e..d199ea9 100644
--- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
@@ -10,7 +10,18 @@
error 403 "Access denied";
}
- call wm_common_recv;
+ call wm_common_recv_early;
+
+<% if @site_tier == "one" -%>
+ // tier-one caches must select an applayer backend
+ call cluster_be_recv_applayer_backend;
+<% else -%>
+ // tier-two caches always have "backend" defined to tier-one
varnish-backend instances
+ set req.backend = backend;
+<% end -%>
+
+ call wm_common_recv_grace;
+
call wm_common_recv_purge;
call cluster_be_recv;
diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
index 396b7d7..f7027bd 100644
--- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
@@ -173,7 +173,7 @@
}
}
-sub wm_common_recv {
+sub wm_common_recv_early {
unset req.http.X-CDIS; // clear internal cache-disposition header
// XFF-appending is non-idempotent for restart purposes..
@@ -195,20 +195,14 @@
if ( req.http.host ~ "^varnishcheck" ) {
error 200 "OK";
}
+}
- <% if @vcl_config.fetch("has_def_backend", "yes") == "yes" -%>
- /* Select the default backend/director, which is always the one named
'backend'.
- * If an instance has no default 'backend', it must declare
has_def_backend==no,
- * and its own VCL must handle all possible req.backend cases.
- */
- set req.backend = backend;
-
+sub wm_common_recv_grace {
if (req.backend.healthy) {
set req.grace = 5m;
} else {
set req.grace = 60m;
}
- <% end %>
}
sub wm_common_hit {
diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
index c41fe0c..5dde162 100644
--- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
@@ -232,7 +232,12 @@
call recv_fe_ip_processing;
}
- call wm_common_recv;
+ call wm_common_recv_early;
+
+ // frontend caches always have "backend" defined to varnish-backend
instances
+ set req.backend = backend;
+
+ call wm_common_recv_grace;
call rewrite_proxy_urls;
diff --git a/templates/varnish/maps-backend.inc.vcl.erb
b/templates/varnish/maps-backend.inc.vcl.erb
index b9b4df6..7067443 100644
--- a/templates/varnish/maps-backend.inc.vcl.erb
+++ b/templates/varnish/maps-backend.inc.vcl.erb
@@ -1,4 +1,11 @@
// Varnish VCL include file for maps backends
+
+<% if @site_tier == "one" -%>
+sub cluster_be_recv_applayer_backend {
+ set req.backend = kartotherian;
+}
+<% end -%>
+
sub cluster_be_recv { }
sub cluster_be_hash { }
sub cluster_be_hit { }
diff --git a/templates/varnish/misc-backend.inc.vcl.erb
b/templates/varnish/misc-backend.inc.vcl.erb
index a510c89..9fe005e 100644
--- a/templates/varnish/misc-backend.inc.vcl.erb
+++ b/templates/varnish/misc-backend.inc.vcl.erb
@@ -1,7 +1,7 @@
include "misc-common.inc.vcl";
-sub cluster_be_recv {
<% if @site_tier == "one" -%>
+sub cluster_be_recv_applayer_backend {
if (req.http.Host == "git.wikimedia.org") {
set req.backend = antimony;
} elsif (req.http.Host == "doc.wikimedia.org" || req.http.Host ==
"integration.wikimedia.org") {
@@ -63,8 +63,10 @@
} else {
error 404 "Domain not served here";
}
-<% end -%> // end tier-1 backend selection
+}
+<% end -%>
+sub cluster_be_recv {
call misc_recv_pass;
}
diff --git a/templates/varnish/text-backend.inc.vcl.erb
b/templates/varnish/text-backend.inc.vcl.erb
index 0a13013..f94bb39 100644
--- a/templates/varnish/text-backend.inc.vcl.erb
+++ b/templates/varnish/text-backend.inc.vcl.erb
@@ -2,9 +2,8 @@
include "text-common.inc.vcl";
-sub cluster_be_recv {
<% if @site_tier == "one" -%>
-
+sub cluster_be_recv_applayer_backend {
if (req.url ~ "^/w/api\.php") {
if (req.http.X-Wikimedia-Debug == "1") {
set req.backend = appservers_debug;
@@ -30,7 +29,7 @@
if (req.http.X-Wikimedia-Debug == "1") {
set req.backend = appservers_debug;
} else {
- set req.backend = backend; // appservers.svc
+ set req.backend = appservers;
}
}
@@ -39,9 +38,10 @@
set req.backend = security_audit;
}
<% end %>
-
+}
<% end %> // end site_tier == one
+sub cluster_be_recv {
call text_common_recv;
}
diff --git a/templates/varnish/upload-backend.inc.vcl.erb
b/templates/varnish/upload-backend.inc.vcl.erb
index 10ec594..279fa2f 100644
--- a/templates/varnish/upload-backend.inc.vcl.erb
+++ b/templates/varnish/upload-backend.inc.vcl.erb
@@ -17,6 +17,12 @@
}
}
+<% if @site_tier == "one" -%>
+sub cluster_be_recv_applayer_backend {
+ set req.backend = swift;
+}
+<% end -%>
+
sub cluster_be_recv {
call upload_common_recv;
--
To view, visit https://gerrit.wikimedia.org/r/274590
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I21035e0d83c4d45d39399cc2b215f535e3b8d9b3
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>
Gerrit-Reviewer: BBlack <bbl...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
