BBlack has submitted this change and it was merged. Change subject: VCL: explicit applayer backend selection ......................................................................
VCL: explicit applayer backend selection 1. Renames the director named "backend" from tier-one varnish-be instances; all applayer backends have explicit descriptive names, even if they're the only one. 2. Separates t1be backend selection into a new subroutine named cluster_be_recv_applayer_backend. 3. Gets rid of "has_def_backend", which was only used by the misc cluster t1be instances for similar purposes. Bug: T127481 Change-Id: I21035e0d83c4d45d39399cc2b215f535e3b8d9b3 --- M modules/role/manifests/cache/maps.pp M modules/role/manifests/cache/misc.pp M modules/role/manifests/cache/text.pp M modules/role/manifests/cache/upload.pp M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb M templates/varnish/maps-backend.inc.vcl.erb M templates/varnish/misc-backend.inc.vcl.erb M templates/varnish/text-backend.inc.vcl.erb M templates/varnish/upload-backend.inc.vcl.erb 11 files changed, 48 insertions(+), 33 deletions(-) Approvals: BBlack: Verified; Looks good to me, approved diff --git a/modules/role/manifests/cache/maps.pp b/modules/role/manifests/cache/maps.pp index 84bbdad..e89fadc 100644 --- a/modules/role/manifests/cache/maps.pp +++ b/modules/role/manifests/cache/maps.pp @@ -22,7 +22,7 @@ $varnish_be_directors = { 'one' => { - 'backend' => { + 'kartotherian' => { 'dynamic' => 'no', 'type' => 'random', # XXX note explicit abnormal hack: service only exists in codfw, but eqiad is Tier-1 in general diff --git a/modules/role/manifests/cache/misc.pp b/modules/role/manifests/cache/misc.pp index c10e791..74d33c3 100644 --- a/modules/role/manifests/cache/misc.pp +++ b/modules/role/manifests/cache/misc.pp @@ -259,22 +259,12 @@ } $fe_vcl_config = merge($common_vcl_config, { - 'layer' => 'frontend', + 'layer' => 'frontend', }) - $be_one_vcl_config = merge($common_vcl_config, { - 'layer' => 'backend', - 'has_def_backend' => 'no', + $be_vcl_config = merge($common_vcl_config, { + 'layer' => 'backend', }) - - $be_two_vcl_config = merge($common_vcl_config, { - 'layer' => 'backend', - }) - - $be_vcl_config = $::site_tier ? { - 'one' => $be_one_vcl_config, - default => $be_two_vcl_config, - } varnish::instance { 'misc-backend': name => '', diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index 88175ce..ff21e85 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -26,7 +26,7 @@ $varnish_be_directors = { 'one' => { - 'backend' => { + 'appservers' => { 'dynamic' => 'no', 'type' => 'random', 'backends' => $role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary], diff --git a/modules/role/manifests/cache/upload.pp b/modules/role/manifests/cache/upload.pp index 8fdf456..0e7eed7 100644 --- a/modules/role/manifests/cache/upload.pp +++ b/modules/role/manifests/cache/upload.pp @@ -23,7 +23,7 @@ $varnish_be_directors = { 'one' => { - 'backend' => { + 'swift' => { 'dynamic' => 'no', 'type' => 'random', 'backends' => $role::cache::configuration::backends[$::realm]['swift'][$::mw_primary], diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb index 6329f5e..d199ea9 100644 --- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb @@ -10,7 +10,18 @@ error 403 "Access denied"; } - call wm_common_recv; + call wm_common_recv_early; + +<% if @site_tier == "one" -%> + // tier-one caches must select an applayer backend + call cluster_be_recv_applayer_backend; +<% else -%> + // tier-two caches always have "backend" defined to tier-one varnish-backend instances + set req.backend = backend; +<% end -%> + + call wm_common_recv_grace; + call wm_common_recv_purge; call cluster_be_recv; diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb index 396b7d7..f7027bd 100644 --- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb @@ -173,7 +173,7 @@ } } -sub wm_common_recv { +sub wm_common_recv_early { unset req.http.X-CDIS; // clear internal cache-disposition header // XFF-appending is non-idempotent for restart purposes.. @@ -195,20 +195,14 @@ if ( req.http.host ~ "^varnishcheck" ) { error 200 "OK"; } +} - <% if @vcl_config.fetch("has_def_backend", "yes") == "yes" -%> - /* Select the default backend/director, which is always the one named 'backend'. - * If an instance has no default 'backend', it must declare has_def_backend==no, - * and its own VCL must handle all possible req.backend cases. - */ - set req.backend = backend; - +sub wm_common_recv_grace { if (req.backend.healthy) { set req.grace = 5m; } else { set req.grace = 60m; } - <% end %> } sub wm_common_hit { diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb index c41fe0c..5dde162 100644 --- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb @@ -232,7 +232,12 @@ call recv_fe_ip_processing; } - call wm_common_recv; + call wm_common_recv_early; + + // frontend caches always have "backend" defined to varnish-backend instances + set req.backend = backend; + + call wm_common_recv_grace; call rewrite_proxy_urls; diff --git a/templates/varnish/maps-backend.inc.vcl.erb b/templates/varnish/maps-backend.inc.vcl.erb index b9b4df6..7067443 100644 --- a/templates/varnish/maps-backend.inc.vcl.erb +++ b/templates/varnish/maps-backend.inc.vcl.erb @@ -1,4 +1,11 @@ // Varnish VCL include file for maps backends + +<% if @site_tier == "one" -%> +sub cluster_be_recv_applayer_backend { + set req.backend = kartotherian; +} +<% end -%> + sub cluster_be_recv { } sub cluster_be_hash { } sub cluster_be_hit { } diff --git a/templates/varnish/misc-backend.inc.vcl.erb b/templates/varnish/misc-backend.inc.vcl.erb index a510c89..9fe005e 100644 --- a/templates/varnish/misc-backend.inc.vcl.erb +++ b/templates/varnish/misc-backend.inc.vcl.erb @@ -1,7 +1,7 @@ include "misc-common.inc.vcl"; -sub cluster_be_recv { <% if @site_tier == "one" -%> +sub cluster_be_recv_applayer_backend { if (req.http.Host == "git.wikimedia.org") { set req.backend = antimony; } elsif (req.http.Host == "doc.wikimedia.org" || req.http.Host == "integration.wikimedia.org") { @@ -63,8 +63,10 @@ } else { error 404 "Domain not served here"; } -<% end -%> // end tier-1 backend selection +} +<% end -%> +sub cluster_be_recv { call misc_recv_pass; } diff --git a/templates/varnish/text-backend.inc.vcl.erb b/templates/varnish/text-backend.inc.vcl.erb index 0a13013..f94bb39 100644 --- a/templates/varnish/text-backend.inc.vcl.erb +++ b/templates/varnish/text-backend.inc.vcl.erb @@ -2,9 +2,8 @@ include "text-common.inc.vcl"; -sub cluster_be_recv { <% if @site_tier == "one" -%> - +sub cluster_be_recv_applayer_backend { if (req.url ~ "^/w/api\.php") { if (req.http.X-Wikimedia-Debug == "1") { set req.backend = appservers_debug; @@ -30,7 +29,7 @@ if (req.http.X-Wikimedia-Debug == "1") { set req.backend = appservers_debug; } else { - set req.backend = backend; // appservers.svc + set req.backend = appservers; } } @@ -39,9 +38,10 @@ set req.backend = security_audit; } <% end %> - +} <% end %> // end site_tier == one +sub cluster_be_recv { call text_common_recv; } diff --git a/templates/varnish/upload-backend.inc.vcl.erb b/templates/varnish/upload-backend.inc.vcl.erb index 10ec594..279fa2f 100644 --- a/templates/varnish/upload-backend.inc.vcl.erb +++ b/templates/varnish/upload-backend.inc.vcl.erb @@ -17,6 +17,12 @@ } } +<% if @site_tier == "one" -%> +sub cluster_be_recv_applayer_backend { + set req.backend = swift; +} +<% end -%> + sub cluster_be_recv { call upload_common_recv; -- To view, visit https://gerrit.wikimedia.org/r/274590 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I21035e0d83c4d45d39399cc2b215f535e3b8d9b3 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits