BBlack has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/339668 )
Change subject: varnish: switch all clusters to req_handling [WIP, 2/4] ...................................................................... varnish: switch all clusters to req_handling [WIP, 2/4] This also adds "debug_director" to support the text cluster's unique debug backends logic. Bug: T134404 Change-Id: I757b420c4791398d947c6e725b5508a7f70db4f3 --- M modules/role/manifests/cache/maps.pp M modules/role/manifests/cache/text.pp M modules/role/manifests/cache/upload.pp M modules/varnish/templates/maps-backend.inc.vcl.erb M modules/varnish/templates/misc-backend.inc.vcl.erb M modules/varnish/templates/misc-common.inc.vcl.erb M modules/varnish/templates/text-backend.inc.vcl.erb M modules/varnish/templates/text-common.inc.vcl.erb M modules/varnish/templates/upload-backend.inc.vcl.erb M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb 12 files changed, 171 insertions(+), 165 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/68/339668/1 diff --git a/modules/role/manifests/cache/maps.pp b/modules/role/manifests/cache/maps.pp index 247fd1b..f49b4bb 100644 --- a/modules/role/manifests/cache/maps.pp +++ b/modules/role/manifests/cache/maps.pp @@ -36,7 +36,6 @@ 'max_connections' => 1000, } - $apps = hiera('cache::maps::apps') $app_directors = { 'kartotherian' => { 'backend' => 'kartotherian.svc.eqiad.wmnet', @@ -47,6 +46,7 @@ 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re, 'ttl_cap' => '1d', 'pass_random' => true, + 'req_handling' => { director => 'kartotherian' }, } $common_runtime_params = ['default_ttl=86400'] diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index 4303d79..8b71c85 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -76,12 +76,41 @@ }, } + $req_handling = { + 'cxserver.wikimedia.org' => { + 'director' => 'cxserver_backend', + 'caching' => 'pass', + }, + 'citoid.wikimedia.org' => { + 'director' => 'citoid_backend', + 'caching' => 'pass', + }, + 'default' => { + 'director' => 'appservers', + 'debug_director' => 'appservers_debug', + 'subpaths' => { + '^/api/rest_v1/' => { + 'director' => 'restbase_backend' + }, + '^/w/api\.php' => { + 'director' => 'api', + 'debug_director' => 'appservers_debug', + }, + '^/w/thumb(_handler)?\.php' => { + 'director' => 'rendering', + 'debug_director' => 'appservers_debug', + } + } + }, + } + $common_vcl_config = { 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re, 'static_host' => $static_host, 'top_domain' => $top_domain, 'shortener_domain' => $shortener_domain, 'pass_random' => true, + 'req_handling' => $req_handling, } $be_vcl_config = $common_vcl_config diff --git a/modules/role/manifests/cache/upload.pp b/modules/role/manifests/cache/upload.pp index 482f5b4..95ed0c9 100644 --- a/modules/role/manifests/cache/upload.pp +++ b/modules/role/manifests/cache/upload.pp @@ -40,7 +40,6 @@ 'max_connections' => 10000, } - $apps = hiera('cache::upload::apps') $app_directors = { 'swift' => { 'backend' => 'ms-fe.svc.eqiad.wmnet', @@ -50,10 +49,20 @@ }, } + $req_handling = { + 'director' => 'swift', + 'subpaths' => { + '^/+[^/]+/[^/]+/thumb/' => { + director => 'swift_thumbs', + }, + }, + } + $common_vcl_config = { 'purge_host_regex' => $::role::cache::base::purge_host_only_upload_re, 'upload_domain' => $upload_domain, 'allowed_methods' => '^(GET|HEAD|OPTIONS|PURGE)$', + 'req_handling' => $req_handling, } # Note pass_random true in BE, false in FE below. diff --git a/modules/varnish/templates/maps-backend.inc.vcl.erb b/modules/varnish/templates/maps-backend.inc.vcl.erb index 3c3b110..cc0330c 100644 --- a/modules/varnish/templates/maps-backend.inc.vcl.erb +++ b/modules/varnish/templates/maps-backend.inc.vcl.erb @@ -1,11 +1,6 @@ // Varnish VCL include file for maps backends sub cluster_be_recv_pre_purge { } - -sub cluster_be_recv_applayer_backend { - set req.backend_hint = kartotherian.backend(); -} - sub cluster_be_recv { } sub cluster_be_hash { } sub cluster_be_hit { } diff --git a/modules/varnish/templates/misc-backend.inc.vcl.erb b/modules/varnish/templates/misc-backend.inc.vcl.erb index 77a4cfe..feebcab 100644 --- a/modules/varnish/templates/misc-backend.inc.vcl.erb +++ b/modules/varnish/templates/misc-backend.inc.vcl.erb @@ -2,45 +2,6 @@ sub cluster_be_recv_pre_purge { } -sub cluster_be_recv_applayer_backend { -<% - def set_director(dirname) - return "set req.backend_hint = #{dirname}.backend();" - end - - def_action = 'return (synth(404, "Domain not served here"));' - if_stmts = [] - @vcl_config['req_handling'].keys.sort.each do |reqhost| - options = @vcl_config['req_handling'][reqhost] - host_action = set_director(options['director']) - if options.has_key?('subpaths') - path_ifs = [] - options['subpaths'].keys.sort.each do |subpath| - path_action = set_director(options['subpaths'][subpath]['director']) - path_ifs.push(%Q[if (req.url ~ "#{subpath}") {\n #{path_action}\n }]) - end - path_ifs.push(%Q[e {\n #{host_action}\n }]) - host_action = path_ifs.join(' els') - end - - if reqhost == 'default' - def_action = host_action - else - if reqhost =~ /^[-.A-Za-z0-9]+$/ - hostop = '==' - else - hostop = '~' - end - hostcmp = %Q[req.http.Host #{hostop} "#{reqhost}"] - if_stmts.push("if (#{hostcmp}) {\n #{host_action}\n }") - end - end - if_stmts.push("e {\n #{def_action}\n }") - set_backend = if_stmts.join(' els') -%> - <%= set_backend %> -} - sub cluster_be_recv { call misc_recv_pass; } diff --git a/modules/varnish/templates/misc-common.inc.vcl.erb b/modules/varnish/templates/misc-common.inc.vcl.erb index 1ca2111..db2e8fd 100644 --- a/modules/varnish/templates/misc-common.inc.vcl.erb +++ b/modules/varnish/templates/misc-common.inc.vcl.erb @@ -1,75 +1,4 @@ sub misc_recv_pass { -<% - def uc_action(uc) - if uc == "pass" - return "return (pass);" - elsif uc == "pipe" - return "return (pipe);" - elsif uc == "websockets" - return %Q[if (req.http.upgrade ~ "(?i)websocket") { return (pipe); } else { return (pass); }] - elsif uc == "normal" - return "" - else - #throw error XXX - end - end - - caching = [] - caching_default = false - @vcl_config['req_handling'].keys.sort.each do |reqhost| - if reqhost == 'default' - host_cmp = %Q[e] # "else" when join(' els') applied later - elsif reqhost =~ /^[-.A-Za-z0-9]+$/ - host_cmp = %Q[if (req.http.host == "#{reqhost}")] - else - host_cmp = %Q[if (req.http.host ~ "#{reqhost}")] - end - - need_host = false - host_action = false - options = @vcl_config['req_handling'][reqhost] - if options.has_key?('caching') - host_action = uc_action(options['caching']) - need_host = true - end - - if options.has_key?('subpaths') - path_ifs = [] - options['subpaths'].keys.sort.each do |subpath| - path_options = options['subpaths'][subpath] - path_cmp = %Q[if (req.url ~ "#{subpath}")] - if path_options.has_key?('caching') - need_host = true - path_action = uc_action(path_options['caching']) - path_ifs.push(%Q[#{path_cmp} {\n #{path_action}\n }]) - else - path_ifs.push(%Q[#{path_cmp} { }]) - end - end - if host_action - path_ifs.push(%Q[e { #{host_action} }]) - end - if need_host - host_action = path_ifs.join(' els') - end - end - - if need_host - if reqhost == 'default' - caching_default = %Q[#{host_cmp} { #{host_action} }] - else - caching.push(%Q[#{host_cmp} { #{host_action} }]) - end - end - end - - if caching_default - caching.push(caching_default) - end - caching_vcl = caching.join("\n "); -%> - <%= caching_vcl %> - if (req.method != "GET" && req.method != "HEAD") { // We only deal with GET and HEAD return (pass); diff --git a/modules/varnish/templates/text-backend.inc.vcl.erb b/modules/varnish/templates/text-backend.inc.vcl.erb index 8a3cfc6..62a54ef 100644 --- a/modules/varnish/templates/text-backend.inc.vcl.erb +++ b/modules/varnish/templates/text-backend.inc.vcl.erb @@ -8,40 +8,14 @@ } } -sub cluster_be_recv_applayer_backend { - if (req.http.Host == "cxserver.wikimedia.org" ) { # LEGACY: to be removed eventually - set req.backend_hint = cxserver_backend.backend(); - } else if (req.http.Host == "citoid.wikimedia.org" ) { # LEGACY: to be removed eventually - set req.backend_hint = citoid_backend.backend(); - } else { // default for all other hostnames - if (req.url ~ "^/api/rest_v1/") { - set req.backend_hint = restbase_backend.backend(); - } else if (req.url ~ "^/w/api\.php") { - set req.backend_hint = api.backend(); - set req.http.X-Backend-is-Mediawiki = 1; - } else if (req.url ~ "^/w/thumb(_handler)?\.php") { - set req.backend_hint = rendering.backend(); - set req.http.X-Backend-is-Mediawiki = 1; - } else { - set req.backend_hint = appservers.backend(); - set req.http.X-Backend-is-Mediawiki = 1; - } - } - - if (req.http.X-Wikimedia-Debug && req.http.X-Backend-is-Mediawiki) { - set req.backend_hint = appservers_debug.backend(); - unset req.http.X-Backend-is-Mediawiki; - } - +sub cluster_be_recv { + call text_common_recv; +// This only applies for beta, which only has a fake "eqiad" and always goes "direct"... <% if @realm != "production" %> if (req.http.X-Wikimedia-Security-Audit == "1") { set req.backend_hint = security_audit.backend(); } <% end %> -} - -sub cluster_be_recv { - call text_common_recv; } sub cluster_be_hash { diff --git a/modules/varnish/templates/text-common.inc.vcl.erb b/modules/varnish/templates/text-common.inc.vcl.erb index 4848fb6..d21afda 100644 --- a/modules/varnish/templates/text-common.inc.vcl.erb +++ b/modules/varnish/templates/text-common.inc.vcl.erb @@ -88,11 +88,6 @@ return (pass); } - # LEGACY: to be removed eventually - if (req.http.Host ~ "^(cxserver|citoid)\.wikimedia\.org$" ) { - return (pass); - } - call evaluate_cookie; call pass_authorization; } diff --git a/modules/varnish/templates/upload-backend.inc.vcl.erb b/modules/varnish/templates/upload-backend.inc.vcl.erb index f8c278a..7635586 100644 --- a/modules/varnish/templates/upload-backend.inc.vcl.erb +++ b/modules/varnish/templates/upload-backend.inc.vcl.erb @@ -29,14 +29,6 @@ sub cluster_be_recv_pre_purge { } -sub cluster_be_recv_applayer_backend { - if (req.url ~ "^/+[^/]+/[^/]+/thumb/") { - set req.backend_hint = swift_thumbs.backend(); - } else { - set req.backend_hint = swift.backend(); - } -} - sub cluster_be_recv { call upload_common_recv; } diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb index bd894e6..71b16cd 100644 --- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb @@ -10,6 +10,58 @@ call wm_common_directors_init; } +sub set_backend_app__ { +<% + def set_director(dirname, debug_dir) + if debug_dir.nil? + return "set req.backend_hint = #{dirname}.backend();" + else + return "if (req.http.X-Wikimedia-Debug) { set req.backend_hint = #{debug_dir}.backend(); } else { set req.backend_hint = #{dirname}.backend(); }" + end + end + + def_action = 'return (synth(404, "Domain not served here"));' + if_stmts = [] + @vcl_config['req_handling'].keys.sort.each do |reqhost| + options = @vcl_config['req_handling'][reqhost] + host_action = set_director(options['director'], options['debug_director']) + if options.has_key?('subpaths') + path_ifs = [] + options['subpaths'].keys.sort.each do |subpath| + path_action = set_director(options['subpaths'][subpath]['director'], options['subpaths'][subpath]['debug_director']) + path_ifs.push(%Q[if (req.url ~ "#{subpath}") {\n #{path_action}\n }]) + end + path_ifs.push(%Q[e {\n #{host_action}\n }]) + host_action = path_ifs.join(' els') + end + + if reqhost == 'default' + def_action = host_action + else + if reqhost =~ /^[-.A-Za-z0-9]+$/ + hostop = '==' + else + hostop = '~' + end + hostcmp = %Q[req.http.Host #{hostop} "#{reqhost}"] + if_stmts.push("if (#{hostcmp}) {\n #{host_action}\n }") + end + end + if_stmts.push("e {\n #{def_action}\n }") + set_backend = if_stmts.join(' els') +%> + <%= set_backend %> +} + +sub set_backend__ { +<% if @cache_route == 'direct' -%> + // tier-one caches must select an applayer backend + call set_backend_app__; +<% else -%> + set req.backend_hint = cache_<%= @cache_route %>.backend(); +<% end -%> +} + sub vcl_recv { if (client.ip !~ wikimedia_trust) { // Do not allow direct access to non-frontend layers @@ -34,17 +86,13 @@ } } -<% if @cache_route == 'direct' -%> - // tier-one caches must select an applayer backend - call cluster_be_recv_applayer_backend; -<% else -%> - set req.backend_hint = cache_<%= @cache_route %>.backend(); -<% end -%> + call set_backend__; call wm_common_recv_grace; call cluster_be_recv_pre_purge; call wm_common_recv_purge; call cluster_be_recv; + call wm_common_recv_pass; return (hash); // no default VCL } diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb index d95dec1..50ea0d2 100644 --- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb @@ -216,6 +216,79 @@ # Functions +sub wm_common_recv_pass { +<% + def uc_action(uc) + if uc == "pass" + return "return (pass);" + elsif uc == "pipe" + return "return (pipe);" + elsif uc == "websockets" + return %Q[if (req.http.upgrade ~ "(?i)websocket") { return (pipe); } else { return (pass); }] + elsif uc == "normal" + return "" + else + #throw error XXX + end + end + + caching = [] + caching_default = false + @vcl_config['req_handling'].keys.sort.each do |reqhost| + if reqhost == 'default' + host_cmp = %Q[e] # "else" when join(' els') applied later + elsif reqhost =~ /^[-.A-Za-z0-9]+$/ + host_cmp = %Q[if (req.http.host == "#{reqhost}")] + else + host_cmp = %Q[if (req.http.host ~ "#{reqhost}")] + end + + need_host = false + host_action = false + options = @vcl_config['req_handling'][reqhost] + if options.has_key?('caching') + host_action = uc_action(options['caching']) + need_host = true + end + + if options.has_key?('subpaths') + path_ifs = [] + options['subpaths'].keys.sort.each do |subpath| + path_options = options['subpaths'][subpath] + path_cmp = %Q[if (req.url ~ "#{subpath}")] + if path_options.has_key?('caching') + need_host = true + path_action = uc_action(path_options['caching']) + path_ifs.push(%Q[#{path_cmp} {\n #{path_action}\n }]) + else + path_ifs.push(%Q[#{path_cmp} { }]) + end + end + if host_action + path_ifs.push(%Q[e { #{host_action} }]) + end + if need_host + host_action = path_ifs.join(' els') + end + end + + if need_host + if reqhost == 'default' + caching_default = %Q[#{host_cmp} { #{host_action} }] + else + caching.push(%Q[#{host_cmp} { #{host_action} }]) + end + end + end + + if caching_default + caching.push(caching_default) + end + caching_vcl = caching.join("\n "); +%> + <%= caching_vcl %> +} + sub wm_common_recv_purge { /* Support HTTP PURGE */ if (req.method == "PURGE") { diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb index 59af6d7..515b799 100644 --- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb +++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb @@ -268,6 +268,7 @@ call cluster_fe_recv_pre_purge; call wm_common_recv_purge; call cluster_fe_recv; + call wm_common_recv_pass; return (hash); // no default VCL } -- To view, visit https://gerrit.wikimedia.org/r/339668 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I757b420c4791398d947c6e725b5508a7f70db4f3 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BBlack <bbl...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits