BBlack has submitted this change and it was merged.
Change subject: Separate HHVM app servers backend.
......................................................................
Separate HHVM app servers backend.
This creates a separate backend for regular app servers with HHVM.
If the request has a certain cookie ('hhvm' for now), the request
is sent to the HHVM app servers, provided it's not already assigned
a different backend like the API or testwiki.
The cache for responses coming from HHVM is separated from the one of
standard Zend appservers.
If a request gets restarted e.g. in vcl_error, the normal Zend
app servers are used instead, and the response in this case
is not cached.
The LVS service has been assigned an IP, but needs to be setup
seperately.
Tested in beta.
Change-Id: Ic9531496b8259bb32b80e3e613fb1df4687a036c
---
M manifests/role/cache.pp
M modules/lvs/manifests/configuration.pp
M modules/varnish/manifests/common/vcl.pp
M templates/varnish/bits.inc.vcl.erb
A templates/varnish/hhvm.inc.vcl.erb
M templates/varnish/mobile-backend.inc.vcl.erb
M templates/varnish/mobile-frontend.inc.vcl.erb
M templates/varnish/text-backend.inc.vcl.erb
M templates/varnish/text-frontend.inc.vcl.erb
9 files changed, 142 insertions(+), 12 deletions(-)
Approvals:
Ori.livneh: Looks good to me, but someone else must approve
BBlack: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/cache.pp b/manifests/role/cache.pp
index 223e773..86d5a2b 100644
--- a/manifests/role/cache.pp
+++ b/manifests/role/cache.pp
@@ -402,9 +402,10 @@
$backends = {
'production' => {
- 'appservers' =>
$lvs::configuration::lvs_service_ips['production']['apaches'],
- 'api' =>
$lvs::configuration::lvs_service_ips['production']['api'],
- 'rendering' =>
$lvs::configuration::lvs_service_ips['production']['rendering'],
+ 'appservers' =>
$lvs::configuration::lvs_service_ips['production']['apaches'],
+ 'hhvm_appservers' =>
$lvs::configuration::lvs_service_ips['production']['hhvm_appservers'],
+ 'api' =>
$lvs::configuration::lvs_service_ips['production']['api'],
+ 'rendering' =>
$lvs::configuration::lvs_service_ips['production']['rendering'],
'bits' => {
'eqiad' =>
flatten([$lvs::configuration::lvs_service_ips['production']['bits']['eqiad']['bitslb']]),
},
@@ -422,6 +423,11 @@
'appservers' => {
'eqiad' => [
'10.68.17.96', # deployment-mediawiki01
+ '10.68.17.208', # deployment-mediawiki02
+ ],
+ },
+ 'hhvm_appservers' => {
+ 'eqiad' => [
'10.68.17.208', # deployment-mediawiki02
],
},
@@ -709,10 +715,11 @@
$varnish_be_directors = {
1 => {
- 'backend' =>
$role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary],
- 'api' =>
$role::cache::configuration::backends[$::realm]['api'][$::mw_primary],
- 'rendering' =>
$role::cache::configuration::backends[$::realm]['rendering'][$::mw_primary],
- 'test_wikipedia'=>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
+ 'backend' =>
$role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary],
+ 'hhvm_appservers' =>
$role::cache::configuration::backends[$::realm]['hhvm_appservers'][$::mw_primary],
+ 'api' =>
$role::cache::configuration::backends[$::realm]['api'][$::mw_primary],
+ 'rendering' =>
$role::cache::configuration::backends[$::realm]['rendering'][$::mw_primary],
+ 'test_wikipedia' =>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
},
2 => {
'eqiad' =>
$role::cache::configuration::active_nodes[$::realm]['text']['eqiad'],
@@ -722,7 +729,6 @@
if $::realm == 'production' {
$storage_size_main = 300
}
-
include standard
include nrpe
@@ -1037,6 +1043,7 @@
1 => {
'backend' =>
$::role::cache::configuration::backends[$::realm]['bits_appservers'][$::mw_primary],
'test_wikipedia' =>
$::role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
+ 'hhvm_appservers' =>
$role::cache::configuration::backends[$::realm]['hhvm_appservers'][$::mw_primary],
},
2 => {
'backend' =>
sort(flatten(values($role::cache::configuration::backends[$::realm]['bits'])))
@@ -1132,9 +1139,11 @@
$varnish_be_directors = {
1 => {
- 'backend' =>
$role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary],
- 'api' =>
$role::cache::configuration::backends[$::realm]['api'][$::mw_primary],
- 'test_wikipedia' =>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
+ 'backend' =>
$role::cache::configuration::backends[$::realm]['appservers'][$::mw_primary],
+ 'api' =>
$role::cache::configuration::backends[$::realm]['api'][$::mw_primary],
+ 'test_wikipedia' =>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
+ 'hhvm_appservers' =>
$role::cache::configuration::backends[$::realm]['hhvm_appservers'][$::mw_primary],
+
},
2 => {
'eqiad' =>
$role::cache::configuration::active_nodes[$::realm]['mobile']['eqiad'],
diff --git a/modules/lvs/manifests/configuration.pp
b/modules/lvs/manifests/configuration.pp
index 1d3799d..2321b2a 100644
--- a/modules/lvs/manifests/configuration.pp
+++ b/modules/lvs/manifests/configuration.pp
@@ -222,6 +222,9 @@
'apaches' => {
'eqiad' => "10.2.2.1",
},
+ 'hhvm_appservers' => {
+ 'eqiad' => "10.2.2.2",
+ },
'rendering' => {
'eqiad' => "10.2.2.21",
},
@@ -294,6 +297,9 @@
'apaches' => {
'pmtpa' => "10.4.0.254",
},
+ 'hhvm_apaches' => {
+ # left empty as probably not used anymore
+ },
'rendering' => {
# Used to be 10.4.0.252
'pmtpa' => [ '10.4.0.166', '10.4.0.187', ],
diff --git a/modules/varnish/manifests/common/vcl.pp
b/modules/varnish/manifests/common/vcl.pp
index fa0e5a7..78209b8 100644
--- a/modules/varnish/manifests/common/vcl.pp
+++ b/modules/varnish/manifests/common/vcl.pp
@@ -17,6 +17,7 @@
content => template('varnish/errorpage.inc.vcl.erb'),
}
+
file { '/etc/varnish/via.inc.vcl':
owner => 'root',
group => 'root',
@@ -24,6 +25,13 @@
content => template('varnish/via.inc.vcl.erb'),
}
+ file { '/etc/varnish/hhvm.inc.vcl':
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template('varnish/hhvm.inc.vcl.erb'),
+ }
+
# VCL unit tests
file { '/usr/local/sbin/varnish-test-geoip':
owner => 'root',
diff --git a/templates/varnish/bits.inc.vcl.erb
b/templates/varnish/bits.inc.vcl.erb
index a4133c7..e8b783e 100644
--- a/templates/varnish/bits.inc.vcl.erb
+++ b/templates/varnish/bits.inc.vcl.erb
@@ -1,12 +1,26 @@
# Varnish VCL include file for bits
include "errorpage.inc.vcl";
+include "hhvm.inc.vcl";
sub vcl_recv {
/* Since we are allowing POST at wikimedia3.vcl.erb, disallow here */
if (req.request == "POST") {
error 403 "HTTP method not allowed.";
}
+
+ /* transform the cookie in an http header */
+ if (req.http.Cookie ~ "hhvm=true") {
+ set req.http.X-Use-HHVM = 1;
+<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
+ if (req.restarts == 0) {
+ set req.backend = hhvm_appservers;
+ }
+<% end -%>
+ } else {
+ unset req.http.X-Use-HHVM;
+ }
+
if (req.http.host == "<%= cluster_options.fetch( "bits_domain",
"bits.wikimedia.org" )%>") {
if (req.url ~ "^/event\.gif") {
error 204;
@@ -38,6 +52,12 @@
}
sub vcl_fetch {
+<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
+ call hhvm_add_vary;
+
+<% end -%>
+ call hhvm_is_cacheable;
+
/* Don't run the default vcl_fetch function */
return (deliver);
}
diff --git a/templates/varnish/hhvm.inc.vcl.erb
b/templates/varnish/hhvm.inc.vcl.erb
new file mode 100644
index 0000000..0843a10
--- /dev/null
+++ b/templates/varnish/hhvm.inc.vcl.erb
@@ -0,0 +1,21 @@
+# Common functions for HHVM handling
+
+sub hhvm_add_vary {
+ /* Set Vary to depend on X-Use-HHVM */
+ if (beresp.http.Vary !~ "X-Use-HHVM") {
+ if (beresp.http.Vary) {
+ set beresp.http.Vary = beresp.http.Vary + ",X-Use-HHVM";
+ } else {
+ set beresp.http.Vary = "X-Use-HHVM";
+ }
+ }
+}
+
+
+sub hhvm_is_cacheable {
+ /* do not cache requests that should go to hhvm but have been
+ * replied by Zend. */
+ if (req.http.X-Use-HHVM == "1" && beresp.http.X-Powered-By !~ "^HHVM") {
+ set beresp.ttl = 0s;
+ }
+}
diff --git a/templates/varnish/mobile-backend.inc.vcl.erb
b/templates/varnish/mobile-backend.inc.vcl.erb
index 7c56953..d4b178c 100644
--- a/templates/varnish/mobile-backend.inc.vcl.erb
+++ b/templates/varnish/mobile-backend.inc.vcl.erb
@@ -1,6 +1,7 @@
# Varnish VCL include file for mobile backends
include "errorpage.inc.vcl";
+include "hhvm.inc.vcl";
sub vcl_recv {
call vcl_recv_purge;
@@ -15,6 +16,14 @@
<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
if (req.url ~ "^/w/api.php") {
set req.backend = api;
+ }
+<% end -%>
+
+<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
+ /* Use the hhvm backend for any request to the standard appservers.
+ On a request restart (often due to an error), fallback to the Zend
pool */
+ if (req.http.X-Use-HHVM == "1" && req.restarts == 0) {
+ set req.backend = hhvm_appservers;
}
<% end -%>
@@ -35,10 +44,17 @@
set beresp.ttl = 60 s;
}
+<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
+ call hhvm_add_vary;
+<% end -%>
+
if (beresp.ttl <= 0s) {
set beresp.ttl = 120s;
return (hit_for_pass);
}
+
+ call hhvm_is_cacheable;
+
return (deliver);
}
diff --git a/templates/varnish/mobile-frontend.inc.vcl.erb
b/templates/varnish/mobile-frontend.inc.vcl.erb
index 0a43cdf..192b1cd 100644
--- a/templates/varnish/mobile-frontend.inc.vcl.erb
+++ b/templates/varnish/mobile-frontend.inc.vcl.erb
@@ -16,6 +16,9 @@
// address) in X-Forwarded-For, even if it's the only value, as in
// the example of traffic sourced directly by satellite or something.
+include "hhvm.inc.vcl";
+
+
sub filter_noise {
if (req.request == "POST" && req.url ~
"index\.php\?option=com_jce&task=plugin&plugin=imgmanager&file=imgmanager&method=form&cid=")
{
error 403 "Noise";
@@ -95,6 +98,17 @@
return (pass);
}
+ /* Set the hhvm header for requests:
+ * a) not for api
+ * b) not for testwiki */
+ if ( req.http.X-Orig-Cookie ~ "hhvm=true"
+ && req.http.host !~ "^test\."
+ && req.url !~ "^/w/api.php" ) {
+ set req.http.X-Use-HHVM = "1";
+ } else {
+ unset req.http.X-Use-HHVM;
+ }
+
set req.hash_ignore_busy = true;
unset req.http.Surrogate-Capability;
@@ -134,6 +148,8 @@
return (hit_for_pass);
}
*/
+
+ call hhvm_is_cacheable;
return (deliver);
}
diff --git a/templates/varnish/text-backend.inc.vcl.erb
b/templates/varnish/text-backend.inc.vcl.erb
index f9222cb..a607ce7 100644
--- a/templates/varnish/text-backend.inc.vcl.erb
+++ b/templates/varnish/text-backend.inc.vcl.erb
@@ -2,6 +2,7 @@
include "errorpage.inc.vcl";
include "text-common.inc.vcl";
+include "hhvm.inc.vcl";
<% if @vcl_config.fetch("cluster_tier", "1") != "1" -%>
# A random director with the same backends as the default 'backend' (chash)
director
@@ -36,7 +37,14 @@
set req.backend = test_wikipedia;
}
+ /* Use the hhvm backend for any request to the standard appservers.
+ On a request restart (often due to an error), fallback to the Zend
pool */
+ if ( req.http.X-Use-HHVM == "1" && req.restarts == 0) {
+ set req.backend = hhvm_appservers;
+ }
+
<% end -%>
+
call pass_requests;
if (req.request == "POST" || req.url ~
"^/wiki/Special:CentralAutoLogin/") {
@@ -63,7 +71,7 @@
/* Make sure Set-Cookie responses are not cacheable, and log violations
*/
/* FIXME: exceptions are ugly; maybe get rid of the whole thing? */
if (beresp.ttl > 0s && beresp.http.Set-Cookie &&
- req.url !~ "^/wiki/Special:HideBanners") {
+ req.url !~ "^/wiki/Special:HideBanners") {
std.log("Cacheable object with Set-Cookie found. req.url: " +
req.url + " Cache-Control: " + beresp.http.Cache-Control + " Set-Cookie: " +
beresp.http.Set-Cookie);
set beresp.http.Cache-Control = "private, max-age=0,
s-maxage=0";
set beresp.ttl = 0s;
@@ -81,10 +89,16 @@
}
}
+<% if vcl_config.fetch("cluster_tier", "1") == "1" -%>
+ call hhvm_add_vary;
+<% end -%>
+
if (beresp.ttl <= 0s) {
set beresp.ttl = 120s;
return (hit_for_pass);
}
+
+ call hhvm_is_cacheable;
return (deliver);
}
diff --git a/templates/varnish/text-frontend.inc.vcl.erb
b/templates/varnish/text-frontend.inc.vcl.erb
index 08de568..8ed710a 100644
--- a/templates/varnish/text-frontend.inc.vcl.erb
+++ b/templates/varnish/text-frontend.inc.vcl.erb
@@ -21,6 +21,8 @@
// mobile-frontend.inc.vcl.erb for (m|zero).wikipedia.org and its
// subdomains.
+include "hhvm.inc.vcl";
+
# A random director with the same backends as the default 'backend' (chash)
director
<%
default_backend = @vcl_config.fetch("default_backend", "backend")
@@ -36,6 +38,21 @@
.weight = 10;
}
<% end -%>
+}
+
+sub set_hhvm_header {
+ /* Set the hhvm header for requests:
+ * a) not for api
+ * b) not for the rendering backend
+ * c) not for testwiki */
+ if ( req.http.Cookie ~ "hhvm=true"
+ && req.url !~ "^/w/api\.php"
+ && req.url !~ "^/w/thumb(_handler)?\.php"
+ && req.http.host !~ "^test\.") {
+ set req.http.X-Use-HHVM = "1";
+ } else {
+ unset req.http.X-Use-HHVM;
+ }
}
sub mobile_redirect {
@@ -104,6 +121,7 @@
unset req.http.If-Modified-Since;
}
+ call set_hhvm_header;
call evaluate_cookie;
return(lookup);
@@ -122,6 +140,8 @@
set beresp.ttl = 120s;
return (hit_for_pass);
}
+
+ call hhvm_is_cacheable;
return (deliver);
}
--
To view, visit https://gerrit.wikimedia.org/r/152903
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic9531496b8259bb32b80e3e613fb1df4687a036c
Gerrit-PatchSet: 26
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mark Bergsma <[email protected]>
Gerrit-Reviewer: BBlack <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: Mark Bergsma <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
