Gilles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/264084
Change subject: Add HAProxy in front of Thumbors
......................................................................
Add HAProxy in front of Thumbors
This get us consistent hash load balancing as well as
abuse throttling.
Bug: T123461
Change-Id: Ibe44bc16262f81cebcbd7c7bfa02c5f7255f065f
---
M puppet/hieradata/common.yaml
M puppet/modules/role/settings/thumbor.yaml
M puppet/modules/thumbor/manifests/init.pp
A puppet/modules/thumbor/templates/haproxy-upstart.erb
A puppet/modules/thumbor/templates/haproxy.conf.erb
R puppet/modules/thumbor/templates/thumbor-upstart.erb
M puppet/modules/thumbor/templates/thumbor.conf.erb
M puppet/modules/thumbor/templates/varnish.vcl.erb
8 files changed, 144 insertions(+), 14 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/84/264084/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index af91aa7..e6dfcc8 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -329,6 +329,7 @@
thumbor::deploy_dir: "%{hiera('mwv::vendor_dir')}/thumbor"
thumbor::cfg_file: /etc/thumbor.conf
+thumbor::haproxy_cfg_file: /etc/haproxy-thumbor.conf
thumbor::statsd_port: "%{hiera('statsd::port')}"
thumbor::sentry_dsn_file: "%{hiera('sentry::dsn_file')}"
diff --git a/puppet/modules/role/settings/thumbor.yaml
b/puppet/modules/role/settings/thumbor.yaml
index 0565c23..71bad5f 100644
--- a/puppet/modules/role/settings/thumbor.yaml
+++ b/puppet/modules/role/settings/thumbor.yaml
@@ -1,5 +1,8 @@
vagrant_ram: 512
forward_ports:
- 8888: 8888 # thumbor
+ 8888: 8888 # haproxy pointing to thumbors
+ 8889: 8889 # haproxy stats
+ 8890: 8890 # thumbor1
+ 8891: 8891 # thumbor2
6081: 6081 # varnish
8040: 8040 # swift
\ No newline at end of file
diff --git a/puppet/modules/thumbor/manifests/init.pp
b/puppet/modules/thumbor/manifests/init.pp
index 8241b44..407a7f3 100644
--- a/puppet/modules/thumbor/manifests/init.pp
+++ b/puppet/modules/thumbor/manifests/init.pp
@@ -13,6 +13,9 @@
# [*cfg_file*]
# Thumbor configuration file. The file will be generated by Puppet.
#
+# [*haproxy_cfg_file*]
+# HAProxy configuration file. The file will be generated by Puppet.
+#
# [*statsd_port*]
# Port the statsd instance runs on.
#
@@ -22,6 +25,7 @@
class thumbor (
$deploy_dir,
$cfg_file,
+ $haproxy_cfg_file,
$statsd_port,
$sentry_dsn_file,
) {
@@ -58,10 +62,17 @@
# For Ghostscript engine (PDF)
require_package('ghostscript')
+ # For load-balancing
+ require_package('haproxy')
+
$statsd_host = 'localhost'
$statsd_prefix = 'Thumbor'
group { 'thumbor':
+ ensure => present,
+ }
+
+ group { 'haproxy':
ensure => present,
}
@@ -70,6 +81,13 @@
home => '/var/run/thumbor',
gid => 'thumbor',
require => Group['thumbor'],
+ }
+
+ user { 'haproxy':
+ ensure => present,
+ home => '/var/run/haproxy',
+ gid => 'haproxy',
+ require => Group['haproxy'],
}
virtualenv::environment { $deploy_dir:
@@ -114,28 +132,46 @@
require => User['thumbor'],
}
+ file { $haproxy_cfg_file:
+ ensure => present,
+ group => 'haproxy',
+ owner => 'haproxy',
+ content => template('thumbor/haproxy.conf.erb'),
+ mode => '0640',
+ require => User['haproxy'],
+ }
+
cgroup::config { 'thumbor':
limits => "perm { task { uid = thumbor; gid = thumbor; } admin { uid
= thumbor; gid = thumbor; } } memory { memory.limit_in_bytes = \"1048576000\";
}", # 1GB
cgrules => '@thumbor memory thumbor',
}
- file { '/etc/init/thumbor.conf':
+ thumbor::service { '8890':
+ deploy_dir => $deploy_dir,
+ cfg_file => $cfg_file,
+ }
+
+ thumbor::service { '8891':
+ deploy_dir => $deploy_dir,
+ cfg_file => $cfg_file,
+ }
+
+ file { '/etc/init/haproxy-thumbor.conf':
ensure => present,
- content => template('thumbor/upstart.erb'),
+ content => template('thumbor/haproxy-upstart.erb'),
mode => '0444',
}
- service { 'thumbor':
+ service { 'haproxy':
ensure => running,
enable => true,
provider => 'upstart',
- require => [
- Virtualenv::Environment[$deploy_dir],
- User['thumbor'],
- ],
+ require => User['haproxy'],
subscribe => [
- File["${deploy_dir}/tinyrgb.icc", $cfg_file,
'/etc/init/thumbor.conf'],
- Cgroup::Config['thumbor'],
+ File[
+ $haproxy_cfg_file,
+ '/etc/init/haproxy-thumbor.conf'
+ ],
],
}
@@ -156,3 +192,32 @@
order => 49, # Needs to be before default for vcl_recv override
}
}
+
+define thumbor::service (
+ $deploy_dir,
+ $cfg_file,
+) {
+ file { "/etc/init/thumbor-${title}.conf":
+ ensure => present,
+ content => template('thumbor/thumbor-upstart.erb'),
+ mode => '0444',
+ }
+
+ service { "thumbor-${title}":
+ ensure => running,
+ enable => true,
+ provider => 'upstart',
+ require => [
+ Virtualenv::Environment[$deploy_dir],
+ User['thumbor'],
+ ],
+ subscribe => [
+ File[
+ "${deploy_dir}/tinyrgb.icc",
+ $cfg_file,
+ "/etc/init/thumbor-${title}.conf"
+ ],
+ Cgroup::Config['thumbor'],
+ ],
+ }
+}
diff --git a/puppet/modules/thumbor/templates/haproxy-upstart.erb
b/puppet/modules/thumbor/templates/haproxy-upstart.erb
new file mode 100644
index 0000000..ef0e5e2
--- /dev/null
+++ b/puppet/modules/thumbor/templates/haproxy-upstart.erb
@@ -0,0 +1,13 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+description "HAProxy"
+
+start on mediawiki-ready
+respawn
+
+setuid haproxy
+setgid haproxy
+
+exec /usr/sbin/haproxy -f <%= @haproxy_cfg_file %>
\ No newline at end of file
diff --git a/puppet/modules/thumbor/templates/haproxy.conf.erb
b/puppet/modules/thumbor/templates/haproxy.conf.erb
new file mode 100644
index 0000000..7cd8494
--- /dev/null
+++ b/puppet/modules/thumbor/templates/haproxy.conf.erb
@@ -0,0 +1,42 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+global
+ maxconn 256
+
+defaults
+ mode http
+ timeout connect 5000ms
+ timeout client 50000ms
+ timeout server 50000ms
+ option http-server-close
+
+listen stats
+ bind *:8889
+ stats uri /stats
+
+frontend fe_thumbs
+ bind *:8888
+ default_backend be_thumbor
+
+ # A client cannot send more than 10 connection per second
+ acl too_fast fe_sess_rate ge 10
+
+ rspadd X-Backend-Server:\ thumbor1 if { srv_id 1 }
+ rspadd X-Backend-Server:\ thumbor2 if { srv_id 2 }
+
+ # Accept connections that aren't too fast
+ tcp-request content accept unless too_fast
+
+ # Add a delay for badly behaved requests (which weren't accepted by the
previous command)
+ tcp-request inspect-delay 5s
+
+ # Accept throttled connections after the wait
+ tcp-request content accept if WAIT_END
+
+backend be_thumbor
+ balance hdr(xkey)
+ hash-type consistent
+ server thumbor1 127.0.0.1:8890 check id 1
+ server thumbor2 127.0.0.1:8891 check id 2
\ No newline at end of file
diff --git a/puppet/modules/thumbor/templates/upstart.erb
b/puppet/modules/thumbor/templates/thumbor-upstart.erb
similarity index 85%
rename from puppet/modules/thumbor/templates/upstart.erb
rename to puppet/modules/thumbor/templates/thumbor-upstart.erb
index 4b7260d..0d69745 100644
--- a/puppet/modules/thumbor/templates/upstart.erb
+++ b/puppet/modules/thumbor/templates/thumbor-upstart.erb
@@ -2,7 +2,7 @@
### THIS FILE IS MANAGED BY PUPPET
#####################################################################
-description "Thumbor"
+description "Thumbor on port <%= @title %>"
start on mediawiki-ready
respawn
@@ -12,4 +12,4 @@
# Wrapping this with cgexec fails silently, which is why we rely on cgrulesengd
# to put the thumbor process into a cgroup
-exec <%= @deploy_dir %>/bin/thumbor -c <%= @cfg_file %> -a tc_core.app.App
\ No newline at end of file
+exec <%= @deploy_dir %>/bin/thumbor -c <%= @cfg_file %> -a tc_core.app.App -p
<%= @title %>
\ No newline at end of file
diff --git a/puppet/modules/thumbor/templates/thumbor.conf.erb
b/puppet/modules/thumbor/templates/thumbor.conf.erb
index 1ebab85..4f5d5c3 100644
--- a/puppet/modules/thumbor/templates/thumbor.conf.erb
+++ b/puppet/modules/thumbor/templates/thumbor.conf.erb
@@ -133,7 +133,7 @@
## The loader thumbor should use to load the original image. This must be the
## full name of a python module (python must be able to import it)
## Defaults to: thumbor.loaders.http_loader
-LOADER = 'wikimedia_thumbor.loader.proxy'
+#LOADER = 'wikimedia_thumbor.loader.proxy'
## The file storage thumbor should use to store original images. This must be
the
## full name of a python module (python must be able to import it)
@@ -558,7 +558,7 @@
################################## Extensions
##################################
COMMUNITY_EXTENSIONS = [
- 'wikimedia_thumbor.handler.multi'
+# 'wikimedia_thumbor.handler.multi'
]
EXIFTOOL_PATH = '/usr/bin/exiftool'
diff --git a/puppet/modules/thumbor/templates/varnish.vcl.erb
b/puppet/modules/thumbor/templates/varnish.vcl.erb
index aa6270c..c6263d8 100644
--- a/puppet/modules/thumbor/templates/varnish.vcl.erb
+++ b/puppet/modules/thumbor/templates/varnish.vcl.erb
@@ -14,6 +14,12 @@
set req.http.X-Url = req.url;
}
+ if (req.http.X-Url ~ "^/images/thumb/") {
+ set req.http.xkey = "File:" + regsub(req.http.X-Url,
"^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1");
+ } else if (req.http.X-Url ~ "^/images/") {
+ set req.http.xkey = "File:" + regsub(req.http.X-Url,
"^/images/[^/]+/[^/]+/(.*)", "\1");
+ }
+
# Since we expose varnish on the default port (6081) we need to rewrite
# requests that are generated using the default wiki port (8080)
# This needs to be done early because it's needed for PURGE calls
--
To view, visit https://gerrit.wikimedia.org/r/264084
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibe44bc16262f81cebcbd7c7bfa02c5f7255f065f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
