Gilles has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/342835 )
Change subject: Set up Thumbor temp folders in a similar way as production
......................................................................
Set up Thumbor temp folders in a similar way as production
Refs T159358
Also enables the memcache-based erroring thumbnail throttling
and points to IM for new IM engine.
Change-Id: Ife83068a3824816c91d501aa34e6c2a8bd8f1d05
---
M puppet/hieradata/common.yaml
M puppet/modules/thumbor/manifests/init.pp
M puppet/modules/thumbor/manifests/service.pp
M puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
M puppet/modules/thumbor/templates/systemd/thumbor.erb
A puppet/modules/thumbor/templates/thumbor.tmpfiles.d.erb
6 files changed, 45 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/35/342835/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index f92f630..cb5c41f 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -476,9 +476,9 @@
swift::object_cfg_file: /etc/swift/object-server.conf
swift::container_cfg_file: /etc/swift/container-server.conf
-thumbor::deploy_dir: "%{hiera('mwv::vendor_dir')}/thumbor"
thumbor::cfg_dir: /etc/thumbor.d
thumbor::log_dir: /vagrant/logs/thumbor
+thumbor::tmp_dir: /tmp/thumbor
thumbor::statsd_port: "%{hiera('statsd::port')}"
thumbor::sentry_dsn_file: "%{hiera('sentry::dsn_file')}"
diff --git a/puppet/modules/thumbor/manifests/init.pp
b/puppet/modules/thumbor/manifests/init.pp
index 52d4fd9..9acf4dd 100644
--- a/puppet/modules/thumbor/manifests/init.pp
+++ b/puppet/modules/thumbor/manifests/init.pp
@@ -7,14 +7,14 @@
#
# === Parameters
#
-# [*deploy_dir*]
-# Path where Thumbor should be installed (example: '/var/thumbor').
-#
# [*cfg_dir*]
# Thumbor configuration directory. The directory will be generated by Puppet.
#
# [*log_dir*]
# Thumbor log directory. The directory will be generated by Puppet.
+#
+# [*tmp_dir*]
+# Thumbor tmp directory. The directory will be generated by Puppet.
#
# [*statsd_port*]
# Port the statsd instance runs on.
@@ -23,9 +23,9 @@
# Path to file containing the sentry dsn file.
#
class thumbor (
- $deploy_dir,
$cfg_dir,
$log_dir,
+ $tmp_dir,
$statsd_port,
$sentry_dsn_file,
) {
@@ -70,7 +70,6 @@
user { 'thumbor':
ensure => present,
- home => '/var/run/thumbor',
gid => 'thumbor',
require => Group['thumbor'],
}
@@ -95,6 +94,10 @@
}
file { $log_dir:
+ ensure => directory,
+ }
+
+ file { $tmp_dir:
ensure => directory,
}
@@ -137,7 +140,7 @@
$ports = sequence_array(8889, inline_template('<%= `nproc` %>'))
thumbor::service { $ports:
- deploy_dir => $deploy_dir,
+ tmp_dir => $tmp_dir,
cfg_files => File[
"${cfg_dir}/10-thumbor.conf",
"${cfg_dir}/20-thumbor-logging.conf",
@@ -187,4 +190,13 @@
],
subscribe => Mediawiki::Settings['swift'],
}
+
+ cron { 'systemd-thumbor-tmpfiles-clean':
+ minute => '*',
+ hour => '*',
+ monthday => '*',
+ month => '*',
+ weekday => '*',
+ command => "/bin/systemd-tmpfiles --clean --prefix=${tmp_dir}",
+ }
}
diff --git a/puppet/modules/thumbor/manifests/service.pp
b/puppet/modules/thumbor/manifests/service.pp
index f570f05..4ff424e 100644
--- a/puppet/modules/thumbor/manifests/service.pp
+++ b/puppet/modules/thumbor/manifests/service.pp
@@ -7,8 +7,8 @@
# [*name*]
# Service port.
#
-# [*deploy_dir*]
-# Path where Thumbor is installed (example: '/var/thumbor').
+# [*tmp_dir*]
+# Path where Thumbor temproary files are kept (example: '/var/thumbor/tmp').
#
# [*cfg_file*]
# Thumbor configuration files.
@@ -16,12 +16,12 @@
# === Examples
#
# thumbor::service { '8888':
-# deploy_dir => '/var/thumbor',
+# tmp_dir => '/var/thumbor-tmp',
# cfg_files => File['/etc/thumbor.d/10-thumbor.conf',
'/etc/thumbor.d/20-thumbor-logging.conf'],
# }
#
define thumbor::service (
- $deploy_dir,
+ $tmp_dir,
$cfg_files
) {
$port = $name
@@ -45,6 +45,16 @@
template_name => 'thumbor',
}
+ file { "/usr/lib/tmpfiles.d/thumbor@${port}.conf":
+ content => template('thumbor/thumbor.tmpfiles.d.erb'),
+ }
+
+ exec { "create-tmp-folder-${port}":
+ command => "/bin/systemd-tmpfiles --create --prefix=${tmp_dir}",
+ creates => "${tmp_dir}/thumbor@${port}",
+ before => Service["thumbor-${port}"],
+ }
+
# Ensure that Sentry is started before Thumbor
Service['sentry-worker'] ~> Service["thumbor-${port}"]
}
diff --git a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
index d1a7e69..c291902 100644
--- a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
+++ b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
@@ -28,6 +28,8 @@
GHOSTSCRIPT_PATH = '/usr/bin/gs'
+CONVERT_PATH = '/usr/bin/convert'
+
SWIFT_HOST = 'http://127.0.0.1:<%= scope['::swift::port'] %>'
SWIFT_API_PATH = '/v1/AUTH_<%= scope['::swift::project'] %>/'
SWIFT_AUTH_PATH = '/auth/v1.0'
@@ -99,4 +101,11 @@
'maxqueue': 100,
'timeout': 8,
'extensions': ['xcf', 'djvu', 'pdf', 'tiff']
-}
\ No newline at end of file
+}
+
+# Thumbnails that fail for a given xkey more than 4 times per hour aren't
+# worth re-attempting that often
+FAILURE_THROTTLING_MEMCACHE = ['127.0.0.1:<%= scope['::memcached::port'] %>']
+FAILURE_THROTTLING_MAX = 4
+FAILURE_THROTTLING_DURATION = 3600
+FAILURE_THROTTLING_PREFIX = 'thumbor-failure-'
diff --git a/puppet/modules/thumbor/templates/systemd/thumbor.erb
b/puppet/modules/thumbor/templates/systemd/thumbor.erb
index dd57ae5..8de1750 100644
--- a/puppet/modules/thumbor/templates/systemd/thumbor.erb
+++ b/puppet/modules/thumbor/templates/systemd/thumbor.erb
@@ -9,7 +9,7 @@
SyslogIdentifier=thumbor
MemoryMax=1G
# To assign CPU affinity we consider that the base port is 8889 (Thumbor's
default + 1)
-ExecStart=/usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail
--profile=/etc/firejail/thumbor.profile -- /usr/bin/thumbor -p <%= @port %> -c
<%= @cfg_dir %>
+ExecStart=/usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail
--profile=/etc/firejail/thumbor.profile --env=TMPDIR=<%= @tmp_dir
%>/thumbor@<%= @port %> --env=MAGICK_TEMPORARY_PATH=<%= @tmp_dir %>/thumbor@<%=
@port %> --env=MAGICK_DISK_LIMIT=900MB --env=MAGICK_MEMORY_LIMIT=900MB --
/usr/bin/thumbor -p <%= @port %> -c <%= @cfg_dir %>
Restart=always
[Install]
diff --git a/puppet/modules/thumbor/templates/thumbor.tmpfiles.d.erb
b/puppet/modules/thumbor/templates/thumbor.tmpfiles.d.erb
new file mode 100644
index 0000000..fc4d8e0
--- /dev/null
+++ b/puppet/modules/thumbor/templates/thumbor.tmpfiles.d.erb
@@ -0,0 +1 @@
+d <%= @tmp_dir %>/thumbor@<%= @port %> 0755 thumbor thumbor 1h
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/342835
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ife83068a3824816c91d501aa34e6c2a8bd8f1d05
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles <gdu...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
