Gilles has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/334002 )
Change subject: Migrate Thumbor role from upstart to systemd
......................................................................
Migrate Thumbor role from upstart to systemd
Bug: T154270
Change-Id: I4aff8987a948381a04633564e41fa52487d58c74
---
M Vagrantfile
M puppet/modules/apache/templates/ports.conf.erb
M puppet/modules/nginx
M puppet/modules/thumbor/manifests/init.pp
M puppet/modules/thumbor/manifests/service.pp
M puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
A puppet/modules/thumbor/templates/systemd.erb
D puppet/modules/thumbor/templates/upstart.erb
A puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
9 files changed, 181 insertions(+), 104 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/02/334002/1
diff --git a/Vagrantfile b/Vagrantfile
index 49554e5..9db9e7d 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -104,7 +104,7 @@
# end
config.vm.network :forwarded_port,
- guest: 80, host: settings[:http_port], host_ip: settings[:host_ip],
+ guest: 8080, host: settings[:http_port], host_ip: settings[:host_ip],
id: 'http'
config.vm.network :forwarded_port,
diff --git a/puppet/modules/apache/templates/ports.conf.erb
b/puppet/modules/apache/templates/ports.conf.erb
index 10b63e9..1af1ef4 100644
--- a/puppet/modules/apache/templates/ports.conf.erb
+++ b/puppet/modules/apache/templates/ports.conf.erb
@@ -1,5 +1,2 @@
# This file is managed by Puppet.
-Listen 80
-<%- if @forwarded_port and @forwarded_port.to_s != "80" -%>
-Listen <%= @forwarded_port %>
-<%- end -%>
+Listen 8080
diff --git a/puppet/modules/nginx b/puppet/modules/nginx
index 10a1e92..d41ab2c 160000
--- a/puppet/modules/nginx
+++ b/puppet/modules/nginx
@@ -1 +1 @@
-Subproject commit 10a1e925fa707654102f39092b46a64c61710614
+Subproject commit d41ab2cc4dd560f1483247bf2e1487d77b7e451c
diff --git a/puppet/modules/thumbor/manifests/init.pp
b/puppet/modules/thumbor/manifests/init.pp
index ea8f4d0..49c7378 100644
--- a/puppet/modules/thumbor/manifests/init.pp
+++ b/puppet/modules/thumbor/manifests/init.pp
@@ -29,47 +29,41 @@
$statsd_port,
$sentry_dsn_file,
) {
- require ::virtualenv
- # Needed by the venv, which clones a few git repos
- require ::git
+ apt::pin { 'gifsicle-jessie-backports':
+ package => 'gifsicle',
+ pin => 'release n=jessie-backports',
+ priority => 1000,
+ }
- # jpegtran
- require_package('libjpeg-progs')
+ apt::pin { 'python-tornado-jessie-backports':
+ package => 'python-tornado',
+ pin => 'release n=jessie-backports',
+ priority => 1000,
+ }
- # exiftool is needed by exif-optimizer
- require_package('libimage-exiftool-perl')
+ apt::pin { 'python-pil-jessie-backports':
+ package => 'python-pil',
+ pin => 'release n=jessie-backports',
+ priority => 1000,
+ }
- # For Pillow
- require_package('libjpeg-dev')
+ apt::pin { 'python-swiftclient-jessie-backports':
+ package => 'python-swiftclient',
+ pin => 'release n=jessie-backports',
+ priority => 1000,
+ }
- # For GIF engine
- require_package('gifsicle')
+ package { 'raven':
+ provider => 'pip',
+ }
- # For Video engine
- require_package('ffmpeg')
+ package { 'python-thumbor-wikimedia':
+ notify => Exec['stop-and-disable-default-thumbor-service']
+ }
- # For XCF engine
- require_package('xcftools')
-
- # For DjVu engine
- require_package('libdjvulibre-dev')
- require_package('cython')
-
- # For Ghostscript engine (PDF)
- require_package('ghostscript')
-
- # For pycurl, a dependency of thumbor
- require_package('libcurl4-gnutls-dev')
-
- # For Mediawiki's IM/RSVG SVG support
- require_package('librsvg2-bin')
-
- # For Mediawiki's DJVU support
- require_package('djvulibre-bin')
- require_package('netpbm')
-
- # For lxml, a dependency of thumbor-plugins
- require_package('libxml2-dev', 'libxslt1-dev')
+ exec { 'stop-and-disable-default-thumbor-service':
+ command => '/bin/systemctl stop thumbor'
+ }
require_package('firejail')
@@ -88,42 +82,18 @@
}
file { '/etc/firejail/thumbor.profile':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0444',
- source => 'puppet:///modules/thumbor/thumbor.profile',
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source => 'puppet:///modules/thumbor/thumbor.profile',
+ require => Package['firejail'],
}
- virtualenv::environment { $deploy_dir:
- ensure => present,
- packages => [
- 'pgi',
- 'raven',
- 'python-swiftclient',
- 'pympler',
- 'git+git://github.com/gi11es/thumbor.git',
- 'git+git://github.com/thumbor-community/core',
-
'git+https://phabricator.wikimedia.org/diffusion/THMBREXT/thumbor-plugins.git',
- ],
- require => Package[
- 'libjpeg-progs',
- # Needs to be an explicit dependency, for the packages pointing to
git repos
- 'git',
- 'libcurl4-gnutls-dev',
- 'libxml2-dev',
- 'libxslt1-dev',
- 'libjpeg-dev',
- 'libdjvulibre-dev',
- 'cython',
- 'firejail'
- ],
- timeout => 600, # This venv can be particularly long to download and
setup
- }
-
- file { "${deploy_dir}/tinyrgb.icc":
- ensure => present,
- source => 'puppet:///modules/thumbor/tinyrgb.icc',
+ file { '/etc/tinyrgb.icc':
+ ensure => present,
+ source => 'puppet:///modules/thumbor/tinyrgb.icc',
+ require => Package['python-thumbor-wikimedia'],
}
file { $cfg_dir:
@@ -166,11 +136,6 @@
File[$cfg_dir],
Group['thumbor'],
],
- }
-
- cgroup::config { 'thumbor':
- limits => "perm { task { uid = thumbor; gid = thumbor; } admin { uid
= thumbor; gid = thumbor; } } memory { memory.limit_in_bytes = \"1048576000\";
}", # 1GB
- cgrules => '@thumbor memory thumbor',
}
# This will generate a list of ports starting at 8889, with
diff --git a/puppet/modules/thumbor/manifests/service.pp
b/puppet/modules/thumbor/manifests/service.pp
index d461a5d..91e42ed 100644
--- a/puppet/modules/thumbor/manifests/service.pp
+++ b/puppet/modules/thumbor/manifests/service.pp
@@ -26,25 +26,34 @@
) {
$port = $name
- file { "/etc/init/thumbor-${port}.conf":
+ file { "/lib/systemd/system/thumbor-${port}.service":
ensure => present,
- content => template('thumbor/upstart.erb'),
+ content => template('thumbor/systemd.erb'),
mode => '0444',
+ }
+
+ exec { "systemd reload for thumbor-${port}":
+ refreshonly => true,
+ command => '/bin/systemctl daemon-reload',
+ subscribe => File["/lib/systemd/system/thumbor-${port}.service"],
}
service { "thumbor-${port}":
ensure => running,
enable => true,
- provider => 'upstart',
+ provider => 'systemd',
require => [
- Virtualenv::Environment[$deploy_dir],
+ Package['python-thumbor-wikimedia'],
User['thumbor'],
File['/etc/firejail/thumbor.profile'],
],
subscribe => [
- File["${deploy_dir}/tinyrgb.icc",
"/etc/init/thumbor-${port}.conf", '/etc/firejail/thumbor.profile'],
+ File[
+ "/etc/tinyrgb.icc",
+ "/lib/systemd/system/thumbor-${port}.service",
+ '/etc/firejail/thumbor.profile'
+ ],
$cfg_files,
- Cgroup::Config['thumbor'],
],
}
diff --git a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
index 78b25e1..111dbb9 100644
--- a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
+++ b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb
@@ -9,7 +9,7 @@
EXIFTOOL_PATH = '/usr/bin/exiftool'
EXIF_FIELDS_TO_KEEP = [ 'Artist', 'Copyright', 'Description' ]
-EXIF_TINYRGB_PATH = '/srv/thumbor/tinyrgb.icc'
+EXIF_TINYRGB_PATH = '/etc/tinyrgb.icc'
EXIF_TINYRGB_ICC_REPLACE = 'sRGB IEC61966-2.1'
SUBPROCESS_USE_TIMEOUT = True
@@ -53,11 +53,13 @@
PROXY_LOADER_LOADERS = [
'wikimedia_thumbor.loader.video',
- 'wikimedia_thumbor.loader.https'
+ 'wikimedia_thumbor.loader.swift'
]
HTTP_LOADER_MAX_BODY_SIZE = 1024 * 1024 * 1024 # 1GB
+LOADER_EXCERPT_LENGTH = 4096
+
# 2 minutes, for situations where an engine failed to clean up after itself
HTTP_LOADER_TEMP_FILE_TIMEOUT = 120
diff --git a/puppet/modules/thumbor/templates/systemd.erb
b/puppet/modules/thumbor/templates/systemd.erb
new file mode 100644
index 0000000..dd57ae5
--- /dev/null
+++ b/puppet/modules/thumbor/templates/systemd.erb
@@ -0,0 +1,16 @@
+[Unit]
+Description=Thumbor service
+Requires=mediawiki-ready.service
+After=mediawiki-ready.service
+
+[Service]
+User=thumbor
+Group=thumbor
+SyslogIdentifier=thumbor
+MemoryMax=1G
+# To assign CPU affinity we consider that the base port is 8889 (Thumbor's
default + 1)
+ExecStart=/usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail
--profile=/etc/firejail/thumbor.profile -- /usr/bin/thumbor -p <%= @port %> -c
<%= @cfg_dir %>
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/puppet/modules/thumbor/templates/upstart.erb
b/puppet/modules/thumbor/templates/upstart.erb
deleted file mode 100644
index 9c00280..0000000
--- a/puppet/modules/thumbor/templates/upstart.erb
+++ /dev/null
@@ -1,16 +0,0 @@
-#####################################################################
-### THIS FILE IS MANAGED BY PUPPET
-#####################################################################
-
-description "Thumbor"
-
-start on mediawiki-ready
-respawn
-
-setuid thumbor
-setgid thumbor
-
-# Wrapping this with cgexec fails silently, which is why we rely on cgrulesengd
-# to put the thumbor process into a cgroup
-# To assign CPU affinity we consider that the base port is 8889 (Thumbor's
default + 1)
-exec /usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail
--profile=/etc/firejail/thumbor.profile -- <%= @deploy_dir %>/bin/thumbor -p
<%= @port %> -c <%= @cfg_dir %>
\ No newline at end of file
diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
new file mode 100644
index 0000000..c105f0a
--- /dev/null
+++ b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb
@@ -0,0 +1,104 @@
+# == Function: os_version( string $version_predicate )
+#
+# Performs semantic OS version comparison.
+#
+# Takes one or more string arguments, each containing one or more predicate
+# expressions. Each expression consts of a distribution name, followed by a
+# comparison operator, followed by a release name or number. Multiple clauses
+# are OR'd together. The arguments are case-insensitive.
+#
+# The host's OS version will be compared to to the comparison target
+# using the specified operator, returning a boolean. If no operator is
+# present, the equality operator is assumed.
+#
+# === Examples
+#
+# # True if Ubuntu Trusty or newer or Debian Jessie or newer
+# os_version('ubuntu >= trusty || debian >= Jessie')
+#
+# # True if exactly Debian Jessie
+# os_version('debian jessie')
+#
+require 'puppet/util/package'
+
+module Puppet::Parser::Functions
+ os_versions = {
+ 'Ubuntu' => {
+ 'Hardy' => '8.04',
+ 'Intrepid' => '8.10',
+ 'Jaunty' => '9.04',
+ 'Karmic' => '9.10',
+ 'Lucid' => '10.04',
+ 'Maverick' => '10.10',
+ 'Natty' => '11.04',
+ 'Oneiric' => '11.10',
+ 'Precise' => '12.04',
+ 'Quantal' => '12.10',
+ 'Raring' => '13.04',
+ 'Saucy' => '13.10',
+ 'Trusty' => '14.04',
+ 'Utopic' => '14.10',
+ 'Vivid' => '15.04',
+ 'Wily' => '15.10',
+ 'Xenial' => '16.04',
+ 'Yakkety' => '16.10',
+ },
+ 'Debian' => {
+ 'Wheezy' => '7',
+ 'Jessie' => '8',
+ 'Stretch' => '9',
+ 'Buster' => '10',
+ }
+ }
+
+ newfunction(:os_version, :type => :rvalue, :arity => 1) do |args|
+ self_release = lookupvar('lsbdistrelease').capitalize
+ self_id = lookupvar('lsbdistid').capitalize
+
+ fail(ArgumentError, 'os_version(): string argument required') unless
args.first.is_a?(String)
+
+ clauses = args.first.split('||').map(&:strip)
+
+ clauses.any? do |clause|
+ unless /^(\w+) *([<>=]*) *([\w\.]+)$/ =~ clause
+ fail(ArgumentError, "os_version(): invalid expression '#{clause}'")
+ end
+ # for ruby 1.8; replace with named groups with ruby >= 1.9
+ other_id = Regexp.last_match(1)
+ operator = Regexp.last_match(2)
+ other_release = Regexp.last_match(3)
+
+ [other_id, other_release].each(&:capitalize!)
+
+ next unless self_id == other_id
+
+ if os_versions[other_id].key?(other_release)
+ other_was_codename = true
+ end
+
+ other_release = os_versions[other_id][other_release] || other_release
+
+ unless /^[\d.]+$/ =~ other_release
+ fail(ArgumentError,
+ "os_version(): unknown #{other_id} release '#{other_release}'")
+ end
+
+ # special-case Debian point-releases, as e.g. jessie is all of 8.x
+ if other_id == "Debian" && other_was_codename
+ self_release = self_release.split(".")[0]
+ end
+
+ cmp = Puppet::Util::Package.versioncmp(self_release, other_release)
+ case operator
+ when '', '==' then cmp == 0
+ when '!=' then cmp != 0
+ when '>' then cmp == 1
+ when '<' then cmp == -1
+ when '>=' then cmp >= 0
+ when '<=' then cmp <= 0
+ else fail(ArgumentError,
+ "os_version(): unknown comparison operator '#{operator}'")
+ end
+ end
+ end
+end
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/334002
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4aff8987a948381a04633564e41fa52487d58c74
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: jessie-migration
Gerrit-Owner: Gilles <gdu...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
