Gilles has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/334002 )
Change subject: Migrate Thumbor role from upstart to systemd ...................................................................... Migrate Thumbor role from upstart to systemd Bug: T154270 Change-Id: I4aff8987a948381a04633564e41fa52487d58c74 --- M Vagrantfile M puppet/modules/apache/templates/ports.conf.erb M puppet/modules/nginx M puppet/modules/thumbor/manifests/init.pp M puppet/modules/thumbor/manifests/service.pp M puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb A puppet/modules/thumbor/templates/systemd.erb D puppet/modules/thumbor/templates/upstart.erb A puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb 9 files changed, 181 insertions(+), 104 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant refs/changes/02/334002/1 diff --git a/Vagrantfile b/Vagrantfile index 49554e5..9db9e7d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -104,7 +104,7 @@ # end config.vm.network :forwarded_port, - guest: 80, host: settings[:http_port], host_ip: settings[:host_ip], + guest: 8080, host: settings[:http_port], host_ip: settings[:host_ip], id: 'http' config.vm.network :forwarded_port, diff --git a/puppet/modules/apache/templates/ports.conf.erb b/puppet/modules/apache/templates/ports.conf.erb index 10b63e9..1af1ef4 100644 --- a/puppet/modules/apache/templates/ports.conf.erb +++ b/puppet/modules/apache/templates/ports.conf.erb @@ -1,5 +1,2 @@ # This file is managed by Puppet. -Listen 80 -<%- if @forwarded_port and @forwarded_port.to_s != "80" -%> -Listen <%= @forwarded_port %> -<%- end -%> +Listen 8080 diff --git a/puppet/modules/nginx b/puppet/modules/nginx index 10a1e92..d41ab2c 160000 --- a/puppet/modules/nginx +++ b/puppet/modules/nginx @@ -1 +1 @@ -Subproject commit 10a1e925fa707654102f39092b46a64c61710614 +Subproject commit d41ab2cc4dd560f1483247bf2e1487d77b7e451c diff --git a/puppet/modules/thumbor/manifests/init.pp b/puppet/modules/thumbor/manifests/init.pp index ea8f4d0..49c7378 100644 --- a/puppet/modules/thumbor/manifests/init.pp +++ b/puppet/modules/thumbor/manifests/init.pp @@ -29,47 +29,41 @@ $statsd_port, $sentry_dsn_file, ) { - require ::virtualenv - # Needed by the venv, which clones a few git repos - require ::git + apt::pin { 'gifsicle-jessie-backports': + package => 'gifsicle', + pin => 'release n=jessie-backports', + priority => 1000, + } - # jpegtran - require_package('libjpeg-progs') + apt::pin { 'python-tornado-jessie-backports': + package => 'python-tornado', + pin => 'release n=jessie-backports', + priority => 1000, + } - # exiftool is needed by exif-optimizer - require_package('libimage-exiftool-perl') + apt::pin { 'python-pil-jessie-backports': + package => 'python-pil', + pin => 'release n=jessie-backports', + priority => 1000, + } - # For Pillow - require_package('libjpeg-dev') + apt::pin { 'python-swiftclient-jessie-backports': + package => 'python-swiftclient', + pin => 'release n=jessie-backports', + priority => 1000, + } - # For GIF engine - require_package('gifsicle') + package { 'raven': + provider => 'pip', + } - # For Video engine - require_package('ffmpeg') + package { 'python-thumbor-wikimedia': + notify => Exec['stop-and-disable-default-thumbor-service'] + } - # For XCF engine - require_package('xcftools') - - # For DjVu engine - require_package('libdjvulibre-dev') - require_package('cython') - - # For Ghostscript engine (PDF) - require_package('ghostscript') - - # For pycurl, a dependency of thumbor - require_package('libcurl4-gnutls-dev') - - # For Mediawiki's IM/RSVG SVG support - require_package('librsvg2-bin') - - # For Mediawiki's DJVU support - require_package('djvulibre-bin') - require_package('netpbm') - - # For lxml, a dependency of thumbor-plugins - require_package('libxml2-dev', 'libxslt1-dev') + exec { 'stop-and-disable-default-thumbor-service': + command => '/bin/systemctl stop thumbor' + } require_package('firejail') @@ -88,42 +82,18 @@ } file { '/etc/firejail/thumbor.profile': - ensure => present, - owner => 'root', - group => 'root', - mode => '0444', - source => 'puppet:///modules/thumbor/thumbor.profile', + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'puppet:///modules/thumbor/thumbor.profile', + require => Package['firejail'], } - virtualenv::environment { $deploy_dir: - ensure => present, - packages => [ - 'pgi', - 'raven', - 'python-swiftclient', - 'pympler', - 'git+git://github.com/gi11es/thumbor.git', - 'git+git://github.com/thumbor-community/core', - 'git+https://phabricator.wikimedia.org/diffusion/THMBREXT/thumbor-plugins.git', - ], - require => Package[ - 'libjpeg-progs', - # Needs to be an explicit dependency, for the packages pointing to git repos - 'git', - 'libcurl4-gnutls-dev', - 'libxml2-dev', - 'libxslt1-dev', - 'libjpeg-dev', - 'libdjvulibre-dev', - 'cython', - 'firejail' - ], - timeout => 600, # This venv can be particularly long to download and setup - } - - file { "${deploy_dir}/tinyrgb.icc": - ensure => present, - source => 'puppet:///modules/thumbor/tinyrgb.icc', + file { '/etc/tinyrgb.icc': + ensure => present, + source => 'puppet:///modules/thumbor/tinyrgb.icc', + require => Package['python-thumbor-wikimedia'], } file { $cfg_dir: @@ -166,11 +136,6 @@ File[$cfg_dir], Group['thumbor'], ], - } - - cgroup::config { 'thumbor': - limits => "perm { task { uid = thumbor; gid = thumbor; } admin { uid = thumbor; gid = thumbor; } } memory { memory.limit_in_bytes = \"1048576000\"; }", # 1GB - cgrules => '@thumbor memory thumbor', } # This will generate a list of ports starting at 8889, with diff --git a/puppet/modules/thumbor/manifests/service.pp b/puppet/modules/thumbor/manifests/service.pp index d461a5d..91e42ed 100644 --- a/puppet/modules/thumbor/manifests/service.pp +++ b/puppet/modules/thumbor/manifests/service.pp @@ -26,25 +26,34 @@ ) { $port = $name - file { "/etc/init/thumbor-${port}.conf": + file { "/lib/systemd/system/thumbor-${port}.service": ensure => present, - content => template('thumbor/upstart.erb'), + content => template('thumbor/systemd.erb'), mode => '0444', + } + + exec { "systemd reload for thumbor-${port}": + refreshonly => true, + command => '/bin/systemctl daemon-reload', + subscribe => File["/lib/systemd/system/thumbor-${port}.service"], } service { "thumbor-${port}": ensure => running, enable => true, - provider => 'upstart', + provider => 'systemd', require => [ - Virtualenv::Environment[$deploy_dir], + Package['python-thumbor-wikimedia'], User['thumbor'], File['/etc/firejail/thumbor.profile'], ], subscribe => [ - File["${deploy_dir}/tinyrgb.icc", "/etc/init/thumbor-${port}.conf", '/etc/firejail/thumbor.profile'], + File[ + "/etc/tinyrgb.icc", + "/lib/systemd/system/thumbor-${port}.service", + '/etc/firejail/thumbor.profile' + ], $cfg_files, - Cgroup::Config['thumbor'], ], } diff --git a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb index 78b25e1..111dbb9 100644 --- a/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb +++ b/puppet/modules/thumbor/templates/20-thumbor-wikimedia.conf.erb @@ -9,7 +9,7 @@ EXIFTOOL_PATH = '/usr/bin/exiftool' EXIF_FIELDS_TO_KEEP = [ 'Artist', 'Copyright', 'Description' ] -EXIF_TINYRGB_PATH = '/srv/thumbor/tinyrgb.icc' +EXIF_TINYRGB_PATH = '/etc/tinyrgb.icc' EXIF_TINYRGB_ICC_REPLACE = 'sRGB IEC61966-2.1' SUBPROCESS_USE_TIMEOUT = True @@ -53,11 +53,13 @@ PROXY_LOADER_LOADERS = [ 'wikimedia_thumbor.loader.video', - 'wikimedia_thumbor.loader.https' + 'wikimedia_thumbor.loader.swift' ] HTTP_LOADER_MAX_BODY_SIZE = 1024 * 1024 * 1024 # 1GB +LOADER_EXCERPT_LENGTH = 4096 + # 2 minutes, for situations where an engine failed to clean up after itself HTTP_LOADER_TEMP_FILE_TIMEOUT = 120 diff --git a/puppet/modules/thumbor/templates/systemd.erb b/puppet/modules/thumbor/templates/systemd.erb new file mode 100644 index 0000000..dd57ae5 --- /dev/null +++ b/puppet/modules/thumbor/templates/systemd.erb @@ -0,0 +1,16 @@ +[Unit] +Description=Thumbor service +Requires=mediawiki-ready.service +After=mediawiki-ready.service + +[Service] +User=thumbor +Group=thumbor +SyslogIdentifier=thumbor +MemoryMax=1G +# To assign CPU affinity we consider that the base port is 8889 (Thumbor's default + 1) +ExecStart=/usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail --profile=/etc/firejail/thumbor.profile -- /usr/bin/thumbor -p <%= @port %> -c <%= @cfg_dir %> +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/puppet/modules/thumbor/templates/upstart.erb b/puppet/modules/thumbor/templates/upstart.erb deleted file mode 100644 index 9c00280..0000000 --- a/puppet/modules/thumbor/templates/upstart.erb +++ /dev/null @@ -1,16 +0,0 @@ -##################################################################### -### THIS FILE IS MANAGED BY PUPPET -##################################################################### - -description "Thumbor" - -start on mediawiki-ready -respawn - -setuid thumbor -setgid thumbor - -# Wrapping this with cgexec fails silently, which is why we rely on cgrulesengd -# to put the thumbor process into a cgroup -# To assign CPU affinity we consider that the base port is 8889 (Thumbor's default + 1) -exec /usr/bin/taskset -c <%= Integer(@port) - 8889 %> /usr/bin/firejail --profile=/etc/firejail/thumbor.profile -- <%= @deploy_dir %>/bin/thumbor -p <%= @port %> -c <%= @cfg_dir %> \ No newline at end of file diff --git a/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb new file mode 100644 index 0000000..c105f0a --- /dev/null +++ b/puppet/modules/wmflib/lib/puppet/parser/functions/os_version.rb @@ -0,0 +1,104 @@ +# == Function: os_version( string $version_predicate ) +# +# Performs semantic OS version comparison. +# +# Takes one or more string arguments, each containing one or more predicate +# expressions. Each expression consts of a distribution name, followed by a +# comparison operator, followed by a release name or number. Multiple clauses +# are OR'd together. The arguments are case-insensitive. +# +# The host's OS version will be compared to to the comparison target +# using the specified operator, returning a boolean. If no operator is +# present, the equality operator is assumed. +# +# === Examples +# +# # True if Ubuntu Trusty or newer or Debian Jessie or newer +# os_version('ubuntu >= trusty || debian >= Jessie') +# +# # True if exactly Debian Jessie +# os_version('debian jessie') +# +require 'puppet/util/package' + +module Puppet::Parser::Functions + os_versions = { + 'Ubuntu' => { + 'Hardy' => '8.04', + 'Intrepid' => '8.10', + 'Jaunty' => '9.04', + 'Karmic' => '9.10', + 'Lucid' => '10.04', + 'Maverick' => '10.10', + 'Natty' => '11.04', + 'Oneiric' => '11.10', + 'Precise' => '12.04', + 'Quantal' => '12.10', + 'Raring' => '13.04', + 'Saucy' => '13.10', + 'Trusty' => '14.04', + 'Utopic' => '14.10', + 'Vivid' => '15.04', + 'Wily' => '15.10', + 'Xenial' => '16.04', + 'Yakkety' => '16.10', + }, + 'Debian' => { + 'Wheezy' => '7', + 'Jessie' => '8', + 'Stretch' => '9', + 'Buster' => '10', + } + } + + newfunction(:os_version, :type => :rvalue, :arity => 1) do |args| + self_release = lookupvar('lsbdistrelease').capitalize + self_id = lookupvar('lsbdistid').capitalize + + fail(ArgumentError, 'os_version(): string argument required') unless args.first.is_a?(String) + + clauses = args.first.split('||').map(&:strip) + + clauses.any? do |clause| + unless /^(\w+) *([<>=]*) *([\w\.]+)$/ =~ clause + fail(ArgumentError, "os_version(): invalid expression '#{clause}'") + end + # for ruby 1.8; replace with named groups with ruby >= 1.9 + other_id = Regexp.last_match(1) + operator = Regexp.last_match(2) + other_release = Regexp.last_match(3) + + [other_id, other_release].each(&:capitalize!) + + next unless self_id == other_id + + if os_versions[other_id].key?(other_release) + other_was_codename = true + end + + other_release = os_versions[other_id][other_release] || other_release + + unless /^[\d.]+$/ =~ other_release + fail(ArgumentError, + "os_version(): unknown #{other_id} release '#{other_release}'") + end + + # special-case Debian point-releases, as e.g. jessie is all of 8.x + if other_id == "Debian" && other_was_codename + self_release = self_release.split(".")[0] + end + + cmp = Puppet::Util::Package.versioncmp(self_release, other_release) + case operator + when '', '==' then cmp == 0 + when '!=' then cmp != 0 + when '>' then cmp == 1 + when '<' then cmp == -1 + when '>=' then cmp >= 0 + when '<=' then cmp <= 0 + else fail(ArgumentError, + "os_version(): unknown comparison operator '#{operator}'") + end + end + end +end \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/334002 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4aff8987a948381a04633564e41fa52487d58c74 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/vagrant Gerrit-Branch: jessie-migration Gerrit-Owner: Gilles <gdu...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits