Gilles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/256473
Change subject: Swift role
......................................................................
Swift role
Bug: T76489
Change-Id: Ia516a964a65e0b224e8d10e909e77d05a7acb1c2
---
M puppet/hieradata/common.yaml
A puppet/modules/role/manifests/swift.pp
M puppet/modules/role/manifests/thumbor.pp
A puppet/modules/role/settings/swift.yaml
M puppet/modules/role/settings/thumbor.yaml
A puppet/modules/role/templates/swift/apache2.conf.erb
M puppet/modules/role/templates/thumbor/local_repo.php.erb
A puppet/modules/swift/manifests/init.pp
A puppet/modules/swift/manifests/ring.pp
A puppet/modules/swift/templates/conf.php.erb
A puppet/modules/swift/templates/proxy-server.conf.erb
A puppet/modules/swift/templates/ring.conf.erb
A puppet/modules/swift/templates/swift.conf.erb
M puppet/modules/thumbor/manifests/init.pp
R puppet/modules/thumbor/templates/varnish.vcl.erb
15 files changed, 377 insertions(+), 22 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/73/256473/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index 8a66302..af91aa7 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -315,6 +315,14 @@
statsd::port: '8125'
+swift::storage_dir: "%{hiera('mwv::files_dir')}/swift"
+swift::port: 8040
+swift::cfg_file: /etc/swift/swift.conf
+swift::proxy_cfg_file: /etc/swift/proxy-server.conf
+swift::account_cfg_file: /etc/swift/account-server.conf
+swift::object_cfg_file: /etc/swift/object-server.conf
+swift::container_cfg_file: /etc/swift/container-server.conf
+
trafficserver::deploy_dir: "%{hiera('mwv::services_dir')}/trafficserver"
trafficserver::version: 6.0.0
trafficserver::port: 6090
diff --git a/puppet/modules/role/manifests/swift.pp
b/puppet/modules/role/manifests/swift.pp
new file mode 100644
index 0000000..dd84806
--- /dev/null
+++ b/puppet/modules/role/manifests/swift.pp
@@ -0,0 +1,17 @@
+# == Class: role::swift
+# Installs a Swift instance
+#
+class role::swift {
+ require ::role::mediawiki
+ require ::role::memcached
+ include ::swift
+
+ mediawiki::settings { 'swift':
+ values => template('swift/conf.php.erb'),
+ }
+
+ apache::site_conf { 'swift':
+ site => $::mediawiki::wiki_name,
+ content => template('role/swift/apache2.conf.erb'),
+ }
+}
diff --git a/puppet/modules/role/manifests/thumbor.pp
b/puppet/modules/role/manifests/thumbor.pp
index 1078d52..c9782f1 100644
--- a/puppet/modules/role/manifests/thumbor.pp
+++ b/puppet/modules/role/manifests/thumbor.pp
@@ -7,18 +7,21 @@
require ::role::statsd
require ::role::memcached
require ::role::sentry
+ require ::role::swift
include ::apache::mod::proxy
include ::apache::mod::proxy_http
include ::apache::mod::headers
include ::thumbor
mediawiki::settings { 'thumbor-repo':
- values => template('role/thumbor/local_repo.php.erb'),
+ values => template('role/thumbor/local_repo.php.erb'),
+ # Needs to be higher priority that swift for the local repo override
+ priority => 20,
}
mediawiki::settings { 'thumbor':
values => {
'wgIgnoreImageErrors' => true,
- }
+ },
}
}
diff --git a/puppet/modules/role/settings/swift.yaml
b/puppet/modules/role/settings/swift.yaml
new file mode 100644
index 0000000..c4837ca
--- /dev/null
+++ b/puppet/modules/role/settings/swift.yaml
@@ -0,0 +1,2 @@
+forward_ports:
+ 8040: 8040 # swift
\ No newline at end of file
diff --git a/puppet/modules/role/settings/thumbor.yaml
b/puppet/modules/role/settings/thumbor.yaml
index 4d1a119..46a23c4 100644
--- a/puppet/modules/role/settings/thumbor.yaml
+++ b/puppet/modules/role/settings/thumbor.yaml
@@ -1,3 +1,4 @@
forward_ports:
- 8888: 8888
- 6081: 6081
+ 8888: 8888 # thumbor
+ 6081: 6081 # varnish
+ 8040: 8040 # swift
\ No newline at end of file
diff --git a/puppet/modules/role/templates/swift/apache2.conf.erb
b/puppet/modules/role/templates/swift/apache2.conf.erb
new file mode 100644
index 0000000..5d90a6c
--- /dev/null
+++ b/puppet/modules/role/templates/swift/apache2.conf.erb
@@ -0,0 +1,4 @@
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LogLevel trace8
+ProxyPassMatch "^/images/(?!thumb/)(.*)$" "http://127.0.0.1:<%=
scope['::swift::port'] %>/v1/AUTH_testproj/wiki-local-public/$1$2"
\ No newline at end of file
diff --git a/puppet/modules/role/templates/thumbor/local_repo.php.erb
b/puppet/modules/role/templates/thumbor/local_repo.php.erb
index 5868ed0..54871fe 100644
--- a/puppet/modules/role/templates/thumbor/local_repo.php.erb
+++ b/puppet/modules/role/templates/thumbor/local_repo.php.erb
@@ -1,14 +1,15 @@
$wgLocalFileRepo = array(
- 'class' => 'LocalRepo',
- 'name' => 'local',
- 'directory' => $wgUploadDirectory,
- 'scriptDirUrl' => $wgScriptPath,
- 'scriptExtension' => $wgScriptExtension,
- 'url' => 'http://127.0.0.1:6081' . $wgUploadPath,
- 'hashLevels' => $wgHashedUploadDirectory ? 2 : 0,
- 'thumbScriptUrl' => $wgThumbnailScriptPath,
- 'transformVia404' => !$wgGenerateThumbnailOnParse,
- 'deletedDir' => $wgDeletedDirectory,
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'backend' => 'swift-backend',
+ 'directory' => $wgUploadDirectory,
+ 'scriptDirUrl' => $wgScriptPath,
+ 'scriptExtension' => $wgScriptExtension,
+ 'url' => 'http://127.0.0.1:6081' . $wgUploadPath,
+ 'hashLevels' => $wgHashedUploadDirectory ? 2 : 0,
+ 'thumbScriptUrl' => $wgThumbnailScriptPath,
+ 'transformVia404' => !$wgGenerateThumbnailOnParse,
+ 'deletedDir' => $wgDeletedDirectory,
'deletedHashLevels' => $wgHashedUploadDirectory ? 3 : 0,
- 'supportsSha1URLs' => true,
+ 'supportsSha1URLs' => true,
);
\ No newline at end of file
diff --git a/puppet/modules/swift/manifests/init.pp
b/puppet/modules/swift/manifests/init.pp
new file mode 100644
index 0000000..8ed825f
--- /dev/null
+++ b/puppet/modules/swift/manifests/init.pp
@@ -0,0 +1,162 @@
+# == Class: Swift
+#
+# This Puppet class installs and configures a Swift instance
+#
+# === Parameters
+#
+# [*storage_dir*]
+# Path where Swift content will be stored (example: '/var/swift').
+#
+# [*port*]
+# Port for the proxy server to listen on.
+#
+# [*cfg_file*]
+# Swift configuration file. The file will be generated by Puppet.
+#
+# [*proxy_cfg_file*]
+# Swift proxy server configuration file. The file will be generated by
Puppet.
+#
+# [*account_cfg_file*]
+# Swift account server configuration file. The file will be generated by
Puppet.
+#
+# [*object_cfg_file*]
+# Swift object server configuration file. The file will be generated by
Puppet.
+#
+# [*container_cfg_file*]
+# Swift container server configuration file. The file will be generated by
Puppet.
+#
+class swift (
+ $storage_dir,
+ $port,
+ $cfg_file,
+ $proxy_cfg_file,
+ $account_cfg_file,
+ $object_cfg_file,
+ $container_cfg_file,
+) {
+ include ::apache::mod::proxy
+ include ::apache::mod::proxy_http
+
+ require_package('swift')
+ require_package('swift-account')
+ require_package('swift-container')
+ require_package('swift-object')
+ require_package('swift-proxy')
+ require_package('python-swiftclient')
+
+ user { 'swift':
+ ensure => present,
+ managehome => true,
+ home => '/home/swift',
+ }
+
+ file { '/etc/swift':
+ ensure => 'directory',
+ owner => 'swift',
+ group => 'swift',
+ }
+
+ file { '/etc/swift/backups':
+ ensure => 'directory',
+ owner => 'swift',
+ group => 'swift',
+ }
+
+ file { $storage_dir:
+ ensure => 'directory',
+ owner => 'swift',
+ group => 'swift',
+ }
+
+ file { "$storage_dir/1":
+ ensure => 'directory',
+ owner => 'swift',
+ group => 'swift',
+ }
+
+ file { $cfg_file:
+ ensure => present,
+ group => 'www-data',
+ content => template('swift/swift.conf.erb'),
+ mode => '0644',
+ }
+
+ file { $proxy_cfg_file:
+ ensure => present,
+ group => 'www-data',
+ content => template('swift/proxy-server.conf.erb'),
+ mode => '0644',
+ }
+
+ swift::ring { $account_cfg_file:
+ ring_type => 'account',
+ cfg_file => $account_cfg_file,
+ storage_dir => $storage_dir,
+ ring_port => 6010,
+ require => [
+ Package['swift'],
+ Package['swift-account'],
+ ],
+ }
+
+ swift::ring { $object_cfg_file:
+ ring_type => 'object',
+ cfg_file => $object_cfg_file,
+ storage_dir => $storage_dir,
+ ring_port => 6020,
+ require => [
+ Package['swift'],
+ Package['swift-object'],
+ ]
+ }
+
+ swift::ring { $container_cfg_file:
+ ring_type => 'container',
+ cfg_file => $container_cfg_file,
+ storage_dir => $storage_dir,
+ ring_port => 6030,
+ require => [
+ Package['swift'],
+ Package['swift-container'],
+ ],
+ }
+
+ exec { 'swift-init':
+ command => 'swift-init start all',
+ user => 'root',
+ unless => "swift -A http://127.0.0.1:$port/auth/v1.0 -U
testproj:testuser -K testpwd stat -v | grep -Pq 'Auth Token'",
+ require => [
+ File[$storage_dir],
+ File["$storage_dir/1"],
+ File[$cfg_file],
+ File[$proxy_cfg_file],
+ Ring[$account_cfg_file],
+ Ring[$object_cfg_file],
+ Ring[$container_cfg_file],
+ ],
+ }
+
+ file { '/tmp/foo':
+ ensure => present,
+ content => 'bar',
+ mode => '0644',
+ }
+
+ exec { 'swift-create-public-container':
+ command => "swift -A http://127.0.0.1:$port/auth/v1.0 -U
testproj:testuser -K testpwd upload wiki-local-public /tmp/foo",
+ user => 'root',
+ unless => "curl -s -o /dev/null -w \"%{http_code}\"
http://127.0.0.1:$port/v1/AUTH_testproj/wiki-local-public/tmp/foo | grep -Pq
'200'",
+ require => [
+ Exec['swift-init'],
+ File['/tmp/foo'],
+ ],
+ notify => Exec['swift-make-container-public'],
+ }
+
+ exec { 'swift-make-container-public':
+ command => "swift -A http://127.0.0.1:$port/auth/v1.0 -U
testproj:testuser -K testpwd post -r '.r:*' wiki-local-public",
+ user => 'root',
+ require => Exec['swift-create-public-container'],
+ refreshonly => true,
+ }
+}
diff --git a/puppet/modules/swift/manifests/ring.pp
b/puppet/modules/swift/manifests/ring.pp
new file mode 100644
index 0000000..3760582
--- /dev/null
+++ b/puppet/modules/swift/manifests/ring.pp
@@ -0,0 +1,64 @@
+# == Define: swift::ring
+#
+# Creates and adds a swift ring.
+#
+# === Parameters
+#
+# [*ring_type*]
+# The type of swift ring .
+#
+# [*cfg_file*]
+# Path to the ring's config file.
+#
+# [*storage_dir*]
+# Path to the swift storage directory.
+#
+# [*ring_port*]
+# Port the ring will run on.
+#
+# === Examples
+#
+# swift::ring { 'account':
+# server_type => 'account',
+# cfg_file => '/etc/swift/account-server.conf',
+# storage_dir => '/srv/swift',
+# ring_port => 6010,
+# }
+#
+define swift::ring(
+ $ring_type,
+ $cfg_file,
+ $storage_dir,
+ $ring_port
+) {
+ file { $cfg_file:
+ ensure => present,
+ group => 'www-data',
+ content => template('swift/ring.conf.erb'),
+ mode => '0644',
+ notify => Exec["${ring_type}/create_ring"],
+ }
+
+ exec { "${ring_type}/create_ring":
+ command => "swift-ring-builder ${ring_type}.builder create 18 3 1",
+ user => 'swift',
+ cwd => '/etc/swift',
+ notify => Exec["${ring_type}/add_ring"],
+ refreshonly => true,
+ }
+
+ exec { "${ring_type}/add_ring":
+ command => "swift-ring-builder ${ring_type}.builder add
z1-127.0.0.1:${ring_port}/1 1",
+ user => 'swift',
+ cwd => '/etc/swift',
+ notify => Exec["${ring_type}/rebalance"],
+ refreshonly => true,
+ }
+
+ exec { "${ring_type}/rebalance":
+ command => "swift-ring-builder ${ring_type}.builder rebalance",
+ user => 'swift',
+ cwd => '/etc/swift',
+ refreshonly => true,
+ }
+}
\ No newline at end of file
diff --git a/puppet/modules/swift/templates/conf.php.erb
b/puppet/modules/swift/templates/conf.php.erb
new file mode 100644
index 0000000..b6180c0
--- /dev/null
+++ b/puppet/modules/swift/templates/conf.php.erb
@@ -0,0 +1,25 @@
+$wgFileBackends[] = array(
+ 'name' => 'swift-backend',
+ 'class' => 'SwiftFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'fileMode' => 0644,
+ 'basePath' => $IP . '/images/gwtoolset',
+ 'swiftAuthUrl' => '127.0.0.1:<%= scope['::swift::port'] %>/auth',
+ 'swiftUser' => 'testproj:testuser',
+ 'swiftKey' => 'testpwd',
+);
+
+$wgLocalFileRepo = array(
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'backend' => 'swift-backend',
+ 'directory' => $wgUploadDirectory,
+ 'url' => $wgUploadPath,
+ 'scriptDirUrl' => $wgScriptPath,
+ 'scriptExtension' => $wgScriptExtension,
+ 'hashLevels' => $wgHashedUploadDirectory ? 2 : 0,
+ 'thumbScriptUrl' => $wgThumbnailScriptPath,
+ 'transformVia404' => !$wgGenerateThumbnailOnParse,
+ 'deletedDir' => $wgDeletedDirectory,
+ 'deletedHashLevels' => $wgHashedUploadDirectory ? 3 : 0,
+);
\ No newline at end of file
diff --git a/puppet/modules/swift/templates/proxy-server.conf.erb
b/puppet/modules/swift/templates/proxy-server.conf.erb
new file mode 100644
index 0000000..d91927e
--- /dev/null
+++ b/puppet/modules/swift/templates/proxy-server.conf.erb
@@ -0,0 +1,26 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+[DEFAULT]
+bind_port = <%= @port %>
+log_facility = LOG_LOCAL1
+
+[pipeline:main]
+pipeline = healthcheck cache tempauth proxy-server
+
+[app:proxy-server]
+use = egg:swift#proxy
+allow_account_management = true
+account_autocreate = true
+
+[filter:tempauth]
+use = egg:swift#tempauth
+user_admin_admin = admin .admin .reseller_admin
+user_testproj_testuser = testpwd .admin
+
+[filter:healthcheck]
+use = egg:swift#healthcheck
+
+[filter:cache]
+use = egg:swift#memcache
diff --git a/puppet/modules/swift/templates/ring.conf.erb
b/puppet/modules/swift/templates/ring.conf.erb
new file mode 100644
index 0000000..487ca3e
--- /dev/null
+++ b/puppet/modules/swift/templates/ring.conf.erb
@@ -0,0 +1,27 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+[DEFAULT]
+devices = <%= @storage_dir %>
+mount_check = false
+bind_port = <%= @ring_port %>
+
+[pipeline:main]
+pipeline = <%= @ring_type %>-server
+
+[app:<%= @ring_type %>-server]
+use = egg:swift#<%= @ring_type %>
+
+[<%= @ring_type %>-replicator]
+vm_test_mode = yes
+
+[<%= @ring_type %>-updater]
+
+[<%= @ring_type %>-auditor]
+
+# Only for container-sync but avoid making more complex template logic
+[<%= @ring_type %>-sync]
+
+# Same for account-reaper
+[<%= @ring_type %>-reaper]
\ No newline at end of file
diff --git a/puppet/modules/swift/templates/swift.conf.erb
b/puppet/modules/swift/templates/swift.conf.erb
new file mode 100644
index 0000000..75b4540
--- /dev/null
+++ b/puppet/modules/swift/templates/swift.conf.erb
@@ -0,0 +1,6 @@
+#####################################################################
+### THIS FILE IS MANAGED BY PUPPET
+#####################################################################
+
+[swift-hash]
+swift_hash_path_suffix = mystuff
diff --git a/puppet/modules/thumbor/manifests/init.pp
b/puppet/modules/thumbor/manifests/init.pp
index 996233c..4f11284 100644
--- a/puppet/modules/thumbor/manifests/init.pp
+++ b/puppet/modules/thumbor/manifests/init.pp
@@ -105,8 +105,14 @@
onlyif => 'req.url ~ "^/images/thumb/.*\.(jpeg|jpg|png)"',
}
+ varnish::backend { 'swift':
+ host => '127.0.0.1',
+ port => $::swift::port,
+ onlyif => 'req.url ~ "^/images/(?!thumb/).*\.(jpeg|jpg|png)"',
+ }
+
varnish::config { 'thumbor':
- source => 'puppet:///modules/thumbor/varnish.vcl',
- order => 49, # Needs to be before default for vcl_recv override
+ content => template('thumbor/varnish.vcl.erb'),
+ order => 49, # Needs to be before default for vcl_recv override
}
}
diff --git a/puppet/modules/thumbor/files/varnish.vcl
b/puppet/modules/thumbor/templates/varnish.vcl.erb
similarity index 80%
rename from puppet/modules/thumbor/files/varnish.vcl
rename to puppet/modules/thumbor/templates/varnish.vcl.erb
index 7c79b49..d54b94b 100644
--- a/puppet/modules/thumbor/files/varnish.vcl
+++ b/puppet/modules/thumbor/templates/varnish.vcl.erb
@@ -30,7 +30,7 @@
if (req.url ~ "^/images/thumb/") {
set req.http.xkey-purge = "File:" + regsub(req.url,
"^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1");
- } elsif (req.url ~ "^/images/") {
+ } else if (req.url ~ "^/images/") {
set req.http.xkey-purge = "File:" + regsub(req.url,
"^/images/[^/]+/[^/]+/(.*)", "\1");
} else {
# Not an identifiable file, regular purge
@@ -59,15 +59,18 @@
# qlow jpg thumbs
if (req.url ~ "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)") {
- set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(40):sharpen(0.6,0.01,false)/127.0.0.1/images/" +
regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(40):sharpen(0.6,0.01,false)/127.0.0.1:<%=
scope['::swift::port'] %>/v1/AUTH_testproj/wiki-local-public/" +
regsub(req.url, "^/images/thumb/(.*)/qlow-(\d+)px-.*\.(jpg|jpeg)", "\1");
return (hash);
# regular jpg thumbs
} else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)") {
- set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(87):sharpen(0.6,0.01,false)/127.0.0.1/images/" +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\2") +
"x/filters:quality(87):sharpen(0.6,0.01,false)/127.0.0.1:<%=
scope['::swift::port'] %>/v1/AUTH_testproj/wiki-local-public/" +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.(jpg|jpeg)", "\1");
return (hash);
# png thumbs
} else if (req.url ~ "^/images/thumb/(.*)/(\d+)px-.*\.png") {
- set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") + "x/127.0.0.1/images/" +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
+ set req.url = "/unsafe/" + regsub(req.url,
"^/images/thumb/(.*)/(\d+)px-.*\.png", "\2") + "x/127.0.0.1:<%=
scope['::swift::port'] %>/v1/AUTH_testproj/wiki-local-public/" +
regsub(req.url, "^/images/thumb/(.*)/(\d+)px-.*\.png", "\1");
+ return (hash);
+ } else if (req.url ~ "^/images/(.*)") {
+ set req.url = "/v1/AUTH_testproj/wiki-local-public/" + regsub(req.url,
"^/images/(.*)", "\1");
return (hash);
}
@@ -82,7 +85,7 @@
if (bereq.http.X-Url ~ "^/images/thumb/") {
set beresp.http.xkey = "File:" + regsub(bereq.http.X-Url,
"^/images/thumb/[^/]+/[^/]+/([^/]+)/[^/]+$", "\1");
- } elsif (bereq.http.X-Url ~ "^/images/") {
+ } else if (bereq.http.X-Url ~ "^/images/") {
set beresp.http.xkey = "File:" + regsub(bereq.http.X-Url,
"^/images/[^/]+/[^/]+/(.*)", "\1");
}
--
To view, visit https://gerrit.wikimedia.org/r/256473
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia516a964a65e0b224e8d10e909e77d05a7acb1c2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Gilles <gdu...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
