Filippo Giunchedi has submitted this change and it was merged. Change subject: swift_new: add hiera data for eqiad/esams ......................................................................
swift_new: add hiera data for eqiad/esams Abstract away swift_new common hiera data and add new hiera data for the eqiad and esams clusters, in preparation for switching them to swift_new. Change-Id: I901a8192f8e89b19a6c8385807fd79006aaf56a8 --- M hieradata/codfw.yaml M hieradata/codfw/swift_new/proxy.yaml A hieradata/common/swift_new/proxy.yaml R hieradata/common/swift_new/storage.yaml A hieradata/eqiad/swift_new.yaml A hieradata/eqiad/swift_new/params.yaml A hieradata/eqiad/swift_new/proxy.yaml A hieradata/esams/swift_new.yaml A hieradata/esams/swift_new/params.yaml A hieradata/esams/swift_new/proxy.yaml M hieradata/regex.yaml M hieradata/role/codfw/swift/proxy.yaml A hieradata/role/eqiad/swift/proxy.yaml A hieradata/role/esams/swift/proxy.yaml 14 files changed, 174 insertions(+), 66 deletions(-) Approvals: Filippo Giunchedi: Verified; Looks good to me, approved diff --git a/hieradata/codfw.yaml b/hieradata/codfw.yaml index 6229b94..d07e4b6 100644 --- a/hieradata/codfw.yaml +++ b/hieradata/codfw.yaml @@ -1,10 +1,3 @@ -swift_storage_drives: [ - '/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', - '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh', - '/dev/sdi', '/dev/sdj', '/dev/sdk', '/dev/sdl' -# no trailing comma! -] -swift_aux_partitions: ['/dev/sdm3', '/dev/sdn3'] swift_check_http_host: 'ms-fe.svc.codfw.wmnet' mediawiki_memcached_servers: - '10.192.0.34:11211:1 "shard01"' diff --git a/hieradata/codfw/swift_new/proxy.yaml b/hieradata/codfw/swift_new/proxy.yaml index 7886caf..73f6ff7 100644 --- a/hieradata/codfw/swift_new/proxy.yaml +++ b/hieradata/codfw/swift_new/proxy.yaml @@ -1,65 +1,8 @@ proxy_address: 'http://ms-fe.svc.codfw.wmnet' -statsd_host: 'localhost' -backend_url_format: 'sitelang' -rewrite_thumb_server: 'rendering.svc.eqiad.wmnet' -dispersion_account: 'dispersion' -rewrite_account: 'mw_media' memcached_servers: [ 'ms-fe2001.codfw.wmnet:11211', 'ms-fe2002.codfw.wmnet:11211', 'ms-fe2003.codfw.wmnet:11211', 'ms-fe2004.codfw.wmnet:11211' # no trailing comma! - ] -shard_container_list: [ - 'global-data-math-render', - - 'wikipedia-commons-local-public', 'wikipedia-commons-local-temp', - 'wikipedia-commons-local-thumb', 'wikipedia-commons-local-transcoded', - - 'wikipedia-de-local-public', 'wikipedia-de-local-temp', - 'wikipedia-de-local-thumb', 'wikipedia-de-local-transcoded', - - 'wikipedia-en-local-public', 'wikipedia-en-local-temp', - 'wikipedia-en-local-thumb', 'wikipedia-en-local-transcoded', - - 'wikipedia-fi-local-public', 'wikipedia-fi-local-temp', - 'wikipedia-fi-local-thumb', 'wikipedia-fi-local-transcoded', - - 'wikipedia-fr-local-public', 'wikipedia-fr-local-temp', - 'wikipedia-fr-local-thumb', 'wikipedia-fr-local-transcoded', - - 'wikipedia-he-local-public', 'wikipedia-he-local-temp', - 'wikipedia-he-local-thumb', 'wikipedia-he-local-transcoded', - - 'wikipedia-hu-local-public', 'wikipedia-hu-local-temp', - 'wikipedia-hu-local-thumb', 'wikipedia-hu-local-transcoded', - - 'wikipedia-id-local-public', 'wikipedia-id-local-temp', - 'wikipedia-id-local-thumb', 'wikipedia-id-local-transcoded', - - 'wikipedia-it-local-public', 'wikipedia-it-local-temp', - 'wikipedia-it-local-thumb', 'wikipedia-it-local-transcoded', - - 'wikipedia-ja-local-public', 'wikipedia-ja-local-temp', - 'wikipedia-ja-local-thumb', 'wikipedia-ja-local-transcoded', - - 'wikipedia-ro-local-public', 'wikipedia-ro-local-temp', - 'wikipedia-ro-local-thumb', 'wikipedia-ro-local-transcoded', - - 'wikipedia-ru-local-public', 'wikipedia-ru-local-temp', - 'wikipedia-ru-local-thumb', 'wikipedia-ru-local-transcoded', - - 'wikipedia-th-local-public', 'wikipedia-th-local-temp', - 'wikipedia-th-local-thumb', 'wikipedia-th-local-transcoded', - - 'wikipedia-tr-local-public', 'wikipedia-tr-local-temp', - 'wikipedia-tr-local-thumb', 'wikipedia-tr-local-transcoded', - - 'wikipedia-uk-local-public', 'wikipedia-uk-local-temp', - 'wikipedia-uk-local-thumb', 'wikipedia-uk-local-transcoded', - - 'wikipedia-zh-local-public', 'wikipedia-zh-local-temp', - 'wikipedia-zh-local-thumb', 'wikipedia-zh-local-transcoded' -# no trailing comma! - ] +] diff --git a/hieradata/common/swift_new/proxy.yaml b/hieradata/common/swift_new/proxy.yaml new file mode 100644 index 0000000..7f50fc6 --- /dev/null +++ b/hieradata/common/swift_new/proxy.yaml @@ -0,0 +1,57 @@ +statsd_host: 'localhost' +backend_url_format: 'sitelang' +rewrite_thumb_server: 'rendering.svc.eqiad.wmnet' +dispersion_account: 'dispersion' +rewrite_account: 'mw_media' +shard_container_list: [ + 'global-data-math-render', + + 'wikipedia-commons-local-public', 'wikipedia-commons-local-temp', + 'wikipedia-commons-local-thumb', 'wikipedia-commons-local-transcoded', + + 'wikipedia-de-local-public', 'wikipedia-de-local-temp', + 'wikipedia-de-local-thumb', 'wikipedia-de-local-transcoded', + + 'wikipedia-en-local-public', 'wikipedia-en-local-temp', + 'wikipedia-en-local-thumb', 'wikipedia-en-local-transcoded', + + 'wikipedia-fi-local-public', 'wikipedia-fi-local-temp', + 'wikipedia-fi-local-thumb', 'wikipedia-fi-local-transcoded', + + 'wikipedia-fr-local-public', 'wikipedia-fr-local-temp', + 'wikipedia-fr-local-thumb', 'wikipedia-fr-local-transcoded', + + 'wikipedia-he-local-public', 'wikipedia-he-local-temp', + 'wikipedia-he-local-thumb', 'wikipedia-he-local-transcoded', + + 'wikipedia-hu-local-public', 'wikipedia-hu-local-temp', + 'wikipedia-hu-local-thumb', 'wikipedia-hu-local-transcoded', + + 'wikipedia-id-local-public', 'wikipedia-id-local-temp', + 'wikipedia-id-local-thumb', 'wikipedia-id-local-transcoded', + + 'wikipedia-it-local-public', 'wikipedia-it-local-temp', + 'wikipedia-it-local-thumb', 'wikipedia-it-local-transcoded', + + 'wikipedia-ja-local-public', 'wikipedia-ja-local-temp', + 'wikipedia-ja-local-thumb', 'wikipedia-ja-local-transcoded', + + 'wikipedia-ro-local-public', 'wikipedia-ro-local-temp', + 'wikipedia-ro-local-thumb', 'wikipedia-ro-local-transcoded', + + 'wikipedia-ru-local-public', 'wikipedia-ru-local-temp', + 'wikipedia-ru-local-thumb', 'wikipedia-ru-local-transcoded', + + 'wikipedia-th-local-public', 'wikipedia-th-local-temp', + 'wikipedia-th-local-thumb', 'wikipedia-th-local-transcoded', + + 'wikipedia-tr-local-public', 'wikipedia-tr-local-temp', + 'wikipedia-tr-local-thumb', 'wikipedia-tr-local-transcoded', + + 'wikipedia-uk-local-public', 'wikipedia-uk-local-temp', + 'wikipedia-uk-local-thumb', 'wikipedia-uk-local-transcoded', + + 'wikipedia-zh-local-public', 'wikipedia-zh-local-temp', + 'wikipedia-zh-local-thumb', 'wikipedia-zh-local-transcoded' +# no trailing comma! + ] diff --git a/hieradata/codfw/swift_new/storage.yaml b/hieradata/common/swift_new/storage.yaml similarity index 100% rename from hieradata/codfw/swift_new/storage.yaml rename to hieradata/common/swift_new/storage.yaml diff --git a/hieradata/eqiad/swift_new.yaml b/hieradata/eqiad/swift_new.yaml new file mode 100644 index 0000000..d4e7f17 --- /dev/null +++ b/hieradata/eqiad/swift_new.yaml @@ -0,0 +1 @@ +hash_path_suffix: '4f93c548a5903a13' diff --git a/hieradata/eqiad/swift_new/params.yaml b/hieradata/eqiad/swift_new/params.yaml new file mode 100644 index 0000000..5239667 --- /dev/null +++ b/hieradata/eqiad/swift_new/params.yaml @@ -0,0 +1,29 @@ +swift_cluster: eqiad-prod +accounts: + super_admin: + access: '.admin .reseller_admin' + account_name: 'AUTH_admin' + auth: 'http://ms-fe.svc.eqiad.wmnet' + user: 'admin:admin' + mw_media: + access: '.admin' + account_name: 'AUTH_mw' + auth: 'http://ms-fe.svc.eqiad.wmnet' + user: 'mw:media' + dispersion: + access: '.admin' + account_name: 'AUTH_dispersion' + auth: 'http://ms-fe.svc.eqiad.wmnet' + user: 'swift:dispersion' + search_backup: + access: '.admin' + account_name: 'AUTH_search' + auth: 'http://ms-fe.svc.eqiad.wmnet' + user: 'search:backup' +# keys are to be defined in private repo, e.g.: +#swift_new::params::account_keys: +# super_admin: <key> +# mw_media: <key> +# dispersion: <key> +# search_backup: <key> + diff --git a/hieradata/eqiad/swift_new/proxy.yaml b/hieradata/eqiad/swift_new/proxy.yaml new file mode 100644 index 0000000..4fa3561 --- /dev/null +++ b/hieradata/eqiad/swift_new/proxy.yaml @@ -0,0 +1,7 @@ +proxy_address: 'http://ms-fe.svc.eqiad.wmnet' +memcached_servers: [ + 'ms-fe1001.eqiad.wmnet:11211', + 'ms-fe1002.eqiad.wmnet:11211', + 'ms-fe1003.eqiad.wmnet:11211', + 'ms-fe1004.eqiad.wmnet:11211' +# no trailing comma! diff --git a/hieradata/esams/swift_new.yaml b/hieradata/esams/swift_new.yaml new file mode 100644 index 0000000..6d71bf9 --- /dev/null +++ b/hieradata/esams/swift_new.yaml @@ -0,0 +1 @@ +hash_path_suffix: 'a0af6563d361f968' diff --git a/hieradata/esams/swift_new/params.yaml b/hieradata/esams/swift_new/params.yaml new file mode 100644 index 0000000..225db86 --- /dev/null +++ b/hieradata/esams/swift_new/params.yaml @@ -0,0 +1,24 @@ +swift_cluster: esams-prod +accounts: + super_admin: + access: '.admin .reseller_admin' + account_name: 'AUTH_admin' + auth: 'http://ms-fe.svc.esams.wmnet' + user: 'admin:admin' + mw_media: + access: '.admin' + account_name: 'AUTH_mw' + auth: 'http://ms-fe.svc.esams.wmnet' + user: 'mw:media' + dispersion: + access: '.admin' + account_name: 'AUTH_dispersion' + auth: 'http://ms-fe.svc.esams.wmnet' + user: 'swift:dispersion' +# keys are to be defined in private repo, e.g.: +#swift_new::params::account_keys: +# super_admin: <key> +# mw_media: <key> +# dispersion: <key> +# search_backup: <key> + diff --git a/hieradata/esams/swift_new/proxy.yaml b/hieradata/esams/swift_new/proxy.yaml new file mode 100644 index 0000000..8033b56 --- /dev/null +++ b/hieradata/esams/swift_new/proxy.yaml @@ -0,0 +1,5 @@ +proxy_address: 'http://ms-fe.svc.esams.wmnet' +memcached_servers: [ + 'ms-fe3001.esams.wmnet:11211', + 'ms-fe3002.esams.wmnet:11211', +# no trailing comma! diff --git a/hieradata/regex.yaml b/hieradata/regex.yaml index cd9bd73..e535ac3 100644 --- a/hieradata/regex.yaml +++ b/hieradata/regex.yaml @@ -38,3 +38,45 @@ - xenon.eqiad.wmnet cassandra::max_heap_size: 4g cassandra::heap_newsize: 1g + +swift_be_codfw: + __regex: !ruby/regexp /^ms-be20.*\.codfw\.wmnet$/ + swift_storage_drives: [ + '/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', + '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh', + '/dev/sdi', '/dev/sdj', '/dev/sdk', '/dev/sdl' + # no trailing comma! + ] + swift_aux_partitions: ['/dev/sdm3', '/dev/sdn3'] + +swift_be_eqiad_dell: + __regex: !ruby/regexp /^ms-be10(0[0-9]|1[0-5])\.eqiad\.wmnet$/ + swift_storage_drives: [ + '/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', + '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh', + '/dev/sdi', '/dev/sdj', '/dev/sdk', '/dev/sdl' + # no trailing comma! + ] + swift_aux_partitions: ['/dev/sdm3', '/dev/sdn3'] + +# HP machines have different disk ordering T90922 +swift_be_eqiad_hp: + __regex: !ruby/regexp /^ms-be101[678]\.eqiad\.wmnet$/ + swift_storage_drives: [ + '/dev/sdm', '/dev/sdn', '/dev/sdc', '/dev/sdd', + '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh', + '/dev/sdi', '/dev/sdj', '/dev/sdk', '/dev/sdl' + # no trailing comma! + ] + swift_aux_partitions: ['/dev/sda3', '/dev/sdb3'] + +# 720xd *without* SSDs; sda & sdb serve both as root and as Swift disks +swift_be_esams: + __regex: !ruby/regexp /^ms-be300[1-4]\.esams\.wmnet$/ + swift_storage_drives: [ + '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', + '/dev/sdg', '/dev/sdh', '/dev/sdi', '/dev/sdj', + '/dev/sdk', '/dev/sdl' + # no trailing comma! + ] + swift_aux_partitions: ['/dev/sda3', '/dev/sdb3'] diff --git a/hieradata/role/codfw/swift/proxy.yaml b/hieradata/role/codfw/swift/proxy.yaml index 1a0ed89..28abdfa 100644 --- a/hieradata/role/codfw/swift/proxy.yaml +++ b/hieradata/role/codfw/swift/proxy.yaml @@ -1,4 +1,4 @@ cluster: swift nagios_group: swift -lvs::realserver::realserver_ips: +lvs::realserver::realserver_ips: - '10.2.1.27' diff --git a/hieradata/role/eqiad/swift/proxy.yaml b/hieradata/role/eqiad/swift/proxy.yaml new file mode 100644 index 0000000..b4371b8 --- /dev/null +++ b/hieradata/role/eqiad/swift/proxy.yaml @@ -0,0 +1,4 @@ +cluster: swift +nagios_group: swift +lvs::realserver::realserver_ips: + - '10.2.2.27' diff --git a/hieradata/role/esams/swift/proxy.yaml b/hieradata/role/esams/swift/proxy.yaml new file mode 100644 index 0000000..21bef47 --- /dev/null +++ b/hieradata/role/esams/swift/proxy.yaml @@ -0,0 +1,2 @@ +cluster: swift +nagios_group: swift -- To view, visit https://gerrit.wikimedia.org/r/231235 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I901a8192f8e89b19a6c8385807fd79006aaf56a8 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org> Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
