BBlack has uploaded a new change for review. https://gerrit.wikimedia.org/r/204068
Change subject: r::c::config::active_nodes -> hiera cache::nodes ...................................................................... r::c::config::active_nodes -> hiera cache::nodes WIP! Not even remotely sane or tested yet! Haven't even looked at the torrus fallout for this one... Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8 --- R hieradata/common/cache/nodes/bits.yaml A hieradata/common/cache/nodes/misc.yaml A hieradata/common/cache/nodes/mobile.yaml A hieradata/common/cache/nodes/parsoid.yaml R hieradata/common/cache/nodes/text.yaml R hieradata/common/cache/nodes/upload.yaml M hieradata/labs.yaml A hieradata/role/common/cache.yaml D hieradata/role/common/cache/misc.yaml D hieradata/role/common/cache/mobile.yaml D hieradata/role/common/cache/parsoid.yaml M manifests/role/analytics/kafkatee.pp M manifests/role/ipsec.pp M modules/role/manifests/cache/configuration.pp M modules/role/manifests/cache/mobile.pp M modules/role/manifests/cache/parsoid.pp M modules/role/manifests/cache/text.pp M modules/role/manifests/cache/upload.pp M templates/udp2log/filters.oxygen.erb 19 files changed, 84 insertions(+), 218 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/68/204068/1 diff --git a/hieradata/role/common/cache/bits.yaml b/hieradata/common/cache/nodes/bits.yaml similarity index 69% rename from hieradata/role/common/cache/bits.yaml rename to hieradata/common/cache/nodes/bits.yaml index 4dab273..49648d3 100644 --- a/hieradata/role/common/cache/bits.yaml +++ b/hieradata/common/cache/nodes/bits.yaml @@ -1,17 +1,15 @@ -cluster: cache_bits -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: +codfw: [] +eqiad: - 'cp1056.eqiad.wmnet' - 'cp1057.eqiad.wmnet' - 'cp1069.eqiad.wmnet' - 'cp1070.eqiad.wmnet' -hosts_esams: +esams: - 'cp3019.esams.wmnet' - 'cp3020.esams.wmnet' - 'cp3021.esams.wmnet' - 'cp3022.esams.wmnet' -hosts_ulsfo: +ulsfo: - 'cp4001.ulsfo.wmnet' - 'cp4002.ulsfo.wmnet' - 'cp4003.ulsfo.wmnet' diff --git a/hieradata/common/cache/nodes/misc.yaml b/hieradata/common/cache/nodes/misc.yaml new file mode 100644 index 0000000..38ea120 --- /dev/null +++ b/hieradata/common/cache/nodes/misc.yaml @@ -0,0 +1,3 @@ +eqiad: + - 'cp1043.eqiad.wmnet' + - 'cp1044.eqiad.wmnet' diff --git a/hieradata/common/cache/nodes/mobile.yaml b/hieradata/common/cache/nodes/mobile.yaml new file mode 100644 index 0000000..98a6d67 --- /dev/null +++ b/hieradata/common/cache/nodes/mobile.yaml @@ -0,0 +1,17 @@ +mobile: + codfw: [] + eqiad: + - 'cp1046.eqiad.wmnet' + - 'cp1047.eqiad.wmnet' + - 'cp1059.eqiad.wmnet' + - 'cp1060.eqiad.wmnet' + esams: + - 'cp3015.esams.wmnet' + - 'cp3016.esams.wmnet' + - 'cp3017.esams.wmnet' + - 'cp3018.esams.wmnet' + ulsfo: + - 'cp4011.ulsfo.wmnet' + - 'cp4012.ulsfo.wmnet' + - 'cp4019.ulsfo.wmnet' + - 'cp4020.ulsfo.wmnet' diff --git a/hieradata/common/cache/nodes/parsoid.yaml b/hieradata/common/cache/nodes/parsoid.yaml new file mode 100644 index 0000000..cef3acc --- /dev/null +++ b/hieradata/common/cache/nodes/parsoid.yaml @@ -0,0 +1,3 @@ +eqiad: + - 'cp1045.eqiad.wmnet' + - 'cp1058.eqiad.wmnet' diff --git a/hieradata/role/common/cache/text.yaml b/hieradata/common/cache/nodes/text.yaml similarity index 84% rename from hieradata/role/common/cache/text.yaml rename to hieradata/common/cache/nodes/text.yaml index d94a3f4..543826d 100644 --- a/hieradata/role/common/cache/text.yaml +++ b/hieradata/common/cache/nodes/text.yaml @@ -1,7 +1,5 @@ -cluster: cache_text -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: +codfw: [] +eqiad: - 'cp1052.eqiad.wmnet' - 'cp1053.eqiad.wmnet' - 'cp1054.eqiad.wmnet' @@ -10,7 +8,7 @@ - 'cp1066.eqiad.wmnet' - 'cp1067.eqiad.wmnet' - 'cp1068.eqiad.wmnet' -hosts_esams: +esams: - 'cp3003.esams.wmnet' - 'cp3004.esams.wmnet' - 'cp3005.esams.wmnet' @@ -27,7 +25,7 @@ - 'cp3031.esams.wmnet' - 'cp3040.esams.wmnet' - 'cp3041.esams.wmnet' -hosts_ulsfo: +ulsfo: - 'cp4008.ulsfo.wmnet' - 'cp4009.ulsfo.wmnet' - 'cp4010.ulsfo.wmnet' diff --git a/hieradata/role/common/cache/upload.yaml b/hieradata/common/cache/nodes/upload.yaml similarity index 86% rename from hieradata/role/common/cache/upload.yaml rename to hieradata/common/cache/nodes/upload.yaml index 1cf9c19..6b3dbf6 100644 --- a/hieradata/role/common/cache/upload.yaml +++ b/hieradata/common/cache/nodes/upload.yaml @@ -1,7 +1,5 @@ -cluster: cache_upload -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: +codfw: [] +eqiad: - 'cp1048.eqiad.wmnet' - 'cp1049.eqiad.wmnet' - 'cp1050.eqiad.wmnet' @@ -14,7 +12,7 @@ - 'cp1072.eqiad.wmnet' - 'cp1073.eqiad.wmnet' - 'cp1074.eqiad.wmnet' -hosts_esams: +esams: - 'cp3032.esams.wmnet' - 'cp3033.esams.wmnet' - 'cp3034.esams.wmnet' @@ -31,7 +29,7 @@ - 'cp3047.esams.wmnet' - 'cp3048.esams.wmnet' - 'cp3049.esams.wmnet' -hosts_ulsfo: +ulsfo: - 'cp4005.ulsfo.wmnet' - 'cp4006.ulsfo.wmnet' - 'cp4007.ulsfo.wmnet' diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml index 88dc142..e703eff 100644 --- a/hieradata/labs.yaml +++ b/hieradata/labs.yaml @@ -9,3 +9,19 @@ archiva::proxy::ssl_enabled: false archiva::proxy::certificate_name: ssl-cert-snakeoil statsite::instance::graphite_host: 'labmon1001.eqiad.wmnet' +cache::nodes: + text: + eqiad: + - '127.0.0.1' + upload: + eqiad: + - '127.0.0.1' + mobile: + eqiad: + - '127.0.0.1' + bits: + eqiad: + - '127.0.0.1' + parsoid: + eqiad: + - '127.0.0.1' diff --git a/hieradata/role/common/cache.yaml b/hieradata/role/common/cache.yaml new file mode 100644 index 0000000..102ce59 --- /dev/null +++ b/hieradata/role/common/cache.yaml @@ -0,0 +1,15 @@ +text: + cluster: cache_text +upload.yaml + cluster: cache_upload +mobile: + cluster: cache_mobile +bits: + cluster: cache_bits +misc: + cluster: cache_misc +parsoid: + cluster: cache_parsoid + admin::groups: + - parsoid-roots + - parsoid-admin diff --git a/hieradata/role/common/cache/misc.yaml b/hieradata/role/common/cache/misc.yaml deleted file mode 100644 index 6494805..0000000 --- a/hieradata/role/common/cache/misc.yaml +++ /dev/null @@ -1 +0,0 @@ -cluster: cache_misc diff --git a/hieradata/role/common/cache/mobile.yaml b/hieradata/role/common/cache/mobile.yaml deleted file mode 100644 index e8197a7..0000000 --- a/hieradata/role/common/cache/mobile.yaml +++ /dev/null @@ -1,19 +0,0 @@ -cluster: cache_mobile -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: - - 'cp1046.eqiad.wmnet' - - 'cp1047.eqiad.wmnet' - - 'cp1059.eqiad.wmnet' - - 'cp1060.eqiad.wmnet' -hosts_esams: - - 'cp3015.esams.wmnet' - - 'cp3016.esams.wmnet' - - 'cp3017.esams.wmnet' - - 'cp3018.esams.wmnet' -hosts_ulsfo: - - 'cp4011.ulsfo.wmnet' - - 'cp4012.ulsfo.wmnet' - - 'cp4019.ulsfo.wmnet' - - 'cp4020.ulsfo.wmnet' - diff --git a/hieradata/role/common/cache/parsoid.yaml b/hieradata/role/common/cache/parsoid.yaml deleted file mode 100644 index b15ab81..0000000 --- a/hieradata/role/common/cache/parsoid.yaml +++ /dev/null @@ -1,4 +0,0 @@ -cluster: cache_parsoid -admin::groups: - - parsoid-roots - - parsoid-admin diff --git a/manifests/role/analytics/kafkatee.pp b/manifests/role/analytics/kafkatee.pp index 0db257e..b8aaf44 100644 --- a/manifests/role/analytics/kafkatee.pp +++ b/manifests/role/analytics/kafkatee.pp @@ -78,7 +78,7 @@ # Include this to infer mobile varnish frontend hostnames on which to filter. include role::cache::configuration - $cache_configuration = $role::cache::configuration::active_nodes['production']['mobile'] + $cache_configuration = $::cache::nodes['mobile'] $mobile_hosts_regex = inline_template('(<%= @cache_configuration.values.flatten.sort.join(\'|\') %>)') # 1/100 sampling of traffic from mobile varnishes diff --git a/manifests/role/ipsec.pp b/manifests/role/ipsec.pp index 73eb4af..7682885 100644 --- a/manifests/role/ipsec.pp +++ b/manifests/role/ipsec.pp @@ -12,18 +12,19 @@ if $hosts != undef { $targets = $hosts } else { + $cluster_nodes = hiera_lookup("cache::nodes::${cluster}", {}) # for 'left' nodes in cache sites, enumerate 'right' nodes in "main" sites if $::site == 'esams' or $::site == 'ulsfo' { $targets = concat( - hiera('hosts_eqiad', []), - hiera('hosts_codfw', []) + $cluster_nodes{'eqiad'} + $cluster_nodes{'codfw'} ) } # for 'left' nodes in "main" sites, enumerate 'right' nodes in cache sites if $::site == 'eqiad' or $::site == 'codfw' { $targets = concat( - hiera('hosts_esams', []), - hiera('hosts_ulsfo', []) + $cluster_nodes{'esams'} + $cluster_nodes{'ulsfo'} ) } } diff --git a/modules/role/manifests/cache/configuration.pp b/modules/role/manifests/cache/configuration.pp index b686b4d..6cdae9b 100644 --- a/modules/role/manifests/cache/configuration.pp +++ b/modules/role/manifests/cache/configuration.pp @@ -3,164 +3,6 @@ $has_ganglia = hiera('has_ganglia', true) - $active_nodes = { - 'production' => { - 'text' => { - 'eqiad' => [ - 'cp1052.eqiad.wmnet', - 'cp1053.eqiad.wmnet', - 'cp1054.eqiad.wmnet', - 'cp1055.eqiad.wmnet', - 'cp1065.eqiad.wmnet', - 'cp1066.eqiad.wmnet', - 'cp1067.eqiad.wmnet', - 'cp1068.eqiad.wmnet', - ], - 'esams' => [ - 'cp3003.esams.wmnet', - 'cp3004.esams.wmnet', - 'cp3005.esams.wmnet', - 'cp3006.esams.wmnet', - 'cp3007.esams.wmnet', - 'cp3008.esams.wmnet', - 'cp3009.esams.wmnet', - 'cp3010.esams.wmnet', - # T92306 'cp3011.esams.wmnet', # needs-jessie-install - 'cp3012.esams.wmnet', - 'cp3013.esams.wmnet', - 'cp3014.esams.wmnet', - 'cp3030.esams.wmnet', - 'cp3031.esams.wmnet', - 'cp3040.esams.wmnet', - 'cp3041.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4008.ulsfo.wmnet', - 'cp4009.ulsfo.wmnet', - 'cp4010.ulsfo.wmnet', - 'cp4016.ulsfo.wmnet', - 'cp4017.ulsfo.wmnet', - 'cp4018.ulsfo.wmnet', - ] - }, - 'bits' => { - 'eqiad' => [ - 'cp1056.eqiad.wmnet', - 'cp1057.eqiad.wmnet', - 'cp1069.eqiad.wmnet', - 'cp1070.eqiad.wmnet', - ], - 'esams' => [ - 'cp3019.esams.wmnet', - 'cp3020.esams.wmnet', - 'cp3021.esams.wmnet', - 'cp3022.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4001.ulsfo.wmnet', - 'cp4002.ulsfo.wmnet', - 'cp4003.ulsfo.wmnet', - 'cp4004.ulsfo.wmnet', - ], - }, - 'upload' => { - 'eqiad' => [ - 'cp1048.eqiad.wmnet', - 'cp1049.eqiad.wmnet', - 'cp1050.eqiad.wmnet', - 'cp1051.eqiad.wmnet', - 'cp1061.eqiad.wmnet', - 'cp1062.eqiad.wmnet', - 'cp1063.eqiad.wmnet', - 'cp1064.eqiad.wmnet', - 'cp1071.eqiad.wmnet', - 'cp1072.eqiad.wmnet', - 'cp1073.eqiad.wmnet', - 'cp1074.eqiad.wmnet', - ], - 'esams' => [ - 'cp3032.esams.wmnet', - 'cp3033.esams.wmnet', - 'cp3034.esams.wmnet', - 'cp3035.esams.wmnet', - 'cp3036.esams.wmnet', - 'cp3037.esams.wmnet', - 'cp3038.esams.wmnet', - 'cp3039.esams.wmnet', - 'cp3042.esams.wmnet', - 'cp3043.esams.wmnet', - 'cp3044.esams.wmnet', - 'cp3045.esams.wmnet', - 'cp3046.esams.wmnet', - 'cp3047.esams.wmnet', - 'cp3048.esams.wmnet', - 'cp3049.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4005.ulsfo.wmnet', - 'cp4006.ulsfo.wmnet', - 'cp4007.ulsfo.wmnet', - 'cp4013.ulsfo.wmnet', - 'cp4014.ulsfo.wmnet', - 'cp4015.ulsfo.wmnet', - ], - }, - 'mobile' => { - 'eqiad' => [ - 'cp1046.eqiad.wmnet', - 'cp1047.eqiad.wmnet', - 'cp1059.eqiad.wmnet', - 'cp1060.eqiad.wmnet', - ], - 'esams' => [ - 'cp3015.esams.wmnet', - 'cp3016.esams.wmnet', - 'cp3017.esams.wmnet', - 'cp3018.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4011.ulsfo.wmnet', - 'cp4012.ulsfo.wmnet', - 'cp4019.ulsfo.wmnet', - 'cp4020.ulsfo.wmnet', - ] - }, - 'parsoid' => { - 'eqiad' => [ - 'cp1045.eqiad.wmnet', - 'cp1058.eqiad.wmnet', - ], - 'esams' => [], - 'ulsfo' => [] - }, - 'misc' => { - 'eqiad' => [ - 'cp1043.eqiad.wmnet', - 'cp1044.eqiad.wmnet', - ], - 'esams' => [], - 'ulsfo' => [], - }, - }, - 'labs' => { - 'bits' => { - 'eqiad' => '127.0.0.1', - }, - 'mobile' => { - 'eqiad' => '127.0.0.1', - }, - 'text' => { - 'eqiad' => '127.0.0.1', - }, - 'upload' => { - 'eqiad' => '127.0.0.1', - }, - 'parsoid' => { - 'eqiad' => '127.0.0.1', - }, - }, - } - $backends = { 'production' => { 'appservers' => $lvs::configuration::lvs_service_ips['production']['apaches'], diff --git a/modules/role/manifests/cache/mobile.pp b/modules/role/manifests/cache/mobile.pp index d76e5b6..2aff7ef 100644 --- a/modules/role/manifests/cache/mobile.pp +++ b/modules/role/manifests/cache/mobile.pp @@ -31,7 +31,7 @@ 'test_wikipedia' => $role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['mobile']['eqiad'], + 'eqiad' => $::cache::nodes['mobile']['eqiad'], } } @@ -145,10 +145,10 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site], + 'backend' => $::cache::nodes['mobile'][$::site], }, director_options => { - 'retries' => $backend_weight_avg * size($::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site]), + 'retries' => $backend_weight_avg * size($::cache::nodes['mobile'][$::site]), }, director_type => 'chash', vcl_config => { diff --git a/modules/role/manifests/cache/parsoid.pp b/modules/role/manifests/cache/parsoid.pp index 2d05599..c1e59fd 100644 --- a/modules/role/manifests/cache/parsoid.pp +++ b/modules/role/manifests/cache/parsoid.pp @@ -83,14 +83,14 @@ port => 80, admin_port => 6082, directors => { - 'backend' => $::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site], + 'backend' => $::cache::nodes['parsoid'][$::site], 'cxserver_backend' => $::role::cache::configuration::backends[$::realm]['cxserver'][$::site], 'citoid_backend' => $::role::cache::configuration::backends[$::realm]['citoid'][$::site], 'restbase_backend' => $::role::cache::configuration::backends[$::realm]['restbase'][$::site], }, director_type => 'chash', director_options => { - 'retries' => $backend_weight_avg * size($::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site]), + 'retries' => $backend_weight_avg * size($::cache::nodes['parsoid'][$::site]), }, vcl_config => { 'retry5xx' => 0, diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index daaf670..af74c8e 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -29,7 +29,7 @@ 'test_wikipedia' => $role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['text']['eqiad'], + 'eqiad' => $::cache::nodes['text']['eqiad'], }, } @@ -114,7 +114,7 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $role::cache::configuration::active_nodes[$::realm]['text'][$::site], + 'backend' => $::cache::nodes['text'][$::site], }, director_type => 'chash', vcl_config => { diff --git a/modules/role/manifests/cache/upload.pp b/modules/role/manifests/cache/upload.pp index 3e22b0d..92b08d4 100644 --- a/modules/role/manifests/cache/upload.pp +++ b/modules/role/manifests/cache/upload.pp @@ -24,7 +24,7 @@ 'rendering' => $role::cache::configuration::backends[$::realm]['rendering'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['upload']['eqiad'] + 'eqiad' => $::cache::nodes['upload']['eqiad'] } } @@ -141,7 +141,7 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $role::cache::configuration::active_nodes[$::realm]['upload'][$::site], + 'backend' => $::cache::nodes['upload'][$::site], }, director_type => 'chash', vcl_config => { diff --git a/templates/udp2log/filters.oxygen.erb b/templates/udp2log/filters.oxygen.erb index 7d06de6..1c9584e 100644 --- a/templates/udp2log/filters.oxygen.erb +++ b/templates/udp2log/filters.oxygen.erb @@ -18,10 +18,9 @@ pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> <%= log_directory %>/edits.tsv.log <% -# pull in $role::cache::configuration::active_nodes -# to find mobile host names and build a regex on which to grep. -cache_configuration = scope.lookupvar('::role::cache::configuration::active_nodes') -mobile_hosts_regex = '(' + cache_configuration['production']['mobile'].values.flatten.sort.join('|') + ')' +# find mobile host names and build a regex on which to grep. +cache_nodes_mobile = scope.lookupvar('::cache::nodes::mobile') +mobile_hosts_regex = '(' + cache_nodes_mobile.values.flatten.sort.join('|') + ')' -%> ### Mobile traffic filter -- To view, visit https://gerrit.wikimedia.org/r/204068 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BBlack <bbl...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits