BBlack has uploaded a new change for review.
https://gerrit.wikimedia.org/r/204068
Change subject: r::c::config::active_nodes -> hiera cache::nodes
......................................................................
r::c::config::active_nodes -> hiera cache::nodes
WIP! Not even remotely sane or tested yet!
Haven't even looked at the torrus fallout for this one...
Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8
---
R hieradata/common/cache/nodes/bits.yaml
A hieradata/common/cache/nodes/misc.yaml
A hieradata/common/cache/nodes/mobile.yaml
A hieradata/common/cache/nodes/parsoid.yaml
R hieradata/common/cache/nodes/text.yaml
R hieradata/common/cache/nodes/upload.yaml
M hieradata/labs.yaml
A hieradata/role/common/cache.yaml
D hieradata/role/common/cache/misc.yaml
D hieradata/role/common/cache/mobile.yaml
D hieradata/role/common/cache/parsoid.yaml
M manifests/role/analytics/kafkatee.pp
M manifests/role/ipsec.pp
M modules/role/manifests/cache/configuration.pp
M modules/role/manifests/cache/mobile.pp
M modules/role/manifests/cache/parsoid.pp
M modules/role/manifests/cache/text.pp
M modules/role/manifests/cache/upload.pp
M templates/udp2log/filters.oxygen.erb
19 files changed, 84 insertions(+), 218 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/68/204068/1
diff --git a/hieradata/role/common/cache/bits.yaml
b/hieradata/common/cache/nodes/bits.yaml
similarity index 69%
rename from hieradata/role/common/cache/bits.yaml
rename to hieradata/common/cache/nodes/bits.yaml
index 4dab273..49648d3 100644
--- a/hieradata/role/common/cache/bits.yaml
+++ b/hieradata/common/cache/nodes/bits.yaml
@@ -1,17 +1,15 @@
-cluster: cache_bits
-# data synced from manifests/role/cache.pp on 2015-04-08
-hosts_codfw: []
-hosts_eqiad:
+codfw: []
+eqiad:
- 'cp1056.eqiad.wmnet'
- 'cp1057.eqiad.wmnet'
- 'cp1069.eqiad.wmnet'
- 'cp1070.eqiad.wmnet'
-hosts_esams:
+esams:
- 'cp3019.esams.wmnet'
- 'cp3020.esams.wmnet'
- 'cp3021.esams.wmnet'
- 'cp3022.esams.wmnet'
-hosts_ulsfo:
+ulsfo:
- 'cp4001.ulsfo.wmnet'
- 'cp4002.ulsfo.wmnet'
- 'cp4003.ulsfo.wmnet'
diff --git a/hieradata/common/cache/nodes/misc.yaml
b/hieradata/common/cache/nodes/misc.yaml
new file mode 100644
index 0000000..38ea120
--- /dev/null
+++ b/hieradata/common/cache/nodes/misc.yaml
@@ -0,0 +1,3 @@
+eqiad:
+ - 'cp1043.eqiad.wmnet'
+ - 'cp1044.eqiad.wmnet'
diff --git a/hieradata/common/cache/nodes/mobile.yaml
b/hieradata/common/cache/nodes/mobile.yaml
new file mode 100644
index 0000000..98a6d67
--- /dev/null
+++ b/hieradata/common/cache/nodes/mobile.yaml
@@ -0,0 +1,17 @@
+mobile:
+ codfw: []
+ eqiad:
+ - 'cp1046.eqiad.wmnet'
+ - 'cp1047.eqiad.wmnet'
+ - 'cp1059.eqiad.wmnet'
+ - 'cp1060.eqiad.wmnet'
+ esams:
+ - 'cp3015.esams.wmnet'
+ - 'cp3016.esams.wmnet'
+ - 'cp3017.esams.wmnet'
+ - 'cp3018.esams.wmnet'
+ ulsfo:
+ - 'cp4011.ulsfo.wmnet'
+ - 'cp4012.ulsfo.wmnet'
+ - 'cp4019.ulsfo.wmnet'
+ - 'cp4020.ulsfo.wmnet'
diff --git a/hieradata/common/cache/nodes/parsoid.yaml
b/hieradata/common/cache/nodes/parsoid.yaml
new file mode 100644
index 0000000..cef3acc
--- /dev/null
+++ b/hieradata/common/cache/nodes/parsoid.yaml
@@ -0,0 +1,3 @@
+eqiad:
+ - 'cp1045.eqiad.wmnet'
+ - 'cp1058.eqiad.wmnet'
diff --git a/hieradata/role/common/cache/text.yaml
b/hieradata/common/cache/nodes/text.yaml
similarity index 84%
rename from hieradata/role/common/cache/text.yaml
rename to hieradata/common/cache/nodes/text.yaml
index d94a3f4..543826d 100644
--- a/hieradata/role/common/cache/text.yaml
+++ b/hieradata/common/cache/nodes/text.yaml
@@ -1,7 +1,5 @@
-cluster: cache_text
-# data synced from manifests/role/cache.pp on 2015-04-08
-hosts_codfw: []
-hosts_eqiad:
+codfw: []
+eqiad:
- 'cp1052.eqiad.wmnet'
- 'cp1053.eqiad.wmnet'
- 'cp1054.eqiad.wmnet'
@@ -10,7 +8,7 @@
- 'cp1066.eqiad.wmnet'
- 'cp1067.eqiad.wmnet'
- 'cp1068.eqiad.wmnet'
-hosts_esams:
+esams:
- 'cp3003.esams.wmnet'
- 'cp3004.esams.wmnet'
- 'cp3005.esams.wmnet'
@@ -27,7 +25,7 @@
- 'cp3031.esams.wmnet'
- 'cp3040.esams.wmnet'
- 'cp3041.esams.wmnet'
-hosts_ulsfo:
+ulsfo:
- 'cp4008.ulsfo.wmnet'
- 'cp4009.ulsfo.wmnet'
- 'cp4010.ulsfo.wmnet'
diff --git a/hieradata/role/common/cache/upload.yaml
b/hieradata/common/cache/nodes/upload.yaml
similarity index 86%
rename from hieradata/role/common/cache/upload.yaml
rename to hieradata/common/cache/nodes/upload.yaml
index 1cf9c19..6b3dbf6 100644
--- a/hieradata/role/common/cache/upload.yaml
+++ b/hieradata/common/cache/nodes/upload.yaml
@@ -1,7 +1,5 @@
-cluster: cache_upload
-# data synced from manifests/role/cache.pp on 2015-04-08
-hosts_codfw: []
-hosts_eqiad:
+codfw: []
+eqiad:
- 'cp1048.eqiad.wmnet'
- 'cp1049.eqiad.wmnet'
- 'cp1050.eqiad.wmnet'
@@ -14,7 +12,7 @@
- 'cp1072.eqiad.wmnet'
- 'cp1073.eqiad.wmnet'
- 'cp1074.eqiad.wmnet'
-hosts_esams:
+esams:
- 'cp3032.esams.wmnet'
- 'cp3033.esams.wmnet'
- 'cp3034.esams.wmnet'
@@ -31,7 +29,7 @@
- 'cp3047.esams.wmnet'
- 'cp3048.esams.wmnet'
- 'cp3049.esams.wmnet'
-hosts_ulsfo:
+ulsfo:
- 'cp4005.ulsfo.wmnet'
- 'cp4006.ulsfo.wmnet'
- 'cp4007.ulsfo.wmnet'
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index 88dc142..e703eff 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -9,3 +9,19 @@
archiva::proxy::ssl_enabled: false
archiva::proxy::certificate_name: ssl-cert-snakeoil
statsite::instance::graphite_host: 'labmon1001.eqiad.wmnet'
+cache::nodes:
+ text:
+ eqiad:
+ - '127.0.0.1'
+ upload:
+ eqiad:
+ - '127.0.0.1'
+ mobile:
+ eqiad:
+ - '127.0.0.1'
+ bits:
+ eqiad:
+ - '127.0.0.1'
+ parsoid:
+ eqiad:
+ - '127.0.0.1'
diff --git a/hieradata/role/common/cache.yaml b/hieradata/role/common/cache.yaml
new file mode 100644
index 0000000..102ce59
--- /dev/null
+++ b/hieradata/role/common/cache.yaml
@@ -0,0 +1,15 @@
+text:
+ cluster: cache_text
+upload.yaml
+ cluster: cache_upload
+mobile:
+ cluster: cache_mobile
+bits:
+ cluster: cache_bits
+misc:
+ cluster: cache_misc
+parsoid:
+ cluster: cache_parsoid
+ admin::groups:
+ - parsoid-roots
+ - parsoid-admin
diff --git a/hieradata/role/common/cache/misc.yaml
b/hieradata/role/common/cache/misc.yaml
deleted file mode 100644
index 6494805..0000000
--- a/hieradata/role/common/cache/misc.yaml
+++ /dev/null
@@ -1 +0,0 @@
-cluster: cache_misc
diff --git a/hieradata/role/common/cache/mobile.yaml
b/hieradata/role/common/cache/mobile.yaml
deleted file mode 100644
index e8197a7..0000000
--- a/hieradata/role/common/cache/mobile.yaml
+++ /dev/null
@@ -1,19 +0,0 @@
-cluster: cache_mobile
-# data synced from manifests/role/cache.pp on 2015-04-08
-hosts_codfw: []
-hosts_eqiad:
- - 'cp1046.eqiad.wmnet'
- - 'cp1047.eqiad.wmnet'
- - 'cp1059.eqiad.wmnet'
- - 'cp1060.eqiad.wmnet'
-hosts_esams:
- - 'cp3015.esams.wmnet'
- - 'cp3016.esams.wmnet'
- - 'cp3017.esams.wmnet'
- - 'cp3018.esams.wmnet'
-hosts_ulsfo:
- - 'cp4011.ulsfo.wmnet'
- - 'cp4012.ulsfo.wmnet'
- - 'cp4019.ulsfo.wmnet'
- - 'cp4020.ulsfo.wmnet'
-
diff --git a/hieradata/role/common/cache/parsoid.yaml
b/hieradata/role/common/cache/parsoid.yaml
deleted file mode 100644
index b15ab81..0000000
--- a/hieradata/role/common/cache/parsoid.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-cluster: cache_parsoid
-admin::groups:
- - parsoid-roots
- - parsoid-admin
diff --git a/manifests/role/analytics/kafkatee.pp
b/manifests/role/analytics/kafkatee.pp
index 0db257e..b8aaf44 100644
--- a/manifests/role/analytics/kafkatee.pp
+++ b/manifests/role/analytics/kafkatee.pp
@@ -78,7 +78,7 @@
# Include this to infer mobile varnish frontend hostnames on which to
filter.
include role::cache::configuration
- $cache_configuration =
$role::cache::configuration::active_nodes['production']['mobile']
+ $cache_configuration = $::cache::nodes['mobile']
$mobile_hosts_regex = inline_template('(<%=
@cache_configuration.values.flatten.sort.join(\'|\') %>)')
# 1/100 sampling of traffic from mobile varnishes
diff --git a/manifests/role/ipsec.pp b/manifests/role/ipsec.pp
index 73eb4af..7682885 100644
--- a/manifests/role/ipsec.pp
+++ b/manifests/role/ipsec.pp
@@ -12,18 +12,19 @@
if $hosts != undef {
$targets = $hosts
} else {
+ $cluster_nodes = hiera_lookup("cache::nodes::${cluster}", {})
# for 'left' nodes in cache sites, enumerate 'right' nodes in "main"
sites
if $::site == 'esams' or $::site == 'ulsfo' {
$targets = concat(
- hiera('hosts_eqiad', []),
- hiera('hosts_codfw', [])
+ $cluster_nodes{'eqiad'}
+ $cluster_nodes{'codfw'}
)
}
# for 'left' nodes in "main" sites, enumerate 'right' nodes in cache
sites
if $::site == 'eqiad' or $::site == 'codfw' {
$targets = concat(
- hiera('hosts_esams', []),
- hiera('hosts_ulsfo', [])
+ $cluster_nodes{'esams'}
+ $cluster_nodes{'ulsfo'}
)
}
}
diff --git a/modules/role/manifests/cache/configuration.pp
b/modules/role/manifests/cache/configuration.pp
index b686b4d..6cdae9b 100644
--- a/modules/role/manifests/cache/configuration.pp
+++ b/modules/role/manifests/cache/configuration.pp
@@ -3,164 +3,6 @@
$has_ganglia = hiera('has_ganglia', true)
- $active_nodes = {
- 'production' => {
- 'text' => {
- 'eqiad' => [
- 'cp1052.eqiad.wmnet',
- 'cp1053.eqiad.wmnet',
- 'cp1054.eqiad.wmnet',
- 'cp1055.eqiad.wmnet',
- 'cp1065.eqiad.wmnet',
- 'cp1066.eqiad.wmnet',
- 'cp1067.eqiad.wmnet',
- 'cp1068.eqiad.wmnet',
- ],
- 'esams' => [
- 'cp3003.esams.wmnet',
- 'cp3004.esams.wmnet',
- 'cp3005.esams.wmnet',
- 'cp3006.esams.wmnet',
- 'cp3007.esams.wmnet',
- 'cp3008.esams.wmnet',
- 'cp3009.esams.wmnet',
- 'cp3010.esams.wmnet',
- # T92306 'cp3011.esams.wmnet', # needs-jessie-install
- 'cp3012.esams.wmnet',
- 'cp3013.esams.wmnet',
- 'cp3014.esams.wmnet',
- 'cp3030.esams.wmnet',
- 'cp3031.esams.wmnet',
- 'cp3040.esams.wmnet',
- 'cp3041.esams.wmnet',
- ],
- 'ulsfo' => [
- 'cp4008.ulsfo.wmnet',
- 'cp4009.ulsfo.wmnet',
- 'cp4010.ulsfo.wmnet',
- 'cp4016.ulsfo.wmnet',
- 'cp4017.ulsfo.wmnet',
- 'cp4018.ulsfo.wmnet',
- ]
- },
- 'bits' => {
- 'eqiad' => [
- 'cp1056.eqiad.wmnet',
- 'cp1057.eqiad.wmnet',
- 'cp1069.eqiad.wmnet',
- 'cp1070.eqiad.wmnet',
- ],
- 'esams' => [
- 'cp3019.esams.wmnet',
- 'cp3020.esams.wmnet',
- 'cp3021.esams.wmnet',
- 'cp3022.esams.wmnet',
- ],
- 'ulsfo' => [
- 'cp4001.ulsfo.wmnet',
- 'cp4002.ulsfo.wmnet',
- 'cp4003.ulsfo.wmnet',
- 'cp4004.ulsfo.wmnet',
- ],
- },
- 'upload' => {
- 'eqiad' => [
- 'cp1048.eqiad.wmnet',
- 'cp1049.eqiad.wmnet',
- 'cp1050.eqiad.wmnet',
- 'cp1051.eqiad.wmnet',
- 'cp1061.eqiad.wmnet',
- 'cp1062.eqiad.wmnet',
- 'cp1063.eqiad.wmnet',
- 'cp1064.eqiad.wmnet',
- 'cp1071.eqiad.wmnet',
- 'cp1072.eqiad.wmnet',
- 'cp1073.eqiad.wmnet',
- 'cp1074.eqiad.wmnet',
- ],
- 'esams' => [
- 'cp3032.esams.wmnet',
- 'cp3033.esams.wmnet',
- 'cp3034.esams.wmnet',
- 'cp3035.esams.wmnet',
- 'cp3036.esams.wmnet',
- 'cp3037.esams.wmnet',
- 'cp3038.esams.wmnet',
- 'cp3039.esams.wmnet',
- 'cp3042.esams.wmnet',
- 'cp3043.esams.wmnet',
- 'cp3044.esams.wmnet',
- 'cp3045.esams.wmnet',
- 'cp3046.esams.wmnet',
- 'cp3047.esams.wmnet',
- 'cp3048.esams.wmnet',
- 'cp3049.esams.wmnet',
- ],
- 'ulsfo' => [
- 'cp4005.ulsfo.wmnet',
- 'cp4006.ulsfo.wmnet',
- 'cp4007.ulsfo.wmnet',
- 'cp4013.ulsfo.wmnet',
- 'cp4014.ulsfo.wmnet',
- 'cp4015.ulsfo.wmnet',
- ],
- },
- 'mobile' => {
- 'eqiad' => [
- 'cp1046.eqiad.wmnet',
- 'cp1047.eqiad.wmnet',
- 'cp1059.eqiad.wmnet',
- 'cp1060.eqiad.wmnet',
- ],
- 'esams' => [
- 'cp3015.esams.wmnet',
- 'cp3016.esams.wmnet',
- 'cp3017.esams.wmnet',
- 'cp3018.esams.wmnet',
- ],
- 'ulsfo' => [
- 'cp4011.ulsfo.wmnet',
- 'cp4012.ulsfo.wmnet',
- 'cp4019.ulsfo.wmnet',
- 'cp4020.ulsfo.wmnet',
- ]
- },
- 'parsoid' => {
- 'eqiad' => [
- 'cp1045.eqiad.wmnet',
- 'cp1058.eqiad.wmnet',
- ],
- 'esams' => [],
- 'ulsfo' => []
- },
- 'misc' => {
- 'eqiad' => [
- 'cp1043.eqiad.wmnet',
- 'cp1044.eqiad.wmnet',
- ],
- 'esams' => [],
- 'ulsfo' => [],
- },
- },
- 'labs' => {
- 'bits' => {
- 'eqiad' => '127.0.0.1',
- },
- 'mobile' => {
- 'eqiad' => '127.0.0.1',
- },
- 'text' => {
- 'eqiad' => '127.0.0.1',
- },
- 'upload' => {
- 'eqiad' => '127.0.0.1',
- },
- 'parsoid' => {
- 'eqiad' => '127.0.0.1',
- },
- },
- }
-
$backends = {
'production' => {
'appservers' =>
$lvs::configuration::lvs_service_ips['production']['apaches'],
diff --git a/modules/role/manifests/cache/mobile.pp
b/modules/role/manifests/cache/mobile.pp
index d76e5b6..2aff7ef 100644
--- a/modules/role/manifests/cache/mobile.pp
+++ b/modules/role/manifests/cache/mobile.pp
@@ -31,7 +31,7 @@
'test_wikipedia' =>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
},
2 => {
- 'eqiad' =>
$role::cache::configuration::active_nodes[$::realm]['mobile']['eqiad'],
+ 'eqiad' => $::cache::nodes['mobile']['eqiad'],
}
}
@@ -145,10 +145,10 @@
admin_port => 6082,
storage => "-s malloc,${memory_storage_size}G",
directors => {
- 'backend' =>
$::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site],
+ 'backend' => $::cache::nodes['mobile'][$::site],
},
director_options => {
- 'retries' => $backend_weight_avg *
size($::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site]),
+ 'retries' => $backend_weight_avg *
size($::cache::nodes['mobile'][$::site]),
},
director_type => 'chash',
vcl_config => {
diff --git a/modules/role/manifests/cache/parsoid.pp
b/modules/role/manifests/cache/parsoid.pp
index 2d05599..c1e59fd 100644
--- a/modules/role/manifests/cache/parsoid.pp
+++ b/modules/role/manifests/cache/parsoid.pp
@@ -83,14 +83,14 @@
port => 80,
admin_port => 6082,
directors => {
- 'backend' =>
$::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site],
+ 'backend' => $::cache::nodes['parsoid'][$::site],
'cxserver_backend' =>
$::role::cache::configuration::backends[$::realm]['cxserver'][$::site],
'citoid_backend' =>
$::role::cache::configuration::backends[$::realm]['citoid'][$::site],
'restbase_backend' =>
$::role::cache::configuration::backends[$::realm]['restbase'][$::site],
},
director_type => 'chash',
director_options => {
- 'retries' => $backend_weight_avg *
size($::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site]),
+ 'retries' => $backend_weight_avg *
size($::cache::nodes['parsoid'][$::site]),
},
vcl_config => {
'retry5xx' => 0,
diff --git a/modules/role/manifests/cache/text.pp
b/modules/role/manifests/cache/text.pp
index daaf670..af74c8e 100644
--- a/modules/role/manifests/cache/text.pp
+++ b/modules/role/manifests/cache/text.pp
@@ -29,7 +29,7 @@
'test_wikipedia' =>
$role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary],
},
2 => {
- 'eqiad' =>
$role::cache::configuration::active_nodes[$::realm]['text']['eqiad'],
+ 'eqiad' => $::cache::nodes['text']['eqiad'],
},
}
@@ -114,7 +114,7 @@
admin_port => 6082,
storage => "-s malloc,${memory_storage_size}G",
directors => {
- 'backend' =>
$role::cache::configuration::active_nodes[$::realm]['text'][$::site],
+ 'backend' => $::cache::nodes['text'][$::site],
},
director_type => 'chash',
vcl_config => {
diff --git a/modules/role/manifests/cache/upload.pp
b/modules/role/manifests/cache/upload.pp
index 3e22b0d..92b08d4 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -24,7 +24,7 @@
'rendering' =>
$role::cache::configuration::backends[$::realm]['rendering'][$::mw_primary],
},
2 => {
- 'eqiad' =>
$role::cache::configuration::active_nodes[$::realm]['upload']['eqiad']
+ 'eqiad' => $::cache::nodes['upload']['eqiad']
}
}
@@ -141,7 +141,7 @@
admin_port => 6082,
storage => "-s malloc,${memory_storage_size}G",
directors => {
- 'backend' =>
$role::cache::configuration::active_nodes[$::realm]['upload'][$::site],
+ 'backend' => $::cache::nodes['upload'][$::site],
},
director_type => 'chash',
vcl_config => {
diff --git a/templates/udp2log/filters.oxygen.erb
b/templates/udp2log/filters.oxygen.erb
index 7d06de6..1c9584e 100644
--- a/templates/udp2log/filters.oxygen.erb
+++ b/templates/udp2log/filters.oxygen.erb
@@ -18,10 +18,9 @@
pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> <%=
log_directory %>/edits.tsv.log
<%
-# pull in $role::cache::configuration::active_nodes
-# to find mobile host names and build a regex on which to grep.
-cache_configuration =
scope.lookupvar('::role::cache::configuration::active_nodes')
-mobile_hosts_regex = '(' +
cache_configuration['production']['mobile'].values.flatten.sort.join('|') + ')'
+# find mobile host names and build a regex on which to grep.
+cache_nodes_mobile = scope.lookupvar('::cache::nodes::mobile')
+mobile_hosts_regex = '(' + cache_nodes_mobile.values.flatten.sort.join('|') +
')'
-%>
### Mobile traffic filter
--
To view, visit https://gerrit.wikimedia.org/r/204068
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
