BBlack has submitted this change and it was merged. Change subject: r::c::config::active_nodes -> hiera cache::$cluster::nodes ......................................................................
r::c::config::active_nodes -> hiera cache::$cluster::nodes This breaks torrus module testing (still relies on local copy of $r::c::config::active_nodes), but given that torrus CDN stuff doesn't seem to work in practice anyways, and that the modules shouldn't have this data access directly either, I'm not too concerned to merge over it. The changes for role::ipsec and role::analytics::kafkatee pass human inspection of "looks right", but neither one is actually applied on any real host currently (for the data access in question), therefore can't be compiler-tested, but also wouldn't break any live node and can be sorted out if necc when one exists in the future. The rest all checks out in puppet-compiler and seems to work! Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8 --- A hieradata/common/cache/bits.yaml A hieradata/common/cache/misc.yaml A hieradata/common/cache/mobile.yaml A hieradata/common/cache/parsoid.yaml A hieradata/common/cache/text.yaml A hieradata/common/cache/upload.yaml M hieradata/labs.yaml M hieradata/role/common/cache/bits.yaml M hieradata/role/common/cache/mobile.yaml M hieradata/role/common/cache/text.yaml M hieradata/role/common/cache/upload.yaml M manifests/role/analytics/kafkatee.pp M manifests/role/ipsec.pp M modules/role/manifests/cache/configuration.pp M modules/role/manifests/cache/mobile.pp M modules/role/manifests/cache/parsoid.pp M modules/role/manifests/cache/text.pp M modules/role/manifests/cache/upload.pp M modules/torrus/manifests/xml_generation/cdn.pp M modules/torrus/templates/cdn-aggregates.xml.erb M modules/torrus/templates/varnish.xml.erb M templates/udp2log/filters.oxygen.erb 22 files changed, 172 insertions(+), 314 deletions(-) Approvals: Giuseppe Lavagetto: Looks good to me, but someone else must approve BBlack: Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/common/cache/bits.yaml b/hieradata/common/cache/bits.yaml new file mode 100644 index 0000000..3b6dcb0 --- /dev/null +++ b/hieradata/common/cache/bits.yaml @@ -0,0 +1,17 @@ +nodes: + codfw: [] + eqiad: + - 'cp1056.eqiad.wmnet' + - 'cp1057.eqiad.wmnet' + - 'cp1069.eqiad.wmnet' + - 'cp1070.eqiad.wmnet' + esams: + - 'cp3019.esams.wmnet' + - 'cp3020.esams.wmnet' + - 'cp3021.esams.wmnet' + - 'cp3022.esams.wmnet' + ulsfo: + - 'cp4001.ulsfo.wmnet' + - 'cp4002.ulsfo.wmnet' + - 'cp4003.ulsfo.wmnet' + - 'cp4004.ulsfo.wmnet' diff --git a/hieradata/common/cache/misc.yaml b/hieradata/common/cache/misc.yaml new file mode 100644 index 0000000..deafd12 --- /dev/null +++ b/hieradata/common/cache/misc.yaml @@ -0,0 +1,4 @@ +nodes: + eqiad: + - 'cp1043.eqiad.wmnet' + - 'cp1044.eqiad.wmnet' diff --git a/hieradata/common/cache/mobile.yaml b/hieradata/common/cache/mobile.yaml new file mode 100644 index 0000000..1d1bd8a --- /dev/null +++ b/hieradata/common/cache/mobile.yaml @@ -0,0 +1,17 @@ +nodes: + codfw: [] + eqiad: + - 'cp1046.eqiad.wmnet' + - 'cp1047.eqiad.wmnet' + - 'cp1059.eqiad.wmnet' + - 'cp1060.eqiad.wmnet' + esams: + - 'cp3015.esams.wmnet' + - 'cp3016.esams.wmnet' + - 'cp3017.esams.wmnet' + - 'cp3018.esams.wmnet' + ulsfo: + - 'cp4011.ulsfo.wmnet' + - 'cp4012.ulsfo.wmnet' + - 'cp4019.ulsfo.wmnet' + - 'cp4020.ulsfo.wmnet' diff --git a/hieradata/common/cache/parsoid.yaml b/hieradata/common/cache/parsoid.yaml new file mode 100644 index 0000000..75c40f4 --- /dev/null +++ b/hieradata/common/cache/parsoid.yaml @@ -0,0 +1,4 @@ +nodes: + eqiad: + - 'cp1045.eqiad.wmnet' + - 'cp1058.eqiad.wmnet' diff --git a/hieradata/common/cache/text.yaml b/hieradata/common/cache/text.yaml new file mode 100644 index 0000000..8ba5d51 --- /dev/null +++ b/hieradata/common/cache/text.yaml @@ -0,0 +1,35 @@ +nodes: + codfw: [] + eqiad: + - 'cp1052.eqiad.wmnet' + - 'cp1053.eqiad.wmnet' + - 'cp1054.eqiad.wmnet' + - 'cp1055.eqiad.wmnet' + - 'cp1065.eqiad.wmnet' + - 'cp1066.eqiad.wmnet' + - 'cp1067.eqiad.wmnet' + - 'cp1068.eqiad.wmnet' + esams: + - 'cp3003.esams.wmnet' + - 'cp3004.esams.wmnet' + - 'cp3005.esams.wmnet' + - 'cp3006.esams.wmnet' + - 'cp3007.esams.wmnet' + - 'cp3008.esams.wmnet' + - 'cp3009.esams.wmnet' + - 'cp3010.esams.wmnet' + # - 'cp3011.esams.wmnet' # T92306 + - 'cp3012.esams.wmnet' + - 'cp3013.esams.wmnet' + - 'cp3014.esams.wmnet' + - 'cp3030.esams.wmnet' + - 'cp3031.esams.wmnet' + - 'cp3040.esams.wmnet' + - 'cp3041.esams.wmnet' + ulsfo: + - 'cp4008.ulsfo.wmnet' + - 'cp4009.ulsfo.wmnet' + - 'cp4010.ulsfo.wmnet' + - 'cp4016.ulsfo.wmnet' + - 'cp4017.ulsfo.wmnet' + - 'cp4018.ulsfo.wmnet' diff --git a/hieradata/common/cache/upload.yaml b/hieradata/common/cache/upload.yaml new file mode 100644 index 0000000..81443e8 --- /dev/null +++ b/hieradata/common/cache/upload.yaml @@ -0,0 +1,39 @@ +nodes: + codfw: [] + eqiad: + - 'cp1048.eqiad.wmnet' + - 'cp1049.eqiad.wmnet' + - 'cp1050.eqiad.wmnet' + - 'cp1051.eqiad.wmnet' + - 'cp1061.eqiad.wmnet' + - 'cp1062.eqiad.wmnet' + - 'cp1063.eqiad.wmnet' + - 'cp1064.eqiad.wmnet' + - 'cp1071.eqiad.wmnet' + - 'cp1072.eqiad.wmnet' + - 'cp1073.eqiad.wmnet' + - 'cp1074.eqiad.wmnet' + esams: + - 'cp3032.esams.wmnet' + - 'cp3033.esams.wmnet' + - 'cp3034.esams.wmnet' + - 'cp3035.esams.wmnet' + - 'cp3036.esams.wmnet' + - 'cp3037.esams.wmnet' + - 'cp3038.esams.wmnet' + - 'cp3039.esams.wmnet' + - 'cp3042.esams.wmnet' + - 'cp3043.esams.wmnet' + - 'cp3044.esams.wmnet' + - 'cp3045.esams.wmnet' + - 'cp3046.esams.wmnet' + - 'cp3047.esams.wmnet' + - 'cp3048.esams.wmnet' + - 'cp3049.esams.wmnet' + ulsfo: + - 'cp4005.ulsfo.wmnet' + - 'cp4006.ulsfo.wmnet' + - 'cp4007.ulsfo.wmnet' + - 'cp4013.ulsfo.wmnet' + - 'cp4014.ulsfo.wmnet' + - 'cp4015.ulsfo.wmnet' diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml index 88dc142..a0157d3 100644 --- a/hieradata/labs.yaml +++ b/hieradata/labs.yaml @@ -9,3 +9,18 @@ archiva::proxy::ssl_enabled: false archiva::proxy::certificate_name: ssl-cert-snakeoil statsite::instance::graphite_host: 'labmon1001.eqiad.wmnet' +cache::text::nodes: + eqiad: + - '127.0.0.1' +cache::upload::nodes: + eqiad: + - '127.0.0.1' +cache::mobile::nodes: + eqiad: + - '127.0.0.1' +cache::bits::nodes: + eqiad: + - '127.0.0.1' +cache::parsoid::nodes: + eqiad: + - '127.0.0.1' diff --git a/hieradata/role/common/cache/bits.yaml b/hieradata/role/common/cache/bits.yaml index 4dab273..4de989d 100644 --- a/hieradata/role/common/cache/bits.yaml +++ b/hieradata/role/common/cache/bits.yaml @@ -1,18 +1 @@ cluster: cache_bits -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: - - 'cp1056.eqiad.wmnet' - - 'cp1057.eqiad.wmnet' - - 'cp1069.eqiad.wmnet' - - 'cp1070.eqiad.wmnet' -hosts_esams: - - 'cp3019.esams.wmnet' - - 'cp3020.esams.wmnet' - - 'cp3021.esams.wmnet' - - 'cp3022.esams.wmnet' -hosts_ulsfo: - - 'cp4001.ulsfo.wmnet' - - 'cp4002.ulsfo.wmnet' - - 'cp4003.ulsfo.wmnet' - - 'cp4004.ulsfo.wmnet' diff --git a/hieradata/role/common/cache/mobile.yaml b/hieradata/role/common/cache/mobile.yaml index e8197a7..d74bad6 100644 --- a/hieradata/role/common/cache/mobile.yaml +++ b/hieradata/role/common/cache/mobile.yaml @@ -1,19 +1 @@ cluster: cache_mobile -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: - - 'cp1046.eqiad.wmnet' - - 'cp1047.eqiad.wmnet' - - 'cp1059.eqiad.wmnet' - - 'cp1060.eqiad.wmnet' -hosts_esams: - - 'cp3015.esams.wmnet' - - 'cp3016.esams.wmnet' - - 'cp3017.esams.wmnet' - - 'cp3018.esams.wmnet' -hosts_ulsfo: - - 'cp4011.ulsfo.wmnet' - - 'cp4012.ulsfo.wmnet' - - 'cp4019.ulsfo.wmnet' - - 'cp4020.ulsfo.wmnet' - diff --git a/hieradata/role/common/cache/text.yaml b/hieradata/role/common/cache/text.yaml index d94a3f4..a526129 100644 --- a/hieradata/role/common/cache/text.yaml +++ b/hieradata/role/common/cache/text.yaml @@ -1,36 +1 @@ cluster: cache_text -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: - - 'cp1052.eqiad.wmnet' - - 'cp1053.eqiad.wmnet' - - 'cp1054.eqiad.wmnet' - - 'cp1055.eqiad.wmnet' - - 'cp1065.eqiad.wmnet' - - 'cp1066.eqiad.wmnet' - - 'cp1067.eqiad.wmnet' - - 'cp1068.eqiad.wmnet' -hosts_esams: - - 'cp3003.esams.wmnet' - - 'cp3004.esams.wmnet' - - 'cp3005.esams.wmnet' - - 'cp3006.esams.wmnet' - - 'cp3007.esams.wmnet' - - 'cp3008.esams.wmnet' - - 'cp3009.esams.wmnet' - - 'cp3010.esams.wmnet' - # - 'cp3011.esams.wmnet' # T92306 - - 'cp3012.esams.wmnet' - - 'cp3013.esams.wmnet' - - 'cp3014.esams.wmnet' - - 'cp3030.esams.wmnet' - - 'cp3031.esams.wmnet' - - 'cp3040.esams.wmnet' - - 'cp3041.esams.wmnet' -hosts_ulsfo: - - 'cp4008.ulsfo.wmnet' - - 'cp4009.ulsfo.wmnet' - - 'cp4010.ulsfo.wmnet' - - 'cp4016.ulsfo.wmnet' - - 'cp4017.ulsfo.wmnet' - - 'cp4018.ulsfo.wmnet' diff --git a/hieradata/role/common/cache/upload.yaml b/hieradata/role/common/cache/upload.yaml index 1cf9c19..b79cf10 100644 --- a/hieradata/role/common/cache/upload.yaml +++ b/hieradata/role/common/cache/upload.yaml @@ -1,40 +1 @@ cluster: cache_upload -# data synced from manifests/role/cache.pp on 2015-04-08 -hosts_codfw: [] -hosts_eqiad: - - 'cp1048.eqiad.wmnet' - - 'cp1049.eqiad.wmnet' - - 'cp1050.eqiad.wmnet' - - 'cp1051.eqiad.wmnet' - - 'cp1061.eqiad.wmnet' - - 'cp1062.eqiad.wmnet' - - 'cp1063.eqiad.wmnet' - - 'cp1064.eqiad.wmnet' - - 'cp1071.eqiad.wmnet' - - 'cp1072.eqiad.wmnet' - - 'cp1073.eqiad.wmnet' - - 'cp1074.eqiad.wmnet' -hosts_esams: - - 'cp3032.esams.wmnet' - - 'cp3033.esams.wmnet' - - 'cp3034.esams.wmnet' - - 'cp3035.esams.wmnet' - - 'cp3036.esams.wmnet' - - 'cp3037.esams.wmnet' - - 'cp3038.esams.wmnet' - - 'cp3039.esams.wmnet' - - 'cp3042.esams.wmnet' - - 'cp3043.esams.wmnet' - - 'cp3044.esams.wmnet' - - 'cp3045.esams.wmnet' - - 'cp3046.esams.wmnet' - - 'cp3047.esams.wmnet' - - 'cp3048.esams.wmnet' - - 'cp3049.esams.wmnet' -hosts_ulsfo: - - 'cp4005.ulsfo.wmnet' - - 'cp4006.ulsfo.wmnet' - - 'cp4007.ulsfo.wmnet' - - 'cp4013.ulsfo.wmnet' - - 'cp4014.ulsfo.wmnet' - - 'cp4015.ulsfo.wmnet' diff --git a/manifests/role/analytics/kafkatee.pp b/manifests/role/analytics/kafkatee.pp index 0db257e..e267ab2 100644 --- a/manifests/role/analytics/kafkatee.pp +++ b/manifests/role/analytics/kafkatee.pp @@ -78,12 +78,12 @@ # Include this to infer mobile varnish frontend hostnames on which to filter. include role::cache::configuration - $cache_configuration = $role::cache::configuration::active_nodes['production']['mobile'] - $mobile_hosts_regex = inline_template('(<%= @cache_configuration.values.flatten.sort.join(\'|\') %>)') + $mobile_nodes = hiera('::cache::mobile::nodes') + $mobile_nodes_regex = inline_template('(<%= @mobile_nodes.values.flatten.sort.join(\'|\') %>)') # 1/100 sampling of traffic from mobile varnishes ::kafkatee::output { 'mobile-sampled-100': - destination => "/bin/grep -P '${mobile_hosts_regex}' >> ${webrequest_log_directory}/mobile-sampled-100.tsv.log", + destination => "/bin/grep -P '${mobile_nodes_regex}' >> ${webrequest_log_directory}/mobile-sampled-100.tsv.log", sample => 100, type => 'pipe', } diff --git a/manifests/role/ipsec.pp b/manifests/role/ipsec.pp index 408bcca..e03b1a8 100644 --- a/manifests/role/ipsec.pp +++ b/manifests/role/ipsec.pp @@ -12,18 +12,20 @@ if $hosts != undef { $targets = $hosts } else { + $cache_cluster = regsubst(hiera('cluster'), '/_/', '::') + $cluster_nodes = hiera("${cache_cluster}::nodes") # for 'left' nodes in cache sites, enumerate 'right' nodes in "main" sites if $::site == 'esams' or $::site == 'ulsfo' { $targets = concat( - hiera('hosts_eqiad', []), - hiera('hosts_codfw', []) + $cluster_nodes['eqiad'], + $cluster_nodes['codfw'] ) } # for 'left' nodes in "main" sites, enumerate 'right' nodes in cache sites if $::site == 'eqiad' or $::site == 'codfw' { $targets = concat( - hiera('hosts_esams', []), - hiera('hosts_ulsfo', []) + $cluster_nodes['esams'], + $cluster_nodes['ulsfo'] ) } } diff --git a/modules/role/manifests/cache/configuration.pp b/modules/role/manifests/cache/configuration.pp index b686b4d..6cdae9b 100644 --- a/modules/role/manifests/cache/configuration.pp +++ b/modules/role/manifests/cache/configuration.pp @@ -3,164 +3,6 @@ $has_ganglia = hiera('has_ganglia', true) - $active_nodes = { - 'production' => { - 'text' => { - 'eqiad' => [ - 'cp1052.eqiad.wmnet', - 'cp1053.eqiad.wmnet', - 'cp1054.eqiad.wmnet', - 'cp1055.eqiad.wmnet', - 'cp1065.eqiad.wmnet', - 'cp1066.eqiad.wmnet', - 'cp1067.eqiad.wmnet', - 'cp1068.eqiad.wmnet', - ], - 'esams' => [ - 'cp3003.esams.wmnet', - 'cp3004.esams.wmnet', - 'cp3005.esams.wmnet', - 'cp3006.esams.wmnet', - 'cp3007.esams.wmnet', - 'cp3008.esams.wmnet', - 'cp3009.esams.wmnet', - 'cp3010.esams.wmnet', - # T92306 'cp3011.esams.wmnet', # needs-jessie-install - 'cp3012.esams.wmnet', - 'cp3013.esams.wmnet', - 'cp3014.esams.wmnet', - 'cp3030.esams.wmnet', - 'cp3031.esams.wmnet', - 'cp3040.esams.wmnet', - 'cp3041.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4008.ulsfo.wmnet', - 'cp4009.ulsfo.wmnet', - 'cp4010.ulsfo.wmnet', - 'cp4016.ulsfo.wmnet', - 'cp4017.ulsfo.wmnet', - 'cp4018.ulsfo.wmnet', - ] - }, - 'bits' => { - 'eqiad' => [ - 'cp1056.eqiad.wmnet', - 'cp1057.eqiad.wmnet', - 'cp1069.eqiad.wmnet', - 'cp1070.eqiad.wmnet', - ], - 'esams' => [ - 'cp3019.esams.wmnet', - 'cp3020.esams.wmnet', - 'cp3021.esams.wmnet', - 'cp3022.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4001.ulsfo.wmnet', - 'cp4002.ulsfo.wmnet', - 'cp4003.ulsfo.wmnet', - 'cp4004.ulsfo.wmnet', - ], - }, - 'upload' => { - 'eqiad' => [ - 'cp1048.eqiad.wmnet', - 'cp1049.eqiad.wmnet', - 'cp1050.eqiad.wmnet', - 'cp1051.eqiad.wmnet', - 'cp1061.eqiad.wmnet', - 'cp1062.eqiad.wmnet', - 'cp1063.eqiad.wmnet', - 'cp1064.eqiad.wmnet', - 'cp1071.eqiad.wmnet', - 'cp1072.eqiad.wmnet', - 'cp1073.eqiad.wmnet', - 'cp1074.eqiad.wmnet', - ], - 'esams' => [ - 'cp3032.esams.wmnet', - 'cp3033.esams.wmnet', - 'cp3034.esams.wmnet', - 'cp3035.esams.wmnet', - 'cp3036.esams.wmnet', - 'cp3037.esams.wmnet', - 'cp3038.esams.wmnet', - 'cp3039.esams.wmnet', - 'cp3042.esams.wmnet', - 'cp3043.esams.wmnet', - 'cp3044.esams.wmnet', - 'cp3045.esams.wmnet', - 'cp3046.esams.wmnet', - 'cp3047.esams.wmnet', - 'cp3048.esams.wmnet', - 'cp3049.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4005.ulsfo.wmnet', - 'cp4006.ulsfo.wmnet', - 'cp4007.ulsfo.wmnet', - 'cp4013.ulsfo.wmnet', - 'cp4014.ulsfo.wmnet', - 'cp4015.ulsfo.wmnet', - ], - }, - 'mobile' => { - 'eqiad' => [ - 'cp1046.eqiad.wmnet', - 'cp1047.eqiad.wmnet', - 'cp1059.eqiad.wmnet', - 'cp1060.eqiad.wmnet', - ], - 'esams' => [ - 'cp3015.esams.wmnet', - 'cp3016.esams.wmnet', - 'cp3017.esams.wmnet', - 'cp3018.esams.wmnet', - ], - 'ulsfo' => [ - 'cp4011.ulsfo.wmnet', - 'cp4012.ulsfo.wmnet', - 'cp4019.ulsfo.wmnet', - 'cp4020.ulsfo.wmnet', - ] - }, - 'parsoid' => { - 'eqiad' => [ - 'cp1045.eqiad.wmnet', - 'cp1058.eqiad.wmnet', - ], - 'esams' => [], - 'ulsfo' => [] - }, - 'misc' => { - 'eqiad' => [ - 'cp1043.eqiad.wmnet', - 'cp1044.eqiad.wmnet', - ], - 'esams' => [], - 'ulsfo' => [], - }, - }, - 'labs' => { - 'bits' => { - 'eqiad' => '127.0.0.1', - }, - 'mobile' => { - 'eqiad' => '127.0.0.1', - }, - 'text' => { - 'eqiad' => '127.0.0.1', - }, - 'upload' => { - 'eqiad' => '127.0.0.1', - }, - 'parsoid' => { - 'eqiad' => '127.0.0.1', - }, - }, - } - $backends = { 'production' => { 'appservers' => $lvs::configuration::lvs_service_ips['production']['apaches'], diff --git a/modules/role/manifests/cache/mobile.pp b/modules/role/manifests/cache/mobile.pp index d76e5b6..2c4e8fc 100644 --- a/modules/role/manifests/cache/mobile.pp +++ b/modules/role/manifests/cache/mobile.pp @@ -1,4 +1,8 @@ class role::cache::mobile inherits role::cache::2layer { + + $mobile_nodes = hiera('::cache::mobile::nodes') + $site_mobile_nodes = $mobile_nodes[$::site] + if $::realm == 'production' { $memory_storage_size = floor((0.125 * $::memorysize_mb / 1024.0) + 0.5) # 1/8 of total mem } @@ -31,7 +35,7 @@ 'test_wikipedia' => $role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['mobile']['eqiad'], + 'eqiad' => $mobile_nodes['eqiad'], } } @@ -145,10 +149,10 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site], + 'backend' => $site_mobile_nodes, }, director_options => { - 'retries' => $backend_weight_avg * size($::role::cache::configuration::active_nodes[$::realm]['mobile'][$::site]), + 'retries' => $backend_weight_avg * size($site_mobile_nodes), }, director_type => 'chash', vcl_config => { diff --git a/modules/role/manifests/cache/parsoid.pp b/modules/role/manifests/cache/parsoid.pp index 2d05599..6c52794 100644 --- a/modules/role/manifests/cache/parsoid.pp +++ b/modules/role/manifests/cache/parsoid.pp @@ -1,5 +1,8 @@ class role::cache::parsoid inherits role::cache::2layer { + $parsoid_nodes = hiera('::cache::parsoid::nodes') + $site_parsoid_nodes = $parsoid_nodes[$::site] + if ( $::realm == 'production' ) { include role::cache::ssl::parsoid class { 'lvs::realserver': @@ -83,14 +86,14 @@ port => 80, admin_port => 6082, directors => { - 'backend' => $::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site], + 'backend' => $site_parsoid_nodes, 'cxserver_backend' => $::role::cache::configuration::backends[$::realm]['cxserver'][$::site], 'citoid_backend' => $::role::cache::configuration::backends[$::realm]['citoid'][$::site], 'restbase_backend' => $::role::cache::configuration::backends[$::realm]['restbase'][$::site], }, director_type => 'chash', director_options => { - 'retries' => $backend_weight_avg * size($::role::cache::configuration::active_nodes[$::realm]['parsoid'][$::site]), + 'retries' => $backend_weight_avg * size($site_parsoid_nodes), }, vcl_config => { 'retry5xx' => 0, diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index daaf670..dbe22dc 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -1,4 +1,8 @@ class role::cache::text inherits role::cache::2layer { + + $text_nodes = hiera('::cache::text::nodes') + $site_text_nodes = $text_nodes[$::site] + if $::realm == 'production' { $memory_storage_size = floor((0.125 * $::memorysize_mb / 1024.0) + 0.5) # 1/8 of total mem } @@ -29,7 +33,7 @@ 'test_wikipedia' => $role::cache::configuration::backends[$::realm]['test_appservers'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['text']['eqiad'], + 'eqiad' => $text_nodes['eqiad'], }, } @@ -114,7 +118,7 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $role::cache::configuration::active_nodes[$::realm]['text'][$::site], + 'backend' => $site_text_nodes, }, director_type => 'chash', vcl_config => { diff --git a/modules/role/manifests/cache/upload.pp b/modules/role/manifests/cache/upload.pp index 687f809..9611a7b 100644 --- a/modules/role/manifests/cache/upload.pp +++ b/modules/role/manifests/cache/upload.pp @@ -1,4 +1,8 @@ class role::cache::upload inherits role::cache::2layer { + + $upload_nodes = hiera('::cache::upload::nodes') + $site_upload_nodes = $upload_nodes[$::site] + if $::realm == 'production' { $memory_storage_size = floor((0.083 * $::memorysize_mb / 1024.0) + 0.5) # 1/12 of total mem } @@ -24,7 +28,7 @@ 'rendering' => $role::cache::configuration::backends[$::realm]['rendering'][$::mw_primary], }, 2 => { - 'eqiad' => $role::cache::configuration::active_nodes[$::realm]['upload']['eqiad'] + 'eqiad' => $upload_nodes['eqiad'], } } @@ -141,7 +145,7 @@ admin_port => 6082, storage => "-s malloc,${memory_storage_size}G", directors => { - 'backend' => $role::cache::configuration::active_nodes[$::realm]['upload'][$::site], + 'backend' => $site_upload_nodes, }, director_type => 'chash', vcl_config => { diff --git a/modules/torrus/manifests/xml_generation/cdn.pp b/modules/torrus/manifests/xml_generation/cdn.pp index 36d02e3..3f8f4fa 100644 --- a/modules/torrus/manifests/xml_generation/cdn.pp +++ b/modules/torrus/manifests/xml_generation/cdn.pp @@ -5,8 +5,6 @@ # # Uses role/cache/cache.pp class torrus::xml_generation::cdn { - require role::cache::configuration - File { owner => 'root', group => 'root', diff --git a/modules/torrus/templates/cdn-aggregates.xml.erb b/modules/torrus/templates/cdn-aggregates.xml.erb index e17d40d..4662ab7 100644 --- a/modules/torrus/templates/cdn-aggregates.xml.erb +++ b/modules/torrus/templates/cdn-aggregates.xml.erb @@ -2,8 +2,8 @@ <% def node_list(role, site) - realm = @realm - return scope.lookupvar("role::cache::configuration::active_nodes")[realm][role][site] + role_nodes = scope.function_hiera(["::cache::#{role}::nodes"]) + return role_nodes[site] end def varnish_rpn(role, site, dual=false, list=nil) diff --git a/modules/torrus/templates/varnish.xml.erb b/modules/torrus/templates/varnish.xml.erb index 1af402b..5609e61 100644 --- a/modules/torrus/templates/varnish.xml.erb +++ b/modules/torrus/templates/varnish.xml.erb @@ -2,8 +2,8 @@ <% def node_lists(role, site) - realm = @realm - return scope.lookupvar("role::cache::configuration::active_nodes")[realm][role][site] + role_nodes = scope.function_hiera(["::cache::#{role}::nodes"]) + return role_nodes[site] end def varnish_nodes(role, site, template="varnish-server") @@ -174,26 +174,6 @@ <param name="site" value="esams"/> <% varnish_nodes('mobile', 'esams', 'dual-layer-varnish-server').each do |line| -%> - <%= line %> -<% end -%> - </subtree> - - <subtree name="parsoid"> - <param name="system-id" value="%hostname%"/> - <param name="varnishcluster" value="Parsoid caches esams"/> - <param name="site" value="esams"/> - -<% varnish_nodes('parsoid', 'esams', 'dual-layer-varnish-server').each do |line| -%> - <%= line %> -<% end -%> - </subtree> - - <subtree name="misc"> - <param name="system-id" value="%hostname%"/> - <param name="varnishcluster" value="Misc caches esams"/> - <param name="site" value="esams"/> - -<% varnish_nodes('misc', 'esams').each do |line| -%> <%= line %> <% end -%> </subtree> diff --git a/templates/udp2log/filters.oxygen.erb b/templates/udp2log/filters.oxygen.erb index 7d06de6..f5c34f6 100644 --- a/templates/udp2log/filters.oxygen.erb +++ b/templates/udp2log/filters.oxygen.erb @@ -18,10 +18,9 @@ pipe 1 /usr/bin/udp-filter -F '\t' -p action=submit,action=edit >> <%= log_directory %>/edits.tsv.log <% -# pull in $role::cache::configuration::active_nodes -# to find mobile host names and build a regex on which to grep. -cache_configuration = scope.lookupvar('::role::cache::configuration::active_nodes') -mobile_hosts_regex = '(' + cache_configuration['production']['mobile'].values.flatten.sort.join('|') + ')' +# find mobile host names and build a regex on which to grep. +mobile_nodes = scope.function_hiera(['::cache::mobile::nodes']) +mobile_nodes_regex = '(' + mobile_nodes.values.flatten.sort.join('|') + ')' -%> ### Mobile traffic filter @@ -29,7 +28,7 @@ # # This filter has been migrated to Hive. It's output is available on # stat1002 at /a/log/webrequest/archive/mobile -pipe 100 /bin/grep -P '<%= mobile_hosts_regex %>' >> <%= log_directory %>/mobile-sampled-100.tsv.log +pipe 100 /bin/grep -P '<%= mobile_nodes_regex %>' >> <%= log_directory %>/mobile-sampled-100.tsv.log ### All 5xx error responses -- domas (now using udp-filter instead of 5xx-filter). # pipe 1 <%= @template_variables['webrequest_filter_directory'] %>/5xx-filter | awk -W interactive '$9 !~ "upload.wikimedia.org"' >> <%= log_directory %>/5xx.tsv.log -- To view, visit https://gerrit.wikimedia.org/r/204068 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I06f3fecf610524e14fd9256c9477931a735419b8 Gerrit-PatchSet: 17 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits