Giuseppe Lavagetto has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/382683 )
Change subject: role::cache::base: convert to profile [1/2] ...................................................................... role::cache::base: convert to profile [1/2] Change-Id: I28093780badaa38713847028cd5b5569dc81d6ca --- M hieradata/labs.yaml M hieradata/role/common/cache/canary.yaml M hieradata/role/common/cache/misc.yaml M hieradata/role/common/cache/text.yaml M hieradata/role/common/cache/upload.yaml R modules/profile/manifests/cache/base.pp M modules/role/manifests/cache/misc.pp M modules/role/manifests/cache/text.pp M modules/role/manifests/cache/upload.pp M modules/varnish/manifests/zero_update.pp 10 files changed, 70 insertions(+), 34 deletions(-) Approvals: Giuseppe Lavagetto: Looks good to me, approved Ema: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml index fed4f4a..7d20b1e 100644 --- a/hieradata/labs.yaml +++ b/hieradata/labs.yaml @@ -57,15 +57,15 @@ cache::upload::nodes: eqiad: - 'deployment-cache-upload04.deployment-prep.eqiad.wmflabs' -role::cache::base::zero_site: 'https://zero.wikimedia.beta.wmflabs.org' -role::cache::base::purge_host_only_upload_re: '^(upload|maps)\.beta\.wmflabs\.org$' -role::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.beta\.wmflabs\.org)' +profile::cache::base::zero_site: 'https://zero.wikimedia.beta.wmflabs.org' +profile::cache::base::purge_host_only_upload_re: '^(upload|maps)\.beta\.wmflabs\.org$' +profile::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.beta\.wmflabs\.org)' role::cache::upload::upload_domain: 'upload.beta.wmflabs.org' role::cache::upload::maps_domain: 'maps.beta.wmflabs.org' role::cache::text::static_host: 'deployment.wikimedia.beta.wmflabs.org' role::cache::text::shortener_domain: 'w-beta.wmflabs.org' role::cache::text::top_domain: 'beta.wmflabs.org' -role::cache::base::storage_parts: +profile::cache::base::storage_parts: - vdb - vdb varnish::dynamic_backend_caches: false diff --git a/hieradata/role/common/cache/canary.yaml b/hieradata/role/common/cache/canary.yaml index 264688c..eb512ed 100644 --- a/hieradata/role/common/cache/canary.yaml +++ b/hieradata/role/common/cache/canary.yaml @@ -73,3 +73,8 @@ debug_director: 'appservers_debug' cache::fe_transient_gb: 5 cache::be_transient_gb: 2 +# Profile::cache::base +profile::cache::base::zero_site: 'https://zero.wikimedia.org' +profile::cache::base::purge_host_only_upload_re: '^(upload|maps)\.wikimedia\.org$' +profile::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.wikimedia\.org)' +profile::cache::base::storage_parts: ['sda3', 'sdb3'] diff --git a/hieradata/role/common/cache/misc.yaml b/hieradata/role/common/cache/misc.yaml index 00bf122..c63f215 100644 --- a/hieradata/role/common/cache/misc.yaml +++ b/hieradata/role/common/cache/misc.yaml @@ -272,3 +272,8 @@ director: 'bromine' yarn.wikimedia.org: director: 'thorium' +# Profile::cache::base +profile::cache::base::zero_site: 'https://zero.wikimedia.org' +profile::cache::base::purge_host_only_upload_re: '^(upload|maps)\.wikimedia\.org$' +profile::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.wikimedia\.org)' +profile::cache::base::storage_parts: ['sda3', 'sdb3'] diff --git a/hieradata/role/common/cache/text.yaml b/hieradata/role/common/cache/text.yaml index 6187ec5..fa794a3 100644 --- a/hieradata/role/common/cache/text.yaml +++ b/hieradata/role/common/cache/text.yaml @@ -78,3 +78,8 @@ debug_director: 'appservers_debug' cache::fe_transient_gb: 5 cache::be_transient_gb: 2 +# Profile::cache::base +profile::cache::base::zero_site: 'https://zero.wikimedia.org' +profile::cache::base::purge_host_only_upload_re: '^(upload|maps)\.wikimedia\.org$' +profile::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.wikimedia\.org)' +profile::cache::base::storage_parts: ['sda3', 'sdb3'] diff --git a/hieradata/role/common/cache/upload.yaml b/hieradata/role/common/cache/upload.yaml index 1387340..f560edd 100644 --- a/hieradata/role/common/cache/upload.yaml +++ b/hieradata/role/common/cache/upload.yaml @@ -49,3 +49,9 @@ director: 'swift_thumbs' maps.wikimedia.org: director: 'kartotherian' +# Profile parameters +# Profile::cache::base +profile::cache::base::zero_site: 'https://zero.wikimedia.org' +profile::cache::base::purge_host_only_upload_re: '^(upload|maps)\.wikimedia\.org$' +profile::cache::base::purge_host_not_upload_re: '^(?!(upload|maps)\.wikimedia\.org)' +profile::cache::base::storage_parts: ['sda3', 'sdb3'] diff --git a/modules/role/manifests/cache/base.pp b/modules/profile/manifests/cache/base.pp similarity index 85% rename from modules/role/manifests/cache/base.pp rename to modules/profile/manifests/cache/base.pp index d9fe0c9..4329056 100644 --- a/modules/role/manifests/cache/base.pp +++ b/modules/profile/manifests/cache/base.pp @@ -1,24 +1,30 @@ -class role::cache::base( - $zero_site = 'https://zero.wikimedia.org', - $purge_host_only_upload_re = '^(upload|maps)\.wikimedia\.org$', - $purge_host_not_upload_re = '^(?!(upload|maps)\.wikimedia\.org)', - $storage_parts = ['sda3', 'sdb3'], +# === Class profile::cache::base +# +# Sets up some common things for cache instances: +# - LVS/conftool +# - monitoring +# - logging/analytics +# - storage +# +class profile::cache::base( + $cache_cluster = hiera('cache::cluster'), + $statsd_host = hiera('statsd'), + $zero_site = hiera('profile::cache::base::zero_site'), + $purge_host_only_upload_re = hiera('profile::cache::base::purge_host_only_upload_re'), + $purge_host_not_upload_re = hiera('profile::cache::base::purge_host_not_upload_re'), + $storage_parts = hiera('profile::cache::base::purge_host_not_upload_re'), ) { - include ::standard + # Needed profiles require ::profile::conftool::client + require ::profile::cache::kafka::webrequest + include ::standard + + # Other includes - to fix include ::nrpe include lvs::configuration include network::constants include conftool::scripts include ::role::prometheus::varnish_exporter - - - $cache_cluster = hiera('cache::cluster') - $statsd_host = hiera('statsd') - - system::role { "cache::${cache_cluster}": - description => "${cache_cluster} Varnish cache server", - } # Only production needs system perf tweaks if $::realm == 'production' { @@ -51,10 +57,6 @@ ########################################################################### # Analytics/Logging stuff ########################################################################### - # Install a varnishkafka producer to send - # varnish webrequest logs to Kafka. - include ::profile::cache::kafka::webrequest - class { '::varnish::logging': cache_cluster => $cache_cluster, statsd_host => $statsd_host, diff --git a/modules/role/manifests/cache/misc.pp b/modules/role/manifests/cache/misc.pp index 05d3ac1..4f3ec97 100644 --- a/modules/role/manifests/cache/misc.pp +++ b/modules/role/manifests/cache/misc.pp @@ -1,5 +1,9 @@ class role::cache::misc { - include role::cache::base + system::role { 'cache::misc': + description => 'misc Varnish cache server', + } + + include profile::cache::base include role::cache::ssl::unified class { 'tlsproxy::prometheus': } @@ -31,7 +35,7 @@ $common_vcl_config = { 'allowed_methods' => '^(GET|DELETE|HEAD|PATCH|POST|PURGE|PUT|OPTIONS)$', - 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re, + 'purge_host_regex' => $::profile::cache::base::purge_host_not_upload_re, 'pass_random' => true, 'req_handling' => hiera('cache::req_handling'), } @@ -53,7 +57,7 @@ be_vcl_config => $common_vcl_config, fe_extra_vcl => ['misc-common', 'zero'], be_extra_vcl => ['misc-common'], - be_storage => $::role::cache::base::file_storage_args, + be_storage => $::profile::cache::base::file_storage_args, fe_cache_be_opts => $fe_cache_be_opts, be_cache_be_opts => $be_cache_be_opts, cluster_nodes => hiera('cache::misc::nodes'), diff --git a/modules/role/manifests/cache/text.pp b/modules/role/manifests/cache/text.pp index a7cfe92..cbd80a2 100644 --- a/modules/role/manifests/cache/text.pp +++ b/modules/role/manifests/cache/text.pp @@ -5,9 +5,14 @@ $top_domain = 'org', $shortener_domain = 'w.wiki', ) { + + system::role { 'cache::text': + description => 'text Varnish cache server', + } + require geoip require geoip::dev # for VCL compilation using libGeoIP - include role::cache::base + include profile::cache::base include role::cache::ssl::unified include ::standard @@ -39,7 +44,7 @@ } $common_vcl_config = { - 'purge_host_regex' => $::role::cache::base::purge_host_not_upload_re, + 'purge_host_regex' => $::profile::cache::base::purge_host_not_upload_re, 'static_host' => $static_host, 'top_domain' => $top_domain, 'shortener_domain' => $shortener_domain, @@ -55,7 +60,7 @@ $common_runtime_params = ['default_ttl=86400'] - $text_storage_args = $::role::cache::base::file_storage_args + $text_storage_args = $::profile::cache::base::file_storage_args class { 'role::cache::instances': cache_type => 'text', diff --git a/modules/role/manifests/cache/upload.pp b/modules/role/manifests/cache/upload.pp index 1fe9832..13240ac 100644 --- a/modules/role/manifests/cache/upload.pp +++ b/modules/role/manifests/cache/upload.pp @@ -3,7 +3,11 @@ $upload_domain = 'upload.wikimedia.org', $maps_domain = 'maps.wikimedia.org', ) { - include role::cache::base + system::role { 'cache::upload': + description => 'upload Varnish cache server', + } + + include profile::cache::base include role::cache::ssl::unified include ::standard @@ -36,7 +40,7 @@ } $common_vcl_config = { - 'purge_host_regex' => $::role::cache::base::purge_host_only_upload_re, + 'purge_host_regex' => $::profile::cache::base::purge_host_only_upload_re, 'upload_domain' => $upload_domain, 'maps_domain' => $maps_domain, 'allowed_methods' => '^(GET|HEAD|OPTIONS|PURGE)$', @@ -59,9 +63,9 @@ }) # See T145661 for storage binning rationale - $sda = $::role::cache::base::storage_parts[0] - $sdb = $::role::cache::base::storage_parts[1] - $ssm = $::role::cache::base::storage_size * 2 * 1024 + $sda = $::profile::cache::base::storage_parts[0] + $sdb = $::profile::cache::base::storage_parts[1] + $ssm = $::profile::cache::base::storage_size * 2 * 1024 $bin0_size = floor($ssm * 0.04) $bin1_size = floor($ssm * 0.23) $bin2_size = floor($ssm * 0.40) diff --git a/modules/varnish/manifests/zero_update.pp b/modules/varnish/manifests/zero_update.pp index e174434..1b010ef 100644 --- a/modules/varnish/manifests/zero_update.pp +++ b/modules/varnish/manifests/zero_update.pp @@ -1,6 +1,6 @@ # Zero-specific update stuff class varnish::zero_update($site) { - require '::varnish::netmapper_update_common' + class { '::varnish::netmapper_update_common': } package { 'python-requests': ensure => installed; -- To view, visit https://gerrit.wikimedia.org/r/382683 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I28093780badaa38713847028cd5b5569dc81d6ca Gerrit-PatchSet: 6 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: BBlack <bbl...@wikimedia.org> Gerrit-Reviewer: Ema <e...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits