Aaron Schulz has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/392221 )
Change subject: [WIP] Add mcrouter module and mcrouter_wancache profile ...................................................................... [WIP] Add mcrouter module and mcrouter_wancache profile Change-Id: I8b881a036c121c010f1f2c8e42ddbf266630c1b2 --- M hieradata/common.yaml M hieradata/labs/deployment-prep/common.yaml A modules/mcrouter/manifests/init.pp A modules/mcrouter/manifests/monitoring.pp A modules/mcrouter/templates/default.erb A modules/profile/manifests/mediawiki/mcrouter_wancache.pp M modules/role/manifests/deployment/mediawiki.pp 7 files changed, 238 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/21/392221/10 diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 099761c..ef0a85e 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -528,3 +528,44 @@ # HTTP proxy (can be used as an env variable) http_proxy: "http://webproxy.%{::site}.wmnet:8080"; + +# Servers pools for multi-datacenter shared cache +mediawiki_wancache_memcached_servers: + eqiad: + - '10.64.0.80:11211:1' + - '10.64.0.81:11211:1' + - '10.64.0.82:11211:1' + - '10.64.0.83:11211:1' + - '10.64.0.84:11211:1' + - '10.64.16.107:11211:1' + - '10.64.16.108:11211:1' + - '10.64.16.109:11211:1' + - '10.64.16.110:11211:1' + - '10.64.32.208:11211:1' + - '10.64.32.209:11211:1' + - '10.64.32.210:11211:1' + - '10.64.32.211:11211:1' + - '10.64.32.212:11211:1' + - '10.64.48.155:11211:1' + - '10.64.48.156:11211:1' + - '10.64.48.157:11211:1' + - '10.64.48.158:11211:1' + codfw: + - '10.192.0.83:11211:1' + - '10.192.0.84:11211:1' + - '10.192.0.85:11211:1' + - '10.192.0.86:11211:1' + - '10.192.16.60:11211:1' + - '10.192.16.61:11211:1' + - '10.192.16.62:11211:1' + - '10.192.16.194:11211:1' + - '10.192.32.159:11211:1' + - '10.192.32.160:11211:1' + - '10.192.32.161:11211:1' + - '10.192.32.162:11211:1' + - '10.192.32.163:11211:1' + - '10.192.48.76:11211:1' + - '10.192.48.77:11211:1' + - '10.192.48.78:11211:1' + - '10.192.48.79:11211:1' + - '10.192.48.80:11211:1' \ No newline at end of file diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 3c3fbe2..95ec636 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -148,6 +148,10 @@ mediawiki_memcached_servers: - 10.68.23.25:11211:1 # deployment-memc04 - 10.68.23.49:11211:1 # deployment-memc05 +mediawiki_wancache_memcached_servers: + eqiad: + - 10.68.23.25:11211:1 # deployment-memc04 + - 10.68.23.49:11211:1 # deployment-memc05 profile::memcached::version: 'present' profile::memcached::growth_factor: '1.05' profile::memcached::size: '3000' diff --git a/modules/mcrouter/manifests/init.pp b/modules/mcrouter/manifests/init.pp new file mode 100644 index 0000000..9c316a9 --- /dev/null +++ b/modules/mcrouter/manifests/init.pp @@ -0,0 +1,104 @@ +# == Class: mcrouter +# +# mcrouter is a fast routing proxy for memcached. +# It can reduce the connection count on the backend caching servers +# and also supports layered pools, replication, and key/operation +# based routing to pools. +# +# === Parameters +# +# [*pools*] +# A hash defining a mcrouter server pool. +# See <https://github.com/facebook/mcrouter/wiki/Config-Files>. +# +# [*route*] +# A hash defining a mcrouter routing policy. +# See <https://github.com/facebook/mcrouter/wiki/List-of-Route-Handles>. +# +# [*region*] +# Datacenter name for the one in this geographical region +# +# [*cluster*] +# Memcached cluster name +# +# [*cross_region_timeout_ms*] +# Timeout, in milliseconds, when performing cross-region memcached operations +# +# [*cross_cluster_timeout_ms*] +# Timeout, in milliseconds, when performing cross-cluster memcached operations +# +# === Examples +# +# class { '::mcrouter': +# pools => { +# cluster-main' => { +# servers => [ '10.68.23.25:11211', '10.68.23.49:11211' ] +# } +# }, +# route => { +# type => 'OperationSelectorRoute', +# default_policy => 'PoolRoute|cluster-main', +# operation_policies => { +# set => 'AllFastestRoute|Pool|cluster-main', +# delete => 'AllSyncRoute|Pool|cluster-main' +# } +# } +# } +# +class mcrouter( + $pools, + $route, + $region, + $cluster, + $cross_region_timeout_ms, + $cross_cluster_timeout_ms, + $ensure = present +) { + validate_hash($pools) + validate_hash($route) + + require_package('mcrouter') + + $config = { 'pools' => $pools, 'route' => $route } + + file { '/etc/mcrouter/mcrouter.json': + ensure => $ensure, + content => ordered_json($config), + owner => 'root', + group => 'root', + mode => '0444', + require => Package['mcrouter'], + } + + if ( + $ensure == 'present' and + versioncmp($::puppetversion, '3.5') >= 0 and + versioncmp($::serverversion, '3.5') >= 0 + ) { + File['/etc/mcrouter/mcrouter.json'] { + validate_cmd => '/usr/sbin/mcrouter --validate-config --conf-file %', + } + } + + file { '/etc/default/mcrouter': + ensure => $ensure, + content => template('mcrouter/default.erb'), + owner => 'root', + group => 'root', + mode => '0444', + notify => Service['mcrouter'], + } + + file { '/etc/init/mcrouter.override': + ensure => $ensure, + content => "limit nofile 64000 64000\n", + owner => 'root', + group => 'root', + mode => '0444', + notify => Service['mcrouter'], + } + + service { 'mcrouter': + ensure => ensure_service($ensure), + } +} diff --git a/modules/mcrouter/manifests/monitoring.pp b/modules/mcrouter/manifests/monitoring.pp new file mode 100644 index 0000000..d0e9cd2 --- /dev/null +++ b/modules/mcrouter/manifests/monitoring.pp @@ -0,0 +1,17 @@ +# == Class: mcrouter::monitoring +# +# Provisions Icinga alerts for mcrouter. +# +class mcrouter::monitoring($port = 11213) { + nrpe::monitor_service { 'mcrouter': + description => 'mcrouter process', + nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -u mcrouter -C mcrouter', + } + + if ($port != 0) { + nrpe::monitor_service { 'mcrouter_port': + description => 'mcrouter port', + nrpe_command => "/usr/lib/nagios/plugins/check_tcp -H 127.0.0.1 -p ${port} --timeout=2", + } + } +} diff --git a/modules/mcrouter/templates/default.erb b/modules/mcrouter/templates/default.erb new file mode 100644 index 0000000..d9d2a2f --- /dev/null +++ b/modules/mcrouter/templates/default.erb @@ -0,0 +1,5 @@ +# Default settings for mcrouter. +# This file is managed by Puppet. + +DAEMON_OPTS="--route-prefix=<%= @region %>/<%= @cluster %> \ +--cross-region-timeout-ms=<%= @cross_region_timeout_ms %> --cross_cluster_timeout_ms=<%= @cross_cluster_timeout_ms %>" diff --git a/modules/profile/manifests/mediawiki/mcrouter_wancache.pp b/modules/profile/manifests/mediawiki/mcrouter_wancache.pp new file mode 100644 index 0000000..de01553 --- /dev/null +++ b/modules/profile/manifests/mediawiki/mcrouter_wancache.pp @@ -0,0 +1,62 @@ +# Class profile::mcrouter_wancache +# +# Configures a mcrouter instance for multi-datacenter caching +class profile::mediawiki::mcrouter_wancache( + $servers_by_datacenter = hiera('mediawiki_wancache_memcached_servers'), + $monitor_port = hiera('profile::mcrouter_wancache::monitor_port'), # set to 0 if no port available +) { + validate_hash($servers_by_datacenter) + + $pools = merge( $servers_by_datacenter.map |$region, $servers| { + { $region => { 'servers' => $servers } } + } ) + + $route = union( + $servers_by_datacenter.map |$region, $servers| { + { + 'aliases' => [ '/' + $region + '/mw/' ], + 'route' => 'PoolRoute|' + $region + } + }, + $servers_by_datacenter.map |$region, $servers| { + { + 'aliases' => [ '/' + $region + '/mw-wan/' ], + 'route' => { + 'type' => 'OperationSelectorRoute', + 'default_policy' => 'PoolRoute|' + $region, + 'operation_policies' => { + 'set' => 'AllFastestRoute|Pool|' + $region, + 'delete' => 'AllFastestRoute|Pool|' + $region + } + } + } + } + ) + + class { '::mcrouter': + pools => $pools, + route => $route, + region => $::site, + cluster => 'mw', + cross_region_timeout_ms => 250, + cross_cluster_timeout_ms => 1000 + } + + class { '::mcrouter::monitoring': + port => $monitor_port, + } + + ferm::rule { 'skip_mcrouter_wancache_conntrack_out': + desc => 'Skip outgoing connection tracking for mcrouter', + table => 'raw', + chain => 'OUTPUT', + rule => 'proto tcp sport (6378:6382 11212) NOTRACK;', + } + + ferm::rule { 'skip_mcrouter_wancache_conntrack_in': + desc => 'Skip incoming connection tracking for mcrouter', + table => 'raw', + chain => 'PREROUTING', + rule => 'proto tcp dport (6378:6382 11212) NOTRACK;', + } +} diff --git a/modules/role/manifests/deployment/mediawiki.pp b/modules/role/manifests/deployment/mediawiki.pp index 8cb17e8..5f28ea0 100644 --- a/modules/role/manifests/deployment/mediawiki.pp +++ b/modules/role/manifests/deployment/mediawiki.pp @@ -14,6 +14,11 @@ include ::profile::scap::dsh include ::scap::ferm + # mcrouter testing (T151466) + if $::realm == 'labs' { + include ::profile::mediawiki::mcrouter_wancache + } + # Keyholder require ::keyholder require ::keyholder::monitoring -- To view, visit https://gerrit.wikimedia.org/r/392221 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I8b881a036c121c010f1f2c8e42ddbf266630c1b2 Gerrit-PatchSet: 10 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org> Gerrit-Reviewer: Imarlier <imarl...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
