[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Sync security patches for MW from deployment to nightli...

[Chad (Code Review)]

Chad has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404892 )

Change subject: WIP: Sync security patches for MW from deployment to nightlies 
server
..

WIP: Sync security patches for MW from deployment to nightlies server

Change-Id: I241438b471c07ca04f9f29b5bb8c131499f291f2
---
A modules/profile/manifests/releases/mediawiki/security.pp
M modules/role/manifests/deployment_server/base.pp
M modules/role/manifests/releases.pp
3 files changed, 16 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/92/404892/1

diff --git a/modules/profile/manifests/releases/mediawiki/security.pp 
b/modules/profile/manifests/releases/mediawiki/security.pp
new file mode 100644
index 000..d3a22f3
--- /dev/null
+++ b/modules/profile/manifests/releases/mediawiki/security.pp
@@ -0,0 +1,14 @@
+# server hosting Mediawiki releases
+# https://releases.wikimedia.org/mediawiki/
+class profile::releases::mediawiki::security (
+$deployment_server = hiera('deployment_server'),
+$releases_server = hiera('releases_server') ) {
+
+rsync::quickdatacopy { 'srv-patches':
+ensure  => present,
+auto_sync   => true,
+source_host => $deployment_server,
+dest_host   => $releases_server,
+module_path => '/srv/patches',
+}
+}
diff --git a/modules/role/manifests/deployment_server/base.pp 
b/modules/role/manifests/deployment_server/base.pp
index 918fc08..5ccb113 100644
--- a/modules/role/manifests/deployment_server/base.pp
+++ b/modules/role/manifests/deployment_server/base.pp
@@ -4,4 +4,5 @@
 include ::base::firewall
 include ::profile::mediawiki::deployment::server
 include ::role::deployment::mediawiki
+include ::profile::releases::mediawiki::security
 }
diff --git a/modules/role/manifests/releases.pp 
b/modules/role/manifests/releases.pp
index 826f26f..f731d43 100644
--- a/modules/role/manifests/releases.pp
+++ b/modules/role/manifests/releases.pp
@@ -12,6 +12,7 @@
 include ::profile::base::firewall
 include ::profile::backup::host
 include ::profile::releases::mediawiki
+include ::profile::releases::mediawiki::security
 include ::profile::releases::reprepro
 
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/404892
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I241438b471c07ca04f9f29b5bb8c131499f291f2
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Chad 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] point eventlogging processes at Kafka jumbo

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404773 )

Change subject: [WIP] point eventlogging processes at Kafka jumbo
..

[WIP] point eventlogging processes at Kafka jumbo

This needs to be merged with https://gerrit.wikimedia.org/r/#/c/403067/

It is in gerrit now to be cherry-picked in deployment-prep

Bug: T183297
Change-Id: Iaf6f898b58a6564d2b22dce88ececfb415dc232e
---
M modules/role/manifests/eventlogging/analytics/files.pp
M modules/role/manifests/eventlogging/analytics/mysql.pp
M modules/role/manifests/eventlogging/analytics/processor.pp
M modules/role/manifests/eventlogging/analytics/server.pp
4 files changed, 6 insertions(+), 36 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/73/404773/1

diff --git a/modules/role/manifests/eventlogging/analytics/files.pp 
b/modules/role/manifests/eventlogging/analytics/files.pp
index 0619e93..bf5e9e9 100644
--- a/modules/role/manifests/eventlogging/analytics/files.pp
+++ b/modules/role/manifests/eventlogging/analytics/files.pp
@@ -45,16 +45,6 @@
 'eventlogging_consumer_files_00'
 )
 
-# Where possible, if this is set, it will be included in client 
configuration
-# to avoid having to do API version for Kafka < 0.10 (where there is not a 
version API).
-$kafka_api_version = 
$role::eventlogging::analytics::server::kafka_config['api_version']
-
-# Append this to query params if set.
-$kafka_api_version_param = $kafka_api_version ? {
-undef   => '',
-default => "_version=${kafka_api_version}"
-}
-
 # These commonly used URIs are defined for DRY purposes in
 # role::eventlogging::analytics::server.
 $kafka_client_side_raw_uri = 
$role::eventlogging::analytics::server::kafka_client_side_raw_uri
@@ -62,7 +52,7 @@
 
 # Raw client side events:
 eventlogging::service::consumer { 'client-side-events.log':
-input  => 
"${kafka_client_side_raw_uri}=True${kafka_api_version_param}",
+input  => "${kafka_client_side_raw_uri}=True",
 output => "file://${out_dir}/client-side-events.log",
 sid=> $kafka_consumer_group,
 }
@@ -71,7 +61,7 @@
 # 'blacklisted' during processing.  Events are blacklisted
 # from these logs for volume reasons.
 eventlogging::service::consumer { 'all-events.log':
-input  =>  "${kafka_mixed_uri}${kafka_api_version_param}",
+input  => $kafka_mixed_uri,
 output => "file://${out_dir}/all-events.log",
 sid=> $kafka_consumer_group,
 }
diff --git a/modules/role/manifests/eventlogging/analytics/mysql.pp 
b/modules/role/manifests/eventlogging/analytics/mysql.pp
index cac5874..081f838 100644
--- a/modules/role/manifests/eventlogging/analytics/mysql.pp
+++ b/modules/role/manifests/eventlogging/analytics/mysql.pp
@@ -32,16 +32,6 @@
 ['mysql-m4-master-00']
 )
 
-# Where possible, if this is set, it will be included in client 
configuration
-# to avoid having to do API version for Kafka < 0.10 (where there is not a 
version API).
-$kafka_api_version = 
$role::eventlogging::analytics::server::kafka_config['api_version']
-
-# Append this to query params if set.
-$kafka_api_version_param = $kafka_api_version ? {
-undef   => '',
-default => "_version=${kafka_api_version}"
-}
-
 $kafka_consumer_scheme = 
$role::eventlogging::analytics::server::kafka_consumer_scheme
 $kafka_brokers_string  = 
$role::eventlogging::analytics::server::kafka_brokers_string
 
@@ -68,7 +58,7 @@
 # Kafka consumer group for this consumer is mysql-m4-master
 eventlogging::service::consumer { $mysql_consumers:
 # auto commit offsets to kafka more often for mysql consumer
-input  => 
"${map_scheme}${kafka_consumer_uri}_commit_interval_ms=1000${kafka_api_version_param}${map_function}",
+input  => 
"${map_scheme}${kafka_consumer_uri}_commit_interval_ms=1000${map_function}",
 output => 
"mysql://${mysql_user}:${mysql_pass}@${mysql_db}?charset=utf8_host=${statsd_host}=True",
 sid=> 'eventlogging_consumer_mysql_00',
 # Restrict permissions on this config file since it contains a 
password.
diff --git a/modules/role/manifests/eventlogging/analytics/processor.pp 
b/modules/role/manifests/eventlogging/analytics/processor.pp
index dbaf521..cb15c0c 100644
--- a/modules/role/manifests/eventlogging/analytics/processor.pp
+++ b/modules/role/manifests/eventlogging/analytics/processor.pp
@@ -6,9 +6,6 @@
 class role::eventlogging::analytics::processor{
 include role::eventlogging::analytics::server
 
-# Where possible, if this is set, it will be included in client 
configuration
-# to avoid having to do API version for Kafka < 0.10 (where there is not a 
version API).
-$kafka_api_version = 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Produce webrequests from varnishkafka to jumbo Kafka c...

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404737 )

Change subject: [WIP] Produce webrequests from varnishkafka to jumbo Kafka 
cluster via TLS
..

[WIP] Produce webrequests from varnishkafka to jumbo Kafka cluster via TLS

This needs a lot of very careful review and coordination to merge in prod.
For now this exists in gerrit and is cherry-picked in deployment-prep.

Bug: T175461
Change-Id: I1760c36ee26f015617472073e4c5ab95d53d3e44
---
M modules/profile/manifests/cache/kafka/webrequest.pp
1 file changed, 27 insertions(+), 18 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/37/404737/1

diff --git a/modules/profile/manifests/cache/kafka/webrequest.pp 
b/modules/profile/manifests/cache/kafka/webrequest.pp
index 655779b..50321b1 100644
--- a/modules/profile/manifests/cache/kafka/webrequest.pp
+++ b/modules/profile/manifests/cache/kafka/webrequest.pp
@@ -1,34 +1,39 @@
 # === class profile::cache::kafka::webrequest
 #
 # Sets up a varnishkafka instance producing varnish
-# webrequest logs to the analytics Kafka brokers in eqiad.
+# webrequest logs to a Kafka cluster via TLS.
 #
 # === Parameters
 #
-# [*monitoring_enabled*]
-#   True if the varnishkafka instance should be monitored.
-#
 # [*cache_cluster*]
-#   the name of the cache cluster
+#   The name of the cache cluster.
 #
 # [*statsd*]
 #   The host:port to send statsd data to.
 #
+# [*kafka_cluster_name*]
+#   Name of the Kafka cluster in the hiera kafka_clusters hash.  This can
+#   be unqualified (without DC suffix) or fully qualified. Default: jumbo
+#
+# [*monitoring_enabled*]
+#   True if the varnishkafka instance should be monitored.  Default: false
+#
 class profile::cache::kafka::webrequest(
-$monitoring_enabled = 
hiera('profile::cache::kafka::webrequest::monitoring_enabled', false),
 $cache_cluster  = hiera('cache::cluster'),
 $statsd = hiera('statsd'),
+$kafka_cluster_name = 
hiera('profile::cache::kafka::webrequest::kafka_cluster_name', 'jumbo'),
+$monitoring_enabled = 
hiera('profile::cache::kafka::webrequest::monitoring_enabled', false),
 ) {
-$config = kafka_config('analytics')
-# NOTE: This is used by inheriting classes role::cache::kafka::*
-$kafka_brokers = $config['brokers']['array']
+# Include this class to get key and certificate for varnishkafka
+# to produce to Kafka over SSL/TLS.
+require ::profile::cache::kafka::certificate
 
-$topic = "webrequest_${cache_cluster}"
-# These used to be parameters, but I don't really see why given we never 
change
-# them
-$varnish_name   = 'frontend'
-$varnish_svc_name   = 'varnish-frontend'
-$kafka_protocol_version = '0.9.0.1'
+$config = kafka_config($kafka_cluster_name)
+$kafka_brokers = $config['brokers']['ssl_array']
+
+$topic= "webrequest_${cache_cluster}"
+$varnish_name = 'frontend'
+$varnish_svc_name = 'varnish-frontend'
 
 # Background task: T136314
 # Background info about the parameters used:
@@ -88,10 +93,7 @@
 $peak_rps_estimate = 9000
 
 varnishkafka::instance { 'webrequest':
-# FIXME - top-scope var without namespace, will break in puppet 2.8
-# lint:ignore:variable_scope
 brokers  => $kafka_brokers,
-# lint:endignore
 topic=> $topic,
 format_type  => 'json',
 compression_codec=> 'snappy',
@@ -122,6 +124,13 @@
 # stats will be fresh when polled from gmetad.
 log_statistics_interval  => 15,
 force_protocol_version   => $kafka_protocol_version,
+#TLS/SSL config
+ssl_enabled  => true,
+ssl_ca_location  => 
$::profile::cache::kafka::certificate::ssl_ca_location,
+ssl_key_password => 
$::profile::cache::kafka::certificate::ssl_key_password,
+ssl_key_location => 
$::profile::cache::kafka::certificate::ssl_key_location,
+ssl_certificate_location => 
$::profile::cache::kafka::certificate::ssl_certificate_location,
+ssl_cipher_suites=> 
$::profile::cache::kafka::certificate::ssl_cipher_suites,
 }
 
 if $monitoring_enabled {

-- 
To view, visit https://gerrit.wikimedia.org/r/404737
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I1760c36ee26f015617472073e4c5ab95d53d3e44
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] cassandra: create parent data directories with exec

[Eevans (Code Review)]

Eevans has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404705 )

Change subject: [WIP] cassandra: create parent data directories with exec
..

[WIP] cassandra: create parent data directories with exec

Bug: T175284
Change-Id: I9136f1410d07efc80f40c2a438621e43d18b8c6c
---
M modules/cassandra/manifests/instance.pp
1 file changed, 7 insertions(+), 8 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/05/404705/1

diff --git a/modules/cassandra/manifests/instance.pp 
b/modules/cassandra/manifests/instance.pp
index 0a31322..ef0a5c4 100644
--- a/modules/cassandra/manifests/instance.pp
+++ b/modules/cassandra/manifests/instance.pp
@@ -356,14 +356,13 @@
 require => Package['cassandra'],
 }
 
-file { [$instance_data_file_directories,
-$commitlog_directory,
-$saved_caches_directory]:
-ensure  => directory,
-owner   => 'cassandra',
-group   => 'cassandra',
-mode=> '0750',
-require => File[$data_directory_base],
+# (instance_)data_file_directories is an array of arbitrary, 
fully-qualified
+# paths. Since we cannot guarantee a common base path, ensure will not 
work.
+[$instance_data_file_directories, $commitlog_directory, 
$saved_caches_directory].each | $data_dir | {
+exec { 'install-data-directory':
+command => 'install -o cassandra -g cassandra -m 750 -d 
${data_dir}',
+path => '/usr/bin/:/bin/'
+}
 }
 
 file { "${config_directory}/cassandra-env.sh":

-- 
To view, visit https://gerrit.wikimedia.org/r/404705
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I9136f1410d07efc80f40c2a438621e43d18b8c6c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Eevans 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: grafana: Enable grafana's LDAP

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404321 )

Change subject: WIP: grafana: Enable grafana's LDAP
..

WIP: grafana: Enable grafana's LDAP

Disable the proxy auth done by apache and enable grafana's builtin LDAP

This is still missing the ldap.toml config file

Bug: T170150
Change-Id: Ie0c92eabfa743be4dab09e603b5af272d9270ee0
---
M hieradata/role/common/webserver_misc_apps.yaml
1 file changed, 10 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/21/404321/1

diff --git a/hieradata/role/common/webserver_misc_apps.yaml 
b/hieradata/role/common/webserver_misc_apps.yaml
index 859e345..bd0c617 100644
--- a/hieradata/role/common/webserver_misc_apps.yaml
+++ b/hieradata/role/common/webserver_misc_apps.yaml
@@ -15,6 +15,16 @@
   - cn=nda,ou=groups,dc=wikimedia,dc=org
   - cn=wmf,ou=groups,dc=wikimedia,dc=org
   - cn=grafana-admin,ou=groups,dc=wikimedia,dc=org
+profile::grafana::config:
+  auth.proxy:
+enabled: false
+  auth.ldap:
+enabled: true
+allow_sign_up: true
+config_file: ADDME # TO BE ADDED
+  auth.anonymous:
+enabled: true
+org_name: Wikimedia
 # In private repo
 #profile::grafana::secret_key:
 #profile::grafana::admin_password:

-- 
To view, visit https://gerrit.wikimedia.org/r/404321
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie0c92eabfa743be4dab09e603b5af272d9270ee0
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: nfsclient: Setup dumps mounts from new servers

[Madhuvishy (Code Review)]

Madhuvishy has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403767 )

Change subject: WIP: nfsclient: Setup dumps mounts from new servers
..

WIP: nfsclient: Setup dumps mounts from new servers

This patch is just a proof of concept that I'm
using to locally test these changes in the
testlabs project.

Bug: T171540
Change-Id: Ifa51ecfe05a360cf9a4560e49d6d386edceba1d2
---
M modules/role/manifests/labs/nfsclient.pp
1 file changed, 33 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/67/403767/1

diff --git a/modules/role/manifests/labs/nfsclient.pp 
b/modules/role/manifests/labs/nfsclient.pp
index 28cfcc9..553a13d 100644
--- a/modules/role/manifests/labs/nfsclient.pp
+++ b/modules/role/manifests/labs/nfsclient.pp
@@ -139,4 +139,37 @@
 require => Labstore::Nfs_mount['scratch-on-labstore1003'],
 }
 }
+
+if $::labsproject == 'testlabs' {
+
+  labstore::nfs_mount { 'dumps-on-labstore1006':
+  mount_name  => 'dumps',
+  project => $::labsproject,
+  options => ['ro', 'soft', 'timeo=300', 'retrans=3'],
+  mount_path  => '/mnt/nfs/labstore1006-dumps',
+  share_path  => '/public',
+  server  => 'labstore1006.wikimedia.org',
+  lookupcache => $lookupcache,
+  }
+
+  labstore::nfs_mount { 'dumps-on-labstore1007':
+  mount_name  => 'dumps',
+  project => $::labsproject,
+  options => ['ro', 'soft', 'timeo=300', 'retrans=3'],
+  mount_path  => '/mnt/nfs/labstore1007-dumps',
+  share_path  => '/public',
+  server  => 'labstore1007.wikimedia.org',
+  lookupcache => $lookupcache,
+  }
+
+  if mount_nfs_volume($::labsproject, 'dumps') {
+  file { '/public/newdumps':
+  ensure  => 'link',
+  force   => true,
+  target  => '/mnt/nfs/labstore1006-dumps',
+  require => Labstore::Nfs_mount['dumps-on-labstore1006'],
+  }
+  }
+}
+
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/403767
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifa51ecfe05a360cf9a4560e49d6d386edceba1d2
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Madhuvishy 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] coal: Consume EventLogging from Kafka instead of ZMQ

[Krinkle (Code Review)]

Krinkle has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403560 )

Change subject: [WIP] coal: Consume EventLogging from Kafka instead of ZMQ
..

[WIP] coal: Consume EventLogging from Kafka instead of ZMQ

Bug: T110903
Change-Id: I3d258f84cc4221a51750f79b5ba2dc4db329e570
---
M modules/coal/files/coal
M modules/coal/manifests/init.pp
M modules/coal/templates/initscripts/coal.systemd.erb
3 files changed, 40 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/60/403560/1

diff --git a/modules/coal/files/coal b/modules/coal/files/coal
index 2ec68da..09aef4f 100755
--- a/modules/coal/files/coal
+++ b/modules/coal/files/coal
@@ -25,6 +25,7 @@
 sys.setdefaultencoding('utf-8')
 
 import argparse
+import json
 import collections
 import dateutil.parser
 import errno
@@ -37,7 +38,7 @@
 import time
 
 import whisper
-import zmq
+from kafka import KafkaConsumer
 
 
 WINDOW_SPAN = 60 * 5  # Size of sliding window, in seconds.
@@ -105,10 +106,10 @@
 default=os.getcwd(),
 help='Path for Whisper files. Defaults to working directory.'
 )
-arg_parser.add_argument(
-'endpoint',
-help='EventLogging endpoint URL.'
-)
+ap.add_argument('--brokers', required=True,
+help='Comma-separated list of kafka brokers')
+ap.add_argument('--consumer-group', required=True,
+help='Consumer group to register with Kafka')
 
 def __init__(self):
 self.args = self.arg_parser.parse_args()
@@ -128,7 +129,20 @@
 def run(self):
 self.create_whisper_files()
 
-self.log.info('Connecting to %s.', self.args.endpoint)
+# Based on webperf/navtiming.py
+kafka_bootstrap_servers = tuple(self.args.brokers.split(','))
+kafka_topics = ('eventlogging_NavigationTiming', 
'eventlogging_SaveTiming')
+kafka_consumer_timeout_seconds = 60
+consumer = KafkaConsumer(
+*kafka_topics,
+bootstrap_servers=kafka_bootstrap_servers,
+group_id=self.args.consumer_group,
+auto_offset_reset='latest',
+enable_auto_commit=False,
+consumer_timeout_ms=kafka_consumer_timeout_seconds * 1000
+)
+
+self.log.info('Starting Kafka consumer')
 socket = zmq.Context().socket(zmq.SUB)
 socket.connect(self.args.endpoint)
 socket.subscribe = b''
diff --git a/modules/coal/manifests/init.pp b/modules/coal/manifests/init.pp
index 10aa0c1..17f8b6f 100644
--- a/modules/coal/manifests/init.pp
+++ b/modules/coal/manifests/init.pp
@@ -1,6 +1,16 @@
 # == Class: coal
 #
-# Store a basic set of Navigation Timing metrics in Whisper files.
+# Captures NavigationTiming events from Kafka and writes
+# a subset of metric directly to Whisper files.
+#
+# This complements webperf::navtiming, which uses StatsD and writes
+# to Graphite's default backend via carbon.  StatsD produces derived metrics,
+# like 'p99' and 'sample_rate'. Graphite aggregates Carbon's Whisper files
+# at varying resolutions as data gets older.
+#
+# Coal, on the other hand, simply retains data for 1 year at a constant
+# resolution of 1-minute.
+#
 # See https://meta.wikimedia.org/wiki/Schema:NavigationTiming &
 # http://www.mediawiki.org/wiki/Extension:NavigationTiming
 #
@@ -10,11 +20,14 @@
 #   URI of EventLogging event publisher to subscribe to.
 #   For example, 'tcp://eventlogging.eqiad.wmnet:8600'.
 #
+# [*kafka_brokers*]
+#   String of comma separated Kafka bootstrap brokers.
+#
 class coal( $endpoint ) {
 require_package('python-flask')
 require_package('python-numpy')
 require_package('python-whisper')
-require_package('python-zmq')
+require_package('python-kafka')
 
 group { 'coal':
 ensure => present,
diff --git a/modules/coal/templates/initscripts/coal.systemd.erb 
b/modules/coal/templates/initscripts/coal.systemd.erb
index d53916b..405d83b 100644
--- a/modules/coal/templates/initscripts/coal.systemd.erb
+++ b/modules/coal/templates/initscripts/coal.systemd.erb
@@ -1,11 +1,14 @@
 [Unit]
-Description=Navigation Timing Whisper logger
+Description=Navigation Timing Whisper writer
 
 [Service]
 User=coal
 Group=coal
 Restart=always
-ExecStart=/usr/local/bin/coal --whisper-dir "/var/lib/coal" "<%= @endpoint %>"
+ExecStart=/usr/local/bin/coal \
+--whisper-dir "/var/lib/coal" \
+--brokers <%= @kafka_brokers %> \
+--consumer-group coal
 
 [Install]
 WantedBy=multi-user.target

-- 
To view, visit https://gerrit.wikimedia.org/r/403560
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3d258f84cc4221a51750f79b5ba2dc4db329e570
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Krinkle 

___
MediaWiki-commits mailing list

[MediaWiki-commits] [Gerrit] operations/puppet[production]: wip: rabbitmq: handling users and initial setup

[Rush (Code Review)]

Rush has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403202 )

Change subject: wip: rabbitmq: handling users and initial setup
..

wip: rabbitmq: handling users and initial setup

A few inconsistencies and issues across our stuff:

* labtestcontrol2001 did not have the management
  plugin enabled which made the cleanup cron invalid
* the queuecleanup script did not handle a queue
  not being present yet and they are created dynamically
  (this is not by necessarily an errant state)
* queuecleanup script had hard coded credentials
* There has been no way to manage rabbitmq users and
  this has largely been done adhoc and outside of
  Puppets purview resulting in inconsistencies
* Rabbitmq can run on the non-primary control node
  and really needs to in order to manage rabbit
  internals as rabbit has a local user database
  and settings.
* guest account had been left in use on installs
* header docs for rabbit/init.pp were confusing

Change-Id: I1870faa07e49603bd7eff9f38cc1202519aeb467
---
M modules/profile/manifests/openstack/base/rabbitmq.pp
M modules/rabbitmq/files/drain_queue
A modules/rabbitmq/files/rabbit_random_guest.sh
M modules/rabbitmq/manifests/cleanup.pp
M modules/rabbitmq/manifests/init.pp
A modules/rabbitmq/manifests/user.pp
6 files changed, 142 insertions(+), 27 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/02/403202/1

diff --git a/modules/profile/manifests/openstack/base/rabbitmq.pp 
b/modules/profile/manifests/openstack/base/rabbitmq.pp
index efd0127..ffececf 100644
--- a/modules/profile/manifests/openstack/base/rabbitmq.pp
+++ b/modules/profile/manifests/openstack/base/rabbitmq.pp
@@ -10,7 +10,6 @@
 ){
 
 class { '::rabbitmq':
-running  => $::fqdn == $nova_controller,
 file_handles => $file_handles,
 }
 contain '::rabbitmq'
diff --git a/modules/rabbitmq/files/drain_queue 
b/modules/rabbitmq/files/drain_queue
index bd14e4e..4b98dbf 100644
--- a/modules/rabbitmq/files/drain_queue
+++ b/modules/rabbitmq/files/drain_queue
@@ -28,17 +28,21 @@
 import urllib.parse
 
 
+def eprint(*args, **kwargs):
+print(*args, file=sys.stderr, **kwargs)
+
 def die(s):
-print('*** {}'.format(s), file=sys.stderr)
+eprint('*** {}'.format(s))
 exit(1)
 
-
-def http_req(verb, path, body=None):
+def http_req(username, password, verb, path, body=None):
 path = '/api%s' % path
 conn = http.client.HTTPConnection('localhost', 15672)
+credentials = '{}:{}'.format(username, password)
+b = credentials.encode()
 headers = {
 'Authorization': 'Basic {}'.format(
-base64.b64encode(b'guest:guest').decode('ascii')),
+base64.b64encode(b).decode('ascii')),
 }
 if body:
 headers['Content-Type'] = 'application/json'
@@ -53,11 +57,20 @@
 if resp.status == 401:
 die('Access refused: {}'.format(path))
 if resp.status == 404:
-die('Not found: {}'.format(path))
+# Rabbitmq manages queues dynamically so
+# the existence of a queue may depend on a message
+# ever needing to be delivered to it.  Even
+# necessary queues are often created on-demand.
+eprint('Queue not found!')
+return json.dumps('')
 if resp.status == 301:
 url = urllib.parse.urlparse(resp.getheader('location'))
 [host, port] = url.netloc.split(':')
-return http_req(verb, url.path + '?' + url.query, body)
+return http_req(username,
+password,
+verb,
+url.path + '?' + url.query,
+body)
 if resp.status < 200 or resp.status > 400:
 raise Exception(
 'Received {:d} {} for path {}\n{}'.format(
@@ -65,12 +78,19 @@
 return resp_body
 
 
-def http_json(verb, path, body=None):
-return json.loads(http_req(verb, path, body))
+def http_json(username, password, verb, path, body=None):
+return json.loads(http_req(username, password, verb, path, body))
 
 
-def message_count(queue):
-return http_json('GET', '/queues/%2F/{}'.format(queue))['messages_ready']
+def message_count(username, password, queue):
+out = http_json(username,
+ password,
+ 'GET',
+ '/queues/%2F/{}'.format(queue))
+if not out:
+return None
+return out['messages_ready']
+
 
 
 def main():
@@ -86,14 +106,29 @@
 parser.add_argument(
 'queue', metavar='QUEUE', nargs=1,
 help='queue to read messages from')
+parser.add_argument(
+'--username', default='drainqueue',
+help='username to connect to rabbitmq')
+parser.add_argument(
+'--password', default='',
+help='password to connect to rabbitmq')
+
 args = parser.parse_args()
 queue = args.queue[0]
+username = args.username
+password = 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: toolforge: ferm hook to restart components post updates

[Rush (Code Review)]

Rush has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403072 )

Change subject: WIP: toolforge: ferm hook to restart components post updates
..

WIP: toolforge: ferm hook to restart components post updates

* Ferm is not playing nice with other iptables tenants
* Tested an /etc/ferm/conf.d/00_hooks to see it run external
  scripts in what seems like a totally post updates state.  This
  hopefully let's kube-proxy, flannel, and docker deal with
  Ferm stomping all around.

This is a midterm fix where other options are being explored
in the context of the task.  Right now any update to Ferm, even
a definition MAC, results in an outage for k8s in Toolforge.

Bug: T182722
Change-Id: I5c700a2c8bce6050e8cb761450d3716a6b3f33c9
---
M modules/role/manifests/toollabs/k8s/master.pp
M modules/role/manifests/toollabs/proxy.pp
A modules/toollabs/files/ferm_restart_handler.sh
A modules/toollabs/manifests/ferm_restart_handler.pp
M modules/toollabs/manifests/proxy.pp
5 files changed, 40 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/72/403072/1

diff --git a/modules/role/manifests/toollabs/k8s/master.pp 
b/modules/role/manifests/toollabs/k8s/master.pp
index 81647b4..1c0d78c 100644
--- a/modules/role/manifests/toollabs/k8s/master.pp
+++ b/modules/role/manifests/toollabs/k8s/master.pp
@@ -2,8 +2,9 @@
 class role::toollabs::k8s::master(
 $use_puppet_certs = false,
 ) {
-include ::base::firewall
 include ::toollabs::infrastructure
+include ::base::firewall
+include ::toollabs::ferm_restart_handler
 
 $master_host = hiera('k8s::master_host', $::fqdn)
 $etcd_url = prefix(suffix(hiera('k8s::etcd_hosts'), ':2379'), 'https://')
diff --git a/modules/role/manifests/toollabs/proxy.pp 
b/modules/role/manifests/toollabs/proxy.pp
index be70d49..c82cfef 100644
--- a/modules/role/manifests/toollabs/proxy.pp
+++ b/modules/role/manifests/toollabs/proxy.pp
@@ -2,6 +2,8 @@
 class role::toollabs::proxy {
 include ::toollabs::proxy
 include ::role::toollabs::k8s::webproxy
+include ::base::firewall
+include ::toollabs::ferm_restart_handler
 
 ferm::service { 'proxymanager':
 proto  => 'tcp',
diff --git a/modules/toollabs/files/ferm_restart_handler.sh 
b/modules/toollabs/files/ferm_restart_handler.sh
new file mode 100644
index 000..9bffa6b
--- /dev/null
+++ b/modules/toollabs/files/ferm_restart_handler.sh
@@ -0,0 +1,16 @@
+#/bin/bash
+
+/usr/bin/logger -t ${0} "restart firewall components post ferm management"
+
+# Ferm expects to handle all firewall state
+# and that does not mesh well with dynamic chain management.
+# We tell the k8s stack here to restart
+#
+# This should be no more invasive than a rescheduling
+# of a POD to another worker.
+#
+# If we are living an nftables world when you read
+# this, then this should be totally rethought.
+sudo service docker restart
+sudo service flannel restart
+sudo service kube-proxy restart
diff --git a/modules/toollabs/manifests/ferm_restart_handler.pp 
b/modules/toollabs/manifests/ferm_restart_handler.pp
new file mode 100644
index 000..2fc3034
--- /dev/null
+++ b/modules/toollabs/manifests/ferm_restart_handler.pp
@@ -0,0 +1,20 @@
+# tldr; hook post ferm updates to let other interested
+#   parties resync their iptables state.
+# See: T182722
+class toollabs::ferm_restart_handler{
+
+file {'/usr/local/sbin/ferm_restart_handler':
+source => 'puppet:///modules/toollabs/ferm_restart_handler.sh',
+owner  => 'root',
+group  => 'root',
+mode   => '0555',
+}
+
+# http://ferm.foo-projects.org/download/2.1/ferm.html#hooks
+# https://phabricator.wikimedia.org/T182722
+ferm::conf{'ferm_firewall_processing':
+prio  => 00,
+content   => '@hook post  "/usr/local/sbin/ferm_restart_handler"',
+subscribe => File['/usr/local/sbin/ferm_restart_handler'], 
+}
+}
diff --git a/modules/toollabs/manifests/proxy.pp 
b/modules/toollabs/manifests/proxy.pp
index 63953dd..9befba2 100644
--- a/modules/toollabs/manifests/proxy.pp
+++ b/modules/toollabs/manifests/proxy.pp
@@ -9,7 +9,6 @@
 
 include ::toollabs::infrastructure
 include ::redis::client::python
-include ::base::firewall
 
 if $ssl_install_certificate {
 sslcert::certificate { $ssl_certificate_name:

-- 
To view, visit https://gerrit.wikimedia.org/r/403072
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5c700a2c8bce6050e8cb761450d3716a6b3f33c9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Refactor cache::kafka::eventlogging into profile and e...

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403067 )

Change subject: [WIP] Refactor cache::kafka::eventlogging into profile and 
enable TLS
..

[WIP] Refactor cache::kafka::eventlogging into profile and enable TLS

Bug: T183297
Change-Id: I4096fe7efda237bac162dfb5dc8af1262c445503
---
A modules/profile/manifests/cache/kafka/eventlogging.pp
M modules/profile/manifests/cache/text.pp
D modules/role/manifests/cache/kafka/eventlogging.pp
M modules/role/manifests/cache/text.pp
4 files changed, 86 insertions(+), 74 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/67/403067/1

diff --git a/modules/profile/manifests/cache/kafka/eventlogging.pp 
b/modules/profile/manifests/cache/kafka/eventlogging.pp
new file mode 100644
index 000..2f4aa2f
--- /dev/null
+++ b/modules/profile/manifests/cache/kafka/eventlogging.pp
@@ -0,0 +1,79 @@
+# === Class profile::cache::kafka::eventlogging
+#
+# Sets up a varnishkafka logging endpoint for collecting
+# analytics events coming from external clients.
+#
+# TODO: This class is still in test mode
+#
+# More info: https://wikitech.wikimedia.org/wiki/Analytics/EventLogging
+#
+# === Parameters
+#
+# [*kafka_cluster_name*]
+#   Name of the Kafka cluster in the kafka_clusters hash to be passed to the
+#   kafka_config() function.  Default: jumbo.
+#
+# [*cache_cluster*]
+#   The name of the cache cluster.
+#
+# [*statsd*]
+#   The host to send statsd data to.
+#
+class profile::cache::kafka::eventlogging(
+$kafka_cluster_name = 
hiera('profile::cache::kafka::eventlogging::kafka_cluster_name', 'jumbo')
+$cache_cluster  = hiera('cache::cluster'),
+$statsd = hiera('statsd'),
+) {
+# Include this class to get key and certificate for varnishkafka
+# to produce to Kafka over SSL/TLS.
+require ::profile::cache::kafka::certificate
+
+# Set varnish.arg.q or varnish.arg.m according to Varnish version
+$varnish_opts = { 'q' => 'ReqURL ~ "^/(beacon/)?event(\.gif)?\?"' }
+
+$config = kafka_config($kafka_cluster_name)
+# Array of kafka brokers in jumbo-eqiad with SSL port 9093
+$kafka_brokers = $config['brokers']['ssl_array']
+
+$topic= "webrequest_${cache_cluster}_test"
+$varnish_name = 'frontend'
+$varnish_svc_name = 'varnish-frontend'
+
+varnishkafka::instance { 'eventlogging':
+brokers => $kafka_brokers,
+# Note that this format uses literal tab characters.
+# The '-' in this string used to be %{X-Client-IP@ip}o.
+# EventLogging clientIp logging has been removed as part of T128407.
+format  => '%q %l  %n  %{%FT%T}t   
-   "%{User-agent}i"',
+format_type => 'string',
+topic   => 'eventlogging-client-side',
+varnish_name=> $varnish_name,
+varnish_svc_name=> $varnish_svc_name,
+varnish_opts=> $varnish_opts,
+topic_request_required_acks => '1',
+}
+
+include ::standard
+
+# Generate icinga alert if varnishkafka is not running.
+nrpe::monitor_service { 'varnishkafka-eventlogging':
+description   => 'eventlogging Varnishkafka log producer',
+nrpe_command  => "/usr/lib/nagios/plugins/check_procs -c 1 -a 
'/usr/bin/varnishkafka -S /etc/varnishkafka/eventlogging.conf'",
+contact_group => 'admins,analytics',
+require   => Varnishkafka::Instance['eventlogging'],
+}
+
+$cache_type = hiera('cache::cluster')
+$graphite_metric_prefix = 
"varnishkafka.${::hostname}.eventlogging.${cache_cluster}"
+
+# Sets up Logster to read from the Varnishkafka instance stats JSON file
+# and report metrics to statsd.
+varnishkafka::monitor::statsd { 'eventlogging':
+graphite_metric_prefix => $graphite_metric_prefix,
+statsd_host_port   => $statsd,
+}
+
+# Make sure varnishes are configured and started for the first time
+# before the instances as well, or they fail to start initially...
+Service <| tag == 'varnish_instance' |> -> 
Varnishkafka::Instance['eventlogging']
+}
diff --git a/modules/profile/manifests/cache/text.pp 
b/modules/profile/manifests/cache/text.pp
index d4225b8..743d017 100644
--- a/modules/profile/manifests/cache/text.pp
+++ b/modules/profile/manifests/cache/text.pp
@@ -87,14 +87,6 @@
 backend_warming  => $backend_warming,
 }
 
-# varnishkafka eventlogging listens for eventlogging
-# requests and logs them to the eventlogging-client-side
-# topic.  EventLogging servers consume and process this
-# topic into many JSON based kafka topics for further
-# consumption.
-# TODO: Move this to profile, include from role::cache::text.
-class { '::role::cache::kafka::eventlogging': }
-
 # 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Don't merge

[Milimetric (Code Review)]

Milimetric has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402907 )

Change subject: [WIP] Don't merge
..

[WIP] Don't merge

Working out details for retention with Roan and team, so this is just
preliminary.  Right now the change is adding all fields from
Schema:ChangesListFilterGrouping and Schema:ChangesListFilters except
userId.

Change-Id: I3e9cc04a32a04283d2722a5d8659e3408c9bdee9
---
M 
modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
1 file changed, 29 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/07/402907/1

diff --git 
a/modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
 
b/modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
index ec3dd9c..aa52059 100644
--- 
a/modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
+++ 
b/modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
@@ -1,3 +1,32 @@
+ChangesListFilterGrouping  action
+ChangesListFilterGrouping  groupIdentifier
+ChangesListFilterGrouping  filter
+ChangesListFilters pagename
+ChangesListFilters enhancedFiltersEnabled
+ChangesListFilters hideminor
+ChangesListFilters hidemajor
+ChangesListFilters hidebots
+ChangesListFilters hidehumans
+ChangesListFilters hideanons
+ChangesListFilters hidepatrolled
+ChangesListFilters hideunpatrolled
+ChangesListFilters hidemyself
+ChangesListFilters hidebyothers
+ChangesListFilters hideliu
+ChangesListFilters hidecategorization
+ChangesListFilters hidepageedits
+ChangesListFilters hidenewpages
+ChangesListFilters hidelog
+ChangesListFilters hideWikibase
+ChangesListFilters hideReviewed
+ChangesListFilters hidenondamaging
+ChangesListFilters damaging
+ChangesListFilters goodfaith
+ChangesListFilters namespace
+ChangesListFilters userExpLevel
+ChangesListFilters tagfilter
+ChangesListFilters invert
+ChangesListFilters associated
 DiacriticsPoll webHost
 DiacriticsPoll wiki
 DiacriticsPoll event_country

-- 
To view, visit https://gerrit.wikimedia.org/r/402907
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3e9cc04a32a04283d2722a5d8659e3408c9bdee9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Milimetric 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Update gerrit login display

[Paladox (Code Review)]

Paladox has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402665 )

Change subject: WIP: Update gerrit login display
..

WIP: Update gerrit login display

This adds a new fancy login UI.

Change-Id: If18ceb87e7df7b2dd72aca834663467bec840496
---
0 files changed, 0 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/65/402665/1


-- 
To view, visit https://gerrit.wikimedia.org/r/402665
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If18ceb87e7df7b2dd72aca834663467bec840496
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Paladox 
Gerrit-Reviewer: jenkins-bot <>

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add all ops members to docker group

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401492 )

Change subject: WIP: Add all ops members to docker group
..

WIP: Add all ops members to docker group

Use the *ops_members anchor to add all ops people to the docker group.
The puppet parser function being the primary user of this data struct
is already doing a flatten(1) so the inclusion of a nested array as is
should not cause any problems.

While the canonical user of the data.yaml struct is doing 1 level
flattening anyway, there seem to be an array of utilities in the admin
module reimplementing the parsing for their own distinct purposes. None
of these seems to be flattening the array

Change-Id: I2e9d6c51bd1d4969c047e076c5949b95f60d061c
---
M modules/admin/data/data.yaml
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/92/401492/1

diff --git a/modules/admin/data/data.yaml b/modules/admin/data/data.yaml
index 73cdcb2..1db8dfc 100644
--- a/modules/admin/data/data.yaml
+++ b/modules/admin/data/data.yaml
@@ -647,7 +647,7 @@
 # This has the potential of conflicting with other users asking 
permissions to run docker. IFF
 # such a need arises we might want to revisit this
 description: Allow releng team to be in the docker group for contint. No 
gid on purpose
-members: [akosiaris, dduvall, demon, gjg, hashar, thcipriani, 
twentyafterfour, zfilipin]
+members: [*ops_members, dduvall, demon, gjg, hashar, thcipriani, 
twentyafterfour, zfilipin]
 
 users:
   rush:

-- 
To view, visit https://gerrit.wikimedia.org/r/401492
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2e9d6c51bd1d4969c047e076c5949b95f60d061c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add loading DCAT-AP data into dcatap namespace on WDQS

[Smalyshev (Code Review)]

Smalyshev has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399954 )

Change subject: [WIP] Add loading DCAT-AP data into dcatap namespace on WDQS
..

[WIP] Add loading DCAT-AP data into dcatap namespace on WDQS

Bug: T178978
Change-Id: Ie1d7eb7567d1409fb1116a71b23b4d5160aa2c7b
---
A modules/wdqs/files/cron/cronUtils.sh
M modules/wdqs/files/cron/reloadCategories.sh
A modules/wdqs/files/cron/reloadDCAT-AP.sh
M modules/wdqs/manifests/gui.pp
4 files changed, 108 insertions(+), 26 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/54/399954/1

diff --git a/modules/wdqs/files/cron/cronUtils.sh 
b/modules/wdqs/files/cron/cronUtils.sh
new file mode 100755
index 000..42a1e04
--- /dev/null
+++ b/modules/wdqs/files/cron/cronUtils.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Cron helper functions
+if [ -r /etc/wdqs/vars.sh ]; then
+  . /etc/wdqs/vars.sh
+fi
+
+if [ -r /etc/wdqs/gui_vars.sh ]; then
+  . /etc/wdqs/gui_vars.sh
+fi
+
+if [ -z "${DATA_DIR}" -o -z "${LOG_DIR}" -o -z "${DEPLOY_DIR}" ]; then
+   echo "Variables not set up right!"
+   exit 1
+fi
+
+HOST="http://localhost:;
+NAMESPACE_URL="/bigdata/namespace/"
+today=$(date -u +'%Y%m%d')
+
+function loadFileIntoBlazegraph {
+   # source URL
+   local URL=$1
+   # local filename (will be in DATA_DIR)
+   local fileName=$2
+   local sparqlEndpoint=$3
+   curl -s -f -XGET $URL -o ${DATA_DIR}/${fileName}
+   if [ ! -s ${DATA_DIR}/${fileName} ]; then
+   echo "Could not download $URL into ${fileName}"
+   exit 1
+   fi  
+   curl -s -XPOST --data-binary update="LOAD " 
$SPARQL
+}
+
+# NOTE: This should be run under user that has rights to
+# sudo systemctl reload nginx
+function replaceNamespace {
+   local mainName=$1
+   local currentAlias=$2
+   local oldNamespace=$(cat $ALIAS_FILE | grep $mainName | cut -d' ' -f2 | 
cut -d ';' -f1)
+   sed -i "/${MAIN_NAME}/c ${mainName} ${currentAlias};" $ALIAS_FILE
+   # Bump nginx to reload config
+   sudo systemctl reload nginx
+   if [ -n "${oldNamespace}" ]; then
+   # Drop old namespace
+   curl -s -X DELETE "${HOST}${NAMESPACE_URL}${oldNamespace}"
+   fi
+}
\ No newline at end of file
diff --git a/modules/wdqs/files/cron/reloadCategories.sh 
b/modules/wdqs/files/cron/reloadCategories.sh
index 33b83e9..2710a19 100755
--- a/modules/wdqs/files/cron/reloadCategories.sh
+++ b/modules/wdqs/files/cron/reloadCategories.sh
@@ -2,35 +2,14 @@
 # This script is reloading categories into a new namespace
 # NOTE: This should be run under user that has rights to
 # sudo systemctl reload nginx
-if [ -r /etc/wdqs/vars.sh ]; then
-  . /etc/wdqs/vars.sh
-fi
+. /usr/local/bin/cronUtils.sh
 
-if [ -r /etc/wdqs/gui_vars.sh ]; then
-  . /etc/wdqs/gui_vars.sh
-fi
-
-if [ -z "${DATA_DIR}" -o -z "${LOG_DIR}" -o -z "${DEPLOY_DIR}" ]; then
-   echo "Variables not set up right!"
-   exit 1
-fi
-
-today=$(date -u +'%Y%m%d')
 newNamespace="categories${today}"
 # Drop old dumps
 rm -f ${DATA_DIR}/*-categories.ttl.gz
 cd $DEPLOY_DIR
+# Create new namespace
 bash createNamespace.sh $newNamespace || exit 1
 # Load the data
 bash forAllCategoryWikis.sh loadCategoryDump.sh $newNamespace >> 
"${LOG_DIR}/${newNamespace}.log"
-# Get old namespace
-oldNamespace=$(cat $ALIAS_FILE | grep categories | cut -d' ' -f2 | cut -d ';' 
-f1)
-# Switch the map
-# NOTE: right now it overrides the map. If we reuse it for other purposes, 
this needs to be made smarter.
-echo "categories ${newNamespace};" > $ALIAS_FILE
-# Bump nginx to reload config
-sudo systemctl reload nginx
-if [ -n "${oldNamespace}" ]; then
-   # Drop old namespace
-   curl -s -X DELETE 
"http://localhost:/bigdata/namespace/${oldNamespace};
-fi
+replaceNamespace categories $newNamespace
diff --git a/modules/wdqs/files/cron/reloadDCAT-AP.sh 
b/modules/wdqs/files/cron/reloadDCAT-AP.sh
new file mode 100755
index 000..618bddc
--- /dev/null
+++ b/modules/wdqs/files/cron/reloadDCAT-AP.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+# This script is reloading DCAT-AP data from Wikidata
+# NOTE: This should be run under user that has rights to
+# sudo systemctl reload nginx
+. /usr/local/bin/cronUtils.sh
+
+DCAT_SOURCE=${DCAT_SOURCE:-"https://dumps.wikimedia.org/wikidatawiki/entities/dcatap.rdf"}
+
+newNamespace="dcatap${today}"
+# Drop old dumps
+rm -f ${DATA_DIR}/dcatap-*.rdf
+cd $DEPLOY_DIR
+# Create new NS
+bash createNamespace.sh $newNamespace || exit 1
+# Load the data
+FILENAME=dcatap-${today}.rdf
+loadFileIntoBlazegraph $DCAT_SOURCE $FILENAME 
"${HOST}${NAMESPACE_URL}${newNamespace}/sparql"
+replaceNamespace dcatap $newNamespace
diff --git a/modules/wdqs/manifests/gui.pp b/modules/wdqs/manifests/gui.pp
index 458e38e..d10bbb3 100644
--- a/modules/wdqs/manifests/gui.pp
+++ b/modules/wdqs/manifests/gui.pp
@@ -32,7 +32,7 @@
 }
 
 # List 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP allow labmon1001 to contact pdns exporters

[Filippo Giunchedi (Code Review)]

Filippo Giunchedi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399439 )

Change subject: WIP allow labmon1001 to contact pdns exporters
..

WIP allow labmon1001 to contact pdns exporters

Change-Id: Ifad13cccf4351a6954c500632cda032e467d0ed1
---
A hieradata/common/profile/openstack/main/pdns/auth/service.yaml
A hieradata/common/profile/openstack/main/pdns/recursor/primary.yaml
M modules/profile/manifests/prometheus/pdns_exporter.pp
M modules/profile/manifests/prometheus/pdns_rec_exporter.pp
4 files changed, 12 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/399439/1

diff --git a/hieradata/common/profile/openstack/main/pdns/auth/service.yaml 
b/hieradata/common/profile/openstack/main/pdns/auth/service.yaml
new file mode 100644
index 000..aabeac6
--- /dev/null
+++ b/hieradata/common/profile/openstack/main/pdns/auth/service.yaml
@@ -0,0 +1,2 @@
+prometheus_nodes_extra:
+  - labmon1001.eqiad.wmnet
diff --git a/hieradata/common/profile/openstack/main/pdns/recursor/primary.yaml 
b/hieradata/common/profile/openstack/main/pdns/recursor/primary.yaml
new file mode 100644
index 000..aabeac6
--- /dev/null
+++ b/hieradata/common/profile/openstack/main/pdns/recursor/primary.yaml
@@ -0,0 +1,2 @@
+prometheus_nodes_extra:
+  - labmon1001.eqiad.wmnet
diff --git a/modules/profile/manifests/prometheus/pdns_exporter.pp 
b/modules/profile/manifests/prometheus/pdns_exporter.pp
index 292d06e..5525365 100644
--- a/modules/profile/manifests/prometheus/pdns_exporter.pp
+++ b/modules/profile/manifests/prometheus/pdns_exporter.pp
@@ -1,5 +1,10 @@
 class profile::prometheus::pdns_exporter (
+$prometheus_nodes = hiera('prometheus_nodes'),
+$prometheus_nodes_extra = hiera('prometheus_nodes_extra', []),
 ) {
+$prometheus_ferm_nodes = join($prometheus_nodes + $prometheus_nodes_extra, 
' ')
+$ferm_srange = "(@resolve((${prometheus_ferm_nodes})) 
@resolve((${prometheus_ferm_nodes}), ))"
+
 require_package('prometheus-pdns-exporter')
 
 service { 'prometheus-pdns-exporter':
@@ -9,6 +14,6 @@
 ferm::service { 'prometheus-pdns-exporter':
 proto  => 'tcp',
 port   => '9192',
-srange => '@resolve(labmon1001.eqiad.wmnet)', # Should be properly 
defined via Hiera for WMCS
+srange => $ferm_srange,
 }
 }
diff --git a/modules/profile/manifests/prometheus/pdns_rec_exporter.pp 
b/modules/profile/manifests/prometheus/pdns_rec_exporter.pp
index 0902ab2..582393a 100644
--- a/modules/profile/manifests/prometheus/pdns_rec_exporter.pp
+++ b/modules/profile/manifests/prometheus/pdns_rec_exporter.pp
@@ -1,7 +1,8 @@
 class profile::prometheus::pdns_rec_exporter (
 $prometheus_nodes = hiera('prometheus_nodes'),
+$prometheus_nodes_extra = hiera('prometheus_nodes_extra', []),
 ) {
-$prometheus_ferm_nodes = join($prometheus_nodes, ' ')
+$prometheus_ferm_nodes = join($prometheus_nodes + $prometheus_nodes_extra, 
' ')
 $ferm_srange = "(@resolve((${prometheus_ferm_nodes})) 
@resolve((${prometheus_ferm_nodes}), ))"
 
 require_package('prometheus-pdns-rec-exporter')

-- 
To view, visit https://gerrit.wikimedia.org/r/399439
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifad13cccf4351a6954c500632cda032e467d0ed1
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]mariadb: Add mysql 8.0-compatible template

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399115 )

Change subject: [WIP]mariadb: Add mysql 8.0-compatible template
..

[WIP]mariadb: Add mysql 8.0-compatible template

Change-Id: I9330498242ef2b45160ee339ee3faa2bdd9583f2
---
A modules/role/templates/mariadb/mysqld_config/core-mysql.my.cnf.erb
1 file changed, 120 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/15/399115/1

diff --git a/modules/role/templates/mariadb/mysqld_config/core-mysql.my.cnf.erb 
b/modules/role/templates/mariadb/mysqld_config/core-mysql.my.cnf.erb
new file mode 100644
index 000..acece1f
--- /dev/null
+++ b/modules/role/templates/mariadb/mysqld_config/core-mysql.my.cnf.erb
@@ -0,0 +1,120 @@
+# Warning: This file is managed by Puppet!
+# Production
+
+[client]
+port   = 3306
+socket = /run/mysqld/mysqld.sock
+# ssl
+#ssl-ca=/etc/ssl/certs/Puppet_Internal_CA.pem
+#ssl-cert=/etc/mysql/ssl/cert.pem
+#ssl-key=/etc/mysql/ssl/server.key
+#ssl-verify-server-cert
+
+[mysqld]
+
+user   = mysql
+socket = /run/mysqld/mysqld.sock
+port   = 3306
+# extra_port = 3307
+basedir= /opt/mysql
+datadir= /srv/sqldata
+tmpdir = /srv/tmp
+server_id  = 1
+
+# Always start in read_only mode.
+# Master selection will be handled by cluster control.
+#read_only = on
+
+log_bin
+log_slave_updates
+skip-external-locking
+skip-name-resolve
+#temp-pool
+
+# thread_handling= pool-of-threads
+# thread_pool_stall_limit= 100
+# thread_pool_size   = 32
+max_connections= 5000
+back_log   = 500
+#extra_max_connections  = 10
+max_connect_errors = 10
+max_allowed_packet = 32M
+connect_timeout= 3
+# query_cache_size   = 0
+# query_cache_type   = 0
+event_scheduler= 1
+#log-warnings   = 0
+thread_stack   = 192K
+thread_cache_size  = 300
+interactive_timeout= 28800
+wait_timeout   = 3600
+transaction-isolation  = REPEATABLE-READ
+#use_stat_tables= preferably
+
+sync_binlog= 1
+binlog_cache_size  = 1M
+max_binlog_size= 1000M
+binlog_format  = ROW
+# expire_logs_days   = 7
+binlog_expire_logs_seconds = 604800
+slave_transaction_retries  = 4294967295
+#sql-mode   = IGNORE_BAD_TABLE_OPTIONS
+tmp_table_size = 64M
+max_heap_table_size= 64M
+
+table_open_cache   = 5
+table_definition_cache = 4
+open-files-limit   = 20
+character_set_server   = binary
+character_set_filesystem   = binary
+collation_server   = binary
+
+default-storage-engine = InnoDB
+innodb_file_per_table  = 1
+innodb_buffer_pool_size= 1G
+innodb_log_file_size   = 100M
+innodb_flush_log_at_trx_commit = 1
+innodb_flush_method= O_DIRECT
+innodb_thread_concurrency  = 0
+innodb_io_capacity = 1000
+#innodb_stats_sample_pages  = 16
+innodb_stats_method= nulls_unequal
+#aria_pagecache_buffer_size = 1G
+#join_cache_level   = 8
+
+# dump and load innodb buffer at start and stop
+innodb_buffer_pool_load_at_startup  = 1
+innodb_buffer_pool_dump_at_shutdown = 1
+
+# ssl
+#ssl-ca=/etc/ssl/certs/Puppet_Internal_CA.pem
+#ssl-cert=/etc/mysql/ssl/cert.pem
+#ssl-key=/etc/mysql/ssl/server.key
+#ssl-cipher=TLSv1.2
+# Enabling performance_schema (disabled by default in MariaDB10)
+performance_schema = 1
+# downsizing performance schema memory usage: T99485
+#performance_schema_digests_size= -1
+#performance_schema_max_thread_instances= 500
+#performance_schema_max_cond_instances  = 1000
+#performance_schema_accounts_size   = 300
+#performance_schema_hosts_size  = 300
+#performance_schema_events_statements_history_size  = 10
+#performance_schema_events_statements_history_long_size = 1000
+#performance_schema_events_waits_history_size   = 10
+#performance_schema_events_waits_history_long_size  = 1000
+#performance_schema_events_stages_history_size  = 10
+#performance_schema_events_stages_history_long_size = 1000
+#performance_schema_max_mutex_instances = 5000
+#performance_schema_max_rwlock_instances= 2000
+#performance_schema_max_socket_instances= 500
+#performance_schema_max_table_instances = 1000
+
+#optimizer_switch = 
'mrr=on,mrr_cost_based=on,mrr_sort_keys=on,optimize_join_buffer_size=on'
+optimizer_switch = 'mrr=on,mrr_cost_based=on'
+
+[mysqldump]
+
+quick

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Update mariadb::proxy to the latest style and path loc...

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398450 )

Change subject: [WIP] Update mariadb::proxy to the latest style and path 
locations
..

[WIP] Update mariadb::proxy to the latest style and path locations

Create profile::mariadb::proxy{,::master,::replicas}, for the base,
master failover and replica load balancing roles. Replica class is
unused, so probably untested and doesn't work.

Make socker and pid configurable, default to /run location, but
it can now be overriden by host by host migration from /tmp to
/run.

The patch is missing the several hiera keys for the individual or
role-based pointing to servers.

Bug: T148507
Change-Id: I71a888b516866ab22174565b00c426fafe55a7d9
---
M manifests/site.pp
M modules/haproxy/manifests/init.pp
R modules/haproxy/templates/check_haproxy.erb
A modules/profile/manifests/mariadb/proxy.pp
A modules/profile/manifests/mariadb/proxy/master.pp
A modules/profile/manifests/mariadb/proxy/replicas.pp
R modules/profile/templates/mariadb/proxy/db-master-stretch.cfg.erb
R modules/profile/templates/mariadb/proxy/db-master.cfg.erb
R modules/profile/templates/mariadb/proxy/db-replicas.cfg.erb
R modules/profile/templates/mariadb/proxy/db.cfg.erb
M modules/role/manifests/mariadb/proxy.pp
M modules/role/manifests/mariadb/proxy/master.pp
A modules/role/manifests/mariadb/proxy/replicas.pp
D modules/role/manifests/mariadb/proxy/slaves.pp
14 files changed, 86 insertions(+), 138 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/50/398450/1

diff --git a/manifests/site.pp b/manifests/site.pp
index 718a2ea..81d21bf 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -626,77 +626,13 @@
 }
 
 # Proxies for misc databases
-node /^dbproxy100(1|6)\.eqiad\.wmnet$/ {
-class { '::role::mariadb::proxy::master':
-shard  => 'm1',
-primary_name   => 'db1016',
-primary_addr   => '10.64.0.20',
-secondary_name => 'db1001',
-secondary_addr => '10.64.0.5',
-}
-}
-
-node /^dbproxy100(2|7)\.eqiad\.wmnet$/ {
-class { '::role::mariadb::proxy::master':
-shard  => 'm2',
-primary_name   => 'db1020',
-primary_addr   => '10.64.16.9',
-secondary_name => 'db2011',
-secondary_addr => '10.192.0.14',
-}
-}
-
-node /^dbproxy100(3|8)\.eqiad\.wmnet$/ {
-class { '::role::mariadb::proxy::master':
-shard  => 'm3',
-primary_name   => 'db1043',
-primary_addr   => '10.64.16.32',
-secondary_name => 'db1059',
-secondary_addr => '10.64.32.29',
-}
-}
-
-# stretch haproxies (1.7) require the port number
-node /^dbproxy100(4|9)\.eqiad\.wmnet$/ {
-class { '::role::mariadb::proxy::master':
-shard  => 'm4',
-primary_name   => 'db1107',
-primary_addr   => '10.64.0.214:3306',
-secondary_name => 'db1108',
-secondary_addr => '10.64.32.71:3306',
-}
-}
-
-node 'dbproxy1005.eqiad.wmnet' {
-role(mariadb::ferm_wmcs)
-class { '::role::mariadb::proxy::master':
-shard  => 'm5',
-primary_name   => 'db1009',
-primary_addr   => '10.64.0.13',
-secondary_name => 'db2030',
-secondary_addr => '10.192.16.18',
-}
+node /^dbproxy10(01|02|03|04|05|06|07|08|09)\.eqiad\.wmnet$/ {
+role(mariadb::proxy)
 }
 
 # labsdb proxies (controling replica service dbs)
-node 'dbproxy1010.eqiad.wmnet' {
-class { '::role::mariadb::proxy::master':
-shard  => 'labsdb',
-primary_name   => 'labsdb1009',
-primary_addr   => '10.64.4.14',
-secondary_name => 'labsdb1010',
-secondary_addr => '10.64.37.23',
-}
-}
-
-node 'dbproxy1011.eqiad.wmnet' {
-class { '::role::mariadb::proxy::master':
-shard  => 'labsdb',
-primary_name   => 'labsdb1011',
-primary_addr   => '10.64.37.24',
-secondary_name => 'labsdb1010',
-secondary_addr => '10.64.37.23',
-}
+node 'dbproxy101[01].eqiad.wmnet' {
+role(mariadb::proxy)
 }
 
 node /^dbmonitor[12]001\.wikimedia\.org$/ {
diff --git a/modules/haproxy/manifests/init.pp 
b/modules/haproxy/manifests/init.pp
index 74c9e53..c70bd00 100644
--- a/modules/haproxy/manifests/init.pp
+++ b/modules/haproxy/manifests/init.pp
@@ -2,6 +2,8 @@
 
 class haproxy(
 $template = 'haproxy/haproxy.cfg.erb',
+$socket   = '/run/haproxy/haproxy.sock',
+$pid  = '/run/haproxy/haproxy.pid',
 ) {
 
 package { [
@@ -27,10 +29,10 @@
 }
 
 file { '/usr/lib/nagios/plugins/check_haproxy':
-owner  => 'root',
-group  => 'root',
-mode   => '0755',
-source => 'puppet:///modules/haproxy/check_haproxy',
+owner   => 'root',
+group   => 'root',
+mode=> '0755',
+content => template('modules/haproxy/check_haproxy'),
 }
 
 if os_version('debian >= 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Populate the docker group in admin module

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398276 )

Change subject: WIP: Populate the docker group in admin module
..

WIP: Populate the docker group in admin module

Let's see if we can reuse the docker group instead of having to devise
sudo rules

Bug: T182860
Change-Id: I115cebfa5ba8592d46fbfa6b42c14f6b92eeed79
---
M hieradata/role/common/ci/master.yaml
M modules/admin/data/data.yaml
2 files changed, 4 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/76/398276/1

diff --git a/hieradata/role/common/ci/master.yaml 
b/hieradata/role/common/ci/master.yaml
index 4acdf78..791ad2f 100644
--- a/hieradata/role/common/ci/master.yaml
+++ b/hieradata/role/common/ci/master.yaml
@@ -2,6 +2,7 @@
   - contint-users
   - contint-admins
   - contint-roots
+  - docker
 cluster: misc
 standard::has_ganglia: false
 contactgroups: 'admins,contint'
diff --git a/modules/admin/data/data.yaml b/modules/admin/data/data.yaml
index 26c5e8c..8dcf9cc 100644
--- a/modules/admin/data/data.yaml
+++ b/modules/admin/data/data.yaml
@@ -648,6 +648,9 @@
 privileges: ['ALL = NOPASSWD: /bin/journalctl *',
  'ALL = NOPASSWD: /bin/dmesg *',
  'ALL = (syslog) NOPASSWD: ALL']
+  docker:
+description: People who can use the docker group as provisioned by docker. 
No gid on purpose
+members: [akosiaris]
 
 users:
   rush:

-- 
To view, visit https://gerrit.wikimedia.org/r/398276
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I115cebfa5ba8592d46fbfa6b42c14f6b92eeed79
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: rework mtail tests

[Filippo Giunchedi (Code Review)]

Filippo Giunchedi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/397889 )

Change subject: WIP: rework mtail tests
..

WIP: rework mtail tests

Change-Id: I7bc5e375e355153c8d5cce9e496cb648deb24cb2
---
M modules/mtail/files/test/exim_test.py
M modules/mtail/files/test/mtail_store.py
2 files changed, 36 insertions(+), 42 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/89/397889/1

diff --git a/modules/mtail/files/test/exim_test.py 
b/modules/mtail/files/test/exim_test.py
index a0de96c..345a9a4 100644
--- a/modules/mtail/files/test/exim_test.py
+++ b/modules/mtail/files/test/exim_test.py
@@ -12,39 +12,40 @@
 os.path.join(test_dir, 'logs/exim.test'))
 
 def testEximMessages(self):
-m = self.store.get_metric('exim_messages_total')
-self.assertEqual(3, m._value)
-self.assertIn('status=out', m._labelpairs)
+s = self.store.get_samples('exim_messages_total')
+self.assertIn(('status=out', 3), s)
 
-m = self.store.get_metric('exim_messages_bytes')
-self.assertEqual(183084, m._value)
-self.assertIn('status=out', m._labelpairs)
+s = self.store.get_samples('exim_messages_bytes')
+self.assertIn(('status=out', 183084), s)
 
 def testDKIM(self):
-m = self.store.get_metric('exim_dkim_failure_total')
-self.assertEqual(2, m._value)
+s = self.store.get_samples('exim_dkim_failure_total')
+self.assertIn(('', 2), s)
 
-m = self.store.get_metric('exim_dkim_success_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_dkim_success_total')
+self.assertIn(('', 1), s)
 
 def testMiscErrors(self):
-m = self.store.get_metric('exim_smtp_errors_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_smtp_errors_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_rejected_rcpt_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_smtp_errors_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_tls_errors_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_rejected_rcpt_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_sender_verify_fail_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_tls_errors_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_sender_verify_defer_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_sender_verify_fail_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_too_many_connections_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_sender_verify_defer_total')
+self.assertIn(('', 1), s)
 
-m = self.store.get_metric('exim_rejected_helo_total')
-self.assertEqual(1, m._value)
+s = self.store.get_samples('exim_too_many_connections_total')
+self.assertIn(('', 1), s)
+
+s = self.store.get_samples('exim_rejected_helo_total')
+self.assertIn(('', 1), s)
diff --git a/modules/mtail/files/test/mtail_store.py 
b/modules/mtail/files/test/mtail_store.py
index 527e315..360611c 100644
--- a/modules/mtail/files/test/mtail_store.py
+++ b/modules/mtail/files/test/mtail_store.py
@@ -29,23 +29,16 @@
 
 self._store = json.loads(''.join(metrics_store))
 
-def get_metric(self, name):
+def get_samples(self, name):
+"""Return all samples for metric name as a list of samples.
+   Each sample is in this form: ("k1=v1,k2=v2", value)"""
+samples = []
 if name not in self._store:
 raise ValueError('metric %s not found in store', name)
-return MtailMetric(self._store[name][0].get('Keys', []),
-   
self._store[name][0]['LabelValues'][0].get('Labels', []),
-   
self._store[name][0]['LabelValues'][0]['Value']['Value'])
-
-
-class MtailMetric(object):
-def __init__(self, keys, labels, value):
-self._keys = keys
-self._labels = labels
-self._value = value
-self._labelpairs = self.get_labelpairs(keys, labels)
-
-def get_labelpairs(self, keys, labels):
-res = []
-for k, v in zip(keys, labels):
-res.append('%s=%s' % (k, v))
-return res
+for metric in self._store[name][0]['LabelValues']:
+label_names = self._store[name][0].get('Keys', [])
+label_values = metric.get('Labels', [])
+value = metric['Value']['Value']
+labelpairs = ["%s=%s" % (k, v) for k, v in zip(label_names, 
label_values)]
+samples.append((','.join(labelpairs), value))

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add mcrouter module and mcrouter_wancache profile

[Aaron Schulz (Code Review)]

Aaron Schulz has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/392221 )

Change subject: [WIP] Add mcrouter module and mcrouter_wancache profile
..

[WIP] Add mcrouter module and mcrouter_wancache profile

Change-Id: I8b881a036c121c010f1f2c8e42ddbf266630c1b2
---
M hieradata/common.yaml
M hieradata/labs/deployment-prep/common.yaml
A modules/mcrouter/manifests/init.pp
A modules/mcrouter/manifests/monitoring.pp
A modules/mcrouter/templates/default.erb
A modules/profile/manifests/mediawiki/mcrouter_wancache.pp
M modules/role/manifests/deployment/mediawiki.pp
7 files changed, 238 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/21/392221/10

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 099761c..ef0a85e 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -528,3 +528,44 @@
 
 # HTTP proxy (can be used as an env variable)
 http_proxy: "http://webproxy.%{::site}.wmnet:8080;
+
+# Servers pools for multi-datacenter shared cache
+mediawiki_wancache_memcached_servers:
+  eqiad:
+- '10.64.0.80:11211:1'
+- '10.64.0.81:11211:1'
+- '10.64.0.82:11211:1'
+- '10.64.0.83:11211:1'
+- '10.64.0.84:11211:1'
+- '10.64.16.107:11211:1'
+- '10.64.16.108:11211:1'
+- '10.64.16.109:11211:1'
+- '10.64.16.110:11211:1'
+- '10.64.32.208:11211:1'
+- '10.64.32.209:11211:1'
+- '10.64.32.210:11211:1'
+- '10.64.32.211:11211:1'
+- '10.64.32.212:11211:1'
+- '10.64.48.155:11211:1'
+- '10.64.48.156:11211:1'
+- '10.64.48.157:11211:1'
+- '10.64.48.158:11211:1'
+  codfw:
+- '10.192.0.83:11211:1'
+- '10.192.0.84:11211:1'
+- '10.192.0.85:11211:1'
+- '10.192.0.86:11211:1'
+- '10.192.16.60:11211:1'
+- '10.192.16.61:11211:1'
+- '10.192.16.62:11211:1'
+- '10.192.16.194:11211:1'
+- '10.192.32.159:11211:1'
+- '10.192.32.160:11211:1'
+- '10.192.32.161:11211:1'
+- '10.192.32.162:11211:1'
+- '10.192.32.163:11211:1'
+- '10.192.48.76:11211:1'
+- '10.192.48.77:11211:1'
+- '10.192.48.78:11211:1'
+- '10.192.48.79:11211:1'
+- '10.192.48.80:11211:1'
\ No newline at end of file
diff --git a/hieradata/labs/deployment-prep/common.yaml 
b/hieradata/labs/deployment-prep/common.yaml
index 3c3fbe2..95ec636 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -148,6 +148,10 @@
 mediawiki_memcached_servers:
 - 10.68.23.25:11211:1  # deployment-memc04
 - 10.68.23.49:11211:1  # deployment-memc05
+mediawiki_wancache_memcached_servers:
+eqiad:
+  - 10.68.23.25:11211:1  # deployment-memc04
+  - 10.68.23.49:11211:1  # deployment-memc05
 profile::memcached::version: 'present'
 profile::memcached::growth_factor: '1.05'
 profile::memcached::size: '3000'
diff --git a/modules/mcrouter/manifests/init.pp 
b/modules/mcrouter/manifests/init.pp
new file mode 100644
index 000..9c316a9
--- /dev/null
+++ b/modules/mcrouter/manifests/init.pp
@@ -0,0 +1,104 @@
+# == Class: mcrouter
+#
+# mcrouter is a fast routing proxy for memcached.
+# It can reduce the connection count on the backend caching servers
+# and also supports layered pools, replication, and key/operation
+# based routing to pools.
+#
+# === Parameters
+#
+# [*pools*]
+#   A hash defining a mcrouter server pool.
+#   See .
+#
+# [*route*]
+#   A hash defining a mcrouter routing policy.
+#   See .
+#
+# [*region*]
+#   Datacenter name for the one in this geographical region
+#
+# [*cluster*]
+#   Memcached cluster name
+#
+# [*cross_region_timeout_ms*]
+#   Timeout, in milliseconds, when performing cross-region memcached operations
+#
+# [*cross_cluster_timeout_ms*]
+#   Timeout, in milliseconds, when performing cross-cluster memcached 
operations
+#
+# === Examples
+#
+#  class { '::mcrouter':
+#pools => {
+#  cluster-main' => {
+#servers => [ '10.68.23.25:11211', '10.68.23.49:11211' ]
+#  }
+#},
+#route => {
+#  type => 'OperationSelectorRoute',
+#  default_policy => 'PoolRoute|cluster-main',
+#  operation_policies => {
+#set => 'AllFastestRoute|Pool|cluster-main',
+#delete => 'AllSyncRoute|Pool|cluster-main'
+#  }
+#}
+#  }
+#
+class mcrouter(
+$pools,
+$route,
+$region,
+$cluster,
+$cross_region_timeout_ms,
+$cross_cluster_timeout_ms,
+$ensure= present
+) {
+validate_hash($pools)
+validate_hash($route)
+
+require_package('mcrouter')
+
+$config = { 'pools' => $pools, 'route' => $route }
+
+file { '/etc/mcrouter/mcrouter.json':
+ensure  => $ensure,
+content => ordered_json($config),
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+require => 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP Update RuboCop Ruby gem

[Zfilipin (Code Review)]

Zfilipin has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395522 )

Change subject: WIP Update RuboCop Ruby gem
..

WIP Update RuboCop Ruby gem

Bug: T180878
Change-Id: I05e1803ece1930df0f670d2a0031d325f2d1d1be
---
M Gemfile
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/22/395522/1

diff --git a/Gemfile b/Gemfile
index 01f6cfc..686a04b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,5 +11,5 @@
 gem 'rake', '~> 12.0.0'
 gem 'git', '1.3.0'
 gem 'puppet-lint', '2.0.2'
-gem 'rubocop', '~> 0.49.1', require: false
+gem 'rubocop', '~> 0.51.0', require: false
 gem 'puppet-lint-wmf_styleguide-check', '1.0.0'

-- 
To view, visit https://gerrit.wikimedia.org/r/395522
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I05e1803ece1930df0f670d2a0031d325f2d1d1be
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Zfilipin 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] php7 manifests for mediawiki on stretch

[ArielGlenn (Code Review)]

ArielGlenn has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394977 )

Change subject: [WIP] php7 manifests for mediawiki on stretch
..

[WIP] php7 manifests for mediawiki on stretch

Did I say [WIP]? Let me say it again:
[WIP] Draft of first bits, probably entirely wrong, placeholder, etc.
But we need somewhere to put this stuff.

Change-Id: Ib7e60fcd98efca8bca749139836125216996a4b9
---
M modules/mediawiki/manifests/packages.pp
A modules/mediawiki/manifests/packages/php7.pp
A modules/mediawiki/manifests/php7.pp
3 files changed, 79 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/77/394977/1

diff --git a/modules/mediawiki/manifests/packages.pp 
b/modules/mediawiki/manifests/packages.pp
index 1136abb..097669a 100644
--- a/modules/mediawiki/manifests/packages.pp
+++ b/modules/mediawiki/manifests/packages.pp
@@ -5,6 +5,9 @@
 if os_version('ubuntu == trusty') {
 include ::mediawiki::packages::php5
 }
+if os_version('debian == stretch') {
+include ::mediawiki::packages::php7
+}
 
 include ::mediawiki::packages::math
 include ::mediawiki::packages::tex
diff --git a/modules/mediawiki/manifests/packages/php7.pp 
b/modules/mediawiki/manifests/packages/php7.pp
new file mode 100644
index 000..541ac94
--- /dev/null
+++ b/modules/mediawiki/manifests/packages/php7.pp
@@ -0,0 +1,43 @@
+# === Class mediawiki::packages::php7
+#
+# This class declares packages that make up Wikimedia's PHP7-based
+# MediaWiki deployment stack. We'll be moving to this by mid-2018.
+# See T172165
+#
+class mediawiki::packages::php7 {
+# We don't need php-apc on php > 5.3
+package { 'php-apc':
+ensure => absent,
+}
+
+# Run-time
+package { [
+'php7.0-cli',
+'php7.0-common',
+'php-dbg',
+]:
+ensure => present,
+}
+
+# Wikimedia PHP extensions
+# THESE NEED TO BE BUILT AND TESTED
+package { [
+'php7-luasandbox',
+'php7-wikidiff2',
+]:
+ensure => present,
+}
+
+# Third-party PHP extensions
+package { [
+'php7.0-curl',
+'php-geoip',
+'php7.0-intl',
+'php-memcached',
+'php7.0-mysql',
+'php-redis',
+'php7.0-xmlrpc',
+]:
+ensure => present,
+}
+}
diff --git a/modules/mediawiki/manifests/php7.pp 
b/modules/mediawiki/manifests/php7.pp
new file mode 100644
index 000..e6d3a18
--- /dev/null
+++ b/modules/mediawiki/manifests/php7.pp
@@ -0,0 +1,33 @@
+# == Class: mediawiki::php
+#
+# Packages and .ini files for PHP7 extensions.
+#
+class mediawiki::php {
+include ::mediawiki::packages
+
+$php_module_conf_dir = '/etc/php7/mods-available'
+mediawiki::php_enmod { ['mail']: }
+
+file { '/etc/php7/apache2/php.ini':
+source  => 'puppet:///modules/mediawiki/php7/php.ini',
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+# libapache2-mod-php7 actually provides the /etc/php7/apache2
+# directory, but we only install it as a side effect of php-dbg.
+require => Package['php-dbg'],
+}
+
+file { '/etc/php7/cli/php.ini':
+source  => 'puppet:///modules/mediawiki/php/php.ini.cli',
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+require => Package['php7.0-cli'],
+}
+
+file { "${php_module_conf_dir}/mail.ini":
+ensure  => absent,
+require => Package['php-mail'],
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/394977
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib7e60fcd98efca8bca749139836125216996a4b9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] role::puppetmaster::puppetdb: add Prometheus monitorin...

[Elukey (Code Review)]

Elukey has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394966 )

Change subject: [WIP] role::puppetmaster::puppetdb: add Prometheus monitoring 
for puppetdb
..

[WIP] role::puppetmaster::puppetdb: add Prometheus monitoring for puppetdb

Change-Id: I58f036e85edb98ef4170580d093c42f0bc8ef786
---
A 
modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
A modules/profile/manifests/puppetmaster/puppetdb/monitoring.pp
M modules/puppetdb/manifests/app.pp
M modules/puppetdb/templates/puppetdb.service.erb
M modules/role/manifests/puppetmaster/puppetdb.pp
5 files changed, 32 insertions(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/66/394966/1

diff --git 
a/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
 
b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
new file mode 100644
index 000..73530b2
--- /dev/null
+++ 
b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git a/modules/profile/manifests/puppetmaster/puppetdb/monitoring.pp 
b/modules/profile/manifests/puppetmaster/puppetdb/monitoring.pp
new file mode 100644
index 000..7766ef2
--- /dev/null
+++ b/modules/profile/manifests/puppetmaster/puppetdb/monitoring.pp
@@ -0,0 +1,20 @@
+# Class: profile::puppetmaster::puppetdb::monitoring
+#
+# Sets up Prometheus based monitoring (only jvm) for the PuppetDB JVM.
+#
+class profile::puppetmaster::puppetdb::monitoring(
+$prometheus_nodes= hiera('prometheus_nodes'),
+) {
+# Prometheus JMX agent for the Puppetdb's JVM
+$jmx_exporter_config_file = 
'/etc/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml'
+$prometheus_jmx_exporter_port = 8181
+$prometheus_java_opts = 
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_port}:${jmx_exporter_config_file}"
+profile::prometheus::jmx_exporter { "puppetdb_${::hostname}":
+hostname => $::hostname,
+port => $prometheus_jmx_exporter_port,
+prometheus_nodes => $prometheus_nodes,
+config_file  => $jmx_exporter_config_file,
+source   => 
'puppet:///modules/profile/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml',
+require  => File['/etc/puppetdb'],
+}
+}
\ No newline at end of file
diff --git a/modules/puppetdb/manifests/app.pp 
b/modules/puppetdb/manifests/app.pp
index 8ef5a79..c58e205 100644
--- a/modules/puppetdb/manifests/app.pp
+++ b/modules/puppetdb/manifests/app.pp
@@ -11,7 +11,7 @@
 $db_user='puppetdb',
 $db_password=undef,
 $perform_gc=false,
-$heap_size='4G',
+$jvm_opts='-Xmx=4G',
 $bind_ip=undef,
 $ssldir=puppet_ssldir(),
 ) {
diff --git a/modules/puppetdb/templates/puppetdb.service.erb 
b/modules/puppetdb/templates/puppetdb.service.erb
index cef26bd..1f662c8 100644
--- a/modules/puppetdb/templates/puppetdb.service.erb
+++ b/modules/puppetdb/templates/puppetdb.service.erb
@@ -6,8 +6,8 @@
 Group=puppetdb
 Environment=CONFIG=/etc/puppetdb/conf.d
 ExecStartPre=/bin/bash -c "test -e /var/log/puppetdb/puppetdb-oom.hprof && mv 
/var/log/puppetdb/puppetdb-oom.hprof /var/log/puppetdb/puppetdb-oom.hprof.prev 
|| exit 0"
-ExecStart=/usr/bin/java -Xmx<%= @heap_size %> -XX:+ExitOnOutOfMemoryError \
--XX:+HeapDumpOnOutOfMemoryError 
-XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof 
-Djava.security.egd=file:/dev/urandom \
+ExecStart=/usr/bin/java ${jvm_opts} \
+-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError 
-XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof 
-Djava.security.egd=file:/dev/urandom \
 -cp /usr/share/puppetdb/puppetdb.jar clojure.main -m 
com.puppetlabs.puppetdb.core services -c ${CONFIG}
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=always
diff --git a/modules/role/manifests/puppetmaster/puppetdb.pp 
b/modules/role/manifests/puppetmaster/puppetdb.pp
index 5ae5b49..38dceb3 100644
--- a/modules/role/manifests/puppetmaster/puppetdb.pp
+++ b/modules/role/manifests/puppetmaster/puppetdb.pp
@@ -1,6 +1,6 @@
 # filtertags: labs-project-deployment-prep labs-project-automation-framework 
labs-project-toolsbeta
 class role::puppetmaster::puppetdb (
-$shared_buffers = '7680MB'
+$shared_buffers   = '7680MB'
 ) {
 include ::standard
 include ::base::firewall
@@ -85,9 +85,13 @@
 master => $master,
 }
 
+include ::profile::puppetmaster::puppetdb::monitoring
+$prometheus_java_opts = 
$::profile::puppetmaster::puppetdb::monitoring::prometheus_java_opts
+
 # The JVM heap size has been raised to 6G for T170740
 class { '::puppetmaster::puppetdb':
 master=> $master,
-heap_size => '6G',
+

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: varnish: prometheus equivalent of statsd metrics daemons

[Ema (Code Review)]

Ema has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394543 )

Change subject: WIP: varnish: prometheus equivalent of statsd metrics daemons
..

WIP: varnish: prometheus equivalent of statsd metrics daemons

Introduce the following mtail scripts:

- xcache.mtail (varnishxcache)
- resourceloader.mtail (varnishrls)
- media.mtail (varnishmedia)

Add and use mtail::script to install mtail scripts.

Bug: T177199
Change-Id: I31115573a5d7f43268eef3a1bcee92e18d5fa957
---
A modules/mtail/manifests/script.pp
A modules/varnish/files/mtail/media.mtail
A modules/varnish/files/mtail/resourceloader.mtail
A modules/varnish/files/mtail/xcache.mtail
M modules/varnish/manifests/logging/media.pp
M modules/varnish/manifests/logging/rls.pp
M modules/varnish/manifests/logging/xcache.pp
7 files changed, 70 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/43/394543/1

diff --git a/modules/mtail/manifests/script.pp 
b/modules/mtail/manifests/script.pp
new file mode 100644
index 000..b544b80
--- /dev/null
+++ b/modules/mtail/manifests/script.pp
@@ -0,0 +1,36 @@
+# Definition: mtail::script
+#
+# Install the given mtail script under the mtail scripts directory. There is no
+# need to notify any running mtail instance, changes are picked up by mtail
+# in a automatic fashion upon file modification.
+#
+# Parameters
+#   $source
+#   The file containing the mtail script to be installed (required).
+#   $destdir
+#   Destination directory. Defaults to /usr/local/share/mtail.
+#
+# Usage example:
+#   mtail::script { 'xcache':
+#   source => 'puppet:///modules/varnish/mtail/xcache.mtail',
+#   }
+#
+define mtail::script($source, $destdir='/usr/local/share/mtail') {
+validate_string($source)
+validate_absolute_path($destdir)
+
+file { $destdir:
+ensure => directory,
+owner  => 'root',
+group  => 'root',
+mode   => '0444',
+}
+
+file { "${destdir}/${title}.mtail":
+source  => $source,
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+require => File[$destdir],
+}
+}
diff --git a/modules/varnish/files/mtail/media.mtail 
b/modules/varnish/files/mtail/media.mtail
new file mode 100644
index 000..a84f184
--- /dev/null
+++ b/modules/varnish/files/mtail/media.mtail
@@ -0,0 +1,5 @@
+counter varnish_thumbnails by status
+
+/^url \/thumb\/.*\thttp_status (?P.*)\tcache_control/ {
+varnish_thumbnails[$http_status]++
+}
diff --git a/modules/varnish/files/mtail/resourceloader.mtail 
b/modules/varnish/files/mtail/resourceloader.mtail
new file mode 100644
index 000..0705e44
--- /dev/null
+++ b/modules/varnish/files/mtail/resourceloader.mtail
@@ -0,0 +1,7 @@
+counter varnish_resourceloader_inm
+
+/^url \/w\/load.php.*\tinm (?P.*)$/ {
+$inm != "-" {
+varnish_resourceloader_inm++
+}
+}
diff --git a/modules/varnish/files/mtail/xcache.mtail 
b/modules/varnish/files/mtail/xcache.mtail
new file mode 100644
index 000..df4a197
--- /dev/null
+++ b/modules/varnish/files/mtail/xcache.mtail
@@ -0,0 +1,5 @@
+counter varnish_x_cache by x_cache
+
+/^.*\tcache_status (?P.*)\thttp_status/ {
+varnish_x_cache[$x_cache]++
+}
diff --git a/modules/varnish/manifests/logging/media.pp 
b/modules/varnish/manifests/logging/media.pp
index a86e001..e9f7e33 100644
--- a/modules/varnish/manifests/logging/media.pp
+++ b/modules/varnish/manifests/logging/media.pp
@@ -1,7 +1,7 @@
 # == Define: varnish::logging::media
 #
 #  Accumulate browser cache hit ratio and total request volume statistics
-#  for Media requests and report to StatsD.
+#  for Media requests and report to StatsD. Expose metrics to prometheus.
 #
 # === Parameters
 #
@@ -42,4 +42,8 @@
 description  => 'Varnish traffic logger - varnishmedia',
 nrpe_command => '/usr/lib/nagios/plugins/check_procs -w 1:1 -a 
"/usr/local/bin/varnishmedia" -u root',
 }
+
+mtail::script { 'media':
+source => 'puppet:///modules/varnish/mtail/media.mtail',
+}
 }
diff --git a/modules/varnish/manifests/logging/rls.pp 
b/modules/varnish/manifests/logging/rls.pp
index 9ee16e7..4c4629a 100644
--- a/modules/varnish/manifests/logging/rls.pp
+++ b/modules/varnish/manifests/logging/rls.pp
@@ -1,7 +1,8 @@
 # == Define: varnish::logging::rls
 #
-#  Accumulate browser cache hit ratio and total request volume statistics
-#  for ResourceLoader requests (/w/load.php) and report to StatsD.
+#  Accumulate browser cache hit ratio and total request volume statistics for
+#  ResourceLoader requests (/w/load.php) and report to StatsD. Expose metrics
+#  to prometheus.
 #
 # === Parameters
 #
@@ -43,4 +44,8 @@
 description  => 'Varnish traffic logger - varnishrls',
 nrpe_command => '/usr/lib/nagios/plugins/check_procs -w 1:1 -a 
"/usr/local/bin/varnishrls" -u root',
 }
+
+mtail::script { 'resourceloader':

[MediaWiki-commits] [Gerrit] operations/puppet[production]: wip: toolforge: follow attended upgrade process

[Rush (Code Review)]

Rush has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394200 )

Change subject: wip: toolforge: follow attended upgrade process
..

wip: toolforge: follow attended upgrade process

relies on changeset 392421

Bug: T181647
Change-Id: Id9e95a58860a0ef786324d62020347bf01a36111
---
M hieradata/labs.yaml
M hieradata/labs/tools/common.yaml
M modules/apt/manifests/unattendedupgrades.pp
M modules/profile/manifests/base/labs.pp
4 files changed, 25 insertions(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/00/394200/1

diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index 9cbccbf..f5582bf 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -17,6 +17,8 @@
   recursor: 'labs-recursor0.wikimedia.org'
   recursor_secondary: 'labs-recursor1.wikimedia.org'
 
+profile::base::labs::unattended_distro: true
+profile::base::labs::unattended_wmf: true
 profile::openstack::main::version: 'liberty'
 profile::openstack::base::region: "%{::site}"
 profile::openstack::main::nova_controller: 'labcontrol1001.wikimedia.org'
diff --git a/hieradata/labs/tools/common.yaml b/hieradata/labs/tools/common.yaml
index c62e87a..6e5eb3f 100644
--- a/hieradata/labs/tools/common.yaml
+++ b/hieradata/labs/tools/common.yaml
@@ -1,3 +1,6 @@
+profile::base::labs::unattended_distro: false
+profile::base::labs::unattended_wmf: false
+
 "profile::base::core_dump_pattern": core
 classes:
 - role::aptly::client
diff --git a/modules/apt/manifests/unattendedupgrades.pp 
b/modules/apt/manifests/unattendedupgrades.pp
index c02745c..41fafe4 100644
--- a/modules/apt/manifests/unattendedupgrades.pp
+++ b/modules/apt/manifests/unattendedupgrades.pp
@@ -1,13 +1,17 @@
-class apt::unattendedupgrades($ensure=present) {
+class apt::unattendedupgrades(
+$unattended_distro=true,
+$unattended_wmf=true,
+) {
+
 # package installation should enable security upgrades by default
 package { 'unattended-upgrades':
-ensure => $ensure,
+ensure => 'present',
 }
 
 # dpkg tries to determine the most conservative default action in case of
 # conffile conflict. This tells dpkg to use that action without asking
 apt::conf { 'dpkg-force-confdef':
-ensure   => present,
+ensure   => 'present',
 priority => '00',
 key  => 'Dpkg::Options::',
 value=> '--force-confdef',
@@ -16,20 +20,21 @@
 # In case of conffile conflicts, tell dpkg to keep the old conffile without
 # asking
 apt::conf { 'dpkg-force-confold':
-ensure   => present,
+ensure   => 'present',
 priority => '00',
 key  => 'Dpkg::Options::',
 value=> '--force-confold',
 }
 
 apt::conf { 'auto-upgrades':
-ensure   => $ensure,
+ensure   => $unattended_distro,
 priority => '20',
 key  => 'APT::Periodic::Unattended-Upgrade',
 value=> '1',
 }
 
 apt::conf { 'unattended-upgrades-wikimedia':
+ensure   => $unattended_wmf,
 priority => '51',
 # Key with trailing '::' to append to potentially existing entry
 key  => 'Unattended-Upgrade::Origins-Pattern::',
diff --git a/modules/profile/manifests/base/labs.pp 
b/modules/profile/manifests/base/labs.pp
index 23816b3..c028c3f 100644
--- a/modules/profile/manifests/base/labs.pp
+++ b/modules/profile/manifests/base/labs.pp
@@ -1,4 +1,13 @@
-class profile::base::labs {
+class profile::base::labs(
+$unattended_distro = hiera('profile::base::labs::unattended_distro'),
+$unattended_wmf = hiera('profile::base::labs::unattended_wmf),
+) {
+
+class {'::apt::unattendedupgrades':
+unattended_distro => $unattended_distro,
+unattended_wmf=> $unattended_wmf,
+}
+
 include ::apt::unattendedupgrades
 include ::apt::noupgrade
 

-- 
To view, visit https://gerrit.wikimedia.org/r/394200
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id9e95a58860a0ef786324d62020347bf01a36111
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] profile::hadoop::common: add Prometheus JMX exporter c...

[Elukey (Code Review)]

Elukey has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394045 )

Change subject: [WIP] profile::hadoop::common: add Prometheus JMX exporter 
configuration
..

[WIP] profile::hadoop::common: add Prometheus JMX exporter configuration

Bug: T177458
Change-Id: Ida0f54b30a5d12210fb990a2bd36f51ce38fd981
---
M hieradata/common/profile/hadoop/common.yaml
A modules/profile/files/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml
M modules/profile/manifests/hadoop/common.pp
A modules/profile/manifests/hadoop/monitoring/datanode.pp
4 files changed, 36 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/45/394045/1

diff --git a/hieradata/common/profile/hadoop/common.yaml 
b/hieradata/common/profile/hadoop/common.yaml
index 15a2b37..3299c50 100644
--- a/hieradata/common/profile/hadoop/common.yaml
+++ b/hieradata/common/profile/hadoop/common.yaml
@@ -33,3 +33,5 @@
 - /var/lib/hadoop/data/k
 - /var/lib/hadoop/data/l
 - /var/lib/hadoop/data/m
+
+profile::hadoop::common::monitoring_enabled: true
diff --git 
a/modules/profile/files/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml 
b/modules/profile/files/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml
new file mode 100644
index 000..73530b2
--- /dev/null
+++ b/modules/profile/files/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/common.pp 
b/modules/profile/manifests/hadoop/common.pp
index 7629982..c05041a 100644
--- a/modules/profile/manifests/hadoop/common.pp
+++ b/modules/profile/manifests/hadoop/common.pp
@@ -108,6 +108,9 @@
 #  [*yarn_scheduler_maximum_allocation_vcores*]
 #Yarn scheduler specific setting. Default: 32
 #
+#  [*monitoring_enabled*]
+#Configure the Prometheus jmx exporter for all the JVM daemons.
+#
 class profile::hadoop::common (
 $zookeeper_clusters   = hiera('zookeeper_clusters'),
 $zookeeper_cluster_name   = 
hiera('profile::hadoop::common::zookeeper_cluster_name'),
@@ -137,6 +140,7 @@
 $yarn_scheduler_maximum_allocation_mb = 
hiera('profile::hadoop::common::yarn_scheduler_maximum_allocation_mb', 57344),
 $yarn_scheduler_minimum_allocation_vcores = 
hiera('profile::hadoop::common::yarn_scheduler_minimum_allocation_vcores', 0),
 $yarn_scheduler_maximum_allocation_vcores = 
hiera('profile::hadoop::common::yarn_scheduler_maximum_allocation_vcores', 32),
+$monitoring_enabled   = 
hiera('profile::hadoop::common::monitoring_enabled'),
 ) {
 # Include Wikimedia's thirdparty/cloudera apt component
 # as an apt source on all Hadoop hosts.  This is needed
@@ -158,6 +162,14 @@
 
 $zookeeper_hosts = 
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
 
+
+if $monitoring_enabled {
+include ::profile::hadoop::monitoring::datanode
+$hadoop_datanode_opts_prometheus = 
$::profile::hadoop::monitoring::datanode::java_opts
+} else {
+$hadoop_datanode_opts_prometheus = ''
+}
+
 class { '::cdh::hadoop':
 # Default to using running resourcemanager on the same hosts
 # as the namenodes.
@@ -178,7 +190,7 @@
 yarn_nodemanager_opts   => $yarn_nodemanager_opts,
 yarn_resourcemanager_opts   => 
$yarn_resourcemanager_opts,
 hadoop_namenode_opts=> $hadoop_namenode_opts,
-hadoop_datanode_opts=> $hadoop_datanode_opts,
+hadoop_datanode_opts=> 
"${hadoop_datanode_opts} ${hadoop_datanode_opts_prometheus}",
 mapreduce_history_java_opts => 
$mapreduce_history_java_opts,
 
 yarn_app_mapreduce_am_resource_mb   => 
$yarn_app_mapreduce_am_resource_mb,
diff --git a/modules/profile/manifests/hadoop/monitoring/datanode.pp 
b/modules/profile/manifests/hadoop/monitoring/datanode.pp
new file mode 100644
index 000..c3cbfa4
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/datanode.pp
@@ -0,0 +1,18 @@
+# Class: profile::hadoop::monitoring::datanode
+#
+# Sets up Prometheus based monitoring for the Hadoop HDFS Datanode
+#
+class profile::hadoop::monitoring::datanode(
+$prometheus_nodes= hiera('prometheus_nodes'),
+) {
+$jmx_exporter_config_file = 
'/etc/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml'
+$prometheus_jmx_exporter_datanode_port = 51010
+$java_opts = 
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_datanode_port}:${jmx_exporter_config_file}"
+profile::prometheus::jmx_exporter { "hdfs_datanode_${::hostname}":
+hostname => $::hostname,
+port => 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] puppet: redirect several wikis per LangCom decission

[MarcoAurelio (Code Review)]

MarcoAurelio has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/393289 )

Change subject: [WIP] puppet: redirect several wikis per LangCom decission
..

[WIP] puppet: redirect several wikis per LangCom decission

Domains affected:

* mo.wikipedia.org   -> ro.wikipedia.org
* mo.wiktionary.org  -> ro.wiktionary.org
* als.wiktionary.org -> https://als.wikipedia.org/wiki/Wort:Houptsyte
* als.wikiquote.org  -> https://als.wikipedia.org/wiki/Spruch:Houptsyte
* als.wikibooks.org  -> https://als.wikipedia.org/wiki/Buech:Houptsyte

Bug: T169450
Change-Id: I23078c8fd643f0db171d607c1968b9bf7453fae7
---
0 files changed, 0 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/89/393289/1


-- 
To view, visit https://gerrit.wikimedia.org/r/393289
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I23078c8fd643f0db171d607c1968b9bf7453fae7
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: MarcoAurelio 
Gerrit-Reviewer: jenkins-bot <>

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WiP] Move puppet CI to puppet 4.8.2

[Giuseppe Lavagetto (Code Review)]

Giuseppe Lavagetto has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/393259 )

Change subject: [WiP] Move puppet CI to puppet 4.8.2
..

[WiP] Move puppet CI to puppet 4.8.2

Change-Id: I9679b14f6bb861603d22e2d2cc5d32e7ccc50a95
---
M Gemfile
M modules/apt/spec/classes/apt_spec.rb
M modules/authdns/spec/classes/authdns_spec.rb
M modules/authdns/spec/spec_helper.rb
M modules/git/spec/defines/clone_spec.rb
M modules/install_server/spec/classes/install_server_dhcp_server_spec.rb
M modules/install_server/spec/classes/install_server_web_server_spec.rb
M modules/install_server/spec/spec_helper.rb
M modules/jenkins/spec/classes/jenkins_spec.rb
M modules/jenkins/spec/hosts/master_and_slave_spec.rb
M modules/jenkins/spec/spec_helper.rb
M modules/mirrors/spec/classes/mirrors_debian_spec.rb
M modules/mirrors/spec/classes/mirrors_ubuntu_spec.rb
D modules/mirrors/spec/fixtures/manifests/site.pp
A modules/mirrors/spec/fixtures/modules/passwords/manifests/mirrors.pp
M modules/monitoring/manifests/host.pp
M modules/monitoring/spec/defines/monitoring_host_spec.rb
M rake_modules/taskgen.rb
18 files changed, 127 insertions(+), 44 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/59/393259/1

diff --git a/Gemfile b/Gemfile
index f5e07ad..871d725 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,12 +1,11 @@
 source 'https://rubygems.org'
 
-gem 'puppet', ENV['PUPPET_GEM_VERSION'] || '~> 3.8.5'
+gem 'puppet', ENV['PUPPET_GEM_VERSION'] || '4.9.2'
 gem 'xmlrpc' if RUBY_VERSION >= '2.4.0'
 gem 'puppet-strings', '~> 1.0.0'
-gem 'rspec-puppet', '~> 2.5.0'
+gem 'rspec-puppet', '~> 2.6.9'
+gem 'rspec-puppet-facts', '~> 1.7', :require => false
 gem 'puppetlabs_spec_helper', '< 2.0.0'
-# Puppet 3.7 fails on ruby 2.2+
-# https://tickets.puppetlabs.com/browse/PUP-3796
 gem 'safe_yaml', '~> 1.0.4'
 
 gem 'rake', '~> 12.0.0'
diff --git a/modules/apt/spec/classes/apt_spec.rb 
b/modules/apt/spec/classes/apt_spec.rb
index ccbb7f6..cfb89c5 100644
--- a/modules/apt/spec/classes/apt_spec.rb
+++ b/modules/apt/spec/classes/apt_spec.rb
@@ -1,22 +1,25 @@
 require 'spec_helper'
 
 describe 'apt' do
+
 os = [
 {
 :lsbdistid => 'Debian',
 :lsbdistrelease => '8.0',
 :operatingsystem => 'Debian',
+:lsbdistcodename => 'jessie'
 },
 {
 :lsbdistid => 'Ubuntu',
 :lsbdistrelease => '14.04',
 :operatingsystem => 'Ubuntu',
+:lsbdistcodename => 'trusty'
 },
 ]
 os.each do |os_facts|
-context "with OS #{os_facts[:lsbdistid]} #{os_facts[:lsbdistrelease]}" 
do
+  context "with OS #{os_facts[:lsbdistid]} #{os_facts[:lsbdistrelease]}" do
 let(:facts) { os_facts }
-
+let(:node_params) { {'site' => 'eqiad'} }
 it { should compile }
 
 context "when not using a proxy" do
diff --git a/modules/authdns/spec/classes/authdns_spec.rb 
b/modules/authdns/spec/classes/authdns_spec.rb
index 0e7595f..4c4bddb 100644
--- a/modules/authdns/spec/classes/authdns_spec.rb
+++ b/modules/authdns/spec/classes/authdns_spec.rb
@@ -1,7 +1,20 @@
 require 'spec_helper'
+test_on = {
+  supported_os: [
+{
+  'operatingsystem'=> 'Debian',
+  'operatingsystemrelease' => ['8'], # we cannot support stretch atm 
because of a bug in the service provider in
+  # the puppet gem
+}
+  ]
+}
 
 describe 'authdns' do
-let(:node) { 'testhost.eqiad.wmnet' }
+  let(:node) { 'testhost.eqiad.wmnet' }
+
+  on_supported_os(test_on).each do |os, facts|
+facts[:initsystem] = 'systemd'
+let(:facts) { facts }
 let(:params) { {
 :lvs_services => {},
 :discovery_services => {},
@@ -14,9 +27,12 @@
 'class confd($prefix) {}',
 'package{ "git": }',
 ] }
-it { should compile }
+it { is_expected.to compile.with_all_deps  }
+  end
 end
 
 describe 'authdns::lint' do
+  on_supported_os(test_on).each do |os, facts|
 it { should compile }
+  end
 end
diff --git a/modules/authdns/spec/spec_helper.rb 
b/modules/authdns/spec/spec_helper.rb
index df9590b..7464689 100644
--- a/modules/authdns/spec/spec_helper.rb
+++ b/modules/authdns/spec/spec_helper.rb
@@ -1,6 +1,8 @@
 require 'rspec-puppet'
 require 'puppetlabs_spec_helper/module_spec_helper'
+require 'rspec-puppet-facts'
 
+include RspecPuppetFacts
 fixture_path = File.expand_path(File.join(__FILE__, '..', 'fixtures'))
 
 RSpec.configure do |c|
diff --git a/modules/git/spec/defines/clone_spec.rb 
b/modules/git/spec/defines/clone_spec.rb
index 20d2271..4a81bfa 100644
--- a/modules/git/spec/defines/clone_spec.rb
+++ b/modules/git/spec/defines/clone_spec.rb
@@ -9,7 +9,7 @@
 } }
 it 'checkouts a workspace' do
 should contain_exec('git_clone_operations/puppet')
-.without_command(/ --bare /)
+ .with_command('/usr/bin/git  

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: cache: size-based cutoff for exp caching policy

[Ema (Code Review)]

Ema has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/393227 )

Change subject: WIP: cache: size-based cutoff for exp caching policy
..

WIP: cache: size-based cutoff for exp caching policy

- Limit the exp caching policy to Varnish 5 (v4 has no HFM)
- Unconditionally return HFM for exp
- Add a size-based HFP cutoff

Bug: T144187
Change-Id: I5a326e128153af9e3f21840eaf53164a4eb586d6
---
M modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
2 files changed, 18 insertions(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/27/393227/1

diff --git a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb 
b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
index a4d32e0..bcbd68c 100644
--- a/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-common.inc.vcl.erb
@@ -447,6 +447,22 @@
return (deliver);
<%- end -%>
}
+
+<% if @varnish_version == 5 && @vcl_config.fetch("admission_policy", "nhw") == 
"exp" -%>
+   // XXX: this should most likely not be defined here, as the exp policy 
need
+   // to be applied on the frontend layer only
+
+   // We want to apply the "exp" caching policy only to objects with CL 
below
+   // 10M (but present). The "exp" caching policy is defined in
+   // wikimedia-frontend.vcl.erb, but the logic below can not be added 
there
+   // because by returning HFP we would skip calling various VCL 
subroutines such
+   // as cluster_fe_backend_response_early.
+   if (beresp.status == 200 && bereq.http.X-CDIS == "miss"
+   && std.integer(beresp.http.Content-Length, 0) >= 1024 * 1024 * 
10) {
+   // HFP
+   return(pass(120s));
+   }
+<%- end -%>
 }
 
 // call just before wm_common_xcache_deliver, but only in vcl_deliver, not 
vcl_synth
diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb 
b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
index a3f5826..7ea9c50 100644
--- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
@@ -2,7 +2,7 @@
 
 vcl 4.0;
 
-<% if @vcl_config.fetch("admission_policy", "nhw") == "exp" -%>
+<% if @varnish_version == 5 && @vcl_config.fetch("admission_policy", "nhw") == 
"exp" -%>
 // Includes for Exp cache admission policy, admission probability exponentially
 // decreasing with size. See vcl_backend_response. T144187
 C{
@@ -358,7 +358,7 @@
if (beresp.status == 503 && bereq.retries == 0 && bereq.method ~ 
"^(GET|HEAD|OPTIONS|PUT|DELETE)$") {
return(retry);
}
-<% if @vcl_config.fetch("admission_policy", "nhw") == "exp" -%>
+<% if @varnish_version == 5 && @vcl_config.fetch("admission_policy", "nhw") == 
"exp" -%>
 if (beresp.status == 200 && bereq.http.X-CDIS == "miss") {
 C{
const struct gethdr_s hdr = { HDR_BERESP, "\017Content-Length:" };
@@ -391,12 +391,8 @@
 
// If admission test succeeds, mark as uncacheable
if (admissionprob < urand) {
-   <%- if @varnish_version == 5 -%>
// HFM with ttl=67 to avoid stalling
VRT_l_beresp_ttl(ctx,67);
-   <%- else -%>
-   VRT_l_beresp_ttl(ctx,0);
-   <%- end -%>
VRT_l_beresp_uncacheable(ctx,1);
}
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/393227
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5a326e128153af9e3f21840eaf53164a4eb586d6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]mariadb: Move hosts to s8 replica set on codfw

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/393065 )

Change subject: [WIP]mariadb: Move hosts to s8 replica set on codfw
..

[WIP]mariadb: Move hosts to s8 replica set on codfw

Bug: T177208
Change-Id: I15e8315a8d611b7d77e3499c342e5c7dcf09f97d
---
M hieradata/hosts/db2038.yaml
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/65/393065/1

diff --git a/hieradata/hosts/db2038.yaml b/hieradata/hosts/db2038.yaml
index 0d8bce2..3ecb13a 100644
--- a/hieradata/hosts/db2038.yaml
+++ b/hieradata/hosts/db2038.yaml
@@ -1 +1 @@
-mariadb::shard: 's5'
+mariadb::shard: 's8'

-- 
To view, visit https://gerrit.wikimedia.org/r/393065
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I15e8315a8d611b7d77e3499c342e5c7dcf09f97d
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Puppetization for superset

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/392978 )

Change subject: [WIP] Puppetization for superset
..

[WIP] Puppetization for superset

Bug: T166689
Change-Id: Ib63b35e409ba4c8fbc77f342220f1f4c4f6fecea
---
A modules/superset/files/superset.profile.firejail
A modules/superset/manifests/init.pp
A modules/superset/templates/gunicorn_config.py.erb
A modules/superset/templates/initscripts/superset.systemd.erb
A modules/superset/templates/superset_config.py.erb
5 files changed, 187 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/78/392978/1

diff --git a/modules/superset/files/superset.profile.firejail 
b/modules/superset/files/superset.profile.firejail
new file mode 100644
index 000..acd4e84
--- /dev/null
+++ b/modules/superset/files/superset.profile.firejail
@@ -0,0 +1,26 @@
+# system directories
+blacklist /sbin
+blacklist /usr/sbin
+blacklist /usr/local/sbin
+
+# system management
+blacklist ${PATH}/umount
+blacklist ${PATH}/mount
+blacklist ${PATH}/fusermount
+blacklist ${PATH}/su
+blacklist ${PATH}/sudo
+blacklist ${PATH}/xinput
+blacklist ${PATH}/evtest
+blacklist ${PATH}/xev
+blacklist ${PATH}/strace
+blacklist ${PATH}/nc
+blacklist ${PATH}/ncat
+
+blacklist /etc/shadow
+blacklist /etc/ssh
+blacklist /root
+blacklist /home
+noroot
+caps.drop all
+seccomp
+private-dev
diff --git a/modules/superset/manifests/init.pp 
b/modules/superset/manifests/init.pp
new file mode 100644
index 000..4307ada
--- /dev/null
+++ b/modules/superset/manifests/init.pp
@@ -0,0 +1,93 @@
+# == Class superset
+#
+class superset(
+$port  = 9080,
+$statsd= undef,
+$database_uri  = 'sqlite:tmp/superset.db',
+$password_mapping  = undef,
+$deployment_user   = 'analytics_deploy',
+$scap_repo = 'analytics/superset/deploy',
+$contact_group = 'admins',
+) {
+requires_os('debian >= jessie')
+require_package('python', 'virtualenv', 'firejail')
+
+$scap_deployment_base_dir = '/srv/deployment'
+$deployment_dir = "${scap_deployment_base_dir}/${scap_repo}"
+$virtualenv_dir = dirname($deployment_dir) + "/virtualenv"
+
+scap::target { 'analytics/superset/deploy':
+deploy_user  => $deployment_user,
+service_name => 'superset',
+}
+
+group { 'superset':
+ensure => present,
+system => true,
+}
+
+user { 'superset':
+gid => 'superset',
+shell   => '/bin/bash',
+system  => true,
+require => Group['superset'],
+}
+
+file { '/etc/firejail/superset.profile':
+ensure => present,
+owner  => 'root',
+group  => 'root',
+mode   => '0444',
+source => 'puppet:///modules/superset/superset.profile.firejail',
+}
+
+file { '/etc/superset':
+ensure => directory,
+owner  => 'root',
+group  => 'root',
+mode   => '0755',
+}
+
+file { '/etc/superset/gunicorn_config.py':
+ensure  => 'present',
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+content => template('superset/gunicorn_config.py.erb'),
+}
+
+file { '/etc/superset/superset_config.py':
+ensure  => 'present',
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+content => template('superset/superset_config.py.erb'),
+}
+
+systemd::syslog { 'superset':
+readable_by => 'all',
+base_dir=> '/var/log',
+group   => 'root',
+}
+
+systemd::service { 'superset':
+ensure  => 'present',
+content => systemd_template('superset'),
+restart => true,
+require => [
+Scap::Target['analytics/superset/deploy'],
+File['/etc/firejail/superset.profile'],
+File['/etc/superset/gunicorn_config.yaml'],
+File['/etc/superset/superset_config.yaml'],
+User['superset'],
+Systemd::Syslog['pivot'],
+],
+}
+
+monitoring::service { 'superset':
+description   => 'superset',
+check_command => "check_tcp!${port}",
+contact_group => $contact_group,
+require   => Systemd::Service['superset'],
+}
+}
diff --git a/modules/superset/templates/gunicorn_config.py.erb 
b/modules/superset/templates/gunicorn_config.py.erb
new file mode 100644
index 000..31cb4b2
--- /dev/null
+++ b/modules/superset/templates/gunicorn_config.py.erb
@@ -0,0 +1,7 @@
+bind = '0.0.0.0:<%= @port %>'
+workers = 4
+worker_class = 'gevent'
+timeout = 120
+<% if @statsd -%>
+statsd_host <%= @statsd %> \
+<% end -%>
diff --git a/modules/superset/templates/initscripts/superset.systemd.erb 
b/modules/superset/templates/initscripts/superset.systemd.erb
new file mode 100644
index 000..62549c4
--- /dev/null
+++ 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] eqsin: cache/lvs/dns/bast site.pp

[BBlack (Code Review)]

BBlack has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/392639 )

Change subject: [WIP] eqsin: cache/lvs/dns/bast site.pp
..

[WIP] eqsin: cache/lvs/dns/bast site.pp

dns500[12] should be here too, but we haven't even sorted out the
software config for dns400[12] yet...

Change-Id: Iebda6bd23697da864f38adfcac59eae95864f6a4
---
M manifests/site.pp
1 file changed, 37 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/392639/1

diff --git a/manifests/site.pp b/manifests/site.pp
index e9c71bd..370797b 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -152,6 +152,19 @@
 }
 }
 
+node 'bast5001.wikimedia.org' {
+role(bastionhost::general,
+ipmi::mgmt,
+installserver::tftp,
+prometheus::ops)
+
+interface::add_ip6_mapped { 'main': }
+
+class { '::ganglia::monitor::aggregator':
+sites =>  'eqsin',
+}
+}
+
 node 'bohrium.eqiad.wmnet' {
 role(piwik::server)
 }
@@ -303,6 +316,20 @@
 }
 
 node /^cp40(2[789]|3[012])\.ulsfo\.wmnet$/ {
+interface::add_ip6_mapped { 'main': }
+role(cache::text, ipsec)
+}
+
+#
+# eqsin varnishes
+#
+
+node /^cp500[1-6]\.eqsin\.wmnet$/ {
+interface::add_ip6_mapped { 'main': }
+role(cache::upload, ipsec)
+}
+
+node /^cp50(0[789]|1[012])\.eqsin\.wmnet$/ {
 interface::add_ip6_mapped { 'main': }
 role(cache::text, ipsec)
 }
@@ -1502,6 +1529,16 @@
 }
 }
 
+# EQSIN lvs servers
+node /^lvs500[123]\.eqsin\.wmnet$/ {
+# ns override for all lvs for now, see T103921
+$nameservers_override = [ '208.80.153.12', '208.80.153.42', 
'208.80.154.254' ]
+
+role(lvs::balancer)
+lvs::interface_tweaks {
+'eth0': bnx2x => true, txqlen => 1;
+}
+}
 
 node 'maerlant.wikimedia.org' {
 role(dnsrecursor, ntp)

-- 
To view, visit https://gerrit.wikimedia.org/r/392639
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iebda6bd23697da864f38adfcac59eae95864f6a4
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] [logstash] Add a way to move some data to debug_blob

[DCausse (Code Review)]

DCausse has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/392591 )

Change subject: [WIP] [logstash] Add a way to move some data to debug_blob
..

[WIP] [logstash] Add a way to move some data to debug_blob

NOTE: do not work with logstash 5.5.
(5.6 is required)

Bug: T180051
Change-Id: Id0e8d3b16c55a937096aef9ad9a81865cf17b544
---
A modules/role/files/logstash/filter-debug_blob.conf
M modules/role/manifests/logstash/collector.pp
2 files changed, 14 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/91/392591/1

diff --git a/modules/role/files/logstash/filter-debug_blob.conf 
b/modules/role/files/logstash/filter-debug_blob.conf
new file mode 100644
index 000..938d967
--- /dev/null
+++ b/modules/role/files/logstash/filter-debug_blob.conf
@@ -0,0 +1,9 @@
+filter {
+  if "es" in [tags] {
+if [channel] == "EventBus" {
+  mutate {
+rename => { "[events][params]" => 
"[debug_blob][eventbus][events][params]" }
+  }
+}
+  }
+}
diff --git a/modules/role/manifests/logstash/collector.pp 
b/modules/role/manifests/logstash/collector.pp
index 7113d5a..941472f 100644
--- a/modules/role/manifests/logstash/collector.pp
+++ b/modules/role/manifests/logstash/collector.pp
@@ -181,6 +181,11 @@
 priority => 70,
 }
 
+logstash::conf { 'filter_debug_blob':
+source   => 'puppet:///modules/role/logstash/filter-debug_blob.conf',
+priority => 70,
+}
+
 ## Outputs (90)
 # Template for Elasticsearch index creation
 file { '/etc/logstash/elasticsearch-template.json':

-- 
To view, visit https://gerrit.wikimedia.org/r/392591
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id0e8d3b16c55a937096aef9ad9a81865cf17b544
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: DCausse 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] profile::redids::jobqueue: stagger redis slave restarts

[Elukey (Code Review)]

Elukey has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391798 )

Change subject: [WIP] profile::redids::jobqueue: stagger redis slave restarts
..

[WIP] profile::redids::jobqueue: stagger redis slave restarts

Change-Id: I58f1fb4b16f5947eecd0f89b075471e335e45de6
---
M modules/profile/manifests/redis/jobqueue.pp
M modules/profile/manifests/redis/jobqueue_slave.pp
2 files changed, 6 insertions(+), 21 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/98/391798/1

diff --git a/modules/profile/manifests/redis/jobqueue.pp 
b/modules/profile/manifests/redis/jobqueue.pp
index da37315..d2a00e2 100644
--- a/modules/profile/manifests/redis/jobqueue.pp
+++ b/modules/profile/manifests/redis/jobqueue.pp
@@ -1,23 +1,3 @@
-# Very simple profile for redis for the MW jobqueue. It works as an addition to
-# profile::redis::multidc
-# This is basically to cope with issues described in
-# https://phabricator.wikimedia.org/T163337 with a ugly workaround: restart
-# periodically the redis slaves in order to force a
-# service restart
 class profile::redis::jobqueue {
 require ::profile::redis::multidc
-file { '/usr/local/bin/restart-redis-if-slave':
-ensure => present,
-source => 'puppet:///modules/profile/redis/restart-redis-if-slave.sh',
-mode   => '0555',
-owner  => 'root',
-group  => 'root',
-}
-
-$instance_str = join($::profile::redis::multidc::instances, ' ')
-cron { 'jobqueue-redis-conditional-restart':
-command => "/usr/local/bin/restart-redis-if-slave ${instance_str}",
-hour=> 1,
-minute  => 0,
-}
 }
diff --git a/modules/profile/manifests/redis/jobqueue_slave.pp 
b/modules/profile/manifests/redis/jobqueue_slave.pp
index 39b3f59..478e859 100644
--- a/modules/profile/manifests/redis/jobqueue_slave.pp
+++ b/modules/profile/manifests/redis/jobqueue_slave.pp
@@ -14,9 +14,14 @@
 group  => 'root',
 }
 $instance_str = join($::profile::redis::slave::instances, ' ')
+if $::site == 'codfw' {
+$restart_hour = 3
+} else {
+$restart_hour = 1
+}
 cron { 'jobqueue-redis-conditional-restart':
 command => "/usr/local/bin/restart-redis-if-slave ${instance_str}",
-hour=> 1,
+hour=> $restart_hour,
 minute  => 0,
 }
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/391798
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I58f1fb4b16f5947eecd0f89b075471e335e45de6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] normalize_path: fully normalize MW+RB URL paths

[BBlack (Code Review)]

BBlack has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391216 )

Change subject: [WIP] normalize_path: fully normalize MW+RB URL paths
..

[WIP] normalize_path: fully normalize MW+RB URL paths

Haven't even compiled this yet, but I think it's right in theory

Bug: T127387
Change-Id: I5887c54a9295e6a344911621b2963c9df9ad4e24
---
M modules/varnish/templates/normalize_path.inc.vcl.erb
1 file changed, 158 insertions(+), 78 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/16/391216/1

diff --git a/modules/varnish/templates/normalize_path.inc.vcl.erb 
b/modules/varnish/templates/normalize_path.inc.vcl.erb
index 98e4242..1c5895d 100644
--- a/modules/varnish/templates/normalize_path.inc.vcl.erb
+++ b/modules/varnish/templates/normalize_path.inc.vcl.erb
@@ -1,92 +1,172 @@
 C{
+
+/***
+ * URL-Path (not query!) normalization:
+ *
+ * For MediaWiki's purposes, the 256 characters can be divided into two sets
+ * named Always-Decode and Always-Encode, which means every path has exactly
+ * one canonical encoding:
+ *
+ * Always-Decode:
+ *   Unreserved Set (RFC 3986): 0-9 A-Z a-z - . _ ~
+ *   MW-specific from wfUrlencode: ! $ ( ) * , : ; / @
+ * Always-Encode:
+ *   Unprintables and space: 0x00-0x20 0x7F-0xFF
+ *   MediaWiki disallowed title chars: # < > [ ] | { }
+ *   Observed canonical encodes: " % & ' + = \ ^ ` ?
+ *
+ * Additional notes:---
+ * Canonical form for percent-encoding hex digits is uppercase.
+ * "Observed canonical encodes" - tested live WP titles containing these
+ *   characters, observed MW rel=canonical uses the percent-encoded form.
+ * Won't ever actually get encoded:
+ *   space ( ) - Can't be transmitted in HTTP request anyways
+ *   question (?) - Starts query part, used to delimit path below
+ * Literal Percent (%) - Obviously, only encode if not followed by hex digits
+ *
+ * Restbase:---
+ * Believed to use MW encoding rules above, but has a special exception for
+ *   forward-slash: We can neither encode nor decode either form of the
+ *   forward-slash for RB; it must be preserved.  This is because RB needs
+ *   forward-slashes from MediaWiki titles to be in %27 form, but still needs
+ *   its own functional path-delimiting slashes unencoded.
+ 
**/
+
+#include 
 #include 
 
-/* DIY hexadecimal conversion, since it is simple enough for a fixed
- * width, and all the relevant standard C library functions promise to
- * malfunction if the locale is set to anything other than "C"
- */
-#define NP_HEX_DIGIT(c) ( \
-   (c) >= '0' && (c) <= '9' ? (c) - '0' : ( \
-   (c) >= 'A' && (c) <= 'F' ? (c) - 'A' + 0x0a : ( \
-   (c) >= 'a' && (c) <= 'f' ? (c) - 'a' + 0x0a : -1 ) ) )
-#define NP_IS_HEX(c) (NP_HEX_DIGIT(c) != -1)
-#define NP_HEXCHAR(c1, c2) (char)( (NP_HEX_DIGIT(c1) << 4) | NP_HEX_DIGIT(c2) )
+static const uintptr_t decoder_ring[256] = {
+  // 0x00-0x1F (all unprintable)
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+  //  ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
+0,1,0,0,1,0,0,0,1,1,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,
+  //@ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,
+  //` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~ 
+0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,1,0
+  // 0x80-0xFF (all unprintable)
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+};
 
-void raw_normalize_path(const struct vrt_ctx *ctx, const int doslash) {
-   /* Rewrite the path part of the URL, replacing unnecessarily escaped
-* punctuation with the actual characters. The character list is from
-* MediaWiki's wfUrlencode(), so the URLs produced here will be the 
same as
-* the ones produced by MediaWiki in href attributes. Doing this reduces
-* cache fragmentation and fixes T29935, i.e. stale cache entries due to
-* MediaWiki purging only the wfUrlencode'd version of the URL.
-*/
-   const char * url = VRT_r_req_url(ctx);
-   size_t i, outPos;
-   const size_t urlLength = strlen(url);
-   // index for the last position %XX can start at:
-   const size_t lastConvertIdx = urlLength > 2 ? urlLength - 3 : 0;
-   char c;
-   int dirty = 0;
-
-   /* Allocate destination memory from the stack using the C99
-* variable-length automatic feature. We know the length in advance
-  

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Have every rdns advertise a private anycast VIP

[Ayounsi (Code Review)]

Ayounsi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391149 )

Change subject: [WIP] Have every rdns advertise a private anycast VIP
..

[WIP] Have every rdns advertise a private anycast VIP

Change-Id: I56b16355ee33cd68a6246f08fc16c20f10da3df2
---
A hieradata/hosts/acamar.yaml
M hieradata/role/common/dnsrecursor.yaml
A modules/bird/manifests/init.pp
A modules/bird/templates/bird.service.erb
A modules/bird/templates/bird_anycast.conf.erb
A modules/profile/manifests/bird/anycast.pp
M modules/role/manifests/dnsrecursor.pp
7 files changed, 167 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/49/391149/1

diff --git a/hieradata/hosts/acamar.yaml b/hieradata/hosts/acamar.yaml
new file mode 100644
index 000..219afcd
--- /dev/null
+++ b/hieradata/hosts/acamar.yaml
@@ -0,0 +1,3 @@
+profile::bird::neighbors_list:
+  - 208.80.153.2 # cr1-codfw ae1:2001
+  - 208.80.153.3 # cr2-codfw ae1:2001
diff --git a/hieradata/role/common/dnsrecursor.yaml 
b/hieradata/role/common/dnsrecursor.yaml
index 49549d1..b2c2243 100644
--- a/hieradata/role/common/dnsrecursor.yaml
+++ b/hieradata/role/common/dnsrecursor.yaml
@@ -1 +1,6 @@
 standard::has_ganglia: false
+
+profile::bird::advertise_vips:
+  rec-dns-anycast-vip: 10.3.0.1/32
+
+profile::bird::bind_service: 'pdns-recursor.service'
diff --git a/modules/bird/manifests/init.pp b/modules/bird/manifests/init.pp
new file mode 100644
index 000..a114dba
--- /dev/null
+++ b/modules/bird/manifests/init.pp
@@ -0,0 +1,52 @@
+# == Class: bird::base
+#
+# Installs Bird
+# Let the option to "bindTo" the Bird service to another service 
(watchdog-like)
+#
+#
+class bird(
+  $neighbors,
+  $bfd = true,
+  $bind_service = '',
+  $routerid= $::ipaddress,
+  ){
+
+  require_package('bird')
+
+  if $bind_service != '' {
+file { '/lib/systemd/system/bird.service':
+ensure  => present,
+owner   => 'root',
+group   => 'root',
+mode=> '0644',
+content => template('bird/bird.service.erb'),
+require => Package['bird'],
+}
+exec { 'bird-systemd-reload':
+command => 'systemctl daemon-reload',
+path=> [ '/usr/bin', '/bin', '/usr/sbin' ],
+refreshonly => true,
+}
+  }
+
+  service { 'bird':
+  ensure  => running,
+  enable  => true,
+  require => Package['bird'],
+  }
+
+  service { 'bird6':
+  ensure  => stopped,
+  enable  => false,
+  require => Package['bird'],
+  }
+
+  file { '/etc/bird/bird.conf':
+  ensure  => present,
+  owner   => 'bird',
+  group   => 'bird',
+  mode=> '0640',
+  content => template('bird/bird_anycast.conf.erb'),
+  }
+
+}
diff --git a/modules/bird/templates/bird.service.erb 
b/modules/bird/templates/bird.service.erb
new file mode 100644
index 000..319ac98
--- /dev/null
+++ b/modules/bird/templates/bird.service.erb
@@ -0,0 +1,17 @@
+[Unit]
+Description=BIRD Internet Routing Daemon (IPv4)
+
+After=<%= @bind_service %>
+BindsTo=<%= @bind_service %>
+
+[Service]
+EnvironmentFile=/etc/bird/envvars
+ExecStartPre=/usr/lib/bird/prepare-environment
+ExecStartPre=/usr/sbin/bird -p
+ExecReload=/usr/sbin/birdc configure
+ExecStart=/usr/sbin/bird -f -u $BIRD_RUN_USER -g $BIRD_RUN_GROUP $BIRD_ARGS
+Restart=on-abort
+
+[Install]
+## https://github.com/systemd/systemd/issues/720
+WantedBy=<%= @bind_service %>
diff --git a/modules/bird/templates/bird_anycast.conf.erb 
b/modules/bird/templates/bird_anycast.conf.erb
new file mode 100644
index 000..f108d07
--- /dev/null
+++ b/modules/bird/templates/bird_anycast.conf.erb
@@ -0,0 +1,47 @@
+
+router id <%= @routerid %>;
+
+protocol device {
+}
+
+protocol kernel {
+   metric 64;
+   import none;
+}
+
+protocol direct {
+   interface "*";
+}
+
+filter vips_filter{
+if ( net.len = 32 && net ~ 10.3.0.0/24 ) then {
+accept;
+}
+else {
+reject;
+}
+}
+
+<%- if @bfd -%>
+protocol bfd {
+interface "*" {
+interval 300 ms;
+multiplier 3;
+};
+}
+<%- end -%>
+
+
+protocol bgp {
+   import none;
+   export filter vips_filter;
+   local as 64605;
+  check link yes;
+  <% @neighbors.each do |neighbor| %>
+  neighbor <%= @neighbor %> as 14907;
+  <% end %>
+  <%- if @bfd -%>
+   bfd yes;
+  <%- end -%>
+
+}
diff --git a/modules/profile/manifests/bird/anycast.pp 
b/modules/profile/manifests/bird/anycast.pp
new file mode 100644
index 000..b048a39
--- /dev/null
+++ b/modules/profile/manifests/bird/anycast.pp
@@ -0,0 +1,42 @@
+# == Class: bird::base
+#
+# Installs and configure Bird
+# Configure Ferm
+#
+#
+class profile::bird::anycast(
+  $bfd = hiera('profile::bird::bfd', true),
+  $neighbors_list = hiera('profile::bird::neighbors_list', []),
+  $bind_service = hiera('profile::bird::bind_service', ''),
+  $advertise_vips = 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add cergen module

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391134 )

Change subject: [WIP] Add cergen module
..

[WIP] Add cergen module

has not been tested, just some ideas atm.

Bug: T166167
Change-Id: I26c3072f4f4d1b8dd73b9e123263b09b5972b045
---
A modules/cergen/manifests/certificate.pp
A modules/cergen/manifests/init.pp
A modules/cergen/manifests/manifest.pp
A modules/cergen/templates/certificate.yaml.erb
4 files changed, 131 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/34/391134/1

diff --git a/modules/cergen/manifests/certificate.pp 
b/modules/cergen/manifests/certificate.pp
new file mode 100644
index 000..1e11b69
--- /dev/null
+++ b/modules/cergen/manifests/certificate.pp
@@ -0,0 +1,78 @@
+# == Define cergen::certificate
+# == Parameters
+# arguments
+#
+define cergen::certificate (
+$destination,
+$manifest,
+# TODO: use ensure
+$ensure  = 'present',
+$owner   = 'root',
+$group   = 'root',
+$include_private_key = false,
+) {
+include ::passwords::certificates
+$password = $::passwords::certificates::certificates[$title]
+
+$defaults = {
+'authority' =>  'puppet_ca',
+'subject' => {
+'country_name' => 'US',
+'state_or_province_name' => 'CA',
+'locality_name' => 'San Francisco',
+'organization_name' => 'Wikimedia Foundation',
+},
+'expiry' => 'null',
+'key' => {
+'algorithm' => 'ec',
+'password' => $password,
+}
+}
+$certificate_manifest = deep_merge($manifest, $defaults)
+
+@@cergen::manifest { $title:
+ensure  => $ensure,
+content => template('cergen/certificate.yaml.erb'),
+}
+
+# TODO: automatically run cergen --generate using puppet generate() 
function?!
+
+# TODO: Assuming the file is on the puppet master, now render it?
+# Or, should this be a separate define?
+
+# base-path: 
/etc/puppet/private/modules/secret/files/certificates/certs/$name/
+# base-private-path: 
/etc/puppet/private/modules/secret/secrets/certficates/private/$name/
+
+# Default subsequent file resources with these.
+File {
+owner => $owner,
+group => $group,
+mode  => '0400',
+}
+
+file { $destination:
+ensure  => 'directory',
+mode=> '0555',
+# Puppet will fully manage this directory.  Any files in
+# this directory that are not managed by puppet will be deleted.
+recurse => true,
+purge   => true,
+}
+
+file { "${destination}":
+ensure  => 'directory',
+mode=> '0555',
+# Puppet will fully manage this directory.  Any files in
+# this directory that are not managed by puppet will be deleted.
+recurse => true,
+purge   => true,
+source  => "puppet:///secret/certificates/certs/${title}"
+}
+
+if $include_private_key {
+file { "${destination}/{title}.key.private.pem":
+ensure  => 'directory',
+content => 
secret("certificates/private/${title}/${title}.key.private.pem"),
+}
+}
+}
diff --git a/modules/cergen/manifests/init.pp b/modules/cergen/manifests/init.pp
new file mode 100644
index 000..8ee3af6
--- /dev/null
+++ b/modules/cergen/manifests/init.pp
@@ -0,0 +1,16 @@
+# == Class cergen
+# Installs cergen and ensure that /etc/cergen/manifests.d exists.
+#
+class cergen
+{
+require_package('cergen')
+
+$manifests_path = '/etc/cergen/manifests.d'
+
+file { ['/etc/cergen', $manifests_path]:
+ensure => 'directory',
+}
+
+# Collect all exported cergen certificate manifests.
+Cergen_manifest <<||>>
+}
diff --git a/modules/cergen/manifests/manifest.pp 
b/modules/cergen/manifests/manifest.pp
new file mode 100644
index 000..3cf8169
--- /dev/null
+++ b/modules/cergen/manifests/manifest.pp
@@ -0,0 +1,35 @@
+# == Define cergen
+# Installs a cergen certificate manifest file into /etc/cergen/manifests.d
+# This does not handle generation of certificates with cergen CLI.
+# You should manually run cergen CLI and commit the resulting files to puppet 
and private
+# repositories.
+#
+# Parameters:
+# [*ensure*]
+#
+# [*source*]
+#
+# [*content*]
+#
+define cergen::manifest(
+$ensure  = 'present',
+$source  = undef,
+$content = undef,
+) {
+require ::cergen
+
+if $source == undef and $content == undef and $ensure == 'present' {
+fail('you must provide either "source" or "content", or ensure must be 
"absent"')
+}
+
+if $source != undef and $content != undef  {
+fail('"source" and "content" are mutually exclusive')
+}
+
+file { "${::cergen::manifests_path}/${title}.yaml":
+ensure  => $ensure,
+mode=> '0400'
+content => 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Bird-lg

[Ayounsi (Code Review)]

Ayounsi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390330 )

Change subject: [WIP] Bird-lg
..

[WIP] Bird-lg

Change-Id: I3bbd8851a67fde8d9d778f6d3c263879ccfd659a
---
A modules/birdlg/manifests/lg_backend.pp
A modules/birdlg/manifests/lg_frontend.pp
A modules/birdlg/templates/lg.cfg.erb
A modules/birdlg/templates/lgproxy.cfg.erb
A modules/profile/manifests/birdlg/lg_backend.pp
A modules/profile/manifests/birdlg/lg_frontend.pp
A modules/profile/templates/birdlg/lg.wikimedia.org.erb
A modules/role/manifests/birdlg/lg_backend.pp
A modules/role/manifests/birdlg/lg_frontend.pp
9 files changed, 351 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/30/390330/1

diff --git a/modules/birdlg/manifests/lg_backend.pp 
b/modules/birdlg/manifests/lg_backend.pp
new file mode 100644
index 000..047ea4f
--- /dev/null
+++ b/modules/birdlg/manifests/lg_backend.pp
@@ -0,0 +1,80 @@
+# == Class: librenms
+#
+# This class installs & manages Bird and lgproxy, the backend part of BirdLG
+#
+class birdlg::lg_backend(
+$install_dir='/srv/deployment/birdlg/',
+$access_list=['127.0.0.1'],
+$port = 5000,
+) {
+
+  package { [
+  'python-flask',
+  'python-dnspython',
+  'python-memcache',
+  'whois',
+  'traceroute',
+  'bird',
+  ]:
+  ensure => present,
+  }
+
+file { '/etc/bird/bird.conf':  # TODO
+ensure  => present,
+owner   => 'bird',
+group   => 'bird',
+mode=> '0440',
+content => template('birdlg/bird.conf.erb'),
+}
+file { '/etc/bird/bird6.conf':  # TODO
+ensure  => present,
+owner   => 'bird',
+group   => 'bird',
+mode=> '0440',
+content => template('birdlg/bird6.conf.erb'),
+}
+
+service { 'bird':
+ensure=> running,
+subscribe => [
+  File['/etc/bird/bird.conf'],
+  File['/etc/bird/bird6.conf'],
+  ],
+require   => Package['bird'],
+}
+
+file { "${install_dir}/lgproxy.cfg":
+ensure  => present,
+owner   => 'bird',
+group   => 'bird',
+mode=> '0440',
+content => template('birdlg/lgproxy.cfg.erb'),
+}
+
+service::uwsgi { 'lgproxy':
+port=> $port,
+deployment_user => 'bird',   # TODO
+config  => {
+need-plugins => 'python',
+chdir=> $install_dir,
+wsgi => 'lgproxy.wsgi',
+vacuum   => true,
+http-socket  => "0.0.0.0:${port}",
+# T170189: make sure Python has a sane default encoding
+env  => [
+'LANG=C.UTF-8',
+'PYTHONENCODING=utf-8',
+],
+},
+healthcheck_url => '/',
+icinga_check=> false,
+sudo_rules  => [
+'ALL=(root) NOPASSWD: /usr/sbin/service uwsgi-lgproxy restart',
+'ALL=(root) NOPASSWD: /usr/sbin/service uwsgi-lgproxy start',
+'ALL=(root) NOPASSWD: /usr/sbin/service uwsgi-lgproxy status',
+'ALL=(root) NOPASSWD: /usr/sbin/service uwsgi-lgproxy stop',
+],
+}
+
+
+}
diff --git a/modules/birdlg/manifests/lg_frontend.pp 
b/modules/birdlg/manifests/lg_frontend.pp
new file mode 100644
index 000..1eeb1ad
--- /dev/null
+++ b/modules/birdlg/manifests/lg_frontend.pp
@@ -0,0 +1,29 @@
+# == Class: librenms
+#
+# This class installs & manages bird-lg frontend
+#
+class birdlg::lg_frontend(
+$session_key, #TODO
+$install_dir='/srv/deployment/birdlg/',
+) {
+
+
+  package { [
+  'python-flask',
+  'python-dnspython',
+  'python-pydot',
+  'python-memcache',
+  'graphviz',
+  ]:
+  ensure => present,
+  }
+
+  file { "${install_dir}/lg.cfg":
+  ensure  => present,
+  owner   => 'bird',
+  group   => 'bird',
+  mode=> '0440',
+  content => template('birdlg/lg.cfg.erb'),
+  }
+
+}
diff --git a/modules/birdlg/templates/lg.cfg.erb 
b/modules/birdlg/templates/lg.cfg.erb
new file mode 100644
index 000..cddcadd
--- /dev/null
+++ b/modules/birdlg/templates/lg.cfg.erb
@@ -0,0 +1,32 @@
+DEBUG = False
+LOG_FILE="<%= @install_dir %>/lg.log"
+LOG_LEVEL="WARNING"
+
+DOMAIN = "lg.wikimedia.org"
+
+BIND_IP = "127.0.0.1"
+BIND_PORT = 5001
+
+## TODO: Need to either add a line to /etc/hosts or a A record for PROXY.DOMAIN
+PROXY = {
+   "codfw": 5000,
+   "eqiad": 5000,
+   }
+
+# Used for bgpmap
+ROUTER_IP = {
+"codfw" : ["208.80.153.192", "2620:0:860:::1", "208.80.153.193", 
"2620:0:860:::2", "208.80.153.198", "2620:0:860:::5"],
+"eqiad" : ["208.80.154.196", "2620:0:861:::1", "208.80.154.197", 
"2620:0:861:::2"],
+}
+
+AS_NUMBER = {
+"codfw" : "14907",
+"eqiad" : 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: varnish: log slow requests

[Ema (Code Review)]

Ema has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390258 )

Change subject: WIP: varnish: log slow requests
..

WIP: varnish: log slow requests

Change-Id: Icb39030c74566cb8bbf7f6440951f3d223f39c62
---
M modules/varnish/manifests/instance.pp
A modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb
2 files changed, 31 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/58/390258/1

diff --git a/modules/varnish/manifests/instance.pp 
b/modules/varnish/manifests/instance.pp
index 1d615c8..13cfeda 100644
--- a/modules/varnish/manifests/instance.pp
+++ b/modules/varnish/manifests/instance.pp
@@ -168,6 +168,20 @@
 },
 }
 
+# Log slow requests to syslog
+
+$slow_req_threshold = '10.0' # XXX: somewhere in hiera?
+
+systemd::service { "varnish${instancesuffix}-slowreqs":
+ensure => present,
+content=> systemd_template('varnish-slowreqs'),
+restart=> true,
+service_params => {
+require => Service["varnish${instancesuffix}"],
+enable  => true,
+},
+}
+
 # This mechanism with the touch/rm conditionals in the pair of execs
 #   below should ensure that reload-vcl failures are retried on
 #   future puppet runs until they succeed.
diff --git a/modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb 
b/modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb
new file mode 100644
index 000..5160e9f
--- /dev/null
+++ b/modules/varnish/templates/initscripts/varnish-slowreqs.systemd.erb
@@ -0,0 +1,17 @@
+[Unit]
+Description=Varnish <%= @inst %> Slow Requests
+After=varnish<%= @instancesuffix %>.service
+Requires=varnish<%= @instancesuffix %>.service
+
+[Service]
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=varnish<%= @instancesuffix %>-slowreqs
+Restart=always
+# Note the usage of the %% specifier here.
+# See systemd.unit(5), section SPECIFIERS.
+ExecStart=/usr/bin/varnishncsa <%= @extraopts %> -q 'ReqMethod ne "PURGE" and 
Timestamp:Resp[2] > <%= @slow_req_threshold %>' -F '%{VSL:Timestamp:Resp}x %%r 
%%s'
+
+[Install]
+WantedBy=multi-user.target
+

-- 
To view, visit https://gerrit.wikimedia.org/r/390258
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Icb39030c74566cb8bbf7f6440951f3d223f39c62
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [wip] logstash: move to role / profiles

[Gehel (Code Review)]

Gehel has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390039 )

Change subject: [wip] logstash: move to role / profiles
..

[wip] logstash: move to role / profiles

Change-Id: I463285567317fef595c3f178310c4b053244597a
---
M manifests/site.pp
R modules/profile/files/logstash/apifeatureusage-template.json
R modules/profile/files/logstash/elasticsearch-template.json
R modules/profile/files/logstash/filter-add-normalized-message.conf
R modules/profile/files/logstash/filter-apifeatureusage.conf
R modules/profile/files/logstash/filter-de_dot.conf
R modules/profile/files/logstash/filter-eventlogging.conf
R modules/profile/files/logstash/filter-gelf.conf
R modules/profile/files/logstash/filter-json-lines.conf
R modules/profile/files/logstash/filter-logback.conf
R modules/profile/files/logstash/filter-mediawiki.conf
R modules/profile/files/logstash/filter-normalize-log-levels.conf
R modules/profile/files/logstash/filter-ores.conf
R modules/profile/files/logstash/filter-striker.conf
R modules/profile/files/logstash/filter-strip-ansi-color.conf
R modules/profile/files/logstash/filter-syslog.conf
R modules/profile/files/logstash/filter-udp2log.conf
R modules/profile/files/logstash/filter-webrequest.conf
A modules/profile/manifests/logstash/apifeatureusage.pp
R modules/profile/manifests/logstash/apifeatureusage/elasticsearch.pp
A modules/profile/manifests/logstash/collector.pp
R modules/profile/manifests/logstash/elasticsearch.pp
A modules/profile/manifests/logstash/eventlogging.pp
M modules/role/manifests/logstash/apifeatureusage.pp
M modules/role/manifests/logstash/eventlogging.pp
A modules/role/manifests/logstash/frontend.pp
A modules/role/manifests/logstash/storage.pp
27 files changed, 332 insertions(+), 27 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/390039/1

diff --git a/manifests/site.pp b/manifests/site.pp
index 48fc866..bd4e756 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1451,23 +1451,23 @@
 }
 
 node /^logstash100[1-2]\.eqiad\.wmnet$/ {
-role(logstash::collector, kibana, logstash::apifeatureusage)
+role(logstash::frontend, kibana)
 include ::lvs::realserver
 }
 
 node /^logstash1003\.eqiad\.wmnet$/ {
 # Before decommissionning logstash1003, role::logstash::eventlogging needs
 # to be moved to another node
-role(logstash::collector, kibana, logstash::apifeatureusage, 
logstash::eventlogging)
+role(logstash::eventlogging, kibana)
 include ::lvs::realserver
 }
 node /^logstash100[4-6]\.eqiad\.wmnet$/ {
-role(logstash::elasticsearch)
+role(logstash::storage)
 }
 
 # logstash collectors (Ganeti VM)
 node /^logstash100[7-9]\.eqiad\.wmnet$/ {
-role(logstash::collector, kibana, logstash::apifeatureusage)
+role(logstash::frontend, kibana)
 include ::lvs::realserver
 }
 
diff --git a/modules/role/files/logstash/apifeatureusage-template.json 
b/modules/profile/files/logstash/apifeatureusage-template.json
similarity index 100%
rename from modules/role/files/logstash/apifeatureusage-template.json
rename to modules/profile/files/logstash/apifeatureusage-template.json
diff --git a/modules/role/files/logstash/elasticsearch-template.json 
b/modules/profile/files/logstash/elasticsearch-template.json
similarity index 100%
rename from modules/role/files/logstash/elasticsearch-template.json
rename to modules/profile/files/logstash/elasticsearch-template.json
diff --git a/modules/role/files/logstash/filter-add-normalized-message.conf 
b/modules/profile/files/logstash/filter-add-normalized-message.conf
similarity index 100%
rename from modules/role/files/logstash/filter-add-normalized-message.conf
rename to modules/profile/files/logstash/filter-add-normalized-message.conf
diff --git a/modules/role/files/logstash/filter-apifeatureusage.conf 
b/modules/profile/files/logstash/filter-apifeatureusage.conf
similarity index 100%
rename from modules/role/files/logstash/filter-apifeatureusage.conf
rename to modules/profile/files/logstash/filter-apifeatureusage.conf
diff --git a/modules/role/files/logstash/filter-de_dot.conf 
b/modules/profile/files/logstash/filter-de_dot.conf
similarity index 100%
rename from modules/role/files/logstash/filter-de_dot.conf
rename to modules/profile/files/logstash/filter-de_dot.conf
diff --git a/modules/role/files/logstash/filter-eventlogging.conf 
b/modules/profile/files/logstash/filter-eventlogging.conf
similarity index 100%
rename from modules/role/files/logstash/filter-eventlogging.conf
rename to modules/profile/files/logstash/filter-eventlogging.conf
diff --git a/modules/role/files/logstash/filter-gelf.conf 
b/modules/profile/files/logstash/filter-gelf.conf
similarity index 100%
rename from modules/role/files/logstash/filter-gelf.conf
rename to modules/profile/files/logstash/filter-gelf.conf
diff --git a/modules/role/files/logstash/filter-json-lines.conf 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] EventLogging analytics capsule discrepency fixes

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/389722 )

Change subject: [WIP] EventLogging analytics capsule discrepency fixes
..

[WIP] EventLogging analytics capsule discrepency fixes

Bug: T179625
Change-Id: I3d5f8650c416c71f7b8cc904a58f979c852a723f
---
D modules/eventlogging/files/filters.py
A modules/eventlogging/files/plugins.py
M modules/role/manifests/eventlogging/analytics/mysql.pp
M modules/role/manifests/eventlogging/analytics/processor.pp
M modules/role/manifests/eventlogging/analytics/server.pp
5 files changed, 89 insertions(+), 32 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/22/389722/1

diff --git a/modules/eventlogging/files/filters.py 
b/modules/eventlogging/files/filters.py
deleted file mode 100644
index e411a89..000
--- a/modules/eventlogging/files/filters.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import json
-
-
-def should_insert_event(e):
-"""
-Given an Event dict e, returns true if this event should be inserted into 
the
-EventLogging storage (MySQL), or false otherwise.  This is used
-to filter out events generated by unwanted bots.
-"""
-# If no userAgent information, then insert anyway.
-if 'userAgent' not in e:
-return True
-
-user_agent_dict = json.loads(e['userAgent'])
-
-is_bot = user_agent_dict.get('is_bot', False)
-is_mediawiki = user_agent_dict.get('is_mediawiki', False)
-
-# Don't insert events generated by bots unless they are mediawiki bots.
-if is_bot and not is_mediawiki:
-return False
-else:
-return True
diff --git a/modules/eventlogging/files/plugins.py 
b/modules/eventlogging/files/plugins.py
new file mode 100644
index 000..4850fd8
--- /dev/null
+++ b/modules/eventlogging/files/plugins.py
@@ -0,0 +1,74 @@
+import json
+import dateutil.parser
+from datetime import datetime
+import unittest
+
+
+# Format string for :func:`datetime.datetime.strptime` for MediaWiki
+# timestamps. See ``_.
+MEDIAWIKI_TIMESTAMP_FORMAT = '%Y%m%d%H%M%S'
+def inject_mediawiki_timestamp(e):
+"""
+Convert dt to backwards compatible Mediawiki timestamp field.
+If dt is not in event, use current time. T179540
+"""
+
+if 'dt' in e:
+dt = dateutil.parser.parse(e['dt'])
+else:
+dt = datetime.utcnow()
+
+e['timestamp'] = dt.strftime(MEDIAWIKI_TIMESTAMP_FORMAT)
+return e
+
+
+def mysql_mapper(e):
+"""
+The WMF EventLogging Analytics MySQL log database has a lot of curious
+legacy compatibility problems.  This function converts an event
+to a format that the MySQL database expects.
+"""
+if 'userAgent' in e and isinstance(e['userAgent'], dict):
+# Get rid of unwanted bots. T67508
+is_bot = e['userAgent'].get('is_bot', False)
+is_mediawiki = e['userAgent'].get('is_mediawiki', False)
+# Don't insert events generated by bots unless they are mediawiki bots.
+if is_bot and not is_mediawiki:
+# Returning None will cause map://
+# reader to exclude this event.
+return None
+
+# MySQL expects that userAgent is a string, so we
+# convert it to JSON string now.  T153207
+e['userAgent'] = json.dumps(e['userAgent'])
+
+# Historicaly, EventCapsule did not have `dt` so we remove it from
+# insertion into MySQL.
+if 'dt' in e:
+del e['dt']
+
+return e
+
+
+# # Tests ##
+# To run:
+#   python -m unittest -v plugins.py
+# Or:
+#   python plugins.py
+#
+class TestEventLoggingPlugins(unittest.TestCase):
+def test_inject_mediawiki_timestamp(self):
+e = {'dt': '2017-11-01T11:00:00', 'userAgent': {}}
+should_be = {'dt': '2017-11-01T11:00:00', 'timestamp': 
'2017110111', 'userAgent': {}}
+self.assertEqual(inject_mediawiki_timestamp(e), should_be)
+
+def test_mysql_mapper(self):
+e1 = {'dt': '2017-11-01T11:00:00', 'timestamp': '2017110111', 
'userAgent': {'browser_family': 'Chrome'}}
+should_be1 = {'timestamp': '2017110111', 'userAgent': 
'{"browser_family": "Chrome"}'}
+self.assertEqual(mysql_mapper(e1), should_be1)
+
+e2 = {'dt': '2017-11-01T11:00:00', 'timestamp': '2017110111', 
'userAgent': {'is_bot': True}}
+self.assertEqual(mysql_mapper(e2), None)
+
+if __name__ == '__main__':
+unittest.main(verbosity=2)
\ No newline at end of file
diff --git a/modules/role/manifests/eventlogging/analytics/mysql.pp 
b/modules/role/manifests/eventlogging/analytics/mysql.pp
index 8ba8972..cac5874 100644
--- a/modules/role/manifests/eventlogging/analytics/mysql.pp
+++ b/modules/role/manifests/eventlogging/analytics/mysql.pp
@@ -25,10 +25,6 @@
 labs   => '127.0.0.1/log',
 }
 
-eventlogging::plugin { 'filters':
-source => 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP DNM role and profile for wdcm dashboards

[Addshore (Code Review)]

Addshore has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387211 )

Change subject: WIP DNM role and profile for wdcm dashboards
..

WIP DNM role and profile for wdcm dashboards

Change-Id: I17366190a9bdc01dc5959e0df3cc8237319e8202
---
A modules/profile/manifests/wdcm_dashboards/base.pp
A modules/profile/manifests/wdcm_dashboards/development.pp
A modules/profile/manifests/wdcm_dashboards/production.pp
A modules/role/manifests/wdcm/README.md
A modules/role/manifests/wdcm/beta_dashboards.pp
A modules/role/manifests/wdcm/dashboards.pp
6 files changed, 133 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/11/387211/2

diff --git a/modules/profile/manifests/wdcm_dashboards/base.pp 
b/modules/profile/manifests/wdcm_dashboards/base.pp
new file mode 100644
index 000..3c8881d
--- /dev/null
+++ b/modules/profile/manifests/wdcm_dashboards/base.pp
@@ -0,0 +1,48 @@
+# Provision Shiny Server and WDCM Dashboards
+#
+# Install and configure Shiny Server, install WDCM-specific R packages,
+# and clone WDCM's dashboards.
+#
+# filtertags: labs-project-wikidataconcepts
+class profile::wdcm_dashboards::base {
+include ::shiny_server
+
+#TODO update
+$cran_packages = [
+# Needed by Search metrics dashboard:
+'sparkline',
+'toOrdinal',
+# Needed by Wikipedia.org portal metrics dashboard:
+'highcharter',
+'countrycode'
+]
+r_lang::cran { $cran_packages:
+mirror => 'https://cran.cnr.berkeley.edu',
+}
+
+#TODO is this needed?
+# 'polloi' contains common functions & data used by all the dashboards
+r_lang::git { 'polloi':
+url => 'https://gerrit.wikimedia.org/r/wikimedia/discovery/polloi',
+}
+
+#TODO is this needed?
+# 'googleCharts' is used on the Wikipedia.org portal metrics dashboard
+r_lang::github { 'googleCharts':
+repo => 'jcheng5/googleCharts',
+}
+
+#TODO setup the landing page somewhere
+
#https://github.com/wikimedia/analytics-wmde-WDCM/tree/master/WDCM_ShinyServerFrontPage
+# Set up a portal to the various dashboards:
+# file { '/srv/shiny-server/index.html':
+# ensure => 'present',
+# owner  => 'shiny',
+# # TODO udpate group? wmde? wmde-analytics?
+# group  => 'staff',
+# mode   => '0440',
+# #TODO udpate the files for the portal!
+# source => 
'puppet:///modules/profile/discovery_dashboards/index.html',
+# }
+
+}
diff --git a/modules/profile/manifests/wdcm_dashboards/development.pp 
b/modules/profile/manifests/wdcm_dashboards/development.pp
new file mode 100644
index 000..d98a4c0
--- /dev/null
+++ b/modules/profile/manifests/wdcm_dashboards/development.pp
@@ -0,0 +1,21 @@
+# Provision Shiny Server and WDCM Dashboards
+#
+# Install and configure Shiny Server, install WDCM-specific R packages,
+# and clone "master" branch of WDCM's dashboards so it has the latest
+# versions (which may have unfinished features).
+#
+# filtertags: labs-project-wikidataconcepts
+class profile::wdcm_dashboards::development {
+require profile::wdcm_dashboards::base
+
+# Set up clones of individual dashboard repos, triggering a restart
+# of the Shiny Server service if any of the clones are updated:
+# TODO clone correct repo
+git::clone { 'wikimedia/discovery/rainbow':
+ensure=> 'latest',
+directory => '/srv/shiny-server/metrics',
+notify=> Service['shiny-server'],
+branch=> 'master',
+}
+
+}
diff --git a/modules/profile/manifests/wdcm_dashboards/production.pp 
b/modules/profile/manifests/wdcm_dashboards/production.pp
new file mode 100644
index 000..0068d44
--- /dev/null
+++ b/modules/profile/manifests/wdcm_dashboards/production.pp
@@ -0,0 +1,22 @@
+# Provision Shiny Server and WDCM Dashboards
+#
+# Install and configure Shiny Server, install WDCM-specific R packages,
+# and clone release-ready versions of WDCM's dashboards.
+#
+# filtertags: labs-project-wikidataconcepts
+class profile::wdcm_dashboards::production {
+require profile::wdcm_dashboards::base
+
+# Set up clones of individual dashboard repos, triggering a restart
+# of the Shiny Server service if any of the clones are updated:
+# TODO currently we only have a master branch but we will have a master & 
production branch
+# TODO switch from present to latest and master to production
+# TODO clone correct repo
+git::clone { 'wikimedia/discovery/rainbow':
+ensure=> 'present',
+directory => '/srv/shiny-server/metrics',
+notify=> Service['shiny-server'],
+branch=> 'master',
+}
+
+}
diff --git a/modules/role/manifests/wdcm/README.md 
b/modules/role/manifests/wdcm/README.md
new file mode 100644
index 000..6bc6414
--- /dev/null
+++ b/modules/role/manifests/wdcm/README.md
@@ -0,0 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: add puppet package version paramater to puppetmaster mo...

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/385999 )

Change subject: WIP: add puppet package version paramater to puppetmaster module
..

WIP: add puppet package version paramater to puppetmaster module

Add paramater to allow more granular selection of puppet package version

Change-Id: I18586d907153606e185cea8061ea478eccce4371
---
M hieradata/role/codfw/puppetmaster/frontend.yaml
M modules/profile/manifests/puppetmaster/frontend.pp
M modules/puppetmaster/manifests/init.pp
3 files changed, 27 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/99/385999/1

diff --git a/hieradata/role/codfw/puppetmaster/frontend.yaml 
b/hieradata/role/codfw/puppetmaster/frontend.yaml
index e124b99..c7705b5 100644
--- a/hieradata/role/codfw/puppetmaster/frontend.yaml
+++ b/hieradata/role/codfw/puppetmaster/frontend.yaml
@@ -1 +1,2 @@
 profile::puppetmaster::common::puppetdb_host: nihal.codfw.wmnet
+profile::puppetmaster::common::puppet_package_version: 4.8.2-5~bpo8+1
diff --git a/modules/profile/manifests/puppetmaster/frontend.pp 
b/modules/profile/manifests/puppetmaster/frontend.pp
index ee86a8c..f86242d 100644
--- a/modules/profile/manifests/puppetmaster/frontend.pp
+++ b/modules/profile/manifests/puppetmaster/frontend.pp
@@ -45,15 +45,16 @@
 }
 
 class { '::puppetmaster':
-bind_address=> '*',
-server_type => 'frontend',
-is_git_master   => true,
-workers => $workers,
-config  => $::profile::puppetmaster::common::config,
-secure_private  => $secure_private,
-prevent_cherrypicks => $prevent_cherrypicks,
-allow_from  => $allow_from,
-extra_auth_rules=> $extra_auth_rules,
+bind_address   => '*',
+server_type=> 'frontend',
+is_git_master  => true,
+workers=> $workers,
+config => $::profile::puppetmaster::common::config,
+secure_private => $secure_private,
+prevent_cherrypicks=> $prevent_cherrypicks,
+allow_from => $allow_from,
+extra_auth_rules   => $extra_auth_rules,
+puppet_package_version => 
hiera('::profile::puppetmaster::common::puppet_package_version', undef),
 }
 
 # Main site to respond to
diff --git a/modules/puppetmaster/manifests/init.pp 
b/modules/puppetmaster/manifests/init.pp
index b7bdafe..1584ad9 100644
--- a/modules/puppetmaster/manifests/init.pp
+++ b/modules/puppetmaster/manifests/init.pp
@@ -33,9 +33,11 @@
 #Bool - use git hooks to prevent cherry picking on top of the git repo
 #- $git_user
 #String - name of user who should own the git repositories
-#
 #- $git_group
 #String - name of group which should own the git repositories
+#- $puppet_package_version
+#String - version of puppet packages to ensure. defaults to "present"
+
 class puppetmaster(
 $server_name='puppet',
 $bind_address='*',
@@ -58,6 +60,7 @@
 $prevent_cherrypicks=true,
 $git_user='gitpuppet',
 $git_group='gitpuppet',
+$puppet_package_version=undef,
 ){
 
 $gitdir = '/var/lib/git'
@@ -76,12 +79,24 @@
 require_package('puppetdb-terminus')
 }
 
+# If puppet_package_version paramater was supplied set
+# puppet_package_ensure accordingly. otherwise default to "present"
+if $puppet_package_version {
+$puppet_package_ensure = $puppet_package_version
+} else {
+$puppet_package_ensure = 'present'
+}
 
 package { [
 'puppetmaster',
 'puppetmaster-common',
 'vim-puppet',
 'puppet-el',
+]:
+ensure => $puppet_package_ensure,
+}
+
+package { [
 'rails',
 'ruby-json',
 'ruby-mysql',

-- 
To view, visit https://gerrit.wikimedia.org/r/385999
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I18586d907153606e185cea8061ea478eccce4371
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]: Support multiinstance in core servers

[Marostegui (Code Review)]

Marostegui has submitted this change and it was merged. ( 
https://gerrit.wikimedia.org/r/384452 )

Change subject: [WIP]: Support multiinstance in core servers
..


[WIP]: Support multiinstance in core servers

First attempt to start working on supporting multiinstance on core
servers
Using db2084 to test it.

Change-Id: I3c2cad302a368a408ffd2bd278c53b9f529b46ec
---
M hieradata/hosts/db2084.yaml
M manifests/site.pp
A modules/profile/manifests/mariadb/core/multiinstance.pp
A modules/profile/templates/mariadb/mysqld_config/core_multiinstance.my.cnf.erb
A modules/role/manifests/mariadb/core_multiinstance.pp
5 files changed, 325 insertions(+), 1 deletion(-)

Approvals:
  Marostegui: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/hieradata/hosts/db2084.yaml b/hieradata/hosts/db2084.yaml
index 0ea867a..4c81f7b 100644
--- a/hieradata/hosts/db2084.yaml
+++ b/hieradata/hosts/db2084.yaml
@@ -1,2 +1,5 @@
-mariadb::shard: 's5'
+# Buffer pool sizes/instance enabled
+profile::mariadb::core_multiinstance::num_instances: 2
+profile::mariadb::core_multiinstance::s4: '185G'
+profile::mariadb::core_multiinstance::s5: '185G'
 profile::base::notifications_enabled: '0'
diff --git a/manifests/site.pp b/manifests/site.pp
index 9df4a6f..3122b60 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -455,6 +455,11 @@
 role(mariadb::core)
 }
 
+node 'db2084.codfw.wmnet' {
+role(mariadb::core_multiinstance)
+}
+
+
 # s6 core production dbs on eqiad
 # eqiad master
 node 'db1061.eqiad.wmnet' {
diff --git a/modules/profile/manifests/mariadb/core/multiinstance.pp 
b/modules/profile/manifests/mariadb/core/multiinstance.pp
new file mode 100644
index 000..5211ad1
--- /dev/null
+++ b/modules/profile/manifests/mariadb/core/multiinstance.pp
@@ -0,0 +1,134 @@
+# TODO: use a data structure for the shards
+class profile::mariadb::core::multiinstance(
+$num_instances = 
hiera('profile::mariadb::core_multiinstance::num_instances', 8),
+$s1 = hiera('profile::mariadb::core_multiinstance::s1', false),
+$s2 = hiera('profile::mariadb::core_multiinstance::s2', false),
+$s3 = hiera('profile::mariadb::core_multiinstance::s3', false),
+$s4 = hiera('profile::mariadb::core_multiinstance::s4', false),
+$s5 = hiera('profile::mariadb::core_multiinstance::s5', false),
+$s6 = hiera('profile::mariadb::core_multiinstance::s6', false),
+$s7 = hiera('profile::mariadb::core_multiinstance::s7', false),
+$x1 = hiera('profile::mariadb::core_multiinstance::x1', false),
+) {
+#FIXME:
+ferm::service { 'core_multiinstance':
+proto  => 'tcp',
+port   => '3311:3320',
+srange => '$PRODUCTION_NETWORKS',
+}
+
+#TODO: define one group per shard
+class {'role::mariadb::groups':
+mysql_group => 'core',
+mysql_shard => 's1',
+mysql_role  => 'slave',
+socket  => '/run/mysqld/mysqld.s1.sock',
+}
+
+class {'mariadb::packages_wmf': }
+class {'mariadb::service':
+override => "[Service]\nExecStartPre=/bin/sh -c \"echo 'mariadb main 
service is \
+disabled, use mariadb@ instead'; exit 1\"",
+}
+
+if os_version('debian >= stretch') {
+$basedir = '/opt/wmf-mariadb101'
+} else {
+$basedir = '/opt/wmf-mariadb10'
+}
+# Read only forced on also for the masters of the primary datacenter
+class { 'mariadb::config':
+basedir   => $basedir,
+config=> 
'profile/mariadb/mysqld_config/core_multiinstance.my.cnf.erb',
+p_s   => 'on',
+ssl   => 'puppet-cert',
+binlog_format => 'ROW',
+}
+
+file {'/etc/mysql/mysqld.conf.d':
+ensure => directory,
+owner  => root,
+group  => root,
+mode   => '0755',
+}
+
+
+if $s1 {
+mariadb::instance { 's1':
+port=> 3311,
+innodb_buffer_pool_size => $s1,
+}
+role::prometheus::mysqld_exporter_instance {'s1': port => 13311, }
+}
+
+if $s2 {
+mariadb::instance { 's2':
+port=> 3312,
+innodb_buffer_pool_size => $s2,
+}
+role::prometheus::mysqld_exporter_instance { 's2': port => 13312, }
+}
+
+if $s3 {
+mariadb::instance { 's3':
+port=> 3313,
+innodb_buffer_pool_size => $s2,
+}
+role::prometheus::mysqld_exporter_instance {'s3': port => 13313, }
+}
+
+if $s4 {
+mariadb::instance { 's4':
+port=> 3314,
+innodb_buffer_pool_size => $s4,
+}
+role::prometheus::mysqld_exporter_instance { 's4': port => 13314, }
+}
+
+if $s5 {
+mariadb::instance { 's5':
+port=> 3315,
+innodb_buffer_pool_size => $s5,
+}
+

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]: Support multiinstance in core servers

[Marostegui (Code Review)]

Marostegui has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/384452 )

Change subject: [WIP]: Support multiinstance in core servers
..

[WIP]: Support multiinstance in core servers

First attempt to start working on supporting multiinstance on core
servers

Change-Id: I3c2cad302a368a408ffd2bd278c53b9f529b46ec
---
A modules/role/manifests/mariadb/core_multiinstance.pp
A modules/role/templates/mariadb/mysqld_config/core_multiinstance.my.cnf.erb
2 files changed, 238 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/52/384452/1

diff --git a/modules/role/manifests/mariadb/core_multiinstance.pp 
b/modules/role/manifests/mariadb/core_multiinstance.pp
new file mode 100644
index 000..ccc3b17
--- /dev/null
+++ b/modules/role/manifests/mariadb/core_multiinstance.pp
@@ -0,0 +1,129 @@
+class role::mariadb::core_multiinstance {
+system::role { 'mariadb::core':
+description => 'Core multi-instance server',
+}
+
+include ::standard
+include ::base::firewall
+#FIXME:
+ferm::service { 'dbstore_multiinstance':
+proto  => 'tcp',
+port   => '3311:3320',
+srange => '$PRODUCTION_NETWORKS',
+}
+
+#TODO: define one group per shard
+class {'mariadb::groups':
+mysql_group => 'core',
+mysql_shard => 's1',
+mysql_role  => 'slave',
+socket  => '/run/mysqld/mysqld.s1.sock',
+}
+
+class {'mariadb::packages_wmf': }
+class {'mariadb::service':
+override => "[Service]\nExecStartPre=/bin/sh -c \"echo 'mariadb main 
service is \
+disabled, use mariadb@ instead'; exit 1\"",
+}
+
+if os_version('debian >= stretch') {
+$basedir = '/opt/wmf-mariadb101'
+} else {
+$basedir = '/opt/wmf-mariadb10'
+}
+# Read only forced on also for the masters of the primary datacenter
+class { 'mariadb::config':
+basedir   => $basedir,
+config=> 
'role/mariadb/mysqld_config/core_multiinstance.my.cnf.erb',
+p_s   => 'on',
+ssl   => 'puppet-cert',
+binlog_format => 'ROW',
+}
+
+file {'/etc/mysql/mysqld.conf.d':
+ensure => directory,
+owner  => root,
+group  => root,
+mode   => '0755',
+}
+
+$s1 = hiera('role::mariadb::core_multiinstance::s1', false)
+if $s1 {
+mariadb::instance { 's1':
+port=> 3311,
+innodb_buffer_pool_size => $s1,
+}
+role::prometheus::mysqld_exporter_instance {'s1': port => 13311, }
+}
+$s2 = hiera('role::mariadb::core_multiinstance::s2', false)
+if $s2 {
+mariadb::instance { 's2':
+port=> 3312,
+innodb_buffer_pool_size => $s2,
+}
+role::prometheus::mysqld_exporter_instance { 's2': port => 13312, }
+}
+$s3 = hiera('role::mariadb::core_multiinstance::s3', false)
+if $s3 {
+mariadb::instance { 's3':
+port=> 3313,
+innodb_buffer_pool_size => $s2,
+}
+role::prometheus::mysqld_exporter_instance {'s3': port => 13313, }
+}
+$s4 = hiera('role::mariadb::core_multiinstance::s4', false)
+if $s4 {
+mariadb::instance { 's4':
+port=> 3314,
+innodb_buffer_pool_size => $s4,
+}
+role::prometheus::mysqld_exporter_instance { 's4': port => 13314, }
+}
+$s5 = hiera('role::mariadb::core_multiinstance::s5', false)
+if $s5 {
+mariadb::instance { 's5':
+port=> 3315,
+innodb_buffer_pool_size => $s5,
+}
+role::prometheus::mysqld_exporter_instance { 's5': port => 13315, }
+}
+$s6 = hiera('role::mariadb::core_multiinstance::s6', false)
+if $s6 {
+mariadb::instance { 's6':
+port=> 3316,
+innodb_buffer_pool_size => $s6,
+}
+role::prometheus::mysqld_exporter_instance { 's6': port => 13316, }
+}
+$s7 = hiera('role::mariadb::core_multiinstance::s7', false)
+if $s7 {
+mariadb::instance { 's7':
+port=> 3317,
+innodb_buffer_pool_size => $s7,
+}
+role::prometheus::mysqld_exporter_instance { 's7': port => 13317, }
+}
+
+$x1 = hiera('role::mariadb::core_multiinstance::x1', false)
+if $x1 {
+mariadb::instance { 'x1':
+port=> 3320,
+innodb_buffer_pool_size => '5G',
+}
+role::prometheus::mysqld_exporter_instance { 'x1': port => 13320, }
+}
+
+require_package ('mydumper')
+
+class { 'mariadb::monitor_disk':
+is_critical   => true,
+contact_group => 'admins',
+}
+
+$num_instances = 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Removing from whitelist tables that no longer exist

[Nuria (Code Review)]

Nuria has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/383185 )

Change subject: [WIP] Removing from whitelist tables that no longer exist
..

[WIP] Removing from whitelist tables that no longer exist

Bug: T171629
Change-Id: I225ea76761c0f8ce6224f164957508ca18c0fa0a
---
M modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
1 file changed, 0 insertions(+), 53 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/85/383185/1

diff --git a/modules/role/files/mariadb/eventlogging_purging_whitelist.tsv 
b/modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
index b0dd574..9069c9b 100644
--- a/modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
+++ b/modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
@@ -14,23 +14,14 @@
 Echo   event_revisionId
 Echo   event_sender
 Echo   event_version
-_EchoInteraction   clientValidated
 EchoInteractionclientValidated
-_EchoInteraction   event_action
 EchoInteractionevent_action
-_EchoInteraction   event_context
 EchoInteractionevent_context
-_EchoInteraction   event_mobile
 EchoInteractionevent_mobile
-_EchoInteraction   event_notificationType
 EchoInteractionevent_notificationType
-_EchoInteraction   event_version
 EchoInteractionevent_version
-_EchoInteraction   isTruncated
 EchoInteractionisTruncated
-_EchoInteraction   webHost
 EchoInteractionwebHost
-_EchoInteraction   wiki
 EchoInteractionwiki
 Echo   isTruncated
 EchoMail   clientValidated
@@ -680,50 +671,6 @@
 PageContentSaveCompleteisTruncated
 PageContentSaveCompletewebHost
 PageContentSaveCompletewiki
-PageCreation   clientValidated
-PageCreation   event_namespace
-PageCreation   event_pageId
-PageCreation   event_revId
-PageCreation   event_title
-PageCreation   event_userId
-PageCreation   event_userText
-PageCreation   isTruncated
-PageCreation   webHost
-PageCreation   wiki
-PageDeletion   clientValidated
-PageDeletion   event_comment
-PageDeletion   event_namespace
-PageDeletion   event_pageId
-PageDeletion   event_title
-PageDeletion   event_userId
-PageDeletion   event_userText
-PageDeletion   isTruncated
-PageDeletion   webHost
-PageDeletion   wiki
-PageMove   clientValidated
-PageMove   event_comment
-PageMove   event_newNamespace
-PageMove   event_newTitle
-PageMove   event_oldNamespace
-PageMove   event_oldTitle
-PageMove   event_pageId
-PageMove   event_redirectId
-PageMove   event_userId
-PageMove   event_userText
-PageMove   isTruncated
-PageMove   webHost
-PageMove   wiki
-PageRestorationclientValidated
-PageRestorationevent_comment
-PageRestorationevent_namespace
-PageRestorationevent_newPageId
-PageRestorationevent_oldPageId
-PageRestorationevent_title
-PageRestorationevent_userId
-PageRestorationevent_userText
-PageRestorationisTruncated
-PageRestorationwebHost
-PageRestorationwiki
 Popups event_sessionToken
 Popups event_version
 Popups event_pageIdSource

-- 
To view, visit https://gerrit.wikimedia.org/r/383185
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I225ea76761c0f8ce6224f164957508ca18c0fa0a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Nuria 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: docker pusher

[Thcipriani (Code Review)]

Thcipriani has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/382608 )

Change subject: WIP: docker pusher
..

WIP: docker pusher

Change-Id: Ifabeec98d672768db80f722cf8df8e4d9d8b1fd0
---
A modules/docker_pusher/files/docker_pusher.py
A modules/docker_pusher/manifests/init.pp
M modules/profile/manifests/ci/docker.pp
A modules/profile/manifests/ci/pipeline.pp
M modules/profile/manifests/ci/slave.pp
M modules/role/manifests/ci/master.pp
6 files changed, 53 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/08/382608/1

diff --git a/modules/docker_pusher/files/docker_pusher.py 
b/modules/docker_pusher/files/docker_pusher.py
new file mode 100644
index 000..49bc32a
--- /dev/null
+++ b/modules/docker_pusher/files/docker_pusher.py
@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+
+
+def main():
+"""
+Placeholder for now.
+"""
+pass
+
+
+if __name__ == '__main__':
+main()
diff --git a/modules/docker_pusher/manifests/init.pp 
b/modules/docker_pusher/manifests/init.pp
new file mode 100644
index 000..ef638ab
--- /dev/null
+++ b/modules/docker_pusher/manifests/init.pp
@@ -0,0 +1,29 @@
+# == Class docker_pusher
+#
+# Installs small push script used by CI
+class docker_pusher {
+file { '/etc/docker-pusher':
+ensure => 'directory',
+owner  => 'root',
+group  => 'root',
+mode   => '0400',
+}
+
+file { '/etc/docker-pusher/config.yaml':
+ensure=> 'present',
+owner => 'root',
+group => 'root',
+mode  => '0400',
+content   => secret('docker-pusher/config.yaml'),
+show_diff => false,
+}
+
+file { '/usr/local/bin/docker-pusher':
+ensure => 'present',
+owner  => 'root',
+group  => 'root',
+mode   => '0700',
+source => 'puppet:///modules/docker_pusher/docker_pusher.py',
+}
+}
+
diff --git a/modules/profile/manifests/ci/docker.pp 
b/modules/profile/manifests/ci/docker.pp
index fa5c784..55107a4 100644
--- a/modules/profile/manifests/ci/docker.pp
+++ b/modules/profile/manifests/ci/docker.pp
@@ -5,6 +5,9 @@
 class profile::ci::docker(
 $jenkins_agent_username = hiera('jenkins_agent_username'),
 ) {
+# We will need to build containers in production
+require_package('blubber')
+
 apt::repository { 'thirdparty-ci':
 uri=> 'http://apt.wikimedia.org/wikimedia',
 dist   => "${::lsbdistcodename}-wikimedia",
diff --git a/modules/profile/manifests/ci/pipeline.pp 
b/modules/profile/manifests/ci/pipeline.pp
new file mode 100644
index 000..391a993
--- /dev/null
+++ b/modules/profile/manifests/ci/pipeline.pp
@@ -0,0 +1,8 @@
+# == profile::ci::pipeline
+#
+# Profile that makes necessary provisions for building containers for
+# production.
+class profile::ci::pipeline() {
+include ::profile::ci::docker
+include ::docker_pusher
+}
diff --git a/modules/profile/manifests/ci/slave.pp 
b/modules/profile/manifests/ci/slave.pp
index 751d01f..adfbb06 100644
--- a/modules/profile/manifests/ci/slave.pp
+++ b/modules/profile/manifests/ci/slave.pp
@@ -10,11 +10,7 @@
 class profile::ci::slave {
 include contint::packages::base
 include contint::slave_scripts
-include ::profile::ci::docker
 include ::profile::zuul::cloner
-
-# We will need to build containers in production
-require_package('blubber')
 
 class { 'jenkins::slave':
 # Master connect to itself via the fqdn / primary IP ipaddress
diff --git a/modules/role/manifests/ci/master.pp 
b/modules/role/manifests/ci/master.pp
index d5a7547..f190e06 100644
--- a/modules/role/manifests/ci/master.pp
+++ b/modules/role/manifests/ci/master.pp
@@ -19,6 +19,7 @@
 include ::profile::ci::jenkins
 include ::profile::ci::slave
 include ::profile::ci::website
+include ::profile::ci::pipeline
 include ::profile::zuul::merger
 include ::profile::zuul::server
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/382608
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifabeec98d672768db80f722cf8df8e4d9d8b1fd0
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Thcipriani 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Prometheus based Kafka broker alerts, take 1

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/381489 )

Change subject: [WIP] Prometheus based Kafka broker alerts, take 1
..

[WIP] Prometheus based Kafka broker alerts, take 1

This refactors the Prometheus JXM exporter just a bit, moving
it to a separate profile::kafka::broker::monitoring class,
along with icinga alerts.

Bug: T175923
Change-Id: I839d5de4110da245f712e23285280c2fd546fe8f
---
M hieradata/role/common/kafka/jumbo/broker.yaml
M modules/profile/manifests/kafka/broker.pp
A modules/profile/manifests/kafka/broker/monitoring.pp
3 files changed, 87 insertions(+), 39 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/89/381489/1

diff --git a/hieradata/role/common/kafka/jumbo/broker.yaml 
b/hieradata/role/common/kafka/jumbo/broker.yaml
index 5fb6770..5b18607 100644
--- a/hieradata/role/common/kafka/jumbo/broker.yaml
+++ b/hieradata/role/common/kafka/jumbo/broker.yaml
@@ -2,8 +2,8 @@
 
 profile::kafka::broker::kafka_cluster_name: jumbo
 
-# Enable the Prometheus JMX Exporter
-profile::kafka::broker::prometheus_monitoring_enabled: true
+# Enable Monitoring (via Prometheus) and icinga alerts
+profile::kafka::broker::monitoring_enabled: true
 
 profile::kafka::broker::log_dirs: [/srv/kafka/data]
 profile::kafka::broker::plaintext: true
@@ -28,5 +28,5 @@
 profile::kafka::broker::num_recovery_threads_per_data_dir: 12
 profile::kafka::broker::num_io_threads: 12
 
-profile::kafka::broker::replica_maxlag_warning: "100"
-profile::kafka::broker::replica_maxlag_critical: "500"
+profile::kafka::broker::monitoring::replica_maxlag_warning: 100
+profile::kafka::broker::monitoring::replica_maxlag_critical: 500
diff --git a/modules/profile/manifests/kafka/broker.pp 
b/modules/profile/manifests/kafka/broker.pp
index 0ce4f1a..e0b3e71 100644
--- a/modules/profile/manifests/kafka/broker.pp
+++ b/modules/profile/manifests/kafka/broker.pp
@@ -87,7 +87,7 @@
 $replica_maxlag_critical   = 
hiera('profile::kafka::broker::replica_maxlag_critical'),
 # This is set via top level hiera variable so it can be synchronized 
between roles and clients.
 $message_max_bytes = hiera('kafka_message_max_bytes'),
-$prometheus_monitoring_enabled = 
hiera('profile::kafka::broker::prometheus_monitoring_enabled'),
+$monitoring_enabled   = 
hiera('profile::kafka::broker::monitoring_enabled'),
 $prometheus_nodes  = hiera('prometheus_nodes'),
 ) {
 # TODO: WIP
@@ -185,39 +185,6 @@
 java_home => '/usr/lib/jvm/java-8-openjdk-amd64',
 }
 
-if $prometheus_monitoring_enabled {
-# Allow automatic generation of config on the
-# Prometheus master
-prometheus::jmx_exporter_instance { $::hostname:
-address => $::ipaddress,
-port=> 7800,
-}
-
-$prometheus_nodes_ferm = join($prometheus_nodes, ' ')
-ferm::service { 'kafka-broker-jmx_exporter':
-proto  => 'tcp',
-port   => '7800',
-srange => "@resolve((${prometheus_nodes_ferm}))",
-}
-
-require_package('prometheus-jmx-exporter')
-
-$jmx_exporter_config_file = 
'/etc/kafka/broker_prometheus_jmx_exporter.yaml'
-$java_opts = 
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:7800:${jmx_exporter_config_file}"
-
-# Create the Prometheus JMX Exporter configuration
-file { $jmx_exporter_config_file:
-ensure  => present,
-source  => 
'puppet:///modules/profile/kafka/broker_prometheus_jmx_exporter.yaml',
-owner   => 'kafka',
-group   => 'kafka',
-mode=> '0400',
-require => Class['::confluent::kafka::broker'],
-}
-} else {
-$java_opts = undef
-}
-
 class { '::confluent::kafka::broker':
 log_dirs => $log_dirs,
 brokers  => $config['brokers']['hash'],
@@ -233,7 +200,6 @@
 # https://kafka.apache.org/documentation/#java
 # Note that MetaspaceSize is a Java 8 setting.
 jvm_performance_opts => '-server -XX:MetaspaceSize=96m 
-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 
-XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50 
-XX:MaxMetaspaceFreeRatio=80',
-java_opts=> $java_opts,
 listeners=> $listeners,
 
 security_inter_broker_protocol   => $security_inter_broker_protocol,
@@ -249,6 +215,15 @@
 message_max_bytes=> $message_max_bytes,
 }
 
+# If monitoring is enabled, then include the monitoring profile and set 
$java_opts
+# for exposing the Prometheus JMX Exporter in the Kafka Broker process.
+if $monitoring_enabled {
+include 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Set up separate druid public-eqiad cluster.

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/380804 )

Change subject: [WIP] Set up separate druid public-eqiad cluster.
..

[WIP] Set up separate druid public-eqiad cluster.

This should not be merged yet, we need to do a lot of manual work
to prepare these servers.

NOTE: It is safe to rename the 'druid-eqiad' Zookeeper cluster
to 'druid-analytics-eqiad', because it is only used in puppet.
There is nothing in zookeeper or druid configurations that
refer to the Zookeeper cluster name.

This also sets up a new Zookeeper cluster colocated on the
druid public-eqiad nodes called 'druid-public-eqiad'.

Bug: T176223
Change-Id: I8624fbc402105cc44818a2ee8d0db7dbab3526ee
---
M hieradata/common.yaml
M hieradata/role/common/druid/analytics/worker.yaml
M manifests/site.pp
M modules/network/manifests/constants.pp
4 files changed, 31 insertions(+), 9 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/04/380804/1

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 1822f98..040d968 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -439,14 +439,22 @@
   conf2002.codfw.wmnet: '2002'
   conf2003.codfw.wmnet: '2003'
 
-  # ZK cluster for Druid (in Analytics cluster),
-  # colocated with Druid.
-  druid-eqiad:
+  # ZK cluster for Druid analytics-eqiad cluster (non public),
+  # colocated on druid hosts.
+  druid-analytics-eqiad:
 hosts:
   druid1001.eqiad.wmnet: '1001'
   druid1002.eqiad.wmnet: '1002'
   druid1003.eqiad.wmnet: '1003'
 
+  # ZK cluster for Druid public-eqiad cluster, (for AQS, wikistats, etc.)
+  # colocated on druid hosts.
+  druid-public-eqiad:
+hosts:
+  druid1004.eqiad.wmnet: '1004'
+  druid1005.eqiad.wmnet: '1005'
+  druid1006.eqiad.wmnet: '1006'
+
 # Used to sync the setting between all Kafka clusters and clients.
 kafka_message_max_bytes: 4194304
 
diff --git a/hieradata/role/common/druid/analytics/worker.yaml 
b/hieradata/role/common/druid/analytics/worker.yaml
index e5715a8..7e4dc04 100644
--- a/hieradata/role/common/druid/analytics/worker.yaml
+++ b/hieradata/role/common/druid/analytics/worker.yaml
@@ -6,7 +6,7 @@
 # Druid nodes get their own Zookeeper cluster to isolate them
 # from the production ones.
 # Configure the zookeeper profile.
-profile::zookeeper::cluster_name: druid-eqiad
+profile::zookeeper::cluster_name: druid-analytics-eqiad
 # Don't page if a zookeeper server in this cluster goes down.
 profile::zookeeper::is_critical: false
 # Max number of connections per IP for Zookeeper
@@ -33,7 +33,7 @@
 # The logical name of this druid cluster
 profile::druid::common::druid_cluster_name: analytics-eqiad
 # The logical name of the zookeeper cluster that druid should use
-profile::druid::common::zookeeper_cluster_name: druid-eqiad
+profile::druid::common::zookeeper_cluster_name: druid-analytics-eqiad
 
 # Make druid build an extension composed of CDH jars.
 profile::druid::common::use_cdh: true
diff --git a/manifests/site.pp b/manifests/site.pp
index d5ebe1c..ca8a884 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -771,9 +771,21 @@
 interface::add_ip6_mapped { 'main': }
 }
 
-# Analytics Druid servers.
+# Druid analytics-eqiad (non public) servers.
+# These power internal backends and queries.
 # https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
-node /^druid100[123456].eqiad.wmnet$/ {
+node /^druid100[123].eqiad.wmnet$/ {
+role(druid::analytics::worker)
+
+include ::base::firewall
+include ::standard
+}
+
+
+# Druid public-eqiad servers.
+# These power AQS and wikistats 2.0 and contain non sensitive datasets.
+# https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
+node /^druid100[456].eqiad.wmnet$/ {
 role(druid::analytics::worker)
 
 include ::base::firewall
diff --git a/modules/network/manifests/constants.pp 
b/modules/network/manifests/constants.pp
index 45c6fac..0d726ea 100644
--- a/modules/network/manifests/constants.pp
+++ b/modules/network/manifests/constants.pp
@@ -160,20 +160,22 @@
 '10.64.53.21',# 
analytics1002.eqiad.wmnet
 '2620:0:861:108:f21f:afff:fee8:bc3f', # 
analytics1002.eqiad.wmnet
 ],
-'druid_hosts' => [
+'druid_analytics_hosts' => [
 '10.64.5.101',# 
druid1001.eqiad.wmnet
 '2620:0:861:104:1e98:ecff:fe29:e298', # 
druid1001.eqiad.wmnet
 '10.64.36.102',   # 
druid1002.eqiad.wmnet
 '2620:0:861:106:1602:ecff:fe06:8bec', # 
druid1002.eqiad.wmnet
 '10.64.53.103',   # 
druid1003.eqiad.wmnet
 '2620:0:861:108:1e98:ecff:fe29:e278', # 
druid1003.eqiad.wmnet
+],
+'druid_public_hosts' => [
 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] webperf: Add navtiming tests to puppet.git:/tox.ini

[Krinkle (Code Review)]

Krinkle has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/379830 )

Change subject: [WIP] webperf: Add navtiming tests to puppet.git:/tox.ini
..

[WIP] webperf: Add navtiming tests to puppet.git:/tox.ini

Change-Id: I2fdb8e0b978b2406ebc386c716c6c4321aeff0d8
---
M modules/webperf/files/navtiming.py
M tox.ini
2 files changed, 5 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/30/379830/1

diff --git a/modules/webperf/files/navtiming.py 
b/modules/webperf/files/navtiming.py
index d288852..b000395 100755
--- a/modules/webperf/files/navtiming.py
+++ b/modules/webperf/files/navtiming.py
@@ -13,8 +13,6 @@
 import unittest
 import yaml
 
-from kafka import KafkaConsumer
-
 handlers = {}
 
 # Mapping of continent names to ISO 3166 country codes.
@@ -402,6 +400,8 @@
 
 
 if __name__ == '__main__':
+from kafka import KafkaConsumer
+
 ap = argparse.ArgumentParser(description='NavigationTiming subscriber')
 ap.add_argument('--brokers', required=True,
 help='Comma-separated list of kafka brokers')
diff --git a/tox.ini b/tox.ini
index b32a194..8f4eb50 100644
--- a/tox.ini
+++ b/tox.ini
@@ -31,6 +31,9 @@
 -rmodules/admin/data/requirements.txt
 commands = nosetests modules/admin/data 
modules/mediawiki/files/apache/sites/redirects
 
+[testenv:webperf]
+commands = nosetests modules/webperf/files
+
 [testenv:commit-message]
 deps = commit-message-validator
 commands = commit-message-validator

-- 
To view, visit https://gerrit.wikimedia.org/r/379830
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2fdb8e0b978b2406ebc386c716c6c4321aeff0d8
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Krinkle 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add druid options to AQS config

[Milimetric (Code Review)]

Milimetric has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/379730 )

Change subject: [WIP] Add druid options to AQS config
..

[WIP] Add druid options to AQS config

Questions before merging:
* better way to list the druid hosts instead of hard-coding them?
* where/how do I make the passwords to include with ::passwords::aqs::..

Change-Id: I5c78ce68e8351eb0c379e6f9d0109c3388b757a7
---
M modules/aqs/manifests/init.pp
M modules/aqs/templates/config.yaml.erb
2 files changed, 26 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/30/379730/1

diff --git a/modules/aqs/manifests/init.pp b/modules/aqs/manifests/init.pp
index 49590a8..501ef87 100644
--- a/modules/aqs/manifests/init.pp
+++ b/modules/aqs/manifests/init.pp
@@ -43,22 +43,30 @@
 # [*statsd_port*]
 #   statsd port. Default: 8125
 #
+include ::passwords::aqs::druid_http_auth
 class aqs(
 $cassandra_user = 'cassandra',
 $cassandra_password = 'cassandra',
 $seeds  = [$::ipaddress],
 $cassandra_default_consistency = 'localQuorum',
-$cassandra_local_dc = 'datacenter1',
-$cassandra_datacenters = [ 'datacenter1' ],
-$port   = 7232,
-$salt_key   = 'secretkey',
-$page_size  = 250,
-$logstash_host  = 'localhost',
-$logstash_port  = 12201,
-$logging_level  = 'warn',
-$statsd_host= 'localhost',
-$statsd_port= 8125,
+$cassandra_local_dc= 'datacenter1',
+$cassandra_datacenters = [ 'datacenter1' ],
+$port  = 7232,
+$salt_key  = 'secretkey',
+$page_size = 250,
+$logstash_host = 'localhost',
+$logstash_port = 12201,
+$logging_level = 'warn',
+$statsd_host   = 'localhost',
+$statsd_port   = 8125,
+$druid_hosts   = [ 'druid1001.eqiad.wmnet', 
'druid1001.eqiad.wmnet', 'druid1001.eqiad.wmnet' ],
+$druid_scheme  = 'https',
+$druid_port= 8082,
+$druid_query_path  = '/druid/v2/',
+$druid_http_auth_user  = $::passwords::aqs::druid_http_auth::user,
+$druid_http_auth_password  = 
$::passwords::aqs::druid_http_auth::password,
 ) {
+# NOTE: didn't know how to make the ::passwords::aqs values, they're in 
the private repo, right?
 
 service::node { 'aqs':
 port=> $port,
diff --git a/modules/aqs/templates/config.yaml.erb 
b/modules/aqs/templates/config.yaml.erb
index 2803fe3..556ff6c 100644
--- a/modules/aqs/templates/config.yaml.erb
+++ b/modules/aqs/templates/config.yaml.erb
@@ -18,6 +18,14 @@
   - name: default.group.local
 domains: /./
 
+  druid:
+  hosts: <%= Array(@druid_hosts).join(',') %>
+  scheme: <%= @druid_scheme %>
+  port: <%= @druid_port %>
+  query_path: <%= @druid_query_path %>
+  http_auth_user: <%= @druid_http_auth_user %>
+  http_auth_password: <%= @druid_http_auth_password %>
+
 # Swagger spec root.
 spec: 
   title: "The Analytics RESTBase root"

-- 
To view, visit https://gerrit.wikimedia.org/r/379730
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5c78ce68e8351eb0c379e6f9d0109c3388b757a7
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Milimetric 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Port statsv from kafka analytics to kafka jumbo

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/379308 )

Change subject: [WIP] Port statsv from kafka analytics to kafka jumbo
..

[WIP] Port statsv from kafka analytics to kafka jumbo

Bug: T176352
Change-Id: I6d34a162601e32164cc18b57377b010010a30084
---
M modules/role/manifests/cache/kafka/statsv.pp
M modules/role/manifests/webperf.pp
2 files changed, 18 insertions(+), 11 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/08/379308/1

diff --git a/modules/role/manifests/cache/kafka/statsv.pp 
b/modules/role/manifests/cache/kafka/statsv.pp
index 5552c60..ece5958 100644
--- a/modules/role/manifests/cache/kafka/statsv.pp
+++ b/modules/role/manifests/cache/kafka/statsv.pp
@@ -13,27 +13,27 @@
 # [*varnish_svc_name*]
 #   The name of the init unit for the above.
 #   Default 'varnish-frontend'
-# [*kafka_protocol_version*]
-#   Kafka API version to use, needed for brokers < 0.10
-#   https://issues.apache.org/jira/browse/KAFKA-3547
 #
 class role::cache::kafka::statsv(
 $varnish_name   = 'frontend',
 $varnish_svc_name   = 'varnish-frontend',
-$kafka_protocol_version = '0.9.0.1',
-) inherits role::cache::kafka
+)
 {
+# Include top role::cache::kafka to get 
+# varnish_instance -> varnishkafka instance dependency.
+include ::role::cache::kafka
+
 $format  = "%{fake_tag0@hostname?${::fqdn}}x %{%FT%T@dt}t 
%{X-Client-IP@ip}o %{@uri_path}U %{@uri_query}q %{User-Agent@user_agent}i"
 
 # Set varnish.arg.q or varnish.arg.m according to Varnish version
 $varnish_opts = { 'q' => 'ReqURL ~ "^/beacon/statsv\?"' }
 $conf_template = 'varnishkafka/varnishkafka_v4.conf.erb'
 
+$kafka_config = kafka_config('jumbo')
+$kafka_brokers = $kafka_config['brokers']['array']
+
 varnishkafka::instance { 'statsv':
-# FIXME - top-scope var without namespace, will break in puppet 2.8
-# lint:ignore:variable_scope
 brokers => $kafka_brokers,
-# lint:endignore
 format  => $format,
 format_type => 'json',
 topic   => 'statsv',
diff --git a/modules/role/manifests/webperf.pp 
b/modules/role/manifests/webperf.pp
index 80b8388..f592dc7 100644
--- a/modules/role/manifests/webperf.pp
+++ b/modules/role/manifests/webperf.pp
@@ -13,8 +13,7 @@
 $statsd_host = $statsd_parts[0]
 $statsd_port = $statsd_parts[1]
 
-# Use brokers from this Kafka cluster to consume metrics.
-$kafka_config  = kafka_config('analytics')
+$kafka_config = kafka_config('jumbo')
 $kafka_brokers = $kafka_config['brokers']['string']
 
 # Consume statsd metrics from Kafka and emit them to statsd.
@@ -23,11 +22,19 @@
 statsd=> $statsd,
 }
 
+# Use brokers from this Kafka cluster to consume metrics.
+# NOTE: We are in the process of migrating clients from
+# analytics Kafka to jumbo Kafka.  This will be removed
+# once all clients here are on jumbo.
+# See: https://phabricator.wikimedia.org/T175461
+$kafka_analytics_config  = kafka_config('analytics')
+$kafka_analytics_brokers = $kafka_config['brokers']['string']
+
 # Aggregate client-side latency measurements collected via the
 # NavigationTiming MediaWiki extension and send them to Graphite.
 # See 
 class { '::webperf::navtiming':
-kafka_brokers => $kafka_brokers,
+kafka_brokers => $kafka_analytics_brokers,
 statsd_host   => $statsd_host,
 statsd_port   => $statsd_port,
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/379308
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6d34a162601e32164cc18b57377b010010a30084
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Allow admin module to ensure system user membership in...

[Ottomata (Code Review)]

Ottomata has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/379004 )

Change subject: [WIP] Allow admin module to ensure system user membership in 
managed groups
..

[WIP] Allow admin module to ensure system user membership in managed groups

Bug: T174465
Change-Id: I495942b6a65db7058a6272277ada9f0286a4ba9e
---
M modules/admin/README
M modules/admin/data/data.yaml
M modules/admin/manifests/groupmembers.pp
M modules/statistics/manifests/discovery.pp
4 files changed, 32 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/04/379004/1

diff --git a/modules/admin/README b/modules/admin/README
index 89388c9..e3dc476 100644
--- a/modules/admin/README
+++ b/modules/admin/README
@@ -54,6 +54,15 @@
   realname: Foo Bar
   ssh_keys: [ssh-rsa mykeyhash foobar@mac]
 
+- Ensuring a system user is in a group: (see note below about system user 
group membership)
+groups:
+  mygroup:
+ensure: present
+gid: 551
+members: [foo, bar]
+system_members: [www-data]
+
+
 # NOTE: To choose the UID for a new user please lookup
 # the existing UID in (labs) LDAP and use that.
 # currently you do this on terbium, example:
@@ -256,3 +265,15 @@
 If you try to apply two groupings with the same posix names on a single node 
you see:
 
 Duplicate definition: Admin::Group[$POSIX_NAME] is already defined
+
+
+System user group membership:
+
+Sometimes it is useful to declare that a system user should be in a group with 
other
+human user accounts.  This module will not manage any system users, but it 
does support
+ensuring that system users are in groups that it does manage.  Add system 
users to a group
+by providing a list of system_members in your group declaration.
+
+Your system user *must* already exist by the time the admin module ensures 
group membership.
+If it doesn't, the groupmembers exec will fail.
+
diff --git a/modules/admin/data/data.yaml b/modules/admin/data/data.yaml
index 386b3dd..694cc0a 100644
--- a/modules/admin/data/data.yaml
+++ b/modules/admin/data/data.yaml
@@ -252,6 +252,7 @@
   flemmerich, mkroetzsch, akrausetud, filippo, pmiazga, faidon,
   piccardi, fdans, eevans, anomie, demon, ladsgroup, musikanimal, 
joewalsh,
   kaldari, goransm, ema, dsaez, shiladsen, rho]
+system_members: [analytics-search]
   analytics-admins:
 gid: 732
 description: Admin access to analytics cluster.
diff --git a/modules/admin/manifests/groupmembers.pp 
b/modules/admin/manifests/groupmembers.pp
index 4a3256a..f22bbdc 100644
--- a/modules/admin/manifests/groupmembers.pp
+++ b/modules/admin/manifests/groupmembers.pp
@@ -15,11 +15,16 @@
 
 $gdata = $::admin::data['groups'][$name]
 $members = $gdata['members']
+$system_members = $gdata['system_members']
 
-if !empty($members) {
-$joined_user_list = join($members,',')
+# This contains all human members, as well as any system account
+# members of this group.
+$all_members = concat($members, $system_members)
+
+if !empty($all_members) {
+$joined_user_list = join($all_members,',')
 } else {
-$joined_user_list = $default_member
+$joined_user_list = join($default_member)
 }
 
 if has_key($gdata, 'posix_name') {
diff --git a/modules/statistics/manifests/discovery.pp 
b/modules/statistics/manifests/discovery.pp
index 0310483..499fc0d 100644
--- a/modules/statistics/manifests/discovery.pp
+++ b/modules/statistics/manifests/discovery.pp
@@ -12,6 +12,8 @@
 # Path in which the R library will reside
 $rlib_dir = "${dir}/r-library"
 
+
+# TODO: User the analytics-search user instead.
 $user = 'discovery-stats'
 # Setting group to 'analytics-privatedata-users' so that Discovery's 
Analysts
 # (as members of analytics-privatedata-users) have some privileges, and so

-- 
To view, visit https://gerrit.wikimedia.org/r/379004
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I495942b6a65db7058a6272277ada9f0286a4ba9e
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] maps: move to vector tiles and cleartables

[Gehel (Code Review)]

Gehel has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/378245 )

Change subject: [WIP] maps: move to vector tiles and cleartables
..

[WIP] maps: move to vector tiles and cleartables

* new OSM update script
* new database name

Bug: T157613
Change-Id: Ida723dee967cffb4ed44b46b14f7257f0a146ffd
---
A hieradata/role/common/maps/test/vectortiles_master.yaml
A modules/osm/files/process-osm-data.sh
A modules/osm/manifests/cleartables_sync.pp
A modules/osm/manifests/meddo.pp
M modules/profile/manifests/maps/osm_master.pp
M modules/profile/templates/maps/grants-gis.sql.erb
A modules/role/manifests/maps/test/vectortiles_master.pp
7 files changed, 456 insertions(+), 19 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/45/378245/1

diff --git a/hieradata/role/common/maps/test/vectortiles_master.yaml 
b/hieradata/role/common/maps/test/vectortiles_master.yaml
new file mode 100644
index 000..6833913
--- /dev/null
+++ b/hieradata/role/common/maps/test/vectortiles_master.yaml
@@ -0,0 +1,32 @@
+# the following passwords are defined in private repo:
+#profile::maps::osm_master::kartotherian_pass: some_password
+#profile::maps::osm_master::tilerator_pass: some_password
+#profile::maps::osm_master::tileratorui_pass: some_password
+#profile::maps::osm_master::osmimporter_pass: some_password
+#profile::maps::osm_master::osmupdater_pass: some_password
+#profile::maps::osm_master::replication_pass: some_password
+#profile::maps::cassandra::kartotherian_pass: some_password
+#profile::maps::cassandra::tilerator_pass: some_password
+#profile::maps::cassandra::tileratorui_pass: some_password
+#profile::cassandra::single_instance::super_pass: some_password
+
+admin::groups:
+  - maps-admins
+  - kartotherian-admin
+  - tilerator-admin
+
+cluster: 'maps-test-vector'
+
+profile::cassandra::single_instance::graphite_host: 'graphite-in.eqiad.wmnet'
+service::configuration::statsd_host: 'statsd.eqiad.wmnet'
+
+cassandra::metrics::blacklist:
+  - .*\.metrics\.Table\..*$
+
+profile::redis::master::instances: ['6379']
+profile::redis::master::settings:
+  bind: "0.0.0.0"
+
+profile::maps::postgresql_common::shared_buffers: '4GB'
+profile::maps::postgresql_common::maintenance_work_mem: '3GB'
+profile::maps::osm_master::cleartables: true
diff --git a/modules/osm/files/process-osm-data.sh 
b/modules/osm/files/process-osm-data.sh
new file mode 100644
index 000..db79fea
--- /dev/null
+++ b/modules/osm/files/process-osm-data.sh
@@ -0,0 +1,283 @@
+#!/usr/bin/env bash
+
+# This script will
+# 1. Download OSM data and load it into a DB
+# 2. Update that DB
+# 3. Keep a local copy of the planet up to date
+
+# Requirements
+# - osmium-tool
+# - osmosis
+# - osm2pgsql
+# - ClearTables
+# - meddo
+
+set -e
+
+CREATE_DB=false
+
+BASE_DIR="/srv/osm_replication"
+CLEARTABLES="/srv/deployment/tilerator/deploy/ClearTables"
+MEDDO="/srv/deployment/tilerator/deploy/meddo"
+
+DATABASE="ct"
+
+PLANET_DIR="$BASE_DIR/planet"
+PLANET_REPLICATION_BASE="$PLANET_DIR/planet-replication"
+DATABASE_REPLICATION_BASE="$PLANET_DIR/database-replication"
+
+# -E 3857 is not required on newer versions of osm2pgsql
+osm2pgsql_common_opts="-E 3857  --flat-nodes $PLANET_DIR/nodes.bin"
+osm2pgsql_import_opts="--cache 1 --number-processes 2"
+osm2pgsql_update_opts="--cache 1000 --number-processes 1"
+
+function show_setup_help() {
+  cat << EOF
+Usage: ${0##*/} setup data_url state_url replication_url
+
+Examples:
+  ${0##*/} setup 
http://download.geofabrik.de/north-america/canada/british-columbia-170101.osm.pbf
 \\
+
http://download.geofabrik.de/north-america/canada/british-columbia-updates/000/001/384.state.txt
 \\
+http://download.geofabrik.de/north-america/canada/british-columbia-updates
+
+EOF
+exit 1
+}
+
+function setup_data() {
+  if [ -z "$1" ]; then
+echo "data_url not set"
+show_setup_help
+exit 0
+  fi
+  if [ -z "$2" ]; then
+echo "state_url not set"
+show_setup_help
+exit 0
+  fi
+  if [ -z "$3" ]; then
+echo "replication_url not set"
+show_setup_help
+exit 0
+  fi
+  PLANET_URL="$1"
+  STATE_URL="$2"
+  REPLICATION_BASE="$3"
+
+  mkdir -p "$PLANET_DIR"
+  mkdir -p "$PLANET_REPLICATION_BASE"
+
+  cat < "$PLANET_REPLICATION_BASE/configuration.txt"
+# The URL of the directory containing change files.
+baseUrl=$REPLICATION_BASE
+
+# Allow 3 days of downloads
+maxInterval = 259200
+EOF
+
+  echo "Downloading files"
+  curl --retry 5 -o "$PLANET_DIR/osm-data.osm.pbf" "$PLANET_URL"
+  curl --retry 5 -o "$PLANET_REPLICATION_BASE/state.txt" "$STATE_URL"
+
+  # Call a function here to update the planet later
+}
+
+function onplanetupdateexit {
+[ -f "$PLANET_REPLICATION_BASE/state-prev.txt" ] && mv 
"$PLANET_REPLICATION_BASE/state-prev.txt" "$PLANET_REPLICATION_BASE/state.txt"
+}
+
+function load_borders() {
+  echo "Loading borders"
+  psql -d ct -v ON_ERROR_STOP=1 -Xq 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Disable cassandra-metrics-collector when Prometheus age...

[Eevans (Code Review)]

Eevans has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/378100 )

Change subject: WIP: Disable cassandra-metrics-collector when Prometheus agent 
is enabled
..

WIP: Disable cassandra-metrics-collector when Prometheus agent is enabled

Moves the `jmx_exporter_enabled` variable up to the cassandra class
where it can be assigned once for a cluster, and then passed into
`cassandra::metrics` to disable the systemd unit.

Bug: T171772
Change-Id: I3ce1a963df3ca89c7a119675820d95c27934aaf9
---
M hieradata/role/common/restbase/dev_cluster.yaml
M hieradata/role/common/restbase/production_ng.yaml
M hieradata/role/common/restbase/test_cluster.yaml
M modules/cassandra/manifests/init.pp
M modules/cassandra/manifests/instance.pp
M modules/cassandra/manifests/metrics.pp
M modules/profile/manifests/cassandra.pp
7 files changed, 22 insertions(+), 29 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/00/378100/1

diff --git a/hieradata/role/common/restbase/dev_cluster.yaml 
b/hieradata/role/common/restbase/dev_cluster.yaml
index 25b274e..98f6e94 100644
--- a/hieradata/role/common/restbase/dev_cluster.yaml
+++ b/hieradata/role/common/restbase/dev_cluster.yaml
@@ -89,7 +89,6 @@
 - /srv/cassandra-a/data_1
 - /srv/cassandra-a/data_2
 - /srv/cassandra-a/data_3
-  jmx_exporter_enabled: true
 b:
   listen_address: 10.64.0.168
   data_file_directories:
@@ -97,33 +96,28 @@
 - /srv/cassandra-b/data_1
 - /srv/cassandra-b/data_2
 - /srv/cassandra-b/data_3
-  jmx_exporter_enabled: true
   "restbase-dev1005.eqiad.wmnet":
 a:
   listen_address: 10.64.16.97
   data_file_directories:
 - /srv/cassandra-a/data_0
 - /srv/cassandra-a/data_1
-  jmx_exporter_enabled: true
 b:
   listen_address: 10.64.16.98
   data_file_directories:
 - /srv/cassandra-b/data_0
 - /srv/cassandra-b/data_1
-  jmx_exporter_enabled: true
   "restbase-dev1006.eqiad.wmnet":
 a:
   listen_address: 10.64.48.168
   data_file_directories:
 - /srv/cassandra-a/data_0
 - /srv/cassandra-a/data_1
-  jmx_exporter_enabled: true
 b:
   listen_address: 10.64.48.169
   data_file_directories:
 - /srv/cassandra-b/data_0
 - /srv/cassandra-b/data_1
-  jmx_exporter_enabled: true
 
 profile::cassandra::settings:
   start_rpc: 'true'
@@ -147,6 +141,7 @@
 internode_encryption: dc
 permissions_validity_in_ms: 60
 trickle_fsync: false
+jmx_exporter_enabled: true
 
 service::configuration::log_dir: "/var/log"
 
diff --git a/hieradata/role/common/restbase/production_ng.yaml 
b/hieradata/role/common/restbase/production_ng.yaml
index 7c8c7e2..8ca3f7f 100644
--- a/hieradata/role/common/restbase/production_ng.yaml
+++ b/hieradata/role/common/restbase/production_ng.yaml
@@ -54,7 +54,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-a/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-a/hints
   saved_caches_directory: 
/srv/cassandra/instance-data/cassandra-a/saved_caches
-  jmx_exporter_enabled: true
 b:
   listen_address: 10.64.32.195
   data_file_directories:
@@ -67,7 +66,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-b/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-b/hints
   saved_caches_directory: 
/srv/cassandra/instance-data/cassandra-b/saved_caches
-  jmx_exporter_enabled: true
 c:
   listen_address: 10.64.32.196
   data_file_directories:
@@ -80,7 +78,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-c/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-c/hints
   saved_caches_directory: 
/srv/cassandra/instance-data/cassandra-c/saved_caches
-  jmx_exporter_enabled: true
   "restbase1009.eqiad.wmnet":
 a:
   listen_address: 10.64.48.120
@@ -94,7 +91,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-a/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-a/hints
   saved_caches_directory: 
/srv/cassandra/instance-data/cassandra-a/saved_caches
-  jmx_exporter_enabled: true
 b:
   listen_address: 10.64.48.130
   data_file_directories:
@@ -107,7 +103,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-b/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-b/hints
   saved_caches_directory: 
/srv/cassandra/instance-data/cassandra-b/saved_caches
-  jmx_exporter_enabled: true
 c:
   listen_address: 10.64.48.131
   data_file_directories:
@@ -120,7 +115,6 @@
   commitlog_directory: /srv/cassandra/instance-data/cassandra-c/commitlog
   hints_directory: /srv/cassandra/instance-data/cassandra-c/hints
   saved_caches_directory: 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] smart: new module

[Filippo Giunchedi (Code Review)]

Filippo Giunchedi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/378039 )

Change subject: [WIP] smart: new module
..

[WIP] smart: new module

Report smart attributes as Prometheus metrics

Bug: T86552
Change-Id: I324f23acb64f5b7c4e8250e9aba3374c5ceba22b
---
A modules/smart/files/20logger
A modules/smart/files/smart-data-dump
A modules/smart/manifests/init.pp
3 files changed, 298 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/378039/1

diff --git a/modules/smart/files/20logger b/modules/smart/files/20logger
new file mode 100644
index 000..4ddc291
--- /dev/null
+++ b/modules/smart/files/20logger
@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+# Send smartd warnings/errors via syslog instead of mail
+
+if ! [ -x /usr/bin/logger ]; then
+echo "Your system does not have /usr/bin/logger.  Install the bsdutils 
package."
+exit 1
+fi
+
+input=$1
+shift
+
+/usr/bin/logger -p local0.warning -t smart_failure < $input
diff --git a/modules/smart/files/smart-data-dump 
b/modules/smart/files/smart-data-dump
new file mode 100644
index 000..c56ed54
--- /dev/null
+++ b/modules/smart/files/smart-data-dump
@@ -0,0 +1,251 @@
+#!/usr/bin/python3
+
+import argparse
+import collections
+import json
+import logging
+import re
+import subprocess
+import sys
+
+from prometheus_client import CollectorRegistry, Gauge, write_to_textfile
+from prometheus_client.exposition import generate_latest
+
+log = logging.getLogger(__name__)
+PD = collections.namedtuple('PD', ['driver', 'smart_args', 'disk_id'])
+REPORT_ATTRIBUTES = [
+'airflow_temperature_cel',
+'command_timeout',
+'current_pending_sector',
+'end_to_end_error',
+'erase_fail_count',
+'g_sense_error_rate',
+'hardware_ecc_recovered',
+'host_reads_mib',
+'host_reads_32mib',
+'host_writes_mib',
+'host_writes_32mib',
+'load_cycle_count',
+'media_wearout_indicator',
+'nand_writes_1gib',
+'offline_uncorrectable',
+'power_cycle_count',
+'power_on_hours',
+'program_fail_count',
+'raw_read_error_rate',
+'reallocated_sector_ct',
+'reported_uncorrect',
+'sata_downshift_count',
+'spin_retry_count',
+'spin_up_time',
+'start_stop_count',
+'temperature_celsius',
+'total_lbas_read',
+'total_lbas_written',
+'udma_crc_error_count',
+'unsafe_shutdown_count',
+'workld_host_reads_perc',
+'workld_media_wear_indic',
+'workload_minutes',
+]
+
+
+def get_fact(fact_name):
+raw_output = subprocess.check_output(['/usr/bin/facter', '--puppet',
+ '--json', fact_name])
+try:
+fact_value = json.loads(raw_output).get(fact_name, None)
+except ValueError:
+return None
+
+log.debug('Fact "raid" discovered: %r', fact_value)
+return fact_value
+
+
+def megaraid_list_pd():
+raw_output = subprocess.check_output(['/usr/bin/timeout', '30', 
'/usr/sbin/smartctl', '--scan-open'])
+for line in raw_output.splitlines():
+if 'megaraid,' not in line:
+continue
+bus, _, device, _ = line.split(' ', 3)
+yield PD(driver='megaraid', smart_args=['-d', device, bus],
+ disk_id=device)
+
+
+def hpsa_list_pd():
+raw_output = subprocess.check_output(['/usr/bin/timeout', '30', 
'/usr/sbin/hpssacli', 'controller',
+'all', 'show', 'config'])
+in_controller = False
+for line in raw_output.splitlines():
+m = re.match('^Smart Array .* in Slot (\d+)', line)
+if m:
+in_controller = True
+disk_id = 0
+
+m = re.match('^\s+physicaldrive', line)
+if m and in_controller:
+device = 'cciss,%s' % disk_id
+# TODO(filippo) assumes /dev/sda
+yield PD(driver='cciss', disk_id=device,
+ smart_args=['-d', device, '/dev/sda'])
+disk_id += 1
+
+
+def noraid_list_pd():
+# starting with stretch, lsblk has --json but not on trusty/jessie
+raw_output = subprocess.check_output(['/usr/bin/timeout', '30',
+'/bin/lsblk', '--noheadings', '--output', 'NAME,TYPE',
+'--raw'])
+for line in raw_output.splitlines():
+name, disk_type = line.split(' ', 1)
+if disk_type != 'disk':
+continue
+yield PD(driver='noraid', smart_args=['-d', 'auto', '/dev/%s' % name],
+disk_id=name)
+
+
+def _run_smartctl(args, timeout=30):
+try:
+cmd_args = ['/usr/bin/timeout', str(timeout), '/usr/sbin/smartctl']
+cmd_args.extend(args)
+log.debug('Running: %s', ' '.join(cmd_args))
+raw_output = subprocess.check_output(cmd_args)
+except subprocess.CalledProcessError as e:
+# TODO(filippo) handle non-fatal exit codes
+raw_output = e.output
+pass
+return raw_output
+
+
+def collect_smart_metrics(disks, registry):
+

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Cassandra: Include only instance DNS' in the list of seeds

[Mobrovac (Code Review)]

Mobrovac has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/377997 )

Change subject: WIP: Cassandra: Include only instance DNS' in the list of seeds
..

WIP: Cassandra: Include only instance DNS' in the list of seeds

Change-Id: I0a00f1d8efb143edfe18a8749dfbda3ba32753e3
---
M modules/cassandra/templates/cassandra.yaml-2.1.erb
M modules/cassandra/templates/cassandra.yaml-2.2.erb
M modules/cassandra/templates/cassandra.yaml-3.x.erb
M modules/profile/manifests/cassandra.pp
M modules/profile/templates/cassandra/seeds.erb
5 files changed, 14 insertions(+), 17 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/97/377997/1

diff --git a/modules/cassandra/templates/cassandra.yaml-2.1.erb 
b/modules/cassandra/templates/cassandra.yaml-2.1.erb
index 63775da..12f9d80 100644
--- a/modules/cassandra/templates/cassandra.yaml-2.1.erb
+++ b/modules/cassandra/templates/cassandra.yaml-2.1.erb
@@ -273,10 +273,7 @@
   x != @hostname \
   && ! (x.start_with? 
"#{@hostname}-") \
   && x != @ipaddress \
-  && x != @fqdn \
-  && 
(/^.+-[a-z]\.(?:eqiad|codfw)\.wmnet$/ =~ x \
-|| @instance_count == 
1 \
-|| 
/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ =~ x)
+  && x != @fqdn)
   }.join(',') : 
@seeds.join(',') %>
  - seeds: <%= clean_seeds %>
 # For workloads with more data than can fit in memory, Cassandra's
diff --git a/modules/cassandra/templates/cassandra.yaml-2.2.erb 
b/modules/cassandra/templates/cassandra.yaml-2.2.erb
index cdf6404..c0ff8d3 100644
--- a/modules/cassandra/templates/cassandra.yaml-2.2.erb
+++ b/modules/cassandra/templates/cassandra.yaml-2.2.erb
@@ -318,10 +318,7 @@
   x != @hostname \
   && ! (x.start_with? 
"#{@hostname}-") \
   && x != @ipaddress \
-  && x != @fqdn \
-  && 
(/^.+-[a-z]\.(?:eqiad|codfw)\.wmnet$/ =~ x \
-|| @instance_count == 
1 \
-|| 
/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ =~ x)
+  && x != @fqdn)
   }.join(',') : 
@seeds.join(',') %>
  - seeds: <%= clean_seeds %>
 # For workloads with more data than can fit in memory, Cassandra's
diff --git a/modules/cassandra/templates/cassandra.yaml-3.x.erb 
b/modules/cassandra/templates/cassandra.yaml-3.x.erb
index 46edbb3..95281db 100644
--- a/modules/cassandra/templates/cassandra.yaml-3.x.erb
+++ b/modules/cassandra/templates/cassandra.yaml-3.x.erb
@@ -404,10 +404,7 @@
   x != @hostname \
   && ! (x.start_with? 
"#{@hostname}-") \
   && x != @ipaddress \
-  && x != @fqdn \
-  && 
(/^.+-[a-z]\.(?:eqiad|codfw)\.wmnet$/ =~ x \
-|| @instance_count == 
1 \
-|| 
/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ =~ x)
+  && x != @fqdn)
   }.join(',') : 
@seeds.join(',') %>
  - seeds: <%= clean_seeds %>
 # For workloads with more data than can fit in memory, Cassandra's
diff --git a/modules/profile/manifests/cassandra.pp 
b/modules/profile/manifests/cassandra.pp
index 629b0f4..217724d 100644
--- a/modules/profile/manifests/cassandra.pp
+++ b/modules/profile/manifests/cassandra.pp
@@ -14,7 +14,9 @@
 $instances = $all_instances[$::fqdn]
 # We get the cassandra seeds from $all_instances, with a template hack
 # This is preferred over a very specialized parser function.
-$seeds = split(template('profile/cassandra/seeds.erb'), '\|')
+$all_seeds = split(template('profile/cassandra/seeds.erb'), '\|')
+$seeds = split($all_seeds[0], ',')
+$ferm_seeds = split($all_seeds[1], ',')
 
 $base_settings = {
 'instances' => $instances,
@@ -70,7 +72,7 @@
 description => 'Cassandra server',
 }
 
-

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] role::kafka::jumbo::broker: enable Prometheus JMX moni...

[Elukey (Code Review)]

Elukey has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/377753 )

Change subject: [WIP] role::kafka::jumbo::broker: enable Prometheus JMX 
monitoring
..

[WIP] role::kafka::jumbo::broker: enable Prometheus JMX monitoring

Bug: T167992
Change-Id: Icb26d40cf4f9b6661d3560f314e073df075d3c1f
---
M hieradata/role/common/kafka/jumbo/broker.yaml
M hieradata/role/common/kafka/simple/broker.yaml
A modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/kafka.default.erb
M modules/profile/manifests/kafka/broker.pp
6 files changed, 67 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/53/377753/1

diff --git a/hieradata/role/common/kafka/jumbo/broker.yaml 
b/hieradata/role/common/kafka/jumbo/broker.yaml
index 2960a76..9e6ea8a 100644
--- a/hieradata/role/common/kafka/jumbo/broker.yaml
+++ b/hieradata/role/common/kafka/jumbo/broker.yaml
@@ -2,6 +2,9 @@
 
 profile::kafka::broker::kafka_cluster_name: jumbo
 
+# Enable the Prometheus JMX Exporter
+profile::kafka::broker::jmx_exporter_enabled: true
+
 profile::kafka::broker::log_dirs: [/srv/kafka/data]
 profile::kafka::broker::plaintext: true
 # TODO: does not yet work.
diff --git a/hieradata/role/common/kafka/simple/broker.yaml 
b/hieradata/role/common/kafka/simple/broker.yaml
index 0ac2c22..a41c40a 100644
--- a/hieradata/role/common/kafka/simple/broker.yaml
+++ b/hieradata/role/common/kafka/simple/broker.yaml
@@ -1,5 +1,6 @@
 kafka_cluster_name: simple
 
+profile::kafka::broker::jmx_exporter_enabled: false
 profile::kafka::broker::log_dirs: [/srv/kafka/data]
 profile::kafka::broker::plaintext: true
 profile::kafka::broker::tls_secrets_path: false
diff --git a/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml 
b/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
new file mode 100644
index 000..1e927ac
--- /dev/null
+++ b/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
@@ -0,0 +1,14 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: true
+# Examples from Cassandra:
+# rules:
+#   - pattern: 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Avoid warnings for invalid lines in reverse-stack mode

[Aaron Schulz (Code Review)]

Aaron Schulz has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/377451 )

Change subject: [WIP] Avoid warnings for invalid lines in reverse-stack mode
..

[WIP] Avoid warnings for invalid lines in reverse-stack mode

Bug: T169249
Change-Id: I43e21d83d8fc8757d418e2f4baec0181cc752062
---
M modules/xenon/files/flamegraph.pl
1 file changed, 5 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/51/377451/1

diff --git a/modules/xenon/files/flamegraph.pl 
b/modules/xenon/files/flamegraph.pl
index bf00a04..4a94fe5 100755
--- a/modules/xenon/files/flamegraph.pl
+++ b/modules/xenon/files/flamegraph.pl
@@ -495,6 +495,11 @@
# there may be an extra samples column for differentials
# XXX todo: redo these REs as one. It's repeated below.
my ($stack, $samples) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/);
+   unless (defined $samples and defined $stack) {
+   # Invalid line will be ignored later on
+   unshift @Data, $line;
+   next;
+   }
my $samples2 = undef;
if ($stack =~ /^(.*)\s+?(\d+(?:\.\d*)?)$/) {
$samples2 = $samples;

-- 
To view, visit https://gerrit.wikimedia.org/r/377451
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I43e21d83d8fc8757d418e2f4baec0181cc752062
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Aaron Schulz 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: nova: turn off hourly instance usage audits

[Andrew Bogott (Code Review)]

Andrew Bogott has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/377187 )

Change subject: WIP: nova: turn off hourly instance usage audits
..

WIP: nova: turn off hourly instance usage audits

These were useful for keeping the 'wikistatus' pages
in sync, but we don't do that anymore.  We can probably
live without these messages entirely, but this needs
some tests on labtest before we can safely merge
in production.  In particular, I'm curious about whether
this will break the System:Hypervisors statistics
in Horizon.

Change-Id: I3255975eddf2acf222bc46d3898c2d9af30f4d8a
---
M modules/openstack/templates/liberty/nova/nova.conf.erb
M modules/openstack/templates/mitaka/nova/nova.conf.erb
M modules/openstack2/templates/liberty/nova/common/nova.conf.erb
3 files changed, 6 insertions(+), 13 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/87/377187/1

diff --git a/modules/openstack/templates/liberty/nova/nova.conf.erb 
b/modules/openstack/templates/liberty/nova/nova.conf.erb
index 89f8fb5..b9def5f 100644
--- a/modules/openstack/templates/liberty/nova/nova.conf.erb
+++ b/modules/openstack/templates/liberty/nova/nova.conf.erb
@@ -3,7 +3,7 @@
 verbose=True
 auth_strategy=keystone
 compute_driver=nova.virt.libvirt.LibvirtDriver
-notification_topics=notifications,ceilometer_notifications
+notification_topics=notifications
 connection_type=libvirt
 root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
 instance_name_template=i-%08x
@@ -77,10 +77,7 @@
 notification_driver = messagingv2
 
 # Ceilometer things:
-instance_usage_audit = True
-instance_usage_audit_period = hour
-notify_on_state_change = vm_and_task_state
-
+instance_usage_audit = False
 
 
 # Deprecated, remove in Kilo:
diff --git a/modules/openstack/templates/mitaka/nova/nova.conf.erb 
b/modules/openstack/templates/mitaka/nova/nova.conf.erb
index 351ee79..8f285f7 100644
--- a/modules/openstack/templates/mitaka/nova/nova.conf.erb
+++ b/modules/openstack/templates/mitaka/nova/nova.conf.erb
@@ -3,7 +3,7 @@
 verbose=True
 auth_strategy=keystone
 compute_driver=nova.virt.libvirt.LibvirtDriver
-notification_topics=notifications,ceilometer_notifications
+notification_topics=notifications
 connection_type=libvirt
 root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
 instance_name_template=i-%08x
@@ -75,9 +75,7 @@
 notification_driver = messagingv2
 
 # Ceilometer things:
-instance_usage_audit = True
-instance_usage_audit_period = hour
-notify_on_state_change = vm_and_task_state
+instance_usage_audit = False
 
 # Deprecated, remove in Kilo:
 node_availability_zone=<%= @novaconfig["zone"] %>
diff --git a/modules/openstack2/templates/liberty/nova/common/nova.conf.erb 
b/modules/openstack2/templates/liberty/nova/common/nova.conf.erb
index 01b072e..7c545a6 100644
--- a/modules/openstack2/templates/liberty/nova/common/nova.conf.erb
+++ b/modules/openstack2/templates/liberty/nova/common/nova.conf.erb
@@ -3,7 +3,7 @@
 verbose=True
 auth_strategy=keystone
 compute_driver=nova.virt.libvirt.LibvirtDriver
-notification_topics=notifications,ceilometer_notifications
+notification_topics=notifications
 connection_type=libvirt
 root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
 instance_name_template=i-%08x
@@ -77,9 +77,7 @@
 notification_driver = messagingv2
 
 # Ceilometer things:
-instance_usage_audit = True
-instance_usage_audit_period = hour
-notify_on_state_change = vm_and_task_state
+instance_usage_audit = False
 
 # When user doesn't specify a scheduling zone, use 'nova' which
 #  has been the only zone (and, hence, the de-facto default)

-- 
To view, visit https://gerrit.wikimedia.org/r/377187
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3255975eddf2acf222bc46d3898c2d9af30f4d8a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Andrew Bogott 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] stabilize backend storage patterns

[BBlack (Code Review)]

BBlack has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/376751 )

Change subject: [WIP] stabilize backend storage patterns
..

[WIP] stabilize backend storage patterns

As discussed on IRC, this may help reduce the mailbox lag problem
and improve the backend hitrates (for local fe traffic) at the
core DCs.  The basic idea is that for be->be fetches, the
receiving backend should operate in a hit-or-pass mode: it will
serve existing object hits, but will not bring new misses into its
cached set.  A hieradata setting should control the
"@backend_warming" flag in the VCL, for the operationally-rare
case where we want to disable this behavior when warming an inner
cache using an outer's misses.

Note this change also modifies the calculation of X-Cache-Status
wrt miss-vs-pass.  We previously only had known cases of
layer-differential pass behavior in a single direction (where the
FE might pass but the BE would attempt real hit/miss), but now we
have layer-differential pass possible in both directions (an outer
be might miss, while the inner be's all pass).  In light of this,
it probably makes more sense to consider X-Cache lines containing
any miss at any layer to be a true miss, and only count a pass
when every layer passes the traffic.  This could cause small
sudden jumps in the existing miss/pass stats due to the
re-interpration.

TODO: hook up the @backend_warming flag, modulo bikeshedding about
various puppet coding rules and ongoing refactors.

Change-Id: I17f02801f1d18ee0b27521f182fd4e041c6dd8fb
---
M modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
M modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
2 files changed, 28 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/51/376751/1

diff --git a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb 
b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
index 5f63062..f745a5d 100644
--- a/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-backend.vcl.erb
@@ -80,8 +80,25 @@
return (synth(403, "Access denied"));
}
 
-   // Used to signal to other logic whether this request jumps next to 
another cache or the applayer
-   unset req.http.X-Next-Is-Cache;
+   // X-Next-Is-Cache is used to signal to other logic whether this
+   // request jumps next to another cache or the applayer.  We also
+   // convert such a signal sent by a previous backend cache to
+   // X-MISS2PASS, which converts misses to passes (we're only looking for
+   // opporunistic hits on these, but don't want to disturb the local
+   // backend cache's object hotness/distribution).  hash_ignore_busy is
+   // to avoid stalling on those passes (like a cheaper hfp).  The
+   // backend_warming setting is meant to be set temporarily from
+   // hieradata to disable this behavior (when we want to use the misses
+   // from an outer-layer backend to pre-warm some data in the next layer
+   // deep of backend).
+   unset req.http.X-MISS2PASS;
+   if (req.http.X-Next-Is-Cache) {
+   unset req.http.X-Next-Is-Cache;
+   <% if not @backend_warming -%> // XXX no implementation of 
setting this flag yet
+   set req.http.X-MISS2PASS = 1;
+   set req.hash_ignore_busy = true;
+   <% end -%>
+   }
 
call wm_common_recv_early;
 
@@ -133,6 +150,10 @@
 sub vcl_miss {
call wm_common_miss;
call cluster_be_miss;
+   if (req.http.X-MISS2PASS) {
+   unset req.http.X-MISS2PASS;
+   return (pass);
+   }
return (fetch); // no default VCL (which is just "return (fetch)" 
anyways)
 }
 
diff --git a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb 
b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
index 1e17089..83b5a06 100644
--- a/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
+++ b/modules/varnish/templates/vcl/wikimedia-frontend.vcl.erb
@@ -229,7 +229,9 @@
error 403 "This server has been disabled, you probably ended up here 
because of hardcoded IPs or broken DNS caching";
 <% end -%>
 
-   unset req.http.X-DCPath; // no injection from outside our stack
+   // no injection from outside our stack allowed for these
+   unset req.http.X-DCPath;
+   unset req.http.X-Next-Is-Cache;
unset req.http.Proxy; // https://httpoxy.org/
 
if (req.restarts == 0) {
@@ -367,15 +369,10 @@
set resp.http.X-Cache-Status = "hit";
} elsif (resp.http.X-Cache ~ "int") {
set resp.http.X-Cache-Status = "int";
-   } elsif (resp.http.X-Cache ~ "pass,[^,]+$") {
-   // The special pass regex above is because we have
-   // layer-differential 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: icinga: add check_sysctl.sh script

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/376566 )

Change subject: WIP: icinga: add check_sysctl.sh script
..

WIP: icinga: add check_sysctl.sh script

Add check_sysctl.sh script to compare configured sysctl values
to running values.

Bug: T160060
Change-Id: I5c6aaa746e81d1733074a102e78372ac55a09901
---
A modules/icinga/files/check_sysctl.sh
1 file changed, 85 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/66/376566/1

diff --git a/modules/icinga/files/check_sysctl.sh 
b/modules/icinga/files/check_sysctl.sh
new file mode 100644
index 000..0618ea6
--- /dev/null
+++ b/modules/icinga/files/check_sysctl.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# check_sysctl.sh - check running sysctl values against configuration files
+# note: this will print only one mismatch to keep alert text concise
+# 2017 Keith Herron 
+
+# Command locations
+sysctl_cmd="/sbin/sysctl"
+
+function print_help() {
+  echo "
+$0 - check sysctl config file(s) against running values
+
+usage: $0 -f 
+
+options:
+  -f  Required - Sysctl configuration file location(s).  To supply 
multiple
+  files use a space delimited list.
+  -h  Print this help text
+"
+
+  exit 3
+}
+
+# Check that options were provided
+if [ $# -lt 1 ]; then
+  print_help
+fi
+
+# Gather options.  -f requres an argument -h does not.
+while getopts 'f:h' OPT; do
+  case $OPT in
+f)  files=$OPTARG;;
+h)  print_help;;
+*)  print_help;;
+  esac
+done
+
+# Check if provided file(s) exist.
+for file in ${files}; do
+  if [ ! -r $file ]; then
+echo "error: config file $file does not exist"
+print_help
+exit 1
+  fi
+done
+
+for file in ${files}; do
+
+  while read -r line; do
+
+# Skip lines that do not begin with an alphanumeric
+[[ "$line" =~ ^[:alnum:] ]] || continue
+
+# Remove whitespace from line
+line=${line//[[:space:]]/}
+
+# Split line into key/val variables using = delimiter
+configured_key=${line%=*};
+configured_val=${line#*=};
+
+running_val=`$sysctl_cmd -b "${configured_key}" 2>/dev/null`
+
+if [ $running_val ]; then
+  if [ ${running_val} != ${configured_val} ]; then
+echo -n "WARNING: "
+echo "${configured_key}" running value ${running_val} does not match 
value of "${configured_val} configured in ${file}"
+exit 1
+  else
+matched=yes
+  fi
+fi
+
+  done < "$file"
+
+done
+
+if [ $matched ]; then
+echo -n "OK: "
+echo "Running sysctl values match config file(s) $files"
+exit 0
+else
+echo "UNKNOWN: No entries in this file match matched running values.  Are 
you sure this is a sysctl config file?"
+exit 3
+fi

-- 
To view, visit https://gerrit.wikimedia.org/r/376566
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5c6aaa746e81d1733074a102e78372ac55a09901
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Change check_ipmi_temp to check_ipmi_sensor and monitor...

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/376048 )

Change subject: WIP: Change check_ipmi_temp to check_ipmi_sensor and monitor 
PSUs
..

WIP: Change check_ipmi_temp to check_ipmi_sensor and monitor PSUs

Repurpose existing ipmi temp check as generic ipmi sensor check.  Rename
check_ipmi_temperature to check_ipmi_sensor.  Monitor Power_Supply sensor.

Bug: T109903
Change-Id: Ic4a764fb92f9f4f75d2afc59b520a1f8faf2b07c
---
M modules/ipmi/manifests/monitor.pp
1 file changed, 9 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/48/376048/1

diff --git a/modules/ipmi/manifests/monitor.pp 
b/modules/ipmi/manifests/monitor.pp
index 964f69f..2690a63 100644
--- a/modules/ipmi/manifests/monitor.pp
+++ b/modules/ipmi/manifests/monitor.pp
@@ -14,14 +14,20 @@
 ensure => present,
 }
 
+# rename nagios_ipmi_temp to nagios_ipmi_sensor
+# this ensure absent can be removed after propagation
 ::sudo::user { 'nagios_ipmi_temp':
+ensure => absent,
+}
+
+::sudo::user { 'nagios_ipmi_sensor':
 user   => 'nagios',
 privileges => ['ALL = NOPASSWD: /usr/sbin/ipmi-sel, 
/usr/sbin/ipmi-sensors'],
 }
 
-nrpe::monitor_service { 'check_ipmi_temp':
-description=> 'IPMI Temperature',
-nrpe_command   => '/usr/local/lib/nagios/plugins/check_ipmi_sensor 
--noentityabsent -T Temperature -ST Temperature --nosel',
+nrpe::monitor_service { 'check_ipmi_sensor':
+description=> 'IPMI Sensor Status',
+nrpe_command   => '/usr/local/lib/nagios/plugins/check_ipmi_sensor 
--noentityabsent -T Temperature -T Power_Supply --nosel',
 check_interval => 30,
 retry_interval => 10,
 timeout=> 60,

-- 
To view, visit https://gerrit.wikimedia.org/r/376048
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic4a764fb92f9f4f75d2afc59b520a1f8faf2b07c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add letsencrypt certs to mx servers

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/375427 )

Change subject: WIP: Add letsencrypt certs to mx servers
..

WIP: Add letsencrypt certs to mx servers

Add letsencrypt cert to mail exchangers with letsencrypt::cert::integrated
and deploy standalone nginx instances to handle letsencrypt validation. Cut
over to new certs with a follow-up exim config change.

WIP because this depends on https://gerrit.wikimedia.org/r/375071 or
an alternate solution to T174720

Bug: T174081
Change-Id: I06575b1883b71fe0870679e3fd6765bacdcf5f51
---
M modules/role/manifests/mail/mx.pp
1 file changed, 15 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/27/375427/1

diff --git a/modules/role/manifests/mail/mx.pp 
b/modules/role/manifests/mail/mx.pp
index b8fdce9..dba4596 100644
--- a/modules/role/manifests/mail/mx.pp
+++ b/modules/role/manifests/mail/mx.pp
@@ -23,6 +23,21 @@
 before => Class['exim4'],
 }
 
+letsencrypt::cert::integrated { "${facts['hostname']}":
+subjects   => $facts['fqdn'],
+key_group  => 'Debian-exim',
+puppet_svc => 'nginx',
+system_svc => 'nginx',
+}
+
+class { 'nginx':
+variant => 'light',
+}
+
+nginx::site { ‘letsencrypt-standalone’:
+ content => 
template(‘letsencrypt/cert/integrated/standalone.nginx.erb'),
+}
+
 class { 'spamassassin':
 required_score   => '4.0',
 use_bayes=> '1',

-- 
To view, visit https://gerrit.wikimedia.org/r/375427
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I06575b1883b71fe0870679e3fd6765bacdcf5f51
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] keystone: Create top-level domain for each new project

[Alex Monk (Code Review)]

Alex Monk has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/375089 )

Change subject: [WIP] keystone: Create top-level domain for each new project
..

[WIP] keystone: Create top-level domain for each new project

I have no way of testing this.

Bug: T162977
Change-Id: I3994c66127a592fe99644782286033ca1eeb29c0
---
M modules/openstack/files/liberty/keystone/wmfkeystonehooks/wmfkeystonehooks.py
M modules/openstack/manifests/horizon/service.pp
M modules/openstack/manifests/keystone/hooks.pp
M modules/openstack2/manifests/keystone/hooks.pp
4 files changed, 24 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/89/375089/1

diff --git 
a/modules/openstack/files/liberty/keystone/wmfkeystonehooks/wmfkeystonehooks.py 
b/modules/openstack/files/liberty/keystone/wmfkeystonehooks/wmfkeystonehooks.py
index f6aaa96..46e5986 100644
--- 
a/modules/openstack/files/liberty/keystone/wmfkeystonehooks/wmfkeystonehooks.py
+++ 
b/modules/openstack/files/liberty/keystone/wmfkeystonehooks/wmfkeystonehooks.py
@@ -18,6 +18,7 @@
 from keystone import exception
 from novaclient import client as nova_client
 from novaclient import exceptions
+import designatemakedomain
 
 from oslo_log import log as logging
 from oslo_config import cfg
@@ -162,6 +163,14 @@
  project_id,
  
roledict[CONF.wmfhooks.observer_role_name])
 
+designatemakedomain.createdomain(
+CONF.wmfhooks.auth_url,
+CONF.wmfhooks.admin_user,
+CONF.wmfhooks.admin_pass,
+project_id,
+'{}.wmflabs.org'.format(project_id)
+)
+
 # Use the nova api to set up security groups for the new project
 auth = generic.Password(
 auth_url=CONF.wmfhooks.auth_url,
diff --git a/modules/openstack/manifests/horizon/service.pp 
b/modules/openstack/manifests/horizon/service.pp
index b7ce1a9..6348815 100644
--- a/modules/openstack/manifests/horizon/service.pp
+++ b/modules/openstack/manifests/horizon/service.pp
@@ -262,7 +262,7 @@
 mode=> '0644',
 }
 
-# Arbitrary handy script that needs to be on the horizon host because it 
only works with Liberty
+# Arbitrary handy script that historically needed to be on the horizon 
host because it only works with Liberty and above. From an era when everything 
else ran Kilo
 file { '/root/makedomain':
 source => 
"puppet:///modules/openstack2/${openstack_version}/admin_scripts/makedomain",
 owner  => 'root',
diff --git a/modules/openstack/manifests/keystone/hooks.pp 
b/modules/openstack/manifests/keystone/hooks.pp
index 5b7cd74..012e06c 100644
--- a/modules/openstack/manifests/keystone/hooks.pp
+++ b/modules/openstack/manifests/keystone/hooks.pp
@@ -3,6 +3,13 @@
 class openstack::keystone::hooks(
 $openstack_version  = $::openstack::version)
 {
+file { '/usr/lib/python2.7/dist-packages/designatemakedomain.py':
+source => 
"puppet:///modules/openstack2/${openstack_version}/admin_scripts/makedomain",
+owner  => 'root',
+group  => 'root',
+mode   => '0744',
+}
+
 file { '/usr/lib/python2.7/dist-packages/wmfkeystonehooks':
 source  => 
"puppet:///modules/openstack/${openstack_version}/keystone/wmfkeystonehooks",
 owner   => 'root',
diff --git a/modules/openstack2/manifests/keystone/hooks.pp 
b/modules/openstack2/manifests/keystone/hooks.pp
index a99fef0..f11541a 100644
--- a/modules/openstack2/manifests/keystone/hooks.pp
+++ b/modules/openstack2/manifests/keystone/hooks.pp
@@ -5,6 +5,13 @@
 ) {
 include openstack2::keystone::service
 
+file { '/usr/lib/python2.7/dist-packages/designatemakedomain.py':
+source => 
"puppet:///modules/openstack2/${openstack_version}/admin_scripts/makedomain",
+owner  => 'root',
+group  => 'root',
+mode   => '0744',
+}
+
 file { '/usr/lib/python2.7/dist-packages/wmfkeystonehooks':
 source  => 
"puppet:///modules/openstack/${version}/keystone/wmfkeystonehooks",
 owner   => 'root',

-- 
To view, visit https://gerrit.wikimedia.org/r/375089
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3994c66127a592fe99644782286033ca1eeb29c0
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alex Monk 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add standalone letsencrypt nginx template

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/375071 )

Change subject: WIP: Add standalone letsencrypt nginx template
..

WIP: Add standalone letsencrypt nginx template

Add a standalone nginx config template intended for systems that utilize
letsencrypt certs for non-http protocols (smtps, ldaps, etc.) and do
not have an existing http server to include the acme challenge file.

Bug: T174720
Change-Id: I17fc758dc63973fba0c1d3ce1cfbba9299d17b79
---
A modules/letsencrypt/templates/cert/integrated/standalone.nginx.erb
1 file changed, 17 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/71/375071/1

diff --git a/modules/letsencrypt/templates/cert/integrated/standalone.nginx.erb 
b/modules/letsencrypt/templates/cert/integrated/standalone.nginx.erb
new file mode 100644
index 000..ee924f8
--- /dev/null
+++ b/modules/letsencrypt/templates/cert/integrated/standalone.nginx.erb
@@ -0,0 +1,17 @@
+# This file managed by Puppet
+#
+# Standalone nginx config file intended for systems that utilize
+# letsencrypt certs for non-http protocols (smtps, ldaps, etc.) and do
+# not have an existing http server to include the acme challenge file.
+
+server {
+listen 80;
+<%
+server_name @facts['hostname']  @facts['fqdn'];
+-%>
+
+include /etc/acme/challenge-nginx.conf;
+location / {
+return 403;
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/375071
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I17fc758dc63973fba0c1d3ce1cfbba9299d17b79
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add reportupdater job to trigger page-creation metrics

[Ottomata (Code Review)]

Ottomata has submitted this change and it was merged. ( 
https://gerrit.wikimedia.org/r/374878 )

Change subject: [WIP] Add reportupdater job to trigger page-creation metrics
..


[WIP] Add reportupdater job to trigger page-creation metrics

Bug: T170850
Change-Id: I4f13143a6c8716726459c445c962e47d19f4b35f
---
M modules/profile/manifests/reportupdater/jobs/mysql.pp
1 file changed, 4 insertions(+), 0 deletions(-)

Approvals:
  Mforns: Looks good to me, but someone else must approve
  Ottomata: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/modules/profile/manifests/reportupdater/jobs/mysql.pp 
b/modules/profile/manifests/reportupdater/jobs/mysql.pp
index cba6fe7..7b02ad0 100644
--- a/modules/profile/manifests/reportupdater/jobs/mysql.pp
+++ b/modules/profile/manifests/reportupdater/jobs/mysql.pp
@@ -71,4 +71,8 @@
 repository => 'discovery-stats',
 output_dir => 'metrics/interactive',
 }
+reportupdater::job { 'page-creation':
+repository => 'reportupdater-queries',
+output_dir => 'metrics/page-creation',
+}
 }
\ No newline at end of file

-- 
To view, visit https://gerrit.wikimedia.org/r/374878
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I4f13143a6c8716726459c445c962e47d19f4b35f
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mforns 
Gerrit-Reviewer: Mforns 
Gerrit-Reviewer: Ottomata 
Gerrit-Reviewer: jenkins-bot <>

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP jbod config for cassandra

[Filippo Giunchedi (Code Review)]

Filippo Giunchedi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/374994 )

Change subject: WIP jbod config for cassandra
..

WIP jbod config for cassandra

Change-Id: I456261d7446ef131ebf6b9c292e8961816f0f5a8
---
M hieradata/hosts/restbase2001.yaml
M modules/cassandra/manifests/init.pp
A modules/cassandra/manifests/jbod_device.pp
3 files changed, 43 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/94/374994/1

diff --git a/hieradata/hosts/restbase2001.yaml 
b/hieradata/hosts/restbase2001.yaml
index 062634b..db38b13 100644
--- a/hieradata/hosts/restbase2001.yaml
+++ b/hieradata/hosts/restbase2001.yaml
@@ -1 +1,2 @@
 profile::cassandra::rack: 'b'
+cassandra::jbod_devices: ['sda', 'sdb', 'sdc', 'sdd', 'sde']
diff --git a/modules/cassandra/manifests/init.pp 
b/modules/cassandra/manifests/init.pp
index 1c40774..a168807 100644
--- a/modules/cassandra/manifests/init.pp
+++ b/modules/cassandra/manifests/init.pp
@@ -457,6 +457,9 @@
 }
 }
 
+$jbod_devices = hiera('cassandra::jbod_devices', [])
+cassandra::jbod_device { $jbod_devices: }
+
 # nodetool wrapper to handle multiple instances, for each instance there
 # will be symlinks from /usr/local/bin/nodetool- to
 # nodetool-instance
diff --git a/modules/cassandra/manifests/jbod_device.pp 
b/modules/cassandra/manifests/jbod_device.pp
new file mode 100644
index 000..4407e58
--- /dev/null
+++ b/modules/cassandra/manifests/jbod_device.pp
@@ -0,0 +1,39 @@
+# == Define: cassandra::jbod_device
+#
+# Configure a block device to be used for Cassandra JBOD.
+# Specifically, create the desired mount point directory and add an entry to
+# /etc/fstab.
+#
+# === Parameters
+#
+# [*mount_point*]
+#   The mount point to use
+#
+# [*mount_options*]
+#   What options to use for /etc/fstab
+
+define cassandra::jbod_device (
+  $mount_point = "/srv/${title}",
+  $mount_options = 'nofail,defaults',
+) {
+$device = $title
+
+file { "mountpoint-${mount_point}":
+ensure => directory,
+path   => $mount_point,
+owner  => 'root',
+group  => 'root',
+mode   => '0750',
+before => Mount[$mount_point],
+}
+
+mount { $mount_point:
+ensure   => mounted,
+device   => $device,
+name => $mount_point,
+fstype   => 'ext4',
+options  => $mount_options,
+atboot   => true,
+remounts => true,
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/374994
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I456261d7446ef131ebf6b9c292e8961816f0f5a8
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] shinkengen for all projects

[Alex Monk (Code Review)]

Alex Monk has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/374897 )

Change subject: [WIP] shinkengen for all projects
..

[WIP] shinkengen for all projects

Untested!

Cleans up some now-unused config and heira etc.

Bug: T166845
Change-Id: Idcf1e283c0dfdd8546cd46c76155b836b7c7677c
---
M hieradata/labs.yaml
M modules/shinken/files/shinkengen
M modules/shinken/templates/shinkengen.yaml.erb
3 files changed, 24 insertions(+), 26 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/97/374897/1

diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index d085d2e..e5b9569 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -86,7 +86,6 @@
 puppetmaster: "labs-puppetmaster.wikimedia.org"
 saltmaster: "labcontrol1001.wikimedia.org"
 mysql::server::use_apparmor: false  # The images we have for debian don't 
support it
-keystone_public_port: '5000'
 
 # All labs instances have internet connectivity, so do not bother with proxies
 profile::base::use_apt_proxy: false
diff --git a/modules/shinken/files/shinkengen b/modules/shinken/files/shinkengen
index 619f172..40b0539 100755
--- a/modules/shinken/files/shinkengen
+++ b/modules/shinken/files/shinkengen
@@ -40,6 +40,7 @@
 from keystoneclient.session import Session as KeystoneSession
 from keystoneclient.auth.identity.v3 import Password as KeystonePassword
 
+from keystoneclient.v3 import Client as KeystoneClient
 from novaclient import client as novaclient
 
 
@@ -61,21 +62,18 @@
  for key, value in self.properties.iteritems())
 return 'define %s {\n%s}\n' % (self.type, config_str)
 
+def get_keystone_session(project, observer_config):
+return KeystoneSession(auth=KeystonePassword(
+auth_url=observer_config['OS_AUTH_URL'],
+username=observer_config['OS_USERNAME'],
+password=observer_config['OS_PASSWORD'],
+project_name=project,
+user_domain_name='default',
+project_domain_name='default'
+))
 
-def get_instances(project, observer_pass, keystone_host, keystone_port):
-client = novaclient.Client(
-"2.0",
-session=KeystoneSession(auth=KeystonePassword(
-auth_url="http://{host}:{port}/v3".format(host=keystone_host,
-  port=keystone_port),
-username="novaobserver",
-password=observer_pass,
-project_name=project,
-user_domain_name='default',
-project_domain_name='default'
-))
-)
-
+def get_instances(project, keystone_session):
+client = novaclient.Client("2.0", session=keystone_session)
 hosts = []
 for instance in client.servers.list():
 hosts.append({
@@ -112,10 +110,14 @@
 nova_observer = yaml.safe_load(n)
 observer_pass = nova_observer['OS_PASSWORD']
 
-for project in config['projects']:
-instances = get_instances(project, observer_pass,
-  config['keystone_host'],
-  config['keystone_port'])
+client = KeystoneClient(
+session=get_keystone_session('bastion', nova_observer),
+endpoint=nova_observer['OS_AUTH_URL'],
+interface='public'
+)
+
+for project in client.projects.list():
+instances = get_instances(project.name, 
get_keystone_session(project.name, nova_observer))
 host_configs = []
 for instance in instances:
 co = ConfigObject('host')
@@ -127,7 +129,7 @@
 # particular roles with checks.  role::labs::instance is
 # implicitly applied to all instances via
 # manifests/site.pp, so we need to add it here explicitly.
-hostgroups = [project, 'role::labs::instance']
+hostgroups = [project.name, 'role::labs::instance']
 hostgroups += get_instance_roles(instance['project'],
  instance['fqdn'],
  config['puppet_enc_host'])
@@ -135,13 +137,13 @@
 # For each project added to monitoring we expect a contactgroup 
with the same
 # name added.
 # FIXME: Implement access mechanism more fine grained than 
per-project
-co.properties['contact_groups'] = project
+co.properties['contact_groups'] = project.name
 # Used to auto derive metric paths in graphite, which is of the 
form
 # ...
-co.properties['notes'] = project  # Used for auto deriving 
graphite path
+co.properties['notes'] = project.name  # Used for auto deriving 
graphite path
 host_configs.append(co)
 hosts_config_path = '%s/%s.cfg' % (
-config['output_path'], project
+config['output_path'], project.name
 )
 file_content = ''.join([str(hc) 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add reportupdater job to trigger page-creation metrics

[Mforns (Code Review)]

Mforns has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/374878 )

Change subject: [WIP] Add reportupdater job to trigger page-creation metrics
..

[WIP] Add reportupdater job to trigger page-creation metrics

Bug: T170850
Change-Id: I4f13143a6c8716726459c445c962e47d19f4b35f
---
M modules/profile/manifests/reportupdater/jobs/mysql.pp
1 file changed, 4 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/78/374878/1

diff --git a/modules/profile/manifests/reportupdater/jobs/mysql.pp 
b/modules/profile/manifests/reportupdater/jobs/mysql.pp
index cba6fe7..7b02ad0 100644
--- a/modules/profile/manifests/reportupdater/jobs/mysql.pp
+++ b/modules/profile/manifests/reportupdater/jobs/mysql.pp
@@ -71,4 +71,8 @@
 repository => 'discovery-stats',
 output_dir => 'metrics/interactive',
 }
+reportupdater::job { 'page-creation':
+repository => 'reportupdater-queries',
+output_dir => 'metrics/page-creation',
+}
 }
\ No newline at end of file

-- 
To view, visit https://gerrit.wikimedia.org/r/374878
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I4f13143a6c8716726459c445c962e47d19f4b35f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mforns 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Optimize EventLogging purging script using timestamps

[Mforns (Code Review)]

Mforns has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/374823 )

Change subject: [WIP] Optimize EventLogging purging script using timestamps
..

[WIP] Optimize EventLogging purging script using timestamps

Bug: T156933
Change-Id: Ib9f7b9559425bf0496c3cca82df63cd101261c8b
---
M modules/role/files/mariadb/eventlogging_cleaner.py
1 file changed, 44 insertions(+), 67 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/23/374823/1

diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py 
b/modules/role/files/mariadb/eventlogging_cleaner.py
index ee08333..df4583e 100644
--- a/modules/role/files/mariadb/eventlogging_cleaner.py
+++ b/modules/role/files/mariadb/eventlogging_cleaner.py
@@ -27,8 +27,8 @@
if any of DB username/password are provided by the user as my.cnf 
configuration
file (the conf file needs to have a [client] section with 'user' and 
'password').
 2) If a table is listed in the whitelist, then some of its fields are 
automatically
-   added to it (see COMMON_PERSISTENT_FIELDS). This ensures that important 
fields
-   like timestamp or primary keys are preserved.
+   added to it (see COMMON_PERSISTENT_FIELDS). This ensures that important 
non-sensitive
+   fields like timestamp or primary keys are preserved.
 3) The script runs updates/deletes in batches to avoid blocking the database 
for too
long creating contention with other write operations (like inserts).
 """
@@ -44,7 +44,6 @@
 import sys
 import time
 import unittest
-import uuid
 
 from datetime import datetime, timedelta
 from unittest.mock import MagicMock, Mock, call, patch
@@ -225,53 +224,26 @@
 result = self.database.execute(command, params, 
dry_run=self.dry_run)
 time.sleep(self.sleep_between_batches)
 
-def _get_uuids_and_last_ts(self, table, start_ts, 
override_batch_size=None):
+def _get_last_ts(self, table, start_ts):
 """
-Return the first  uuids of the events between start_ts
-and self.end. Also return the timestamp of the last of those events.
-NOTE: If there exist several events that share the last timestamp,
-it might be that some of them are listed in the uuid batch, and some
-others aren't (do not fit in the batch size limit). In the next 
iteration
-start_ts will be this iteration's last_ts, and so the script might
-re-purge some events, which is OK, because the outcome does not change.
+Return the timestamp of the Nth event between start_ts and self.end,
+where N is equal to the batch size. If there are less than N events
+between start_ts and self.end, return the timestamp of the last one.
+If there are no events between start_ts and self.end, return None.
 """
-batch_size = override_batch_size or self.batch_size
-# July 2017
-# There are currently some tables on analtics-store that have their 
uuid
-# field set as 'binary', not 'char' as in the master
-# and the analytics-slave.
-# Since altering all the inconsistent tables is a demanding task for 
the
-# current hardware, we just force an explicit cast to char in the 
query.
 command = (
-"SELECT timestamp, CAST(uuid AS CHAR) from {} WHERE timestamp >= 
%(start_ts)s "
+"SELECT timestamp from `{}` WHERE timestamp >= %(start_ts)s "
 "AND timestamp < %(end_ts)s ORDER BY timestamp LIMIT 
%(batch_size)s"
 .format(table)
 )
 params = {
 'start_ts': start_ts,
 'end_ts': self.end,
-'batch_size': batch_size,
+'batch_size': self.batch_size,
 }
 result = self.database.execute(command, params, self.dry_run)
 if result['rows']:
-last_ts = result['rows'][-1][0]
-if last_ts == start_ts:
-if batch_size > 4 * self.batch_size:
-raise RuntimeError(
-"The number of events with the same timestamp ({}) "
-"for table {} exceeded 4 times the configured batch 
size. "
-"Aborting as a precautionary measure."
-.format(start_ts, table)
-)
-log.warning("All events in the batch have the same timestamp 
({}) for table {}. "
-"Growing the batch size to {}."
-.format(start_ts, table, 2 * batch_size))
-return self._get_uuids_and_last_ts(table, start_ts,
-   
override_batch_size=batch_size*2)
-uuids = [x[1] for x in result['rows']]
-return (uuids, last_ts)
-else:
-return ([], None)
+return result['rows'][-1][0]
 
 def sanitize(self, table):
 """
@@ 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add Matxin MT service for ContentTranslation

[KartikMistry (Code Review)]

KartikMistry has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/374706 )

Change subject: WIP: Add Matxin MT service for ContentTranslation
..

WIP: Add Matxin MT service for ContentTranslation

Change-Id: I43f0203c69e256e106a8657f78f94f0db9f839fd
---
M modules/cxserver/manifests/init.pp
1 file changed, 4 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/06/374706/1

diff --git a/modules/cxserver/manifests/init.pp 
b/modules/cxserver/manifests/init.pp
index 206214a..a43ad71 100644
--- a/modules/cxserver/manifests/init.pp
+++ b/modules/cxserver/manifests/init.pp
@@ -7,6 +7,8 @@
 #
 # [*apertium*]
 #   Url to Apertium service.
+# [*matxin*]
+#   Url to Matxin service.
 # [*yandex_api_key*]
 #   API key for Yandex service.
 # [*youdao_api_key*]
@@ -15,6 +17,7 @@
 #   JWT secret token.
 class cxserver(
 $apertium = "http://apertium.svc.${::site}.wmnet:2737;,
+$matxin = undef,
 $yandex_api_key = undef,
 $youdao_api_key = undef,
 $jwt_secret = undef,
@@ -29,6 +32,7 @@
 deployment_vars   => {
 jwt_token=> $jwt_secret,
 apertium_uri => $apertium,
+matxin_uri   => $matxin,
 yandex_key   => $yandex_api_key,
 youdao_key   => $youdao_api_key,
 },

-- 
To view, visit https://gerrit.wikimedia.org/r/374706
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I43f0203c69e256e106a8657f78f94f0db9f839fd
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: KartikMistry 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add RDF dumps for categories

[Smalyshev (Code Review)]

Smalyshev has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/373354 )

Change subject: [WIP] Add RDF dumps for categories
..

[WIP] Add RDF dumps for categories

Creates RDF dump for each wiki configured in categories-rdf dblist
in other/categoriesrdf/YYMMDD.
Keeps old dumps back for 70 days.
The dumps are kept in TTL format.

Bug:
Change-Id: Idc3710f13d2ab03006011850bec98ee168e247c5
---
A modules/snapshot/files/cron/dumpcategoriesrdf.sh
A modules/snapshot/files/cron/logrotate.categoriesrdf
M modules/snapshot/manifests/cron.pp
A modules/snapshot/manifests/cron/categoriesrdf.pp
4 files changed, 173 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/54/373354/1

diff --git a/modules/snapshot/files/cron/dumpcategoriesrdf.sh 
b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
new file mode 100755
index 000..b128a62
--- /dev/null
+++ b/modules/snapshot/files/cron/dumpcategoriesrdf.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# This file is maintained by puppet!
+# modules/snapshot/cron/dumpcategoriesrdf.sh
+#
+#
+# Generate an RDF dump of categories for all wikis in 
+# categories-rdf list and remove old ones.
+
+source /usr/local/etc/set_dump_dirs.sh
+
+usage() {
+   echo "Usage: $0 [--config ] [--dryrun]"
+   echo
+   echo "  --config  path to configuration file for dump generation"
+   echo "(default value: ${confsdir}/wikidump.conf"
+   echo "  --dryrun  don't run dump, show what would have been done"
+   exit 1
+}
+
+configFile="${confsdir}/wikidump.conf"
+dryrun="false"
+dumpFormat="ttl"
+dbList="categories-rdf"
+
+while [ $# -gt 0 ]; do
+   if [ $1 == "--config" ]; then
+   configFile="$2"
+   shift; shift;
+   elif [ $1 == "--dryrun" ]; then
+   dryrun="true"
+   shift
+   else
+   echo "$0: Unknown option $1"
+   usage
+   fi
+done
+
+if [ ! -f "$configFile" ]; then
+   echo "Could not find config file: $configFile"
+   echo "Exiting..."
+   exit 1
+fi
+
+deployDir=$(egrep "^dir=" "$configFile" | mawk -Fdir= '{ print $2 }')
+gzip=$(egrep "^gzip=" "$configFile" | mawk -Fgzip= '{ print $2 }')
+privateList=$(egrep "^privatelist=" "$configFile" | mawk -Fprivatelist= '{ 
print $2 }')
+publicDir=$(egrep "^public=" "$configFile" | mawk -Fpublic= '{ print $2 }')
+
+if [ -z "$deployDir" -o -z "$gzip" -o -z "$privateList" -o -z "$publicDir" ]; 
then
+   echo "failed to find value of one of the following from config file 
$configFile:"
+   echo "gzip: $gzip"
+   echo "dir: $deployDir"
+   echo "privatelist: $privateList"
+   echo "public: $publicDir"
+   echo "exiting..."
+   exit 1
+fi
+
+today=$(date +'%Y%m%d')
+targetDirBase="$publicDir/other/categoriesrdf"
+targetDir="$targetDirBase/$today"
+timestampsDir="$targetDirBase/lastdump"
+multiVersionScript="$deployDir/multiversion/MWScript.php"
+
+# remove old datasets
+daysToKeep=70
+cutOff=$(( $(date +%s) - $(( $daysToKeep + 1 )) * 24 * 3600))
+if [ -d "$targetDirBase" ]; then
+   for folder in $(ls -d -r $targetDirBase/*); do
+   creationTime=$(date --utc --date="$(basename $folder)" +%s 
2>/dev/null)
+   if [ -n "$creationTime" ]; then
+   if [ "$cutOff" -gt "$creationTime" ]; then
+   if [ "$dryrun" == "true" ]; then
+   echo rm $folder/*.$dumpFormat.gz
+   echo rmdir $folder
+   else
+   rm -f $folder/*.$dumpFormat.gz
+   rmdir $folder
+   fi
+   fi
+   fi
+   done
+fi
+
+# create todays folder
+if [ "$dryrun" == "true" ]; then
+   echo mkdir -p "$targetDir"
+   echo mkdir -p "$timestampsDir"
+else
+   if ! mkdir -p "$targetDir"; then
+   echo "Can't make output directory: $targetDir"
+   echo "Exiting..."
+   exit 1
+   fi
+   if ! mkdir -p "$timestampsDir"; then
+   echo "Can't make output directory: $timestampsDir"
+   echo "Exiting..."
+   exit 1
+   fi
+fi
+
+# iterate over configured wikis
+/usr/local/bin/expanddblist $dbList | while read wiki; do
+   # exclude all private wikis
+   if ! egrep -q "^$wiki$" $privateList; then
+   filename="$wiki-$today-categories"
+   targetFile="$targetDir/$filename.$dumpFormat.gz"
+   tsFile="$timestampsDir/$wiki-categories.last"
+   if [ "$dryrun" == "true" ]; then
+   echo "php $multiVersionScript 
maintenance/dumpCategoriesAsRdf.php --wiki=$wiki 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Allow silencing notifications for hosts

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/373291 )

Change subject: WIP: Allow silencing notifications for hosts
..

WIP: Allow silencing notifications for hosts

Add a notifications_enabled parameter to monitoring::host and
monitoring::service, defaulting to 1. This is passed straight to the
nagios_host and nagios_service resource respectively allowing to
selectively have disabled notification on a per host level. In the case
of monitoring::host the aforementioned parameter gets passed directly
from profile::base, effectively curbing the extent of this patch only to
puppet enabled hosts. This protects all "virtual" hosts (e.g. LVS) from
being accidentaly silenced. On the monitoring::service side, things are
not so easy since that define is used in a myriad places. Instead of
violating our puppet coding policy and adding one more hiera lookup in a
module class, lookup directly $::profile::base::notifications_enabled
instead, then safeguard it in the case it's not defined

There is one caveat with this approach. Setting the hiera parameter on
the icinga host level will cause all virtual host (e.g. LVS) bound
services to be silenced.

TODO: Actually create an event handler

Bug: T151632
Change-Id: I3047d9421c035c7004ca42b248f584216b8e5bdb
---
M modules/monitoring/manifests/host.pp
M modules/monitoring/manifests/service.pp
M modules/profile/manifests/base.pp
3 files changed, 53 insertions(+), 30 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/91/373291/1

diff --git a/modules/monitoring/manifests/host.pp 
b/modules/monitoring/manifests/host.pp
index 9e99a6d..ce02077 100644
--- a/modules/monitoring/manifests/host.pp
+++ b/modules/monitoring/manifests/host.pp
@@ -2,14 +2,15 @@
 # Exports the resource that monitors hosts in icinga/shinken
 #
 define monitoring::host (
-$ip_address= $facts['ipaddress'],
-$os= $facts['operatingsystem'],
-$host_fqdn = undef,
-$group = undef,
-$ensure= present,
-$critical  = false,
-$parents   = undef,
-$contact_group = hiera('contactgroups', 'admins'),
+$ip_address= $facts['ipaddress'],
+$os= $facts['operatingsystem'],
+$host_fqdn = undef,
+$group = undef,
+$ensure= present,
+$critical  = false,
+$parents   = undef,
+$contact_group = hiera('contactgroups', 'admins'),
+$notifications_enabled = 1,
 ) {
 
 $nagios_address = $host_fqdn ? {
@@ -64,6 +65,7 @@
 $real_parents = undef
 }
 # We have a BMC, and the BMC is configured and it has an IP address
+# We always monitor the BMC so never skip notifications
 if $facts['has_ipmi'] and $facts['ipmi_lan'] and 'ipaddress' in 
$facts['ipmi_lan'] {
 $mgmt_host = {
 "${title}.mgmt" => {
@@ -100,6 +102,7 @@
 check_command => 'check_ping!500,20%!2000,100%',
 check_period  => '24x7',
 max_check_attempts=> 2,
+notifications_enabled => $notifications_enabled,
 contact_groups=> $real_contact_groups,
 notification_interval => 0,
 notification_period   => '24x7',
@@ -119,21 +122,24 @@
 create_resources($rtype, $host)
 if $mgmt_host {
 create_resources($rtype, $mgmt_host)
+# We always monitor the BMC so never skip notifications
 monitoring::service { "dns_${title}.mgmt":
-description=> "DNS ${title}.mgmt",
-host   => "${title}.mgmt",
-check_command  => "check_fqdn!${title}.mgmt.${::site}.wmnet",
-group  => 'mgmt',
-check_interval => 60,
-retry_interval => 60,
+description   => "DNS ${title}.mgmt",
+host  => "${title}.mgmt",
+check_command => 
"check_fqdn!${title}.mgmt.${::site}.wmnet",
+notifications_enabled => 1,
+group => 'mgmt',
+check_interval=> 60,
+retry_interval=> 60,
 }
 monitoring::service { "ssh_${title}.mgmt":
-description=> "SSH ${title}.mgmt",
-host   => "${title}.mgmt",
-check_command  => 'check_ssh',
-group  => 'mgmt',
-check_interval => 60,
-retry_interval => 60,
+description   => "SSH ${title}.mgmt",
+host  => "${title}.mgmt",
+check_command => 'check_ssh',
+notifications_enabled => 1,
+group => 'mgmt',
+check_interval=> 60,
+retry_interval=> 60,
 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add shiladsen shell account

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/373115 )

Change subject: WIP: Add shiladsen shell account
..

WIP: Add shiladsen shell account

Add shiladsen to groups researchers, statistics-privatedata-users and 
statistics-users.

Change-Id: Id95b7ff8b2553893408ca977866671fb0cfb070f
---
M modules/admin/data/data.yaml
1 file changed, 13 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/15/373115/1

diff --git a/modules/admin/data/data.yaml b/modules/admin/data/data.yaml
index fab540e..5437e15 100644
--- a/modules/admin/data/data.yaml
+++ b/modules/admin/data/data.yaml
@@ -120,7 +120,7 @@
   nikerabbit, nschaaf, dstrine, joewalsh, mpany, jsamra,
   jdittrich, chelsyx, ovasileva, mtizzoni, panisson, paolotti, 
ciro, debt,
   samwalton9, fdans, samtar, mlitn, shrlak, niharika29, goransm,
-  pmiazga, diego]
+  pmiazga, diego, shiladsen]
   ldap-admins:
 gid: 715
 description: ldap admins
@@ -189,7 +189,7 @@
   leila, ellery, nettrom, mforns, bmansurov, tbayer, joal,
   madhuvishy, tjones, legoktm, dcausse, bearloga, dpatrick, 
atgomez, dstrine,
   joewalsh, marktraceur, mtizzoni, panisson, paolotti, ciro, 
melodykramer,
-  fdans]
+  fdans, shiladsen]
   statistics-users:
 gid: 726
 description: Access statistics number crunching hosts. NO PRIVS.
@@ -200,7 +200,7 @@
   deskana, dduvall, nettrom, mforns, jkatz, ebernhardson,
   mlitn, tbayer, joal, kartik, nikerabbit, pcoombe,
   neilpquinn-wmf, maxsem, jminor, dpatrick, atgomez, dstrine,
-  ladsgroup, ovasileva]
+  ladsgroup, ovasileva, shiladsen]
   statistics-admins:
 posix_name: stats
 description: access files created by stats user cron jobs
@@ -2536,6 +2536,16 @@
   - ssh-rsa 
B3NzaC1yc2EDAQABAAABAQC/oA9hk3snx7Y66ZR3sEMukK6tOk4esFT02irhqB0jr9osstyZN9KhPVeWMzhip+93ToDzX+aDHeXqsu5grXsqGQZBZSU850GSNt0pgi8+4E1yGQngLNhFv+z7EemWUQH0XT4atoDXbmfXdRv6NpVlcr1vAPBQpjBZuFe5vaxLKRrhZpm+YNvl4RHdGbZorz6WI0NvzuOTerKUOyUZS/KQpT4FxlvVRoTIO3H05EtJEv3745rUH5wHCcyr7m9Hdsjh8RMrWF3okKLk9WOxQTesfvvstmu8GpBbauzmQYMKwRcKQqoc0/qo3ZGvwXPwrYjv4wpPZhDexjOfUPLGvtGf
 dsaeztrumper@dsaeztrumper
 uid: 17677
 email: di...@wikimedia.org
+  shiladsen:
+ensure: present
+gid: 500
+name: shiladsen
+realname: Shilad Sen
+ssh_keys:
+  - ssh-dss 
B3NzaC1kc3MAAACBAKBspbywXptKB4djp8jYjfk0fAQUAhsEM03zvRhuCpIwB5BYQl2mIeIwADHqM5DA0plGtFZLLwZvFR/LpHIiK3zcDuvz5N6LBkTulKQ5TrjnMkAeTk1SA900u6jCoKitF7j6ZO3Q4diLgFSY5F4EJI80GiWkOx+JAnzhS3kHbkibFQDzFcnzFRA7bawBb0ZVhCYDU2v+2wAAAIBzWSGg2rEvV0UT+cDzGZMl6LGWT+3oC1pJviW8vilOhIKvdbXYeQeGpqpJjxZToN/5Ok+P0kAMNTacdPWyYiDDepb+zgB9tbW+DPB3HgH2y6u7SMNWnOXK+C9VAT62LEX4zQsD41NC3kMijDjLuAzAkyKPAVmgtFWCXpYDDU/+zgAAAIAW66EVt/6tp7o6GlfU3TS3JnYLA3cFzWqmbuHuV2dFhW3h7OAbmCRivhOVuhJuu56C/AJeKdGzIA10p/eo39YXUX3iOjUTO8/YFFAAnh9m4Fb1YDTMG3JzwBi8jT6r8iOm9414ITX48y9zzD3smXku3o3At/w5Up6rl/lDeywI4g==
 a558989@600308a4c4c6
+uid: 10339
+email: shi...@gmail.com
+
 ldap_only_users:
   abartov:
 ensure: present

-- 
To view, visit https://gerrit.wikimedia.org/r/373115
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id95b7ff8b2553893408ca977866671fb0cfb070f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: new prometheus instance 'services'

[Filippo Giunchedi (Code Review)]

Filippo Giunchedi has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/372357 )

Change subject: WIP: new prometheus instance 'services'
..

WIP: new prometheus instance 'services'

Bug: T173490
Change-Id: I88c11a302f0c430cebe38aef56ac2c160c180e74
---
M manifests/site.pp
A modules/role/files/prometheus/rules_services.conf
A modules/role/manifests/prometheus/services.pp
3 files changed, 63 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/57/372357/1

diff --git a/manifests/site.pp b/manifests/site.pp
index 64f9d64..47e2c60 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1949,7 +1949,7 @@
 }
 
 node /^prometheus200[34]\.codfw\.wmnet$/ {
-role(prometheus::ops, prometheus::global)
+role(prometheus::ops, prometheus::global, prometheus::services)
 
 include ::base::firewall
 include ::standard
@@ -1959,7 +1959,7 @@
 }
 
 node /^prometheus100[34]\.eqiad\.wmnet$/ {
-role(prometheus::ops, prometheus::global)
+role(prometheus::ops, prometheus::global, prometheus::services)
 
 include ::lvs::realserver
 
diff --git a/modules/role/files/prometheus/rules_services.conf 
b/modules/role/files/prometheus/rules_services.conf
new file mode 100644
index 000..dcf2c80
--- /dev/null
+++ b/modules/role/files/prometheus/rules_services.conf
@@ -0,0 +1 @@
+# Placeholder
diff --git a/modules/role/manifests/prometheus/services.pp 
b/modules/role/manifests/prometheus/services.pp
new file mode 100644
index 000..6261759
--- /dev/null
+++ b/modules/role/manifests/prometheus/services.pp
@@ -0,0 +1,60 @@
+# Uses the prometheus module and generates the specific configuration
+# needed for WMF production
+#
+# filtertags: labs-project-monitoring
+class role::prometheus::services {
+include ::standard
+include ::base::firewall
+
+$targets_path = '/srv/prometheus/services/targets'
+$storage_retention = hiera('prometheus::server::storage_retention', 
'2190h0m0s')
+$max_chunks_to_persist = 
hiera('prometheus::server::max_chunks_to_persist', '524288')
+$memory_chunks = hiera('prometheus::server::memory_chunks', '1048576')
+
+$config_extra = {
+# All metrics will get an additional 'site' label when queried by
+# external systems (e.g. via federation)
+'external_labels' => {
+'site' => $::site,
+},
+}
+
+$cassandra_jobs = [
+  {
+'job_name'=> 'cassandra_restbase',
+'scheme'  => 'http',
+'file_sd_configs' => [
+  { 'files' => [ "${targets_path}/cassandra_restbase_*.yaml" ]}
+],
+  },
+]
+
+# Gather etcd metrics from machines exposing them via http
+prometheus::class_config{ "cassandra_restbase_${::site}":
+dest   => "${targets_path}/cassandra_restbase_${::site}.yaml",
+site   => $::site,
+class_name => 'cassandra::instance::monitoring',
+port   => 7800,
+}
+
+prometheus::server { 'services':
+storage_encoding  => '2',
+listen_address=> '127.0.0.1:9903',
+storage_retention => $storage_retention,
+max_chunks_to_persist => $max_chunks_to_persist,
+memory_chunks => $memory_chunks,
+scrape_configs_extra  => array_concat(
+$cassandra_jobs,
+),
+global_config_extra   => $config_extra,
+}
+
+prometheus::web { 'services':
+proxy_pass => 'http://localhost:9903/services',
+}
+
+prometheus::rule { 'rules_services.conf':
+instance => 'services',
+source   => 'puppet:///modules/role/prometheus/rules_services.conf',
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/372357
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I88c11a302f0c430cebe38aef56ac2c160c180e74
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Filippo Giunchedi 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Add libraryupgrader puppet module

[Legoktm (Code Review)]

Legoktm has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/372213 )

Change subject: [WIP] Add libraryupgrader puppet module
..

[WIP] Add libraryupgrader puppet module

For use in the libraryupgrader Cloud VPS project.



Bug: T173478
Change-Id: Ifb8fbf27b0ecce251c29cb254fda9a916e1d0dea
---
A modules/libraryupgrader/manifests/init.pp
1 file changed, 71 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/13/372213/1

diff --git a/modules/libraryupgrader/manifests/init.pp 
b/modules/libraryupgrader/manifests/init.pp
new file mode 100644
index 000..54e5b35
--- /dev/null
+++ b/modules/libraryupgrader/manifests/init.pp
@@ -0,0 +1,71 @@
+# = Class: role::libraryupgrader
+#
+# This class sets up a the Cloud VPS project libraryupgrader.
+#
+class libraryupgrader(
+$base_dir = '/srv'
+){
+$clone_dir= "${base_dir}/libraryupgrader"
+
+user { 'libraryupgrader':
+ensure => present,
+system => true,
+}
+
+file { '/home/libraryupgrader':
+ensure  => directory,
+owner   => 'libraryupgrader',
+require => User['libraryupgrader'],
+}
+
+file { [$clone_dir]:
+ensure => directory,
+owner  => 'extdist',
+group  => 'www-data',
+mode   => '0755',
+}
+
+git::clone {'labs/libraryupgrader':
+ensure=> latest,
+directory => $clone_dir,
+branch=> 'master',
+require   => [File[$clone_dir], User['extdist']],
+owner => 'libraryupgrader',
+group => 'libraryupgrader',
+}
+
+package { 'docker-engine':
+ensure => present,
+}
+
+package { 'apache2':
+ensure => present,
+}
+
+package { 'python3-jinja2':
+ensure => present,
+}
+
+cron { 'libraryupgrader-build':
+command => "/bin/bash ${clone_dir}/build.sh",
+user=> 'root',
+minute  => '0',
+hour=> '0',
+require => [
+Git::Clone['labs/libraryupgrader'],
+Package['docker-engine'],
+],
+}
+
+cron { 'libraryupgrader-run':
+command => "/usr/bin/python3 ${clone_dir}/run.py",
+user=> 'root',
+minute  => '0',
+hour=> '1',
+require => [
+Git::Clone['labs/libraryupgrader'],
+Package['python3-jinja2'],
+Package['apache2'],
+],
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/372213
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifb8fbf27b0ecce251c29cb254fda9a916e1d0dea
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Legoktm 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Increase max kafka message size

[Ppchelko (Code Review)]

Ppchelko has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/372179 )

Change subject: WIP: Increase max kafka message size
..

WIP: Increase max kafka message size

Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07
---
M hieradata/common.yaml
M modules/changeprop/manifests/init.pp
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/server.properties.erb
4 files changed, 15 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/79/372179/1

diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 4922ec7..d6be87f 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -438,6 +438,7 @@
   main-eqiad:
 api_version: 0.9
 zookeeper_cluster_name: main-eqiad
+kafka_message_max_bytes: 200
 brokers:
   kafka1001.eqiad.wmnet:
 id: 1001
@@ -449,6 +450,7 @@
   main-codfw:
 api_version: 0.9
 zookeeper_cluster_name: main-codfw
+kafka_message_max_bytes: 200
 brokers:
   kafka2001.codfw.wmnet:
 id: 2001
diff --git a/modules/changeprop/manifests/init.pp 
b/modules/changeprop/manifests/init.pp
index ef3b397..d83b107 100644
--- a/modules/changeprop/manifests/init.pp
+++ b/modules/changeprop/manifests/init.pp
@@ -48,7 +48,7 @@
 include ::service::configuration
 
 require ::changeprop::packages
-
+
 service::node { 'changeprop':
 enable=> true,
 port  => 7272,
@@ -66,6 +66,7 @@
 site => $::site,
 redis_path   => $redis_path,
 redis_pass   => $redis_pass,
+kafka_max_bytes => 
$::kafka_clusters::main_eqiad::kafka_message_max_bytes,
 },
 auto_refresh  => false,
 init_restart  => false,
diff --git a/modules/confluent/manifests/kafka/broker.pp 
b/modules/confluent/manifests/kafka/broker.pp
index 99a7203..e2d3414 100644
--- a/modules/confluent/manifests/kafka/broker.pp
+++ b/modules/confluent/manifests/kafka/broker.pp
@@ -104,6 +104,9 @@
 #   partition in the fetch requests the replicas send to the leader.
 #   Default: undef (1048576)
 #
+# [*kafka_message_max_bytes*]
+#   The maximum size of a message that the server can receive. Default: undef
+#
 # [*num_network_threads*]
 #   The number of threads handling network requests.  Default: undef (3)
 #
@@ -237,6 +240,7 @@
 $replica_socket_receive_buffer_bytes = undef,
 $num_replica_fetchers= 1,
 $replica_fetch_max_bytes = undef,
+$kafka_message_max_bytes = undef,
 
 $num_network_threads = undef,
 $num_io_threads  = size($log_dirs),
diff --git a/modules/confluent/templates/kafka/server.properties.erb 
b/modules/confluent/templates/kafka/server.properties.erb
index c2a8fe6..aec3cc3 100644
--- a/modules/confluent/templates/kafka/server.properties.erb
+++ b/modules/confluent/templates/kafka/server.properties.erb
@@ -273,3 +273,10 @@
 # that additional support metrics will be collected and reported.
 #
 confluent.support.customer.id=anonymous
+
+<% if @kafka_message_max_bytes -%>
+
+# For the JobQueue we need larger messages then default
+message.max.bytes=<%= @kafka_message_max_bytes %>
+
+<% end -%>
\ No newline at end of file

-- 
To view, visit https://gerrit.wikimedia.org/r/372179
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ppchelko 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Add acl to warn on forged HELO messages on lists.wikime...

[Herron (Code Review)]

Herron has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/372174 )

Change subject: WIP: Add acl to warn on forged HELO messages on 
lists.wikimedia.org
..

WIP: Add acl to warn on forged HELO messages on lists.wikimedia.org

For testing/prep purposes log a warning if remote side is not in allowed list
and sends HELO claming to be a domain, hostname or IP that belongs to us.

Change from warn to delay & drop after testing.

Bug: T173338
Change-Id: I8b9fd41177cc681aa64228f96a83257ef0781ffe
---
M modules/profile/templates/exim/exim4.conf.mailman.erb
1 file changed, 19 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/74/372174/1

diff --git a/modules/profile/templates/exim/exim4.conf.mailman.erb 
b/modules/profile/templates/exim/exim4.conf.mailman.erb
index e5b52da..ca3c1ec 100644
--- a/modules/profile/templates/exim/exim4.conf.mailman.erb
+++ b/modules/profile/templates/exim/exim4.conf.mailman.erb
@@ -32,6 +32,7 @@
 
 # Policy control
 acl_smtp_connect = acl_check_connect
+acl_smtp_helo = acl_check_helo
 acl_smtp_rcpt = acl_check_rcpt
 acl_smtp_data = acl_check_data
 
@@ -130,6 +131,24 @@
 acl_check_connect:
accept
 
+acl_check_helo:
+
+   # warn if remote tries to send HELO < a local domain > (change from 
warn to delay & drop after testing)
+   warn message = HELO failure ref1
+   condition   = ${if 
match_domain{$sender_helo_name}{+local_domains}{yes}{no}}
+   !hosts  = +wikimedia_nets : +relay_from_hosts
+   log_message = HELO failure ref1 (Remote said: HELO 
$sender_helo_name which matches a domain listed in local_domains)
+   #delay   = 45s
+
+   # warn if remote tries to send HELO < my primary hostname > (change 
from warn to delay & drop after testing)
+   warn message = HELO failure ref2
+   condition   = ${if match{$sender_helo_name}{$primary_hostname}}
+   !hosts  = +wikimedia_nets : +relay_from_hosts
+   log_message = HELO failure ref2 (Remote said: HELO 
$sender_helo_name which equals my primary_hostname of $primary_hostname)
+   #delay   = 45s
+
+   accept
+
 acl_check_data:
# Let's trust local senders to not send out spam
accept hosts = +wikimedia_nets

-- 
To view, visit https://gerrit.wikimedia.org/r/372174
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I8b9fd41177cc681aa64228f96a83257ef0781ffe
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Herron 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]mariadb: First attempt at a mydumper-based dump script

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/371944 )

Change subject: [WIP]mariadb: First attempt at a mydumper-based dump script
..

[WIP]mariadb: First attempt at a mydumper-based dump script

Bug: T169516
Change-Id: I995cc704b5368df5a92a4763fcbde8e171e3d705
---
A modules/role/files/mariadb/dump_shards.sh
1 file changed, 31 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/44/371944/1

diff --git a/modules/role/files/mariadb/dump_shards.sh 
b/modules/role/files/mariadb/dump_shards.sh
new file mode 100644
index 000..e996d64
--- /dev/null
+++ b/modules/role/files/mariadb/dump_shards.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+numthreads=16
+backupdir=backups
+shards="s1 s4 s5 s2 s6 s7 s3 x1"
+rowsmax=1
+
+[ -z "$backupdir" ] && { echo "backupdir variable is empty"; exit 1 }
+[ "$backupdir" == "/" ] && { echo "backupdir variable cannot be the root 
directory"; exit 1 }
+[[ "$HOST" =~ ^sqldata.* ]] && { echo "backupdir variable cannot start with 
sqldata"; exit 1 }
+
+mkdir -p "/srv/$backupdir"
+chmod go-rwx "/srv/$backupdir"
+
+find "/srv/$backupdir" -mtime +14 -type f -delete
+find "/srv/$backupdir" -mtime +14 -type d -exec rmdir \{\} \;
+
+# stopping all replication activity to save iops
+for stop_shard in $shards; do 
+/usr/local/bin/mysql --socket="/run/mysqld/mysqld.$stop_shard.sock" -e 
"STOP SLAVE"
+done
+
+for backup_shard in $shards; do
+
+/usr/bin/mydumper/mydumper --compress --host=localhost 
--threads="$numthreads" --user="`whoami`" 
--socket="/run/mysqld/mysqld.$backup_shard.sock" --triggers --routines --events 
--rows="$rowsmax" --logfile="/srv/$backupdir/dump.$shard.log" 
--outputdir="/srv/$backupdir/$backup_shard.`date +%Y%m%d%H%M%S`"
+done
+
+# Restarting replication
+for start_shard in $shards; do
+/usr/local/bin/mysql --socket="/run/mysqld/mysqld.$start_shard.sock" -e 
"START SLAVE"
+done

-- 
To view, visit https://gerrit.wikimedia.org/r/371944
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I995cc704b5368df5a92a4763fcbde8e171e3d705
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Reshape RESTBase Cassandra production cluster; Provisio...

[Eevans (Code Review)]

Eevans has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/370098 )

Change subject: WIP: Reshape RESTBase Cassandra production cluster; Provision 
new 3.x cluster
..

WIP: Reshape RESTBase Cassandra production cluster; Provision new 3.x cluster

Basic steps:

- 9T of storage will be freed
- restbase1016.eqiad.wmnet is decommissioned
- this changeset is merged
- restbase1016.eqiad.wmnet is re-imaged
- restbase1016.eqiad.wmnet is bootstrapped
- new changeset to comment/uncomment instances is merged
  - lather
  - rinse
  - repeat

Todo:

- Figure out what to do w/ restbase::seeds

Bug: T169939
Change-Id: Ife09538ec52e8f48b8d0899d81b1c961a8476b94
---
M hieradata/role/common/restbase/production.yaml
A hieradata/role/common/restbase/production_new.yaml
2 files changed, 123 insertions(+), 7 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/98/370098/1

diff --git a/hieradata/role/common/restbase/production.yaml 
b/hieradata/role/common/restbase/production.yaml
index 82f3b2b..4e99d3b 100644
--- a/hieradata/role/common/restbase/production.yaml
+++ b/hieradata/role/common/restbase/production.yaml
@@ -127,13 +127,13 @@
   listen_address: 10.64.48.139
 c:
   listen_address: 10.64.48.140
-  "restbase1016.eqiad.wmnet":
-a:
-  listen_address: 10.64.0.32
-b:
-  listen_address: 10.64.0.33
-c:
-  listen_address: 10.64.0.34
+  # "restbase1016.eqiad.wmnet":
+  #   a:
+  # listen_address: 10.64.0.32
+  #   b:
+  # listen_address: 10.64.0.33
+  #   c:
+  # listen_address: 10.64.0.34
   "restbase1017.eqiad.wmnet":
 a:
   listen_address: 10.64.32.130
diff --git a/hieradata/role/common/restbase/production_new.yaml 
b/hieradata/role/common/restbase/production_new.yaml
new file mode 100644
index 000..29b0487
--- /dev/null
+++ b/hieradata/role/common/restbase/production_new.yaml
@@ -0,0 +1,116 @@
+#
+# Cassandra
+#
+graphite_host: 'graphite1003.eqiad.wmnet'
+profile::cassandra::allow_analytics: false
+
+profile::cassandra::metrics_whitelist:
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.CoordinatorReadLatency\..*$
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.CoordinatorScanLatency\..*$
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.EstimatedColumnCountHistogram\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.EstimatedRowCount\..*$
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.EstimatedRowSizeHistogram\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.LiveDiskSpaceUsed\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.LiveSSTableCount\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.LiveScannedHistogram\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.MaxRowSize\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.MeanRowSize\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.MinRowSize\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.PendingCompactions\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.PendingFlushes\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.RangeLatency\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.ReadLatency\..*$
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.SSTablesPerReadHistogram\..*$
+  - 
.*\.metrics\.ColumnFamily\.local_group_.*\.meta\.TombstoneScannedHistogram\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.TotalDiskSpaceUsed\..*$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\.WriteLatency\..*$
+profile::cassandra::metrics_blacklist:
+  - .*\.15MinuteRate$
+  - .*\.5MinuteRate$
+  - .*\.98percentile$
+  - .*\.999percentile$
+  - .*\.mean$
+  - .*\.meanRate$
+  - .*\.min$
+  - .*\.stddev$
+  - .*\.metrics\.ColumnFamily\.local_group_.*\.meta\..*$
+  - .*\.metrics\.Keyspace\..*$
+  - .*\.metrics\.Client\..*$
+  - .*\.metrics\.Table\..*$
+## Instances
+profile::cassandra::instances:
+  "restbase1016.eqiad.wmnet":
+a:
+  listen_address: 10.64.0.32
+b:
+  listen_address: 10.64.0.33
+c:
+  listen_address: 10.64.0.34
+  # "restbase1017.eqiad.wmnet":
+  #   a:
+  # listen_address: 10.64.32.130
+  #   b:
+  # listen_address: 10.64.32.131
+  #   c:
+  # listen_address: 10.64.32.132
+  # "restbase1018.eqiad.wmnet":
+  #   a:
+  # listen_address: 10.64.48.98
+  #   b:
+  # listen_address: 10.64.48.99
+  #   c:
+  # listen_address: 10.64.48.100
+  # "restbase2010.codfw.wmnet":
+  #   a:
+  # listen_address: 10.192.16.186
+  #   b:
+  # listen_address: 10.192.16.187
+  #   c:
+  # listen_address: 10.192.16.188
+  # "restbase2011.codfw.wmnet":
+  #   a:
+  # listen_address: 10.192.32.152
+  #   b:
+  # listen_address: 10.192.32.153
+  #   c:
+  # listen_address: 10.192.32.154
+  # "restbase2012.codfw.wmnet":
+  #   a:
+  # listen_address: 10.192.48.68
+  #   b:
+  # listen_address: 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: (WIP) trigger all modules

[Hashar (Code Review)]

Hashar has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/369923 )

Change subject: (WIP) trigger all modules
..

(WIP) trigger all modules

Change-Id: I802507bd38be03e1acb39ca7815be44e63326efd
---
A modules/aptrepo/spec/trigger.foo
A modules/authdns/spec/trigger.foo
A modules/backup/spec/trigger.foo
A modules/bacula/spec/trigger.foo
A modules/base/spec/trigger.foo
A modules/contint/spec/trigger.foo
A modules/elasticsearch/spec/trigger.foo
A modules/git/spec/trigger.foo
A modules/graphite/spec/trigger.foo
A modules/install_server/spec/trigger.foo
A modules/interface/spec/trigger.foo
A modules/jenkins/spec/trigger.foo
A modules/lvm/spec/trigger.foo
A modules/mirrors/spec/trigger.foo
A modules/monitoring/spec/trigger.foo
A modules/mysql/spec/trigger.foo
A modules/nagios_common/spec/trigger.foo
A modules/network/spec/trigger.foo
A modules/nrpe/spec/trigger.foo
A modules/osm/spec/trigger.foo
A modules/postgresql/spec/trigger.foo
A modules/profile/spec/trigger.foo
A modules/puppetdbquery/spec/trigger.foo
A modules/rsync/spec/trigger.foo
A modules/scap/spec/trigger.foo
A modules/servermon/spec/trigger.foo
A modules/service/spec/trigger.foo
A modules/squid3/spec/trigger.foo
A modules/stdlib/spec/trigger.foo
A modules/strongswan/spec/trigger.foo
A modules/systemd/spec/trigger.foo
A modules/tilerator/spec/trigger.foo
A modules/wdqs/spec/trigger.foo
A modules/wmflib/spec/trigger.foo
A modules/zuul/spec/trigger.foo
35 files changed, 0 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/23/369923/1

diff --git a/modules/aptrepo/spec/trigger.foo b/modules/aptrepo/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/aptrepo/spec/trigger.foo
diff --git a/modules/authdns/spec/trigger.foo b/modules/authdns/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/authdns/spec/trigger.foo
diff --git a/modules/backup/spec/trigger.foo b/modules/backup/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/backup/spec/trigger.foo
diff --git a/modules/bacula/spec/trigger.foo b/modules/bacula/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/bacula/spec/trigger.foo
diff --git a/modules/base/spec/trigger.foo b/modules/base/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/base/spec/trigger.foo
diff --git a/modules/contint/spec/trigger.foo b/modules/contint/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/contint/spec/trigger.foo
diff --git a/modules/elasticsearch/spec/trigger.foo 
b/modules/elasticsearch/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/elasticsearch/spec/trigger.foo
diff --git a/modules/git/spec/trigger.foo b/modules/git/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/git/spec/trigger.foo
diff --git a/modules/graphite/spec/trigger.foo 
b/modules/graphite/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/graphite/spec/trigger.foo
diff --git a/modules/install_server/spec/trigger.foo 
b/modules/install_server/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/install_server/spec/trigger.foo
diff --git a/modules/interface/spec/trigger.foo 
b/modules/interface/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/interface/spec/trigger.foo
diff --git a/modules/jenkins/spec/trigger.foo b/modules/jenkins/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/jenkins/spec/trigger.foo
diff --git a/modules/lvm/spec/trigger.foo b/modules/lvm/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/lvm/spec/trigger.foo
diff --git a/modules/mirrors/spec/trigger.foo b/modules/mirrors/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/mirrors/spec/trigger.foo
diff --git a/modules/monitoring/spec/trigger.foo 
b/modules/monitoring/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/monitoring/spec/trigger.foo
diff --git a/modules/mysql/spec/trigger.foo b/modules/mysql/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/mysql/spec/trigger.foo
diff --git a/modules/nagios_common/spec/trigger.foo 
b/modules/nagios_common/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/nagios_common/spec/trigger.foo
diff --git a/modules/network/spec/trigger.foo b/modules/network/spec/trigger.foo
new file mode 100644
index 000..e69de29
--- /dev/null
+++ b/modules/network/spec/trigger.foo
diff --git a/modules/nrpe/spec/trigger.foo b/modules/nrpe/spec/trigger.foo
new file mode 100644

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] wdqs - moving to role / profiles

[Gehel (Code Review)]

Gehel has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/369682 )

Change subject: [WIP] wdqs - moving to role / profiles
..

[WIP] wdqs - moving to role / profiles

Change-Id: Id2adbd2cd86e61831a16f7abbb73135f47352bc0
---
M hieradata/labs/wikidata-query/common.yaml
M hieradata/role/codfw/wdqs.yaml
M hieradata/role/common/wdqs.yaml
M manifests/site.pp
A modules/profile/manifests/wdqs.pp
M modules/role/manifests/wdqs.pp
A modules/role/manifests/wdqs/labs.pp
M modules/wdqs/manifests/init.pp
M modules/wdqs/manifests/service.pp
M modules/wdqs/manifests/updater.pp
10 files changed, 82 insertions(+), 57 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/82/369682/1

diff --git a/hieradata/labs/wikidata-query/common.yaml 
b/hieradata/labs/wikidata-query/common.yaml
index bc94915..d6ce9c1 100644
--- a/hieradata/labs/wikidata-query/common.yaml
+++ b/hieradata/labs/wikidata-query/common.yaml
@@ -1,7 +1,9 @@
-"wdqs::gui::log_aggregator":  
'deployment-logstash2.deployment-prep.eqiad.wmflabs:10514'
-"wdqs::updater::options": '-n wdq -s'
-"wdqs::use_git_deploy": false
-"wdqs::package_dir": '/srv/wdqs/blazegraph'
-"wdqs::data_dir": '/srv/wdqs/blazegraph'
-"wdqs::endpoint": ''
-"wdqs::blazegraph_heap_size": '8g'
+profile::wdqs::log_aggregator: 
'deployment-logstash2.deployment-prep.eqiad.wmflabs:10514'
+profile::wdqs::use_git_deploy: false
+profile::wdqs::package_dir: '/srv/wdqs/blazegraph'
+profile::wdqs::data_dir: '/srv/wdqs/blazegraph'
+profile::wdqs::endpoint: ''
+profile::wdqs::updater_options: '-n wdq -s'
+profile::wdqs::config_file: 'RWStore.properties'
+profile::wdqs::blazegraph_options: ''
+profile::wdqs::blazegraph_heap_size: '8g'
diff --git a/hieradata/role/codfw/wdqs.yaml b/hieradata/role/codfw/wdqs.yaml
index 9083c4b..57d9e1e 100644
--- a/hieradata/role/codfw/wdqs.yaml
+++ b/hieradata/role/codfw/wdqs.yaml
@@ -1,3 +1,3 @@
 lvs::realserver::realserver_ips:
 - '10.2.1.32' # wdqs.svc.codfw.wmnet
-wdqs::blazegraph_options: '-Dhttp.proxyHost=webproxy.codfw.wmnet 
-Dhttp.proxyPort=8080 -XX:+ExitOnOutOfMemoryError'
+profile::wdqs::blazegraph_options: '-Dhttp.proxyHost=webproxy.codfw.wmnet 
-Dhttp.proxyPort=8080 -XX:+ExitOnOutOfMemoryError'
diff --git a/hieradata/role/common/wdqs.yaml b/hieradata/role/common/wdqs.yaml
index 1dbf8d2..1012c03 100644
--- a/hieradata/role/common/wdqs.yaml
+++ b/hieradata/role/common/wdqs.yaml
@@ -1,7 +1,11 @@
-wdqs::gui::log_aggregator:  'logstash1001.eqiad.wmnet:10514'
-wdqs::updater::options: '-n wdq -- -b 500 -T 1200'
-wdqs::service::config_file: /etc/wdqs/RWStore.properties
-wdqs::endpoint: 'https://query.wikidata.org'
+profile::wdqs::log_aggregator: 'logstash1001.eqiad.wmnet:10514'
+profile::wdqs::use_git_deploy: true
+profile::wdqs::package_dir: '/srv/deployment/wdqs/wdqs'
+profile::wdqs::data_dir: '/srv/wdqs'
+profile::wdqs::endpoint: 'https://query.wikidata.org'
+profile::wdqs::updater_options: '-n wdq -- -b 500 -T 1200'
+profile::wdqs::config_file: '/etc/wdqs/RWStore.properties'
+profile::wdqs::blazegraph_heap_size: '16g'
 admin::groups:
   - wdqs-admins
 cluster: wdqs
diff --git a/manifests/site.pp b/manifests/site.pp
index 8f559b5..8d4dc8e 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2323,12 +2323,10 @@
 # Wikidata query service
 node /^wdqs100[1-3]\.eqiad\.wmnet$/ {
 role(wdqs)
-include ::lvs::realserver
 }
 
 node /^wdqs200[1-3]\.codfw\.wmnet$/ {
 role(wdqs)
-include ::lvs::realserver
 }
 
 node 'wezen.codfw.wmnet' {
diff --git a/modules/profile/manifests/wdqs.pp 
b/modules/profile/manifests/wdqs.pp
new file mode 100644
index 000..862b2eb
--- /dev/null
+++ b/modules/profile/manifests/wdqs.pp
@@ -0,0 +1,47 @@
+class profile::wdqs (
+$log_aggregator = hiera('profile::wdqs::log_aggregator'),
+$use_git_deploy = hiera('profile::wdqs::use_git_deploy'),
+$package_dir = hiera('profile::wdqs::package_dir'),
+$data_dir = hiera('profile::wdqs::data_dir'),
+$endpoint = hiera('profile::wdqs::endpoint'),
+$blazegraph_options = hiera('profile::wdqs::blazegraph_options'),
+$blazegraph_heap_size = hiera('profile::wdqs::blazegraph_heap_size'),
+$blazegraph_config_file = hiera('profile::wdqs::blazegraph_config_file'),
+) {
+$nagios_contact_group = 'admins,wdqs-admins'
+
+# Install services - both blazegraph and the updater
+class { '::wdqs':
+use_git_deploy => false,
+package_dir => $package_dir,
+data_dir => $data_dir,
+endpoint => $endpoint,
+blazegraph_options => $blazegraph_options,
+blazegraph_heap_size => $blazegraph_heap_size,
+blazegraph_config_file => $blazegraph_config_file,
+}
+
+# Service Web proxy
+class { '::wdqs::gui':
+log_aggregator => $log_aggregator,
+}
+
+# Firewall
+ferm::service {
+'wdqs_http':
+proto => 'tcp',
+port  => '80';
+

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: [labs] Puppetize XTools

[MaxSem (Code Review)]

MaxSem has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/368101 )

Change subject: WIP: [labs] Puppetize XTools
..

WIP: [labs] Puppetize XTools

Bug: T170514
Change-Id: Ia5320d52dd6934232a51ea5ad1d65288b8666b79
---
A modules/xtools/files/update-xtools
A modules/xtools/manifests/code.pp
A modules/xtools/manifests/init.pp
A modules/xtools/manifests/packages.pp
A modules/xtools/manifests/web.pp
A modules/xtools/templates/xtools.conf.erb
6 files changed, 110 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/01/368101/1

diff --git a/modules/xtools/files/update-xtools 
b/modules/xtools/files/update-xtools
new file mode 100644
index 000..a1d51bf
--- /dev/null
+++ b/modules/xtools/files/update-xtools
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export SYMFONY_ENV=prod
+cd /var/www
+
+## See if there's any update.
+GITFETCH=$(git fetch 2>&1)
+if [ -z "$GITFETCH" ]; then
+exit 0
+fi
+
+## If there's an update, pull and install it.
+git checkout master
+git pull origin master
+/usr/local/bin/composer install --no-dev --optimize-autoloader
+./bin/console cache:clear --env prod
+./bin/console doctrine:migrations:migrate --env prod --no-interaction
+./bin/console assetic:dump --env prod
diff --git a/modules/xtools/manifests/code.pp b/modules/xtools/manifests/code.pp
new file mode 100644
index 000..9cca0a0
--- /dev/null
+++ b/modules/xtools/manifests/code.pp
@@ -0,0 +1,40 @@
+class xtools::code {
+group { 'xtools':
+ensure => present,
+}
+
+user { 'xtools-update':
+ensure  => present,
+group   => 'xtools',
+system  => true,
+require => Group['xtools'],
+}
+
+file { '/srv/xtools':
+ensure => directory,
+owner  => 'xtools-update',
+mode   => '0775',
+}
+
+::git::clone { 'xtools':
+directory => '/srv/xtools',
+owner => 'xtools-update',
+origin=> 'https://github.com/x-tools/xtools-rebirth.git',
+require   => [ User['xtools-update'], File['/srv/xtools'] ]
+}
+# todo: composer install. Requires user input so far
+
+file { '/usr/local/update-xtools':
+source => 'xtools/update-xtools',
+owner  => 'root',
+mode   => '0555',
+}
+
+cron { 'update-xtools':
+ensure  => present,
+command => '/usr/local/update-xtools',
+user=> 'xtools-update',
+hour=> '*',
+require => [ File['/usr/local/update-xtools'], Git::Clone['xtools'] ],
+}
+}
diff --git a/modules/xtools/manifests/init.pp b/modules/xtools/manifests/init.pp
new file mode 100644
index 000..deb54d8
--- /dev/null
+++ b/modules/xtools/manifests/init.pp
@@ -0,0 +1,11 @@
+class xtools(
+$host = 'xtools.wmflabs.org',
+) {
+requires_realm('labs')
+
+require ::xtools::packages
+require ::xtools::code
+class { 'xtools::web':
+host => $host,
+}
+}
diff --git a/modules/xtools/manifests/packages.pp 
b/modules/xtools/manifests/packages.pp
new file mode 100644
index 000..9def3b9
--- /dev/null
+++ b/modules/xtools/manifests/packages.pp
@@ -0,0 +1,7 @@
+class xtools::packages {
+require_package(
+'percona-toolkit',
+'mysql-client',
+'git'
+)
+}
diff --git a/modules/xtools/manifests/web.pp b/modules/xtools/manifests/web.pp
new file mode 100644
index 000..3f7e9ad
--- /dev/null
+++ b/modules/xtools/manifests/web.pp
@@ -0,0 +1,10 @@
+class xtools::web(
+$host
+) {
+::apache::site { "xtools-$host":
+ensure => present,
+content => template('xtools/xtools.conf.erb'),
+}
+
+require apache::mod::php7
+}
diff --git a/modules/xtools/templates/xtools.conf.erb 
b/modules/xtools/templates/xtools.conf.erb
new file mode 100644
index 000..afce181
--- /dev/null
+++ b/modules/xtools/templates/xtools.conf.erb
@@ -0,0 +1,24 @@
+
+DocumentRoot /srv/xtools/web
+ServerName <%= @host %>
+
+ErrorLog ${APACHE_LOG_DIR}/error.log
+CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+
+ Options Indexes FollowSymLinks
+ AllowOverride All
+ Require all granted
+
+
+Alias /awstatsclasses "/usr/share/awstats/lib/"
+Alias /awstats-icon/ "/usr/share/awstats/icon/"
+Alias /awstatscss "/usr/share/doc/awstats/examples/css"
+ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/
+ScriptAlias /awstats/ /usr/lib/cgi-bin/
+
+Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch
+Require all granted
+
+
+

-- 
To view, visit https://gerrit.wikimedia.org/r/368101
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia5320d52dd6934232a51ea5ad1d65288b8666b79
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Configure Cassandra for restbase-dev[4-6]

[Eevans (Code Review)]

Eevans has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/366334 )

Change subject: WIP: Configure Cassandra for restbase-dev[4-6]
..

WIP: Configure Cassandra for restbase-dev[4-6]

- Needs actual IP addresses plugged in
- Should old nodes/hosts be removed when the new ones are added?

Bug: T171104
Change-Id: I676ee062227d02b130d6d939b8e6d0da5fce330c
---
M hieradata/role/common/restbase/dev_cluster.yaml
1 file changed, 43 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/34/366334/1

diff --git a/hieradata/role/common/restbase/dev_cluster.yaml 
b/hieradata/role/common/restbase/dev_cluster.yaml
index 6633e34..cfff9d2 100644
--- a/hieradata/role/common/restbase/dev_cluster.yaml
+++ b/hieradata/role/common/restbase/dev_cluster.yaml
@@ -32,10 +32,19 @@
   - restbase-dev1002-b.eqiad.wmnet
   - restbase-dev1003-a.eqiad.wmnet
   - restbase-dev1003-b.eqiad.wmnet
+  - restbase-dev1004-a.eqiad.wmnet
+  - restbase-dev1004-b.eqiad.wmnet
+  - restbase-dev1005-a.eqiad.wmnet
+  - restbase-dev1005-b.eqiad.wmnet
+  - restbase-dev1006-a.eqiad.wmnet
+  - restbase-dev1006-b.eqiad.wmnet
 profile::restbase::hosts:
   - restbase-dev1001.eqiad.wmnet
   - restbase-dev1002.eqiad.wmnet
   - restbase-dev1003.eqiad.wmnet
+  - restbase-dev1004.eqiad.wmnet
+  - restbase-dev1005.eqiad.wmnet
+  - restbase-dev1006.eqiad.wmnet
 
 
 #
@@ -99,6 +108,40 @@
 b:
   listen_address: 10.64.48.118
 
+  "restbase-dev1004.eqiad.wmnet":
+a:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+b:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+  "restbase-dev1005.eqiad.wmnet":
+a:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+b:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+  "restbase-dev1006.eqiad.wmnet":
+a:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+b:
+  listen_address: xx.xx.x.xx
+  data_file_directories:
+- data_0
+- data_1
+
 profile::cassandra::settings:
   start_rpc: 'true'
   max_heap_size: 12g

-- 
To view, visit https://gerrit.wikimedia.org/r/366334
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I676ee062227d02b130d6d939b8e6d0da5fce330c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Eevans 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Configure an additional data file directory

[Eevans (Code Review)]

Eevans has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/365081 )

Change subject: WIP: Configure an additional data file directory
..

WIP: Configure an additional data file directory

Bug: 170276
Change-Id: I67edd8d78a6522c82e9dd1a7fe1d3bf41faeed27
---
M hieradata/role/common/restbase/dev_cluster.yaml
1 file changed, 3 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/81/365081/1

diff --git a/hieradata/role/common/restbase/dev_cluster.yaml 
b/hieradata/role/common/restbase/dev_cluster.yaml
index 3cbda43..6633e34 100644
--- a/hieradata/role/common/restbase/dev_cluster.yaml
+++ b/hieradata/role/common/restbase/dev_cluster.yaml
@@ -83,6 +83,9 @@
   "restbase-dev1001.eqiad.wmnet":
 a:
   listen_address: 10.64.0.36
+  data_file_directories:
+- data
+- data_0
 b:
   listen_address: 10.64.0.37
   "restbase-dev1002.eqiad.wmnet":

-- 
To view, visit https://gerrit.wikimedia.org/r/365081
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I67edd8d78a6522c82e9dd1a7fe1d3bf41faeed27
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Eevans 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]mariadb: Add grants for rddmark to m1

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/365035 )

Change subject: [WIP]mariadb: Add grants for rddmark to m1
..

[WIP]mariadb: Add grants for rddmark to m1

Add user account and backups for rddmark.

Blocked on missing source ip/ips and predicted traffic to confirm
it should go to m1.

Bug: T170158
Change-Id: I87878898b24d0688cf67e6b945cab9e6fbcd9421
---
M modules/role/manifests/mariadb.pp
M modules/role/templates/mariadb/backups/dumps-misc.sh.erb
M modules/role/templates/mariadb/grants/production-m1.sql.erb
3 files changed, 5 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/35/365035/1

diff --git a/modules/role/manifests/mariadb.pp 
b/modules/role/manifests/mariadb.pp
index f22bac9..23ef20c 100644
--- a/modules/role/manifests/mariadb.pp
+++ b/modules/role/manifests/mariadb.pp
@@ -66,6 +66,7 @@
 $striker_admin_pass  = $passwords::striker::admin_db_password
 $labspuppet_pass = hiera('labspuppetbackend_mysql_password')
 $labsdbaccounts_pass = $passwords::labsdbaccounts::db_password
+$rddmark_pass= $passwords::rddmark::db_password
 
 file { '/etc/mysql/production-grants-shard.sql':
 ensure  => present,
diff --git a/modules/role/templates/mariadb/backups/dumps-misc.sh.erb 
b/modules/role/templates/mariadb/backups/dumps-misc.sh.erb
index 3fc977a..1d02b65 100644
--- a/modules/role/templates/mariadb/backups/dumps-misc.sh.erb
+++ b/modules/role/templates/mariadb/backups/dumps-misc.sh.erb
@@ -5,7 +5,7 @@
 my="/usr/local/bin/mysql --skip-column-names --skip-ssl"
 dump="/usr/local/bin/mysqldump --single-transaction --quick --master-data=2 
--skip-ssl"
 
-$dump -h m1-slave --databases bacula etherpadlite librenms puppet racktables 
rt | \
+$dump -h m1-slave --databases bacula etherpadlite librenms puppet racktables 
rt rddmarc | \
 pigz > /srv/backups/m1-$(date +%Y%m%d%H%M%S).sql.gz
 
 $dump -h m2-slave --databases iegreview reviewdb scholarships | \
diff --git a/modules/role/templates/mariadb/grants/production-m1.sql.erb 
b/modules/role/templates/mariadb/grants/production-m1.sql.erb
index 29e9132..ff96ad6 100644
--- a/modules/role/templates/mariadb/grants/production-m1.sql.erb
+++ b/modules/role/templates/mariadb/grants/production-m1.sql.erb
@@ -112,4 +112,6 @@
 SELECT, TRIGGER, UPDATE
 ON `puppet`.* TO 'servermon'@'10.64.16.159';
 
-
+-- rddmarc
+GRANT ALTER, CREATE, SELECT, INSERT, UPDATE, DELETE
+ON rddmarc.* TO 'rddmarc'@'' IDENTIFIED BY 
'<%=@rddmark_pass%>';

-- 
To view, visit https://gerrit.wikimedia.org/r/365035
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I87878898b24d0688cf67e6b945cab9e6fbcd9421
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: base::kernel: add base::kernel::module

[Ema (Code Review)]

Ema has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/365030 )

Change subject: WIP: base::kernel: add base::kernel::module
..

WIP: base::kernel: add base::kernel::module

Change-Id: Ib2bd63df23dcc6e33ad51b444c45cc731ea25f0c
---
M modules/base/manifests/kernel.pp
1 file changed, 30 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/30/365030/1

diff --git a/modules/base/manifests/kernel.pp b/modules/base/manifests/kernel.pp
index 01c9c76..f058eb8 100644
--- a/modules/base/manifests/kernel.pp
+++ b/modules/base/manifests/kernel.pp
@@ -68,3 +68,33 @@
 }
 }
 }
+
+define base::kernel::module($ensure=present) {
+case $ensure {
+'present': {
+file { "/etc/modules-load.d/${name}.conf":
+ensure  => present,
+owner   => 'root',
+group   => 'root',
+mode=> '0444',
+content => "${name}\n",
+require => File['/etc/modules-load.d/'],
+notify  => Exec["modprobe ${name}"],
+}
+
+exec { "modprobe ${name}":
+unless => "/bin/lsmod | /bin/grep -q '^${name}'",
+}
+}
+'absent': {
+file { "/etc/modules-load.d/${name}.conf":
+ensure => absent,
+notify => Exec["modprobe -r ${name}"],
+}
+
+exec { "modprobe -r ${name}":
+onlyif => "/bin/lsmod | /bin/grep -q '^${name}'",
+}
+}
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/365030
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib2bd63df23dcc6e33ad51b444c45cc731ea25f0c
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ema 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: (WIP): Add SNMP classes

[Faidon Liambotis (Code Review)]

Faidon Liambotis has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/364753 )

Change subject: (WIP): Add SNMP classes
..

(WIP): Add SNMP classes

Add an snmp class and, more importantly, and snmp::mibs::source define
that can use snmp-mibs-downloader to fetch third-party MIBs. Populate
the Juniper MIBs in a new snmp::mibs class.

Change-Id: Id34099a3abe8e97cc8fbd023ce810e8658c7f7cf
---
A modules/snmp/files/juniper-mibs
A modules/snmp/manifests/init.pp
A modules/snmp/manifests/mibs.pp
A modules/snmp/manifests/mibs/source.pp
A modules/snmp/templates/mib.conf.erb
5 files changed, 229 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/53/364753/1

diff --git a/modules/snmp/files/juniper-mibs b/modules/snmp/files/juniper-mibs
new file mode 100644
index 000..2b1e68f
--- /dev/null
+++ b/modules/snmp/files/juniper-mibs
@@ -0,0 +1,156 @@
+# Up to date as of 16.1R3.10
+#
+# Generate this list with:
+#   for mib in *; do
+# new=$(sed -nr 's/\s*(.*) DEFINITIONS.*/\1/p' $mib);
+# old=$(echo $mib |sed 's/^mib-//;s/\.txt$//');
+# echo $old $new;
+#   done | column -t | sort
+jnx-alarmJUNIPER-ALARM-EXT-MIB
+jnx-analyzer JUNIPER-ANALYZER-MIB
+jnx-atm-cos  JUNIPER-ATM-COS-MIB
+jnx-atm  JUNIPER-ATM-MIB
+jnx-bfd-exp  BFD-STD-MIB
+jnx-bfd  JUNIPER-BFD-MIB
+jnx-bgpmib2  BGP4-V2-MIB-JUNIPER
+jnx-bl   JNX-OPT-IF-EXT-MIB
+jnx-cfgmgmt  JUNIPER-CFGMGMT-MIB
+jnx-chas-defines JUNIPER-CHASSIS-DEFINES-MIB
+jnx-chassis-alarmJUNIPER-ALARM-MIB
+jnx-chassis-fwdd JUNIPER-CHASSIS-FWDD-MIB
+jnx-chassis  JUNIPER-MIB
+jnx-coll JUNIPER-COLLECTOR-MIB
+jnx-cos  JUNIPER-COS-MIB
+jnx-dcu  JUNIPER-DCU-MIB
+jnx-dfc  JUNIPER-DFC-MIB
+jnx-dom  JUNIPER-DOM-MIB
+jnx-dot3oam-capability   JNX-DOT3OAM-CAPABILITY
+jnx-eventJUNIPER-EVENT-MIB
+jnx-ex-mac-notification  JUNIPER-EX-MAC-NOTIFICATION-MIB
+jnx-exp  JUNIPER-EXPERIMENT-MIB
+jnx-ex-smi   JUNIPER-EX-SMI
+jnx-fabric-chassis   JUNIPER-FABRIC-CHASSIS
+jnx-fabric   JUNIPER-FABRIC-MIB
+jnx-firewall JUNIPER-FIREWALL-MIB
+jnx-fru  JUNIPER-FRU-MIB
+jnx-gen-set  JUNIPER-SNMP-SET-MIB
+jnx-hostresourcesJUNIPER-HOSTRESOURCES-MIB
+jnx-if-accountingJUNIPER-IF-ACCOUNTING-MIB
+jnx-if-capabilityJNX-IF-CAPABILITY
+jnx-if-extensionsJUNIPER-IF-MIB
+jnx-ifotnJUNIPER-IFOTN-MIB
+jnx-ip-capabilityJNX-IP-CAPABILITY
+jnx-ipforwardJUNIPER-IPFORWARD-MIB
+jnx-ipmcast-capability   IPMCAST-MIB-CAPABILITY
+jnx-ipsec-flow-mon   JUNIPER-IPSEC-FLOW-MON-MIB
+jnx-ipsec-monitor-aspJNX-IPSEC-MONITOR-MIB
+jnx-ipv4 JUNIPER-IPv4-MIB
+jnx-ipv6 JUNIPER-IPv6-MIB
+jnx-jdhcpJUNIPER-JDHCP-MIB
+jnx-jdhcpv6  JUNIPER-JDHCPV6-MIB
+jnx-js-auth  JUNIPER-JS-AUTH-MIB
+jnx-js-cert  JUNIPER-JS-CERT-MIB
+jnx-js-dns   JUNIPER-JS-DNS-MIB
+jnx-js-idp   JUNIPER-JS-IDP-MIB
+jnx-js-if-extJUNIPER-JS-IF-EXT-MIB
+jnx-js-ipsec-vpn JUNIPER-JS-IPSEC-VPN-MIB
+jnx-js-nat   JUNIPER-JS-NAT-MIB
+jnx-js-packet-mirror JUNIPER-JS-PACKET-MIRROR-MIB
+jnx-js-policyJUNIPER-JS-POLICY-MIB
+jnx-jsrpdJUNIPER-CHASSIS-CLUSTER-MIB
+jnx-js-screening JUNIPER-JS-SCREENING-MIB
+jnx-js-smi   JUNIPER-JS-SMI
+jnx-js-spu-monitoringJUNIPER-SRX5000-SPU-MONITORING-MIB
+jnx-js-utm-avJUNIPER-JS-UTM-AV-MIB
+jnx-jvae-infra   JUNIPER-JVAE-INFRA-MIB
+jnx-jvae-nodeJUNIPER-JVAE-NODE-MIB
+jnx-l2aldJUNIPER-L2ALD-MIB
+jnx-l2cp-featuresJUNIPER-L2CP-FEATURES-MIB
+jnx-l2tp JNX-L2TP-MIB
+jnx-ldp  JUNIPER-LDP-MIB
+jnx-license  JUNIPER-LICENSE-MIB
+jnx-lsys-securityprofile JUNIPER-LSYS-SECURITYPROFILE-MIB
+jnx-lsys-sp-authentryJUNIPER-LSYSSPAUTHENTRY-MIB
+jnx-lsys-sp-cpu  JUNIPER-LSYSSP-CPU-MIB
+jnx-lsys-sp-flowgate JUNIPER-LSYSSP-FLOWGATE-MIB
+jnx-lsys-sp-flowsess JUNIPER-LSYSSP-FLOWSESS-MIB

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP: Monitoring: Simplify BMC hostnames

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/364708 )

Change subject: WIP: Monitoring: Simplify BMC hostnames
..

WIP: Monitoring: Simplify BMC hostnames

Drop the $::site.wmnet suffix from all hostnames, making the defined BMC
hostnames more aligned to the current status quo of only using hostnames
and not FQDN. Amend check_fqdn respectively to now check to FQDN against
the IP

Change-Id: Ie77daf21c8c9d8716ffc15d316ec936de9572bbe
---
M modules/monitoring/manifests/host.pp
M modules/nagios_common/files/checkcommands.cfg
2 files changed, 10 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/08/364708/1

diff --git a/modules/monitoring/manifests/host.pp 
b/modules/monitoring/manifests/host.pp
index 2f76db3..9e99a6d 100644
--- a/modules/monitoring/manifests/host.pp
+++ b/modules/monitoring/manifests/host.pp
@@ -66,9 +66,9 @@
 # We have a BMC, and the BMC is configured and it has an IP address
 if $facts['has_ipmi'] and $facts['ipmi_lan'] and 'ipaddress' in 
$facts['ipmi_lan'] {
 $mgmt_host = {
-"${title}.mgmt.${::site}.wmnet" => {
+"${title}.mgmt" => {
 ensure=> $ensure,
-host_name => "${title}.mgmt.${::site}.wmnet",
+host_name => "${title}.mgmt",
 address   => $facts['ipmi_lan']['ipaddress'],
 hostgroups=> 'mgmt',
 check_command => 'check_ping!500,20%!2000,100%',
@@ -119,17 +119,17 @@
 create_resources($rtype, $host)
 if $mgmt_host {
 create_resources($rtype, $mgmt_host)
-monitoring::service { "dns_${title}.mgmt.${::site}.wmnet":
-description=> "DNS ${title}.mgmt.${::site}.wmnet",
-host   => "${title}.mgmt.${::site}.wmnet",
-check_command  => 'check_fqdn',
+monitoring::service { "dns_${title}.mgmt":
+description=> "DNS ${title}.mgmt",
+host   => "${title}.mgmt",
+check_command  => "check_fqdn!${title}.mgmt.${::site}.wmnet",
 group  => 'mgmt',
 check_interval => 60,
 retry_interval => 60,
 }
-monitoring::service { "ssh_${title}.mgmt.${::site}.wmnet":
-description=> "SSH ${title}.mgmt.${::site}.wmnet",
-host   => "${title}.mgmt.${::site}.wmnet",
+monitoring::service { "ssh_${title}.mgmt":
+description=> "SSH ${title}.mgmt",
+host   => "${title}.mgmt",
 check_command  => 'check_ssh',
 group  => 'mgmt',
 check_interval => 60,
diff --git a/modules/nagios_common/files/checkcommands.cfg 
b/modules/nagios_common/files/checkcommands.cfg
index a01fb98..8c4f9d1 100644
--- a/modules/nagios_common/files/checkcommands.cfg
+++ b/modules/nagios_common/files/checkcommands.cfg
@@ -62,7 +62,7 @@
 
 define command {
 command_name   check_fqdn
-command_line   $USER1$/check_dns -H $HOSTNAME$ -a $HOSTADDRESS$
+command_line   $USER1$/check_dns -H $ARG1$ -a $HOSTADDRESS$
 }
 
 # 'check_ftp' command definition

-- 
To view, visit https://gerrit.wikimedia.org/r/364708
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie77daf21c8c9d8716ffc15d316ec936de9572bbe
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Alexandros Kosiaris 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP]prometheus: Convert mysqld-exporter into multi-instance

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/364396 )

Change subject: [WIP]prometheus: Convert mysqld-exporter into multi-instance
..

[WIP]prometheus: Convert mysqld-exporter into multi-instance

We need to monitor several independent mysql instances on the same
host. Add an additional resource that actually can be duplicated
so that more than one instance can be run at the same time.

This is still work in progress, but uploaded so it can be discussed
before more refactoring is done. Needs architecture feedback.

Bug: T169514
Change-Id: I664f04c7b026aedf32a1ef1ec648fea81dd47436
---
A modules/prometheus/manifests/mysqld_exporter_multiinstance.pp
1 file changed, 65 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/96/364396/1

diff --git a/modules/prometheus/manifests/mysqld_exporter_multiinstance.pp 
b/modules/prometheus/manifests/mysqld_exporter_multiinstance.pp
new file mode 100644
index 000..7094872
--- /dev/null
+++ b/modules/prometheus/manifests/mysqld_exporter_multiinstance.pp
@@ -0,0 +1,65 @@
+# == Define: prometheus::mysqld_exporter
+#
+# Prometheus exporter for MySQL server metrics. The exporter is most effective
+# when ran alongside the MySQL server to be monitored, connecting via a local
+# UNIX socket is supported.
+#
+# = Parameters
+#
+# [*client_socket*]
+#   The socket to connect to.
+#
+# [*client_user*]
+#   MySQL user
+#
+# [*client_password*]
+#   MySQL password
+#
+# [*arguments*]
+#   Additional command line arguments for prometheus-mysqld-exporter.
+
+define prometheus::mysqld_exporter (
+$client_socket = '/tmp/mysql.sock',
+$client_user = 'prometheus',
+$client_password = '',
+$port = 9104,
+$arguments = '',
+) {
+#require_package('prometheus-mysqld-exporter')
+#
+#file { '/var/lib/prometheus':
+#ensure => directory,
+#mode   => '0550',
+#owner  => 'prometheus',
+#group  => 'prometheus',
+#}
+
+# default .my.cnf location (i.e. $HOME/.my.cnf)
+file { "/var/lib/prometheus/{$title}.my.cnf":
+ensure  => present,
+mode=> '0400',
+owner   => 'prometheus',
+group   => 'prometheus',
+content => template('prometheus/mysqld_exporter.cnf.erb'),
+require => [
+  Package['prometheus-mysqld-exporter'],
+  File['/var/lib/prometheus'],
+],
+notify  => Service['prometheus-mysqld-exporter@{$title}'],
+}
+
+#file { '/etc/default/prometheus-mysqld-exporter':
+#ensure  => present,
+#mode=> '0444',
+#owner   => 'root',
+#group   => 'root',
+#content => "ARGS=\"${arguments}\"",
+#notify  => Service['prometheus-mysqld-exporter'],
+#}
+
+service { "prometheus-mysqld-exporter@{$title}":
+ensure  => running,
+require => [Package['prometheus-mysqld-exporter'],
+File['/var/lib/prometheus/{$title}.my.cnf']],
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/364396
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I664f04c7b026aedf32a1ef1ec648fea81dd47436
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: (WIP) packages for R

[Hashar (Code Review)]

Hashar has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/363337 )

Change subject: (WIP) packages for R
..

(WIP) packages for R

Should probably factor out code from shiny_server instead of copy
pasting.

Bug: T153856
Change-Id: I72825701f624b495376cd9967dfff53b91632b97
---
A modules/contint/manifests/packages/r.pp
1 file changed, 11 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/37/363337/1

diff --git a/modules/contint/manifests/packages/r.pp 
b/modules/contint/manifests/packages/r.pp
new file mode 100644
index 000..99c3c1d
--- /dev/null
+++ b/modules/contint/manifests/packages/r.pp
@@ -0,0 +1,11 @@
+class contint::packages::r {
+packages { [
+'r-base',
+'r-base-dev',
+'libcurl4-openssl-dev',
+'libssh-dev',
+'libssl-dev',
+]:
+ensure => present,
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/363337
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I72825701f624b495376cd9967dfff53b91632b97
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Hashar 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Support multiple instances on the mariadb module

[Jcrespo (Code Review)]

Jcrespo has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/363195 )

Change subject: [WIP] Support multiple instances on the mariadb module
..

[WIP] Support multiple instances on the mariadb module

This is just a first idea that doesn't work yet, but could be
one of the ways to implement it.

The idea is to have a common mariadb::config, and then one
resouces mariadb::instance usage for each mysql run, with its own
datadir, tmpdir, port and socket. The custom config should include
the extra config files on /etc/mysql/mysqld.config.d and either
init.d or systemd (with mariadb@.service) should be able to
handle the different configs.

Many questions are still left about the details.

Bug: T169514
Change-Id: Ib313d12e6784700e0afaccd443acb181b7a0e1d4
---
A modules/mariadb/manifests/instance.pp
A modules/mariadb/templates/instance.cnf.erb
A modules/role/manifests/mariadb/dbstore3.pp
3 files changed, 98 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/95/363195/1

diff --git a/modules/mariadb/manifests/instance.pp 
b/modules/mariadb/manifests/instance.pp
new file mode 100644
index 000..ef6474b
--- /dev/null
+++ b/modules/mariadb/manifests/instance.pp
@@ -0,0 +1,39 @@
+# Setups additional instances for hosts that hosts more
+# than one instance
+define mariadb::instance(
+$port,
+$datadir = 'undefined',
+$tmpdir  = 'undefined',
+$socket  = 'undefined',
+) {
+if $datadir == 'undefined' {
+$datadir = "/srv/sqldata.${title}",
+}
+if $tmpdir == 'undefined' {
+$tmpdir  = '/srv/tmp.${title}',
+}
+if $tmpdir == 'undefined' {
+$socket  = '/run/mysqld/mysqld.${title}.sock',
+}
+
+file { $datadir:
+ensure => directory,
+owner  => 'mysql',
+group  => 'mysql',
+mode   => '0755',
+}
+
+file { $tmpdir:
+ensure => directory,
+owner  => 'mysql',
+group  => 'mysql',
+mode   => '0755',
+}
+
+file { "/etc/mysql/mysql.conf.d/${title}.cnf"
+owner   => 'root',
+group   => 'root',
+mode=> '0644',
+content => template('mariadb/instance.cnf.erb'),
+}
+}
diff --git a/modules/mariadb/templates/instance.cnf.erb 
b/modules/mariadb/templates/instance.cnf.erb
new file mode 100644
index 000..9aefd5f
--- /dev/null
+++ b/modules/mariadb/templates/instance.cnf.erb
@@ -0,0 +1,4 @@
+[mysqld@<%= @title %>]
+datadir=<%= @datadir %>
+socket=<%= @socket %>
+port=<%= @port %>
diff --git a/modules/role/manifests/mariadb/dbstore3.pp 
b/modules/role/manifests/mariadb/dbstore3.pp
new file mode 100644
index 000..f72f94d
--- /dev/null
+++ b/modules/role/manifests/mariadb/dbstore3.pp
@@ -0,0 +1,55 @@
+class role::mariadb::dbstore3 {
+system::role { 'mariadb::core':
+description => "DBStore multi-instance Server",
+}
+
+include ::standard
+include ::base::firewall
+include role::mariadb::monitor
+#TODO: Custom firewall rules
+
+#TODO: define one group per shard
+class {'mariadb::groups':
+mysql_group => 'dbstore',
+mysql_shard => 's1',
+mysql_role  => 'slave',
+socket  => '/run/mysqld/mysqld.s1.sock',
+}
+
+class {'mariadb::packages_wmf': }
+class {'mariadb::service':
+# for now we will keep things simple, we probably should have a
+# higher-level interface with templates
+override => "[Service]\nLimitNOFILE=20",
+}
+
+# Read only forced on also for the masters of the primary datacenter
+class { 'mariadb::config':
+config   => 'role/mariadb/mysqld_config/dbstore3.my.cnf.erb',
+p_s  => 'on',
+ssl  => 'puppet-cert',
+binlog_format=> 'ROW',
+}
+
+mariadb::instance {'s1':
+port => 3311,
+}
+mariadb::instance {'s2':
+port => 3312,
+}
+mariadb::instance {'s3':
+port => 3313,
+}
+mariadb::instance {'s4':
+port => 3314,
+}
+mariadb::instance {'s5':
+port => 3315,
+}
+mariadb::instance {'s6':
+port => 3316,
+}
+mariadb::instance {'s7':
+port => 3317,
+}
+}

-- 
To view, visit https://gerrit.wikimedia.org/r/363195
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib313d12e6784700e0afaccd443acb181b7a0e1d4
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Jcrespo 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: WIP monitoring: provide basic Rspec

[Alexandros Kosiaris (Code Review)]

Alexandros Kosiaris has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/363186 )

Change subject: WIP monitoring: provide basic Rspec
..

WIP monitoring: provide basic Rspec

Add RSpec for monitoring module.

Change-Id: I04db8fe2051dc9f667da0e4e557c740174c352a1
---
A modules/monitoring/.fixtures.yml
A modules/monitoring/.rspec
A modules/monitoring/Rakefile
A modules/monitoring/spec/defines/monitoring_host_spec.rb
A modules/monitoring/spec/fixtures/modules/bacula
A modules/monitoring/spec/fixtures/modules/base
A modules/monitoring/spec/fixtures/modules/stdlib
A modules/monitoring/spec/fixtures/modules/wmflib
A modules/monitoring/spec/spec_helper.rb
9 files changed, 133 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/86/363186/1

diff --git a/modules/monitoring/.fixtures.yml b/modules/monitoring/.fixtures.yml
new file mode 100644
index 000..ef1bcbf
--- /dev/null
+++ b/modules/monitoring/.fixtures.yml
@@ -0,0 +1,8 @@
+fixtures:
+symlinks:
+monitoring: "#{source_dir}"
+icinga: "../../../../icinga"
+nagios_common: "../../../../nagios_common"
+wmflib: "../../../../wmflib"
+logrotate: "../../../../logrotate"
+secret: "../../../../../../labs/private/modules/secret"
diff --git a/modules/monitoring/.rspec b/modules/monitoring/.rspec
new file mode 100644
index 000..f449dae
--- /dev/null
+++ b/modules/monitoring/.rspec
@@ -0,0 +1,2 @@
+--format doc
+--color
diff --git a/modules/monitoring/Rakefile b/modules/monitoring/Rakefile
new file mode 100644
index 000..cd3d379
--- /dev/null
+++ b/modules/monitoring/Rakefile
@@ -0,0 +1 @@
+require 'puppetlabs_spec_helper/rake_tasks'
diff --git a/modules/monitoring/spec/defines/monitoring_host_spec.rb 
b/modules/monitoring/spec/defines/monitoring_host_spec.rb
new file mode 100644
index 000..f863008
--- /dev/null
+++ b/modules/monitoring/spec/defines/monitoring_host_spec.rb
@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'monitoring::host' do
+  context 'with a standard physical host' do
+let(:facts) {
+  {
+:hostname=> 'ahost',
+:operatingsystem => 'Debian',
+:ipaddress   => '1.2.3.4',
+:is_virtual  => false,
+:lldp_parent => 'ahosts_parent',
+  }
+}
+let(:title) { 'ahost' }
+it { should compile }
+describe 'with no parameters' do
+  subject { exported_resources }
+  it do
+should contain_nagios_host('ahost').with(
+  'host_name'  => 'ahost',
+  'parents'=> 'ahosts_parent',
+  'icon_image' => 'vendors/debian.png',
+  'address'=> '1.2.3.4',
+)
+end
+end
+describe 'with a parents parameters' do
+  let(:params) {
+{
+  :parents => 'aparent',
+}
+  }
+  subject { exported_resources }
+  it do
+should contain_nagios_host('ahost').with(
+  'host_name'  => 'ahost',
+  'parents'=> 'aparent',
+  'icon_image' => 'vendors/debian.png',
+  'address'=> '1.2.3.4',
+)
+  end
+end
+  end
+  context 'with a standard virtual host' do
+let(:facts) {
+  {
+:hostname=> 'ahost',
+:operatingsystem => 'Debian',
+:ipaddress   => '1.2.3.4',
+:is_virtual  => true,
+:lldp_parent => 'ahosts_parent',
+  }
+}
+let(:title) { 'ahost' }
+it { should compile }
+describe 'with no parameters' do
+  subject { exported_resources }
+  it do
+should contain_nagios_host('ahost').with(
+  'host_name'  => 'ahost',
+  'parents'=> nil,
+  'icon_image' => 'vendors/debian.png',
+  'address'=> '1.2.3.4',
+)
+  end
+end
+describe 'with a parents parameters' do
+  let(:params) {
+{
+  :parents => 'aparent',
+}
+  }
+  subject { exported_resources }
+  it do
+should contain_nagios_host('ahost').with(
+  'host_name'  => 'ahost',
+  'parents'=> 'aparent',
+  'icon_image' => 'vendors/debian.png',
+  'address'=> '1.2.3.4',
+)
+  end
+end
+  end
+  context 'with an icinga host monitoring itself' do
+let(:facts) {
+  {
+:hostname=> 'ahost',
+:operatingsystem => 'Debian',
+:ipaddress   => '1.2.3.4',
+:is_virtual  => false,
+:lldp_parent => 'ahosts_parent',
+  }
+}
+let(:title) { 'ahost' }
+let(:pre_condition) { 'include icinga'}
+it { should compile }
+describe 'with no parameters' do
+  subject { exported_resources }
+  it do
+should contain_nagios_host('ahost').with(
+  'host_name'  => 'ahost',
+  'parents'=> 'ahosts_parent',
+  'icon_image' => 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] numa_networking: new state "isolate"

[BBlack (Code Review)]

BBlack has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/362438 )

Change subject: [WIP] numa_networking: new state "isolate"
..

[WIP] numa_networking: new state "isolate"

This causes kernel cmdline params to be added (for future reboots)
which isolates the CPUs in the primary interface's NUMA node,
reserving them exclusively for tasks which are explicitly
configured there (e.g. via cset, taskset, and/or numactl).

TODOs:
  1. Sort out the writeback masking stuff
  2. Make it safer: ideally via facts we should recognize the case where 
"isolate" doesn't really make sense because it would isolate every CPU in the 
system.  Right now the only protection is "user should not set this parameter 
on hardware it doesn't make sense on"

Change-Id: I11027be1b9bcb66bf82dba0cf69c9c034a1d114e
---
M hieradata/hosts/cp4021.yaml
M manifests/realm.pp
M modules/interface/manifests/rps/modparams.pp
M modules/interface/templates/interface-rps-config.erb
M modules/profile/manifests/base.pp
M modules/tlsproxy/manifests/instance.pp
6 files changed, 26 insertions(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/38/362438/1

diff --git a/hieradata/hosts/cp4021.yaml b/hieradata/hosts/cp4021.yaml
index 84968de..4c422e3 100644
--- a/hieradata/hosts/cp4021.yaml
+++ b/hieradata/hosts/cp4021.yaml
@@ -1,2 +1,2 @@
 bbr_congestion_control: true
-numa_networking: true
+numa_networking: isolate
diff --git a/manifests/realm.pp b/manifests/realm.pp
index 900ca27..383a960 100644
--- a/manifests/realm.pp
+++ b/manifests/realm.pp
@@ -60,7 +60,15 @@
 }
 
 # Hiera->Global to configure various classes for NUMA-aware networking
-$numa_networking = hiera('numa_networking', false)
+# 3 possible values:
+# off: default, no NUMA awareness
+# on: try confine network stuff to the NUMA node of the adapter
+# isolate: also exclude all other tasks from the NUMA node of the adapter
+#   Note that "isolate" will probably be dysfunctional on nodes which do not
+#   have true multi-node NUMA hardware with the relevant interface(s) attached
+#   to less than all nodes!  Therefore setting 'isolate' should only be done in
+#   cases with known-compatible hardware.
+$numa_networking = hiera('numa_networking', 'off')
 
 # TODO: create hash of all LVS service IPs
 
diff --git a/modules/interface/manifests/rps/modparams.pp 
b/modules/interface/manifests/rps/modparams.pp
index a5fc7d0..b020730 100644
--- a/modules/interface/manifests/rps/modparams.pp
+++ b/modules/interface/manifests/rps/modparams.pp
@@ -1,7 +1,7 @@
 class interface::rps::modparams {
 include initramfs
 
-if $::numa_networking {
+if $::numa_networking != 'off' {
 # note this assumes if bnx2x queue counts matter at all, that the
 # primary interface is bnx2x.  This is true for current cases, but may
 # need to evolve later for hosts with multiple interfaces with distinct
diff --git a/modules/interface/templates/interface-rps-config.erb 
b/modules/interface/templates/interface-rps-config.erb
index a1c9e80..3c6892e 100644
--- a/modules/interface/templates/interface-rps-config.erb
+++ b/modules/interface/templates/interface-rps-config.erb
@@ -1,4 +1,4 @@
 [Options]
 <% if @rss_pattern != '' %>rss_pattern = <%= @rss_pattern %><% end %>
 <% if @qdisc != '' %>qdisc = <%= @qdisc %><% end %>
-<%- if @numa_networking %>numa_filter = yes<% end -%>
+<%- if @numa_networking != 'off' %>numa_filter = yes<% end -%>
diff --git a/modules/profile/manifests/base.pp 
b/modules/profile/manifests/base.pp
index 280f629..4e4591d 100644
--- a/modules/profile/manifests/base.pp
+++ b/modules/profile/manifests/base.pp
@@ -106,4 +106,17 @@
 source => 'puppet:///modules/base/logrotate/upstart',
 }
 }
+
+if $::numa_networking == 'isolate' {
+grub::bootparam { 'isolcpus':
+value => 
join(sort(flatten($facts['numa']['device_to_htset'][$facts['interface_primary']])),
 ',')
+}
+# XXX TODO: move disk writeback off the isolated node, needs inverted 
(or opposite-node) cpumask...
+# sysfs::parameters { 'cache_numa_isolate':
+# values => {
+# 'bus/workqueue/devices/writeback/numa'=> 0,
+# 'bus/workqueue/devices/writeback/cpumask' => XXX,
+# }
+# }
+}
 }
diff --git a/modules/tlsproxy/manifests/instance.pp 
b/modules/tlsproxy/manifests/instance.pp
index 959255d..57a7e11 100644
--- a/modules/tlsproxy/manifests/instance.pp
+++ b/modules/tlsproxy/manifests/instance.pp
@@ -27,7 +27,7 @@
 # otherwise use 'lo' for this purpose.  Assumes NUMA data has "lo" 
interface
 # mapped to all cpu cores in the non-NUMA case.  The numa_iface variable is
 # in turn consumed by the systemd unit and config templates.
-if $::numa_networking {
+if $::numa_networking != 'off' {
 $numa_iface = $facts['interface_primary']
 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] NUMA binding for cache frontends under 'isolate'

[BBlack (Code Review)]

BBlack has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/362439 )

Change subject: [WIP] NUMA binding for cache frontends under 'isolate'
..

[WIP] NUMA binding for cache frontends under 'isolate'

Binding the cache frontends to the network node won't make sense
on many of our legacy hardware configs due to limitations.  It
will make sense only on our latest config and possibly the other
most-recent one as well (the 256GB symmetric nodes).  On the older
machines that shouldn't bind varnish-frontend, it's also too
wasteful to isolate the node just for nginx and nothing else.
Therefore, tie the binding of varnish to the 'isolate' setting.

Change-Id: Ie67991ab785714e4c0e27f14b5081b8148e3fba6
---
M modules/role/manifests/cache/instances.pp
M modules/varnish/manifests/instance.pp
M modules/varnish/templates/initscripts/varnish.systemd.erb
3 files changed, 11 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/39/362439/1

diff --git a/modules/role/manifests/cache/instances.pp 
b/modules/role/manifests/cache/instances.pp
index 99daba4..8683f97 100644
--- a/modules/role/manifests/cache/instances.pp
+++ b/modules/role/manifests/cache/instances.pp
@@ -103,6 +103,13 @@
 # Set a reduced keep value for frontends
 $fe_keep_vcl_config = merge($fe_vcl_config, { 'keep' => '1d', })
 
+if $::numa_networking == 'isolate' {
+$nnodes = 
$facts['numa']['device_to_node'][$facts['interface_primary']].join(',')
+$fe_cmd_prefix = "/usr/bin/numactl -a -N ${nnodes} -m ${nnodes}"
+} else {
+$fe_cmd_prefix = ''
+}
+
 # lint:ignore:arrow_alignment
 varnish::instance { "${title}-frontend":
 instance_name  => 'frontend',
@@ -123,6 +130,7 @@
 },
 },
 vcl_config => $fe_keep_vcl_config,
+start_cmd_prefix   => $fe_cmd_prefix,
 }
 # lint:endignore
 }
diff --git a/modules/varnish/manifests/instance.pp 
b/modules/varnish/manifests/instance.pp
index a6324c2..f1dff96 100644
--- a/modules/varnish/manifests/instance.pp
+++ b/modules/varnish/manifests/instance.pp
@@ -11,7 +11,8 @@
 $app_directors={},
 $app_def_be_opts={},
 $backend_caches={},
-$extra_vcl = []
+$extra_vcl = [],
+$start_cmd_prefix = '',
 ) {
 
 include ::varnish::common
diff --git a/modules/varnish/templates/initscripts/varnish.systemd.erb 
b/modules/varnish/templates/initscripts/varnish.systemd.erb
index 5b3d4c3..6a67cb3 100644
--- a/modules/varnish/templates/initscripts/varnish.systemd.erb
+++ b/modules/varnish/templates/initscripts/varnish.systemd.erb
@@ -16,7 +16,7 @@
 Environment="MALLOC_CONF=<%= @jemalloc_conf %>"
 <% end -%>
 ExecReload=/usr/share/varnish/reload-vcl <%= @extraopts %> -q
-ExecStart=/usr/sbin/varnishd \
+ExecStart=<%= @start_cmd_prefix %>/usr/sbin/varnishd \
 -P %t/%p.pid \
 <%= @ports.map { |p| "-a :"+p }.join(" ") -%> \
 -T 127.0.0.1:<%= @admin_port -%> \

-- 
To view, visit https://gerrit.wikimedia.org/r/362439
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie67991ab785714e4c0e27f14b5081b8148e3fba6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP don't merge yet, still working on the actual code to de...

[Milimetric (Code Review)]

Milimetric has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/362118 )

Change subject: [WIP don't merge yet, still working on the actual code to 
deploy] Clone wikistats v2 repository and link it to v2
..

[WIP don't merge yet, still working on the actual code to deploy]
Clone wikistats v2 repository and link it to v2

This sets up wikistats v2 to serve at stats.wikimedia.org/v2

NOTE: the depth argument I used is supported here, not sure if I used it
properly as I didn't find any examples of it in the rest of the repo:
https://github.com/wikimedia/puppet/blob/production/modules/git/manifests/clone.pp#L127

Bug: T167684
Change-Id: I6cff8fcf70a91a922f9c37ba3b5f48dee2bd7835
---
M modules/statistics/manifests/sites/stats.pp
1 file changed, 25 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/18/362118/1

diff --git a/modules/statistics/manifests/sites/stats.pp 
b/modules/statistics/manifests/sites/stats.pp
index 9c54e55..4274b9b 100644
--- a/modules/statistics/manifests/sites/stats.pp
+++ b/modules/statistics/manifests/sites/stats.pp
@@ -6,7 +6,10 @@
 require ::geowiki::private_data
 include ::geowiki::params
 
-$geowiki_private_directory = 
'/srv/stats.wikimedia.org/htdocs/geowiki-private'
+$wikistats_web_directory   = '/srv/stats.wikimedia.org'
+$wikistats_v2_link = "${wikistats_web_directory}/v2"
+$wikistats_source_directory= '/srv/src/wikistats-v2'
+$geowiki_private_directory = 
"${wikistats_web_directory}/htdocs/geowiki-private"
 $geowiki_private_htpasswd_file = '/etc/apache2/htpasswd.stats-geowiki'
 
 # add htpasswd file for stats.wikimedia.org
@@ -37,4 +40,25 @@
 apache::site { 'stats.wikimedia.org':
 content => template('statistics/stats.wikimedia.org.erb'),
 }
+
+# stats.wikimedia.org/v2 (Wikistats 2.0) setup
+# wikistats v2 is cloned and has a built version available in /dist
+git::clone { 'wikistats-v2':
+ensure=> 'latest',
+depth => '1',
+directory => $wikistats_source_directory,
+origin=> 'https://phabricator.wikimedia.org/source/wikistats.git',
+owner => 'root',
+group => 'statistics-web-users',
+mode  => '0775',
+}
+
+# it is then linked to serve at http://stats.wikimedia.org/v2
+file { $wikistats_v2_link:
+ensure => 'link',
+target => "${wikistats_source_directory}/dist",
+owner  => 'root',
+group  => 'www-data',
+mode   => '0750',
+}
 }

-- 
To view, visit https://gerrit.wikimedia.org/r/362118
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6cff8fcf70a91a922f9c37ba3b5f48dee2bd7835
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Milimetric 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Fix timestamp infinite loop in EL purging script (1)

[Mforns (Code Review)]

Mforns has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/362103 )

Change subject: [WIP] Fix timestamp infinite loop in EL purging script (1)
..

[WIP] Fix timestamp infinite loop in EL purging script (1)

Change-Id: Ib4ab6edb0c7be445e24724b756564f8c1eb2fd41
---
M modules/role/files/mariadb/eventlogging_cleaner.py
1 file changed, 17 insertions(+), 10 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/03/362103/1

diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py 
b/modules/role/files/mariadb/eventlogging_cleaner.py
index 2de0160..dcdbe62 100644
--- a/modules/role/files/mariadb/eventlogging_cleaner.py
+++ b/modules/role/files/mariadb/eventlogging_cleaner.py
@@ -215,7 +215,7 @@
 result = self.database.execute(command, params, 
dry_run=self.dry_run)
 time.sleep(self.sleep_between_batches)
 
-def _get_uuids_and_last_ts(self, table, start_ts):
+def _get_uuid_info(self, table, start_ts, already_processed):
 """
 Return the first  uuids of the events between start_ts
 and self.end. Also return the timestamp of the last of those events.
@@ -227,7 +227,8 @@
 """
 command = (
 "SELECT timestamp, uuid from {} WHERE timestamp >= %(start_ts)s "
-"AND timestamp < %(end_ts)s ORDER BY timestamp LIMIT 
%(batch_size)s"
+"AND timestamp < %(end_ts)s ORDER BY timestamp, uuid "
+"LIMIT %(batch_size)s OFFSET %(already_processed)s"
 .format(table)
 )
 params = {
@@ -235,13 +236,17 @@
 'end_ts': self.end,
 'batch_size': self.batch_size,
 }
-result = self.database.execute(command, params, self.dry_run)
-if result['rows']:
-uuids = [x[1] for x in result['rows']]
-last_ts = result['rows'][-1][0]
-return (uuids, last_ts)
+results = self.database.execute(command, params, self.dry_run)['rows']
+if results:
+uuids = [x[1] for x in results]
+last_ts = results[-1][0]
+if start_ts == last_ts:
+uuids_with_last_ts = already_processed + len(results)
+else:
+uuids_with_last_ts = sum(x[0] == last_ts for x in results)
+return (uuids, last_ts, uuids_with_last_ts)
 else:
-return ([], None)
+return ([], None, 0)
 
 def sanitize(self, table):
 """
@@ -265,7 +270,8 @@
 fields_to_purge = filter(lambda f: f not in fields_to_keep, fields)
 
 values_string = ','.join([field + ' = NULL' for field in 
fields_to_purge])
-uuids_current_batch, last_ts = self._get_uuids_and_last_ts(table, 
self.start)
+uuid_info = self._get_uuid_info(table, self.start, 0)
+uuids_current_batch, last_ts, uuids_with_last_ts = uuid_info
 command_template = (
 "UPDATE {0} "
 "SET {1} "
@@ -300,7 +306,8 @@
 # to sanitize.
 uuids_current_batch = []
 else:
-uuids_current_batch, last_ts = 
self._get_uuids_and_last_ts(table, last_ts)
+uuid_info = self._get_uuid_info(table, last_ts, 
uuids_with_last_ts)
+uuids_current_batch, last_ts, uuids_with_last_ts = uuid_info
 time.sleep(self.sleep_between_batches)
 
 

-- 
To view, visit https://gerrit.wikimedia.org/r/362103
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib4ab6edb0c7be445e24724b756564f8c1eb2fd41
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mforns 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Fix timestamp infinite loop in EL purging script

[Mforns (Code Review)]

Mforns has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/362101 )

Change subject: [WIP] Fix timestamp infinite loop in EL purging script
..

[WIP] Fix timestamp infinite loop in EL purging script

Change-Id: I63a6d8158b1f957f38a21ff86e1878af03c270f6
---
M modules/role/files/mariadb/eventlogging_cleaner.py
1 file changed, 22 insertions(+), 33 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/01/362101/1

diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py 
b/modules/role/files/mariadb/eventlogging_cleaner.py
index 2de0160..0b38f64 100644
--- a/modules/role/files/mariadb/eventlogging_cleaner.py
+++ b/modules/role/files/mariadb/eventlogging_cleaner.py
@@ -215,33 +215,31 @@
 result = self.database.execute(command, params, 
dry_run=self.dry_run)
 time.sleep(self.sleep_between_batches)
 
-def _get_uuids_and_last_ts(self, table, start_ts):
-"""
-Return the first  uuids of the events between start_ts
-and self.end. Also return the timestamp of the last of those events.
-NOTE: If there exist several events that share the last timestamp,
-it might be that some of them are listed in the uuid batch, and some
-others aren't (do not fit in the batch size limit). In the next 
iteration
-start_ts will be this iteration's last_ts, and so the script might
-re-purge some events, which is OK, because the outcome does not change.
-"""
+def _get_sanitize_batches(self):
 command = (
-"SELECT timestamp, uuid from {} WHERE timestamp >= %(start_ts)s "
-"AND timestamp < %(end_ts)s ORDER BY timestamp LIMIT 
%(batch_size)s"
-.format(table)
+"SELECT timestamp, uuid FROM {} "
+"WHERE timestamp >= %(batch_start_timestamp)s AND timestamp < '{}' 
"
+"ORDER BY timestamp, uuid LIMIT {} OFFSET 
%(already_processed_events)s"
+.format(table, self.end, self.batch_size)
 )
 params = {
-'start_ts': start_ts,
-'end_ts': self.end,
-'batch_size': self.batch_size,
+'batch_start_timestamp': self.start,
+'already_processed_events': 0
 }
-result = self.database.execute(command, params, self.dry_run)
-if result['rows']:
-uuids = [x[1] for x in result['rows']]
-last_ts = result['rows'][-1][0]
-return (uuids, last_ts)
-else:
-return ([], None)
+results = self.database.execute(command, params, self.dry_run)['rows']
+yield [x[1] for x in results]
+while len(results) == self.batch_size:
+last_event_timestamp = result[-1][0]
+params = {
+'batch_start_timestamp': last_event_timestamp,
+'already_processed_events': (
+params['already_processed_events'] + len(results)
+if last_event_timestamp == params['batch_start_timestamp'] 
else
+sum(x[0] == last_event_timestamp for x in results)
+)
+}
+results = self.database.execute(command, params, 
self.dry_run)['rows']
+yield [x[1] for x in results]
 
 def sanitize(self, table):
 """
@@ -265,14 +263,13 @@
 fields_to_purge = filter(lambda f: f not in fields_to_keep, fields)
 
 values_string = ','.join([field + ' = NULL' for field in 
fields_to_purge])
-uuids_current_batch, last_ts = self._get_uuids_and_last_ts(table, 
self.start)
 command_template = (
 "UPDATE {0} "
 "SET {1} "
 "WHERE uuid IN ({{}})"
 ).format(table, values_string)
 
-while uuids_current_batch:
+for uuids_current_batch in self._get_sanitize_batches():
 uuids_no = len(uuids_current_batch)
 if uuids_no > self.batch_size:
 log.error("The number of uuids to sanitize {} is bigger "
@@ -293,14 +290,6 @@
   .format(uuids_no, result['numrows']))
 raise RuntimeError('Sanitization stopped as precautionary 
step.')
 
-if uuids_no < self.batch_size:
-# Avoid an extra SQL query to the database if the number of
-# uuids returned are less than BATCH_SIZE, since this value
-# means that we have already reached the last batch of uuids
-# to sanitize.
-uuids_current_batch = []
-else:
-uuids_current_batch, last_ts = 
self._get_uuids_and_last_ts(table, last_ts)
 time.sleep(self.sleep_between_batches)
 
 

-- 
To view, visit https://gerrit.wikimedia.org/r/362101
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: 

[MediaWiki-commits] [Gerrit] operations/puppet[production]: [WIP] Modify EL purging script to not use limit/offset

[Mforns (Code Review)]

Mforns has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/359442 )

Change subject: [WIP] Modify EL purging script to not use limit/offset
..

[WIP] Modify EL purging script to not use limit/offset

Bug: T168071
Change-Id: Ic4c1e5a30ce41d8bc5ea3429f716d145118e3e65
---
M modules/role/files/mariadb/eventlogging_cleaner.py
1 file changed, 58 insertions(+), 87 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/42/359442/1

diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py 
b/modules/role/files/mariadb/eventlogging_cleaner.py
index 4e536f0..f4fc85e 100644
--- a/modules/role/files/mariadb/eventlogging_cleaner.py
+++ b/modules/role/files/mariadb/eventlogging_cleaner.py
@@ -167,11 +167,12 @@
 self.start = self.relative_ts(newer_than)
 self.end = self.relative_ts(older_than)
 self.batch_size = batch_size
+self.max_interval_size = timedelta(days=30)
 self.sleep_between_batches = sleep_between_batches
 self.dry_run = dry_run
 
 def relative_ts(self, days):
-return (self.reference_time - 
timedelta(days=days)).strftime(DATE_FORMAT)
+return self.reference_time - timedelta(days=days)
 
 def purge(self, table):
 """
@@ -184,8 +185,8 @@
 "LIMIT %(batch_size)s".format(table)
 )
 params = {
-'start_ts': self.start,
-'end_ts': self.end,
+'start_ts': self.start.strftime(DATE_FORMAT),
+'end_ts': self.end.strftime(DATE_FORMAT),
 'batch_size': self.batch_size,
 }
 result = self.database.execute(command, params, dry_run=self.dry_run)
@@ -196,105 +197,75 @@
 result = self.database.execute(command, params, 
dry_run=self.dry_run)
 time.sleep(self.sleep_between_batches)
 
-def _get_old_uuids(self, table, offset):
+def get_interval_factor(self, numrows):
+"""The adapt factor tries to maintain the size of the update interval
+close to the defined batch size. If the given table has lots of events
+the update interval should be smaller, and if the table has less events
+then the interval can be greater, so that we need less update queries.
+The initial update interval is 1 minute, and then depending on the
+number of updated rows, the adapt factor will multiply the interval 
size
+to make it closer to the batch size.
 """
-Return a list of uuids between self.start and self.end limiting
-the batch with an offset.
-"""
-command = (
-"SELECT uuid from {} WHERE timestamp >= %(start_ts)s "
-"AND timestamp < %(end_ts)s LIMIT %(batch_size)s OFFSET %(offset)s"
-.format(table)
+return min( # the min ensures that the factor is not too big
+# batch_size / numrows should be 1 if the last update had
+# a number of rows equal to batch_size. If the update was smaller,
+# the factor should be proportionally greater, and viceversa.
+self.batch_size / float(max(numrows, 1)), # max prevents division 
by 0
+10
 )
-params = {
-'start_ts': self.start,
-'end_ts': self.end,
-'batch_size': self.batch_size,
-'offset': offset,
-}
-result = self.database.execute(command, params, self.dry_run)
-if result['rows']:
-return [x[0] for x in result['rows']]
-else:
-return []
+
+def interval_size_is_ok(self, interval_size, numrows):
+return (
+numrows < self.batch_size * 1.2 and (
+numrows > self.batch_size * 0.8 or
+interval_size == self.max_interval_size
+)
+)
+
+def get_new_interval_size(self, table, interval_start, interval_size):
+command_template = (
+"SELECT count(*) FROM {}"
+"WHERE timestamp >= {} AND timestamp < {{}}"
+.format(table, interval_start.strftime(DATE_FORMAT))
+)
+interval_end = interval_start + interval_size
+command = command_template.format(interval_end.strftime(DATE_FORMAT))
+numrows = self.database.execute(command)['rows'][0][0]
+while not self.interval_size_is_ok(interval_size, numrows):
+interval_size = min(
+interval_size * self.get_interval_factor(numrows),
+self.max_interval_size
+)
+interval_end = interval_start + interval_size
+command = 
command_template.format(interval_end.strftime(DATE_FORMAT))
+numrows = self.database.execute(command)['rows'][0][0]
+return interval_size
 
 def sanitize(self, table):
-"""
-Set all the fields not in the whitelist (for a given table) to NULL.
-The