Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/385173 )
Change subject: Introduce mariadb eventlogging profiles for master/replica
......................................................................
Introduce mariadb eventlogging profiles for master/replica
Bug: T177405
Change-Id: Ibf31d8f5976e96d7334f9db81bc3ad2f152000cc
---
D hieradata/role/common/mariadb/misc/eventlogging.yaml
A hieradata/role/common/mariadb/misc/eventlogging/master.yaml
A hieradata/role/common/mariadb/misc/eventlogging/replica.yaml
M manifests/site.pp
R modules/profile/files/mariadb/misc/eventlogging/eventlogging_cleaner.py
R
modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
R modules/profile/files/mariadb/misc/eventlogging/eventlogging_sync.sh
A modules/profile/manifests/mariadb/misc/eventlogging/database.pp
A modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
R modules/profile/templates/mariadb/misc/eventlogging/eventlogging.my.cnf.erb
R
modules/profile/templates/mariadb/misc/eventlogging/eventlogging_sync.init.erb.sh
M modules/role/manifests/mariadb.pp
A modules/role/manifests/mariadb/misc/eventlogging/master.pp
A modules/role/manifests/mariadb/misc/eventlogging/replica.pp
14 files changed, 198 insertions(+), 106 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/73/385173/1
diff --git a/hieradata/role/common/mariadb/misc/eventlogging.yaml
b/hieradata/role/common/mariadb/misc/eventlogging.yaml
deleted file mode 100644
index 4bfa7b0..0000000
--- a/hieradata/role/common/mariadb/misc/eventlogging.yaml
+++ /dev/null
@@ -1 +0,0 @@
-monitor_screens: false
diff --git a/hieradata/role/common/mariadb/misc/eventlogging/master.yaml
b/hieradata/role/common/mariadb/misc/eventlogging/master.yaml
new file mode 100644
index 0000000..3ee5995
--- /dev/null
+++ b/hieradata/role/common/mariadb/misc/eventlogging/master.yaml
@@ -0,0 +1,3 @@
+monitor_screens: false
+profile::mariadb::eventlogging::database::shard: 'm4'
+profile::mariadb::eventlogging::database::master: true
\ No newline at end of file
diff --git a/hieradata/role/common/mariadb/misc/eventlogging/replica.yaml
b/hieradata/role/common/mariadb/misc/eventlogging/replica.yaml
new file mode 100644
index 0000000..8005eda
--- /dev/null
+++ b/hieradata/role/common/mariadb/misc/eventlogging/replica.yaml
@@ -0,0 +1,3 @@
+monitor_screens: false
+profile::mariadb::eventlogging::database::shard: 'm4'
+profile::mariadb::eventlogging::database::master: false
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index eb5d35a..dfed584 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -611,19 +611,12 @@
# m4 shard
node 'db1046.eqiad.wmnet' {
- class { '::role::mariadb::misc::eventlogging':
- shard => 'm4',
- master => true,
- }
+ role(mariadb::analytics, mariadb::misc::eventlogging::master)
}
node 'db1047.eqiad.wmnet' {
# this slave has an m4 custom replication protocol
# this slave additionally replicates s1 and s2
- role(mariadb::analytics, mariadb::analytics::custom_repl_slave)
- class { '::role::mariadb::misc::eventlogging':
- shard => 'm4',
- master => false,
- }
+ role(mariadb::analytics, mariadb::misc::eventlogging::replica)
}
# m5 shard
@@ -672,7 +665,7 @@
node 'dbstore1002.eqiad.wmnet' {
# this slave has an m4 custom replication protocol
- role(mariadb::dbstore, mariadb::analytics::custom_repl_slave)
+ role(mariadb::dbstore, mariadb::misc::eventlogging::replica)
}
node 'dbstore2001.codfw.wmnet' {
diff --git a/modules/role/files/mariadb/eventlogging_cleaner.py
b/modules/profile/files/mariadb/misc/eventlogging/eventlogging_cleaner.py
similarity index 100%
rename from modules/role/files/mariadb/eventlogging_cleaner.py
rename to
modules/profile/files/mariadb/misc/eventlogging/eventlogging_cleaner.py
diff --git a/modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
b/modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
similarity index 100%
rename from modules/role/files/mariadb/eventlogging_purging_whitelist.tsv
rename to
modules/profile/files/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv
diff --git a/modules/role/files/mariadb/eventlogging_sync.sh
b/modules/profile/files/mariadb/misc/eventlogging/eventlogging_sync.sh
old mode 100755
new mode 100644
similarity index 100%
rename from modules/role/files/mariadb/eventlogging_sync.sh
rename to modules/profile/files/mariadb/misc/eventlogging/eventlogging_sync.sh
diff --git a/modules/profile/manifests/mariadb/misc/eventlogging/database.pp
b/modules/profile/manifests/mariadb/misc/eventlogging/database.pp
new file mode 100644
index 0000000..95f668b
--- /dev/null
+++ b/modules/profile/manifests/mariadb/misc/eventlogging/database.pp
@@ -0,0 +1,71 @@
+# == Class profile::mariadb::eventlogging::database
+#
+# Configures the database settings for the Eventlogging master/replica.
+#
+# [*shard*]
+# Database shard
+# [*master*]
+# Boolean value to establish if the host is acting as Master or Replica.
+#
+class profile::mariadb::misc::eventlogging::database (
+ $shard = hiera('profile::mariadb::eventlogging::database::shard'),
+ $master = hiera('profile::mariadb::eventlogging::database::master'),
+) {
+ $mysql_role = $master ? {
+ true => 'master',
+ false => 'slave',
+ }
+
+ include ::standard
+ include passwords::misc::scripts
+ include ::base::firewall
+
+ # FIXME: including a role in a profile is not
+ # allowed by our coding standard, but these needs to be
+ # refactored on a separate change since they are broadly used.
+ include role::mariadb::ferm
+ include role::mariadb::monitor::dba
+
+ class {'role::mariadb::groups':
+ mysql_group => 'misc',
+ mysql_shard => $shard,
+ mysql_role => $mysql_role,
+ socket => '/tmp/mysql.sock',
+ }
+
+ include mariadb::packages_wmf
+ include mariadb::service
+
+
+ # History context: there used to be a distinction between
+ # EL master and slaves, namely that only the master was not
+ # in read only mode. The Analytics team removed this constraint
+ # before deploying the eventlogging_cleaner script (T156933),
+ # that needed to DELETE/UPDATE rows on the job database without
+ # running as root for obvious reasons.
+ class { 'mariadb::config':
+ config =>
'profile/mariadb/misc/eventlogging/eventlogging.my.cnf.erb',
+ datadir => '/srv/sqldata',
+ tmpdir => '/srv/tmp',
+ read_only => 0,
+ ssl => 'puppet-cert',
+ p_s => 'off',
+ binlog_format => 'MIXED',
+ }
+
+ # FIXME: instanciating a role in a profile is not
+ # allowed by our coding standard, but it needs to be
+ # refactored on a separate change since it is broadly used.
+ class { 'role::mariadb::grants::production':
+ shard => $shard,
+ prompt => "EVENTLOGGING ${shard}",
+ password => $passwords::misc::scripts::mysql_root_pass,
+ }
+
+ class { 'mariadb::heartbeat':
+ shard => $shard,
+ datacenter => $::site,
+ enabled => $master,
+ socket => '/tmp/mysql.sock',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
b/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
new file mode 100644
index 0000000..d82f1ce
--- /dev/null
+++ b/modules/profile/manifests/mariadb/misc/eventlogging/replication.pp
@@ -0,0 +1,111 @@
+# == Class profile::mariadb::eventlogging::replication
+#
+# This profile is responsible to add the custom replication service for
+# the Eventlogging database (assumed to be 'log' for historical reasons).
+# It consists in a bash script executed regularly that runs a mysqldump on the
+# master host, copies it on localhost and then uns it against the local
database.
+# This profile will also install the eventlogging_cleaner Python script,
+# responsible to apply the Analytics' sanitization/purging rules.
+#
+# [*master_host*]
+# The host that runs the log database to be replicated.
+#
+# [*cutoff_days*]
+# Don't try to 'replicate' tables with no events more recent than this many
days ago.
+#
+# [*batch_size*]
+# Replicate this many rows at a time.
+#
+class profile::mariadb::misc::eventlogging::replication (
+ $master_host =
hiera('profile::mariadb::eventlogging::replication::master_host'),
+ $cutoff_days =
hiera('profile::mariadb::eventlogging::replication::cutoff_days'),
+ $batch_size =
hiera('profile::mariadb::eventlogging::replication::batch_size'),
+) {
+ $slave_host = 'localhost'
+ $database = 'log'
+
+ # TODO: hiera
+ #$master_host = 'm4-master.eqiad.wmnet'
+ #$cutoff_days = 90
+ #$batch_size = 1000
+
+ group { 'eventlog':
+ ensure => 'present',
+ system => true,
+ }
+
+ user { 'eventlogcleaner':
+ gid => 'eventlog',
+ shell => '/bin/false',
+ home => '/nonexistent',
+ comment => 'EventLogging cleaner user',
+ system => true,
+ managehome => false,
+ require => Group['eventlog'],
+ }
+
+ require_package('python3-pymysql')
+
+ file { '/usr/local/bin/eventlogging_cleaner':
+ ensure => present,
+ owner => 'eventlogcleaner',
+ group => 'eventlog',
+ mode => '0550',
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_cleaner.py',
+ require => Package['python3-pymysql'],
+ }
+
+ file { '/etc/eventlogging':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ }
+
+ file { '/etc/eventlogging/whitelist.tsv':
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_purging_whitelist.tsv',
+ require => File['/etc/eventlogging'],
+ }
+
+ file { '/usr/local/bin/eventlogging_sync.sh':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0700',
+ source =>
'puppet:///modules/profile/mariadb/misc/eventlogging/eventlogging_sync.sh',
+ }
+ file { '/etc/init.d/eventlogging_sync':
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ content =>
template('profile/mariadb/misc/eventlogging/eventlogging_sync.init.erb'),
+ require => File['/usr/local/bin/eventlogging_sync.sh'],
+ notify => Service['eventlogging_sync'],
+ }
+
+ logrotate::rule { 'eventlogging_sync':
+ ensure => present,
+ file_glob => '/var/log/eventlogging_sync.*',
+ frequency => 'daily',
+ copy_truncate => true,
+ compress => true,
+ missing_ok => true,
+ not_if_empty => true,
+ rotate => 14,
+ }
+
+ service { 'eventlogging_sync':
+ ensure => running,
+ enable => true,
+ }
+ nrpe::monitor_service { 'eventlogging_sync':
+ description => 'eventlogging_sync processes',
+ nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:2 -u root
-a "/bin/bash /usr/local/bin/eventlogging_sync.sh"',
+ critical => false,
+ contact_group => 'admins', # show on icinga/irc only
+ }
+}
\ No newline at end of file
diff --git
a/modules/role/templates/mariadb/mysqld_config/eventlogging.my.cnf.erb
b/modules/profile/templates/mariadb/misc/eventlogging/eventlogging.my.cnf.erb
similarity index 100%
rename from modules/role/templates/mariadb/mysqld_config/eventlogging.my.cnf.erb
rename to
modules/profile/templates/mariadb/misc/eventlogging/eventlogging.my.cnf.erb
diff --git a/modules/role/templates/mariadb/eventlogging_sync.init.erb
b/modules/profile/templates/mariadb/misc/eventlogging/eventlogging_sync.init.erb.sh
old mode 100755
new mode 100644
similarity index 100%
rename from modules/role/templates/mariadb/eventlogging_sync.init.erb
rename to
modules/profile/templates/mariadb/misc/eventlogging/eventlogging_sync.init.erb.sh
diff --git a/modules/role/manifests/mariadb.pp
b/modules/role/manifests/mariadb.pp
index 801e278..7a503ab 100644
--- a/modules/role/manifests/mariadb.pp
+++ b/modules/role/manifests/mariadb.pp
@@ -168,99 +168,4 @@
}
}
-class role::mariadb::analytics::custom_repl_slave {
- # Sync eventlogging tables from m4-master.eqiad.wmnet to localhost
- # using a custom bash 'replication' script, that looks for new records on
'master',
- # and inserts into 'slave'.
- $master_host = 'm4-master.eqiad.wmnet'
- $slave_host = 'localhost'
- $database = 'log'
-
- # Don't try to 'replicate' tables with no events more recent than this
many days ago.
- $cutoff_days = 90
-
- # Only 'replicate' this many rows at a time.
- $batch_size = 1000
-
- group { 'eventlog':
- ensure => 'present',
- system => true,
- }
-
- user { 'eventlogcleaner':
- gid => 'eventlog',
- shell => '/bin/false',
- home => '/nonexistent',
- comment => 'EventLogging cleaner user',
- system => true,
- managehome => false,
- require => Group['eventlog'],
- }
-
- require_package('python3-pymysql')
-
- file { '/usr/local/bin/eventlogging_cleaner':
- ensure => present,
- owner => 'eventlogcleaner',
- group => 'eventlog',
- mode => '0550',
- source => 'puppet:///modules/role/mariadb/eventlogging_cleaner.py',
- require => Package['python3-pymysql'],
- }
-
- file { '/etc/eventlogging':
- ensure => 'directory',
- owner => 'root',
- group => 'root',
- mode => '0755',
- }
-
- file { '/etc/eventlogging/whitelist.tsv':
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0444',
- source =>
'puppet:///modules/role/mariadb/eventlogging_purging_whitelist.tsv',
- require => File['/etc/eventlogging'],
- }
-
- file { '/usr/local/bin/eventlogging_sync.sh':
- ensure => present,
- owner => 'root',
- group => 'root',
- mode => '0700',
- source => 'puppet:///modules/role/mariadb/eventlogging_sync.sh',
- }
- file { '/etc/init.d/eventlogging_sync':
- owner => 'root',
- group => 'root',
- mode => '0755',
- content => template('role/mariadb/eventlogging_sync.init.erb'),
- require => File['/usr/local/bin/eventlogging_sync.sh'],
- notify => Service['eventlogging_sync'],
- }
-
- logrotate::rule { 'eventlogging_sync':
- ensure => present,
- file_glob => '/var/log/eventlogging_sync.*',
- frequency => 'daily',
- copy_truncate => true,
- compress => true,
- missing_ok => true,
- not_if_empty => true,
- rotate => 14,
- }
-
- service { 'eventlogging_sync':
- ensure => running,
- enable => true,
- }
- nrpe::monitor_service { 'eventlogging_sync':
- description => 'eventlogging_sync processes',
- nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:2 -u root
-a "/bin/bash /usr/local/bin/eventlogging_sync.sh"',
- critical => false,
- contact_group => 'admins', # show on icinga/irc only
- }
-}
-
# lint:endignore
diff --git a/modules/role/manifests/mariadb/misc/eventlogging/master.pp
b/modules/role/manifests/mariadb/misc/eventlogging/master.pp
new file mode 100644
index 0000000..cadb5fd
--- /dev/null
+++ b/modules/role/manifests/mariadb/misc/eventlogging/master.pp
@@ -0,0 +1,3 @@
+class role::mariadb::misc::eventlogging::master() {
+ include ::profile::mariadb::eventlogging::database
+}
\ No newline at end of file
diff --git a/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
b/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
new file mode 100644
index 0000000..be70d7b
--- /dev/null
+++ b/modules/role/manifests/mariadb/misc/eventlogging/replica.pp
@@ -0,0 +1,4 @@
+class role::mariadb::misc::eventlogging::replica() {
+ include ::profile::mariadb::eventlogging::database
+ include ::profile::mariadb::eventlogging::replication
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/385173
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibf31d8f5976e96d7334f9db81bc3ad2f152000cc
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
