Ori.livneh has submitted this change and it was merged.
Change subject: Add rsyslog module and port existing usage
......................................................................
Add rsyslog module and port existing usage
* Make Puppet manage /etc/rsyslog.d recursively.
* This means that packages that include their own rsyslog.d config will need
to have that config file declared in Puppet or it will get clobbered on the
next run.
* This is the same decision we made with respect to sysctl.d.
* Per apt-file, the set of packages with rsyslog.d files are:
arno-iptables-firewall, cloud-init, gforge-shell-postgresql, maas,
maas-region-controller, mythtv-common, octopussy, postfix, qpidd, sagan,
sympa, ubuntu-orchestra-client, ubuntu-orchestra-logging-server, and ufw
We don't run any of these in production. (We do have ufw installed because
it's part of the 'standard' tasksel task for Ubuntu, but we don't use it.)
* Remove reference to i-000003a9.pmtpa.wmflabs:5544 as a syslog sink for
non-beta-cluster hosts; the instance no longer exists.
Change-Id: Ibceb48bd8c475696b900e89f38dea686394069d8
---
D files/misc/scripts/apache-syslog-rotate
M manifests/ganglia.pp
M manifests/role/analytics/kafkatee.pp
M manifests/role/cache.pp
M modules/base/manifests/init.pp
A modules/mediawiki/files/logrotate.d_mediawiki_apache
M modules/mediawiki/manifests/syslog.pp
A modules/mediawiki/templates/apache/rsyslog.conf.erb
M modules/ocg/manifests/init.pp
A modules/rsyslog/files/rsyslog.d-empty/README
A modules/rsyslog/manifests/conf.pp
A modules/rsyslog/manifests/init.pp
D templates/rsyslog/40-appserver.conf.erb
13 files changed, 161 insertions(+), 101 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/files/misc/scripts/apache-syslog-rotate
b/files/misc/scripts/apache-syslog-rotate
deleted file mode 100644
index b141b67..0000000
--- a/files/misc/scripts/apache-syslog-rotate
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#####################################################################
-### THIS FILE IS MANAGED BY PUPPET
-### puppet:///files/misc/scripts/apache-syslog-rotate
-#####################################################################
-
-mv -f /var/log/apache2.log.4 /var/log/apache2.log.5
-mv -f /var/log/apache2.log.3 /var/log/apache2.log.4
-mv -f /var/log/apache2.log.2 /var/log/apache2.log.3
-mv -f /var/log/apache2.log.1 /var/log/apache2.log.2
-mv -f /var/log/apache2.log /var/log/apache2.log.1
diff --git a/manifests/ganglia.pp b/manifests/ganglia.pp
index a24f25c..1e815c3 100644
--- a/manifests/ganglia.pp
+++ b/manifests/ganglia.pp
@@ -388,11 +388,9 @@
}
# log gmetad messages to /var/log/ganglia.log
- file { '/etc/rsyslog.d/30-ganglia.conf':
- ensure => present,
- source => 'puppet:///files/ganglia/rsyslog.d/30-ganglia.conf',
- mode => '0444',
- notify => Service['rsyslog'],
+ rsyslog::conf { 'ganglia':
+ source => 'puppet:///files/ganglia/rsyslog.d/30-ganglia.conf',
+ priority => 30,
}
file { '/etc/logrotate.d/ganglia':
diff --git a/manifests/role/analytics/kafkatee.pp
b/manifests/role/analytics/kafkatee.pp
index 8b7be0c..06132e8 100644
--- a/manifests/role/analytics/kafkatee.pp
+++ b/manifests/role/analytics/kafkatee.pp
@@ -15,6 +15,12 @@
log_statistics_interval => 15,
}
+ # Declare packaged rsyslog config to ensure it isn't purged.
+ file { '/etc/rsyslog.d/75-kafkatee.conf':
+ ensure => file,
+ require => Class['::kafkatee'],
+ }
+
$log_directory = '/srv/log'
file { $log_directory:
ensure => 'directory',
diff --git a/manifests/role/cache.pp b/manifests/role/cache.pp
index 1f74079..e0c386e 100644
--- a/manifests/role/cache.pp
+++ b/manifests/role/cache.pp
@@ -473,6 +473,13 @@
log_statistics_interval => 15,
}
+ # Persist the rsyslog conf bundled with the varnishkafka .deb
+ # by declaring it to Puppet.
+ file { '/etc/rsyslog.d/75-kafkatee.conf':
+ ensure => file,
+ require => Class['::varnishkafka'],
+ }
+
class { '::varnishkafka::monitoring': }
# Generate icinga alert if varnishkafka is not running.
diff --git a/modules/base/manifests/init.pp b/modules/base/manifests/init.pp
index e968597..b10a013 100644
--- a/modules/base/manifests/init.pp
+++ b/modules/base/manifests/init.pp
@@ -238,53 +238,14 @@
($::hostname != 'aluminium') and
($::instancename != 'deployment-bastion') {
- package { 'rsyslog':
- ensure => present,
+ $syslog_host = $::realm ? {
+ 'production' => 'syslog.eqiad.wmnet',
+ 'labs' => "deployment-bastion.${::site}.wmflabs",
}
- # remote syslog destination
- case $::realm {
- 'production': {
- if( $::site != '(undefined)' ) {
- $syslog_remote_real = "syslog.${::site}.wmnet"
- }
- }
- 'labs': {
- # Per labs project syslog:
- case $::instanceproject {
- 'deployment-prep': {
- $syslog_remote_real =
"deployment-bastion.${::site}.wmflabs"
- }
- default: {
- $syslog_remote_real = 'i-000003a9.pmtpa.wmflabs:5544'
- }
- }
- }
- }
-
- $ensure_remote = $syslog_remote_real ? {
- '' => absent,
- default => present,
- }
-
- file { '/etc/rsyslog.d/90-remote-syslog.conf':
- ensure => absent,
- }
-
- file { '/etc/rsyslog.d/30-remote-syslog.conf':
- ensure => $ensure_remote,
- require => Package['rsyslog'],
- owner => 'root',
- group => 'root',
- mode => '0444',
- content => "*.info;mail.none;authpriv.none;cron.none
@${syslog_remote_real}\n",
- }
-
- service { 'rsyslog':
- ensure => running,
- require => Package['rsyslog'],
- subscribe => File['/etc/rsyslog.d/30-remote-syslog.conf'],
- provider => 'upstart',
+ rsyslog::conf { 'remote_syslog':
+ content => "*.info;mail.none;authpriv.none;cron.none
@${syslog_host}",
+ priority => 30,
}
}
}
diff --git a/modules/mediawiki/files/logrotate.d_mediawiki_apache
b/modules/mediawiki/files/logrotate.d_mediawiki_apache
new file mode 100644
index 0000000..b72fa57
--- /dev/null
+++ b/modules/mediawiki/files/logrotate.d_mediawiki_apache
@@ -0,0 +1,15 @@
+# logrotate config for MediaWiki Apache logs
+# This file is managed by Puppet
+/var/log/apache2.log
+{
+ rotate 4
+ weekly
+ missingok
+ notifempty
+ compress
+ delaycompress
+ sharedscripts
+ postrotate
+ reload rsyslog >/dev/null 2>&1 || true
+ endscript
+}
diff --git a/modules/mediawiki/manifests/syslog.pp
b/modules/mediawiki/manifests/syslog.pp
index a1747f5..85465f7 100644
--- a/modules/mediawiki/manifests/syslog.pp
+++ b/modules/mediawiki/manifests/syslog.pp
@@ -1,20 +1,14 @@
# syslog instance and configuration for applicationservers
class mediawiki::syslog( $apache_log_aggregator ) {
- require base::remote-syslog
-
- file { '/etc/rsyslog.d/40-appserver.conf':
- ensure => present,
- require => Package[rsyslog],
- owner => root,
- group => root,
+ file { '/etc/logrotate.d/mediawiki_apache':
+ source => 'puppet:///modules/mediawiki/logrotate.d_mediawiki_apache',
+ owner => 'root',
+ group => 'root',
mode => '0444',
- content => template('rsyslog/40-appserver.conf.erb'),
}
- file { '/usr/local/bin/apache-syslog-rotate':
- ensure => present,
- owner => root,
- group => root,
- mode => '0555',
- source => 'puppet:///files/misc/scripts/apache-syslog-rotate',
+
+ rsyslog::conf { 'mediawiki_apache':
+ content => template('mediawiki/apache/rsyslog.conf.erb'),
+ priority => 40,
}
}
diff --git a/modules/mediawiki/templates/apache/rsyslog.conf.erb
b/modules/mediawiki/templates/apache/rsyslog.conf.erb
new file mode 100644
index 0000000..9782053
--- /dev/null
+++ b/modules/mediawiki/templates/apache/rsyslog.conf.erb
@@ -0,0 +1,17 @@
+# rsyslog configuration for MediaWiki on Apache
+# This file is managed by Puppet
+
+# Emulate MediaWiki's wfDebugLog / wfErrorLog format
+$template MediaWiki,"%programname% %timegenerated% %HOSTNAME%: %msg%\n"
+
+# Maximum 100MB log file size
+$outchannel apache2,/var/log/apache2.log,100000000,/usr/sbin/logrotate -f
/etc/logrotate.d/rsyslog-apache2
+
+# Send apache2 messages to the apache output channel
+:programname,isequal,"apache2" $apache2
+
+# ..and to the MediaWiki log aggregator
+& @<%= @apache_log_aggregator %>;MediaWiki
+
+# Discard apache2 messages before they get to 50-default.conf
+& ~
diff --git a/modules/ocg/manifests/init.pp b/modules/ocg/manifests/init.pp
index f5a8cf9..715138f 100644
--- a/modules/ocg/manifests/init.pp
+++ b/modules/ocg/manifests/init.pp
@@ -139,12 +139,8 @@
group => 'root',
}
- file { '/etc/rsyslog.d/20-ocg.conf':
- ensure => present,
- source => 'puppet:///modules/ocg/ocg.rsyslog.conf',
- mode => '0444',
- owner => 'root',
- group => 'root',
- notify => Service['rsyslog'],
+ rsyslog::conf { 'ocg':
+ source => 'puppet:///modules/ocg/ocg.rsyslog.conf',
+ priority => 20,
}
}
diff --git a/modules/rsyslog/files/rsyslog.d-empty/README
b/modules/rsyslog/files/rsyslog.d-empty/README
new file mode 100644
index 0000000..93f86f8
--- /dev/null
+++ b/modules/rsyslog/files/rsyslog.d-empty/README
@@ -0,0 +1,2 @@
+This directory is managed by Puppet. Any configuration file in this directory
+that is not explicitly managed by Puppet will be purged.
diff --git a/modules/rsyslog/manifests/conf.pp
b/modules/rsyslog/manifests/conf.pp
new file mode 100644
index 0000000..6f5f7c6
--- /dev/null
+++ b/modules/rsyslog/manifests/conf.pp
@@ -0,0 +1,59 @@
+# == Define: rsyslog::conf
+#
+# Represents an rsyslogd config file. See rsyslog.conf(5).
+#
+# === Parameters
+#
+# [*content*]
+# The content of the file provided as a string. Either this or
+# 'source' must be specified.
+#
+# [*source*]
+# The content of the file provided as a puppet:/// file reference.
+# Either this or 'content' must be specified.
+#
+# [*priority*]
+# A numeric value in range 0 - 99. Files with a lower priority value
+# are evaluated first.
+#
+# If you're not sure, leave this unspecified. The default value of 60
+# should suit most cases.
+#
+# === Examples
+#
+# rsyslog::conf { 'hadoop':
+# content => template('hadoop/hadoop.conf.erb'),
+# priority => 90,
+# }
+#
+define rsyslog::conf(
+ $ensure = present,
+ $content = undef,
+ $source = undef,
+ $priority = 60
+) {
+ include ::rsyslog
+
+ if $priority !~ /^\d?\d$/ {
+ fail("'priority' must be an integer between 0 - 99 (got:
${priority}).")
+ }
+
+ $basename = regsubst($title, '[\W_]', '-', 'G')
+ $filename = sprintf('/etc/rsyslog.d/%02d-%s.conf', $priority, $basename)
+
+ # append a trailing newline if omitted
+ $content_formatted = $content ? {
+ undef => undef,
+ default => regsubst($content, "\n?$", "\n")
+ }
+
+ file { $filename:
+ ensure => $ensure,
+ content => $content_formatted,
+ source => $source,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ notify => Service['rsyslog'],
+ }
+}
diff --git a/modules/rsyslog/manifests/init.pp
b/modules/rsyslog/manifests/init.pp
new file mode 100644
index 0000000..9d430fe
--- /dev/null
+++ b/modules/rsyslog/manifests/init.pp
@@ -0,0 +1,35 @@
+# == Class: rsyslog
+#
+# rsyslogd is a full-featured kernel logging daemon. It is the default
+# syslogd implementation on Debian systems.
+#
+class rsyslog {
+ package { 'rsyslog':
+ ensure => present,
+ }
+
+ file { '/etc/rsyslog.d':
+ ensure => directory,
+ source => 'puppet:///modules/rsyslog/rsyslog.d-empty',
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ recurse => true,
+ purge => true,
+ force => true,
+ require => Package['rsyslog'],
+ notify => Service['rsyslog'],
+ }
+
+ service { 'rsyslog':
+ ensure => running,
+ provider => 'upstart',
+ require => Package['rsyslog'],
+ }
+
+ rsyslog::conf { 'default':
+ source => '/usr/share/rsyslog/50-default.conf',
+ priority => 50,
+ require => Package['rsyslog'],
+ }
+}
diff --git a/templates/rsyslog/40-appserver.conf.erb
b/templates/rsyslog/40-appserver.conf.erb
deleted file mode 100644
index a528f8d..0000000
--- a/templates/rsyslog/40-appserver.conf.erb
+++ /dev/null
@@ -1,19 +0,0 @@
-#####################################################################
-### THIS FILE IS MANAGED BY PUPPET
-### puppet:///rsyslog/40-appserver.conf.erb
-#####################################################################
-
-# Emulate MediaWiki's wfDebugLog / wfErrorLog format
-$template MediaWiki,"%programname% %timegenerated% %HOSTNAME%: %msg%\n"
-
-# Maximum 100MB log file size
-$outchannel
apache2,/var/log/apache2.log,100000000,/usr/local/bin/apache-syslog-rotate
-
-# Send apache2 messages to the apache output channel
-:programname,isequal,"apache2" $apache2
-
-# ..and to the MediaWiki log aggregator
-& @<%= @apache_log_aggregator %>;MediaWiki
-
-# Discard apache2 messages before they get to 50-default.conf
-& ~
--
To view, visit https://gerrit.wikimedia.org/r/135447
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ibceb48bd8c475696b900e89f38dea686394069d8
Gerrit-PatchSet: 12
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: Chasemp <chas...@gmail.com>
Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Ori.livneh <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
