Gehel has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/390039 )
Change subject: [wip] logstash: move to role / profiles
......................................................................
[wip] logstash: move to role / profiles
Change-Id: I463285567317fef595c3f178310c4b053244597a
---
M manifests/site.pp
R modules/profile/files/logstash/apifeatureusage-template.json
R modules/profile/files/logstash/elasticsearch-template.json
R modules/profile/files/logstash/filter-add-normalized-message.conf
R modules/profile/files/logstash/filter-apifeatureusage.conf
R modules/profile/files/logstash/filter-de_dot.conf
R modules/profile/files/logstash/filter-eventlogging.conf
R modules/profile/files/logstash/filter-gelf.conf
R modules/profile/files/logstash/filter-json-lines.conf
R modules/profile/files/logstash/filter-logback.conf
R modules/profile/files/logstash/filter-mediawiki.conf
R modules/profile/files/logstash/filter-normalize-log-levels.conf
R modules/profile/files/logstash/filter-ores.conf
R modules/profile/files/logstash/filter-striker.conf
R modules/profile/files/logstash/filter-strip-ansi-color.conf
R modules/profile/files/logstash/filter-syslog.conf
R modules/profile/files/logstash/filter-udp2log.conf
R modules/profile/files/logstash/filter-webrequest.conf
A modules/profile/manifests/logstash/apifeatureusage.pp
R modules/profile/manifests/logstash/apifeatureusage/elasticsearch.pp
A modules/profile/manifests/logstash/collector.pp
R modules/profile/manifests/logstash/elasticsearch.pp
A modules/profile/manifests/logstash/eventlogging.pp
M modules/role/manifests/logstash/apifeatureusage.pp
M modules/role/manifests/logstash/eventlogging.pp
A modules/role/manifests/logstash/frontend.pp
A modules/role/manifests/logstash/storage.pp
27 files changed, 332 insertions(+), 27 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/39/390039/1
diff --git a/manifests/site.pp b/manifests/site.pp
index 48fc866..bd4e756 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1451,23 +1451,23 @@
}
node /^logstash100[1-2]\.eqiad\.wmnet$/ {
- role(logstash::collector, kibana, logstash::apifeatureusage)
+ role(logstash::frontend, kibana)
include ::lvs::realserver
}
node /^logstash1003\.eqiad\.wmnet$/ {
# Before decommissionning logstash1003, role::logstash::eventlogging needs
# to be moved to another node
- role(logstash::collector, kibana, logstash::apifeatureusage,
logstash::eventlogging)
+ role(logstash::eventlogging, kibana)
include ::lvs::realserver
}
node /^logstash100[4-6]\.eqiad\.wmnet$/ {
- role(logstash::elasticsearch)
+ role(logstash::storage)
}
# logstash collectors (Ganeti VM)
node /^logstash100[7-9]\.eqiad\.wmnet$/ {
- role(logstash::collector, kibana, logstash::apifeatureusage)
+ role(logstash::frontend, kibana)
include ::lvs::realserver
}
diff --git a/modules/role/files/logstash/apifeatureusage-template.json
b/modules/profile/files/logstash/apifeatureusage-template.json
similarity index 100%
rename from modules/role/files/logstash/apifeatureusage-template.json
rename to modules/profile/files/logstash/apifeatureusage-template.json
diff --git a/modules/role/files/logstash/elasticsearch-template.json
b/modules/profile/files/logstash/elasticsearch-template.json
similarity index 100%
rename from modules/role/files/logstash/elasticsearch-template.json
rename to modules/profile/files/logstash/elasticsearch-template.json
diff --git a/modules/role/files/logstash/filter-add-normalized-message.conf
b/modules/profile/files/logstash/filter-add-normalized-message.conf
similarity index 100%
rename from modules/role/files/logstash/filter-add-normalized-message.conf
rename to modules/profile/files/logstash/filter-add-normalized-message.conf
diff --git a/modules/role/files/logstash/filter-apifeatureusage.conf
b/modules/profile/files/logstash/filter-apifeatureusage.conf
similarity index 100%
rename from modules/role/files/logstash/filter-apifeatureusage.conf
rename to modules/profile/files/logstash/filter-apifeatureusage.conf
diff --git a/modules/role/files/logstash/filter-de_dot.conf
b/modules/profile/files/logstash/filter-de_dot.conf
similarity index 100%
rename from modules/role/files/logstash/filter-de_dot.conf
rename to modules/profile/files/logstash/filter-de_dot.conf
diff --git a/modules/role/files/logstash/filter-eventlogging.conf
b/modules/profile/files/logstash/filter-eventlogging.conf
similarity index 100%
rename from modules/role/files/logstash/filter-eventlogging.conf
rename to modules/profile/files/logstash/filter-eventlogging.conf
diff --git a/modules/role/files/logstash/filter-gelf.conf
b/modules/profile/files/logstash/filter-gelf.conf
similarity index 100%
rename from modules/role/files/logstash/filter-gelf.conf
rename to modules/profile/files/logstash/filter-gelf.conf
diff --git a/modules/role/files/logstash/filter-json-lines.conf
b/modules/profile/files/logstash/filter-json-lines.conf
similarity index 100%
rename from modules/role/files/logstash/filter-json-lines.conf
rename to modules/profile/files/logstash/filter-json-lines.conf
diff --git a/modules/role/files/logstash/filter-logback.conf
b/modules/profile/files/logstash/filter-logback.conf
similarity index 100%
rename from modules/role/files/logstash/filter-logback.conf
rename to modules/profile/files/logstash/filter-logback.conf
diff --git a/modules/role/files/logstash/filter-mediawiki.conf
b/modules/profile/files/logstash/filter-mediawiki.conf
similarity index 100%
rename from modules/role/files/logstash/filter-mediawiki.conf
rename to modules/profile/files/logstash/filter-mediawiki.conf
diff --git a/modules/role/files/logstash/filter-normalize-log-levels.conf
b/modules/profile/files/logstash/filter-normalize-log-levels.conf
similarity index 100%
rename from modules/role/files/logstash/filter-normalize-log-levels.conf
rename to modules/profile/files/logstash/filter-normalize-log-levels.conf
diff --git a/modules/role/files/logstash/filter-ores.conf
b/modules/profile/files/logstash/filter-ores.conf
similarity index 100%
rename from modules/role/files/logstash/filter-ores.conf
rename to modules/profile/files/logstash/filter-ores.conf
diff --git a/modules/role/files/logstash/filter-striker.conf
b/modules/profile/files/logstash/filter-striker.conf
similarity index 100%
rename from modules/role/files/logstash/filter-striker.conf
rename to modules/profile/files/logstash/filter-striker.conf
diff --git a/modules/role/files/logstash/filter-strip-ansi-color.conf
b/modules/profile/files/logstash/filter-strip-ansi-color.conf
similarity index 100%
rename from modules/role/files/logstash/filter-strip-ansi-color.conf
rename to modules/profile/files/logstash/filter-strip-ansi-color.conf
diff --git a/modules/role/files/logstash/filter-syslog.conf
b/modules/profile/files/logstash/filter-syslog.conf
similarity index 100%
rename from modules/role/files/logstash/filter-syslog.conf
rename to modules/profile/files/logstash/filter-syslog.conf
diff --git a/modules/role/files/logstash/filter-udp2log.conf
b/modules/profile/files/logstash/filter-udp2log.conf
similarity index 100%
rename from modules/role/files/logstash/filter-udp2log.conf
rename to modules/profile/files/logstash/filter-udp2log.conf
diff --git a/modules/role/files/logstash/filter-webrequest.conf
b/modules/profile/files/logstash/filter-webrequest.conf
similarity index 100%
rename from modules/role/files/logstash/filter-webrequest.conf
rename to modules/profile/files/logstash/filter-webrequest.conf
diff --git a/modules/profile/manifests/logstash/apifeatureusage.pp
b/modules/profile/manifests/logstash/apifeatureusage.pp
new file mode 100644
index 0000000..6759c41
--- /dev/null
+++ b/modules/profile/manifests/logstash/apifeatureusage.pp
@@ -0,0 +1,33 @@
+# vim:sw=4 ts=4 sts=4 et:
+# == Class: role::logstash::apifeatureusage
+#
+# Builds on role::logstash to insert sanitized data for
+# Extension:ApiFeatureUsage into Elasticsearch.
+#
+# filtertags: labs-project-deployment-prep
+class profile::logstash::apifeatureusage(
+ $hosts = hiera('role::logstash::apifeatureusage::elastic_hosts'),
+) {
+
+ validate_array($hosts)
+
+ # Template for Elasticsearch index creation
+ file { '/etc/logstash/apifeatureusage-template.json':
+ ensure => present,
+ source =>
'puppet:///modules/profile/logstash/apifeatureusage-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+
+ # Add configuration to logstash
+ # Needs to come after 'filter_mediawiki' (priority 50)
+ logstash::conf { 'filter_apifeatureusage':
+ source =>
'puppet:///modules/profile/logstash/filter-apifeatureusage.conf',
+ priority => 55,
+ }
+
+ # Output destined for separate Elasticsearch cluster from Logstash cluster
+ profile::logstash::apifeatureusage::elasticsearch { $hosts: }
+
+}
diff --git a/modules/role/manifests/logstash/apifeatureusage/elasticsearch.pp
b/modules/profile/manifests/logstash/apifeatureusage/elasticsearch.pp
similarity index 91%
rename from modules/role/manifests/logstash/apifeatureusage/elasticsearch.pp
rename to modules/profile/manifests/logstash/apifeatureusage/elasticsearch.pp
index 7bb1849..0a6ae63 100644
--- a/modules/role/manifests/logstash/apifeatureusage/elasticsearch.pp
+++ b/modules/profile/manifests/logstash/apifeatureusage/elasticsearch.pp
@@ -3,7 +3,7 @@
# This is used to do a pseudo iterator. It might be replaced by a nicer syntax
# once we are fully Puppet 4 compatible.
#
-define role::logstash::apifeatureusage::elasticsearch {
+define profile::logstash::apifeatureusage::elasticsearch {
logstash::output::elasticsearch { "apifeatureusage-${title}":
host => $title,
index => 'apifeatureusage-%{+YYYY.MM.dd}',
@@ -14,4 +14,4 @@
template => '/etc/logstash/apifeatureusage-template.json',
require => File['/etc/logstash/apifeatureusage-template.json'],
}
-}
\ No newline at end of file
+}
diff --git a/modules/profile/manifests/logstash/collector.pp
b/modules/profile/manifests/logstash/collector.pp
new file mode 100644
index 0000000..0436c27
--- /dev/null
+++ b/modules/profile/manifests/logstash/collector.pp
@@ -0,0 +1,233 @@
+# vim:sw=4 ts=4 sts=4 et:
+# == Class: profile::logstash::collector
+#
+# Provisions Logstash and an Elasticsearch node to proxy requests to ELK stack
+# Elasticsearch cluster.
+#
+# == Parameters:
+# - $statsd_host: Host to send statsd data to.
+#
+# filtertags: labs-project-deployment-prep
+class profile::logstash::collector (
+ $statsd_host,
+) {
+ include ::logstash
+
+ nrpe::monitor_service { 'logstash':
+ description => 'logstash process',
+ nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -u
logstash -C java -a logstash',
+ }
+
+ ## Inputs (10)
+
+ logstash::input::udp2log { 'mediawiki':
+ port => 8324,
+ }
+
+ ferm::service { 'logstash_udp2log':
+ proto => 'udp',
+ port => '8324',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
+ logstash::input::syslog { 'syslog':
+ port => 10514,
+ }
+
+ ferm::service { 'logstash_syslog_udp':
+ proto => 'udp',
+ port => '10514',
+ notrack => true,
+ srange => '($DOMAIN_NETWORKS $NETWORK_INFRA $MGMT_NETWORKS)',
+ }
+
+ ferm::service { 'logstash_syslog_tcp':
+ proto => 'tcp',
+ port => '10514',
+ notrack => true,
+ srange => '($DOMAIN_NETWORKS $NETWORK_INFRA $MGMT_NETWORKS)',
+ }
+
+ ferm::service { 'grafana_dashboard_definition_storage':
+ proto => 'tcp',
+ port => '9200',
+ srange => '@resolve(krypton.eqiad.wmnet)',
+ }
+
+ ferm::service { 'logstash_canary_checker_reporting':
+ proto => 'tcp',
+ port => '9200',
+ srange => '($DEPLOYMENT_HOSTS $MAINTENANCE_HOSTS)',
+ }
+
+ logstash::input::gelf { 'gelf':
+ port => 12201,
+ }
+
+ ferm::service { 'logstash_gelf':
+ proto => 'udp',
+ port => '12201',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
+ logstash::input::log4j { 'log4j': }
+
+ ferm::service { 'logstash_log4j':
+ proto => 'tcp',
+ port => '4560',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
+ logstash::input::udp { 'logback':
+ port => 11514,
+ codec => 'json',
+ }
+
+ ferm::service { 'logstash_udp':
+ proto => 'udp',
+ port => '11514',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
+ logstash::input::tcp { 'json_lines':
+ port => 11514,
+ codec => 'json_lines',
+ }
+
+ ferm::service { 'logstash_json_lines':
+ proto => 'tcp',
+ port => '11514',
+ notrack => true,
+ srange => '$DOMAIN_NETWORKS',
+ }
+
+ ## Global pre-processing (15)
+
+ # move files into module?
+ # lint:ignore:puppet_url_without_modules
+ logstash::conf { 'filter_strip_ansi_color':
+ source =>
'puppet:///modules/profile/logstash/filter-strip-ansi-color.conf',
+ priority => 15,
+ }
+
+ ## Input specific processing (20)
+
+ logstash::conf { 'filter_syslog':
+ source => 'puppet:///modules/profile/logstash/filter-syslog.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_udp2log':
+ source => 'puppet:///modules/profile/logstash/filter-udp2log.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_gelf':
+ source => 'puppet:///modules/profile/logstash/filter-gelf.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_logback':
+ source => 'puppet:///modules/profile/logstash/filter-logback.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_json_lines':
+ source =>
'puppet:///modules/profile/logstash/filter-json-lines.conf',
+ priority => 20,
+ }
+ ## Application specific processing (50)
+
+ logstash::conf { 'filter_mediawiki':
+ source => 'puppet:///modules/profile/logstash/filter-mediawiki.conf',
+ priority => 50,
+ }
+
+ logstash::conf { 'filter_striker':
+ source => 'puppet:///modules/profile/logstash/filter-striker.conf',
+ priority => 50,
+ }
+
+ logstash::conf { 'filter_ores':
+ source => 'puppet:///modules/profile/logstash/filter-ores.conf',
+ priority => 50,
+ }
+
+ logstash::conf { 'filter_webrequest':
+ source =>
'puppet:///modules/profile/logstash/filter-webrequest.conf',
+ priority => 50,
+ }
+
+ ## Global post-processing (70)
+
+ logstash::conf { 'filter_add_normalized_message':
+ source =>
'puppet:///modules/profile/logstash/filter-add-normalized-message.conf',
+ priority => 70,
+ }
+
+ logstash::conf { 'filter_normalize_log_levels':
+ source =>
'puppet:///modules/profile/logstash/filter-normalize-log-levels.conf',
+ priority => 70,
+ }
+
+ logstash::conf { 'filter_de_dot':
+ source => 'puppet:///modules/profile/logstash/filter-de_dot.conf',
+ priority => 70,
+ }
+
+ ## Outputs (90)
+ # Template for Elasticsearch index creation
+ file { '/etc/logstash/elasticsearch-template.json':
+ ensure => present,
+ source =>
'puppet:///modules/profile/logstash/elasticsearch-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+ # lint:endignore
+
+ logstash::output::elasticsearch { 'logstash':
+ host => '127.0.0.1',
+ guard_condition => '"es" in [tags]',
+ manage_indices => true,
+ priority => 90,
+ template => '/etc/logstash/elasticsearch-template.json',
+ require => File['/etc/logstash/elasticsearch-template.json'],
+ }
+
+ logstash::output::statsd { 'MW_channel_rate':
+ host => $statsd_host,
+ guard_condition => '[type] == "mediawiki" and "es" in [tags]',
+ namespace => 'logstash.rate',
+ sender => 'mediawiki',
+ increment => [ '%{channel}.%{level}' ],
+ }
+
+ logstash::output::statsd { 'OOM_channel_rate':
+ host => $statsd_host,
+ guard_condition => '[type] == "hhvm" and [message] =~ "request has
exceeded memory limit"',
+ namespace => 'logstash.rate',
+ sender => 'oom',
+ increment => [ '%{level}' ],
+ }
+
+ logstash::output::statsd { 'HHVM_channel_rate':
+ host => $statsd_host,
+ guard_condition => '[type] == "hhvm" and [message] !~ "request has
exceeded memory limit"',
+ namespace => 'logstash.rate',
+ sender => 'hhvm',
+ increment => [ '%{level}' ],
+ }
+
+ logstash::output::statsd { 'Apache2_channel_rate':
+ host => $statsd_host,
+ guard_condition => '[type] == "apache2" and "syslog" in [tags]',
+ namespace => 'logstash.rate',
+ sender => 'apache2',
+ increment => [ '%{level}' ],
+ }
+}
diff --git a/modules/role/manifests/logstash/elasticsearch.pp
b/modules/profile/manifests/logstash/elasticsearch.pp
similarity index 89%
rename from modules/role/manifests/logstash/elasticsearch.pp
rename to modules/profile/manifests/logstash/elasticsearch.pp
index c81695c..fb730a7 100644
--- a/modules/role/manifests/logstash/elasticsearch.pp
+++ b/modules/profile/manifests/logstash/elasticsearch.pp
@@ -3,10 +3,10 @@
#
# Provisions Elasticsearch backend node for a Logstash cluster.
#
-class role::logstash::elasticsearch {
- include ::standard
+class profile::logstash::elasticsearch(
+ $logstash_nodes = hiera('logstash::cluster_hosts'),
+) {
include ::elasticsearch::monitor::diamond
- include ::base::firewall
# the logstash cluster has 3 data nodes, and each shard has 3 replica (each
#shard is present on each node). If one node is lost, 1/3 of the shards
@@ -29,7 +29,6 @@
curator_uses_unicast_hosts => false, # elasticsearch API is only
exposed to localhost
}
- $logstash_nodes = hiera('logstash::cluster_hosts')
$logstash_nodes_ferm = join($logstash_nodes, ' ')
ferm::service { 'logstash_elastic_internode':
diff --git a/modules/profile/manifests/logstash/eventlogging.pp
b/modules/profile/manifests/logstash/eventlogging.pp
new file mode 100644
index 0000000..71ff5fa
--- /dev/null
+++ b/modules/profile/manifests/logstash/eventlogging.pp
@@ -0,0 +1,18 @@
+class profile::logstash::eventlogging {
+ require ::profile::logstash::collector
+
+ $topic = 'eventlogging_EventError'
+ $kafka_config = kafka_config('analytics')
+
+ logstash::input::kafka { $topic:
+ tags => [$topic, 'kafka'],
+ type => 'eventlogging',
+ bootstrap_servers => $kafka_config['brokers']['string'],
+ }
+
+ logstash::conf { 'filter_eventlogging':
+ source =>
'puppet:///modules/profile/logstash/filter-eventlogging.conf',
+ priority => 50,
+ }
+
+}
diff --git a/modules/role/manifests/logstash/apifeatureusage.pp
b/modules/role/manifests/logstash/apifeatureusage.pp
index df69e4f..4ea6445 100644
--- a/modules/role/manifests/logstash/apifeatureusage.pp
+++ b/modules/role/manifests/logstash/apifeatureusage.pp
@@ -15,7 +15,7 @@
# lint:ignore:puppet_url_without_modules
file { '/etc/logstash/apifeatureusage-template.json':
ensure => present,
- source =>
'puppet:///modules/role/logstash/apifeatureusage-template.json',
+ source =>
'puppet:///modules/profile/logstash/apifeatureusage-template.json',
owner => 'root',
group => 'root',
mode => '0444',
@@ -24,12 +24,12 @@
# Add configuration to logstash
# Needs to come after 'filter_mediawiki' (priority 50)
logstash::conf { 'filter_apifeatureusage':
- source =>
'puppet:///modules/role/logstash/filter-apifeatureusage.conf',
+ source =>
'puppet:///modules/profile/logstash/filter-apifeatureusage.conf',
priority => 55,
}
# lint:endignore
# Output destined for separate Elasticsearch cluster from Logstash cluster
- role::logstash::apifeatureusage::elasticsearch { $hosts: }
+ profile::logstash::apifeatureusage::elasticsearch { $hosts: }
}
diff --git a/modules/role/manifests/logstash/eventlogging.pp
b/modules/role/manifests/logstash/eventlogging.pp
index 7cc5cc6..043d8a2 100644
--- a/modules/role/manifests/logstash/eventlogging.pp
+++ b/modules/role/manifests/logstash/eventlogging.pp
@@ -5,20 +5,16 @@
#
# filtertags: labs-project-deployment-prep
class role::logstash::eventlogging {
- include ::role::logstash::collector
+ include ::standard
+ include ::base::firewall
+ include ::role::lvs::realserver
+ include ::profile::logstash::elasticsearch
+ include ::profile::logstash::collector
+ include ::profile::logstash::apifeatureusage
+ include ::profile::logstash::eventlogging
- $topic = 'eventlogging_EventError'
- $kafka_config = kafka_config('analytics')
-
- logstash::input::kafka { $topic:
- tags => [$topic, 'kafka'],
- type => 'eventlogging',
- bootstrap_servers => $kafka_config['brokers']['string'],
+ system::role { 'logstash::eventlogging':
+ ensure => 'present',
+ description => 'logstash frontend and eventlogging collector',
}
- # lint:ignore:puppet_url_without_modules
- logstash::conf { 'filter_eventlogging':
- source => 'puppet:///modules/role/logstash/filter-eventlogging.conf',
- priority => 50,
- }
- # lint:endignore
}
diff --git a/modules/role/manifests/logstash/frontend.pp
b/modules/role/manifests/logstash/frontend.pp
new file mode 100644
index 0000000..aa434f1
--- /dev/null
+++ b/modules/role/manifests/logstash/frontend.pp
@@ -0,0 +1,14 @@
+class role::logstash::frontend {
+ include ::standard
+ include ::base::firewall
+ include ::role::lvs::realserver
+ include ::profile::logstash::elasticsearch
+ include ::profile::logstash::collector
+ include ::profile::logstash::apifeatureusage
+
+ system::role { 'logstash::frontend':
+ ensure => 'present',
+ description => 'logstash frontend',
+ }
+
+}
diff --git a/modules/role/manifests/logstash/storage.pp
b/modules/role/manifests/logstash/storage.pp
new file mode 100644
index 0000000..a51dea0
--- /dev/null
+++ b/modules/role/manifests/logstash/storage.pp
@@ -0,0 +1,12 @@
+class role::logstash::storage {
+ include ::standard
+ include ::base::firewall
+ include ::role::lvs::realserver
+ include ::profile::logstash::elasticsearch
+
+ system::role { 'logstash::storage':
+ ensure => 'present',
+ description => 'elasticsearch data node backing logstash',
+ }
+
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/390039
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I463285567317fef595c3f178310c4b053244597a
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Gehel <guillaume.leder...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
