Filippo Giunchedi has submitted this change and it was merged.
Change subject: labs: new role::logstash::stashbot class
......................................................................
labs: new role::logstash::stashbot class
Add a new role and configuration for use in the stashbot labs project to
create instances for logging irc messages.
The irc filters have special handling for messages with certain prefixes:
* !log for server admin log (SAL) logging
* !bash for quips logging
* Remove role::logstash::ircbot which was previously only used in the
beta cluster.
* Refactor role::logstash to include role::logstash::elasticsearch
rather than duplicating its configuration.
* Update role::logstash::elasticsearch to only apply
::elasticsearch::ganglia when $::standard::has_ganglia is true. This
mirrors usage in role::elasticsearch.
* Refactor role::kibana to allow fine grained configuration via hiera.
Change-Id: Ia04219138f3dab667d7c2a667994681aeeabc563
---
D files/logstash/filter-irc-banglog.conf
A files/logstash/filter-stashbot-bash.conf
A files/logstash/filter-stashbot-sal.conf
A files/logstash/filter-stashbot.conf
A files/logstash/stashbot-bash-template.json
A files/logstash/stashbot-sal-template.json
A files/logstash/stashbot-template.json
M hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
A hieradata/labs/stashbot/common.yaml
A hieradata/labs/stashbot/host/stashbot-logstash.yaml
M hieradata/role/common/logstash.yaml
M manifests/role/kibana.pp
M manifests/role/logstash.pp
M modules/logstash/manifests/conf.pp
M modules/logstash/manifests/init.pp
M modules/logstash/manifests/input/irc.pp
A templates/kibana/apache-auth-none.erb
M templates/kibana/apache.conf.erb
18 files changed, 437 insertions(+), 129 deletions(-)
Approvals:
Filippo Giunchedi: Verified; Looks good to me, approved
diff --git a/files/logstash/filter-irc-banglog.conf
b/files/logstash/filter-irc-banglog.conf
deleted file mode 100644
index 44cfb5c..0000000
--- a/files/logstash/filter-irc-banglog.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-filter {
- if [type] == "irc" and [message] =~ /^!log / {
- mutate {
- add_tag => [ "es", "banglog" ]
- }
-
- if [channel] == "#wikimedia-labs" {
- grok {
- match => [ "message", "^!log %{NOTSPACE:project} " ]
- named_captures_only => true
- }
- }
-
- if [channel] == "#wikimedia-releng" {
- mutate {
- add_field => [ "project", "qa" ]
- }
- }
-
- if [channel] == "#wikimedia-operations" {
- mutate {
- add_field => [ "project", "production" ]
- }
- }
- }
-}
diff --git a/files/logstash/filter-stashbot-bash.conf
b/files/logstash/filter-stashbot-bash.conf
new file mode 100644
index 0000000..0c02ecc
--- /dev/null
+++ b/files/logstash/filter-stashbot-bash.conf
@@ -0,0 +1,21 @@
+filter {
+ if [type] == "bash" {
+ # Strip !bash from start of message
+ grok {
+ match => [ "message", "^!bash %{GREEDYDATA:message}$" ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ }
+
+ mutate {
+ # Replace tabs with newlines
+ # NOTE: a literal newline is used as Logstash doesn't properly expand
+ # escape codes in the replacement string.
+ gsub => [ "message", "\t", "
+" ]
+
+ # Trim leading/trailing whitespace
+ strip => [ "message" ]
+ }
+ } # end [type] == "bash"
+}
diff --git a/files/logstash/filter-stashbot-sal.conf
b/files/logstash/filter-stashbot-sal.conf
new file mode 100644
index 0000000..ce8a717
--- /dev/null
+++ b/files/logstash/filter-stashbot-sal.conf
@@ -0,0 +1,38 @@
+filter {
+ if [type] == "sal" {
+ if [channel] == "#wikimedia-labs" {
+ grok {
+ match => [ "message", "^!log %{NOTSPACE:project}
%{GREEDYDATA:message}$" ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ }
+ }
+
+ if [channel] == "#wikimedia-releng" {
+ grok {
+ match => [ "message", "^!log %{GREEDYDATA:message}$" ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ add_field => [ "project", "releng" ]
+ }
+ }
+
+ if [channel] == "#wikimedia-operations" {
+ grok {
+ match => [ "message", "^!log %{GREEDYDATA:message}$" ]
+ overwrite => [ "message" ]
+ named_captures_only => true
+ add_field => [ "project", "production" ]
+ }
+
+ if [nick] == "logmsgbot" {
+ # Scrape shell sender out of logmsgbot relayed messages
+ grok {
+ match => [ "message", "^%{NOTSPACE:nick} %{GREEDYDATA:message}$" ]
+ overwrite => [ "nick", "message" ]
+ named_captures_only => true
+ }
+ }
+ }
+ } # end [type] == "sal"
+}
diff --git a/files/logstash/filter-stashbot.conf
b/files/logstash/filter-stashbot.conf
new file mode 100644
index 0000000..3ef1c26
--- /dev/null
+++ b/files/logstash/filter-stashbot.conf
@@ -0,0 +1,25 @@
+filter {
+ if [type] == "irc" {
+ # Remove IRC formatting codes and tag for indexing
+ mutate {
+ gsub => [ "message", "[\x02\x0F\x16\x1D\x1F]|\x03(\d{,2}(,\d{,2})?)?",
"" ]
+ add_tag => [ "es" ]
+ }
+
+ if [message] =~ /^!bash / {
+ # Clone for custom bash message processing
+ clone {
+ clones => [ "bash" ]
+ remove_tag => [ "es" ]
+ }
+ } # end [message] =~ /^!bash /
+
+ if [message] =~ /^!log / {
+ # Clone for custom SAL message processing
+ clone {
+ clones => [ "sal" ]
+ remove_tag => [ "es" ]
+ }
+ } # end [message] =~ /^!log /
+ } # end [type] == "irc"
+}
diff --git a/files/logstash/stashbot-bash-template.json
b/files/logstash/stashbot-bash-template.json
new file mode 100644
index 0000000..abc1bc4
--- /dev/null
+++ b/files/logstash/stashbot-bash-template.json
@@ -0,0 +1,39 @@
+{
+ "template" : "bash",
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 1,
+ "index.cache.field.type" : "soft",
+ "index.refresh_interval" : "5s",
+ "analysis" : {
+ "analyzer" : {
+ "default" : {
+ "type" : "standard",
+ "stopwords" : "_none_"
+ }
+ }
+ }
+ },
+ "mappings" : {
+ "sal" : {
+ "_all" : {"enabled" : false},
+ "dynamic_templates" : [ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms"
: true }
+ }
+ } ],
+ "properties" : {
+ "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+ "@version" : { "type" : "string", "index" : "not_analyzed" },
+ "channel" : { "type" : "string", "index" : "not_analyzed" },
+ "message" : { "type" : "string", "index" : "analyzed", "omit_norms" :
true },
+ "nick" : { "type" : "string", "index" : "not_analyzed" },
+ "server" : { "type" : "string", "index" : "not_analyzed" },
+ "tags" : { "type" : "string", "index" : "not_analyzed", "index_name" :
"tag" },
+ "type" : { "type" : "string", "index" : "not_analyzed" }
+ }
+ }
+ }
+}
diff --git a/files/logstash/stashbot-sal-template.json
b/files/logstash/stashbot-sal-template.json
new file mode 100644
index 0000000..b7bb818
--- /dev/null
+++ b/files/logstash/stashbot-sal-template.json
@@ -0,0 +1,40 @@
+{
+ "template" : "sal",
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 1,
+ "index.cache.field.type" : "soft",
+ "index.refresh_interval" : "5s",
+ "analysis" : {
+ "analyzer" : {
+ "default" : {
+ "type" : "standard",
+ "stopwords" : "_none_"
+ }
+ }
+ }
+ },
+ "mappings" : {
+ "sal" : {
+ "_all" : {"enabled" : false},
+ "dynamic_templates" : [ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms"
: true }
+ }
+ } ],
+ "properties" : {
+ "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+ "@version" : { "type" : "string", "index" : "not_analyzed" },
+ "channel" : { "type" : "string", "index" : "not_analyzed" },
+ "message" : { "type" : "string", "index" : "analyzed", "omit_norms" :
true },
+ "nick" : { "type" : "string", "index" : "not_analyzed" },
+ "project" : { "type" : "string", "index" : "not_analyzed" },
+ "server" : { "type" : "string", "index" : "not_analyzed" },
+ "tags" : { "type" : "string", "index" : "not_analyzed", "index_name" :
"tag" },
+ "type" : { "type" : "string", "index" : "not_analyzed" }
+ }
+ }
+ }
+}
diff --git a/files/logstash/stashbot-template.json
b/files/logstash/stashbot-template.json
new file mode 100644
index 0000000..4dbdd7d
--- /dev/null
+++ b/files/logstash/stashbot-template.json
@@ -0,0 +1,39 @@
+{
+ "template" : "logstash-*",
+ "settings" : {
+ "number_of_shards" : 1,
+ "number_of_replicas" : 1,
+ "index.cache.field.type" : "soft",
+ "index.refresh_interval" : "5s",
+ "analysis" : {
+ "analyzer" : {
+ "default" : {
+ "type" : "standard",
+ "stopwords" : "_none_"
+ }
+ }
+ }
+ },
+ "mappings" : {
+ "_default_" : {
+ "_all" : {"enabled" : false},
+ "dynamic_templates" : [ {
+ "string_fields" : {
+ "match" : "*",
+ "match_mapping_type" : "string",
+ "mapping" : { "type" : "string", "index" : "analyzed", "omit_norms"
: true }
+ }
+ } ],
+ "properties" : {
+ "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
+ "@version" : { "type" : "string", "index" : "not_analyzed" },
+ "channel" : { "type" : "string", "index" : "not_analyzed" },
+ "message" : { "type" : "string", "index" : "analyzed", "omit_norms" :
true },
+ "nick" : { "type" : "string", "index" : "not_analyzed" },
+ "server" : { "type" : "string", "index" : "not_analyzed" },
+ "tags" : { "type" : "string", "index" : "not_analyzed", "index_name" :
"tag" },
+ "type" : { "type" : "string", "index" : "not_analyzed" }
+ }
+ }
+ }
+}
diff --git a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
index 06a3b3e..d1a9013 100644
--- a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
+++ b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml
@@ -26,3 +26,10 @@
- logstash1004.eqiad.wmnet
- logstash1005.eqiad.wmnet
- logstash1006.eqiad.wmnet
+
+# Kibana
+role::kibana::vhost: logstash-beta.wmflabs.org
+role::kibana::serveradmin:
root@deployment-logstash2.deployment-prep.eqiad.wmflabs
+role::kibana::auth_type: local
+role::kibana::auth_realm: "Logstash (ssh deployment-bastion.eqiad.wmflabs sudo
cat /root/secrets.txt)"
+role::kibana::auth_file: /etc/logstash/htpasswd
diff --git a/hieradata/labs/stashbot/common.yaml
b/hieradata/labs/stashbot/common.yaml
new file mode 100644
index 0000000..06251d3
--- /dev/null
+++ b/hieradata/labs/stashbot/common.yaml
@@ -0,0 +1,23 @@
+---
+# Elasticsearch
+elasticsearch::auto_create_index: true
+elasticsearch::plugins_dir: /srv/deployment/elasticsearch/plugins
+elasticsearch::script_disable_dynamic: true
+elasticsearch::cluster_name: stashbot
+elasticsearch::expected_nodes: 2
+elasticsearch::heap_memory: '8G'
+elasticsearch::minimum_master_nodes: 1
+elasticsearch::recover_after_nodes: 1
+elasticsearch::recover_after_time: 1m
+elasticsearch::unicast_hosts:
+ - stashbot-elastic01.stashbot.eqiad.wmflabs
+ - stashbot-elastic02.stashbot.eqiad.wmflabs
+
+# Logstash
+logstash::filter_workers: 2
+logstash::heap_memory_mb: 256
+
+# Kibana
+role::kibana::vhost: stashbot.wmflabs.org
+role::kibana::serveradmin: bd...@wikimedia.org
+role::kibana::auth_type: none
diff --git a/hieradata/labs/stashbot/host/stashbot-logstash.yaml
b/hieradata/labs/stashbot/host/stashbot-logstash.yaml
new file mode 100644
index 0000000..f75cc6a
--- /dev/null
+++ b/hieradata/labs/stashbot/host/stashbot-logstash.yaml
@@ -0,0 +1,7 @@
+---
+# Elasticsearch
+# The ES nodes that are run on the same box as Logstash+Kibana are only used
+# as client nodes to communicate with the backing cluster.
+elasticsearch::holds_data: false
+elasticsearch::master_eligible: false
+elasticsearch::heap_memory: '2G'
diff --git a/hieradata/role/common/logstash.yaml
b/hieradata/role/common/logstash.yaml
index d9c08f7..f77985e 100644
--- a/hieradata/role/common/logstash.yaml
+++ b/hieradata/role/common/logstash.yaml
@@ -30,6 +30,7 @@
logstash::filter_workers: 1
logstash::heap_memory_mb: 256
+
logstash::cluster_hosts:
- logstash1001.eqiad.wmnet
- logstash1002.eqiad.wmnet
@@ -37,3 +38,16 @@
- logstash1004.eqiad.wmnet
- logstash1005.eqiad.wmnet
- logstash1006.eqiad.wmnet
+
+# Kibana
+role::kibana::vhost: logstash.wmflabs.org
+role::kibana::serveradmin: n...@wikimedia.org
+role::kibana::auth_type: ldap
+role::kibana::auth_realm: WMF Labs (use wiki login name not shell) -
nda/ops/wmf
+role::kibana::ldap_authurl: ldaps://ldap-eqiad.wikimedia.org
ldap-codfw.wikimedia.org/ou=people,dc=wikimedia,dc=org?cn
+role::kibana::ldap_bindpass:
"%{scope('passwords::ldap::production::proxypass')}"
+role::kibana::ldap_binddn: cn=proxyagent,ou=profile,dc=wikimedia,dc=org
+role::kibana::ldap_groups:
+ - cn=ops,ou=groups,dc=wikimedia,dc=org
+ - cn=nda,ou=groups,dc=wikimedia,dc=org
+ - cn=wmf,ou=groups,dc=wikimedia,dc=org
diff --git a/manifests/role/kibana.pp b/manifests/role/kibana.pp
index 2604fdd..5b8d5e8 100644
--- a/manifests/role/kibana.pp
+++ b/manifests/role/kibana.pp
@@ -4,76 +4,62 @@
#
# Provisions Kibana
#
-class role::kibana {
+# == Parameters:
+# - $vhost: Apache vhost name
+# - $serveradmin: Email address for contacting server administrator
+# - $auth_type: Vhost auth type. One of ldap, local, none
+# - $es_host: Elasticsearch host to proxy to
+# - $es_port: Elasticsearch port to proxy to
+# - $require_ssl: Require SSL connection to vhost?
+# - $auth_realm: HTTP basic auth realm description
+# - $auth_file: Path to htpasswd file for $auth_type == 'local'
+# - $ldap_authurl: AuthLDAPURL for $auth_type == 'ldap'
+# - $ldap_bindpass: AuthLDAPBindPassword for $auth_type == 'ldap'
+# - $ldap_binddn: AuthLDAPBindDN for $auth_type == 'ldap'
+# - $ldap_groups: List of ldap-group names for $auth_type == 'ldap'
+#
+class role::kibana (
+ $vhost,
+ $serveradmin,
+ $auth_type,
+ $es_host = '127.0.0.1',
+ $es_port = 9200,
+ $require_ssl = true,
+ $auth_realm = undef,
+ $auth_file = undef,
+ $ldap_authurl = undef,
+ $ldap_bindpass = undef,
+ $ldap_binddn = undef,
+ $ldap_groups = [],
+) {
include ::apache
-
- if ($::realm == 'labs') {
- include ::apache::mod::authz_groupfile
- include ::apache::mod::authz_user
-
- if ($::hostname =~ /^deployment-/) {
- # Beta
- $hostname = 'logstash.beta.wmflabs.org'
- $deploy_dir = '/srv/deployment/kibana/kibana'
- $auth_realm = 'Logstash (ssh deployment-bastion.eqiad.wmflabs
sudo cat /root/secrets.txt)'
- $auth_file = '/etc/logstash/htpasswd'
- $require_ssl = true
- } else {
- # Regular labs instance
- $hostname = $::kibana_hostname ? {
- undef => $::hostname,
- default => $::kibana_hostname,
- }
- $deploy_dir = $::kibana_deploydir ? {
- undef => '/srv/deployment/kibana/kibana',
- default => $::kibana_deploydir,
- }
- $auth_realm = $::kibana_authrealm ? {
- undef => 'Logstash',
- default => $::kibana_authrealm,
- }
- $auth_file = $::kibana_authfile ? {
- undef => '/etc/logstash/htpasswd',
- default => $::kibana_authfile,
- }
- $require_ssl = false
- }
- $serveradmin = "root@${hostname}"
- $apache_auth = template('kibana/apache-auth-local.erb')
- } else {
- # Production
- include ::apache::mod::authnz_ldap
- include ::passwords::ldap::production
-
- $hostname = 'logstash.wikimedia.org'
- $deploy_dir = '/srv/deployment/kibana/kibana'
- $serveradmin = 'n...@wikimedia.org'
- $require_ssl = true
-
- $ldap_authurl = 'ldaps://ldap-eqiad.wikimedia.org
ldap-codfw.wikimedia.org/ou=people,dc=wikimedia,dc=org?cn'
- $ldap_bindpass = $passwords::ldap::production::proxypass
- $ldap_binddn = 'cn=proxyagent,ou=profile,dc=wikimedia,dc=org'
- $ldap_groups = [
- 'cn=ops,ou=groups,dc=wikimedia,dc=org',
- 'cn=nda,ou=groups,dc=wikimedia,dc=org',
- 'cn=wmf,ou=groups,dc=wikimedia,dc=org',
- ]
- $auth_realm = 'WMF Labs (use wiki login name not shell) -
nda/ops/wmf'
- $apache_auth = template('kibana/apache-auth-ldap.erb')
- }
- $es_host = '127.0.0.1'
- $es_port = 9200
-
- class { '::kibana':
- default_route => '/dashboard/elasticsearch/default',
- }
-
include ::apache::mod::alias
include ::apache::mod::headers
include ::apache::mod::proxy
include ::apache::mod::proxy_http
include ::apache::mod::rewrite
+ # Directory trebuchet puts Kibana files in
+ $deploy_dir = '/srv/deployment/kibana/kibana'
+
+ if $auth_type == 'ldap' {
+ include ::apache::mod::authnz_ldap
+ include ::passwords::ldap::production
+
+ } elsif $auth_type == 'local' {
+ include ::apache::mod::authz_groupfile
+ include ::apache::mod::authz_user
+
+ } elsif $auth_type != 'none' {
+ fail('role::kibana::auth_type must be one of ldap, local, none')
+ }
+
+ $apache_auth = template("kibana/apache-auth-${auth_type}.erb")
+
+ class { '::kibana':
+ default_route => '/dashboard/elasticsearch/default',
+ }
+
ferm::service { 'kibana_frontend':
proto => 'tcp',
port => 80,
diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp
index 805fe25..2834399 100644
--- a/manifests/role/logstash.pp
+++ b/manifests/role/logstash.pp
@@ -6,18 +6,8 @@
# Provisions Logstash and ElasticSearch.
#
class role::logstash {
- include standard
- include ::elasticsearch::ganglia
- include ::elasticsearch::nagios::check
+ include ::role::logstash::elasticsearch
include ::logstash
-
- package { 'elasticsearch/plugins':
- provider => 'trebuchet',
- }
-
- class { '::elasticsearch':
- require => Package['elasticsearch/plugins'],
- }
## Inputs (10)
@@ -85,6 +75,9 @@
file { '/etc/logstash/elasticsearch-template.json':
ensure => present,
source => 'puppet:///files/logstash/elasticsearch-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
}
logstash::output::elasticsearch { 'logstash':
@@ -103,8 +96,11 @@
#
class role::logstash::elasticsearch {
include standard
- include ::elasticsearch::ganglia
include ::elasticsearch::nagios::check
+
+ if $::standard::has_ganglia {
+ include ::elasticsearch::ganglia
+ }
package { 'elasticsearch/plugins':
provider => 'trebuchet',
@@ -112,29 +108,6 @@
class { '::elasticsearch':
require => Package['elasticsearch/plugins'],
- }
-}
-
-# == Class: role::logstash::ircbot
-#
-# Sets up an IRC Bot to log messages from certain IRC channels
-class role::logstash::ircbot {
- require ::role::logstash
-
- $irc_name = $::logstash_irc_name ? {
- undef => "logstash-${::labsproject}",
- default => $::logstash_irc_name,
- }
-
- logstash::input::irc { 'freenode':
- user => $irc_name,
- nick => $irc_name,
- channels => ['#wikimedia-labs', '#wikimedia-releng',
'#wikimedia-operations'],
- }
-
- logstash::conf { 'filter_irc_banglog':
- source => 'puppet:///files/logstash/filter-irc-banglog.conf',
- priority => 50,
}
}
@@ -179,6 +152,9 @@
file { '/etc/logstash/apifeatureusage-template.json':
ensure => present,
source => 'puppet:///files/logstash/apifeatureusage-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
}
# Add configuration to logstash
@@ -198,3 +174,113 @@
require => File['/etc/logstash/apifeatureusage-template.json'],
}
}
+
+# == Class: role::logstash::stashbot
+#
+# Configure logstash to record IRC channel messages
+#
+# == Parameters:
+# [*irc_user*]
+# IRC username
+#
+# [*irc_pass*]
+# IRC password
+#
+# [*irc_nick*]
+# IRC nick
+#
+# [*irc_real*]
+# IRC real name
+#
+# [*channels*]
+# List of channels to join and log
+#
+class role::logstash::stashbot (
+ $irc_user = 'stashbot',
+ $irc_pass = undef,
+ $irc_nick = 'stashbot',
+ $irc_real = 'Wikimedia Tool Labs IRC bot',
+ $channels = [],
+) {
+ include ::role::logstash::elasticsearch
+ include ::logstash
+
+ logstash::input::irc { 'freenode':
+ user => $irc_user,
+ password => $irc_pass,
+ nick => $irc_nick,
+ real => $irc_real,
+ channels => $channels,
+ }
+
+ logstash::conf { 'filter_strip_ansi_color':
+ source => 'puppet:///files/logstash/filter-strip-ansi-color.conf',
+ priority => 15,
+ }
+
+ logstash::conf { 'filter_stashbot':
+ source => 'puppet:///files/logstash/filter-stashbot.conf',
+ priority => 20,
+ }
+
+ logstash::conf { 'filter_stashbot_sal':
+ source => 'puppet:///files/logstash/filter-stashbot-sal.conf',
+ priority => 50,
+ }
+
+ logstash::conf { 'filter_stashbot_bash':
+ source => 'puppet:///files/logstash/filter-stashbot-bash.conf',
+ priority => 50,
+ }
+
+ file { '/etc/logstash/stashbot-template.json':
+ ensure => present,
+ source => 'puppet:///files/logstash/stashbot-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+ logstash::output::elasticsearch { 'logstash':
+ host => '127.0.0.1',
+ index => "logstash-%{+YYYY.MM}",
+ guard_condition => '"es" in [tags]',
+ priority => 90,
+ template => '/etc/logstash/stashbot-template.json',
+ require => File['/etc/logstash/stashbot-template.json'],
+ }
+
+ # Special indexing for SAL messages
+ file { '/etc/logstash/stashbot-sal-template.json':
+ ensure => present,
+ source => 'puppet:///files/logstash/stashbot-sal-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+ logstash::output::elasticsearch { 'sal':
+ host => $host,
+ index => 'sal',
+ guard_condition => '[type] == "sal"',
+ priority => 95,
+ template => '/etc/logstash/stashbot-sal-template.json',
+ require => File['/etc/logstash/stashbot-sal-template.json'],
+ }
+
+ # Special indexing for bash messages
+ file { '/etc/logstash/stashbot-bash-template.json':
+ ensure => present,
+ source => 'puppet:///files/logstash/stashbot-bash-template.json',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ }
+ logstash::output::elasticsearch { 'bash':
+ host => $host,
+ index => 'bash',
+ guard_condition => '[type] == "bash"',
+ priority => 95,
+ template => '/etc/logstash/stashbot-bash-template.json',
+ require => File['/etc/logstash/stashbot-bash-template.json'],
+ }
+}
+
diff --git a/modules/logstash/manifests/conf.pp
b/modules/logstash/manifests/conf.pp
index 542bbff..72bd1f5 100644
--- a/modules/logstash/manifests/conf.pp
+++ b/modules/logstash/manifests/conf.pp
@@ -26,13 +26,17 @@
$priority = 10,
$ensure = present,
) {
+ include ::logstash
+
$config_name = inline_template('<%= @title.gsub(/\W/, "-") %>')
file { "/etc/logstash/conf.d/${priority}-${config_name}.conf":
ensure => $ensure,
content => $content,
source => $source,
- require => File['/etc/logstash/conf.d'],
+ owner => 'logstash',
+ group => 'logstash',
+ mode => '0440',
notify => Service['logstash'],
}
}
diff --git a/modules/logstash/manifests/init.pp
b/modules/logstash/manifests/init.pp
index 214ac74..09cb486 100644
--- a/modules/logstash/manifests/init.pp
+++ b/modules/logstash/manifests/init.pp
@@ -32,6 +32,9 @@
file { '/etc/default/logstash':
content => template('logstash/default.erb'),
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
require => Package['logstash'],
notify => Service['logstash'],
}
@@ -41,6 +44,8 @@
recurse => true,
purge => true,
force => true,
+ owner => 'logstash',
+ group => 'logstash',
source => 'puppet:///modules/logstash/conf.d',
require => Package['logstash'],
}
diff --git a/modules/logstash/manifests/input/irc.pp
b/modules/logstash/manifests/input/irc.pp
index d0d25a1..8c6bc11 100644
--- a/modules/logstash/manifests/input/irc.pp
+++ b/modules/logstash/manifests/input/irc.pp
@@ -34,7 +34,7 @@
$real = 'logstash',
$priority = 10,
) {
- logstash::conf { "input-irc${title}":
+ logstash::conf { "input-irc-${title}":
ensure => $ensure,
content => template('logstash/input/irc.erb'),
priority => $priority,
diff --git a/templates/kibana/apache-auth-none.erb
b/templates/kibana/apache-auth-none.erb
new file mode 100644
index 0000000..0048cb1
--- /dev/null
+++ b/templates/kibana/apache-auth-none.erb
@@ -0,0 +1,2 @@
+ Allow from all
+ Satisfy Any
diff --git a/templates/kibana/apache.conf.erb b/templates/kibana/apache.conf.erb
index 96255dd..0cc4391 100644
--- a/templates/kibana/apache.conf.erb
+++ b/templates/kibana/apache.conf.erb
@@ -5,7 +5,7 @@
#####################################################################
<VirtualHost *:80>
- ServerName <%= @hostname %>
+ ServerName <%= @vhost %>
ServerAdmin <%= @serveradmin %>
DocumentRoot <%= @deploy_dir %>/src
@@ -34,11 +34,9 @@
Allow from all
</Directory>
-<%- if @apache_auth -%>
<Location />
<%= @apache_auth -%>
</Location>
-<%- end -%>
Alias /config.js /etc/kibana/config.js
--
To view, visit https://gerrit.wikimedia.org/r/227175
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia04219138f3dab667d7c2a667994681aeeabc563
Gerrit-PatchSet: 27
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: BryanDavis <bda...@wikimedia.org>
Gerrit-Reviewer: Chasemp <chas...@gmail.com>
Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Rush <r...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
