Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/377753 )
Change subject: [WIP] role::kafka::jumbo::broker: enable Prometheus JMX
monitoring
......................................................................
[WIP] role::kafka::jumbo::broker: enable Prometheus JMX monitoring
Bug: T167992
Change-Id: Icb26d40cf4f9b6661d3560f314e073df075d3c1f
---
M hieradata/role/common/kafka/jumbo/broker.yaml
M hieradata/role/common/kafka/simple/broker.yaml
A modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/kafka.default.erb
M modules/profile/manifests/kafka/broker.pp
6 files changed, 67 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/53/377753/1
diff --git a/hieradata/role/common/kafka/jumbo/broker.yaml
b/hieradata/role/common/kafka/jumbo/broker.yaml
index 2960a76..9e6ea8a 100644
--- a/hieradata/role/common/kafka/jumbo/broker.yaml
+++ b/hieradata/role/common/kafka/jumbo/broker.yaml
@@ -2,6 +2,9 @@
profile::kafka::broker::kafka_cluster_name: jumbo
+# Enable the Prometheus JMX Exporter
+profile::kafka::broker::jmx_exporter_enabled: true
+
profile::kafka::broker::log_dirs: [/srv/kafka/data]
profile::kafka::broker::plaintext: true
# TODO: does not yet work.
diff --git a/hieradata/role/common/kafka/simple/broker.yaml
b/hieradata/role/common/kafka/simple/broker.yaml
index 0ac2c22..a41c40a 100644
--- a/hieradata/role/common/kafka/simple/broker.yaml
+++ b/hieradata/role/common/kafka/simple/broker.yaml
@@ -1,5 +1,6 @@
kafka_cluster_name: simple
+profile::kafka::broker::jmx_exporter_enabled: false
profile::kafka::broker::log_dirs: [/srv/kafka/data]
profile::kafka::broker::plaintext: true
profile::kafka::broker::tls_secrets_path: false
diff --git a/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
b/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
new file mode 100644
index 0000000..1e927ac
--- /dev/null
+++ b/modules/confluent/files/kafka/prometheus_jmx_exporter.yaml
@@ -0,0 +1,14 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: true
+# Examples from Cassandra:
+# rules:
+# - pattern: org.apache.cassandra.metrics<type=(Connection|Streaming),
scope=(\S*), name=(\S*)><>(Count|Value)
+# name: cassandra_$1_$3
+# labels:
+# address: "$2"
+# - pattern: org.apache.cassandra.metrics<type=(\S*)(?:,
((?!scope)\S*)=(\S*))?(?:, scope=(\S*))?, name=(\S*)><>(Count|Value)
+# name: cassandra_$1_$5
+# labels:
+# "$1": "$4"
+# "$2": "$3"
diff --git a/modules/confluent/manifests/kafka/broker.pp
b/modules/confluent/manifests/kafka/broker.pp
index 3bb4935..9b1019f 100644
--- a/modules/confluent/manifests/kafka/broker.pp
+++ b/modules/confluent/manifests/kafka/broker.pp
@@ -206,6 +206,14 @@
# The maximum message size allowed.
# Default: 1048576
#
+# [*jmx_exporter_enabled*]
+# Enable the Prometheus Jmx exporter.
+# Default: false
+#
+# [*jmx_exporter_jar_version*]
+# Prometheus Jmx exporter version to use.
+# Default: 0.8-20170117.190412-1
+#
class confluent::kafka::broker(
$enabled = true,
$brokers = {
@@ -279,6 +287,9 @@
$log4j_properties_template =
'confluent/kafka/log4j.properties.erb',
$message_max_bytes = 1048576,
+
+ $jmx_exporter_enabled = false,
+ $jmx_exporter_jar_version = '0.8-20170117.190412-1',
) {
# confluent::kafka::client installs the kafka package
# and a handy wrapper script.
@@ -337,6 +348,23 @@
content => template($server_properties_template),
}
+ if $jmx_exporter_enabled {
+ # Allow the host to be deployable
+ scap::target { 'prometheus/jmx_exporter':
+ deploy_user => 'deploy-service',
+ manage_user => true,
+ }
+ # Create the Prometheus JMX Exporter configuration
+ file { '/etc/kafka/prometheus_jmx_exporter.yaml':
+ ensure => present,
+ source =>
'puppet:///modules/confluent/kafka/prometheus_jmx_exporter.yaml',
+ owner => 'kafka',
+ group => 'kafka',
+ mode => '0400',
+ require => Package[$confluent::kafka::client::package],
+ }
+ }
+
# log4j configuration for Kafka daemon
# process logs in /var/log/kafka.
file { '/etc/kafka/log4j.properties':
diff --git a/modules/confluent/templates/kafka/kafka.default.erb
b/modules/confluent/templates/kafka/kafka.default.erb
index a2dcdbe..e8f55ba 100644
--- a/modules/confluent/templates/kafka/kafka.default.erb
+++ b/modules/confluent/templates/kafka/kafka.default.erb
@@ -25,6 +25,10 @@
JMX_PORT=<%= @jmx_port %>
#KAFKA_JMX_OPTS=${KAFKA_JMX_OPTS:="-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.authenticate=false
-Dcom.sun.management.jmxremote.ssl=false"}
+<% if @jmx_exporter_enabled -%>
+JVM_OPTS="$JVM_OPTS
-javaagent:/srv/deployment/prometheus/jmx_exporter/lib/jmx_prometheus_javaagent-<%=
@jmx_exporter_jar_version %>.jar=<%= @ipaddress
%>:7800:/etc/kafka/prometheus_jmx_exporter.yaml"
+<% end -%>
+
<% if @jvm_performance_opts -%>
KAFKA_JVM_PERFORMANCE_OPTS="<%= @jvm_performance_opts %>"
<% else -%>
diff --git a/modules/profile/manifests/kafka/broker.pp
b/modules/profile/manifests/kafka/broker.pp
index 4d4b183..42ec63c 100644
--- a/modules/profile/manifests/kafka/broker.pp
+++ b/modules/profile/manifests/kafka/broker.pp
@@ -65,6 +65,10 @@
# than 0.10.2, the consumers' fetch size must also be increased
# so that the they can fetch record batches this large.
#
+# [*prometheus_jmx_exporter_enabled*]
+# Enable the prometheus jmx exporter.
+#
+#
class profile::kafka::broker(
$kafka_cluster_name =
hiera('profile::kafka::broker::kafka_cluster_name'),
$statsd = hiera('statsd'),
@@ -83,7 +87,8 @@
$replica_maxlag_warning =
hiera('profile::kafka::broker::replica_maxlag_warning'),
$replica_maxlag_critical =
hiera('profile::kafka::broker::replica_maxlag_critical'),
# This is set via top level hiera variable so it can be synchronized
between roles and clients.
- $message_max_bytes = hiera('kafka_message_max_bytes')
+ $message_max_bytes = hiera('kafka_message_max_bytes'),
+ $jmx_exporter_enabled =
hiera('profile::kafka::broker::jmx_exporter_enabled'),
) {
# TODO: WIP
$tls_secrets_path = undef
@@ -180,6 +185,15 @@
java_home => '/usr/lib/jvm/java-8-openjdk-amd64',
}
+ if $jmx_exporter_enabled {
+ # Allow automatic generation of config on the
+ # Prometheus master
+ prometheus::jmx_exporter_instance { $::hostname:
+ address => $::ipaddress,
+ port => 7800,
+ }
+ }
+
class { '::confluent::kafka::broker':
log_dirs => $log_dirs,
brokers => $config['brokers']['hash'],
@@ -195,7 +209,6 @@
# https://kafka.apache.org/documentation/#java
# Note that MetaspaceSize is a Java 8 setting.
jvm_performance_opts => '-server -XX:MetaspaceSize=96m
-XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35
-XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50
-XX:MaxMetaspaceFreeRatio=80',
-
listeners => $listeners,
security_inter_broker_protocol => $security_inter_broker_protocol,
@@ -209,6 +222,8 @@
auto_leader_rebalance_enable => $auto_leader_rebalance_enable,
num_replica_fetchers => $num_replica_fetchers,
message_max_bytes => $message_max_bytes,
+
+ jmx_exporter_enabled => $jmx_exporter_enabled,
}
class { '::confluent::kafka::broker::jmxtrans':
--
To view, visit https://gerrit.wikimedia.org/r/377753
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icb26d40cf4f9b6661d3560f314e073df075d3c1f
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <ltosc...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
