Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/72004
Change subject: Adding icinga check to make sure kafka brokers aren't getting too many produce requests ...................................................................... Adding icinga check to make sure kafka brokers aren't getting too many produce requests Change-Id: If8bf706692331d1e2963e5dda289b5be819d309a --- M manifests/misc/analytics.pp M templates/icinga/checkcommands.cfg.erb 2 files changed, 18 insertions(+), 6 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/04/72004/1 diff --git a/manifests/misc/analytics.pp b/manifests/misc/analytics.pp index cf598fa..c190eea 100644 --- a/manifests/misc/analytics.pp +++ b/manifests/misc/analytics.pp @@ -29,10 +29,16 @@ class misc::analytics::monitoring::kafka::server { # Set up icinga monitoring of Kafka broker server produce requests per second. # If this drops too low, trigger an alert - # for this udp2log instance. - monitor_service { "kakfa-broker-ProduceRequestsPerSecond": - description => "kafka_network_SocketServerStats.ProduceRequestsPerSecond", - check_command => "check_kafka_broker_produce_requests!2!1", - contact_group => "analytics", + monitor_service { 'kafka_broker-ProduceRequestsPerSecond': + description => 'kafka_network_SocketServerStats.ProduceRequestsPerSecond', + check_command => 'check_kafka_broker_produce_requests_min!2!1', + contact_group => 'analytics', + } + # Set up icinga monitoring of Kafka broker server produce requests per second. + # If this drops too high, trigger an alert + monitor_service { 'kafka_broker-ProduceRequestsPerSecond': + description => 'kafka_network_SocketServerStats.ProduceRequestsPerSecond', + check_command => 'check_kafka_broker_produce_requests_max!15!20', + contact_group => 'analytics', } } diff --git a/templates/icinga/checkcommands.cfg.erb b/templates/icinga/checkcommands.cfg.erb index b156f7e..b40ea35 100644 --- a/templates/icinga/checkcommands.cfg.erb +++ b/templates/icinga/checkcommands.cfg.erb @@ -617,10 +617,16 @@ # Analytics Kraken checks # Check that Kafka Brokers are getting ProduceRequests. define command{ - command_name check_kafka_broker_produce_requests + command_name check_kafka_broker_produce_requests_min command_line $USER1$/check_ganglios_generic_value -H $HOSTADDRESS$ -m kafka_network_SocketServerStats.ProduceRequestsPerSecond -w $ARG1$ -c $ARG2$ -o lt } +# Check that Kafka Brokers aren't getting TOO many ProduceRequests +define command{ + command_name check_kafka_producer_produce_requests_max + command_line $USER1$/check_ganglios_generic_value -H $HOSTADDRESS$ -m kafka_network_SocketServerStats.ProduceRequestsPerSecond -w $ARG1$ -c $ARG2$ -o gt +} + # Alerts for data loss in Kraken HDFS. define command{ command_name check_kraken_webrequest_loss_positive -- To view, visit https://gerrit.wikimedia.org/r/72004 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If8bf706692331d1e2963e5dda289b5be819d309a Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits