Elukey has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/363307 )
Change subject: role::analytics_cluster::hadoop::master: add icinga check for HDFS topology ...................................................................... role::analytics_cluster::hadoop::master: add icinga check for HDFS topology Bug: T163909 Change-Id: I6f1c259eb872abcca60d96add2bfa89a0546a14d --- A modules/role/files/analytics_cluster/hadoop/check_hdfs_topology M modules/role/manifests/analytics_cluster/hadoop/master.pp 2 files changed, 31 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/07/363307/1 diff --git a/modules/role/files/analytics_cluster/hadoop/check_hdfs_topology b/modules/role/files/analytics_cluster/hadoop/check_hdfs_topology new file mode 100644 index 0000000..919d806 --- /dev/null +++ b/modules/role/files/analytics_cluster/hadoop/check_hdfs_topology @@ -0,0 +1,10 @@ +#!/bin/bash + +# The HDFS topology should not contain any host in the 'default' rack. +sudo -u hdfs dfsadmin -printTopology | grep -q 'Rack: default' + +if [ $? -eq 1 ]; then + echo "OK" +else + echo "CRITICAL: There is at least one node in the default rack." +fi \ No newline at end of file diff --git a/modules/role/manifests/analytics_cluster/hadoop/master.pp b/modules/role/manifests/analytics_cluster/hadoop/master.pp index 2ec2da4..4e8cce6 100644 --- a/modules/role/manifests/analytics_cluster/hadoop/master.pp +++ b/modules/role/manifests/analytics_cluster/hadoop/master.pp @@ -46,6 +46,14 @@ require => Class['cdh::hadoop::master'], } + file { '/usr/local/lib/nagios/plugins/check_hdfs_topology': + ensure => present, + source => 'puppet:///modules/role/analytics_cluster/hadoop/check_hdfs_topology', + mode => '0555', + owner => 'root', + group => 'root', + } + # Include icinga alerts if production realm. if $::realm == 'production' { # Icinga process alerts for NameNode, ResourceManager and HistoryServer @@ -76,10 +84,21 @@ require => Class['cdh::hadoop::master'], } - # Allow nagios to run the check_hdfs_active_namenode as hdfs user. + # Allow nagios to run some scripts as hdfs user. sudo::user { 'nagios-check_hdfs_active_namenode': user => 'nagios', - privileges => ['ALL = NOPASSWD: /usr/local/bin/check_hdfs_active_namenode'], + privileges => [ + 'ALL = NOPASSWD: /usr/local/bin/check_hdfs_active_namenode', + 'ALL = NOPASSWD: /usr/local/lib/nagios/plugins/check_hdfs_topology', + ], + } + # Alert if the HDFS topology shows any inconsistency. + nrpe::monitor_service { 'check_hdfs_topology': + description => 'HDFS topology check', + nrpe_command => '/usr/bin/sudo /usr/local/lib/nagios/plugins/check_hdfs_topology', + check_interval => 30, + retries => 2, + require => File['/usr/local/lib/nagios/plugins/check_hdfs_topology'], } # Alert if there is no active NameNode nrpe::monitor_service { 'hadoop-hdfs-active-namenode': -- To view, visit https://gerrit.wikimedia.org/r/363307 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6f1c259eb872abcca60d96add2bfa89a0546a14d Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Elukey <ltosc...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits