Here are the logs for the triggered alerts:
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:mavericks.hostname.xyz Path: /home USED PERCENT:
85.16991232825805","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=mavericks.hostname.xyz,path=/home\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/VolGroup01-LogVol02\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;mavericks.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/home\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:151598505984,\u0026#34;inodes_free\u0026#34;:133300363,\u0026#34;inodes_total\u0026#34;:133562368,\u0026#34;inodes_used\u0026#34;:262005,\u0026#34;total\u0026#34;:1076941541376,\u0026#34;used\u0026#34;:870637568000,\u0026#34;used_percent\u0026#34;:85.16991232825805},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:05Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:mavericks.hostname.xyz Path: /home USED PERCENT:
85.16991232825805\u0026#34;}\n","time":"2017-02-22T23:27:05Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/VolGroup01-LogVol02","fstype":"ext3","host":"mavericks.hostname.xyz","path":"/home"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:05Z",151598505984,133300363,133562368,262005,1076941541376,870637568000,85.16991232825805]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:fusion.hostname.xyz Path: / USED PERCENT:
70.27302461097585","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=fusion.hostname.xyz,path=/\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-root\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;fusion.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:1118322688,\u0026#34;inodes_free\u0026#34;:154449,\u0026#34;inodes_total\u0026#34;:245760,\u0026#34;inodes_used\u0026#34;:91311,\u0026#34;total\u0026#34;:3963305984,\u0026#34;used\u0026#34;:2643656704,\u0026#34;used_percent\u0026#34;:70.27302461097585},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:06Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:fusion.hostname.xyz Path: / USED PERCENT:
70.27302461097585\u0026#34;}\n","time":"2017-02-22T23:27:06Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"fusion.hostname.xyz","path":"/"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:06Z",1118322688,154449,245760,91311,3963305984,2643656704,70.27302461097585]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:wiki.hostname.xyz Path: /home USED PERCENT:
83.31287638084197","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=wiki.hostname.xyz,path=/home\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vg_wikidev-lv_home\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext4\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;wiki.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/home\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:73585696768,\u0026#34;inodes_free\u0026#34;:27882405,\u0026#34;inodes_total\u0026#34;:28819456,\u0026#34;inodes_used\u0026#34;:937051,\u0026#34;total\u0026#34;:464559513600,\u0026#34;used\u0026#34;:367387226112,\u0026#34;used_percent\u0026#34;:83.31287638084197},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:06Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:wiki.hostname.xyz Path: /home USED PERCENT:
83.31287638084197\u0026#34;}\n","time":"2017-02-22T23:27:06Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vg_wikidev-lv_home","fstype":"ext4","host":"wiki.hostname.xyz","path":"/home"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:06Z",73585696768,27882405,28819456,937051,464559513600,367387226112,83.31287638084197]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:name-chat.hostname.xyz Path: / USED PERCENT:
2.5504234284876333","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=name-chat.hostname.xyz,path=/\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;rootfs\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;rootfs\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;name-chat.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:10452328448,\u0026#34;inodes_free\u0026#34;:10470655,\u0026#34;inodes_total\u0026#34;:10484736,\u0026#34;inodes_used\u0026#34;:14081,\u0026#34;total\u0026#34;:10725883904,\u0026#34;used\u0026#34;:273555456,\u0026#34;used_percent\u0026#34;:2.5504234284876333},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:07Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:name-chat.hostname.xyz Path: / USED PERCENT:
2.5504234284876333\u0026#34;}\n","time":"2017-02-22T23:27:07Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"rootfs","fstype":"rootfs","host":"name-chat.hostname.xyz","path":"/"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:07Z",10452328448,10470655,10484736,14081,10725883904,273555456,2.5504234284876333]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:spork.hostname.xyz Path: /home USED PERCENT:
83.46179768411558","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=spork.hostname.xyz,path=/home\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-home\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;spork.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/home\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:110579269632,\u0026#34;inodes_free\u0026#34;:42103018,\u0026#34;inodes_total\u0026#34;:43679744,\u0026#34;inodes_used\u0026#34;:1576726,\u0026#34;total\u0026#34;:704386506752,\u0026#34;used\u0026#34;:558050050048,\u0026#34;used_percent\u0026#34;:83.46179768411558},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:20Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:spork.hostname.xyz Path: /home USED PERCENT:
83.46179768411558\u0026#34;}\n","time":"2017-02-22T23:27:20Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-home","fstype":"ext3","host":"spork.hostname.xyz","path":"/home"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:20Z",110579269632,42103018,43679744,1576726,704386506752,558050050048,83.46179768411558]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:spork.hostname.xyz Path: /tmp USED PERCENT:
12.152787701855745","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=spork.hostname.xyz,path=/tmp\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-temp\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;spork.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/tmp\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:881258496,\u0026#34;inodes_free\u0026#34;:65490,\u0026#34;inodes_total\u0026#34;:65536,\u0026#34;inodes_used\u0026#34;:46,\u0026#34;total\u0026#34;:1056858112,\u0026#34;used\u0026#34;:121913344,\u0026#34;used_percent\u0026#34;:12.152787701855745},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:20Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:spork.hostname.xyz Path: /tmp USED PERCENT:
12.152787701855745\u0026#34;}\n","time":"2017-02-22T23:27:20Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-temp","fstype":"ext3","host":"spork.hostname.xyz","path":"/tmp"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:20Z",881258496,65490,65536,46,1056858112,121913344,12.152787701855745]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:name-fusion.hostname.xyz Path: /var USED PERCENT:
87.1589951462336","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=name-fusion.hostname.xyz,path=/var\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-var\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;name-fusion.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/var\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:1288216576,\u0026#34;inodes_free\u0026#34;:636548,\u0026#34;inodes_total\u0026#34;:655360,\u0026#34;inodes_used\u0026#34;:18812,\u0026#34;total\u0026#34;:10568916992,\u0026#34;used\u0026#34;:8743837696,\u0026#34;used_percent\u0026#34;:87.1589951462336},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:30Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:name-fusion.hostname.xyz Path: /var USED PERCENT:
87.1589951462336\u0026#34;}\n","time":"2017-02-22T23:27:30Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-var","fstype":"ext3","host":"name-fusion.hostname.xyz","path":"/var"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:30Z",1288216576,636548,655360,18812,10568916992,8743837696,87.1589951462336]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:name-fusion.hostname.xyz Path: /var/log USED PERCENT:
20.560194353142926","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=name-fusion.hostname.xyz,path=/var/log\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-varlog\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;name-fusion.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/var/log\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:796917760,\u0026#34;inodes_free\u0026#34;:65316,\u0026#34;inodes_total\u0026#34;:65536,\u0026#34;inodes_used\u0026#34;:220,\u0026#34;total\u0026#34;:1056858112,\u0026#34;used\u0026#34;:206254080,\u0026#34;used_percent\u0026#34;:20.560194353142926},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:30Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:name-fusion.hostname.xyz Path: /var/log USED PERCENT:
20.560194353142926\u0026#34;}\n","time":"2017-02-22T23:27:30Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-varlog","fstype":"ext3","host":"name-fusion.hostname.xyz","path":"/var/log"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:30Z",796917760,65316,65536,220,1056858112,206254080,20.560194353142926]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:chat.hostname.xyz Path: /rootfs/etc USED PERCENT:
88.9189638171997","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=chat.hostname.xyz,path=/rootfs/etc\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;fstype\u0026#34;:\u0026#34;xfs\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;chat.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/rootfs/etc\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:13551599616,\u0026#34;inodes_free\u0026#34;:52943076,\u0026#34;inodes_total\u0026#34;:53049792,\u0026#34;inodes_used\u0026#34;:106716,\u0026#34;total\u0026#34;:122295418880,\u0026#34;used\u0026#34;:108743819264,\u0026#34;used_percent\u0026#34;:88.9189638171997},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:20Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:chat.hostname.xyz Path: /rootfs/etc USED PERCENT:
88.9189638171997\u0026#34;}\n","time":"2017-02-22T23:27:20Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/rootfs/etc"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:20Z",13551599616,52943076,53049792,106716,122295418880,108743819264,88.9189638171997]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:jira.hostname.xyz Path: / USED PERCENT:
33.5491961887984","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=jira.hostname.xyz,path=/\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-root\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;jira.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:15277121536,\u0026#34;inodes_free\u0026#34;:1373361,\u0026#34;inodes_total\u0026#34;:1495552,\u0026#34;inodes_used\u0026#34;:122191,\u0026#34;total\u0026#34;:24210894848,\u0026#34;used\u0026#34;:7713001472,\u0026#34;used_percent\u0026#34;:33.5491961887984},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:10Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:jira.hostname.xyz Path: / USED PERCENT:
33.5491961887984\u0026#34;}\n","time":"2017-02-22T23:27:10Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"jira.hostname.xyz","path":"/"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:10Z",15277121536,1373361,1495552,122191,24210894848,7713001472,33.5491961887984]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:jira.hostname.xyz Path: /var/log USED PERCENT:
82.31917195761795","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=jira.hostname.xyz,path=/var/log\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-varlog\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;jira.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/var/log\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:177369088,\u0026#34;inodes_free\u0026#34;:65441,\u0026#34;inodes_total\u0026#34;:65536,\u0026#34;inodes_used\u0026#34;:95,\u0026#34;total\u0026#34;:1056858112,\u0026#34;used\u0026#34;:825802752,\u0026#34;used_percent\u0026#34;:82.31917195761795},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:10Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:jira.hostname.xyz Path: /var/log USED PERCENT:
82.31917195761795\u0026#34;}\n","time":"2017-02-22T23:27:10Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-varlog","fstype":"ext3","host":"jira.hostname.xyz","path":"/var/log"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:10Z",177369088,65441,65536,95,1056858112,825802752,82.31917195761795]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:jira.hostname.xyz Path: /var/log/audit USED PERCENT:
5.053099187894773","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=jira.hostname.xyz,path=/var/log/audit\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-audit\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;jira.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/var/log/audit\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:1904992256,\u0026#34;inodes_free\u0026#34;:131056,\u0026#34;inodes_total\u0026#34;:131072,\u0026#34;inodes_used\u0026#34;:16,\u0026#34;total\u0026#34;:2113748992,\u0026#34;used\u0026#34;:101384192,\u0026#34;used_percent\u0026#34;:5.053099187894773},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:10Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:jira.hostname.xyz Path: /var/log/audit USED PERCENT:
5.053099187894773\u0026#34;}\n","time":"2017-02-22T23:27:10Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-audit","fstype":"ext3","host":"jira.hostname.xyz","path":"/var/log/audit"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:10Z",1904992256,131056,131072,16,2113748992,101384192,5.053099187894773]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:calendar.hostname.xyz Path: / USED PERCENT:
83.05497801561063","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=calendar.hostname.xyz,path=/\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;mapper/vgroup1-root\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;calendar.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:610897920,\u0026#34;inodes_free\u0026#34;:146007,\u0026#34;inodes_total\u0026#34;:235712,\u0026#34;inodes_used\u0026#34;:89705,\u0026#34;total\u0026#34;:3798114304,\u0026#34;used\u0026#34;:2994278400,\u0026#34;used_percent\u0026#34;:83.05497801561063},\u0026#34;Level\u0026#34;:\u0026#34;WARNING\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:18Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:calendar.hostname.xyz Path: / USED PERCENT:
83.05497801561063\u0026#34;}\n","time":"2017-02-22T23:27:18Z","duration":0,"level":"WARNING","data":{"Series":[{"name":"disk","tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"calendar.hostname.xyz","path":"/"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:18Z",610897920,146007,235712,89705,3798114304,2994278400,83.05497801561063]]}],"Messages":null,"Err":null}}
{"id":"DISK SPACE WARNING","message":"DISK SPACE WARNING
Server:calendar.hostname.xyz Path: /boot USED PERCENT:
22.332737205713027","details":"{\u0026#34;Name\u0026#34;:\u0026#34;disk\u0026#34;,\u0026#34;TaskName\u0026#34;:\u0026#34;disk_alert_warn\u0026#34;,\u0026#34;Group\u0026#34;:\u0026#34;host=calendar.hostname.xyz,path=/boot\u0026#34;,\u0026#34;Tags\u0026#34;:{\u0026#34;device\u0026#34;:\u0026#34;sda1\u0026#34;,\u0026#34;fstype\u0026#34;:\u0026#34;ext3\u0026#34;,\u0026#34;host\u0026#34;:\u0026#34;calendar.hostname.xyz\u0026#34;,\u0026#34;path\u0026#34;:\u0026#34;/boot\u0026#34;},\u0026#34;ID\u0026#34;:\u0026#34;DISK
SPACE
WARNING\u0026#34;,\u0026#34;Fields\u0026#34;:{\u0026#34;free\u0026#34;:389566464,\u0026#34;inodes_free\u0026#34;:32717,\u0026#34;inodes_total\u0026#34;:32768,\u0026#34;inodes_used\u0026#34;:51,\u0026#34;total\u0026#34;:528424960,\u0026#34;used\u0026#34;:112017408,\u0026#34;used_percent\u0026#34;:22.332737205713027},\u0026#34;Level\u0026#34;:\u0026#34;OK\u0026#34;,\u0026#34;Time\u0026#34;:\u0026#34;2017-02-22T23:27:18Z\u0026#34;,\u0026#34;Message\u0026#34;:\u0026#34;DISK
SPACE WARNING Server:calendar.hostname.xyz Path: /boot USED PERCENT:
22.332737205713027\u0026#34;}\n","time":"2017-02-22T23:27:18Z","duration":9223372036854775807,"level":"OK","data":{"Series":[{"name":"disk","tags":{"device":"sda1","fstype":"ext3","host":"calendar.hostname.xyz","path":"/boot"},"columns":["time","free","inodes_free","inodes_total","inodes_used","total","used","used_percent"],"values":[["2017-02-22T23:27:18Z",389566464,32717,32768,51,528424960,112017408,22.332737205713027]]}],"Messages":null,"Err":null}}
On Wednesday, February 22, 2017 at 11:33:57 AM UTC-8, [email protected]
wrote:
>
> Hmm, I see. Could you put a log node before the alert node and share the
> logs along with the logs for triggered alerts after startup?
>
> Also this may be a bug with the most recent alerting system. Do you get
> the same behavior if you configure slack directly in the TICKscript instead
> of via the topic handler?
>
> On Wednesday, February 22, 2017 at 12:23:53 PM UTC-7, Archie Archbold
> wrote:
>>
>> Thanks so much for the reply. I do want the recovery alerts, but the
>> problem is that when I start kapacitor, the task sees *any* server/path
>> in an up status as a recovery of a *different* server/path's down
>> status. So if 7 server/paths are in a down status at the time of start-up I
>> get 7 down alerts (expected) but they are immediately followed by 7
>> recovery messages from different server/paths. Please let me know if I am
>> not being clear enough.
>>
>> On Wednesday, February 22, 2017 at 11:10:09 AM UTC-8,
>> [email protected] wrote:
>>>
>>> If you want to ignore the OK alerts use the `.noRecoveries` property of
>>> the alert node. This will suppress the OK alerts.
>>>
>>> On Friday, February 17, 2017 at 3:33:16 PM UTC-7, Archie Archbold wrote:
>>>>
>>>> Hey all. Pretty new to TICK but I have a problem that I can't wrap my
>>>> head around.
>>>>
>>>> I am monitoring multiple servers all sending data to one influxdb
>>>> database and using the 'host' tag to separate the servers in the DB
>>>>
>>>> My 'disk' measurement is taking in mulitiple disk paths from the
>>>> servers (HOSTS) which each have a respective 'PATH' tag.
>>>>
>>>> So basically each server is assigned a HOST tag and each HOST has
>>>> multiple PATH tags.
>>>>
>>>> EXPECTED FUNCTIONALITY: kapacitor should alert upon state change of a
>>>> HOST's PATH if that path is within the alerting Lambda.
>>>> PROBLEM: When I start the kapacitor service, it looks like it's sensing
>>>> a state change any time it sees another host/path with a opposite status.
>>>>
>>>> This is a simplified example of the alerts I am getting:
>>>>
>>>> Host: host1 Path: /path1 Status: UP
>>>> Host: host1 Path: /path2 Status: DOWN
>>>> Host: host1 Path: /path3 Status: UP
>>>> Host: host2 Path: /path1 Status: DOWN
>>>> Host: host2 Path: /path2 Status: UP
>>>>
>>>> These alerts happen once for each host/path combination and then the
>>>> service performs as expected, alerting properly when lambda is achieved.
>>>>
>>>> The result of this is that I receive a slew of up/down alerts every
>>>> time I restart the kapacitor service
>>>>
>>>> Here is my current tick:
>>>> var data = stream
>>>> |from()
>>>> .measurement('disk')
>>>> .groupBy('host','path')
>>>> |alert()
>>>> .message('{{ .ID }} Server:{{ index .Tags "host" }} Path: {{
>>>> index .Tags "path" }} USED PERCENT: {{ index .Fields "used_percent" }}')
>>>> .warn(lambda: "used_percent" >= 80)
>>>> .id('DISK SPACE WARNING')
>>>> .email($DISK_WARN_GRP)
>>>>
>>>> And the corresponding DOT
>>>>
>>>> ID: disk_alert_warn
>>>>
>>>> Error:
>>>>
>>>> Template:
>>>>
>>>> Type: stream
>>>>
>>>> Status: enabled
>>>>
>>>> Executing: true
>>>>
>>>> Created: 17 Feb 17 22:27 UTC
>>>>
>>>> Modified: 17 Feb 17 22:27 UTC
>>>>
>>>> LastEnabled: 17 Feb 17 22:27 UTC
>>>>
>>>> Databases Retention Policies: ["main"."autogen"]
>>>>
>>>> TICKscript:
>>>>
>>>> var data = stream
>>>>
>>>> |from()
>>>>
>>>> .measurement('disk')
>>>>
>>>> .groupBy('host', 'path')
>>>>
>>>> |alert()
>>>>
>>>> .message('{{ .ID }} Server:{{ index .Tags "host" }} Path: {{
>>>> index .Tags "path" }} USED PERCENT: {{ index .Fields "used_percent" }}')
>>>>
>>>> .warn(lambda: "used_percent" >= 80)
>>>>
>>>> .id('DISK SPACE WARNING')
>>>>
>>>> .email()
>>>>
>>>>
>>>> DOT:
>>>>
>>>> digraph disk_alert_warn {
>>>>
>>>> graph [throughput="38.00 points/s"];
>>>>
>>>>
>>>> stream0 [avg_exec_time_ns="0s" ];
>>>>
>>>> stream0 -> from1 [processed="284"];
>>>>
>>>>
>>>> from1 [avg_exec_time_ns="3.9µs" ];
>>>>
>>>> from1 -> alert2 [processed="284"];
>>>>
>>>>
>>>> alert2 [alerts_triggered="14" avg_exec_time_ns="72.33µs"
>>>> crits_triggered="0" infos_triggered="0" oks_triggered="7"
>>>> warns_triggered="7" ];
>>>>
>>>> }
>>>>
>>>> As you can see, I get 7 oks triggered (for host/path groups that are
>>>> not in alert range) and 7 warns triggered (for the 7 host/path groups that
>>>> are within the alert range) upon start up.
>>>> Then it behaves as normal.
>>>>
>>>> I understand that it should be alerting for the 7 host/path groups that
>>>> are over 80 but why follow it with an alert about the ok groups?
>>>>
>>>> MORE INFO: When I raise the lambda to 90% (out of range for all
>>>> host/paths) I get no alerts at all (which is expected)
>>>>
>>>> Thanks to anyone who can help me understand this
>>>>
>>>
--
Remember to include the version number!
---
You received this message because you are subscribed to the Google Groups
"InfluxData" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at https://groups.google.com/group/influxdb.
To view this discussion on the web visit
https://groups.google.com/d/msgid/influxdb/9a5b3da9-7a58-4ed2-a71f-7b21806dded2%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
