kap |
'##:::'##::::'###::::'########:::::'###:::::'######::'####:'########::'#######::'########::
kap | ##::'##::::'## ##::: ##.... ##:::'## ##:::'##...
##:. ##::... ##..::'##.... ##: ##.... ##:
kap | ##:'##::::'##:. ##:: ##:::: ##::'##:. ##::
##:::..::: ##::::: ##:::: ##:::: ##: ##:::: ##:
kap | #####::::'##:::. ##: ########::'##:::. ##:
##:::::::: ##::::: ##:::: ##:::: ##: ########::
kap | ##. ##::: #########: ##.....::: #########:
##:::::::: ##::::: ##:::: ##:::: ##: ##.. ##:::
kap | ##:. ##:: ##.... ##: ##:::::::: ##.... ##: ##:::
##:: ##::::: ##:::: ##:::: ##: ##::. ##::
kap | ##::. ##: ##:::: ##: ##:::::::: ##:::: ##:.
######::'####:::: ##::::. #######:: ##:::. ##:
kap |
..::::..::..:::::..::..:::::::::..:::::..:::......:::....:::::..::::::.......:::..:::::..::
kap |
kap | 2017/02/22 22:20:21 Using configuration at:
/etc/kapacitor/kapacitor.conf
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "PATCH
/kapacitor/v1/tasks/disk_alert_dead HTTP/1.1" 200 788 "-" "KapacitorClient"
1f51cf6b-f94d-11e6-805d-000000000000 16607
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "GET
/kapacitor/v1/tasks?dot-view=attributes&fields=link&limit=100&offset=0&pattern=disk_alert_warn&replay-id=&script-format=formatted
HTTP/1.1" 200 131 "-" "KapacitorClient"
1f553781-f94d-11e6-805e-000000000000 5815
kap | [task_master:main] 2017/02/22 22:20:31 I! Started
task: disk_alert_warn
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "PATCH
/kapacitor/v1/tasks/disk_alert_warn HTTP/1.1" 200 754 "-" "KapacitorClient"
1f5640c5-f94d-11e6-805f-000000000000 18441
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "GET
/kapacitor/v1/tasks?dot-view=attributes&fields=link&limit=100&offset=0&pattern=http_status_alert_crit&replay-id=&script-format=formatted
HTTP/1.1" 200 137 "-" "KapacitorClient"
1f59f50c-f94d-11e6-8060-000000000000 754
kap | [task_master:main] 2017/02/22 22:20:31 I! Started
task: http_status_alert_crit
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "PATCH
/kapacitor/v1/tasks/http_status_alert_crit HTTP/1.1" 200 1207 "-"
"KapacitorClient" 1f5a3816-f94d-11e6-8061-000000000000 49713
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "GET
/kapacitor/v1/tasks?dot-view=attributes&fields=link&limit=100&offset=0&pattern=http_status_alert_dead&replay-id=&script-format=formatted
HTTP/1.1" 200 137 "-" "KapacitorClient"
1f62be2e-f94d-11e6-8062-000000000000 716
kap | [task_master:main] 2017/02/22 22:20:31 I! Started
task: http_status_alert_dead
kap | [httpd] ::1 - - [22/Feb/2017:22:20:31 +0000] "PATCH
/kapacitor/v1/tasks/http_status_alert_dead HTTP/1.1" 200 729 "-"
"KapacitorClient" 1f62fb2e-f94d-11e6-8063-000000000000 17563
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:31 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 1f90e7a5-f94d-11e6-8064-000000000000 439
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"jira.hostname.xyx","path":"/"},"Fields":{"free":15286812672,"inodes_free":1373362,"inodes_total":1495552,"inodes_used":122190,"total":24210894848,"used":7703310336,"used_percent":33.50704271273119},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/boot","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"sda1","fstype":"ext3","host":"jira.hostname.xyx","path":"/boot"},"Fields":{"free":377835520,"inodes_free":32715,"inodes_total":32768,"inodes_used":53,"total":528424960,"used":123748352,"used_percent":24.671517348946978},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-home","fstype":"ext3","host":"jira.hostname.xyx","path":"/home"},"Fields":{"free":12615372800,"inodes_free":884437,"inodes_total":884736,"inodes_used":299,"total":14268055552,"used":928071680,"used_percent":6.85255277097721},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/tmp","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-temp","fstype":"ext3","host":"jira.hostname.xyx","path":"/tmp"},"Fields":{"free":880750592,"inodes_free":65424,"inodes_total":65536,"inodes_used":112,"total":1056858112,"used":122421248,"used_percent":12.20341751219811},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/var","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-var","fstype":"ext3","host":"jira.hostname.xyx","path":"/var"},"Fields":{"free":26661912576,"inodes_free":4707430,"inodes_total":4792320,"inodes_used":84890,"total":77218316288,"used":46634835968,"used_percent":63.62469945035165},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/var/log","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-varlog","fstype":"ext3","host":"jira.hostname.xyx","path":"/var/log"},"Fields":{"free":206135296,"inodes_free":65441,"inodes_total":65536,"inodes_used":95,"total":1056858112,"used":797036544,"used_percent":79.45164648959843},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:31 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyx,path=/var/log/audit","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-audit","fstype":"ext3","host":"jira.hostname.xyx","path":"/var/log/audit"},"Fields":{"free":1905217536,"inodes_free":131056,"inodes_total":131072,"inodes_used":16,"total":2113748992,"used":101158912,"used_percent":5.041870985917793},"Time":"2017-02-22T22:20:30Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:31 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 1f90ff3b-f94d-11e6-8065-000000000000 381
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:45 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=calendar.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"calendar.hostname.xyz","path":"/"},"Fields":{"free":610889728,"inodes_free":146007,"inodes_total":235712,"inodes_used":89705,"total":3798114304,"used":2994286592,"used_percent":83.05520524444141},"Time":"2017-02-22T22:20:48Z"}
kap |
kap UP | [disk_alert_warn:log2] 2017/02/22 22:20:45 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=calendar.hostname.xyz,path=/boot","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"sda1","fstype":"ext3","host":"calendar.hostname.xyz","path":"/boot"},"Fields":{"free":389566464,"inodes_free":32717,"inodes_total":32768,"inodes_used":51,"total":528424960,"used":112017408,"used_percent":22.332737205713027},"Time":"2017-02-22T22:20:48Z"}
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=mavericks.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/VolGroup01-LogVol02","fstype":"ext3","host":"mavericks.hostname.xyz","path":"/home"},"Fields":{"free":151598505984,"inodes_free":133300363,"inodes_total":133562368,"inodes_used":262005,"total":1076941541376,"used":870637568000,"used_percent":85.16991232825805},"Time":"2017-02-22T22:20:45Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:46 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 280b1ecb-f94d-11e6-80ab-000000000000 265
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=mavericks.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/VolGroup00-LogVol00","fstype":"ext3","host":"mavericks.hostname.xyz","path":"/"},"Fields":{"free":33492471808,"inodes_free":11372969,"inodes_total":11555808,"inodes_used":182839,"total":45829500928,"used":9971441664,"used_percent":22.941886423604558},"Time":"2017-02-22T22:20:45Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=mavericks.hostname.xyz,path=/boot","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"sda1","fstype":"ext3","host":"mavericks.hostname.xyz","path":"/boot"},"Fields":{"free":55656448,"inodes_free":26045,"inodes_total":26104,"inodes_used":59,"total":103512064,"used":42511360,"used_percent":43.304786840101386},"Time":"2017-02-22T22:20:45Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=mavericks.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/VolGroup01-LogVol02","fstype":"ext3","host":"mavericks.hostname.xyz","path":"/home"},"Fields":{"free":151598505984,"inodes_free":133300363,"inodes_total":133562368,"inodes_used":262005,"total":1076941541376,"used":870637568000,"used_percent":85.16991232825805},"Time":"2017-02-22T22:20:45Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:46 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 28399894-f94d-11e6-80ac-000000000000 357
kap UP | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=fusion.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"fusion.hostname.xyz","path":"/"},"Fields":{"free":1118441472,"inodes_free":154449,"inodes_total":245760,"inodes_used":91311,"total":3963305984,"used":2643537920,"used_percent":70.26986712424818},"Time":"2017-02-22T22:20:46Z"}
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=wiki.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vg_wikidev-lv_home","fstype":"ext4","host":"wiki.hostname.xyz","path":"/home"},"Fields":{"free":73774489600,"inodes_free":27882718,"inodes_total":28819456,"inodes_used":936738,"total":464559513600,"used":367198433280,"used_percent":83.27006358617717},"Time":"2017-02-22T22:20:46Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:46 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 284241d0-f94d-11e6-80af-000000000000 352
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=wiki.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vg_wikidev-lv_root","fstype":"ext4","host":"wiki.hostname.xyz","path":"/"},"Fields":{"free":57767124992,"inodes_free":6351092,"inodes_total":6553600,"inodes_used":202508,"total":105555197952,"used":42420019200,"used_percent":42.34078088772119},"Time":"2017-02-22T22:20:46Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=wiki.hostname.xyz,path=/boot","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"sda1","fstype":"ext4","host":"wiki.hostname.xyz","path":"/boot"},"Fields":{"free":362619904,"inodes_free":127963,"inodes_total":128016,"inodes_used":53,"total":499355648,"used":110521344,"used_percent":23.359059153515187},"Time":"2017-02-22T22:20:46Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=wiki.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vg_wikidev-lv_home","fstype":"ext4","host":"wiki.hostname.xyz","path":"/home"},"Fields":{"free":73774489600,"inodes_free":27882718,"inodes_total":28819456,"inodes_used":936738,"total":464559513600,"used":367198433280,"used_percent":83.27006358617717},"Time":"2017-02-22T22:20:46Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:46 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 285f27f4-f94d-11e6-80b0-000000000000 1608
kap UP | [disk_alert_warn:log2] 2017/02/22 22:20:46 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=name-chat.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"rootfs","fstype":"rootfs","host":"name-chat.hostname.xyz","path":"/"},"Fields":{"free":10452328448,"inodes_free":10470655,"inodes_total":10484736,"inodes_used":14081,"total":10725883904,"used":273555456,"used_percent":2.5504234284876333},"Time":"2017-02-22T22:20:47Z"}
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:48 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=spork.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-home","fstype":"ext3","host":"spork.hostname.xyz","path":"/home"},"Fields":{"free":110653030400,"inodes_free":42103075,"inodes_total":43679744,"inodes_used":1576669,"total":704386506752,"used":557976289280,"used_percent":83.45076604583276},"Time":"2017-02-22T22:21:00Z"}
kap |
kap UP | [disk_alert_warn:log2] 2017/02/22 22:20:48 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=spork.hostname.xyz,path=/tmp","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-temp","fstype":"ext3","host":"spork.hostname.xyz","path":"/tmp"},"Fields":{"free":881258496,"inodes_free":65490,"inodes_total":65536,"inodes_used":46,"total":1056858112,"used":121913344,"used_percent":12.152787701855745},"Time":"2017-02-22T22:21:00Z"}
kap |
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:50 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=name-fusion.hostname.xyz,path=/var","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-var","fstype":"ext3","host":"name-fusion.hostname.xyz","path":"/var"},"Fields":{"free":1289412608,"inodes_free":636979,"inodes_total":655360,"inodes_used":18381,"total":10568916992,"used":8742641664,"used_percent":87.14707304167185},"Time":"2017-02-22T22:21:10Z"}
kap |
kap UP | [disk_alert_warn:log2] 2017/02/22 22:20:50 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=name-fusion.hostname.xyz,path=/var/log","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-varlog","fstype":"ext3","host":"name-fusion.hostname.xyz","path":"/var/log"},"Fields":{"free":799748096,"inodes_free":65316,"inodes_total":65536,"inodes_used":220,"total":1056858112,"used":203423744,"used_percent":20.278055651960887},"Time":"2017-02-22T22:21:10Z"}
.
.
.
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=chat.hostname.xyz,path=/rootfs/etc","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/rootfs/etc"},"Fields":{"free":13547388928,"inodes_free":52926628,"inodes_total":53033344,"inodes_used":106716,"total":122295418880,"used":108748029952,"used_percent":88.9224068635857},"Time":"2017-02-22T22:21:00Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=chat.hostname.xyz,path=/etc/resolv.conf","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/etc/resolv.conf"},"Fields":{"free":13547388928,"inodes_free":52926628,"inodes_total":53033344,"inodes_used":106716,"total":122295418880,"used":108748029952,"used_percent":88.9224068635857},"Time":"2017-02-22T22:21:00Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=chat.hostname.xyz,path=/etc/hostname","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/etc/hostname"},"Fields":{"free":13547388928,"inodes_free":52926628,"inodes_total":53033344,"inodes_used":106716,"total":122295418880,"used":108748029952,"used_percent":88.9224068635857},"Time":"2017-02-22T22:21:00Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=chat.hostname.xyz,path=/etc/hosts","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/etc/hosts"},"Fields":{"free":13547388928,"inodes_free":52926628,"inodes_total":53033344,"inodes_used":106716,"total":122295418880,"used":108748029952,"used_percent":88.9224068635857},"Time":"2017-02-22T22:21:00Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=chat.hostname.xyz,path=/etc/telegraf/telegraf.conf","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"fstype":"xfs","host":"chat.hostname.xyz","path":"/etc/telegraf/telegraf.conf"},"Fields":{"free":13547388928,"inodes_free":52926628,"inodes_total":53033344,"inodes_used":106716,"total":122295418880,"used":108748029952,"used_percent":88.9224068635857},"Time":"2017-02-22T22:21:00Z"}
kap |
kap | [httpd] 172.18.0.2 - - [22/Feb/2017:22:20:51 +0000]
"POST /write?consistency=&db=main&precision=ns&rp=autogen HTTP/1.1" 204 0
"-" "InfluxDBClient" 2b7cc1e0-f94d-11e6-80d4-000000000000 470
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-root","fstype":"ext3","host":"jira.hostname.xyz","path":"/"},"Fields":{"free":15286771712,"inodes_free":1373362,"inodes_total":1495552,"inodes_used":122190,"total":24210894848,"used":7703351296,"used_percent":33.50722087619724},"Time":"2017-02-22T22:20:50Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/boot","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"sda1","fstype":"ext3","host":"jira.hostname.xyz","path":"/boot"},"Fields":{"free":377835520,"inodes_free":32715,"inodes_total":32768,"inodes_used":53,"total":528424960,"used":123748352,"used_percent":24.671517348946978},"Time":"2017-02-22T22:20:50Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/home","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-home","fstype":"ext3","host":"jira.hostname.xyz","path":"/home"},"Fields":{"free":12615372800,"inodes_free":884437,"inodes_total":884736,"inodes_used":299,"total":14268055552,"used":928071680,"used_percent":6.85255277097721},"Time":"2017-02-22T22:20:50Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/tmp","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-temp","fstype":"ext3","host":"jira.hostname.xyz","path":"/tmp"},"Fields":{"free":880750592,"inodes_free":65424,"inodes_total":65536,"inodes_used":112,"total":1056858112,"used":122421248,"used_percent":12.20341751219811},"Time":"2017-02-22T22:20:50Z"}
kap |
kap | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/var","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-var","fstype":"ext3","host":"jira.hostname.xyz","path":"/var"},"Fields":{"free":26661879808,"inodes_free":4707430,"inodes_total":4792320,"inodes_used":84890,"total":77218316288,"used":46634868736,"used_percent":63.62474415629107},"Time":"2017-02-22T22:20:50Z"}
kap |
kap DOWN | [disk_alert_warn:log2] 2017/02/22 22:20:51 I! LOGGER
{"Name":"disk","Database":"main","RetentionPolicy":"autogen","Group":"host=jira.hostname.xyz,path=/var/log","Dimensions":{"ByName":false,"TagNames":["host","path"]},"Tags":{"device":"mapper/vgroup1-varlog","fstype":"ext3","host":"jira.hostname.xyz","path":"/var/log"},"Fields":{"free":206041088,"inodes_free":65441,"inodes_total":65536,"inodes_used":95,"total":1056858112,"used":797130752,"used_percent":79.4610375028071},"Time":"2017-02-22T22:20:50Z"}
Here is my log. I edited down to just the entries that fired alerts as
there was a lot to sift through. I marked which entries fired the DOWN
alert and the lines that triggered recoveries. The last two alerts fired
DOWN but never fired recoveries.
On Wednesday, February 22, 2017 at 11:33:57 AM UTC-8, [email protected]
wrote:
>
> Hmm, I see. Could you put a log node before the alert node and share the
> logs along with the logs for triggered alerts after startup?
>
> Also this may be a bug with the most recent alerting system. Do you get
> the same behavior if you configure slack directly in the TICKscript instead
> of via the topic handler?
>
> On Wednesday, February 22, 2017 at 12:23:53 PM UTC-7, Archie Archbold
> wrote:
>>
>> Thanks so much for the reply. I do want the recovery alerts, but the
>> problem is that when I start kapacitor, the task sees *any* server/path
>> in an up status as a recovery of a *different* server/path's down
>> status. So if 7 server/paths are in a down status at the time of start-up I
>> get 7 down alerts (expected) but they are immediately followed by 7
>> recovery messages from different server/paths. Please let me know if I am
>> not being clear enough.
>>
>> On Wednesday, February 22, 2017 at 11:10:09 AM UTC-8,
>> [email protected] wrote:
>>>
>>> If you want to ignore the OK alerts use the `.noRecoveries` property of
>>> the alert node. This will suppress the OK alerts.
>>>
>>> On Friday, February 17, 2017 at 3:33:16 PM UTC-7, Archie Archbold wrote:
>>>>
>>>> Hey all. Pretty new to TICK but I have a problem that I can't wrap my
>>>> head around.
>>>>
>>>> I am monitoring multiple servers all sending data to one influxdb
>>>> database and using the 'host' tag to separate the servers in the DB
>>>>
>>>> My 'disk' measurement is taking in mulitiple disk paths from the
>>>> servers (HOSTS) which each have a respective 'PATH' tag.
>>>>
>>>> So basically each server is assigned a HOST tag and each HOST has
>>>> multiple PATH tags.
>>>>
>>>> EXPECTED FUNCTIONALITY: kapacitor should alert upon state change of a
>>>> HOST's PATH if that path is within the alerting Lambda.
>>>> PROBLEM: When I start the kapacitor service, it looks like it's sensing
>>>> a state change any time it sees another host/path with a opposite status.
>>>>
>>>> This is a simplified example of the alerts I am getting:
>>>>
>>>> Host: host1 Path: /path1 Status: UP
>>>> Host: host1 Path: /path2 Status: DOWN
>>>> Host: host1 Path: /path3 Status: UP
>>>> Host: host2 Path: /path1 Status: DOWN
>>>> Host: host2 Path: /path2 Status: UP
>>>>
>>>> These alerts happen once for each host/path combination and then the
>>>> service performs as expected, alerting properly when lambda is achieved.
>>>>
>>>> The result of this is that I receive a slew of up/down alerts every
>>>> time I restart the kapacitor service
>>>>
>>>> Here is my current tick:
>>>> var data = stream
>>>> |from()
>>>> .measurement('disk')
>>>> .groupBy('host','path')
>>>> |alert()
>>>> .message('{{ .ID }} Server:{{ index .Tags "host" }} Path: {{
>>>> index .Tags "path" }} USED PERCENT: {{ index .Fields "used_percent" }}')
>>>> .warn(lambda: "used_percent" >= 80)
>>>> .id('DISK SPACE WARNING')
>>>> .email($DISK_WARN_GRP)
>>>>
>>>> And the corresponding DOT
>>>>
>>>> ID: disk_alert_warn
>>>>
>>>> Error:
>>>>
>>>> Template:
>>>>
>>>> Type: stream
>>>>
>>>> Status: enabled
>>>>
>>>> Executing: true
>>>>
>>>> Created: 17 Feb 17 22:27 UTC
>>>>
>>>> Modified: 17 Feb 17 22:27 UTC
>>>>
>>>> LastEnabled: 17 Feb 17 22:27 UTC
>>>>
>>>> Databases Retention Policies: ["main"."autogen"]
>>>>
>>>> TICKscript:
>>>>
>>>> var data = stream
>>>>
>>>> |from()
>>>>
>>>> .measurement('disk')
>>>>
>>>> .groupBy('host', 'path')
>>>>
>>>> |alert()
>>>>
>>>> .message('{{ .ID }} Server:{{ index .Tags "host" }} Path: {{
>>>> index .Tags "path" }} USED PERCENT: {{ index .Fields "used_percent" }}')
>>>>
>>>> .warn(lambda: "used_percent" >= 80)
>>>>
>>>> .id('DISK SPACE WARNING')
>>>>
>>>> .email()
>>>>
>>>>
>>>> DOT:
>>>>
>>>> digraph disk_alert_warn {
>>>>
>>>> graph [throughput="38.00 points/s"];
>>>>
>>>>
>>>> stream0 [avg_exec_time_ns="0s" ];
>>>>
>>>> stream0 -> from1 [processed="284"];
>>>>
>>>>
>>>> from1 [avg_exec_time_ns="3.9µs" ];
>>>>
>>>> from1 -> alert2 [processed="284"];
>>>>
>>>>
>>>> alert2 [alerts_triggered="14" avg_exec_time_ns="72.33µs"
>>>> crits_triggered="0" infos_triggered="0" oks_triggered="7"
>>>> warns_triggered="7" ];
>>>>
>>>> }
>>>>
>>>> As you can see, I get 7 oks triggered (for host/path groups that are
>>>> not in alert range) and 7 warns triggered (for the 7 host/path groups that
>>>> are within the alert range) upon start up.
>>>> Then it behaves as normal.
>>>>
>>>> I understand that it should be alerting for the 7 host/path groups that
>>>> are over 80 but why follow it with an alert about the ok groups?
>>>>
>>>> MORE INFO: When I raise the lambda to 90% (out of range for all
>>>> host/paths) I get no alerts at all (which is expected)
>>>>
>>>> Thanks to anyone who can help me understand this
>>>>
>>>
--
Remember to include the version number!
---
You received this message because you are subscribed to the Google Groups
"InfluxData" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at https://groups.google.com/group/influxdb.
To view this discussion on the web visit
https://groups.google.com/d/msgid/influxdb/a39273dd-fed9-456e-b70f-674022f003ad%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
