Repository: incubator-senssoft Updated Branches: refs/heads/ansible ae3ebdfd2 -> c4b63e90b
Added support for monitoring kafka, zookeeper, and updated config for postgresql to monitor TAPs database. Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/c4b63e90 Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/c4b63e90 Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/c4b63e90 Branch: refs/heads/ansible Commit: c4b63e90b1bda7807490d33b0cde19726831f786 Parents: ae3ebdf Author: msbeard <msbe...@apache.org> Authored: Mon Jun 4 18:10:20 2018 -0400 Committer: msbeard <msbe...@apache.org> Committed: Mon Jun 4 18:10:20 2018 -0400 ---------------------------------------------------------------------- .gitignore | 2 + docker/beats/metricbeat/metricbeat.yml | 2 +- docker/beats/metricbeat/modules.d/kafka.yml | 9 ++++ docker/beats/metricbeat/modules.d/kibana.yml | 2 +- .../beats/metricbeat/modules.d/kubernetes.yml | 47 ++++++++++---------- .../beats/metricbeat/modules.d/postgresql.yml | 22 +++------ docker/beats/metricbeat/modules.d/zookeeper.yml | 6 +++ docker/docker-compose.yml | 8 +++- 8 files changed, 56 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index ba433d6..c8facd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +env +venv .idea release/incubator-* release/apache-* http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/metricbeat.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/metricbeat.yml b/docker/beats/metricbeat/metricbeat.yml index 3c61d77..a987a43 100644 --- a/docker/beats/metricbeat/metricbeat.yml +++ b/docker/beats/metricbeat/metricbeat.yml @@ -4,7 +4,7 @@ metricbeat.config.modules: path: ${path.config}/modules.d/*.yml # Period on which files under path should be checked for changes - reload.period: 10s + reload.period: 30s # Set to true to enable config reloading reload.enabled: true http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kafka.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/modules.d/kafka.yml b/docker/beats/metricbeat/modules.d/kafka.yml new file mode 100644 index 0000000..b1954cc --- /dev/null +++ b/docker/beats/metricbeat/modules.d/kafka.yml @@ -0,0 +1,9 @@ +#---------------------------- Kafka Module --------------------------- +- module: kafka + metricsets: ["consumergroup", "partition"] + period: 10s + hosts: ["${KAFKA_URL}"] + enabled: false + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/modules.d/kibana.yml b/docker/beats/metricbeat/modules.d/kibana.yml index 2134a62..0d83b05 100644 --- a/docker/beats/metricbeat/modules.d/kibana.yml +++ b/docker/beats/metricbeat/modules.d/kibana.yml @@ -1,6 +1,6 @@ #------------------------------- Kibana Module ------------------------------- - module: kibana metricsets: ["status"] - period: 10s + period: 30s enabled: true hosts: ["${KIBANA_URL}"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kubernetes.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/modules.d/kubernetes.yml b/docker/beats/metricbeat/modules.d/kubernetes.yml index b3e018d..f58ea52 100644 --- a/docker/beats/metricbeat/modules.d/kubernetes.yml +++ b/docker/beats/metricbeat/modules.d/kubernetes.yml @@ -1,29 +1,30 @@ #----------------------------- Kubernetes Module ----------------------------- # Node metrics, from kubelet: -#- module: kubernetes -# metricsets: -# - node -# - system -# - pod -# - container -# - volume -# period: 10s -# hosts: ["localhost:10255"] +- module: kubernetes + enabled: false + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] # State metrics from kube-state-metrics service: -#- module: kubernetes -# enabled: false -# metricsets: -# - state_node -# - state_deployment -# - state_replicaset -# - state_pod -# - state_container -# period: 10s -# hosts: ["kube-state-metrics:8080"] +- module: kubernetes + enabled: false + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + period: 10s + hosts: ["kube-state-metrics:8080"] # Kubernetes events -#- module: kubernetes -# enabled: false -# metricsets: -# - event \ No newline at end of file +- module: kubernetes + enabled: false + metricsets: + - event \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/postgresql.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/modules.d/postgresql.yml b/docker/beats/metricbeat/modules.d/postgresql.yml index 2b02360..363fe4b 100644 --- a/docker/beats/metricbeat/modules.d/postgresql.yml +++ b/docker/beats/metricbeat/modules.d/postgresql.yml @@ -1,25 +1,17 @@ #----------------------------- PostgreSQL Module ----------------------------- -#- module: postgresql -# metricsets: -# # Stats about every PostgreSQL database -# - database -# -# # Stats about the background writer process's activity -# - bgwriter -# -# # Stats about every PostgreSQL process -# - activity -# -# period: 10s +- module: postgresql + metricsets: ["database", "activity"] + period: 10s + enabled: true # The host must be passed as PostgreSQL URL. Example: # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters -# hosts: ["postgres://localhost:5432"] + hosts: ["${POSTGRESQL_URL}"] # Username to use when connecting to PostgreSQL. Empty by default. - #username: user + username: "${POSTGRESQL_USERNAME}" # Password to use when connecting to PostgreSQL. Empty by default. - #password: pass \ No newline at end of file + password: "${POSTGRESQL_PASSWORD}" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/zookeeper.yml ---------------------------------------------------------------------- diff --git a/docker/beats/metricbeat/modules.d/zookeeper.yml b/docker/beats/metricbeat/modules.d/zookeeper.yml new file mode 100644 index 0000000..ca67993 --- /dev/null +++ b/docker/beats/metricbeat/modules.d/zookeeper.yml @@ -0,0 +1,6 @@ +#---------------------------- ZooKeeper Module --------------------------- +- module: zookeeper + enabled: false + metricsets: ["mntr"] + period: 10s + hosts: ["${ZOOKEEPER_URL}"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 30507c6..316b7f5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -123,7 +123,7 @@ services: # Specific container and configuration to monitor the HOST (filesystem, memory, processes,...) metricbeat: image: docker.elastic.co/beats/metricbeat:6.2.4 - restart: unless-stopped +# restart: unless-stopped user: root pid: host volumes: @@ -133,7 +133,7 @@ services: - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro #We mount the host filesystem so we can report on disk usage with the system module - /:/hostfs:ro - #Mount the metricbeat configuration so users can make edit + #Mount the metricbeat configuration so users can make edits - ./beats/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml #Mount the modules.d directory into the container. This allows user to #potentially make changes to the modules and they will be dynamically loaded. @@ -146,10 +146,14 @@ services: - ELASTICSEARCH_URL=http://elasticsearch:9200 - KIBANA_URL=http://kibana:5601 - LOGSTASH_URL=http://logstash:9600 + - POSTGRESQL_URL="postgresql://db:5432/tapdb?sslmode=disable" + - POSTGRESQL_USERNAME=tapuser + - POSTGRESQL_PASSWORD="Dr@p3rUs3r" extra_hosts: - "elasticsearch:172.17.0.1" # The IP of docker0 interface to access host from container - "kibana:172.17.0.1" # The IP of docker0 interface to access host from container - "logstash:172.17.0.1" + - "db:172.17.0.1" command: metricbeat -e -system.hostfs=/hostfs -strict.perms=false network_mode: host # Mandatory to monitor HOST filesystem, memory, processes,...