This is an automated email from the ASF dual-hosted git repository. wusheng pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/skywalking-showcase.git
The following commit(s) were added to refs/heads/main by this push: new e56df1c MongoDB monitoring showcase (#144) e56df1c is described below commit e56df1c009bdabe8817f45103ebda4138cbd2831 Author: weixiang1862 <652048...@qq.com> AuthorDate: Sat Jul 22 23:01:17 2023 +0800 MongoDB monitoring showcase (#144) --- Makefile.in | 4 +- .../config/mongodb/otel-collector-config.yaml | 47 +++++++++++ deploy/platform/docker/config/mongodb/setup.sh | 46 +++++++++++ deploy/platform/docker/docker-compose.cluster.yaml | 2 +- .../docker/docker-compose.mongodb-monitor.yaml | 95 ++++++++++++++++++++++ .../docker/docker-compose.single-node.yaml | 2 +- deploy/platform/kubernetes/Chart.yaml | 5 ++ deploy/platform/kubernetes/Makefile | 6 ++ deploy/platform/kubernetes/Makefile.in | 2 +- .../feature-mongodb/opentelemetry-config.yaml} | 25 ++++-- .../templates/otel-collector-config.yaml | 1 + deploy/platform/kubernetes/values.yaml | 21 ++++- docs/readme.md | 1 + 13 files changed, 245 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index 4d823ae..b578d17 100644 --- a/Makefile.in +++ b/Makefile.in @@ -29,10 +29,10 @@ ES_IMAGE ?= docker.elastic.co/elasticsearch/elasticsearch-oss ES_IMAGE_TAG ?= 7.10.2 SW_OAP_IMAGE ?= ghcr.io/apache/skywalking/oap -SW_OAP_IMAGE_TAG ?= 8b2fe884152d7bb03e0a8ac0753fdc84df541ce9 +SW_OAP_IMAGE_TAG ?= e0068154c1253500720651dca5c80c3ae10ca87b SW_UI_IMAGE ?= ghcr.io/apache/skywalking/ui -SW_UI_IMAGE_TAG ?= 8b2fe884152d7bb03e0a8ac0753fdc84df541ce9 +SW_UI_IMAGE_TAG ?= e0068154c1253500720651dca5c80c3ae10ca87b SW_CLI_IMAGE ?= ghcr.io/apache/skywalking-cli/skywalking-cli:0883266bfaa36612927b69e35781b64ea181758d SW_EVENT_EXPORTER_IMAGE ?= ghcr.io/apache/skywalking-kubernetes-event-exporter/skywalking-kubernetes-event-exporter:8a012a3f968cb139f817189afb9b3748841bba22 diff --git a/deploy/platform/docker/config/mongodb/otel-collector-config.yaml b/deploy/platform/docker/config/mongodb/otel-collector-config.yaml new file mode 100644 index 0000000..a56a0e7 --- /dev/null +++ b/deploy/platform/docker/config/mongodb/otel-collector-config.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +receivers: + prometheus: + config: + scrape_configs: + - job_name: 'mongodb-monitoring' + metrics_path: "/metrics" + scrape_interval: 10s + static_configs: + - targets: ['mongodb-exporter-1:9216','mongodb-exporter-2:9216'] + labels: + cluster: replset + +processors: + batch: + +exporters: + #################################################################################### + # If you want to use otlp exporter please ensure that your OAP version is >= 9.2.0 # + #################################################################################### + otlp: + endpoint: oap:11800 + tls: + insecure: true +service: + pipelines: + metrics: + receivers: + - prometheus + processors: + - batch + exporters: + - otlp diff --git a/deploy/platform/docker/config/mongodb/setup.sh b/deploy/platform/docker/config/mongodb/setup.sh new file mode 100644 index 0000000..ba9a5d8 --- /dev/null +++ b/deploy/platform/docker/config/mongodb/setup.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Waiting for startup.." +until mongosh --host mongodb-1:27017 --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do + printf '.' + sleep 1 +done +until mongosh --host mongodb-2:27017 --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do + printf '.' + sleep 1 +done + +echo "Started.." + +mongosh --host mongodb-1:27017 <<EOF + var cfg = { + "_id": "rs1", + "protocolVersion": 1, + "members": [ + { + "_id": 0, + "host": "mongodb-1:27017" + }, + { + "_id": 1, + "host": "mongodb-2:27017" + } + ] + }; + rs.initiate(cfg, { force: true }); + rs.reconfig(cfg, { force: true }); +EOF \ No newline at end of file diff --git a/deploy/platform/docker/docker-compose.cluster.yaml b/deploy/platform/docker/docker-compose.cluster.yaml index 6bf0cbd..856d83c 100644 --- a/deploy/platform/docker/docker-compose.cluster.yaml +++ b/deploy/platform/docker/docker-compose.cluster.yaml @@ -53,7 +53,7 @@ services: SW_CLUSTER_ZK_HOST_PORT: zookeeper:2181 # @feature: cluster; set up the cluster coordinator address SW_HEALTH_CHECKER: default # @feature: health-check; SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics - SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/* # @feature: vm; enable the OC rules that analyse the VM metrics + SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/* # @feature: vm; enable the OC rules that analyse the VM metrics SW_STORAGE: elasticsearch SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200 SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus diff --git a/deploy/platform/docker/docker-compose.mongodb-monitor.yaml b/deploy/platform/docker/docker-compose.mongodb-monitor.yaml new file mode 100644 index 0000000..640c9f3 --- /dev/null +++ b/deploy/platform/docker/docker-compose.mongodb-monitor.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# @feature: mongodb; set up mongodb_exporter and OpenTelemetry collector + +version: '2.1' + +services: + mongodb-1: &mongodb + image: mongo:6.0.8 + networks: + - sw + command: mongod --replSet rs1 --shardsvr --port 27017 + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/27017" ] + interval: 10s + timeout: 10s + retries: 120 + mongodb-2: *mongodb + mongodb-repl-setup: + image: mongo:6.0.8 + networks: + - sw + depends_on: + mongodb-1: + condition: service_healthy + mongodb-2: + condition: service_healthy + volumes: + - ./config/mongodb/setup.sh:/setup.sh + entrypoint: [ "bash", "/setup.sh" ] + mongodb-exporter-1: + image: percona/mongodb_exporter:0.39 + expose: + - 9216 + networks: + - sw + depends_on: + mongodb-1: + condition: service_healthy + command: + - '--mongodb.uri=mongodb://mongodb-1:27017' + - '--discovering-mode' + - '--compatible-mode' + - '--collector.dbstats' + - '--collector.diagnosticdata' + - '--collector.replicasetstatus' + mongodb-exporter-2: + image: percona/mongodb_exporter:0.39 + expose: + - 9216 + networks: + - sw + depends_on: + mongodb-2: + condition: service_healthy + command: + - '--mongodb.uri=mongodb://mongodb-2:27017' + - '--discovering-mode' + - '--compatible-mode' + - '--collector.dbstats' + - '--collector.diagnosticdata' + - '--collector.replicasetstatus' + otel-collector: + image: otel/opentelemetry-collector:0.72.0 + networks: + - sw + command: [ "--config=/etc/otel-collector-config.yaml" ] + volumes: + - ./config/mongodb/otel-collector-config.yaml:/etc/otel-collector-config.yaml + expose: + - 55678 + depends_on: + mongodb-exporter-1: + condition: service_started + mongodb-exporter-2: + condition: service_started + +networks: + sw: diff --git a/deploy/platform/docker/docker-compose.single-node.yaml b/deploy/platform/docker/docker-compose.single-node.yaml index caabd2d..8ffc978 100644 --- a/deploy/platform/docker/docker-compose.single-node.yaml +++ b/deploy/platform/docker/docker-compose.single-node.yaml @@ -43,7 +43,7 @@ services: environment: SW_HEALTH_CHECKER: default # @feature: health-check; SW_OTEL_RECEIVER: default # @feature: vm; enable the OC receiver that receives the VM metrics - SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics + SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,mysql/*,postgresql/*,elasticsearch/*,rabbitmq/*,mongodb/* # @feature: vm,mysql,postgresql,elasticsearch; enable the OC rules that analyse the metrics SW_STORAGE: elasticsearch SW_STORAGE_ES_CLUSTER_NODES: elasticsearch:9200 SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus diff --git a/deploy/platform/kubernetes/Chart.yaml b/deploy/platform/kubernetes/Chart.yaml index bae834d..24b2847 100644 --- a/deploy/platform/kubernetes/Chart.yaml +++ b/deploy/platform/kubernetes/Chart.yaml @@ -55,6 +55,11 @@ dependencies: repository: https://charts.bitnami.com/bitnami condition: rabbitmq.enabled + - name: mongodb + version: 13.16.0 + repository: https://charts.bitnami.com/bitnami + condition: mongodb.enabled + - name: base version: 1.14.4 repository: https://istio-release.storage.googleapis.com/charts diff --git a/deploy/platform/kubernetes/Makefile b/deploy/platform/kubernetes/Makefile index e8ee96d..0f3111d 100644 --- a/deploy/platform/kubernetes/Makefile +++ b/deploy/platform/kubernetes/Makefile @@ -150,6 +150,12 @@ feature-rabbitmq-monitor: $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set rabbitmq.enabled=true) $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true) +.PHONY: feature-mongodb-monitor +feature-mongodb-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.mongodbMonitor.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set mongodb.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true) + .PHONY: feature-function feature-function: $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.function.enabled=true) diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/Makefile.in index 9860066..7f32a08 100644 --- a/deploy/platform/kubernetes/Makefile.in +++ b/deploy/platform/kubernetes/Makefile.in @@ -23,4 +23,4 @@ SAMPLE_SERVICES_NAMESPACE ?= sample-services RELEASE ?= demo AGENTLESS ?= false -FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql,elasticsearch-monitor,rabbitmq-monitor +FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/templates/feature-mongodb/opentelemetry-config.yaml similarity index 54% copy from deploy/platform/kubernetes/Makefile.in copy to deploy/platform/kubernetes/templates/feature-mongodb/opentelemetry-config.yaml index 9860066..e0fc690 100644 --- a/deploy/platform/kubernetes/Makefile.in +++ b/deploy/platform/kubernetes/templates/feature-mongodb/opentelemetry-config.yaml @@ -16,11 +16,24 @@ # under the License. # -.EXPORT_ALL_VARIABLES: +{{- define "opentelemetry-config-mongodb-monitor" }} +{{- if .Values.features.mongodbMonitor.enabled }} -NAMESPACE ?= skywalking-showcase -SAMPLE_SERVICES_NAMESPACE ?= sample-services -RELEASE ?= demo -AGENTLESS ?= false +- job_name: mongodb-monitoring + scrape_interval: 10s + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [ __meta_kubernetes_pod_controller_name, __meta_kubernetes_pod_container_name, __meta_kubernetes_pod_container_port_name ] + action: keep + regex: .+mongodb;metrics;metrics + - source_labels: [ ] + target_label: cluster + replacement: replset + - source_labels: [ __meta_kubernetes_pod_name ] + target_label: service_instance_id + regex: (.+) + replacement: $$1 -FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql,elasticsearch-monitor,rabbitmq-monitor +{{- end }} +{{- end }} diff --git a/deploy/platform/kubernetes/templates/otel-collector-config.yaml b/deploy/platform/kubernetes/templates/otel-collector-config.yaml index 7ab890e..1b3b49b 100644 --- a/deploy/platform/kubernetes/templates/otel-collector-config.yaml +++ b/deploy/platform/kubernetes/templates/otel-collector-config.yaml @@ -37,6 +37,7 @@ data: {{- include "opentelemetry-config-vm" . | indent 12 }} {{- include "opentelemetry-config-elasticsearch-monitor" . | indent 12 }} {{- include "opentelemetry-config-rabbitmq-monitor" . | indent 12 }} + {{- include "opentelemetry-config-mongodb-monitor" . | indent 12 }} exporters: otlp: diff --git a/deploy/platform/kubernetes/values.yaml b/deploy/platform/kubernetes/values.yaml index 9751650..903b4a4 100644 --- a/deploy/platform/kubernetes/values.yaml +++ b/deploy/platform/kubernetes/values.yaml @@ -39,7 +39,7 @@ skywalking: # @feature: istiod-monitor; enable rules to analyze Istio control plane metrics # @feature: mysql; enable mysql rules to analyze MySQL metrics # @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics - SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/* + SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix,elasticsearch/*,rabbitmq/*,mongodb/* SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS: "mx-mesh,persistence" # @feature: als; enable mesh analyzer (mx-mesh) to analyze ALS logs K8S_SERVICE_NAME_RULE: "mesh-svr::${service.metadata.name}" @@ -225,6 +225,22 @@ rabbitmq: password: guest erlangCookie: skywalking-showcase +mongodb: + enabled: false + architecture: replicaset + replicaCount: 2 + auth: + enabled: false + persistence: + enabled: false + metrics: + enabled: true + compatibleMode: true + collector: + dbstats: true + diagnosticdata: true + replicasetstatus: true + prometheus-elasticsearch-exporter: enabled: false fullnameOverride: prometheus-elasticsearch-exporter @@ -287,6 +303,9 @@ features: rabbitmqMonitor: enabled: false + mongodbMonitor: + enabled: false + mysqlMonitor: enabled: false diff --git a/docs/readme.md b/docs/readme.md index c6cda23..07aeb24 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -105,6 +105,7 @@ Currently, the features supported are: | `mysql-monitor` | Start a MySQL server and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | | `postgresql-monitor` | Start a PostgreSQL server, and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | | `elasticsearch-monitor` | Deploy OpenTelemetry and export Elasticsearch monitoring metrics to SkyWalking for analysis and display on UI. | | +| `mongodb-monitor` | Deploy OpenTelemetry and export MongoDB monitoring metrics to SkyWalking for analysis and display on UI. | | | `apisix-monitor` | Deploy OpenTelemetry and export APISIX metrics to SkyWalking for analysis and display on UI | | | `mesh-with-agent` | Deploy services with java agent in the service mesh environment. | Only support deployment in the Kubernetes environment, docker is not supported. | | `promql` | Deploy a Grafana to use promql service and show SkyWalking UI on the Grafana. | Feel free to modify the Grafana config when deploy your own environment. |