This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 06016fc0f extend CLI by monitoring stack (#2256)
06016fc0f is described below
commit 06016fc0f3d09a656799e3e3d8f666df58ad5b32
Author: Tim <[email protected]>
AuthorDate: Fri Dec 1 10:17:48 2023 +0100
extend CLI by monitoring stack (#2256)
* chore: extend CLI by monitoring stack
* chore: extend dependabot config by new docker images
* chore: add env variable to backend & extensions service container
---
.github/dependabot.yml | 24 ++++++++---
installer/cli/.env | 5 +++
.../deploy/standalone/backend/docker-compose.yml | 1 +
.../extensions-all-jvm/docker-compose.yml | 2 +
.../alertmanager.yml} | 23 +----------
.../docker-compose.dev.yml} | 28 ++++---------
.../docker-compose.yml | 24 ++++-------
.../docker-compose.dev.yml} | 28 ++++---------
.../docker-compose.yml | 25 +++++-------
.../deploy/standalone/prometheus/prometheus.yml | 47 ++++++++++++++++++++++
.../full-monitoring} | 36 +++++++----------
.../lite-monitoring} | 34 ++++++----------
12 files changed, 133 insertions(+), 144 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 822b365e9..d48dddaa5 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -22,7 +22,6 @@ updates:
schedule:
interval: "daily"
open-pull-requests-limit: 1
-
- package-ecosystem: "docker"
directory: "/installer/compose"
labels:
@@ -31,7 +30,6 @@ updates:
schedule:
interval: "daily"
open-pull-requests-limit: 1
-
- package-ecosystem: "docker"
directory: "/installer/cli/deploy/standalone/grafana"
labels:
@@ -40,6 +38,22 @@ updates:
schedule:
interval: "daily"
open-pull-requests-limit: 1
+ - package-ecosystem: "docker"
+ directory: "/installer/cli/deploy/standalone/prometheus"
+ labels:
+ - "dependencies"
+ - "installer"
+ schedule:
+ interval: "daily"
+ open-pull-requests-limit: 1
+ - package-ecosystem: "docker"
+ directory: "/installer/cli/deploy/standalone/prometheus-alertmanager"
+ labels:
+ - "dependencies"
+ - "installer"
+ schedule:
+ interval: "daily"
+ open-pull-requests-limit: 1
- package-ecosystem: "github-actions"
directory: "/"
labels:
@@ -61,7 +75,7 @@ updates:
interval: "daily"
ignore:
- dependency-name: "*"
- update-types: ["version-update:semver-patch"]
+ update-types: [ "version-update:semver-patch" ]
open-pull-requests-limit: 1
- package-ecosystem: "npm"
directory: "/ui/projects/streampipes/platform-services"
@@ -69,7 +83,7 @@ updates:
interval: "daily"
ignore:
- dependency-name: "*"
- update-types: ["version-update:semver-patch"]
+ update-types: [ "version-update:semver-patch" ]
open-pull-requests-limit: 1
- package-ecosystem: "npm"
directory: "/ui/projects/streampipes/shared-ui"
@@ -77,7 +91,7 @@ updates:
interval: "daily"
ignore:
- dependency-name: "*"
- update-types: ["version-update:semver-patch"]
+ update-types: [ "version-update:semver-patch" ]
open-pull-requests-limit: 1
- package-ecosystem: "pip"
directory: "/streampipes-client-python"
diff --git a/installer/cli/.env b/installer/cli/.env
index 4f899ce21..ce976949e 100644
--- a/installer/cli/.env
+++ b/installer/cli/.env
@@ -39,3 +39,8 @@ SP_PRIORITIZED_PROTOCOL=kafka
# For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an
existing installation
SP_INFLUX_INIT_MODE=setup
#SP_INFLUX_INIT_MODE=upgrade
+
+# Monitoring
+# set to true if you want StreamPipes to expose Prometheus metrics
+#SP_SETUP_PROMETHEUS_ENDPOINT=true
+SP_SETUP_PROMETHEUS_ENDPOINT=false
diff --git a/installer/cli/deploy/standalone/backend/docker-compose.yml
b/installer/cli/deploy/standalone/backend/docker-compose.yml
index c6f320022..9264132f6 100644
--- a/installer/cli/deploy/standalone/backend/docker-compose.yml
+++ b/installer/cli/deploy/standalone/backend/docker-compose.yml
@@ -25,6 +25,7 @@ services:
environment:
- SP_PRIORITIZED_PROTOCOL=${SP_PRIORITIZED_PROTOCOL:-kafka}
- SP_MQTT_HOST=${SP_MQTT_HOST:-activemq}
+ - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT}
logging:
driver: "json-file"
options:
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
index a44655139..90686788d 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
@@ -19,6 +19,8 @@ services:
image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
depends_on:
- couchdb
+ environment:
+ - SP_SETUP_PROMETHEUS_ENDPOINT=${SP_SETUP_PROMETHEUS_ENDPOINT}
volumes:
- files:/spImages
logging:
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml
similarity index 68%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml
index a44655139..bbc7ee7fd 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/deploy/standalone/prometheus-alertmanager/alertmanager.yml
@@ -13,25 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
-services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
+global:
-volumes:
- files:
-
-networks:
- spnet:
- external: true
+## Add your configuration here
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml
similarity index 68%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to
installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml
index a44655139..c6e6c834a 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++
b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.dev.yml
@@ -13,25 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
+version: "3"
services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-volumes:
- files:
-
-networks:
- spnet:
- external: true
+ prometheus-alertmanager:
+ ports:
+ - "9093:9093"
+ command:
+ - --config.file=/etc/prometheus-alertmanager/alertmanager.yml
+ - --log.level=debug
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml
similarity index 74%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to
installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml
index a44655139..4f7ed5316 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/deploy/standalone/prometheus-alertmanager/docker-compose.yml
@@ -13,25 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
+version: "3"
services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
+ prometheus-alertmanager:
+ image: prom/alertmanager:v0.26.0
networks:
spnet:
-
-volumes:
- files:
+ volumes:
+ -
../prometheus-alertmanager/alertmanager.yml:/etc/prometheus-alertmanager/alertmanager.yml
+ command:
+ - --config.file=/etc/prometheus-alertmanager/alertmanager.yml
networks:
spnet:
- external: true
+ external: true
\ No newline at end of file
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml
similarity index 68%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml
index a44655139..d5728b61d 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/deploy/standalone/prometheus/docker-compose.dev.yml
@@ -13,25 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
+version: "3"
services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-volumes:
- files:
-
-networks:
- spnet:
- external: true
+ prometheus:
+ ports:
+ - "9090:9090"
+ command:
+ - --config.file=/etc/prometheus/prometheus.yml
+ - --log.level=debug
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/deploy/standalone/prometheus/docker-compose.yml
similarity index 74%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to installer/cli/deploy/standalone/prometheus/docker-compose.yml
index a44655139..15bcfd0ca 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/deploy/standalone/prometheus/docker-compose.yml
@@ -13,25 +13,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
+version: "3"
services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
+ prometheus:
+ image: prom/prometheus:v2.48.0
networks:
spnet:
-
-volumes:
- files:
+ volumes:
+ - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+ - ../prometheus/rules:/etc/prometheus/rules
+ command:
+ - --config.file=/etc/prometheus/prometheus.yml
networks:
spnet:
- external: true
+ external: true
\ No newline at end of file
diff --git a/installer/cli/deploy/standalone/prometheus/prometheus.yml
b/installer/cli/deploy/standalone/prometheus/prometheus.yml
new file mode 100644
index 000000000..daf866fcc
--- /dev/null
+++ b/installer/cli/deploy/standalone/prometheus/prometheus.yml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+global:
+ scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default
is every 1 minute.
+ evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is
every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets:
+ - prometheus-alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global
'evaluation_interval'.
+rule_files:
+ # add your rules files here
+ # - ./rules/demo.yml
+
+scrape_configs:
+ - job_name: "prometheus"
+ static_configs:
+ - targets: [ "localhost:9090" ]
+
+ - job_name: "backend"
+ metrics_path: "/streampipes-backend/actuator/prometheus"
+ static_configs:
+ - targets: [ "localhost:8030" ]
+ #bearer_token: ""
+ #bearer_token_file: ""
+
+ - job_name: "extensions-all-iiot"
+ metrics_path: "/actuator/prometheus"
+ static_configs:
+ - targets: [ "localhost:8090" ]
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/environments/full-monitoring
similarity index 68%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to installer/cli/environments/full-monitoring
index a44655139..3cc41edb3 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/environments/full-monitoring
@@ -13,25 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
-services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-volumes:
- files:
-
-networks:
- spnet:
- external: true
+[environment:full-monitoring]
+ui
+backend
+extensions-all-jvm
+couchdb
+jobmanager
+taskmanager
+zookeeper
+kafka
+influxdb
+pipeline-elements-all-flink
+grafana
+prometheus
+prometheus-alertmanager
diff --git
a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
b/installer/cli/environments/lite-monitoring
similarity index 68%
copy from installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
copy to installer/cli/environments/lite-monitoring
index a44655139..d99f12c3e 100644
--- a/installer/cli/deploy/standalone/extensions-all-jvm/docker-compose.yml
+++ b/installer/cli/environments/lite-monitoring
@@ -13,25 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-version: "3.4"
-services:
- extensions-all-jvm:
- image: ${SP_DOCKER_REGISTRY}/extensions-all-jvm:${SP_VERSION}
- depends_on:
- - couchdb
- volumes:
- - files:/spImages
- logging:
- driver: "json-file"
- options:
- max-size: "1m"
- max-file: "1"
- networks:
- spnet:
-
-volumes:
- files:
-
-networks:
- spnet:
- external: true
+[environment:lite-monitoring]
+ui
+backend
+consul
+couchdb
+kafka
+zookeeper
+influxdb
+extensions-all-jvm
+grafana
+prometheus
+prometheus-alertmanager
