This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch helm in repository https://gitbox.apache.org/repos/asf/skywalking-showcase.git
commit d1eb146e5f123010774887fc1f63811d8a8d37d1 Author: kezhenxu94 <kezhenx...@apache.org> AuthorDate: Sat Mar 25 20:24:05 2023 +0800 Migrate to Helm Chart --- .gitignore | 5 + deploy/platform/docker/Makefile.in | 2 +- ...ysql.yaml => docker-compose.mysql-monitor.yaml} | 0 ...yaml => docker-compose.postgresql-monitor.yaml} | 0 ...pose.vm.yaml => docker-compose.vm-monitor.yaml} | 0 deploy/platform/kubernetes/Chart.yaml | 54 ++++ deploy/platform/kubernetes/Makefile | 145 +++++++--- deploy/platform/kubernetes/Makefile.in | 5 +- .../kubernetes/feature-cluster/permissions.yaml | 57 ---- .../kubernetes/feature-cluster/resources.yaml | 310 --------------------- .../feature-elasticsearch/resources.yaml | 65 ----- .../kubernetes/feature-satellite/permissions.yaml | 48 ---- .../kubernetes/feature-satellite/resources.yaml | 75 ----- .../feature-single-node/permissions.yaml | 52 ---- .../kubernetes/feature-single-node/resources.yaml | 221 --------------- deploy/platform/kubernetes/features.mk | 104 +------ deploy/platform/kubernetes/templates/NOTES.txt | 56 ++++ deploy/platform/kubernetes/templates/_helpers.tpl | 35 +++ .../{ => templates}/feature-agent/resources.yaml | 87 ++++-- .../feature-apisix-monitor/permissions.yaml | 6 +- .../feature-apisix-monitor/resources.yaml | 4 +- .../{ => templates}/feature-event/permissions.yaml | 5 +- .../{ => templates}/feature-event/resources.yaml | 36 +-- .../{ => templates}/feature-event/scaler.yaml | 11 +- .../{ => templates}/feature-function/resource.yaml | 13 +- .../feature-istiod-monitor/open-telemetry.yaml | 4 +- .../feature-istiod-monitor/permissions.yaml | 6 +- .../feature-kubernetes-monitor/open-telemetry.yaml | 6 +- .../feature-kubernetes-monitor/permissions.yaml | 5 +- .../feature-mysql-monitor}/fluent-bit.yaml | 6 +- .../feature-mysql-monitor}/mysql.yaml | 2 + .../feature-mysql-monitor}/open-telemetry.yaml | 2 + .../feature-mysql-monitor}/permissions.yaml | 6 +- .../feature-postgresql-monitor}/fluent-bit.yaml | 5 +- .../open-telemetry.yaml | 3 + .../feature-postgresql-monitor}/permissions.yaml | 6 +- .../feature-postgresql-monitor}/psql.yaml | 3 + .../feature-promql/configmaps.yaml} | 17 +- .../{ => templates}/feature-promql/resources.yaml | 21 +- .../{ => templates}/feature-rover/permissions.yaml | 6 +- .../{ => templates}/feature-rover/profiling.yaml | 21 +- .../{ => templates}/feature-rover/resources.yaml | 18 +- .../feature-so11y/open-telemetry.yaml | 3 + .../{ => templates}/feature-so11y/permissions.yaml | 6 +- .../feature-trace-profiling/profiling.yaml | 9 +- .../feature-vm-monitor}/open-telemetry.yaml | 3 + .../feature-vm-monitor}/permissions.yaml | 6 +- .../feature-vm-monitor}/vm.yaml | 3 + deploy/platform/kubernetes/values.yaml | 259 +++++++++++++++++ docs/readme.md | 8 +- scripts/wait-cert-manager-ready.sh | 59 ---- 51 files changed, 744 insertions(+), 1145 deletions(-) diff --git a/.gitignore b/.gitignore index 6388ee9..49fe0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,8 @@ out/ ### VS Code ### .vscode/ + +deploy/platform/kubernetes/Chart.lock +deploy/platform/kubernetes/charts/ + +deploy/platform/kubernetes/generated diff --git a/deploy/platform/docker/Makefile.in b/deploy/platform/docker/Makefile.in index a0bea00..6c8ef8f 100644 --- a/deploy/platform/docker/Makefile.in +++ b/deploy/platform/docker/Makefile.in @@ -17,4 +17,4 @@ # .EXPORT_ALL_VARIABLES: -FEATURE_FLAGS ?= agent,cluster,so11y,vm,event,mysql,apisix-monitor,promql +FEATURE_FLAGS ?= agent,cluster,so11y,vm-monitor,event,mysql-monitor,apisix-monitor,promql diff --git a/deploy/platform/docker/docker-compose.mysql.yaml b/deploy/platform/docker/docker-compose.mysql-monitor.yaml similarity index 100% rename from deploy/platform/docker/docker-compose.mysql.yaml rename to deploy/platform/docker/docker-compose.mysql-monitor.yaml diff --git a/deploy/platform/docker/docker-compose.postgresql.yaml b/deploy/platform/docker/docker-compose.postgresql-monitor.yaml similarity index 100% rename from deploy/platform/docker/docker-compose.postgresql.yaml rename to deploy/platform/docker/docker-compose.postgresql-monitor.yaml diff --git a/deploy/platform/docker/docker-compose.vm.yaml b/deploy/platform/docker/docker-compose.vm-monitor.yaml similarity index 100% rename from deploy/platform/docker/docker-compose.vm.yaml rename to deploy/platform/docker/docker-compose.vm-monitor.yaml diff --git a/deploy/platform/kubernetes/Chart.yaml b/deploy/platform/kubernetes/Chart.yaml new file mode 100644 index 0000000..612daf4 --- /dev/null +++ b/deploy/platform/kubernetes/Chart.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: skywalking-showcase +home: https://skywalking.apache.org +version: 0.0.0 +description: Apache SkyWalking showcase +icon: https://raw.githubusercontent.com/apache/skywalking-kubernetes/master/logo/sw-logo-for-chart.jpg +sources: + - https://github.com/apache/skywalking-showcase +dependencies: + - name: skywalking-helm + version: 0.0.0-8cf4be873ca693340408b854d128d97fd088614e + repository: oci://ghcr.io/apache/skywalking-kubernetes + + # @feature: java-agent-injector; the java agent injector is a component of the swck operator, so we need to deploy the swck operator firstly + - name: skywalking-helm-swck-operator + version: 0.0.0-8cf4be873ca693340408b854d128d97fd088614e + repository: oci://ghcr.io/apache/skywalking-kubernetes + condition: agentInjector.enabled + + # @feature: function; install open function resources + - name: openfunction + version: 0.5.0 + repository: https://openfunction.github.io/charts + condition: openfunction.enabled + + - name: kube-state-metrics + version: 5.0.1 + repository: https://prometheus-community.github.io/helm-charts + condition: kube-state-metrics.enabled + + - name: base + version: 1.14.4 + repository: https://istio-release.storage.googleapis.com/charts + condition: istio.enabled + + - name: istiod + version: 1.14.4 + repository: https://istio-release.storage.googleapis.com/charts + condition: istio.enabled diff --git a/deploy/platform/kubernetes/Makefile b/deploy/platform/kubernetes/Makefile index e7c1565..e3d456b 100644 --- a/deploy/platform/kubernetes/Makefile +++ b/deploy/platform/kubernetes/Makefile @@ -26,47 +26,118 @@ ifneq (,$(findstring satellite,$(features))) BACKEND_SERVICE := satellite endif -# Deploy -deploy_features := $(foreach r,$(features),deploy.$(r)) -.PHONY: $(deploy_features) -$(deploy_features): %: %.resources - -deploy_resources := $(foreach r,$(deploy_features),$(r).resources) -.PHONY: $(deploy_resources) -$(deploy_resources): deploy.%.resources: % - $(eval resources := $(wildcard $(shell pwd)/$</*)) - @$(foreach resource, $(resources), docker run --rm -v "$(pwd)/$(resource)":/input \ - $(foreach v,$(EXPORTED_VARS),-e $(v)) \ - bhgedigital/envsubst sh -c "envsubst < /input" | kubectl -n $(NAMESPACE) apply -f - ; \ - ) - -.PHONY: deploy -deploy: $(deploy_features) -ifneq ($(SHOW_TIPS),false) - @echo - @echo "Now wait the Pods to be ready and use the following command to expose the port of SkyWalking UI" - @echo - @echo " $$ kubectl port-forward svc/ui 8080:8080 --namespace $(NAMESPACE)" - @echo - @echo "Then open your browser and navigate to http://localhost:8080" - @echo +DEBUG ?= false +HELM_OPTIONS := --set skywalking-helm.fullnameOverride=$(RELEASE) +ifeq ($(DEBUG),true) + HELM_OPTIONS := $(HELM_OPTIONS) --dry-run endif -# Undeploy -undeploy_features := $(foreach r,$(features),undeploy.$(r)) -.PHONY: $(undeploy_features) -$(undeploy_features): %: %.resources +.PHONY: helm +helm: + @helm version >/dev/null 2>&1 || (echo "No helm in PATH, please make sure helm is available in PATH."; exit 1) + +# @feature: java-agent-injector; use the java agent injector to inject the java agent more natively +.PHONY: feature-java-agent-injector +feature-java-agent-injector: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set agentInjector.enabled=true) + +.PHONY: feature-single-node +feature-single-node: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.oap.replicas=1) + +.PHONY: feature-cluster +feature-cluster: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.oap.replicas=2) + +.PHONY: feature-elasticsearch +feature-elasticsearch: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.oap.storageType=elasticsearch) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.elasticsearch.enabled=true) + +.PHONY: feature-postgresql +feature-postgresql: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.oap.storageType=postgresql) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set skywalking-helm.postgresql.enabled=true) + +.PHONY: feature-so11y +feature-so11y: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.so11y.enabled=true) + +.PHONY: feature-vm-monitor +feature-vm-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.vmMonitor.enabled=true) + +.PHONY: feature-event +feature-event: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.event.enabled=true) + +.PHONY: feature-istiod-monitor +feature-istiod-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.istiodMonitor.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set istio.enabled=true --set istiod.global.istioNamespace=$(NAMESPACE) --set base.global.istioNamespace=$(NAMESPACE)) + +.PHONY: feature-als +feature-als: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.als.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set istio.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set istiod.meshConfig.defaultConfig.tracing.zipkin.address=$(RELEASE)-oap.$(NAMESPACE).svc:9411) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set istiod.meshConfig.defaultConfig.envoyMetricsService.address=$(RELEASE)-oap.$(NAMESPACE).svc:11800) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set istiod.meshConfig.defaultConfig.envoyAccessLogService.address=$(RELEASE)-oap.$(NAMESPACE).svc:11800) -undeploy_resources := $(foreach r,$(undeploy_features),$(r).resources) -.PHONY: $(undeploy_resources) -$(undeploy_resources): undeploy.%.resources: % - $(eval resources := $(wildcard $(shell pwd)/$</*)) - @$(foreach resource, $(resources), docker run --rm -v "$(pwd)/$(resource)":/input \ - $(foreach v,$(EXPORTED_VARS),-e $(v)) \ - bhgedigital/envsubst sh -c "envsubst < /input" | kubectl -n $(NAMESPACE) delete --ignore-not-found -f - ; \ - ) +.PHONY: feature-satellite +feature-satellite: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.satellite.enabled=true) + +.PHONY: feature-rover +feature-rover: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.rover.enabled=true) + +.PHONY: feature-trace-profiling +feature-trace-profiling: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.traceProfiling.enabled=true) + +.PHONY: feature-mysql-monitor +feature-mysql-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.mysqlMonitor.enabled=true) + +.PHONY: feature-postgresql-monitor +feature-postgresql-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.postgresqlMonitor.enabled=true) + +.PHONY: feature-apisix-monitor +feature-apisix-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.apisixMonitor.enabled=true) + +# @feature: kubernetes-monitor; extra resources to install for kubernetes monitoring, standard kube-state-metrics +.PHONY: feature-kubernetes-monitor +feature-kubernetes-monitor: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.kubernetesMonitor.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set kube-state-metrics.enabled=true) + +.PHONY: feature-function +feature-function: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.function.enabled=true) + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set openfunction.enabled=true) + +.PHONY: feature-promql +feature-promql: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.promql.enabled=true) + @mkdir -p ./generated/promql-configs/ + @cp ../config/promql/dashboards/*.json ./generated/promql-configs/ + +.PHONY: feature-mesh-with-agent +feature-mesh-with-agent: + $(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.meshWithAgent.enabled=true) + +.PHONY: deploy +deploy: $(features) + helm dep up . + helm -n $(NAMESPACE) upgrade --install $(RELEASE) . --create-namespace $(HELM_OPTIONS) \ + --set sampleServices.hub=ghcr.io/apache/skywalking-showcase \ + --set sampleServices.tag=f36d4b4 .PHONY: undeploy -undeploy: $(undeploy_features) +undeploy: + helm -n $(NAMESPACE) delete $(RELEASE) include features.mk diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/Makefile.in index 6ddc6e6..67beb61 100644 --- a/deploy/platform/kubernetes/Makefile.in +++ b/deploy/platform/kubernetes/Makefile.in @@ -18,8 +18,9 @@ .EXPORT_ALL_VARIABLES: -NAMESPACE ?= default +NAMESPACE ?= skywalking-showcase +RELEASE ?= demo AGENTLESS ?= false BACKEND_SERVICE ?= oap -FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql,apisix-monitor,promql +FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,promql diff --git a/deploy/platform/kubernetes/feature-cluster/permissions.yaml b/deploy/platform/kubernetes/feature-cluster/permissions.yaml deleted file mode 100644 index dad1c9d..0000000 --- a/deploy/platform/kubernetes/feature-cluster/permissions.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: skywalking-oap-sa-cluster - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: skywalking-oap-sa-cluster-role -rules: - - apiGroups: [ "" ] - resources: - - "pods" # @feature: cluster; OAP needs to read other OAP Pods information to form a cluster - # @feature: als; OAP needs to read Pods metadata to analyze the access logs - - "pods/log" # @feature: on-demand-pod-logs; in feature on demand Pod logs, OAP needs to read pods logs - - "services" # @feature: als; OAP needs to read services metadata to analyze the access logs - - "endpoints" # @feature: als; OAP needs to read endpoints metadata to analyze the access logs - - "nodes" # @feature: als; OAP needs to read nodes metadata to analyze the access logs - - "configmaps" - verbs: [ "get", "watch", "list" ] - - apiGroups: [ "batch" ] - resources: - - "jobs" # @feature: cluster; OAP needs to wait for the init job to complete - verbs: [ "get", "watch", "list" ] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: skywalking-oap-sa-cluster-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: skywalking-oap-sa-cluster-role -subjects: - - kind: ServiceAccount - name: skywalking-oap-sa-cluster - namespace: ${NAMESPACE} diff --git a/deploy/platform/kubernetes/feature-cluster/resources.yaml b/deploy/platform/kubernetes/feature-cluster/resources.yaml deleted file mode 100644 index 0fc9099..0000000 --- a/deploy/platform/kubernetes/feature-cluster/resources.yaml +++ /dev/null @@ -1,310 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - ---- -apiVersion: v1 -data: - metadata-service-mapping.yaml: | - serviceName: mesh-svr::${LABELS."service.istio.io/canonical-name",LABELS."app.kubernetes.io/name",LABELS.app} - serviceInstanceName: ${NAME,NAME} # Just to skip the "envsubst" replacement - alarm-settings.yml: | - rules: - service_resp_time_rule: - metrics-name: service_resp_time - threshold: 20 # 20ms is just for demo - op: ">" - period: 5 - count: 1 - message: Response time of service {name} is more than 20ms. -kind: ConfigMap -metadata: - name: oap-static-config - ---- -apiVersion: v1 -data: - core.default.log4j-xml: |- - <Configuration status="WARN"> - <Appenders> - <Console name="Console" target="SYSTEM_OUT"> - <PatternLayout charset="UTF-8" pattern="%d - %c - %L [%t] %-5p %x - %m%n"/> - </Console> - </Appenders> - <Loggers> - <logger name="io.grpc.netty" level="INFO"/> - <logger name="org.apache.skywalking.oap.server.configuration.api" level="TRACE"/> - <logger name="org.apache.skywalking.oap.server.configuration.configmap" level="DEBUG"/> - <Root level="INFO"> - <AppenderRef ref="Console"/> - </Root> - </Loggers> - </Configuration> -kind: ConfigMap -metadata: - labels: - app: oap - name: oap-dynamic-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: oap -spec: - selector: - app: oap - ports: - - name: metrics - port: 1234 - - name: grpc - port: 11800 - - name: http - port: 12800 - - name: zipkin-receiver - port: 9411 - - name: zipkin-query - port: 9412 - - name: promql - port: 9090 - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: oap-init-job # @feature: cluster; set up an init job to initialize ES templates and indices -spec: - template: - metadata: - name: oap-init-job - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: skywalking-oap-sa-cluster - restartPolicy: Never - initContainers: - - name: wait-for-es - image: busybox:1.30 - command: - - "sh" - - "-c" - - "for i in $(seq 1 60); do nc -z -w3 elasticsearch 9200 && exit 0 || sleep 5; done; exit 1" - containers: - - name: oap-init - image: ${SW_OAP_IMAGE} - imagePullPolicy: Always - env: # @feature: cluster; make sure all env vars are the same with the cluster nodes as this will affect templates / indices - - name: JAVA_OPTS - value: "-Dmode=init" # @feature: cluster; set the OAP mode to "init" so the job can complete - - name: SW_OTEL_RECEIVER - value: default - - name: SW_OTEL_RECEIVER_ENABLED_OTEL_RULES - # @feature: vm; enable vm rules to analyze VM metrics - # @feature: so11y; enable oap rules to analyze OAP metrics - # @feature: kubernetes-monitor; enable rules to analyze Kubernetes Cluster/Node/Service metrics - # @feature: istiod-monitor; enable rules to analyze Istio control plane metrics - # @feature: mysql; enable mysql rules to analyze MySQL metrics - # @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics - value: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix - - name: SW_STORAGE - value: elasticsearch - - name: SW_STORAGE_ES_CLUSTER_NODES - value: elasticsearch:9200 - - name: SW_STORAGE_ES_INDEX_REPLICAS_NUMBER - value: "0" - - name: SW_TELEMETRY - value: prometheus - - name: SW_METER_ANALYZER_ACTIVE_FILES - value: datasource,threadpool,satellite,network-profiling,spring-sleuth - - name: SW_STORAGE_ES_RESPONSE_TIMEOUT - value: "50000" - - name: SW_STORAGE_ES_INDEX_SHARDS_NUMBER - value: "6" - - name: SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR - value: "2" - - name: SW_RECEIVER_ZIPKIN - value: default - - name: SW_QUERY_ZIPKIN - value: default - volumeMounts: - - name: config-volume - mountPath: /skywalking/ext-config - volumes: - - name: config-volume - configMap: - name: oap-static-config ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: oap-deployment - labels: - app: oap -spec: - replicas: 2 # @feature: cluster; set OAP replicas to >1 - selector: - matchLabels: - app: oap - template: - metadata: - labels: - app: oap - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: skywalking-oap-sa-cluster # @feature: cluster; set a service account with Pod "read" permission - initContainers: - - name: wait-for-oap-init - image: bitnami/kubectl:1.20.12 - command: - - "kubectl" - - "wait" - - "--for=condition=complete" - - "job/oap-init-job" - containers: - - name: oap - image: ${SW_OAP_IMAGE} - imagePullPolicy: Always - resources: - limits: - cpu: 4000m - memory: "4096Mi" - requests: - cpu: 1000m - memory: "1024Mi" - ports: - - name: metrics # @feature: so11y; set a name for the metrics port that can be referenced in otel config - containerPort: 1234 - - name: grpc - containerPort: 11800 - - name: http - containerPort: 12800 - - name: zipkin-receiver - containerPort: 9411 - - name: zipkin-query - containerPort: 9412 - - name: promql - containerPort: 9090 - env: - - name: JAVA_OPTS - value: "-Dmode=no-init" - - name: SW_CLUSTER - value: kubernetes # @feature: cluster; set cluster coordinator to kubernetes - - name: SW_CLUSTER_K8S_NAMESPACE - value: ${NAMESPACE} - - name: SW_CLUSTER_K8S_LABEL - value: app=oap # @feature: cluster; set label selectors to select OAP Pods as a cluster - - name: SKYWALKING_COLLECTOR_UID # @feature: cluster; set collector uid to identify the OAP node - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: SW_HEALTH_CHECKER - value: default - - name: SW_OTEL_RECEIVER - value: default # @feature: so11y;,mysql;kubernetes-monitor enable OpenTelemetry receiver to receive OpenTelemetry metrics - - name: SW_OTEL_RECEIVER_ENABLED_OTEL_RULES - # @feature: vm; enable vm rules to analyze VM metrics - # @feature: so11y; enable oap rules to analyze OAP metrics - # @feature: kubernetes-monitor; enable rules to analyze Kubernetes Cluster/Node/Service metrics - # @feature: istiod-monitor; enable rules to analyze Istio control plane metrics - # @feature: mysql; enable mysql rules to analyze MySQL metrics - # @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics - value: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix - - name: SW_STORAGE - value: elasticsearch - - name: SW_STORAGE_ES_CLUSTER_NODES - value: elasticsearch:9200 - - name: SW_TELEMETRY - value: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus - - name: SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS - value: "mx-mesh,persistence" # @feature: als; enable mesh analyzer (mx-mesh) to analyze ALS logs - - name: K8S_SERVICE_NAME_RULE - value: "mesh-svr::${service.metadata.name}" - - name: SW_CONFIGURATION - value: k8s-configmap - - name: SW_ENVOY_METRIC_ALS_TCP_ANALYSIS - value: "mx-mesh" - - name: SW_ENABLE_UPDATE_UI_TEMPLATE - value: "false" - - name: SW_SLOW_DB_THRESHOLD - value: default:0,mongodb:100 - - name: SW_ENABLE_ON_DEMAND_POD_LOG - value: "true" - - name: SW_METER_ANALYZER_ACTIVE_FILES - value: datasource,threadpool,satellite,network-profiling,spring-sleuth - - name: SW_STORAGE_ES_RESPONSE_TIMEOUT - value: "50000" - - name: SW_STORAGE_ES_INDEX_SHARDS_NUMBER - value: "6" - - name: SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR - value: "2" - - name: SW_RECEIVER_ZIPKIN - value: default - - name: SW_QUERY_ZIPKIN - value: default - volumeMounts: - - name: config-volume - mountPath: /skywalking/ext-config - volumes: - - name: config-volume - configMap: - name: oap-static-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: ui -spec: - selector: - app: ui - ports: - - protocol: TCP - port: 8080 - targetPort: 8080 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ui-deployment - labels: - app: ui -spec: - replicas: 1 - selector: - matchLabels: - app: ui - template: - metadata: - labels: - app: ui - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: skywalking-oap-sa-cluster - containers: - - name: ui - image: ${SW_UI_IMAGE} - imagePullPolicy: Always - ports: - - containerPort: 8080 - env: - - name: SW_OAP_ADDRESS - value: http://oap:12800 - - name: SW_ZIPKIN_ADDRESS - value: http://oap:9412 diff --git a/deploy/platform/kubernetes/feature-elasticsearch/resources.yaml b/deploy/platform/kubernetes/feature-elasticsearch/resources.yaml deleted file mode 100644 index 755b362..0000000 --- a/deploy/platform/kubernetes/feature-elasticsearch/resources.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch -spec: - selector: - app: elasticsearch - ports: - - protocol: TCP - port: 9200 - targetPort: 9200 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: es-deployment - labels: - app: elasticsearch -spec: - replicas: 1 - selector: - matchLabels: - app: elasticsearch - template: - metadata: - labels: - app: elasticsearch - annotations: - sidecar.istio.io/inject: "false" - spec: - containers: - - name: elasticsearch - image: ${ES_IMAGE} - imagePullPolicy: Always - resources: - limits: - cpu: 4000m - memory: "4096Mi" - requests: - cpu: 4000m - memory: "4096Mi" - ports: - - containerPort: 9200 - env: - - name: "discovery.type" - value: "single-node" diff --git a/deploy/platform/kubernetes/feature-satellite/permissions.yaml b/deploy/platform/kubernetes/feature-satellite/permissions.yaml deleted file mode 100644 index 43f90fc..0000000 --- a/deploy/platform/kubernetes/feature-satellite/permissions.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: skywalking-satellite-sa - namespace: ${NAMESPACE} - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: skywalking-satellite-sa-role -rules: - - apiGroups: [ "" ] - resources: - - "pods" - verbs: [ "get", "watch", "list" ] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: skywalking-satellite-sa-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: skywalking-satellite-sa-role -subjects: - - kind: ServiceAccount - name: skywalking-satellite-sa - namespace: ${NAMESPACE} \ No newline at end of file diff --git a/deploy/platform/kubernetes/feature-satellite/resources.yaml b/deploy/platform/kubernetes/feature-satellite/resources.yaml deleted file mode 100644 index cad94d9..0000000 --- a/deploy/platform/kubernetes/feature-satellite/resources.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -apiVersion: v1 -kind: Service -metadata: - name: satellite -spec: - selector: - app: satellite - ports: - - name: grpc - port: 11800 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: satellite-deployment - labels: - app: satellite -spec: - replicas: 1 - selector: - matchLabels: - app: satellite - template: - metadata: - labels: - app: satellite - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: skywalking-satellite-sa - containers: - - name: satellite - image: ${SW_SATELLITE_IMAGE} - imagePullPolicy: Always - resources: - limits: - cpu: 500m - memory: "512Mi" - requests: - cpu: 500m - memory: "512Mi" - ports: - - name: grpc - containerPort: 11800 - env: - - name: SATELLITE_GRPC_CLIENT_FINDER - value: kubernetes - - name: SATELLITE_GRPC_CLIENT_KUBERNETES_NAMESPACE - value: ${NAMESPACE} - - name: SATELLITE_GRPC_CLIENT_KUBERNETES_KIND - value: pod - - name: SATELLITE_GRPC_CLIENT_KUBERNETES_SELECTOR_LABEL - value: app=oap - - name: SATELLITE_GRPC_CLIENT_KUBERNETES_EXTRA_PORT - value: "11800" - - name: SATELLITE_TELEMETRY_EXPORT_TYPE - value: metrics_service \ No newline at end of file diff --git a/deploy/platform/kubernetes/feature-single-node/permissions.yaml b/deploy/platform/kubernetes/feature-single-node/permissions.yaml deleted file mode 100644 index 4b4148a..0000000 --- a/deploy/platform/kubernetes/feature-single-node/permissions.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: skywalking-oap-sa - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: skywalking-oap-sa-role -rules: - - apiGroups: [ "" ] - resources: - - "pods" # @feature: als; OAP needs to read pods metadata to analyze the access logs - - "pods/log" # @feature: on-demand-pod-logs; in feature on demand Pod logs, OAP needs to read pods logs - - "services" # @feature: als; OAP needs to read services metadata to analyze the access logs - - "endpoints" # @feature: als; OAP needs to read endpoints metadata to analyze the access logs - - "nodes" # @feature: als; OAP needs to read nodes metadata to analyze the access logs - - "configmaps" - verbs: [ "get", "watch", "list" ] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: skywalking-oap-sa-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: skywalking-oap-sa-role -subjects: - - kind: ServiceAccount - name: skywalking-oap-sa - namespace: ${NAMESPACE} diff --git a/deploy/platform/kubernetes/feature-single-node/resources.yaml b/deploy/platform/kubernetes/feature-single-node/resources.yaml deleted file mode 100644 index 31c8149..0000000 --- a/deploy/platform/kubernetes/feature-single-node/resources.yaml +++ /dev/null @@ -1,221 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -apiVersion: v1 -data: - metadata-service-mapping.yaml: | - serviceName: mesh-svr::${LABELS."service.istio.io/canonical-name",LABELS."app.kubernetes.io/name",LABELS.app} - serviceInstanceName: ${NAME,NAME} # Just to skip the "envsubst" replacement - alarm-settings.yml: | - rules: - service_resp_time_rule: - metrics-name: service_resp_time - threshold: 20 # 20ms is just for demo - op: ">" - period: 5 - count: 1 - message: Response time of service {name} is more than 20ms. -kind: ConfigMap -metadata: - name: oap-static-config - ---- -apiVersion: v1 -data: - core.default.log4j-xml: |- - <Configuration status="WARN"> - <Appenders> - <Console name="Console" target="SYSTEM_OUT"> - <PatternLayout charset="UTF-8" pattern="%d - %c - %L [%t] %-5p %x - %m%n"/> - </Console> - </Appenders> - <Loggers> - <logger name="io.grpc.netty" level="INFO"/> - <logger name="org.apache.skywalking.oap.server.configuration.api" level="TRACE"/> - <logger name="org.apache.skywalking.oap.server.configuration.configmap" level="DEBUG"/> - <Root level="INFO"> - <AppenderRef ref="Console"/> - </Root> - </Loggers> - </Configuration> -kind: ConfigMap -metadata: - labels: - app: oap - name: oap-dynamic-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: oap -spec: - selector: - app: oap - ports: - - name: metrics - port: 1234 - - name: grpc - port: 11800 - - name: http - port: 12800 - - name: zipkin-receiver - port: 9411 - - name: zipkin-query - port: 9412 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: oap-deployment - labels: - app: oap -spec: - replicas: 1 - selector: - matchLabels: - app: oap - template: - metadata: - labels: - app: oap - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: skywalking-oap-sa # @feature: als; set a service account with Pods/Endpoints/Services/Nodes permissions to analyze Envoy access logs - containers: - - name: oap - image: ${SW_OAP_IMAGE} - imagePullPolicy: Always - resources: - limits: - cpu: 4000m - memory: "4096Mi" - requests: - cpu: 1000m - memory: "1024Mi" - ports: - - name: metrics # @feature: so11y; set a name for the metrics port that can be referenced in otel config - containerPort: 1234 - - name: grpc - containerPort: 11800 - - name: http - containerPort: 12800 - - name: zipkin-receiver - containerPort: 9411 - - name: zipkin-query - containerPort: 9412 - env: - - name: SW_CLUSTER_K8S_NAMESPACE - value: ${NAMESPACE} - - name: SW_CLUSTER_K8S_LABEL - value: app=oap - - name: SW_HEALTH_CHECKER - value: default - - name: SW_OTEL_RECEIVER - value: default # @feature: so11y;mysql;vm;kubernetes-monitor enable OpenTelemetry receiver to receive OpenTelemetry metrics - - name: SW_OTEL_RECEIVER_ENABLED_OTEL_RULES - # @feature: vm; enable vm rules to analyze VM metrics - # @feature: so11y; enable oap rules to analyze OAP metrics - # @feature: kubernetes-monitor; enable rules to analyze Kubernetes Cluster/Node/Service metrics - # @feature: istiod-monitor; enable rules to analyze Istio metrics - # @feature: mysql; enable rules to analyze MySQL metrics - value: vm,oap,k8s/*,istio-controlplane,mysql/* - - name: SW_STORAGE - value: elasticsearch - - name: SW_STORAGE_ES_CLUSTER_NODES - value: elasticsearch:9200 - - name: SW_STORAGE_ES_INDEX_REPLICAS_NUMBER - value: "0" - - name: SW_TELEMETRY - value: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus - - name: SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS - value: "k8s-mesh,persistence" # @feature: als; enable mesh analyzer (k8s-mesh) to analyze ALS logs - - name: K8S_SERVICE_NAME_RULE - value: "mesh-svr::${service.metadata.name}" - - name: SW_CONFIGURATION - value: k8s-configmap - - name: SW_ENABLE_ON_DEMAND_POD_LOG - value: "true" - - name: SW_METER_ANALYZER_ACTIVE_FILES - value: datasource,threadpool,satellite,network-profiling,spring-sleuth - - name: SW_STORAGE_ES_RESPONSE_TIMEOUT - value: "50000" - - name: SW_RECEIVER_ZIPKIN - value: default - - name: SW_QUERY_ZIPKIN - value: default - volumeMounts: - - name: config-volume - mountPath: /skywalking/ext-config - volumes: - - name: config-volume - configMap: - name: oap-static-config - ---- -apiVersion: v1 -kind: Service -metadata: - name: ui -spec: - selector: - app: ui - ports: - - protocol: TCP - port: 8080 - targetPort: 8080 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ui-deployment - labels: - app: ui -spec: - replicas: 1 - selector: - matchLabels: - app: ui - template: - metadata: - labels: - app: ui - annotations: - sidecar.istio.io/inject: "false" - spec: - containers: - - name: ui - image: ${SW_UI_IMAGE} - imagePullPolicy: Always - ports: - - containerPort: 8080 - readinessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 3 - periodSeconds: 3 - failureThreshold: 10 - env: - - name: SW_OAP_ADDRESS - value: http://oap:12800 - - name: SW_ZIPKIN_ADDRESS - value: http://oap:9412 diff --git a/deploy/platform/kubernetes/features.mk b/deploy/platform/kubernetes/features.mk index 159fccb..c9eaac8 100644 --- a/deploy/platform/kubernetes/features.mk +++ b/deploy/platform/kubernetes/features.mk @@ -22,38 +22,13 @@ include ../../../Makefile.in -.PHONY: istioctl -istioctl: - @istioctl version >/dev/null 2>&1 || (echo "No istioctl in PATH, please make sure istioctl is available in PATH."; exit 1) - -.PHONY: istio -istio: istioctl - @istioctl install -y --set profile=demo \ - --set meshConfig.enableEnvoyAccessLogService=true `# @feature: als; enable Envoy access log service` \ - `# @feature: als; be careful to only emit wanted metrics otherwise the traffic is HUGE` \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[0]=.*membership_healthy.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[1]=.*upstream_cx_active.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[2]=.*upstream_cx_total.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[3]=.*upstream_rq_active.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[4]=.*upstream_rq_total.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[5]=.*upstream_rq_pending_active.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[6]=.*lb_healthy_panic.*' \ - --set 'meshConfig.defaultConfig.proxyStatsMatcher.inclusionRegexps[7]=.*upstream_cx_none_healthy.*' \ - --set meshConfig.defaultConfig.envoyMetricsService.address=$(BACKEND_SERVICE).$(NAMESPACE):11800 `# @feature: als; set MetricsService address to Backend Service so Envoy emits metrics to Backend Service` \ - --set meshConfig.defaultConfig.envoyAccessLogService.address=$(BACKEND_SERVICE).$(NAMESPACE):11800 `# @feature: als; set AccessLogService address to Backend Service so Envoy emits logs to Backend Service` \ - --set meshConfig.defaultConfig.tracing.sampling=100 \ - --set meshConfig.defaultConfig.tracing.zipkin.address=oap.$(NAMESPACE).svc.cluster.local:9411 \ - --set meshConfig.enableTracing=true - .PHONY: namespace namespace: @kubectl get namespace $(NAMESPACE)-agentless > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)-agentless @kubectl label namespace --overwrite $(NAMESPACE)-agentless istio-injection=enabled # @feature: als; label the namespace to allow Istio sidecar injection .PHONY: prerequisites -prerequisites: istio namespace - -feature-als: +prerequisites: namespace .PHONY: deploy.feature-als deploy.feature-als: prerequisites @@ -69,7 +44,7 @@ undeploy.feature-als: istioctl feature-mesh-with-agent: .PHONY: deploy.feature-mesh-with-agent -deploy.feature-mesh-with-agent: prerequisites install-cert-manager +deploy.feature-mesh-with-agent: @curl -Ls https://archive.apache.org/dist/skywalking/swck/${SWCK_OPERATOR_VERSION}/skywalking-swck-${SWCK_OPERATOR_VERSION}-bin.tgz | tar -zxf - -O ./config/operator-bundle.yaml | kubectl apply -f - @kubectl label namespace --overwrite $(NAMESPACE) swck-injection=enabled @kubectl get configmap skywalking-swck-java-agent-configmap -n skywalking-swck-system -oyaml | sed "s/127.0.0.1/$(NAMESPACE)-$(BACKEND_SERVICE).$(NAMESPACE)/" | kubectl apply -f - @@ -83,39 +58,9 @@ undeploy.feature-mesh-with-agent: istioctl $(MAKE) undeploy FEATURE_FLAGS=agent TAG=$(TAG) NAMESPACE=$(NAMESPACE) AGENTLESS=true istioctl x uninstall --purge -y -# @feature: kubernetes-monitor; extra resources to install for kubernetes monitoring, standard kube-state-metrics -.PHONY: feature-kubernetes-monitor -feature-kubernetes-monitor: - -.PHONY: deploy.feature-kubernetes-monitor -deploy.feature-kubernetes-monitor: - @kubectl apply -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/service-account.yaml - @kubectl apply -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/cluster-role.yaml - @kubectl apply -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/cluster-role-binding.yaml - @kubectl apply -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/service.yaml - @kubectl apply -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/deployment.yaml - -.PHONY: undeploy.feature-kubernetes-monitor -undeploy.feature-kubernetes-monitor: - @kubectl delete --ignore-not-found -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/service-account.yaml - @kubectl delete --ignore-not-found -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/cluster-role.yaml - @kubectl delete --ignore-not-found -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/cluster-role-binding.yaml - @kubectl delete --ignore-not-found -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/service.yaml - @kubectl delete --ignore-not-found -f https://raw.githubusercontent.com/kubernetes/kube-state-metrics/v2.2.4/examples/standard/deployment.yaml - -# @feature: java-agent-injector; use the java agent injector to inject the java agent more natively -.PHONY: feature-java-agent-injector -feature-java-agent-injector: - -# @feature: java-agent-injector; the swck operator depends on the certificate management of the cert-manager -.PHONY: install-cert-manager -install-cert-manager: - @kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml - @sh ../../../scripts/wait-cert-manager-ready.sh - # @feature: java-agent-injector; the java agent injector is a component of the swck operator, so we need to deploy the swck operator firstly .PHONY: deploy.feature-java-agent-injector -deploy.feature-java-agent-injector: install-cert-manager +deploy.feature-java-agent-injector: @curl -Ls https://archive.apache.org/dist/skywalking/swck/${SWCK_OPERATOR_VERSION}/skywalking-swck-${SWCK_OPERATOR_VERSION}-bin.tgz | tar -zxf - -O ./config/operator-bundle.yaml | kubectl apply -f - @kubectl label namespace --overwrite $(NAMESPACE) swck-injection=enabled # @feature: java-agent-injector; we can update the agent's backend address in a single-node cluster firstly so that we don't need to add the same backend env for every java agent @@ -128,46 +73,3 @@ undeploy.feature-java-agent-injector: @curl -Ls https://archive.apache.org/dist/skywalking/swck/${SWCK_OPERATOR_VERSION}/skywalking-swck-${SWCK_OPERATOR_VERSION}-bin.tgz | tar -zxf - -O ./config/operator-bundle.yaml | kubectl delete --ignore-not-found -f - @kubectl delete --ignore-not-found -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml $(MAKE) undeploy FEATURE_FLAGS=agent AGENTLESS=false SHOW_TIPS=false BACKEND_SERVICE=$(BACKEND_SERVICE) - -.PHONY: helm -helm: - @helm version >/dev/null 2>&1 || (echo "No helm in PATH, please make sure helm is available in PATH."; exit 1) - -.PHONY: open-function -open-function: helm - @helm repo add openfunction https://openfunction.github.io/charts/ - @helm repo update - @helm upgrade openfunction openfunction/openfunction -n openfunction --version 0.3.1 --install --create-namespace \ - --set global.ShipwrightBuild.enabled=false \ - --set global.TektonPipelines.enabled=false \ - --set global.Keda.enabled=false \ - --set global.Dapr.enabled=false \ - --set contour.envoy.useHostPort=false \ - --set contour.envoy.service.type=ClusterIP \ - --set contour.envoy.service.externalTrafficPolicy="" \ - --wait - -# @feature: function; install open function resources -.PHONY: deploy.feature-function -deploy.feature-function: open-function - @echo "deploy.feature-function" - -.PHONY: feature-function -feature-function: - -.PHONY: undeploy.feature-function -undeploy.feature-function: helm - @helm uninstall openfunction -n openfunction --timeout 20m || kubectl delete namespace openfunction --ignore-not-found --timeout 20m || true - -# @feature: promql; extra configmaps to create for grafana dashboards -.PHONY: feature-promql -feature-promql: -dashboards := $(wildcard ../config/promql/dashboards/*.json) - -.PHONY: deploy.feature-promql -deploy.feature-promql: - @$(foreach d, $(dashboards), kubectl create configmap grafana-dashboards-$(basename $(notdir $d)) --from-file=$d;) - -.PHONY: undeploy.feature-promql -undeploy.feature-promql: - @$(foreach d, $(dashboards), kubectl delete configmap grafana-dashboards-$(basename $(notdir $d)) --ignore-not-found;) diff --git a/deploy/platform/kubernetes/templates/NOTES.txt b/deploy/platform/kubernetes/templates/NOTES.txt new file mode 100644 index 0000000..7c13563 --- /dev/null +++ b/deploy/platform/kubernetes/templates/NOTES.txt @@ -0,0 +1,56 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +************************************************************************* +* * +* SkyWalking Showcase * +* * +************************************************************************* + +Thank you for installing {{ .Chart.Name }}. + +Your release is named {{ .Release.Name }}. + +Get the UI URL by running these commands: +{{- if index .Values "skywalking-helm" "ui" "ingress" "enabled" }} +{{- range index .Values "skywalking-helm" "ui" "ingress" "hosts" }} + http{{ if index $.Values "ui" "ingress" "tls" }}s{{ end }}://{{ . }}{{ index $.Values "ui" "ingress" "path" }} +{{- end }} +{{- else if contains "NodePort" (index .Values "skywalking-helm" "ui" "service" "type") }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "skywalking.ui.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" (index .Values "skywalking-helm" "ui" "service" "type") }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "skywalking.ui.fullname" . }} -n {{ .Release.Namespace }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "skywalking.ui.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ index .Values "skywalking-helm" "ui" "service" "externalPort" }} +{{- else if contains "ClusterIP" (index .Values "skywalking-helm" "ui" "service" "type") }} + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward svc/{{ printf "%s-skywalking-helm-ui" .Release.Name }} 8080:{{ index .Values "skywalking-helm" "ui" "service" "externalPort" }} --namespace {{ .Release.Namespace }} +{{- end }} + +{{- if index .Values "skywalking-helm" "elasticsearch" "enabled" }} +{{- if index .Values "skywalking-helm" "elasticsearch" "persistence" "enabled" }} +{{- else }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the SkyWalking's storage ES pod is terminated. ##### +################################################################################# + +{{- end }} +{{- end }} diff --git a/deploy/platform/kubernetes/templates/_helpers.tpl b/deploy/platform/kubernetes/templates/_helpers.tpl new file mode 100644 index 0000000..6e64b34 --- /dev/null +++ b/deploy/platform/kubernetes/templates/_helpers.tpl @@ -0,0 +1,35 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +The collector address, might be OAP or Satellite +*/}} +{{- define "skywalking.collector.address" -}} +{{- if index .Values "skywalking-helm" "satellite" "enabled" }} +{{- printf "%s-satellite.%s.svc.cluster.local:%d" (index .Values "skywalking-helm" "fullnameOverride") .Release.Namespace 11800 }} +{{- else }} +{{- printf "%s-oap.%s.svc.cluster.local:%d" (index .Values "skywalking-helm" "fullnameOverride") .Release.Namespace 11800 }} +{{- end -}} +{{- end -}} + +{{- define "skywalking.oap.address.grpc" -}} +{{- printf "%s-oap.%s.svc.cluster.local:%d" (index .Values "skywalking-helm" "fullnameOverride") .Release.Namespace 11800 }} +{{- end -}} + +{{- define "skywalking.oap.address.http" -}} +{{- printf "http://%s-oap.%s.svc.cluster.local:%d" (index .Values "skywalking-helm" "fullnameOverride") .Release.Namespace 12800 }} +{{- end -}} diff --git a/deploy/platform/kubernetes/feature-agent/resources.yaml b/deploy/platform/kubernetes/templates/feature-agent/resources.yaml similarity index 69% rename from deploy/platform/kubernetes/feature-agent/resources.yaml rename to deploy/platform/kubernetes/templates/feature-agent/resources.yaml index f62f5bc..aaa2791 100644 --- a/deploy/platform/kubernetes/feature-agent/resources.yaml +++ b/deploy/platform/kubernetes/templates/feature-agent/resources.yaml @@ -16,11 +16,25 @@ # under the License. # +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.sampleServices.namespace }} + labels: + {{- if .Values.agentInjector.enabled }} + swck-injection: enabled + {{- end }} + {{- if or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }} + istio-injection: enabled + {{- end }} + --- apiVersion: v1 kind: Service metadata: name: gateway + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: gateway @@ -34,6 +48,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: gateway-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: gateway spec: @@ -48,17 +63,17 @@ spec: swck-java-agent-injected: "true" app: gateway annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" # @feature: java-agent-injector; set the java agent configuration strategy.skywalking.apache.org/inject.Container: "gateway" strategy.skywalking.apache.org/agent.Overlay: "true" agent.skywalking.apache.org/agent.service_name: "agent::gateway" optional.skywalking.apache.org: "spring-cloud-gateway-3.x" - sidecar.skywalking.apache.org/initcontainer.Image: "${SW_AGENT_JAVA_IMAGE}" + sidecar.skywalking.apache.org/initcontainer.Image: "{{ .Values.agentInjector.javaImage }}" spec: containers: - name: gateway - image: ${HUB}/gateway-service:${TAG} + image: "{{ .Values.sampleServices.hub }}/gateway-service:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent ports: - containerPort: 80 @@ -67,13 +82,14 @@ spec: - name: SW_AGENT_NAME value: agent::gateway - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} --- apiVersion: v1 kind: Service metadata: name: songs + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: songs @@ -87,6 +103,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: songs-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: songs spec: @@ -101,17 +118,17 @@ spec: swck-java-agent-injected: "true" app: songs annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" # @feature: java-agent-injector; set the java agent configuration strategy.skywalking.apache.org/inject.Container: "songs" strategy.skywalking.apache.org/agent.Overlay: "true" agent.skywalking.apache.org/agent.service_name: "agent::songs" optional.skywalking.apache.org: "guava-cache-plugin" - sidecar.skywalking.apache.org/initcontainer.Image: "${SW_AGENT_JAVA_IMAGE}" + sidecar.skywalking.apache.org/initcontainer.Image: "{{ .Values.agentInjector.javaImage }}" spec: containers: - name: songs - image: ${HUB}/songs-service:${TAG} + image: "{{ .Values.sampleServices.hub }}/songs-service:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent ports: - containerPort: 80 @@ -120,17 +137,18 @@ spec: - name: SW_AGENT_NAME value: agent::songs - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} - name: ACTIVE_MQ_URL value: tcp://activemq:61616 - name: ACTIVE_MQ_QUEUE - value: queue-songs-ping + value: queue-songs-ping --- apiVersion: v1 kind: Service metadata: name: rcmd + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: recommendation @@ -144,6 +162,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: recommendation-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: recommendation spec: @@ -156,11 +175,11 @@ spec: labels: app: recommendation annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" spec: containers: - name: recommendation - image: ${HUB}/recommendation-service:${TAG} + image: "{{ .Values.sampleServices.hub }}/recommendation-service:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent ports: - containerPort: 80 @@ -168,13 +187,14 @@ spec: - name: SW_AGENT_NAME value: agent::recommendation - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} --- apiVersion: v1 kind: Service metadata: name: app + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: app @@ -188,6 +208,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: app-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: app spec: @@ -200,11 +221,11 @@ spec: labels: app: app annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" spec: containers: - name: app - image: ${HUB}/app:${TAG} + image: "{{ .Values.sampleServices.hub }}/app:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent ports: - containerPort: 80 @@ -214,13 +235,14 @@ spec: - name: REACT_APP_SW_AGENT_NAME_UI value: agent::ui - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} --- apiVersion: apps/v1 kind: Deployment metadata: name: loadgen-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: loadgen spec: @@ -233,11 +255,11 @@ spec: labels: app: loadgen annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" spec: containers: - name: app - image: ${HUB}/load-gen:${TAG} + image: "{{ .Values.sampleServices.hub }}/load-gen:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent --- @@ -245,6 +267,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: apisix-config + namespace: {{ .Values.sampleServices.namespace }} labels: app: apisix-config data: @@ -258,16 +281,16 @@ data: service_id: 2 plugins: skywalking: - service_name: "${{SW_AGENT_SERVICE}}" - service_instance_name: "${{SW_AGENT_SERVICE_INSTANCE}}" - endpoint_addr: http://oap:12800 + service_name: ${{ "{{" }}}SW_AGENT_SERVICE{{ "}}" }} + service_instance_name: ${{ "{{" }}SW_AGENT_SERVICE_INSTANCE{{ "}}" }} + endpoint_addr: {{ template "skywalking.oap.address.http" . }} - uri: /test service_id: 2 plugins: skywalking: - service_name: "${{SW_AGENT_SERVICE}}" - service_instance_name: "${{SW_AGENT_SERVICE_INSTANCE}}" - endpoint_addr: http://oap:12800 + service_name: ${{"{{"}}SW_AGENT_SERVICE{{"}}"}} + service_instance_name: ${{"{{"}}SW_AGENT_SERVICE_INSTANCE{{"}}"}} + endpoint_addr: {{ template "skywalking.oap.address.http" . }} services: - id: 1 upstream_id: 1 @@ -304,9 +327,9 @@ data: ip: "0.0.0.0" port: 9091 skywalking: - service_name: "${{SW_AGENT_SERVICE}}" - service_instance_name: "${{SW_AGENT_SERVICE_INSTANCE}}" - endpoint_addr: http://oap:12800 + service_name: ${{"{{"}}SW_AGENT_SERVICE{{"}}"}} + service_instance_name: ${{"{{"}}SW_AGENT_SERVICE_INSTANCE{{"}}"}} + endpoint_addr: {{ template "skywalking.oap.address.http" . }} plugins: - prometheus - skywalking @@ -321,6 +344,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: frontend-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: frontend spec: @@ -333,11 +357,11 @@ spec: labels: app: frontend annotations: - sidecar.istio.io/inject: "${AGENTLESS}" + sidecar.istio.io/inject: "{{ or .Values.features.als.enabled .Values.features.istiodMonitor.enabled .Values.features.meshWithAgent }}" spec: containers: - name: frontend - image: ${HUB}/frontend:${TAG} + image: "{{ .Values.sampleServices.hub }}/frontend:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent resources: limits: @@ -353,7 +377,7 @@ spec: name: metrics env: - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} - name: SW_SERVICE_INSTANCE valueFrom: fieldRef: @@ -374,6 +398,7 @@ apiVersion: v1 kind: Service metadata: name: frontend + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: frontend @@ -387,6 +412,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: activemq-deployment + namespace: {{ .Values.sampleServices.namespace }} labels: app: activemq spec: @@ -418,10 +444,11 @@ apiVersion: v1 kind: Service metadata: name: activemq + namespace: {{ .Values.sampleServices.namespace }} spec: selector: app: activemq ports: - protocol: TCP port: 61616 - targetPort: 61616 \ No newline at end of file + targetPort: 61616 diff --git a/deploy/platform/kubernetes/feature-apisix-monitor/permissions.yaml b/deploy/platform/kubernetes/templates/feature-apisix-monitor/permissions.yaml similarity index 92% rename from deploy/platform/kubernetes/feature-apisix-monitor/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-apisix-monitor/permissions.yaml index ac17148..2881866 100644 --- a/deploy/platform/kubernetes/feature-apisix-monitor/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-apisix-monitor/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.apisixMonitor.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -44,4 +46,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-apisix - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-apisix-monitor/resources.yaml b/deploy/platform/kubernetes/templates/feature-apisix-monitor/resources.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-apisix-monitor/resources.yaml rename to deploy/platform/kubernetes/templates/feature-apisix-monitor/resources.yaml index 9c82bca..fef831d 100644 --- a/deploy/platform/kubernetes/feature-apisix-monitor/resources.yaml +++ b/deploy/platform/kubernetes/templates/feature-apisix-monitor/resources.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. # - +{{- if .Values.features.apisixMonitor.enabled }} apiVersion: v1 kind: ConfigMap @@ -102,3 +102,5 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-event/permissions.yaml b/deploy/platform/kubernetes/templates/feature-event/permissions.yaml similarity index 92% rename from deploy/platform/kubernetes/feature-event/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-event/permissions.yaml index 616661f..c06e465 100644 --- a/deploy/platform/kubernetes/feature-event/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-event/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # + +{{- if .Values.features.event.enabled }} --- apiVersion: v1 kind: ServiceAccount @@ -32,5 +34,6 @@ roleRef: name: view subjects: - kind: ServiceAccount - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} name: skywalking-event-exporter-sa +{{- end }} diff --git a/deploy/platform/kubernetes/feature-event/resources.yaml b/deploy/platform/kubernetes/templates/feature-event/resources.yaml similarity index 60% rename from deploy/platform/kubernetes/feature-event/resources.yaml rename to deploy/platform/kubernetes/templates/feature-event/resources.yaml index c7e5065..287f33c 100644 --- a/deploy/platform/kubernetes/feature-event/resources.yaml +++ b/deploy/platform/kubernetes/templates/feature-event/resources.yaml @@ -15,35 +15,17 @@ # specific language governing permissions and limitations # under the License. # -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# +{{- if .Values.features.event.enabled }} --- apiVersion: v1 kind: ConfigMap metadata: name: skywalking-event-exporter-cm - namespace: ${NAMESPACE} data: config.yaml: | filters: - - namespace: ${NAMESPACE} + - namespace: {{ .Release.Namespace }} exporters: - skywalking @@ -51,18 +33,17 @@ data: skywalking: template: source: - service: "agent::{{ .Service.Name }}" - serviceInstance: "{{ .Pod.Name }}" + service: "agent::{{"{{"}} .Service.Name {{"}}"}}" + serviceInstance: "{{"{{"}} .Pod.Name {{"}}"}}" endpoint: "" - message: "{{ .Event.Message }}" - address: "oap:11800" + message: "{{"{{"}} .Event.Message {{"}}"}}" + address: "{{ .Release.Name }}-skywalking-helm:11800" --- apiVersion: apps/v1 kind: Deployment metadata: name: skywalking-event-exporter-deployment - namespace: ${NAMESPACE} spec: replicas: 1 template: @@ -73,8 +54,8 @@ spec: serviceAccountName: skywalking-event-exporter-sa containers: - name: skywalking-event-exporter - image: ${SW_EVENT_EXPORTER_IMAGE} - imagePullPolicy: Always + image: "{{ .Values.eventExporter.image }}" + imagePullPolicy: IfNotPresent args: - start - -v=debug @@ -89,3 +70,4 @@ spec: selector: matchLabels: app: skywalking-event-exporter +{{- end }} diff --git a/deploy/platform/kubernetes/feature-event/scaler.yaml b/deploy/platform/kubernetes/templates/feature-event/scaler.yaml similarity index 80% rename from deploy/platform/kubernetes/feature-event/scaler.yaml rename to deploy/platform/kubernetes/templates/feature-event/scaler.yaml index 03cd3a3..b991aa4 100644 --- a/deploy/platform/kubernetes/feature-event/scaler.yaml +++ b/deploy/platform/kubernetes/templates/feature-event/scaler.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.event.enabled }} --- # @feature: event; a cron job to mimic the restart of Pods @@ -22,6 +23,7 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: event-role + namespace: {{ .Values.sampleServices.namespace }} rules: - apiGroups: - "" @@ -37,6 +39,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: event-role-binding + namespace: {{ .Values.sampleServices.namespace }} subjects: - kind: ServiceAccount name: event-scaler-sa @@ -50,12 +53,14 @@ apiVersion: v1 kind: ServiceAccount metadata: name: event-scaler-sa + namespace: {{ .Values.sampleServices.namespace }} --- apiVersion: batch/v1 kind: CronJob metadata: name: event-scale-up + namespace: {{ .Values.sampleServices.namespace }} spec: schedule: "0,10,20,30,40,50 * * * *" successfulJobsHistoryLimit: 1 @@ -74,7 +79,7 @@ spec: command: - sh - -c - - kubectl -n ${NAMESPACE} scale deployment songs-deployment gateway-deployment app-deployment --replicas=2 + - kubectl -n {{ .Values.sampleServices.namespace }} scale deployment songs-deployment gateway-deployment app-deployment --replicas=2 restartPolicy: Never --- @@ -82,6 +87,7 @@ apiVersion: batch/v1 kind: CronJob metadata: name: event-scale-down + namespace: {{ .Values.sampleServices.namespace }} spec: schedule: "5,15,25,35,45,55 * * * *" successfulJobsHistoryLimit: 1 @@ -100,5 +106,6 @@ spec: command: - sh - -c - - kubectl -n ${NAMESPACE} scale deployment songs-deployment gateway-deployment app-deployment --replicas=1 + - kubectl -n {{ .Values.sampleServices.namespace }} scale deployment songs-deployment gateway-deployment app-deployment --replicas=1 restartPolicy: Never +{{- end }} diff --git a/deploy/platform/kubernetes/feature-function/resource.yaml b/deploy/platform/kubernetes/templates/feature-function/resource.yaml similarity index 81% rename from deploy/platform/kubernetes/feature-function/resource.yaml rename to deploy/platform/kubernetes/templates/feature-function/resource.yaml index c70b13c..af182b4 100644 --- a/deploy/platform/kubernetes/feature-function/resource.yaml +++ b/deploy/platform/kubernetes/templates/feature-function/resource.yaml @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. # + +{{- if .Values.features.function.enabled }} + --- apiVersion: apps/v1 kind: Deployment @@ -34,11 +37,11 @@ spec: spec: containers: - name: app - image: ${HUB}/function-load-gen:${TAG} + image: "{{ .Values.sampleServices.hub }}/function-load-gen:{{ .Values.sampleServices.tag }}" imagePullPolicy: IfNotPresent env: - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} - name: URL value: http://function.default.svc.cluster.local/World - name: SW_AGENT_LAYER @@ -53,10 +56,10 @@ metadata: enabled: true provider: name: "skywalking" - oapServer: ${BACKEND_SERVICE}:11800 + oapServer: {{ template "skywalking.collector.address" . }} spec: version: "v2.0.0" - image: ${HUB}/function:${TAG} + image: {{ .Values.sampleServices.hub }}/function:{{ .Values.sampleServices.tag }} port: 8080 serving: template: @@ -64,3 +67,5 @@ spec: - name: function imagePullPolicy: IfNotPresent runtime: "knative" + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-istiod-monitor/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-istiod-monitor/open-telemetry.yaml similarity index 95% rename from deploy/platform/kubernetes/feature-istiod-monitor/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-istiod-monitor/open-telemetry.yaml index db98d5d..743a9df 100644 --- a/deploy/platform/kubernetes/feature-istiod-monitor/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-istiod-monitor/open-telemetry.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.istiodMonitor.enabled }} # @feature: istiod-monitor; set OpenTelemetry config to scrape the Istio control plane metrics --- @@ -33,7 +34,7 @@ data: exporters: [ logging, otlp ] exporters: otlp: - endpoint: "oap:11800" + endpoint: "{{ include "skywalking.oap.address.grpc" . }}" tls: insecure: true logging: @@ -92,3 +93,4 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml +{{- end }} diff --git a/deploy/platform/kubernetes/feature-istiod-monitor/permissions.yaml b/deploy/platform/kubernetes/templates/feature-istiod-monitor/permissions.yaml similarity index 93% rename from deploy/platform/kubernetes/feature-istiod-monitor/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-istiod-monitor/permissions.yaml index 34924f4..4dfb9a7 100644 --- a/deploy/platform/kubernetes/feature-istiod-monitor/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-istiod-monitor/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.istiodMonitor.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -47,4 +49,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-istio-monitor - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-kubernetes-monitor/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-kubernetes-monitor/open-telemetry.yaml similarity index 96% rename from deploy/platform/kubernetes/feature-kubernetes-monitor/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-kubernetes-monitor/open-telemetry.yaml index e352eb9..b2e5e43 100644 --- a/deploy/platform/kubernetes/feature-kubernetes-monitor/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-kubernetes-monitor/open-telemetry.yaml @@ -16,6 +16,8 @@ # under the License. # +{{- if .Values.features.kubernetesMonitor.enabled }} + # @feature: kubernetes-monitor; set OpenTelemetry config to scrape the Kubernetes metrics --- apiVersion: v1 @@ -33,7 +35,7 @@ data: exporters: [ logging, otlp ] exporters: otlp: - endpoint: "oap:11800" + endpoint: "{{ include "skywalking.oap.address.grpc" . }}" tls: insecure: true logging: @@ -120,3 +122,5 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-kubernetes-monitor/permissions.yaml b/deploy/platform/kubernetes/templates/feature-kubernetes-monitor/permissions.yaml similarity index 93% rename from deploy/platform/kubernetes/feature-kubernetes-monitor/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-kubernetes-monitor/permissions.yaml index 80f9e93..a5f8af7 100644 --- a/deploy/platform/kubernetes/feature-kubernetes-monitor/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-kubernetes-monitor/permissions.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.kubernetesMonitor.enabled }} --- apiVersion: v1 kind: ServiceAccount @@ -50,4 +51,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-kubernetes-monitor - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-mysql/fluent-bit.yaml b/deploy/platform/kubernetes/templates/feature-mysql-monitor/fluent-bit.yaml similarity index 97% rename from deploy/platform/kubernetes/feature-mysql/fluent-bit.yaml rename to deploy/platform/kubernetes/templates/feature-mysql-monitor/fluent-bit.yaml index 6a48c40..adc2685 100644 --- a/deploy/platform/kubernetes/feature-mysql/fluent-bit.yaml +++ b/deploy/platform/kubernetes/templates/feature-mysql-monitor/fluent-bit.yaml @@ -16,6 +16,8 @@ # under the License. # +{{- if .Values.features.mysqlMonitor.enabled }} + # @feature: slowsql-mysql; fluent bit log configurations --- apiVersion: v1 @@ -69,7 +71,7 @@ data: log = record["log"] record["log"] = nil record["date"] = nil - record["tags"] = {data={{key="LOG_KIND", value="SLOW_SQL"}}} + record["tags"] = { data={ { key="LOG_KIND", value="SLOW_SQL" } } } arr = split(log,"\n") re1 = {} @@ -170,3 +172,5 @@ data: rule "second_state" "^(# User@Host.*)" "third_state" rule "third_state" "^(# Query_time: .*)" "statement" rule "statement" "^\w+.*" "statement" + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-mysql/mysql.yaml b/deploy/platform/kubernetes/templates/feature-mysql-monitor/mysql.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-mysql/mysql.yaml rename to deploy/platform/kubernetes/templates/feature-mysql-monitor/mysql.yaml index 795cdca..b5f15f5 100644 --- a/deploy/platform/kubernetes/feature-mysql/mysql.yaml +++ b/deploy/platform/kubernetes/templates/feature-mysql-monitor/mysql.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.mysqlMonitor.enabled }} apiVersion: v1 kind: ConfigMap @@ -197,3 +198,4 @@ spec: items: - key: mock-sql path: mock.sql +{{- end }} diff --git a/deploy/platform/kubernetes/feature-mysql/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-mysql-monitor/open-telemetry.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-mysql/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-mysql-monitor/open-telemetry.yaml index 8ec063c..00fc27e 100644 --- a/deploy/platform/kubernetes/feature-mysql/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-mysql-monitor/open-telemetry.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.mysqlMonitor.enabled }} # @feature: mysql; set OpenTelemetry config to scrape the mysql metrics --- @@ -100,3 +101,4 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml +{{- end }} diff --git a/deploy/platform/kubernetes/feature-mysql/permissions.yaml b/deploy/platform/kubernetes/templates/feature-mysql-monitor/permissions.yaml similarity index 92% rename from deploy/platform/kubernetes/feature-mysql/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-mysql-monitor/permissions.yaml index d5ed9d4..651a720 100644 --- a/deploy/platform/kubernetes/feature-mysql/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-mysql-monitor/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.mysqlMonitor.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -44,4 +46,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-mysql - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-postgresql/fluent-bit.yaml b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/fluent-bit.yaml similarity index 97% rename from deploy/platform/kubernetes/feature-postgresql/fluent-bit.yaml rename to deploy/platform/kubernetes/templates/feature-postgresql-monitor/fluent-bit.yaml index f69acae..b58f0a0 100644 --- a/deploy/platform/kubernetes/feature-postgresql/fluent-bit.yaml +++ b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/fluent-bit.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.postgresqlMonitor.enabled }} # @feature: slowsql-psql; fluent bit log configurations --- @@ -60,7 +61,7 @@ data: log = record["log"] record["log"] = nil record["date"] = nil - record["tags"] = {data={{key="LOG_KIND", value="SLOW_SQL"}}} + record["tags"] = { data={ { key="LOG_KIND", value="SLOW_SQL" } } } arr = split(log,"\n") re1 = {} @@ -145,3 +146,5 @@ data: name my-log-format format regex regex \w*-\w*-\w* \w*:\w*:\w*.\w* UTCLOG: duration: \w*.\w* ms statement.* + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-postgresql/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/open-telemetry.yaml similarity index 97% rename from deploy/platform/kubernetes/feature-postgresql/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-postgresql-monitor/open-telemetry.yaml index 3a55902..8601343 100644 --- a/deploy/platform/kubernetes/feature-postgresql/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/open-telemetry.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.postgresqlMonitor.enabled }} # @feature: postgresql; set OpenTelemetry config to scrape the psql metrics --- @@ -102,3 +103,5 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-postgresql/permissions.yaml b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/permissions.yaml similarity index 92% rename from deploy/platform/kubernetes/feature-postgresql/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-postgresql-monitor/permissions.yaml index 79d6b41..781ea41 100644 --- a/deploy/platform/kubernetes/feature-postgresql/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.postgresqlMonitor.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -44,4 +46,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-psql - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-postgresql/psql.yaml b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/psql.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-postgresql/psql.yaml rename to deploy/platform/kubernetes/templates/feature-postgresql-monitor/psql.yaml index 047f173..122b796 100644 --- a/deploy/platform/kubernetes/feature-postgresql/psql.yaml +++ b/deploy/platform/kubernetes/templates/feature-postgresql-monitor/psql.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.postgresqlMonitor.enabled }} --- apiVersion: v1 @@ -220,3 +221,5 @@ spec: items: - key: mock-sql path: mock.sql + +{{- end }} diff --git a/deploy/platform/kubernetes/Makefile.in b/deploy/platform/kubernetes/templates/feature-promql/configmaps.yaml similarity index 71% copy from deploy/platform/kubernetes/Makefile.in copy to deploy/platform/kubernetes/templates/feature-promql/configmaps.yaml index 6ddc6e6..d12d9c1 100644 --- a/deploy/platform/kubernetes/Makefile.in +++ b/deploy/platform/kubernetes/templates/feature-promql/configmaps.yaml @@ -15,11 +15,16 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.promql.enabled }} -.EXPORT_ALL_VARIABLES: +apiVersion: v1 +kind: ConfigMap +metadata: + name: promql-configs +data: +{{ range $path, $bytes := .Files.Glob "generated/promql-configs/*.json" }} +{{- $path | replace "generated/promql-configs/" "" | indent 2 }}: | +{{ $.Files.Get $path | indent 4 }} +{{ end }} -NAMESPACE ?= default -AGENTLESS ?= false -BACKEND_SERVICE ?= oap - -FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql,apisix-monitor,promql +{{- end }} diff --git a/deploy/platform/kubernetes/feature-promql/resources.yaml b/deploy/platform/kubernetes/templates/feature-promql/resources.yaml similarity index 87% rename from deploy/platform/kubernetes/feature-promql/resources.yaml rename to deploy/platform/kubernetes/templates/feature-promql/resources.yaml index 6c66cf3..df74201 100644 --- a/deploy/platform/kubernetes/feature-promql/resources.yaml +++ b/deploy/platform/kubernetes/templates/feature-promql/resources.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.promql.enabled }} #This config is for the showcase, when deploy your own environment fell free to modify it. --- @@ -84,7 +85,7 @@ spec: spec: containers: - name: grafana - image: ${GRAFANA_IMAGE} + image: {{ .Values.grafana.image }} imagePullPolicy: IfNotPresent ports: - name: http-grafana @@ -99,28 +100,22 @@ spec: - name: grafana-config mountPath: /etc/grafana/provisioning/dashboards/providers.yaml subPath: providers.yaml - - name: dashboards-general-service + - name: promql-configs mountPath: /var/lib/grafana/dashboards/skywalking/general-service.json subPath: general-service.json - - name: dashboards-service-mesh + - name: promql-configs mountPath: /var/lib/grafana/dashboards/skywalking/service-mesh.json subPath: service-mesh.json - - name: dashboards-virtual-database + - name: promql-configs mountPath: /var/lib/grafana/dashboards/skywalking/virtual-database.json subPath: virtual-database.json volumes: # require create configmap from deploy/platform/config/promql/dashboards - name: grafana-config configMap: name: grafana-config - - name: dashboards-general-service + - name: promql-configs configMap: - name: grafana-dashboards-general-service - - name: dashboards-service-mesh - configMap: - name: grafana-dashboards-service-mesh - - name: dashboards-virtual-database - configMap: - name: grafana-dashboards-virtual-database + name: promql-configs --- apiVersion: v1 @@ -134,3 +129,5 @@ spec: - port: 3000 protocol: TCP targetPort: http-grafana + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-rover/permissions.yaml b/deploy/platform/kubernetes/templates/feature-rover/permissions.yaml similarity index 93% rename from deploy/platform/kubernetes/feature-rover/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-rover/permissions.yaml index 8b1da58..6542cd5 100644 --- a/deploy/platform/kubernetes/feature-rover/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-rover/permissions.yaml @@ -15,12 +15,12 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.rover.enabled}} apiVersion: v1 kind: ServiceAccount metadata: name: skywalking-rover-sa - namespace: ${NAMESPACE} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -42,4 +42,6 @@ roleRef: subjects: - kind: ServiceAccount name: skywalking-rover-sa - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-rover/profiling.yaml b/deploy/platform/kubernetes/templates/feature-rover/profiling.yaml similarity index 68% rename from deploy/platform/kubernetes/feature-rover/profiling.yaml rename to deploy/platform/kubernetes/templates/feature-rover/profiling.yaml index 9a1d005..de1a862 100644 --- a/deploy/platform/kubernetes/feature-rover/profiling.yaml +++ b/deploy/platform/kubernetes/templates/feature-rover/profiling.yaml @@ -15,11 +15,13 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.rover.enabled}} apiVersion: v1 kind: ConfigMap metadata: name: profiling-config + namespace: {{ .Values.sampleServices.namespace }} labels: app: profiling-config data: @@ -37,6 +39,7 @@ apiVersion: batch/v1 kind: CronJob metadata: name: ebpf-profiling-trgigger + namespace: {{ .Values.sampleServices.namespace }} spec: schedule: "10 */2 * * *" successfulJobsHistoryLimit: 1 @@ -50,28 +53,29 @@ spec: spec: containers: - name: ebpf-profiling-trigger - image: ${SW_CLI_IMAGE} + image: "{{ .Values.swctl.image }}" + imagePullPolicy: IfNotPresent command: - /bin/sh - -c - > - /swctl --base-url=http://oap:12800/graphql profiling ebpf create fixed \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create fixed \ --service-name mesh-svr::recommendation --labels mesh-envoy --duration=9m --target-type ON_CPU; - /swctl --base-url=http://oap:12800/graphql profiling ebpf create fixed \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create fixed \ --service-name mesh-svr::app --labels mesh-envoy --duration=9m --target-type ON_CPU; - /swctl --base-url=http://oap:12800/graphql profiling ebpf create fixed \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create fixed \ --service-name mesh-svr::songs --labels mesh-envoy --duration=9m --target-type ON_CPU; - /swctl --base-url=http://oap:12800/graphql profiling ebpf create fixed \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create fixed \ --service-name mesh-svr::gateway --labels mesh-envoy --duration=9m --target-type ON_CPU; - /swctl --base-url=http://oap:12800/graphql profiling ebpf create fixed \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create fixed \ --service-name mesh-svr::songs --labels mesh-envoy --duration=9m --target-type OFF_CPU; apk update && apk add yq; - /swctl --base-url=http://oap:12800/graphql profiling ebpf create network \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling ebpf create network \ --sampling-config=/profiling/network-profiling-sampling.yaml \ --service-name=mesh-svr::recommendation \ - --instance-name=$(/swctl --base-url=http://oap:12800/graphql instance ls --service-name mesh-svr::recommendation|yq e '.[0].name' -); + --instance-name=$(/swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql instance ls --service-name mesh-svr::recommendation|yq e '.[0].name' -); volumeMounts: - name: profiling-config mountPath: /profiling @@ -80,3 +84,4 @@ spec: - name: profiling-config configMap: name: profiling-config +{{- end }} diff --git a/deploy/platform/kubernetes/feature-rover/resources.yaml b/deploy/platform/kubernetes/templates/feature-rover/resources.yaml similarity index 78% rename from deploy/platform/kubernetes/feature-rover/resources.yaml rename to deploy/platform/kubernetes/templates/feature-rover/resources.yaml index a7fa9f8..ce4d75e 100644 --- a/deploy/platform/kubernetes/feature-rover/resources.yaml +++ b/deploy/platform/kubernetes/templates/feature-rover/resources.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.rover.enabled}} + apiVersion: apps/v1 kind: DaemonSet metadata: @@ -33,8 +35,8 @@ spec: serviceAccountName: skywalking-rover-sa containers: - name: rover - image: ${SW_ROVER_IMAGE} - imagePullPolicy: Always + image: {{ .Values.rover.image }} + imagePullPolicy: IfNotPresent securityContext: capabilities: add: @@ -53,13 +55,13 @@ spec: fieldRef: fieldPath: spec.nodeName - name: ROVER_BACKEND_ADDR - value: ${BACKEND_SERVICE}:11800 + value: {{ template "skywalking.collector.address" . }} - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_ISTIO_ENVOY_SERVICE_NAME - value: mesh-svr::{{.Pod.LabelValue "service.istio.io/canonical-name,app.kubernetes.io/name,app"}} + value: mesh-svr::{{"{{"}}.Pod.LabelValue "service.istio.io/canonical-name,app.kubernetes.io/name,app"{{"}}"}} - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_ISTIO_APPLICATION_SERVICE_NAME - value: mesh-svr::{{.Pod.LabelValue "service.istio.io/canonical-name,app.kubernetes.io/name,app"}} + value: mesh-svr::{{"{{"}}.Pod.LabelValue "service.istio.io/canonical-name,app.kubernetes.io/name,app"{{"}}"}} - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_K8S_SERVICE_NAME - value: skywalking-showcase::{{.Pod.ServiceName}}.{{.Pod.Namespace}} + value: skywalking-showcase::{{"{{"}}.Pod.ServiceName{{"}}"}}.{{"{{"}}.Pod.Namespace{{"}}"}} - name: ROVER_HOST_MAPPING value: /host hostPID: true @@ -69,4 +71,6 @@ spec: - name: host hostPath: path: / - type: Directory \ No newline at end of file + type: Directory + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-so11y/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-so11y/open-telemetry.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-so11y/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-so11y/open-telemetry.yaml index afa7832..d3c336f 100644 --- a/deploy/platform/kubernetes/feature-so11y/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-so11y/open-telemetry.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.so11y.enabled }} # @feature: so11y; in Kubernetes, it's better to set service discovery so the OAP instances are not always "localhost:1234" @@ -97,3 +98,5 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-so11y/permissions.yaml b/deploy/platform/kubernetes/templates/feature-so11y/permissions.yaml similarity index 93% rename from deploy/platform/kubernetes/feature-so11y/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-so11y/permissions.yaml index bfdc6e5..7df3a3b 100644 --- a/deploy/platform/kubernetes/feature-so11y/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-so11y/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.so11y.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -44,4 +46,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-so11y - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-trace-profiling/profiling.yaml b/deploy/platform/kubernetes/templates/feature-trace-profiling/profiling.yaml similarity index 82% rename from deploy/platform/kubernetes/feature-trace-profiling/profiling.yaml rename to deploy/platform/kubernetes/templates/feature-trace-profiling/profiling.yaml index 797da6c..f065988 100644 --- a/deploy/platform/kubernetes/feature-trace-profiling/profiling.yaml +++ b/deploy/platform/kubernetes/templates/feature-trace-profiling/profiling.yaml @@ -16,10 +16,13 @@ # under the License. # +{{- if .Values.features.traceProfiling.enabled }} + apiVersion: batch/v1 kind: CronJob metadata: name: trace-profiling-trgigger + namespace: {{ .Values.sampleServices.namespace }} spec: schedule: "8 0 * * *" successfulJobsHistoryLimit: 1 @@ -33,11 +36,13 @@ spec: spec: containers: - name: trace-profiling-trigger - image: ${SW_CLI_IMAGE} + image: "{{ .Values.swctl.image }}" command: - /bin/sh - -c - > - /swctl --base-url=http://oap:12800/graphql profiling trace create --service-name agent::songs \ + /swctl --base-url={{ template "skywalking.oap.address.http" . }}/graphql profiling trace create --service-name agent::songs \ --endpoint-name GET:/songs --duration=15 --min-duration-threshold=0 --dump-period=10 --max-sampling-count=9 restartPolicy: Never + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-vm/open-telemetry.yaml b/deploy/platform/kubernetes/templates/feature-vm-monitor/open-telemetry.yaml similarity index 98% rename from deploy/platform/kubernetes/feature-vm/open-telemetry.yaml rename to deploy/platform/kubernetes/templates/feature-vm-monitor/open-telemetry.yaml index a1996da..06126e0 100644 --- a/deploy/platform/kubernetes/feature-vm/open-telemetry.yaml +++ b/deploy/platform/kubernetes/templates/feature-vm-monitor/open-telemetry.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.vmMonitor.enabled }} # @feature: vm; set OpenTelemetry config to scrape the VM metrics --- @@ -94,3 +95,5 @@ spec: items: - key: otel-collector-config path: otel-collector-config.yaml + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-vm/permissions.yaml b/deploy/platform/kubernetes/templates/feature-vm-monitor/permissions.yaml similarity index 93% rename from deploy/platform/kubernetes/feature-vm/permissions.yaml rename to deploy/platform/kubernetes/templates/feature-vm-monitor/permissions.yaml index 02fb039..bae8a37 100644 --- a/deploy/platform/kubernetes/feature-vm/permissions.yaml +++ b/deploy/platform/kubernetes/templates/feature-vm-monitor/permissions.yaml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.vmMonitor.enabled }} + --- apiVersion: v1 kind: ServiceAccount @@ -44,4 +46,6 @@ roleRef: subjects: - kind: ServiceAccount name: otel-sa-vm - namespace: ${NAMESPACE} + namespace: {{ .Release.Namespace }} + +{{- end }} diff --git a/deploy/platform/kubernetes/feature-vm/vm.yaml b/deploy/platform/kubernetes/templates/feature-vm-monitor/vm.yaml similarity index 96% rename from deploy/platform/kubernetes/feature-vm/vm.yaml rename to deploy/platform/kubernetes/templates/feature-vm-monitor/vm.yaml index b7844e1..c5b2f05 100644 --- a/deploy/platform/kubernetes/feature-vm/vm.yaml +++ b/deploy/platform/kubernetes/templates/feature-vm-monitor/vm.yaml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +{{- if .Values.features.vmMonitor.enabled }} # @feature: pretend these Pods are virtual machines @@ -59,3 +60,5 @@ spec: ports: - containerPort: 9100 name: metrics + +{{- end }} diff --git a/deploy/platform/kubernetes/values.yaml b/deploy/platform/kubernetes/values.yaml new file mode 100644 index 0000000..e77ea11 --- /dev/null +++ b/deploy/platform/kubernetes/values.yaml @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +skywalking-helm: + fullnameOverride: "sw-demo" + oap: + name: oap + image: + repository: ghcr.io/apache/skywalking/oap + tag: af657356ed61c2595d70dd18eae5a6e5dbc39a82 + pullPolicy: IfNotPresent + replicas: 1 + storageType: ~ + env: + SW_HEALTH_CHECKER: default + SW_OTEL_RECEIVER: default # @feature: so11y;,mysql;kubernetes-monitor enable OpenTelemetry receiver to receive OpenTelemetry metrics + # @feature: vm; enable vm rules to analyze VM metrics + # @feature: so11y; enable oap rules to analyze OAP metrics + # @feature: kubernetes-monitor; enable rules to analyze Kubernetes Cluster/Node/Service metrics + # @feature: istiod-monitor; enable rules to analyze Istio control plane metrics + # @feature: mysql; enable mysql rules to analyze MySQL metrics + # @feature: apisix-monitor; enable APISIX rules to analyze APISIX metrics + SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm,oap,k8s/*,istio-controlplane,mysql/*,postgresql/*,apisix + SW_TELEMETRY: prometheus # @feature: so11y; expose the metrics of self o11y through prometheus + SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS: "mx-mesh,persistence" # @feature: als; enable mesh analyzer (mx-mesh) to analyze ALS logs + K8S_SERVICE_NAME_RULE: "mesh-svr::${service.metadata.name}" + SW_CONFIGURATION: k8s-configmap + SW_ENVOY_METRIC_ALS_TCP_ANALYSIS: "mx-mesh" + SW_ENABLE_UPDATE_UI_TEMPLATE: "false" + SW_SLOW_DB_THRESHOLD: default:0,mongodb:100 + SW_ENABLE_ON_DEMAND_POD_LOG: "true" + SW_METER_ANALYZER_ACTIVE_FILES: datasource,threadpool,satellite,network-profiling,spring-sleuth + SW_STORAGE_ES_RESPONSE_TIMEOUT: "50000" + SW_STORAGE_ES_INDEX_SHARDS_NUMBER: "6" + SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR: "2" + SW_RECEIVER_ZIPKIN: default + SW_QUERY_ZIPKIN: default + + config: + metadata-service-mapping.yaml: | + serviceName: mesh-svr::${LABELS."service.istio.io/canonical-name",LABELS."app.kubernetes.io/name",LABELS.app} + serviceInstanceName: ${NAME} + alarm-settings.yml: | + rules: + service_resp_time_rule: + metrics-name: service_resp_time + threshold: 20 # 20ms is just for demo + op: ">" + period: 5 + count: 1 + message: Response time of service {name} is more than 20ms. + core.default.log4j-xml: |- + <Configuration status="WARN"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout charset="UTF-8" pattern="%d - %c - %L [%t] %-5p %x - %m%n"/> + </Console> + </Appenders> + <Loggers> + <logger name="io.grpc.netty" level="INFO"/> + <logger name="org.apache.skywalking.oap.server.configuration.api" level="TRACE"/> + <logger name="org.apache.skywalking.oap.server.configuration.configmap" level="DEBUG"/> + <logger name="org.apache.skywalking.oap.server.storage.plugin.jdbc" level="DEBUG"/> + <Root level="INFO"> + <AppenderRef ref="Console"/> + </Root> + </Loggers> + </Configuration> + ui: + name: ui + replicas: 1 + image: + repository: ghcr.io/apache/skywalking/ui + tag: af657356ed61c2595d70dd18eae5a6e5dbc39a82 + pullPolicy: IfNotPresent + + elasticsearch: + enabled: false + + postgresql: + enabled: false # Whether to start a demo postgresql deployment, don't use this for production. + config: + # The hostname of your own postgresql service, this only takes effect when postgresql.enabled is false. + host: postgresql-service.your-awesome-company.com + auth: + username: postgres + password: "123456" + postgresPassword: "123456" + database: skywalking + containerPorts: + postgresql: 5432 + primary: + persistence: + enabled: false + readReplicas: + persistence: + enabled: false + + satellite: + name: satellite + replicas: 1 + enabled: true + image: + repository: ghcr.io/apache/skywalking-satellite/skywalking-satellite + tag: v672ec32414c38f3589124c6131b0b4473cda0de5 + pullPolicy: IfNotPresent + ports: + grpc: 11800 + prometheus: 1234 + env: + SATELLITE_TELEMETRY_EXPORT_TYPE: metrics_service + podAnnotations: + sidecar.istio.io/inject: "false" + +rover: + image: ghcr.io/apache/skywalking-rover/skywalking-rover:668abb2fbc38250d2bd825e62ec5678bb8017864 + +eventExporter: + image: ghcr.io/apache/skywalking-kubernetes-event-exporter/skywalking-kubernetes-event-exporter:8a012a3f968cb139f817189afb9b3748841bba22 + +swctl: + image: ghcr.io/apache/skywalking-cli/skywalking-cli:0883266bfaa36612927b69e35781b64ea181758d + +sampleServices: + hub: ~ + tag: ~ + namespace: sample-services + +grafana: + image: grafana/grafana:9.1.0 + +# @feature: java-agent-injector; enable java agent injector and set the java agent image +agentInjector: + enabled: false + javaImage: ghcr.io/apache/skywalking-java/skywalking-java:51161ae6a5b8e266eef39162cc4e23440d36ab38-java8 + +# @feature: java-agent-injector; the java agent injector is a component of the swck operator, so we need to deploy the swck operator firstly +skywalking-helm-swck-operator: + image: + repository: docker.io/apache/skywalking-swck + tag: v0.7.0 + # @feature: java-agent-injector; the swck operator depends on the certificate management of the cert-manager + cert-manager: + enabled: true + fullnameOverride: "swck-demo" + +# @feature: function; install open function resources +openfunction: + enabled: false + global: + ShipwrightBuild: + enabled: false + TektonPipelines: + enabled: false + Keda: + enabled: false + Dapr: + enabled: false + contour: + envoy: + useHostPort: false + service: + type: ClusterIP + externalTrafficPolicy: "" + +istio: + enabled: false + +base: + global: + istioNamespace: demo + +istiod: + global: + istioNamespace: demo + meshConfig: + enableEnvoyAccessLogService: false + enableTracing: false + defaultConfig: + tracing: + sampling: 100 + zipkin: + address: ~ + envoyMetricsService: + address: ~ + envoyAccessLogService: + address: ~ + proxyStatsMatcher: + inclusionRegexps: + - ".*membership_healthy.*" + - ".*upstream_cx_active.*" + - ".*upstream_cx_total.*" + - ".*upstream_rq_active.*" + - ".*upstream_rq_total.*" + - ".*upstream_rq_pending_active.*" + - ".*lb_healthy_panic.*" + - ".*upstream_cx_none_healthy.*" + +# @feature: kubernetes-monitor; extra resources to install for kubernetes monitoring, standard kube-state-metrics +kube-state-metrics: + enabled: false + +features: + kubernetesMonitor: + enabled: false + + mysqlMonitor: + enabled: false + + postgresqlMonitor: + enabled: false + + event: + enabled: false + + istiodMonitor: + enabled: false + + apisixMonitor: + enabled: false + + traceProfiling: + enabled: false + + rover: + enabled: false + + vmMonitor: + enabled: false + + promql: + enabled: false + + function: + enabled: false + + so11y: + enabled: false + + als: + enabled: false + + meshWithAgent: + enabled: false + + agent: + enabled: false diff --git a/docs/readme.md b/docs/readme.md index cd99032..83a346c 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -92,7 +92,7 @@ Currently, the features supported are: | `single-node` | Deploy only one single node of SkyWalking OAP, and SkyWalking UI, ElasticSearch as storage. | Only one of `cluster` or `single-node` can be enabled. | | `elasticsearch` | Deploy a single-node ElasticSearch as storage, you may want to disable this if you want to use your own ElasticSearch deployments. | | | `so11y` | Enable SkyWalking self observability. | This is enabled by default for platform [Docker Compose](#docker-compose). | -| `vm` | Start 2 virtual machines and export their metrics to SkyWalking. | The "virtual machines" are mimicked by Docker containers or Pods. | +| `vm-monitor` | Start 2 virtual machines and export their metrics to SkyWalking. | The "virtual machines" are mimicked by Docker containers or Pods. | | `als` | Start microservices WITHOUT SkyWalking agent enabled, and configure SkyWalking to analyze the topology and metrics from their access logs. | Command `istioctl` is required to run this feature. The agentless microservices will be running at namespace `${NAMESPACE}-agentless` | | `kubernetes-monitor` | Deploy OpenTelemetry and export Kubernetes monitoring metrics to SkyWalking for analysis and display on UI. | | | `istiod-monitor` | Deploy OpenTelemetry and export Istio control plane metrics to SkyWalking for analysis and display on UI. | | @@ -101,11 +101,11 @@ Currently, the features supported are: | `function` | Deploy [OpenFunction](https://openfunction.dev/) and export trace to SkyWalking. | Command `helm` is required to run this feature. | | `trace-profiling` | Deploy tools to submit trace profiling tasks. | Only support deployment with SkyWalking agents installed, currently Java agent and Python agent support trace profiling. | | `rover` | Deploy SkyWalking Rover and detect the processes in the Kubernetes environment. | Only support deployment in the Kubernetes environment, docker is not supported. | -| `mysql` | Start a MySQL server and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | -| `postgresql` | Start a PostgreSQL server, and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | +| `mysql-monitor` | Start a MySQL server and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | +| `postgresql-monitor` | Start a PostgreSQL server, and load generator to execute the sample SQLs periodically, set up fluent bit to fetch slow logs and export to OAP, and export their metrics to SkyWalking. | | | `apisix-monitor` | Deploy OpenTelemetry and export APISIX metrics to SkyWalking for analysis and display on UI | | | `mesh-with-agent` | Deploy services with java agent in the service mesh environment. | Only support deployment in the Kubernetes environment, docker is not supported. | -| `promql` | Deploy a Grafana to use promql service and show SkyWalking UI on the Grafana. | Feel free to modify the Grafana config when deploy your own environment. | +| `promql` | Deploy a Grafana to use promql service and show SkyWalking UI on the Grafana. | Feel free to modify the Grafana config when deploy your own environment. | ### Kubernetes diff --git a/scripts/wait-cert-manager-ready.sh b/scripts/wait-cert-manager-ready.sh deleted file mode 100644 index 2067170..0000000 --- a/scripts/wait-cert-manager-ready.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -TIMEOUT=120 - -MANIFEST=$(mktemp) - -cat << EOF > $MANIFEST -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: test-selfsigned - namespace: default -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: selfsigned-cert - namespace: default -spec: - dnsNames: - - example.com - secretName: selfsigned-cert-tls - issuerRef: - name: test-selfsigned -EOF - -( bash -c -- "\ - while ! kubectl apply -f $MANIFEST 2> /dev/null; \ - do \ - sleep 0.1; \ - done" ) & pid=$! -( sleep $TIMEOUT && pkill -HUP $pid ) 2>/dev/null & watcher=$! -if wait $pid 2>/dev/null; then - pkill -HUP -P $watcher - wait $watcher -fi - -# make sure the dummy Issuer and Certificate will be deleted -trap "kubectl delete -f $MANIFEST; rm $MANIFEST" 0 2 3 15