This is an automated email from the ASF dual-hosted git repository. liuhan pushed a commit to branch node-registry in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb-helm.git
commit ddc597567fbcafe1254b2e543b97e344dfc1dc93 Author: mrproliu <[email protected]> AuthorDate: Tue Jan 6 14:14:07 2026 -0800 Support node discovery configuration in cluster mode --- CHANGES.md | 7 ++ chart/templates/_helpers.tpl | 118 +++++++++++++++++++++++ chart/templates/cluster_data_statefulset.yaml | 6 +- chart/templates/cluster_liaison_statefulset.yaml | 1 + chart/values.yaml | 50 ++++++++++ test/e2e/e2e-banyandb-node-registry-dns.yaml | 73 ++++++++++++++ test/e2e/expected/nodes-data.yaml | 67 +++++++++++++ test/e2e/expected/nodes-liaison.yaml | 109 +++++++++++++++++++++ test/e2e/values.dns.registry.yaml | 116 ++++++++++++++++++++++ 9 files changed, 545 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index da5aac9..7473d59 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,13 @@ Changes by Version ================== Release Notes. +0.5.4 +----------------- + +#### Features + +- Support configure node discovery in cluster mode. + 0.5.3 ----------------- diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl index 5d8f674..4063051 100644 --- a/chart/templates/_helpers.tpl +++ b/chart/templates/_helpers.tpl @@ -124,3 +124,121 @@ Generate data node names list for "hot" role only {{- end }} {{- $dataNodes | join "," -}} {{- end }} + +{{/* +Generate DNS SRV address for a component +Format: _<port-name>._<proto>.<service-name>.<namespace>.svc.cluster.local +*/}} +{{- define "banyandb.dnsSrvAddress" -}} +{{- $component := .component }} +{{- $role := .role }} +{{- $fullname := include "banyandb.fullname" .root }} +{{- $namespace := .root.Release.Namespace }} +{{- $serviceName := "" }} +{{- $portName := "grpc" }} +{{- if eq $component "liaison" }} + {{- $serviceName = printf "%s-liaison-headless" $fullname }} + {{- $portName = "internal-grpc" }} +{{- else if eq $component "data" }} + {{- $serviceName = printf "%s-data-%s-headless" $fullname $role }} +{{- end }} +{{- printf "_%s._tcp.%s.%s.svc.cluster.local" $portName $serviceName $namespace }} +{{- end }} + +{{/* +Generate all DNS SRV addresses for node discovery +Returns comma-separated list of SRV addresses +*/}} +{{- define "banyandb.allDnsSrvAddresses" -}} +{{- $addresses := list }} +{{- /* Add liaison SRV address */}} +{{- $liaisonSrv := include "banyandb.dnsSrvAddress" (dict "root" . "component" "liaison") }} +{{- $addresses = append $addresses $liaisonSrv }} +{{- /* Add data node SRV addresses for each role */}} +{{- range $roleName, $roleConfig := .Values.cluster.data.roles }} + {{- $dataSrv := include "banyandb.dnsSrvAddress" (dict "root" $ "component" "data" "role" $roleName) }} + {{- $addresses = append $addresses $dataSrv }} +{{- end }} +{{- $addresses | join "," }} +{{- end }} + +{{/* +Generate node registry environment variables for a component +*/}} +{{- define "banyandb.nodeRegistryEnv" -}} +{{- $config := .root.Values.cluster.nodeRegistry | default dict }} +{{- $mode := $config.mode | default "etcd" }} + +- name: BYDB_NODE_DISCOVERY_MODE + value: {{ $mode | quote }} + +{{- if eq $mode "etcd" }} +{{- /* Etcd mode configuration */}} +{{- if $config.etcd.namespace }} +- name: BYDB_NAMESPACE + value: {{ $config.etcd.namespace | quote }} +{{- end }} +{{- if $config.etcd.nodeRegistryTimeout }} +- name: BYDB_NODE_REGISTRY_TIMEOUT + value: {{ $config.etcd.nodeRegistryTimeout | quote }} +{{- end }} +{{- if $config.etcd.fullSyncInterval }} +- name: BYDB_ETCD_FULL_SYNC_INTERVAL + value: {{ $config.etcd.fullSyncInterval | quote }} +{{- end }} + +{{- else if eq $mode "dns" }} +{{- /* DNS mode configuration */}} +{{- /* Always auto-generate SRV addresses for all data nodes */}} +{{- $srvAddresses := include "banyandb.allDnsSrvAddresses" .root }} +{{- if $srvAddresses }} +- name: BYDB_NODE_DISCOVERY_DNS_SRV_ADDRESSES + value: {{ $srvAddresses | quote }} +{{- end }} + +{{- if $config.dns.fetchInitInterval }} +- name: BYDB_NODE_DISCOVERY_DNS_FETCH_INIT_INTERVAL + value: {{ $config.dns.fetchInitInterval | quote }} +{{- end }} +{{- if $config.dns.fetchInitDuration }} +- name: BYDB_NODE_DISCOVERY_DNS_FETCH_INIT_DURATION + value: {{ $config.dns.fetchInitDuration | quote }} +{{- end }} +{{- if $config.dns.fetchInterval }} +- name: BYDB_NODE_DISCOVERY_DNS_FETCH_INTERVAL + value: {{ $config.dns.fetchInterval | quote }} +{{- end }} +{{- if $config.dns.grpcTimeout }} +- name: BYDB_NODE_DISCOVERY_GRPC_TIMEOUT + value: {{ $config.dns.grpcTimeout | quote }} +{{- end }} + +{{- /* Auto-generate TLS configuration based on existing gRPC TLS settings */}} +{{- $tlsEnabled := false }} +{{- $caCerts := list }} +{{- /* Check liaison TLS */}} +{{- if and .root.Values.cluster.liaison .root.Values.cluster.liaison.tls .root.Values.cluster.liaison.tls.grpcSecretName }} + {{- $tlsEnabled = true }} + {{- $caCerts = append $caCerts (printf "/etc/tls/%s/ca.crt" .root.Values.cluster.liaison.tls.grpcSecretName) }} +{{- end }} +{{- /* Check data nodes TLS for each role */}} +{{- range $roleName, $roleConfig := .root.Values.cluster.data.roles }} + {{- if and $roleConfig.tls $roleConfig.tls.grpcSecretName }} + {{- $tlsEnabled = true }} + {{- $caCerts = append $caCerts (printf "/etc/tls/%s/ca.crt" $roleConfig.tls.grpcSecretName) }} + {{- else if and $.root.Values.cluster.data.nodeTemplate.tls $.root.Values.cluster.data.nodeTemplate.tls.grpcSecretName }} + {{- $tlsEnabled = true }} + {{- $caCerts = append $caCerts (printf "/etc/tls/%s/ca.crt" $.root.Values.cluster.data.nodeTemplate.tls.grpcSecretName) }} + {{- end }} +{{- end }} + +{{- if $tlsEnabled }} +- name: BYDB_NODE_DISCOVERY_DNS_TLS + value: "true" +{{- if $caCerts }} +- name: BYDB_NODE_DISCOVERY_DNS_CA_CERTS + value: {{ $caCerts | join "," | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/chart/templates/cluster_data_statefulset.yaml b/chart/templates/cluster_data_statefulset.yaml index 9db3538..0810e31 100644 --- a/chart/templates/cluster_data_statefulset.yaml +++ b/chart/templates/cluster_data_statefulset.yaml @@ -193,10 +193,11 @@ spec: {{- else }} {{- include "banyandb.etcdEndpoints" $ | nindent 12 }} {{- end }} + {{- include "banyandb.nodeRegistryEnv" (dict "root" $) | nindent 12 }} {{- $mergedEnv := concat $.Values.cluster.data.nodeTemplate.env $roleConfig.env }} {{- range $env := $mergedEnv }} - name: {{ $env.name }} - value: {{ $env.value }} + value: {{ .value | toString | quote }} {{- end }} args: @@ -362,7 +363,8 @@ spec: value: "{{- $.Values.cluster.etcdEndpoints | join "," -}}" {{- else }} {{- include "banyandb.etcdEndpoints" $ | nindent 12 }} - {{- end }} + {{- end }} + {{- include "banyandb.nodeRegistryEnv" (dict "root" $) | nindent 12 }} command: - "/lifecycle" - "--schedule={{ $roleConfig.lifecycleSidecar.schedule }}" diff --git a/chart/templates/cluster_liaison_statefulset.yaml b/chart/templates/cluster_liaison_statefulset.yaml index 705e903..b903645 100644 --- a/chart/templates/cluster_liaison_statefulset.yaml +++ b/chart/templates/cluster_liaison_statefulset.yaml @@ -164,6 +164,7 @@ spec: {{- else }} {{- include "banyandb.etcdEndpoints" . | nindent 12 }} {{- end }} + {{- include "banyandb.nodeRegistryEnv" (dict "root" .) | nindent 12 }} {{- if include "banyandb.hasDataNodeListValue" . }} - name: BYDB_DATA_NODE_LIST value: "{{ include "banyandb.dataNodeListValue" . }}" diff --git a/chart/values.yaml b/chart/values.yaml index 90f425e..9acd285 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -248,6 +248,56 @@ cluster: ## @param cluster.etcdEndpoints List of etcd endpoints ## etcdEndpoints: [] + + ## @section Node Registry Configuration for Service Discovery + ## + nodeRegistry: + ## @param cluster.nodeRegistry.mode Node discovery mode (etcd, dns) + ## Default: etcd + ## + mode: "etcd" + + ## @section Etcd Mode Configuration + ## + etcd: + ## @param cluster.nodeRegistry.etcd.namespace Namespace in etcd for node registration + ## Default: banyandb + ## + namespace: "banyandb" + + ## @param cluster.nodeRegistry.etcd.nodeRegistryTimeout Timeout for node registration + ## Default: 2m + ## + nodeRegistryTimeout: "2m" + + ## @param cluster.nodeRegistry.etcd.fullSyncInterval Interval for full state synchronization + ## Default: 30m + ## + fullSyncInterval: "30m" + + ## @section DNS Mode Configuration + ## + dns: + ## @param cluster.nodeRegistry.dns.fetchInitInterval Query interval during initialization + ## Default: 5s + ## + fetchInitInterval: "5s" + + ## @param cluster.nodeRegistry.dns.fetchInitDuration Duration of initialization phase + ## Default: 5m + ## + fetchInitDuration: "5m" + + ## @param cluster.nodeRegistry.dns.fetchInterval Query interval after initialization + ## Default: 15s + ## + fetchInterval: "15s" + + ## @param cluster.nodeRegistry.dns.grpcTimeout Timeout for gRPC metadata fetch + ## Default: 5s + ## + grpcTimeout: "5s" + ## @section Configuration for liaison component ## liaison: diff --git a/test/e2e/e2e-banyandb-node-registry-dns.yaml b/test/e2e/e2e-banyandb-node-registry-dns.yaml new file mode 100644 index 0000000..20b06cf --- /dev/null +++ b/test/e2e/e2e-banyandb-node-registry-dns.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to test BanyanDB with lifecycle features enabled. + +setup: + env: kind + file: kind.yaml + init-system-environment: env + kind: + expose-ports: + - namespace: istio-system + resource: pod/banyandb-liaison-0 + port: 17913 + - namespace: istio-system + resource: pod/banyandb-liaison-1 + port: 17913 + - namespace: istio-system + resource: pod/banyandb-data-hot-0 + port: 17913 + - namespace: istio-system + resource: pod/banyandb-data-hot-1 + port: 17913 + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e/setup-e2e-shell/install.sh swctl + - name: install kubectl + command: bash test/e2e/setup-e2e-shell/install.sh kubectl + - name: Install helm + command: bash test/e2e/setup-e2e-shell/install.sh helm + - name: Install dependency + command: | + helm repo add bitnami https://charts.bitnami.com/bitnami + cd chart + helm dependency update + - name: Install BanyanDB with lifecycle features + command: helm -n istio-system install banyandb chart/ -f test/e2e/values.dns.registry.yaml --create-namespace + wait: + - namespace: istio-system + resource: pod + for: condition=ready + label-selector: app.kubernetes.io/name=banyandb + timeout: 25m + +verify: + retry: + count: 2 + interval: 3s + cases: + - query: curl -s http://${pod_banyandb_liaison_0_host}:${pod_banyandb_liaison_0_17913}/api/v1/cluster/state | yq -P + expected: expected/nodes-liaison.yaml + - query: curl -s http://${pod_banyandb_liaison_1_host}:${pod_banyandb_liaison_1_17913}/api/v1/cluster/state | yq -P + expected: expected/nodes-liaison.yaml + - query: curl -s http://${pod_banyandb_data_hot_0_host}:${pod_banyandb_data_hot_0_17913}/api/v1/cluster/state | yq -P + expected: expected/nodes-data.yaml + - query: curl -s http://${pod_banyandb_data_hot_1_host}:${pod_banyandb_data_hot_1_17913}/api/v1/cluster/state | yq -P + expected: expected/nodes-data.yaml diff --git a/test/e2e/expected/nodes-data.yaml b/test/e2e/expected/nodes-data.yaml new file mode 100644 index 0000000..7a436dc --- /dev/null +++ b/test/e2e/expected/nodes-data.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +routeTables: + property: + registered: + {{- contains .routeTables.property.registered}} + - metadata: + group: "" + name: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_DATA + grpcAddress: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17912 + httpAddress: "" + createdAt: {{ notEmpty .createdAt }} + labels: + type: hot + propertyRepairGossipGrpcAddress: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17932 + - metadata: + group: "" + name: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_DATA + grpcAddress: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17912 + httpAddress: "" + createdAt: {{ notEmpty .createdAt }} + labels: + type: hot + propertyRepairGossipGrpcAddress: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17932 + {{- end}} + active: [] + evictable: [] diff --git a/test/e2e/expected/nodes-liaison.yaml b/test/e2e/expected/nodes-liaison.yaml new file mode 100644 index 0000000..7d5de49 --- /dev/null +++ b/test/e2e/expected/nodes-liaison.yaml @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +routeTables: + tire1: + registered: + {{- contains .routeTables.tire1.registered}} + - metadata: + group: "" + name: banyandb-liaison-0.banyandb-liaison-headless.istio-system:18912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_LIAISON + grpcAddress: banyandb-liaison-0.banyandb-liaison-headless.istio-system:18912 + httpAddress: banyandb-liaison-0.banyandb-liaison-headless.istio-system:17913 + createdAt: {{ notEmpty .createdAt }} + labels: {} + propertyRepairGossipGrpcAddress: "" + - metadata: + group: "" + name: banyandb-liaison-1.banyandb-liaison-headless.istio-system:18912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_LIAISON + grpcAddress: banyandb-liaison-1.banyandb-liaison-headless.istio-system:18912 + httpAddress: banyandb-liaison-1.banyandb-liaison-headless.istio-system:17913 + createdAt: {{ notEmpty .createdAt }} + labels: {} + propertyRepairGossipGrpcAddress: "" + {{- end}} + active: + {{- contains .routeTables.tire1.active}} + - banyandb-liaison-0.banyandb-liaison-headless.istio-system:18912 + - banyandb-liaison-1.banyandb-liaison-headless.istio-system:18912 + {{- end}} + evictable: [] + tire2: + registered: + {{- contains .routeTables.tire2.registered}} + - metadata: + group: "" + name: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_DATA + grpcAddress: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17912 + httpAddress: "" + createdAt: {{ notEmpty .createdAt }} + labels: + type: hot + propertyRepairGossipGrpcAddress: banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17932 + - metadata: + group: "" + name: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17912 + id: 0 + createRevision: "0" + modRevision: "0" + roles: + - ROLE_META + - ROLE_DATA + grpcAddress: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17912 + httpAddress: "" + createdAt: {{ notEmpty .createdAt }} + labels: + type: hot + propertyRepairGossipGrpcAddress: banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17932 + {{- end}} + active: + {{- contains .routeTables.tire2.active}} + - banyandb-data-hot-1.banyandb-data-hot-headless.istio-system:17912 + - banyandb-data-hot-0.banyandb-data-hot-headless.istio-system:17912 + {{- end}} + evictable: [] diff --git a/test/e2e/values.dns.registry.yaml b/test/e2e/values.dns.registry.yaml new file mode 100644 index 0000000..81736ff --- /dev/null +++ b/test/e2e/values.dns.registry.yaml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for banyandb. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: mrproliu/skywalking-banyandb + tag: node-registry5 + pullPolicy: IfNotPresent + +cluster: + enabled: true + etcdEndpoints: [] + nodeRegistry: + mode: "dns" + liaison: + replicas: 2 + data: + nodeTemplate: + replicas: 2 + roles: + hot: {} + +storage: + data: + enabled: true + persistentVolumeClaims: + - mountTargets: [ "measure", "stream", "property", "trace" ] + nodeRole: hot + existingClaimName: null + claimName: data + size: 5Gi + accessModes: + - ReadWriteOnce + storageClass: null + volumeMode: Filesystem + - mountTargets: [ "backups" ] + nodeRole: hot + existingClaimName: null + claimName: backups + size: 5Gi + accessModes: + - ReadWriteOnce + storageClass: null + volumeMode: Filesystem + liaison: + enabled: true + persistentVolumeClaims: + - mountTargets: [ "measure", "stream", "trace" ] + claimName: liaison-data + size: 10Gi + accessModes: + - ReadWriteOnce + storageClass: null + volumeMode: Filesystem + standalone: + enabled: false + persistentVolumeClaims: + - mountTargets: [ "measure", "stream", "property", "trace" ] + claimName: standalone-data + size: 200Gi + accessModes: + - ReadWriteOnce + storageClass: null + volumeMode: Filesystem + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +etcd: + enabled: true + replicaCount: 1 + auth: + rbac: + create: true + allowNoneAuthentication: false + rootPassword: banyandb + client: + secureTransport: false + existingSecret: "" + enableAuthentication: false + certFilename: tls.crt + certKeyFilename: tls.key + caFilename: "" + # extraEnvVars: + # - name: ETCDCTL_CACERT + # value: /opt/bitnami/etcd/certs/client/ca.crt + token: + enabled: true + type: simple + +fullnameOverride: "" +nameOverride: "banyandb" + +auth: + enabled: false
