This is an automated email from the ASF dual-hosted git repository.
mfordjody pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new cd068a07 Add supplementary tests and performance tests (#963)
cd068a07 is described below
commit cd068a07e113117ec74c673f5f730e3bd983d1bd
Author: mfordjody <[email protected]>
AuthorDate: Sat Jul 18 20:07:41 2026 +0800
Add supplementary tests and performance tests (#963)
---
.github/workflows/benchmark.yml | 66 ++++
.github/workflows/ci.yml | 18 ++
PERFORMANCE.md | 115 +++++++
README.md | 7 +-
.../pkg/model/authentication_resolution_test.go | 293 +++++++++++++++++
dubbod/discovery/pkg/xds/ads_benchmark_test.go | 346 +++++++++++++++++++++
tools/make/test.mk | 14 +
7 files changed, 858 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 00000000..f7d3d53d
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Performance Benchmarks
+
+on:
+ schedule:
+ - cron: '0 4 * * 3'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ benchmark:
+ name: Control-Plane Scale Matrix
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ if: github.repository == 'apache/dubbo-kubernetes'
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache-dependency-path: go.sum
+
+ - name: Capture benchmark environment
+ run: |
+ {
+ go version
+ go env GOOS GOARCH
+ uname -a
+ lscpu
+ } > benchmark-environment.txt
+
+ - name: Run benchmark matrix
+ run: make benchmark BENCHMARK_COUNT=5 BENCHMARK_TIME=1s
BENCHMARK_CPU=1 | tee benchmark-results.txt
+
+ - name: Upload benchmark evidence
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: control-plane-benchmarks-${{ github.run_id }}
+ path: |
+ benchmark-environment.txt
+ benchmark-results.txt
+ retention-days: 30
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 17efd658..ee734d31 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -120,6 +120,24 @@ jobs:
name: codecov-umbrella
fail_ci_if_error: false
+ performance:
+ name: Performance Scale Smoke
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ if: github.repository == 'apache/dubbo-kubernetes'
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ cache-dependency-path: go.sum
+
+ - name: Exercise representative xDS push scale
+ run: make benchmark-smoke
+
build:
name: Build (${{ matrix.goos }}/${{ matrix.goarch }})
runs-on: ubuntu-latest
diff --git a/PERFORMANCE.md b/PERFORMANCE.md
new file mode 100644
index 00000000..d20b631b
--- /dev/null
+++ b/PERFORMANCE.md
@@ -0,0 +1,115 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Performance Engineering
+
+The performance suite keeps two layers separate:
+
+1. Control-plane Go benchmarks measure indexing, client selection, and xDS
+ response generation without Kubernetes or network noise.
+2. Cluster load tests measure end-to-end request latency and resource usage.
+ Those results must identify the cluster, traffic generator, workload, and
+ observability setup; they must not be compared directly with Go benchmark
+ timings.
+
+## Control-plane scale matrix
+
+`BenchmarkProxylessPushScale` exercises the real targeted endpoint-update path:
+
+```text
+clientsForPush -> pushConnection -> EDS generation -> DiscoveryResponse send
+```
+
+Every synthetic proxyless connection watches every generated service. A
+single service endpoint update therefore selects every connection and emits
+one EDS response per connection. The matrix changes one dimension at a time
+from the `100 services / 10 endpoints / 100 connections` baseline:
+
+| Dimension | Cases |
+|---|---|
+| Services | 10, 100, 1,000 |
+| Endpoints per service | 1, 10, 100 |
+| Connections | 10, 100, 1,000 |
+
+`BenchmarkProxylessFullPushScale` uses the same dimensions but forces a full
+push that emits every watched service. Its baseline uses 10 connections, and
+its largest cases cap the generated work at 100,000 endpoint copies per
+operation:
+
+| Dimension | Cases |
+|---|---|
+| Services | 10, 100, 1,000 |
+| Endpoints per service | 1, 10, 100 |
+| Connections | 1, 10, 100 |
+
+Both benchmarks report `ns/op`, `B/op`, `allocs/op`, the three scale values,
+and the number of generated endpoint copies per operation.
+`ns/op` is the serial cost of completing one synthetic push wave on the
+benchmark CPU. Production pushes may execute concurrently, so this number is
+for regression and scaling comparisons rather than a direct production SLO.
+
+The existing `BenchmarkClientsForPushProxylessTargetedScale` separately
+measures connection selection through 100,000 connected clients, while
+`BenchmarkKRTFetch` measures indexed and label-scan lookups over 10,000
+workloads.
+
+## Running benchmarks
+
+Run the representative targeted and full-push scale cases used by pull-request
+CI:
+
+```bash
+make benchmark-smoke
+```
+
+Run the complete matrix with repeated samples:
+
+```bash
+make benchmark BENCHMARK_COUNT=5 BENCHMARK_TIME=1s BENCHMARK_CPU=1
+```
+
+The Make variables can narrow the benchmark expression or adjust measurement
+time and CPU count:
+
+```bash
+make benchmark \
+ BENCHMARK_PATTERN=BenchmarkProxylessPushScale \
+ BENCHMARK_COUNT=10 \
+ BENCHMARK_TIME=2s \
+ BENCHMARK_CPU=1
+```
+
+Use the same Go version, CPU model, power mode, `BENCHMARK_CPU`, and background
+load for baseline and candidate runs. Keep at least five samples and compare
+them with `benchstat`; do not draw a capacity conclusion from one run.
+
+## Continuous evidence
+
+The main CI workflow runs `make benchmark-smoke` to catch fixture failures and
+gross scale regressions on every change. The scheduled `Performance
+Benchmarks` workflow runs five samples of the complete matrix each week and
+retains the raw output plus runner details for 30 days. Shared runner variance
+makes a universal absolute latency threshold unreliable; compare like-for-like
+runs and investigate changes in both time and allocations.
+
+## Cluster load tests
+
+Use `samples/httpbin/httpbin.yaml` as the minimal data-plane target and the
+Prometheus sample under `samples/addons/` for control-plane metrics. Record at
+least request rate, p50/p95/p99 latency, error rate, CPU, memory, connected xDS
+clients, services, and endpoints. Change one scale dimension at a time and
+include warm-up, steady-state, and configuration-churn phases in the result.
diff --git a/README.md b/README.md
index de489cf8..f75b9413 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ Dubbo inherent mesh implemented for the underlying cluster
management platform c
> [!WARNING]
> Current version is in the **Alpha** phase. The `0.5.0` release will be the
> first **RC** phase.
->
+>
Dubbo’s control plane provides an abstraction layer over the underlying
cluster management platform.
Dubbo component composition:
@@ -38,6 +38,11 @@ Projects are distributed across the code directory
repositories:
Refer to [CONTRIBUTING.md](./CONTRIBUTING.md)
+## Performance
+
+See [PERFORMANCE.md](./PERFORMANCE.md) for the control-plane scale matrix,
+repeatable benchmark commands, CI evidence, and cluster load-test methodology.
+
## License
Apache License 2.0, see
[LICENSE](https://github.com/apache/dubbo-kubernetes/blob/master/LICENSE).
diff --git a/dubbod/discovery/pkg/model/authentication_resolution_test.go
b/dubbod/discovery/pkg/model/authentication_resolution_test.go
new file mode 100644
index 00000000..eb65abd0
--- /dev/null
+++ b/dubbod/discovery/pkg/model/authentication_resolution_test.go
@@ -0,0 +1,293 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import (
+ "testing"
+
+ "github.com/apache/dubbo-kubernetes/pkg/config"
+ "github.com/apache/dubbo-kubernetes/pkg/config/schema/gvk"
+ security "github.com/kdubbo/api/security/v1alpha3"
+ typev1alpha3 "github.com/kdubbo/api/type/v1alpha3"
+)
+
+func selectorFromLabels(labels map[string]string)
*typev1alpha3.WorkloadSelector {
+ if len(labels) == 0 {
+ return nil
+ }
+ return &typev1alpha3.WorkloadSelector{MatchLabels: labels}
+}
+
+// peerAuth builds a PeerAuthentication config. A nil selector produces a
+// namespace/mesh-level policy; a non-nil one produces a workload-level policy.
+func peerAuth(namespace, name string, selector map[string]string, mtls
security.PeerAuthentication_MutualTLS_Mode, portLevel
map[uint32]security.PeerAuthentication_MutualTLS_Mode) config.Config {
+ spec := &security.PeerAuthentication{}
+ if mtls != security.PeerAuthentication_MutualTLS_UNSET {
+ spec.Mtls = &security.PeerAuthentication_MutualTLS{Mode: mtls}
+ }
+ if sel := selectorFromLabels(selector); sel != nil {
+ spec.Selector = sel
+ }
+ if len(portLevel) > 0 {
+ spec.PortLevelMtls =
map[uint32]*security.PeerAuthentication_MutualTLS{}
+ for port, mode := range portLevel {
+ spec.PortLevelMtls[port] =
&security.PeerAuthentication_MutualTLS{Mode: mode}
+ }
+ }
+ return config.Config{
+ Meta: config.Meta{GroupVersionKind: gvk.PeerAuthentication,
Name: name, Namespace: namespace},
+ Spec: spec,
+ }
+}
+
+func newPolicies(rootNamespace string, configs ...config.Config)
*AuthenticationPolicies {
+ policy := &AuthenticationPolicies{
+ requestAuthentications: map[string][]config.Config{},
+ authorizationPolicies: map[string][]config.Config{},
+ peerAuthentications: map[string][]config.Config{},
+ globalMutualTLSMode: MTLSUnknown,
+ rootNamespace: rootNamespace,
+ namespaceMutualTLSMode: map[string]MutualTLSMode{},
+ }
+ policy.addPeerAuthentication(configs)
+ return policy
+}
+
+func TestConvertToMutualTLSModeAllModes(t *testing.T) {
+ cases := map[security.PeerAuthentication_MutualTLS_Mode]MutualTLSMode{
+ security.PeerAuthentication_MutualTLS_DISABLE: MTLSDisable,
+ security.PeerAuthentication_MutualTLS_PERMISSIVE:
MTLSPermissive,
+ security.PeerAuthentication_MutualTLS_STRICT: MTLSStrict,
+ security.PeerAuthentication_MutualTLS_UNSET: MTLSUnknown,
+ }
+ for mode, want := range cases {
+ if got := ConvertToMutualTLSMode(mode); got != want {
+ t.Errorf("ConvertToMutualTLSMode(%v) = %v, want %v",
mode, got, want)
+ }
+ }
+}
+
+func TestEffectiveMutualTLSModePrecedence(t *testing.T) {
+ const root = "dubbo-system"
+ const ns = "app"
+ labels := map[string]string{"app": "httpbin"}
+
+ tests := []struct {
+ name string
+ configs []config.Config
+ labels map[string]string
+ port uint32
+ want MutualTLSMode
+ }{
+ {
+ name: "workload selector beats namespace default",
+ configs: []config.Config{
+ peerAuth(ns, "ns-default", nil,
security.PeerAuthentication_MutualTLS_PERMISSIVE, nil),
+ peerAuth(ns, "wl", labels,
security.PeerAuthentication_MutualTLS_STRICT, nil),
+ },
+ labels: labels,
+ port: 8080,
+ want: MTLSStrict,
+ },
+ {
+ name: "namespace default beats mesh global",
+ configs: []config.Config{
+ peerAuth(root, "mesh", nil,
security.PeerAuthentication_MutualTLS_STRICT, nil),
+ peerAuth(ns, "ns-default", nil,
security.PeerAuthentication_MutualTLS_DISABLE, nil),
+ },
+ labels: labels,
+ port: 8080,
+ want: MTLSDisable,
+ },
+ {
+ name: "mesh global applies when namespace has none",
+ configs: []config.Config{
+ peerAuth(root, "mesh", nil,
security.PeerAuthentication_MutualTLS_STRICT, nil),
+ },
+ labels: labels,
+ port: 8080,
+ want: MTLSStrict,
+ },
+ {
+ name: "port level override wins over workload spec
level",
+ configs: []config.Config{
+ peerAuth(ns, "wl", labels,
security.PeerAuthentication_MutualTLS_STRICT,
+
map[uint32]security.PeerAuthentication_MutualTLS_Mode{8080:
security.PeerAuthentication_MutualTLS_DISABLE}),
+ },
+ labels: labels,
+ port: 8080,
+ want: MTLSDisable,
+ },
+ {
+ name: "port level ignored for a different port",
+ configs: []config.Config{
+ peerAuth(ns, "wl", labels,
security.PeerAuthentication_MutualTLS_STRICT,
+
map[uint32]security.PeerAuthentication_MutualTLS_Mode{9090:
security.PeerAuthentication_MutualTLS_DISABLE}),
+ },
+ labels: labels,
+ port: 8080,
+ want: MTLSStrict,
+ },
+ {
+ name: "no policy resolves to unknown",
+ configs: nil,
+ labels: labels,
+ port: 8080,
+ want: MTLSUnknown,
+ },
+ {
+ name: "empty namespace falls through to mesh global",
+ configs: []config.Config{
+ peerAuth(root, "mesh", nil,
security.PeerAuthentication_MutualTLS_PERMISSIVE, nil),
+ },
+ labels: labels,
+ port: 0,
+ want: MTLSPermissive,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ namespace := ns
+ if tt.name == "empty namespace falls through to mesh
global" {
+ namespace = ""
+ }
+ policy := newPolicies(root, tt.configs...)
+ if got := policy.EffectiveMutualTLSMode(namespace,
tt.labels, tt.port); got != tt.want {
+ t.Fatalf("EffectiveMutualTLSMode = %v, want
%v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestEffectiveMutualTLSModeRootNamespaceWorkloadSelector(t *testing.T) {
+ const root = "dubbo-system"
+ labels := map[string]string{"app": "httpbin"}
+ // A workload-selector PeerAuthentication living in the root namespace
applies
+ // to matching workloads in other namespaces when their own namespace
has no
+ // matching policy.
+ policy := newPolicies(root,
+ peerAuth(root, "root-wl", labels,
security.PeerAuthentication_MutualTLS_STRICT, nil),
+ )
+ if got := policy.EffectiveMutualTLSMode("app", labels, 8080); got !=
MTLSStrict {
+ t.Fatalf("effective mode = %v, want MTLSStrict (root ns
workload selector)", got)
+ }
+ // A workload in the same namespaces whose labels do not match falls
through
+ // to unknown.
+ if got := policy.EffectiveMutualTLSMode("app", map[string]string{"app":
"other"}, 8080); got != MTLSUnknown {
+ t.Fatalf("non-matching mode = %v, want MTLSUnknown", got)
+ }
+}
+
+func TestEffectiveMutualTLSModeNilPolicy(t *testing.T) {
+ var policy *AuthenticationPolicies
+ if got := policy.EffectiveMutualTLSMode("app", nil, 80); got !=
MTLSUnknown {
+ t.Fatalf("nil policy mode = %v, want MTLSUnknown", got)
+ }
+}
+
+func TestNamespaceLevelInheritsGlobalWhenUnset(t *testing.T) {
+ const root = "dubbo-system"
+ // A namespace-level PeerAuthentication with no mtls mode (UNSET)
inherits
+ // the mesh-global mode rather than resetting to unknown.
+ policy := newPolicies(root,
+ peerAuth(root, "mesh", nil,
security.PeerAuthentication_MutualTLS_STRICT, nil),
+ peerAuth("app", "ns-unset", nil,
security.PeerAuthentication_MutualTLS_UNSET, nil),
+ )
+ if got := policy.namespaceMutualTLSMode["app"]; got != MTLSStrict {
+ t.Fatalf("inherited namespace mode = %v, want MTLSStrict", got)
+ }
+}
+
+func TestDuplicateNamespaceLevelPeerAuthIgnored(t *testing.T) {
+ const ns = "app"
+ // Two namespace-level policies in the same namespace: the first by
creation
+ // order wins and the second must be ignored.
+ first := peerAuth(ns, "first", nil,
security.PeerAuthentication_MutualTLS_STRICT, nil)
+ second := peerAuth(ns, "second", nil,
security.PeerAuthentication_MutualTLS_DISABLE, nil)
+ policy := newPolicies("dubbo-system", first, second)
+
+ if got := policy.EffectiveMutualTLSMode(ns, nil, 0); got != MTLSStrict {
+ t.Fatalf("effective mode = %v, want MTLSStrict (first policy
wins)", got)
+ }
+}
+
+func TestPeerAuthenticationModeForPort(t *testing.T) {
+ specStrict := &security.PeerAuthentication{
+ Mtls: &security.PeerAuthentication_MutualTLS{Mode:
security.PeerAuthentication_MutualTLS_STRICT},
+ PortLevelMtls:
map[uint32]*security.PeerAuthentication_MutualTLS{
+ 8080: {Mode:
security.PeerAuthentication_MutualTLS_DISABLE},
+ 9090: {Mode:
security.PeerAuthentication_MutualTLS_UNSET},
+ },
+ }
+
+ tests := []struct {
+ name string
+ spec *security.PeerAuthentication
+ port uint32
+ want MutualTLSMode
+ }{
+ {"nil spec", nil, 8080, MTLSUnknown},
+ {"port level override", specStrict, 8080, MTLSDisable},
+ {"port level unset falls back to spec", specStrict, 9090,
MTLSStrict},
+ {"unmapped port uses spec level", specStrict, 7070, MTLSStrict},
+ {"port zero ignores port level", specStrict, 0, MTLSStrict},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := peerAuthenticationModeForPort(tt.spec,
tt.port); got != tt.want {
+ t.Fatalf("peerAuthenticationModeForPort = %v,
want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestSelectorMatchesWorkload(t *testing.T) {
+ tests := []struct {
+ name string
+ selector map[string]string
+ labels map[string]string
+ want bool
+ }{
+ {"empty selector matches all", nil, map[string]string{"app":
"x"}, true},
+ {"empty labels with selector fails", map[string]string{"app":
"x"}, nil, false},
+ {"exact match", map[string]string{"app": "x"},
map[string]string{"app": "x", "v": "1"}, true},
+ {"value mismatch", map[string]string{"app": "x"},
map[string]string{"app": "y"}, false},
+ {"missing key", map[string]string{"app": "x", "tier": "t"},
map[string]string{"app": "x"}, false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got :=
selectorMatchesWorkload(selectorFromLabels(tt.selector), tt.labels); got !=
tt.want {
+ t.Fatalf("selectorMatchesWorkload = %v, want
%v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestHasPeerAuthSelector(t *testing.T) {
+ if hasPeerAuthSelector(nil) {
+ t.Fatal("nil spec should have no selector")
+ }
+ noSel := &security.PeerAuthentication{}
+ if hasPeerAuthSelector(noSel) {
+ t.Fatal("spec without selector should report false")
+ }
+ withSel := &security.PeerAuthentication{Selector:
selectorFromLabels(map[string]string{"app": "x"})}
+ if !hasPeerAuthSelector(withSel) {
+ t.Fatal("spec with selector should report true")
+ }
+}
diff --git a/dubbod/discovery/pkg/xds/ads_benchmark_test.go
b/dubbod/discovery/pkg/xds/ads_benchmark_test.go
new file mode 100644
index 00000000..dd1595a3
--- /dev/null
+++ b/dubbod/discovery/pkg/xds/ads_benchmark_test.go
@@ -0,0 +1,346 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xds
+
+import (
+ "fmt"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/model"
+ v1 "github.com/apache/dubbo-kubernetes/dubbod/discovery/pkg/xds/v1"
+ "github.com/apache/dubbo-kubernetes/pkg/config/host"
+ "github.com/apache/dubbo-kubernetes/pkg/config/mesh"
+ "github.com/apache/dubbo-kubernetes/pkg/config/mesh/meshwatcher"
+ "github.com/apache/dubbo-kubernetes/pkg/config/protocol"
+ "github.com/apache/dubbo-kubernetes/pkg/config/schema/kind"
+ "github.com/apache/dubbo-kubernetes/pkg/kube/krt"
+ dubbolog "github.com/apache/dubbo-kubernetes/pkg/log"
+ "github.com/apache/dubbo-kubernetes/pkg/util/sets"
+ endpoint "github.com/kdubbo/xds-api/endpoint/v1"
+ discovery "github.com/kdubbo/xds-api/service/discovery/v1"
+)
+
+type proxylessPushScale struct {
+ services int
+ endpointsPerService int
+ connections int
+}
+
+func (s proxylessPushScale) name() string {
+ return fmt.Sprintf(
+ "services=%d/endpoints_per_service=%d/connections=%d",
+ s.services,
+ s.endpointsPerService,
+ s.connections,
+ )
+}
+
+var proxylessPushScaleMatrix = []proxylessPushScale{
+ {services: 10, endpointsPerService: 10, connections: 100},
+ {services: 100, endpointsPerService: 10, connections: 100},
+ {services: 1000, endpointsPerService: 10, connections: 100},
+ {services: 100, endpointsPerService: 1, connections: 100},
+ {services: 100, endpointsPerService: 100, connections: 100},
+ {services: 100, endpointsPerService: 10, connections: 10},
+ {services: 100, endpointsPerService: 10, connections: 1000},
+}
+
+var proxylessFullPushScaleMatrix = []proxylessPushScale{
+ {services: 10, endpointsPerService: 10, connections: 10},
+ {services: 100, endpointsPerService: 10, connections: 10},
+ {services: 1000, endpointsPerService: 10, connections: 10},
+ {services: 100, endpointsPerService: 1, connections: 10},
+ {services: 100, endpointsPerService: 100, connections: 10},
+ {services: 100, endpointsPerService: 10, connections: 1},
+ {services: 100, endpointsPerService: 10, connections: 100},
+}
+
+var benchmarkPushResponse *discovery.DiscoveryResponse
+
+func BenchmarkProxylessPushScale(b *testing.B) {
+ for _, scale := range proxylessPushScaleMatrix {
+ b.Run(scale.name(), func(b *testing.B) {
+ fixture := newProxylessPushBenchmarkFixture(b, scale)
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ responses, resources, lastResponse :=
fixture.pushOnce(b)
+ if responses != scale.connections || resources
!= scale.connections {
+ b.Fatalf(
+ "push produced
responses/resources = %d/%d, want %d/%d",
+ responses,
+ resources,
+ scale.connections,
+ scale.connections,
+ )
+ }
+ benchmarkPushResponse = lastResponse
+ }
+ b.StopTimer()
+ b.ReportMetric(float64(scale.services), "services")
+ b.ReportMetric(float64(scale.endpointsPerService),
"endpoints/service")
+ b.ReportMetric(float64(scale.connections),
"connections")
+
b.ReportMetric(float64(scale.endpointsPerService*scale.connections),
"endpoint-copies/op")
+ })
+ }
+}
+
+func BenchmarkProxylessFullPushScale(b *testing.B) {
+ for _, scale := range proxylessFullPushScaleMatrix {
+ b.Run(scale.name(), func(b *testing.B) {
+ fixture := newProxylessPushBenchmarkFixture(b, scale)
+ fixture.request.Full = true
+ fixture.request.Forced = true
+ fixture.request.ConfigsUpdated = nil
+ wantResources := scale.services * scale.connections
+ b.ReportAllocs()
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ responses, resources, lastResponse :=
fixture.pushOnce(b)
+ if responses != scale.connections || resources
!= wantResources {
+ b.Fatalf(
+ "full push produced
responses/resources = %d/%d, want %d/%d",
+ responses,
+ resources,
+ scale.connections,
+ wantResources,
+ )
+ }
+ benchmarkPushResponse = lastResponse
+ }
+ b.StopTimer()
+ b.ReportMetric(float64(scale.services), "services")
+ b.ReportMetric(float64(scale.endpointsPerService),
"endpoints/service")
+ b.ReportMetric(float64(scale.connections),
"connections")
+
b.ReportMetric(float64(scale.services*scale.endpointsPerService*scale.connections),
"endpoint-copies/op")
+ })
+ }
+}
+
+func TestProxylessPushBenchmarkFixture(t *testing.T) {
+ scale := proxylessPushScale{services: 3, endpointsPerService: 2,
connections: 4}
+ fixture := newProxylessPushBenchmarkFixture(t, scale)
+
+ responses, resources, lastResponse := fixture.pushOnce(t)
+ if responses != scale.connections || resources != scale.connections {
+ t.Fatalf(
+ "push produced responses/resources = %d/%d, want %d/%d",
+ responses,
+ resources,
+ scale.connections,
+ scale.connections,
+ )
+ }
+ if lastResponse == nil || len(lastResponse.Resources) != 1 {
+ t.Fatalf("last response has %d resources, want 1",
len(lastResponse.GetResources()))
+ }
+ assignment := &endpoint.ClusterLoadAssignment{}
+ if err := lastResponse.Resources[0].UnmarshalTo(assignment); err != nil
{
+ t.Fatalf("unmarshal endpoint assignment: %v", err)
+ }
+ gotEndpoints := 0
+ for _, locality := range assignment.Endpoints {
+ gotEndpoints += len(locality.LbEndpoints)
+ }
+ if gotEndpoints != scale.endpointsPerService {
+ t.Fatalf("endpoint assignment has %d endpoints, want %d",
gotEndpoints, scale.endpointsPerService)
+ }
+}
+
+type proxylessPushBenchmarkFixture struct {
+ server *DiscoveryServer
+ request *model.PushRequest
+ stream *fakeADSStream
+ connections int
+}
+
+func newProxylessPushBenchmarkFixture(tb testing.TB, scale proxylessPushScale)
*proxylessPushBenchmarkFixture {
+ tb.Helper()
+ if scale.services <= 0 || scale.endpointsPerService <= 0 ||
scale.connections <= 0 {
+ tb.Fatalf("benchmark scale values must all be positive: %+v",
scale)
+ }
+ silenceProxylessPushBenchmarkLogs(tb)
+
+ services := make([]*model.Service, 0, scale.services)
+ serviceByHostname := make(map[host.Name]*model.Service, scale.services)
+ clusterNames := make([]string, 0, scale.services)
+ for serviceIndex := 0; serviceIndex < scale.services; serviceIndex++ {
+ serviceName := fmt.Sprintf("svc-%d", serviceIndex)
+ hostname := host.Name(serviceName + ".app.svc.cluster.local")
+ service := &model.Service{
+ Hostname: hostname,
+ Ports: model.PortList{{
+ Name: "grpc",
+ Port: 80,
+ Protocol: protocol.GRPC,
+ }},
+ Attributes: model.ServiceAttributes{
+ Name: serviceName,
+ Namespace: "app",
+ },
+ }
+ services = append(services, service)
+ serviceByHostname[hostname] = service
+ clusterNames = append(clusterNames,
model.BuildSubsetKey(model.TrafficDirectionOutbound, "", hostname, 80))
+ }
+
+ env := model.NewEnvironment()
+ env.ConfigStore = testConfigStore{}
+ env.ServiceDiscovery = benchmarkServiceDiscovery{services: services,
byHostname: serviceByHostname}
+ env.Watcher =
meshwatcher.ConfigAdapter(krt.NewStatic(&meshwatcher.MeshConfigResource{
+ MeshConfig: mesh.DefaultMeshConfig(),
+ }, true))
+ env.Init()
+
+ for serviceIndex, service := range services {
+ endpoints := make([]*model.DubboEndpoint, 0,
scale.endpointsPerService)
+ for endpointIndex := 0; endpointIndex <
scale.endpointsPerService; endpointIndex++ {
+ endpoints = append(endpoints, &model.DubboEndpoint{
+ Addresses:
[]string{benchmarkEndpointAddress(serviceIndex, endpointIndex)},
+ EndpointPort: 80,
+ ServicePortName: "grpc",
+ HealthStatus: model.Healthy,
+ Namespace: "app",
+ WorkloadName: fmt.Sprintf("%s-%d",
service.Attributes.Name, endpointIndex),
+ })
+ }
+ env.EndpointIndex.UpdateServiceEndpoints(model.ShardKey{},
string(service.Hostname), "app", endpoints, false)
+ }
+
+ push := model.NewPushContext()
+ push.PushVersion = "benchmark"
+ push.InitContext(env, nil, nil)
+ env.SetPushContext(push)
+
+ server := NewDiscoveryServer(env, nil, nil)
+ server.Generators[v1.EndpointType] = &EdsGenerator{
+ Cache: model.DisabledCache{},
+ EndpointIndex: env.EndpointIndex,
+ }
+ stream := newFakeADSStream()
+ for connectionIndex := 0; connectionIndex < scale.connections;
connectionIndex++ {
+ connectionID := fmt.Sprintf("proxyless-%d", connectionIndex)
+ connection := newConnection("127.0.0.1:26010", stream)
+ connection.SetID(connectionID)
+ connection.s = server
+ connection.proxy = &model.Proxy{
+ ID: connectionID,
+ Type: model.Proxyless,
+ ConfigNamespace: "app",
+ Metadata: &model.NodeMetadata{
+ Generator: "grpc",
+ Namespace: "app",
+ },
+ WatchedResources: map[string]*model.WatchedResource{
+ v1.EndpointType: {
+ TypeUrl: v1.EndpointType,
+ ResourceNames:
sets.New(clusterNames...),
+ },
+ },
+ LastPushContext: push,
+ LastPushTime: time.Now(),
+ }
+ server.adsClients[connectionID] = connection
+ }
+
+ targetService := services[0]
+ return &proxylessPushBenchmarkFixture{
+ server: server,
+ request: &model.PushRequest{
+ Push: push,
+ Start: time.Now(),
+ Reason: model.NewReasonStats(model.EndpointUpdate),
+ ConfigsUpdated: sets.New(model.ConfigKey{
+ Kind: kind.Service,
+ Name: string(targetService.Hostname),
+ Namespace: targetService.Attributes.Namespace,
+ }),
+ },
+ stream: stream,
+ connections: scale.connections,
+ }
+}
+
+func silenceProxylessPushBenchmarkLogs(tb testing.TB) {
+ tb.Helper()
+ for _, scopeName := range []string{"ads", "model"} {
+ scope := dubbolog.FindScope(scopeName)
+ if scope == nil {
+ continue
+ }
+ previousOutput := scope.GetOutput()
+ scope.SetOutput(io.Discard)
+ tb.Cleanup(func() {
+ scope.SetOutput(previousOutput)
+ })
+ }
+}
+
+func (f *proxylessPushBenchmarkFixture) pushOnce(tb testing.TB) (int, int,
*discovery.DiscoveryResponse) {
+ tb.Helper()
+ clients := f.server.clientsForPush(f.request)
+ if len(clients) != f.connections {
+ tb.Fatalf("clientsForPush() returned %d clients, want %d",
len(clients), f.connections)
+ }
+
+ responses := 0
+ resources := 0
+ var lastResponse *discovery.DiscoveryResponse
+ for _, connection := range clients {
+ if err := f.server.pushConnection(connection,
&Event{pushRequest: f.request}); err != nil {
+ tb.Fatalf("pushConnection(%s): %v", connection.ID(),
err)
+ }
+ select {
+ case response := <-f.stream.sendCh:
+ responses++
+ resources += len(response.Resources)
+ lastResponse = response
+ default:
+ tb.Fatalf("pushConnection(%s) did not send a response",
connection.ID())
+ }
+ }
+ return responses, resources, lastResponse
+}
+
+func benchmarkEndpointAddress(serviceIndex, endpointIndex int) string {
+ return fmt.Sprintf(
+ "10.%d.%d.%d",
+ (serviceIndex/250)%250+1,
+ serviceIndex%250+1,
+ endpointIndex%250+1,
+ )
+}
+
+type benchmarkServiceDiscovery struct {
+ services []*model.Service
+ byHostname map[host.Name]*model.Service
+}
+
+func (d benchmarkServiceDiscovery) Services() []*model.Service {
+ return d.services
+}
+
+func (d benchmarkServiceDiscovery) GetService(hostname host.Name)
*model.Service {
+ return d.byHostname[hostname]
+}
+
+func (benchmarkServiceDiscovery) GetProxyServiceTargets(*model.Proxy)
[]model.ServiceTarget {
+ return nil
+}
diff --git a/tools/make/test.mk b/tools/make/test.mk
index 235b3a05..51796d21 100644
--- a/tools/make/test.mk
+++ b/tools/make/test.mk
@@ -15,6 +15,12 @@
##@ Test
+BENCHMARK_PACKAGES ?= ./pkg/kube/krt ./dubbod/discovery/pkg/xds
+BENCHMARK_PATTERN ?=
Benchmark(KRTFetch|ClientsForPushProxylessTargeted|Proxyless(Push|FullPush)Scale)
+BENCHMARK_TIME ?= 1s
+BENCHMARK_COUNT ?= 3
+BENCHMARK_CPU ?= 1
+
.PHONY: test
test: ## Run unit tests (GOTESTFLAGS defaults to -race).
go test $(GOTESTFLAGS) ./...
@@ -26,3 +32,11 @@ test-coverage: ## Run unit tests and write coverage.txt.
.PHONY: test-e2e
test-e2e: ## Run the kind smoke test (needs docker, kind, kubectl, helm; knobs
in tests/e2e/run.sh).
bash tests/e2e/run.sh
+
+.PHONY: benchmark
+benchmark: ## Run repeatable control-plane microbenchmarks (override
BENCHMARK_* variables).
+ go test $(BENCHMARK_PACKAGES) -run '^$$' -bench '$(BENCHMARK_PATTERN)'
-benchmem -benchtime=$(BENCHMARK_TIME) -count=$(BENCHMARK_COUNT)
-cpu=$(BENCHMARK_CPU)
+
+.PHONY: benchmark-smoke
+benchmark-smoke: ## Run representative targeted/full xDS scale cases as a
correctness gate.
+ go test ./dubbod/discovery/pkg/xds -run
'^TestProxylessPushBenchmarkFixture$$' -bench
'^BenchmarkProxyless(Push|FullPush)Scale$$/^services=100$$/^endpoints_per_service=10$$/^connections=100$$'
-benchmem -benchtime=1x -count=1 -cpu=1
