This is an automated email from the ASF dual-hosted git repository. shreemaan-abhishek pushed a commit to branch feat/cp-ca-bundle in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
commit e0828c8c62e36f490520b76388449da96cea7301 Author: Abhishek Choudhary <[email protected]> AuthorDate: Mon Jul 27 12:56:19 2026 +0545 feat: support a CA bundle for the control plane connection tlsVerify offered only two states: verify against the system trust store, or do not verify at all. A control plane using a self-signed or private-CA certificate has no way to satisfy the first, so the only escape from the connection error is tlsVerify: false -- which turns the insecure opt-out into copy-paste boilerplate. Add the missing third state: an optional PEM-encoded caBundle on GatewayProxy.spec.provider.controlPlane, carried through the translated config to the ADC server, which verifies the control plane against it in place of the system trust store. Unusable CA material is rejected up front -- by a CEL rule at admission and by a PEM parse in the translator -- rather than surfacing later as an opaque TLS failure. --- api/adc/types.go | 7 ++ api/v1alpha1/gatewayproxy_types.go | 8 ++ .../bases/apisix.apache.org_gatewayproxies.yaml | 10 ++ docs/en/latest/reference/api-reference.md | 1 + internal/adc/client/executor.go | 9 +- internal/adc/client/executor_test.go | 31 +++++++ internal/adc/translator/gatewayproxy.go | 12 +++ internal/adc/translator/gatewayproxy_test.go | 101 +++++++++++++++++++++ 8 files changed, 178 insertions(+), 1 deletion(-) diff --git a/api/adc/types.go b/api/adc/types.go index 1ae74ad4..0a2f0506 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -807,6 +807,11 @@ type Config struct { TlsVerify bool BackendType string + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane, in place of the system trust store. Only meaningful when + // TlsVerify is true. + CaBundle string + // BypassCache makes the ADC server drop the in-memory baseline it holds for this // cacheKey and re-derive it from the data plane before computing the diff. It is a // per-request flag set on the sync path, not part of the translated configuration. @@ -820,10 +825,12 @@ func (c Config) MarshalJSON() ([]byte, error) { Name string `json:"name"` ServerAddrs []string `json:"serverAddrs"` TlsVerify bool `json:"tlsVerify"` + HasCaBundle bool `json:"hasCaBundle"` }{ Name: c.Name, ServerAddrs: c.ServerAddrs, TlsVerify: c.TlsVerify, + HasCaBundle: c.CaBundle != "", }) } diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index 680fa8a9..629c780c 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -120,6 +120,7 @@ type ControlPlaneAuth struct { // ControlPlaneProvider defines configuration for control plane provider. // +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)" // +kubebuilder:validation:XValidation:rule="oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode",message="mode is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.caBundle) || self.caBundle.contains('-----BEGIN CERTIFICATE-----')",message="caBundle must be a PEM-encoded certificate" type ControlPlaneProvider struct { // Mode specifies the mode of control plane provider. // Can be `apisix` or `apisix-standalone`. @@ -136,6 +137,13 @@ type ControlPlaneProvider struct { // +optional TlsVerify *bool `json:"tlsVerify,omitempty"` + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane's TLS certificate, in place of the system trust store. + // Set it when the control plane uses a self-signed or private CA certificate. + // It has no effect when tlsVerify is false. + // +optional + CaBundle string `json:"caBundle,omitempty"` + // Auth specifies the authentication configuration. // +kubebuilder:validation:Required Auth ControlPlaneAuth `json:"auth"` diff --git a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml index 23a7ed50..617226d5 100644 --- a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml +++ b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml @@ -120,6 +120,13 @@ spec: - message: adminKey must be specified when type is AdminKey rule: 'self.type == ''AdminKey'' ? has(self.adminKey) : true' + caBundle: + description: |- + CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + control plane's TLS certificate, in place of the system trust store. + Set it when the control plane uses a self-signed or private CA certificate. + It has no effect when tlsVerify is false. + type: string endpoints: description: Endpoints specifies the list of control plane endpoints. @@ -158,6 +165,9 @@ spec: - message: mode is immutable rule: oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode + - message: caBundle must be a PEM-encoded certificate + rule: '!has(self.caBundle) || self.caBundle.contains(''-----BEGIN + CERTIFICATE-----'')' type: description: Type specifies the type of provider. Can only be `ControlPlane`. diff --git a/docs/en/latest/reference/api-reference.md b/docs/en/latest/reference/api-reference.md index 5ccedfde..1f568054 100644 --- a/docs/en/latest/reference/api-reference.md +++ b/docs/en/latest/reference/api-reference.md @@ -313,6 +313,7 @@ ControlPlaneProvider defines configuration for control plane provider. | `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. | | `service` _[ProviderService](#providerservice)_ | | | `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. | +| `caBundle` _string_ | CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the control plane's TLS certificate, in place of the system trust store. Set it when the control plane uses a self-signed or private CA certificate. It has no effect when tlsVerify is false. | | `auth` _[ControlPlaneAuth](#controlplaneauth)_ | Auth specifies the authentication configuration. | diff --git a/internal/adc/client/executor.go b/internal/adc/client/executor.go index 2fe32009..980a429b 100644 --- a/internal/adc/client/executor.go +++ b/internal/adc/client/executor.go @@ -84,7 +84,11 @@ type ADCServerOpts struct { LabelSelector map[string]string `json:"labelSelector,omitempty"` IncludeResourceType []string `json:"includeResourceType,omitempty"` TlsSkipVerify *bool `json:"tlsSkipVerify,omitempty"` - CacheKey string `json:"cacheKey"` + // CaCert is the PEM-encoded CA certificate (or bundle) the ADC server verifies + // the control plane against. Older ADC servers ignore it, and omitempty keeps + // requests without a CA bundle byte for byte what they were. + CaCert string `json:"caCert,omitempty"` + CacheKey string `json:"cacheKey"` // BypassCache is only accepted by the /sync task of ADC >= 0.27.0. Both ADC task // schemas reject unknown fields, so omitempty is what keeps every other request -- // /validate, and every sync that is not recovering from a rejection -- byte for byte @@ -103,6 +107,7 @@ func (r ADCServerRequest) MarshalLog() any { "labelSelector": r.Task.Opts.LabelSelector, "includeResourceType": r.Task.Opts.IncludeResourceType, "tlsSkipVerify": r.Task.Opts.TlsSkipVerify, + "hasCaCert": r.Task.Opts.CaCert != "", "cacheKey": r.Task.Opts.CacheKey, "config": r.Task.Config.MarshalLog(), } @@ -362,6 +367,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin LabelSelector: labels, IncludeResourceType: types, TlsSkipVerify: ptr.To(!tlsVerify), + CaCert: config.CaBundle, CacheKey: config.Name, BypassCache: bypassCache, }, @@ -385,6 +391,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin "labelSelector", labels, "includeResourceType", types, "tlsSkipVerify", !tlsVerify, + "hasCaCert", config.CaBundle != "", ) // Create HTTP request diff --git a/internal/adc/client/executor_test.go b/internal/adc/client/executor_test.go index 9e7ee71c..4b80c432 100644 --- a/internal/adc/client/executor_test.go +++ b/internal/adc/client/executor_test.go @@ -70,6 +70,37 @@ func TestHTTPADCExecutorBuildHTTPRequestBypassCache(t *testing.T) { assert.NotContains(t, raw, "bypassCache") } +func TestHTTPADCExecutorBuildHTTPRequestCaCert(t *testing.T) { + e := &HTTPADCExecutor{ + serverURL: "http://127.0.0.1:3000";, + log: logr.Discard(), + } + + build := func(config adctypes.Config) (ADCServerOpts, string) { + req, err := e.buildHTTPRequest(context.Background(), "https://apisix:9180";, config, nil, nil, + &adctypes.Resources{}, http.MethodPut, pathSync) + require.NoError(t, err) + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + var parsed ADCServerRequest + require.NoError(t, json.Unmarshal(body, &parsed)) + return parsed.Task.Opts, string(body) + } + + // Without a CA bundle the request stays what an ADC server that predates caCert + // already accepts. + opts, raw := build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true}) + assert.Empty(t, opts.CaCert) + assert.NotContains(t, raw, "caCert") + + const caCert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----" + opts, raw = build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true, CaBundle: caCert}) + assert.Equal(t, caCert, opts.CaCert) + assert.Contains(t, raw, "caCert") + // verification stays on, otherwise the bundle would be pointless + assert.Equal(t, false, *opts.TlsSkipVerify) +} + // confVersionError is what a push carrying a conf_version older than the data plane's // comes back as, once the ADC server has relayed the rejection to us. func confVersionError() error { diff --git a/internal/adc/translator/gatewayproxy.go b/internal/adc/translator/gatewayproxy.go index 13ace18d..ad2999d6 100644 --- a/internal/adc/translator/gatewayproxy.go +++ b/internal/adc/translator/gatewayproxy.go @@ -18,6 +18,7 @@ package translator import ( + "crypto/x509" "fmt" "net" "strconv" @@ -56,6 +57,17 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte cfg.TlsVerify = *cp.TlsVerify } + if cp.CaBundle != "" { + // reject unusable CA material here rather than at connect time + if !x509.NewCertPool().AppendCertsFromPEM([]byte(cp.CaBundle)) { + return nil, errors.New("invalid caBundle: no PEM-encoded certificate found") + } + if !cfg.TlsVerify { + t.Log.Info("caBundle is ignored because tlsVerify is disabled", "gatewayproxy", utils.NamespacedNameKind(gatewayProxy)) + } + cfg.CaBundle = cp.CaBundle + } + if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil { if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef diff --git a/internal/adc/translator/gatewayproxy_test.go b/internal/adc/translator/gatewayproxy_test.go new file mode 100644 index 00000000..205d621b --- /dev/null +++ b/internal/adc/translator/gatewayproxy_test.go @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func newGatewayProxy(tlsVerify *bool, caBundle string) *v1alpha1.GatewayProxy { + return &v1alpha1.GatewayProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gp", + }, + Spec: v1alpha1.GatewayProxySpec{ + Provider: &v1alpha1.GatewayProxyProvider{ + Type: v1alpha1.ProviderTypeControlPlane, + ControlPlane: &v1alpha1.ControlPlaneProvider{ + Endpoints: []string{"https://cp.example.com:9180"}, + TlsVerify: tlsVerify, + CaBundle: caBundle, + Auth: v1alpha1.ControlPlaneAuth{ + Type: v1alpha1.AuthTypeAdminKey, + AdminKey: &v1alpha1.AdminKeyAuth{ + Value: "admin-key", + }, + }, + }, + }, + }, + } +} + +func TestTranslateGatewayProxyToConfigCaBundle(t *testing.T) { + t.Run("carries the CA bundle into the config", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) + + t.Run("leaves the CA bundle empty when unset", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), ""), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Empty(t, cfg.CaBundle) + }) + + t.Run("rejects a CA bundle that is not PEM encoded", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), "not-a-certificate"), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid caBundle") + assert.Nil(t, cfg) + }) + + t.Run("still carries the CA bundle when verification is off", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(false), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.False(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) +}
